diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ad8c456508..8c957cc720 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,4 +6,5 @@ SPDX-License-Identifier: MIT - [ ] If this PR fixes a bug, searched `tests/e2e-cucumber/expectations.toml` for the fixed ticket ID and removed/narrowed any now-stale xfail rows. - [ ] If this PR adds a new subcommand or subsystem, its domain implementation lives in its own file per `docs/architecture.md` (the clap declaration in `cli.rs` and the dispatch wiring in `main.rs`/`lib.rs` are expected, not a violation). +- [ ] If this PR adds, renames or removes a production source file (or a `.rs` file under `xtask/src` or `tests/e2e-cucumber`), `docs/architecture.md` has the matching entry (`cargo xtask check-architecture-doc` fails otherwise), and the prose of any entry whose file changed role still describes it, which the check cannot verify. - [ ] Every new or changed user-facing message was read against the code path that runs after it, and its test asserts the resulting state — not only the wording — per AGENTS.md §3. Covers claims of an outcome, remediation advice naming a command, and promises that something will *not* happen. diff --git a/.github/scan_tools_configs/.gitleaksignore b/.github/scan_tools_configs/.gitleaksignore new file mode 100644 index 0000000000..1e3cd3c5f5 --- /dev/null +++ b/.github/scan_tools_configs/.gitleaksignore @@ -0,0 +1,13 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Reference: https://github.com/gitleaks/gitleaks#gitleaksignore +# +# One finding fingerprint per line, copied from a gitleaks report +# (`:::`). A line starting with # is a +# comment and suppresses nothing. Remove the leading # to activate an +# example: +# +# 3f2a9c1e...:tests/fixtures/creds.py:generic-api-key:12 +# +# Entries here combine with a .gitleaksignore at the repository root. diff --git a/.github/scan_tools_configs/.trivyignore b/.github/scan_tools_configs/.trivyignore new file mode 100644 index 0000000000..c8341ec5dd --- /dev/null +++ b/.github/scan_tools_configs/.trivyignore @@ -0,0 +1,19 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Reference: https://trivy.dev/latest/docs/configuration/filtering/#trivyignore +# +# One vulnerability, misconfiguration, or secret rule ID per line. +# A line starting with # is a comment and suppresses nothing. Remove the +# leading # to activate an example: +# +# CVE-2019-14697 exp:2027-01-01 +# AVD-DS-0002 +# generic-unwanted-rule +# +# `exp:YYYY-MM-DD` limits a suppression to that date. After it, trivy +# reports the finding again. +# +# NOTE: trivy.yml sets `ignorefile: .trivyignore.yaml`, so this plain file +# is NOT read by the scan. It is kept only as a format reference; add real +# suppressions to .trivyignore.yaml. diff --git a/.github/scan_tools_configs/.trivyignore.yaml b/.github/scan_tools_configs/.trivyignore.yaml new file mode 100644 index 0000000000..0acfc5d0ae --- /dev/null +++ b/.github/scan_tools_configs/.trivyignore.yaml @@ -0,0 +1,22 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Path-scoped trivy suppressions, selected by `ignorefile` in trivy.yml. +# Reference: https://trivy.dev/latest/docs/configuration/filtering/#by-finding-ids +# (the plain-text .trivyignore beside this file cannot scope by path, so it is +# not used.) + +misconfigurations: + # tests/remote-ssh/Dockerfile is a throwaway sshd stand-in for a remote GPU + # machine, built and run only by the remote scenarios (tests/remote-ssh/run.sh) + # and never published. It has to run as root (sshd, adduser/chpasswd, root's + # authorized_keys) and listen on port 22, because the point is to exercise a + # real OpenSSH server the way `rocm remote` meets one. + - id: AVD-DS-0002 # image user should not be 'root' + paths: + - tests/remote-ssh/Dockerfile + statement: Test-only sshd stand-in; needs root to run sshd and provision accounts. + - id: AVD-DS-0004 # port 22 exposed + paths: + - tests/remote-ssh/Dockerfile + statement: Test-only sshd stand-in; port 22 is the thing under test. diff --git a/.github/scan_tools_configs/bandit.yml b/.github/scan_tools_configs/bandit.yml new file mode 100644 index 0000000000..aa800efe64 --- /dev/null +++ b/.github/scan_tools_configs/bandit.yml @@ -0,0 +1,32 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# bandit configuration +# Reference: https://bandit.readthedocs.io/en/latest/config.html +# Local scan: bandit --recursive . --configfile .github/scan_tools_configs/bandit.yml +# +# Bandit configuration for this repository's security scans. +# +# This file defines the complete scanner policy; +# Keep repository-specific suppressions and tuning here. +# +# Bandit has no ignore file. Suppress a finding with: +# +# 1. `# nosec ` on the flagged line: +# subprocess.run(cmd, shell=True) # nosec B602 # cmd is a constant +# +# 2. `skips:` for test IDs disabled across the repository: +# skips: [B101] +# +# 3. `exclude_dirs:` below for paths Bandit should not scan. + +# Environment and generated-output exclusions support local scans; these +# paths are absent from clean CI checkouts. +exclude_dirs: + - .git + - .venv* + - venv* + - build + - dist + - node_modules + - __pycache__ diff --git a/.github/scan_tools_configs/gitleaks.toml b/.github/scan_tools_configs/gitleaks.toml new file mode 100644 index 0000000000..13c7b2f37f --- /dev/null +++ b/.github/scan_tools_configs/gitleaks.toml @@ -0,0 +1,109 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# gitleaks.toml +# Reference: https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml +# Local scan: gitleaks detect --source . --config .github/scan_tools_configs/gitleaks.toml +# +# Gitleaks configuration for this repository's security scans. +# +# This file defines the repository's complete Gitleaks policy. Recurring +# exclusions belong in this file. +# +# The `[extend]` section inherits Gitleaks' built-in rules, and the +# `[[allowlists]]` below apply to every rule. +# +# Ways to suppress a finding, narrowest first: +# +# 1. `gitleaks:allow` on the offending line: +# api_key = "not-a-real-key" # gitleaks:allow +# +# 2. `.gitleaksignore` with one finding fingerprint per line: +# 3f2a9c1e...:tests/fixtures/creds.py:generic-api-key:12 +# Reference: https://github.com/gitleaks/gitleaks#gitleaksignore +# Keep it beside this file (`.github/scan_tools_configs/.gitleaksignore`) +# or at the repository root. When both exist, their entries combine. +# The `.gitleaksignore` here is a commented placeholder. +# +# 3. `[[allowlists]]` below for recurring paths, regexes or stopwords. + +title = "gitleaks config" + +[extend] +# Inherit gitleaks' built-in ruleset (rotations, AWS, GCP, Slack, ...). +# Setting this to false would mean rolling our own detections, which +# is almost never what you want. +useDefault = true + + + +# Allowlist field-vs-target reference: +# stopwords -> always matched against the SECRET value. +# regexes -> matched against `regexTarget` (default: "secret"). +# regexTarget -> "secret" | "match" | "line". +# Use "match" or "line" when the "tell" is the variable name or +# surrounding syntax, not the value itself. + +[[allowlists]] +# Allowlist by location only where a real first-party secret structurally +# can't live. Dependency lock files qualify: their contents are generated +# from a manifest, and the high-entropy strings they carry are artifact +# digests rather than credentials. Add vendored third-party trees here as +# callers bring them in, one explicit path per entry. +description = "Generated dependency lock files" +paths = [ + '''.*\.lock$''', +] + +[[allowlists]] +description = "Test fixtures using well-known dummy variable names" +# The "tell" here is the variable name on the same line, not the value +# (the value is a random-looking string by design). Match against the +# full rule match (variable + value) so we anchor on the identifier. +regexTarget = "match" +regexes = [ + '''(?i)\bINVALID_TOKEN\s*=''', + '''(?i)\bwrong_secret\s*=''', + # Explicit test-only secret identifiers + '''(?i)\btest_secret\s*=''', + '''(?i)\bdummy_secret\s*=''', + '''(?i)\bfake_secret\s*=''', + '''(?i)\bexample_secret\s*=''' +] + +[[allowlists]] +description = "Public artifact digests (variable names ending in SHA256/512/MD5)" +# SHA-256/512/MD5 hashes of public artifacts are not secrets. +regexTarget = "line" +regexes = [ + '''[A-Z][A-Z0-9_]*SHA256\s*[=:]''', + '''[A-Z][A-Z0-9_]*SHA512\s*[=:]''', + '''[A-Z][A-Z0-9_]*MD5\s*[=:]''', +] + +[[allowlists]] +description = "Model names and runtime keys assigned in first-party Rust code" +# The generic-api-key rule fires on `api_model = "Qwen3-0.6B-GGUF"` and +# `runtime_key: "wheel-gfx942-7.13.0"` because the variable name contains +# "key" or "api" and the value is a short mixed-case identifier. Both are +# public identifiers (a model repository name, a runtime install key), not +# credentials. Matched against the rule's own match, not the whole line, so +# a real credential on the same line is still reported under its own match. +# Only the two names `api_model` and `runtime_key` are listed, so an +# `api_key = "..."` assignment is still scanned; the Rust-only path limit keeps +# the exception out of other file types. Both `:` (struct field) and `=` +# (assignment) are accepted for both names, and the value may hold the +# characters a model name or a runtime key uses (`Qwen3-0.6B-GGUF`, +# `therock-release:gfx120X-all`), up to 64 characters, so a long token +# assigned to one of the two names is still reported. A JSON-style quoted key +# such as `"runtime_key": "..."` is not covered. +# `condition = "AND"` is load-bearing: under the default OR, the `paths` entry +# alone would allowlist every rule in every `.rs` file. Do not remove it. +condition = "AND" +paths = [ + '''.*\.rs$''', +] +regexTarget = "match" +regexes = [ + '''\b(api_model|runtime_key)\s*[:=]\s*"[A-Za-z0-9._/:-]{1,64}"''', +] diff --git a/.github/scan_tools_configs/trivy.yml b/.github/scan_tools_configs/trivy.yml new file mode 100644 index 0000000000..55d919c97a --- /dev/null +++ b/.github/scan_tools_configs/trivy.yml @@ -0,0 +1,40 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# trivy configuration +# Reference: https://trivy.dev/latest/docs/references/configuration/config-file/ +# Local scan: trivy fs --config .github/scan_tools_configs/trivy.yml . +# +# Trivy configuration for this repository's security scans. +# +# This file defines the complete scanner policy; +# Keep repository-specific suppressions and tuning here. +# +# Ways to suppress a finding, narrowest first: +# +# 1. `.trivyignore` -- one vulnerability, misconfiguration or secret rule +# ID per line. Add `exp:YYYY-MM-DD` for a temporary suppression: +# CVE-2019-14697 exp:2027-01-01 +# Reference: https://trivy.dev/latest/docs/configuration/filtering/#trivyignore +# NOTE: this config sets `ignorefile:` to `.trivyignore.yaml` (below), +# so trivy reads only that file; IDs added to the plain `.trivyignore` +# are not read. Add ID-only suppressions to `.trivyignore.yaml` too. +# +# 2. `scan.skip-dirs` / `scan.skip-files` below -- paths trivy never +# reads, for vendored trees and generated output. Anything found only +# there is not scanned at all, so keep entries narrow. + +# Path-scoped suppressions live in .trivyignore.yaml (each with a rationale). +ignorefile: .github/scan_tools_configs/.trivyignore.yaml + +# Environment and generated-output exclusions support local scans; these +# paths are absent from clean CI checkouts. +scan: + skip-dirs: + - .git + - .venv* + - venv* + - build + - dist + - node_modules + - __pycache__ diff --git a/.github/scan_tools_configs/zizmor.yml b/.github/scan_tools_configs/zizmor.yml new file mode 100644 index 0000000000..5b3444195d --- /dev/null +++ b/.github/scan_tools_configs/zizmor.yml @@ -0,0 +1,35 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# zizmor configuration +# Reference: https://docs.zizmor.sh/configuration/ +# Local scan: zizmor --config .github/scan_tools_configs/zizmor.yml . +# +# Zizmor configuration for this repository's security scans. +# +# This file defines the complete scanner policy; +# Keep repository-specific suppressions and tuning here. +# +# Zizmor has no ignore file. Suppress a finding with: +# +# 1. An inline comment naming the rule: +# run: | # zizmor: ignore[template-injection] input is validated +# +# 2. `rules..ignore` with `[:[:]]`: +# rules: +# template-injection: +# ignore: +# - safe.yml +# +# 3. `rules..disable: true` to disable a rule repository-wide. +# +# Add rule-specific settings and suppressions under `rules:`. + +rules: + dangerous-triggers: + ignore: + # Deliberate: the privileged half of dependabot-manifests.yml. It runs + # the default branch's definition, never checks out or executes PR code, + # and commits only two fixed paths after verifying the run's actor, event, + # live PR head and artifact provenance (see the guard step in the file). + - dependabot-manifests-commit.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bf61e7b7e..230544bfa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,8 @@ jobs: windows: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.build_binaries == 'true' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Filter changed paths (pull requests only) id: filter @@ -144,6 +146,11 @@ jobs: # rocm-deps compiles the pins into constants, so a pin change is a # source change even though no *.rs file moved. - 'runtime-deps.toml' + # A proptest saved-failure file is read by the seed guard in xtask + # (`cargo xtask affected` adds xtask when one changes). Renaming or + # deleting one moves no *.rs file, so without this entry a seed-only + # change would skip every rust-gated job. + - '**/proptest-regressions/**' # pinned_tool_versions_match_across_docs_and_workflows (xtask) reads # this file's hawkeye/cargo-about pins. This entry only gates # whether the rust-flagged jobs run at all; `cargo xtask affected` @@ -165,15 +172,20 @@ jobs: # `heavy`-gated) doesn't silently skip the test that reads it. - 'docs/ci-hardware-testing.md' - 'tests/e2e-cucumber/README.md' + # `cargo xtask check-architecture-doc` (clippy job) compares this + # module map with the source tree; deleting or rewording an entry + # can break it though no *.rs file moved. + - 'docs/architecture.md' # license_headers_hook_has_no_file_type_filter (xtask) reads this # file's license-headers hook config; an edit here that reopens the # file-type-filter gap must still run that test on the PR. - '.pre-commit-config.yaml' - '.github/workflows/**' - # build-and-test runs cargo AND the python/shell smoke steps plus the - # install-lifecycle E2E (`cargo xtask package` + the real installer), - # so it depends on the Rust set, the packaging xtask, the lifecycle - # feature/steps, and the installer inputs. + # build-and-test runs cargo, the smoke gate and the remaining + # python/shell self-tests, plus the install-lifecycle E2E (`cargo + # xtask package` + the real installer), so it depends on the Rust + # set, the packaging xtask, the lifecycle feature/steps, and the + # installer inputs. heavy: - '**/*.rs' - '**/Cargo.toml' @@ -263,6 +275,7 @@ jobs: with: # Need the PR range to read its commit messages. fetch-depth: 0 + persist-credentials: false - name: Warn on referenced bugs that still have xfail rows # Title/body are passed via env (never interpolated into the shell) so a # crafted PR description can't inject commands. @@ -353,6 +366,8 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install native build deps if: needs.changes.outputs.heavy == 'true' @@ -427,7 +442,7 @@ jobs: disable fi - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: needs.changes.outputs.heavy == 'true' with: # rust-cache keys on "Linux-x64" only; name the image so a target dir @@ -457,7 +472,7 @@ jobs: - name: Local no-fallback smoke if: needs.changes.outputs.heavy == 'true' - run: python scripts/smoke_local.py --skip-build + run: cargo xtask smoke --skip-build - name: Acceptance harness self-tests if: needs.changes.outputs.heavy == 'true' @@ -490,10 +505,21 @@ jobs: if: needs.changes.outputs.heavy == 'true' run: python scripts/release_readiness.py --self-test + - name: Nightly canary self-test + if: needs.changes.outputs.heavy == 'true' + run: python scripts/nightly_canary.py --self-test + - name: Pinned key consistency check if: needs.changes.outputs.heavy == 'true' run: cargo xtask verify-pinned-keys + # Hermetic: a signed local bundle and a stub `rocm`, no network and no + # GPU, so a fish-profile regression fails in this fast step instead of + # only inside a full release install. + - name: Installer fish profile checks + if: needs.changes.outputs.heavy == 'true' + run: sh tests/install-sh/run.sh + - name: Acceptance install lifecycle if: needs.changes.outputs.heavy == 'true' # The install lifecycle (package + real installer + install/uninstall) now @@ -548,12 +574,13 @@ jobs: # base history and the real PR head (not the synthetic merge ref). fetch-depth: 0 ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }} + persist-credentials: false - name: Install native build deps if: needs.changes.outputs.rust == 'true' run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: needs.changes.outputs.rust == 'true' with: # Keep the PR (subset) and push/merge (`--workspace`) builds in separate @@ -566,7 +593,7 @@ jobs: - name: Install cargo-nextest if: needs.changes.outputs.rust == 'true' - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 + uses: taiki-e/install-action@b80e2bb2635f28eb98b9fce1e2f0887e1535c6b2 # v2.90.0 with: tool: nextest @@ -577,8 +604,8 @@ jobs: # The `cargo xtask` alias builds xtask in release (target/release), so # computing the selection doesn't perturb the debug feature resolution # the nextest build below uses. - if [ "${{ github.event_name }}" = "pull_request" ]; then - sel=$(cargo xtask affected --base "origin/${{ github.base_ref }}") + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + sel=$(cargo xtask affected --base "origin/${GITHUB_BASE_REF}") else sel="--workspace" fi @@ -611,7 +638,11 @@ jobs: # fields and the distro name, in # `os_release_reads_follow_the_simulated_host_root` and, for the # `/usr/lib/os-release` fallback, in - # `os_release_reads_fall_back_to_usr_lib_when_etc_is_absent`). The other routed + # `os_release_reads_fall_back_to_usr_lib_when_etc_is_absent`), and the + # driver-presence reads `/sys/class/kfd/kfd`, `/dev/kfd` and + # `/sys/bus/pci/drivers/amdgpu` (`kfd_present_on_host`, + # `amdgpu_present_on_host`, `driver_states` and the three + # `probe_modules_reports_*` tests). The other routed # reads (`/dev/dri`, `/sys/class/drm`, `/sys/module/amdgpu`, the other # `/proc` files, the modprobe directories, the `/usr/lib/wsl/lib` and # ROCDXG checks) have no planted-root unit assertion; they are covered @@ -676,6 +707,8 @@ jobs: if: needs.changes.outputs.windows == 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Installs sccache. Runs BEFORE setup-rust-toolchain, whose rust-cache step # runs `cargo metadata` and honours RUSTC_WRAPPER. `version` is pinned so @@ -735,7 +768,7 @@ jobs: } exit 0 - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: needs.changes.outputs.heavy == 'true' with: cache-on-failure: true @@ -788,7 +821,7 @@ jobs: - name: Local no-fallback smoke if: needs.changes.outputs.heavy == 'true' shell: pwsh - run: python .\scripts\smoke_local.py --skip-build + run: cargo xtask smoke --skip-build # One command per step: pwsh carries on past a failing native command and # the step reports only the last command's exit code, so a multi-line @@ -850,6 +883,8 @@ jobs: if: needs.changes.outputs.windows == 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Installs sccache. Runs BEFORE setup-rust-toolchain, whose rust-cache step # runs `cargo metadata` and honours RUSTC_WRAPPER. `version` is pinned so @@ -909,7 +944,7 @@ jobs: } exit 0 - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: needs.changes.outputs.heavy == 'true' with: cache-on-failure: true @@ -1132,8 +1167,10 @@ jobs: && needs.changes.outputs.heavy == 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. @@ -1232,9 +1269,11 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # cargo-fmt hook needs rustfmt (no build). - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: components: rustfmt cache: false @@ -1286,6 +1325,8 @@ jobs: if: github.event_name != 'workflow_dispatch' && needs.changes.outputs.heavy == 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Docker and the OpenSSH client are both present on the hosted Ubuntu # images; jq is the only extra, and only for asserting on JSON contracts. @@ -1299,7 +1340,7 @@ jobs: # other cargo job in this workflow uses. Without it this lane builds the # whole workspace cold against a different compiler from the rest of CI, # inside a 30-minute timeout. - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. @@ -1347,6 +1388,8 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Installs sccache. Runs BEFORE setup-rust-toolchain, whose rust-cache step # runs `cargo metadata` and honours RUSTC_WRAPPER. `version` is pinned so @@ -1393,7 +1436,7 @@ jobs: disable fi - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: needs.changes.outputs.rust == 'true' with: # rust-cache keys on "Linux-x64" only; name the image so a target dir @@ -1420,6 +1463,10 @@ jobs: if: needs.changes.outputs.rust == 'true' run: cargo xtask check-crate-edges + - name: Every production source file has an entry in docs/architecture.md + if: needs.changes.outputs.rust == 'true' + run: cargo xtask check-architecture-doc + - name: Published Doctor catalog is current if: needs.changes.outputs.rust == 'true' run: cargo xtask catalog --check @@ -1428,33 +1475,13 @@ jobs: if: needs.changes.outputs.rust == 'true' run: cargo xtask eval-fixtures --check - # Advisory for now. `cargo xtask check-module-size` is strict on its own; - # only this step is non-blocking. `continue-on-error` keeps the job green - # while the warning annotation and the job summary list the offenders. When - # files are over the limit the step itself shows as failed (red) in the - # job's step list, because it ends in `exit 1`, while the job and the check - # stay green: that is the advisory phase, not a failure to chase. - # Making the limit blocking means deleting `continue-on-error` and the - # capture below, leaving the bare command. - - name: Source files are within the module-size limit (advisory) + # Blocking: a source file over the limit fails this job, which is a required + # check. `merge_group` runs it too (`changes` forces `rust` on every event + # but a pull request), so two PRs that each pass alone cannot together + # push a file over the limit. + - name: Source files are within the module-size limit if: needs.changes.outputs.rust == 'true' - continue-on-error: true - run: | - report="$RUNNER_TEMP/module-size.txt" - if cargo xtask check-module-size > "$report" 2>&1; then - cat "$report" - else - cat "$report" - echo "::warning title=Module size (advisory)::$(sed 's/%/%25/g' "$report" | tr '\n' ' ')" - { - echo '### Source files over the module-size limit (advisory)' - echo - echo '```' - cat "$report" - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - exit 1 - fi + run: cargo xtask check-module-size # Surface the cache hit rate for the clippy compile so the effect is visible # in the logs, on the local-only path too — see build-and-test for why this @@ -1474,8 +1501,10 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: needs.changes.outputs.tpn == 'true' with: # rust-cache keys on "Linux-x64" only; name the image so a target dir @@ -1523,6 +1552,8 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install native build deps if: needs.changes.outputs.rust == 'true' @@ -1535,7 +1566,7 @@ jobs: - name: Install cargo-llvm-cov if: needs.changes.outputs.rust == 'true' - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 + uses: taiki-e/install-action@b80e2bb2635f28eb98b9fce1e2f0887e1535c6b2 # v2.90.0 with: tool: cargo-llvm-cov @@ -1579,8 +1610,10 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: needs.changes.outputs.lint == 'true' - name: Install PSScriptAnalyzer (PowerShell 7) @@ -1624,7 +1657,7 @@ jobs: # fires, so this job is not a substitute for them. # # skills/rocm-cli-assistant/ is deliberately out of scope. Despite its path it - # is not a published skill: apps/rocm/src/main.rs embeds it verbatim into the + # is not a published skill: apps/rocm/src/chat/turn.rs embeds it verbatim into the # chat assistant's system prompt with include_str!, so giving it the YAML # frontmatter this check requires would inject that frontmatter into a live # prompt. @@ -1637,6 +1670,8 @@ jobs: if: github.event_name != 'workflow_dispatch' && needs.changes.outputs.skills == 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # The `skillscope` step below names skills/rocm-doctor explicitly, so a # third skills// folder added later would pass this job green @@ -1705,6 +1740,8 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Cache hawkeye binary id: cache-hawkeye @@ -1741,6 +1778,8 @@ jobs: if: github.event_name != 'workflow_dispatch' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Check markdown links uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 @@ -1761,6 +1800,8 @@ jobs: if: github.event_name != 'workflow_dispatch' && needs.changes.outputs.docs == 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Cache pip packages id: cache-pip @@ -1804,8 +1845,9 @@ jobs: # so these are normal commits that preserve their Signed-off-by trailer # and are GitHub "Verified" (web-flow signed). ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. @@ -1868,6 +1910,8 @@ jobs: || inputs.platform == 'all' || inputs.platform == 'mock') steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Whether the real work runs this time. On push/PR/merge_group: only when # `clippy` passed, build-and-test ran to a verdict, and the change is @@ -1914,7 +1958,7 @@ jobs: if: steps.gate.outputs.run == 'true' run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 if: steps.gate.outputs.run == 'true' with: # rust-cache keys on "Linux-x64" only; name the image so a target dir @@ -1988,8 +2032,10 @@ jobs: || github.event_name == 'workflow_dispatch') steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6934c4833d..66f5b7f7d1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,11 +27,13 @@ jobs: language: [actions, python, rust] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + - uses: github/codeql-action/init@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2 with: languages: ${{ matrix.language }} - - uses: github/codeql-action/autobuild@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + - uses: github/codeql-action/autobuild@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2 - - uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + - uses: github/codeql-action/analyze@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2 diff --git a/.github/workflows/demo-gifs.yml b/.github/workflows/demo-gifs.yml index f86e868144..fd255fe2f9 100644 --- a/.github/workflows/demo-gifs.yml +++ b/.github/workflows/demo-gifs.yml @@ -30,12 +30,14 @@ jobs: runs-on: ubuntu-24.04 # one LTS: renders documentation GIFs steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: - # rust-cache keys on "Linux-x64" only; name the image so a target dir - # built against one Ubuntu release's glibc is never restored on another. - cache-key: ubuntu-24.04 + # No rust-cache restore/save: builds start from a clean tree, so a + # poisoned cache entry cannot reach this job (zizmor cache-poisoning). + cache: false # VHS + its runtime deps. ttyd/ffmpeg from apt; vhs pinned via `go install` # (Go is preinstalled on the runner). Chrome (used by VHS to rasterize diff --git a/.github/workflows/dependabot-manifests.yml b/.github/workflows/dependabot-manifests.yml index 11f54efdb5..e4ce76b9c0 100644 --- a/.github/workflows/dependabot-manifests.yml +++ b/.github/workflows/dependabot-manifests.yml @@ -55,7 +55,7 @@ jobs: # — untrusted code and a write-scoped token never coexist here. generate: name: Regenerate manifests - if: github.actor == 'dependabot[bot]' + if: github.event.pull_request.user.login == 'dependabot[bot]' runs-on: ubuntu-24.04 # one LTS: regenerates manifests, nothing here depends on the image permissions: contents: read @@ -65,8 +65,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 1103d3ccbc..96b63f1b9c 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -9,11 +9,18 @@ name: E2E self-hosted # THIS workflow's supersession, never the required checks in ci.yml. Only # pull_request runs are superseded (see `concurrency:` below). # +# Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, +# MI300X: the only lane where vLLM is the effective engine) and one Windows GPU +# pool (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix +# Ubuntu lane keeps its push / dispatch triggers; it does not start on +# pull_request or merge_group. There is no W7900 lane here: W7900 is covered only +# by the scheduled queue nightly's `w7900-dual` lane (docs/nightly-on-the-queue.md), +# not on push, release/** or dispatch. +# # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not -# because of `continue-on-error`, which is `false` on every lane but -# one, so a real hardware regression shows red instead of always green. The -# exception is `e2e-gpu-w7900`, `true` only while that hardware is proven out. PRs #334, #335 and #343 all +# because of `continue-on-error`, which is `false` on every lane, so a real +# hardware regression shows red instead of always green. PRs #334, #335 and #343 all # landed while `E2E tests (Strix Halo, Ubuntu)` reported `cancelled` on the # merge_group ref, which a required check under ALLGREEN would have blocked; # renaming a lane is therefore safe. @@ -31,6 +38,13 @@ on: # Mirror ci.yml: run on PRs against ANY base branch so a stacked PR still gets # the GPU lanes. Affected-crate/heavy selection diffs against github.base_ref. pull_request: + # `closed` (merge OR close) joins the PR's concurrency group below, so it + # supersedes that PR's earlier run. It runs only the no-op `pr-closed` job; + # every GPU lane skips because `changes` skips. This frees runners only where + # the cancel completes: a job queued for a runner that is not picking it up + # (a lane whose runner is busy) may not finish cancelling, so this + # does NOT clear such queued runs. + types: [opened, synchronize, reopened, closed] merge_group: branches: [main] types: [checks_requested] @@ -44,7 +58,7 @@ on: type: choice default: all # Each value also selectable on its own; `all` covers every lane below. - options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, w7900] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -93,13 +107,18 @@ jobs: # forced true, matching ci.yml. changes: runs-on: ubuntu-24.04 # one LTS: path filter only, nothing here depends on the image + # Skipped on `closed`, and with it every lane and `e2e-report`: the lanes gate + # on `needs.changes.result == 'success'`, the report on + # `needs.changes.outputs.serve`, which is empty for a skipped job. + if: github.event_name != 'pull_request' || github.event.action != 'closed' permissions: contents: read pull-requests: read outputs: heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} # Narrow GPU-serve gate; forced true off-PR so the merge queue runs the - # full matrix (every lane but MI300X, which skips merge_group) and its + # full matrix (every lane but MI300X and Strix Ubuntu, which skip + # merge_group) and its # required checks are never starved. serve: ${{ steps.filter.outputs.serve || steps.all.outputs.forced }} # ROCMAI-430: the three most recent rows of the official release list @@ -110,6 +129,8 @@ jobs: sdk_version_matrix: ${{ steps.sdk_matrix.outputs.matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Filter changed paths (pull requests only) id: filter @@ -175,12 +196,25 @@ jobs: - name: Compute sdk_version matrix (ROCMAI-430) id: sdk_matrix run: | - if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == refs/heads/release/* ]]; then + if [[ "${GITHUB_EVENT_NAME}" == "push" && "${GITHUB_REF}" == refs/heads/release/* ]]; then echo 'matrix=["current","n-1","n-2"]' >> "$GITHUB_OUTPUT" else echo 'matrix=["current"]' >> "$GITHUB_OUTPUT" fi + # Exists only to start a run in the PR's concurrency group when the PR is + # merged or closed: that run cancels the PR's older run. A GPU job of that run + # which is queued for a runner that is not picking it up may not finish + # cancelling (observed on queued runs). + pr-closed: + name: Supersede runs of a closed PR + if: github.event_name == 'pull_request' && github.event.action == 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - run: echo "PR closed; its earlier E2E runs are superseded by this run." + e2e-gpu: name: E2E tests (MI300X, ${{ matrix.sdk_version }}) timeout-minutes: 90 @@ -197,7 +231,7 @@ jobs: # single-element `["current"]` matrix, so the ordinary PR/merge-queue path # runs exactly one job, unchanged from before this matrix existed. matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # No build-and-test gate (cross-workflow needs is unavailable): the job builds # the rocm binary itself, and this check isn't in the required list (see the # file header), so a broken build here still can't block a merge; ci.yml's @@ -232,6 +266,8 @@ jobs: E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine @@ -262,7 +298,7 @@ jobs: # Swatinem/rust-cache otherwise tries to SAVE the large target dir to # GitHub's cache service in a post-step (slow/hangs) and its cleanup wipes # the local target. - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: cache: false @@ -375,17 +411,20 @@ jobs: HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" \ - --release-label "${{ matrix.sdk_version }}" + --release-label "${MATRIX_SDK_VERSION}" # Optional scenario-name filter for a scoped dispatch — lets a manual # run select only the large-model scenario instead of the whole suite. - NAME_FILTER="${{ github.event.inputs.name_filter }}" + NAME_FILTER="${GITHUB_EVENT_INPUTS_NAME_FILTER}" if [ -n "$NAME_FILTER" ]; then echo "name filter active: $NAME_FILTER" cargo xtask e2e -- --name "$NAME_FILTER" else cargo xtask e2e fi + env: + MATRIX_SDK_VERSION: ${{ matrix.sdk_version }} + GITHUB_EVENT_INPUTS_NAME_FILTER: ${{ github.event.inputs.name_filter }} # Three mutually-exclusive steps, not one templated `name:` (ROCMAI-430): # `xtask::e2e_report`'s static contract test greps `name: e2e-*` lines @@ -434,16 +473,24 @@ jobs: # ratio the 35 was chosen for. # An include_nightly dispatch also runs the AIM serve scenarios (serve-26/27, # @nightly, up to 1200s readiness each on a cold pull) and the large-model - # serve, which cannot fit the per-PR 45min budget. + # serve, which cannot fit the 45min budget of an ordinary push run. timeout-minutes: ${{ inputs.include_nightly && 105 || 45 }} runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. + # Not on pull_request: a PR gets exactly one Linux GPU node (MI300X, the only + # lane where vLLM runs) and one Windows pool (Strix Halo). Not on + # merge_group either: over 7 days 35% of its merge-queue jobs (98 of 281) were + # cancelled at the 45 min cap, and the queue does not wait for this + # non-required check. Still runs on push (main, release/**), dispatch and + # nightly (nightly.yml, e2e-gpu-nightly-strix, same pool labels). if: >- always() && needs.changes.result == 'success' && ( (github.event_name != 'workflow_dispatch' + && github.event_name != 'pull_request' + && github.event_name != 'merge_group' && needs.changes.outputs.serve == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')) @@ -455,8 +502,8 @@ jobs: channel: [release, nightly] # See `e2e-gpu` (ROCMAI-430): layered alongside the pre-existing # `channel` dimension above, not replacing it — this lane is the only - # per-PR job with two independent matrix axes. - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + # job with two independent matrix axes. + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # The nightly channel only carries the latest (pre-)release builds, so # it cannot resolve the n-1/n-2 released versions the matrix pins # (`rocm install sdk` fails with "no mutually compatible TheRock ... @@ -477,10 +524,10 @@ jobs: # Match the MI300X dispatch path: opt into the platform-adaptive large-model # scenario only when the manual include_nightly input is enabled. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - # Heavy @merge-queue serves run only in the merge queue. - E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine @@ -550,17 +597,21 @@ jobs: # See the e2e-gpu lane and `xtask e2e-prewarm` for why the old # existence-only guard froze this lane on its first runtime (EAI-8057). HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" \ - --release-label "${{ matrix.sdk_version }}" + cargo xtask e2e-prewarm --channel "${MATRIX_CHANNEL}" --prewarm-dir "$prewarm" \ + --release-label "${MATRIX_SDK_VERSION}" # Optional scenario-name filter for a scoped manual dispatch. - NAME_FILTER="${{ github.event.inputs.name_filter }}" + NAME_FILTER="${GITHUB_EVENT_INPUTS_NAME_FILTER}" if [ -n "$NAME_FILTER" ]; then echo "name filter active: $NAME_FILTER" cargo xtask e2e -- --name "$NAME_FILTER" else cargo xtask e2e fi + env: + MATRIX_CHANNEL: ${{ matrix.channel }} + MATRIX_SDK_VERSION: ${{ matrix.sdk_version }} + GITHUB_EVENT_INPUTS_NAME_FILTER: ${{ github.event.inputs.name_filter }} # Three mutually-exclusive steps per the `sdk_version` leg, same as # `e2e-gpu` (ROCMAI-430); the `current` leg's name is unchanged from @@ -605,7 +656,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-windows. if: >- always() @@ -634,6 +685,8 @@ jobs: E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # setup-rust-toolchain runs an internal bash script, which this Windows # runner lacks (bash: command not found). Bootstrap rustup with the @@ -887,17 +940,20 @@ jobs: # decision lives in `xtask e2e-prewarm` rather than being reimplemented # here in PowerShell, so this lane and the Linux lanes cannot drift # (EAI-8057; the old existence-only guard never reinstalled). - cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" --release-label "${{ matrix.sdk_version }}" + cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" --release-label "$env:MATRIX_SDK_VERSION" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Optional scenario-name filter for a scoped manual dispatch. - $nameFilter = "${{ github.event.inputs.name_filter }}" + $nameFilter = "$env:GITHUB_EVENT_INPUTS_NAME_FILTER" if ($nameFilter) { Write-Host "name filter active: $nameFilter" cargo xtask e2e -- --name "$nameFilter" } else { cargo xtask e2e } + env: + MATRIX_SDK_VERSION: ${{ matrix.sdk_version }} + GITHUB_EVENT_INPUTS_NAME_FILTER: ${{ github.event.inputs.name_filter }} - name: Upload E2E report (current) if: always() && matrix.sdk_version == 'current' @@ -948,7 +1004,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-wsl. Gated on # `serve` like the sibling lanes rather than the broader `heavy`: the # consolidated report gates on `serve` too, so a heavy-but-not-serve change @@ -989,6 +1045,8 @@ jobs: ROCM_CLI_RELEASE_REF: "${{ github.head_ref || github.ref_name }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # `--id Microsoft.WSL --exact`, not the fuzzy `--name` match the # `ubuntu/WSL` composite action's own install step uses (with its own @@ -1422,7 +1480,7 @@ jobs: # Forward the job-level E2E_* env vars into the guest: `wsl.exe` # does not share the calling process's environment unless a # variable is named in WSLENV. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF' + $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli @@ -1456,10 +1514,10 @@ jobs: # existence-only guard froze this lane on its first runtime (EAI-8057). HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" \ - --release-label "${{ matrix.sdk_version }}" + --release-label "${MATRIX_SDK_VERSION}" # Optional scenario-name filter for a scoped manual dispatch. - NAME_FILTER="${{ github.event.inputs.name_filter }}" + NAME_FILTER="${GITHUB_EVENT_INPUTS_NAME_FILTER}" if [ -n "$NAME_FILTER" ]; then echo "name filter active: $NAME_FILTER" cargo xtask e2e -- --name "$NAME_FILTER" @@ -1468,6 +1526,9 @@ jobs: fi '@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + env: + MATRIX_SDK_VERSION: ${{ matrix.sdk_version }} + GITHUB_EVENT_INPUTS_NAME_FILTER: ${{ github.event.inputs.name_filter }} # actions/upload-artifact runs on the Windows host, not inside the WSL # guest, so the results need copying out first. Best-effort: `if: @@ -1537,140 +1598,6 @@ jobs: path: tests/e2e-cucumber/results/ if-no-files-found: error - e2e-gpu-w7900: - name: E2E tests (W7900, ${{ matrix.sdk_version }}) - # 90min: matches every other Linux GPU lane — same collapsed suite. The card - # is smaller than the Instinct parts, so do not tighten this below their cap. - timeout-minutes: 90 - # `w7900` names the hardware rather than reusing the generic `amd-gpu`: - # GitHub matches a job to any runner whose labels are a superset, so a shared - # label would let this workstation card pick up Instinct-targeted work and - # vice versa. - runs-on: [self-hosted, linux, w7900] - needs: [changes] - strategy: - fail-fast: false - matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} - if: >- - always() - && needs.changes.result == 'success' - && ( - (github.event_name != 'workflow_dispatch' - && needs.changes.outputs.serve == 'true') - || (github.event_name == 'workflow_dispatch' - && (inputs.platform == 'all' || inputs.platform == 'w7900')) - ) - # Non-blocking while this hardware is proven out, as the MI350P lane is. - # Flip to `false` once it has a track record — that is what the older lanes - # run with, so a real regression on this silicon shows red instead of - # always green. - continue-on-error: true - env: - E2E_SERVE_TIMEOUT_SECS: "300" - E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - # Full serve matrix at the merge-queue gate. - E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait, as in e2e-gpu, then an assertion that no leaked E2E engine - # process survives. One card per job, so rocm-smi sees exactly the GPU this - # lane runs on. The floor stays at the shared 16 GiB default, which is a - # flat constant rather than a proportion of capacity: it only has to be - # high enough to catch a leftover serve still holding the card, and low - # enough that a clean host passes on the first poll. e2e-gpu uses the same - # 16 GiB against a very different VRAM size for exactly that reason — do - # NOT scale it down for a smaller part. On a 48GB card that floor only - # trips past ~32 GiB held, so the floor alone would miss a leftover serve - # holding less; the process gate is what catches it (#446). The floor is - # kept for the case the process gate cannot see: a non-E2E tenant - # exhausting the host. See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - # cache: false for the same reason as e2e-gpu — the build cache is kept in - # CARGO_TARGET_DIR on the work volume, not in GitHub's cache service. - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 - with: - cache: false - - - name: Run E2E tests on W7900 - run: | - # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the - # cargo target dir and the shared caches are still warm on the next one. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the directory the runner's uv-cache-gc - # initContainer bounds — see the e2e-gpu lane for both mechanisms and - # why a cache one level deeper keeps the links but loses the floor. - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - # Pre-warm one shared runtime in place before the suite. Not an - # optimization — `install sdk` bakes absolute paths into the runtime - # manifest, so installing anywhere temporary breaks every later serve. - # See e2e-gpu for the full rationale. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane for why the e2e-test-hooks feature is the - # floor a pre-built binary must carry (xtask builds a superset). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has moved on — the tree is a cache, not a one-shot. - # See the e2e-gpu lane and `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" \ - --release-label "${{ matrix.sdk_version }}" - - # Optional scenario-name filter for a scoped manual dispatch — a cheap - # way to exercise this lane against a single scenario without the full suite. - NAME_FILTER="${{ github.event.inputs.name_filter }}" - if [ -n "$NAME_FILTER" ]; then - echo "name filter active: $NAME_FILTER" - cargo xtask e2e -- --name "$NAME_FILTER" - else - cargo xtask e2e - fi - - - name: Upload E2E report (current) - if: always() && matrix.sdk_version == 'current' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-report - path: tests/e2e-cucumber/results/ - - - name: Upload E2E report (n-1) - if: always() && matrix.sdk_version == 'n-1' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-n-1-report - path: tests/e2e-cucumber/results/ - - - name: Upload E2E report (n-2) - if: always() && matrix.sdk_version == 'n-2' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-n-2-report - path: tests/e2e-cucumber/results/ - # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -1691,7 +1618,6 @@ jobs: - e2e-gpu-strix-ubuntu - e2e-gpu-strix-windows - e2e-wsl - - e2e-gpu-w7900 # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was @@ -1702,12 +1628,14 @@ jobs: || github.event_name == 'workflow_dispatch') steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: - # rust-cache keys on "Linux-x64" only; name the image so a target dir - # built against one Ubuntu release's glibc is never restored on another. - cache-key: ubuntu-24.04 + # No rust-cache restore/save: builds start from a clean tree, so a + # poisoned cache entry cannot reach this job (zizmor cache-poisoning). + cache: false # Pull every self-hosted e2e artifact matching the glob; `xtask e2e-report` # turns each into one labeled platform and the `*-report` glob picks up any @@ -1761,7 +1689,6 @@ jobs: e2e-gpu-strix-ubuntu=e2e-gpu-strix-ubuntu-release-report,e2e-gpu-strix-ubuntu-nightly-report e2e-gpu-strix-windows=e2e-gpu-strix-windows-report e2e-wsl=e2e-gpu-strix-wsl-report - e2e-gpu-w7900=e2e-gpu-w7900-report run: | uploaded=$(gh api --paginate \ "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \ diff --git a/.github/workflows/nightly-canary.yml b/.github/workflows/nightly-canary.yml new file mode 100644 index 0000000000..82a1f78401 --- /dev/null +++ b/.github/workflows/nightly-canary.yml @@ -0,0 +1,52 @@ +name: Nightly stream canary + +# Metadata only: resolves the current TheRock nightly stream with +# `rocm install sdk --channel nightly --dry-run` and fails when the plan no +# longer comes from that stream, or the newest build it can offer is stale. +# No GPU and no install; see scripts/nightly_canary.py and docs/testing.md +# ("Nightly stream canary") for what it checks, who to tell and how to rerun. + +on: + schedule: + - cron: "23 7 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: -D warnings + AWS_LC_SYS_USE_SYSTEM: "0" + +jobs: + canary: + name: Resolve nightly (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2022] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + with: + cache-key: ${{ matrix.os }} + + - name: Build rocm + run: cargo build --release -p rocm + + - name: Canary self-test + run: python scripts/nightly_canary.py --self-test + + - name: Resolve the current nightly stream + shell: bash + run: python scripts/nightly_canary.py --rocm target/release/rocm diff --git a/.github/workflows/nightly-queue.yml b/.github/workflows/nightly-queue.yml new file mode 100644 index 0000000000..065dbb67de --- /dev/null +++ b/.github/workflows/nightly-queue.yml @@ -0,0 +1,51 @@ +# GPU E2E on the shared queue — publish a run that has already finished. +# +# queue-run.yml starts, waits for and publishes runs. This publishes one again +# by its id, for instance one whose queue-run.yml job was cancelled while the +# run went on: once the run's lanes are done, it has the queue side finish the +# run if no one has, prints its report and lane matrix, uploads the lanes' +# files, and passes or fails with the run's verdict. It needs no credential, so +# the shared self-hosted runner never holds the queue's. What that protects is +# in docs/nightly-on-the-queue.md. +name: Nightly E2E (queue) + +on: + # Manual only. + workflow_dispatch: + inputs: + request_id: + description: "The id of the finished run to publish" + type: string + required: true + +# Nothing here reads the repository. Uploading files uses the per-job runtime +# token GitHub issues to every job regardless of these permissions. +permissions: {} + +concurrency: + group: nightly-queue + cancel-in-progress: false + +jobs: + publish: + name: Publish nightly results + runs-on: [queue-report] + # Nothing here waits on GPUs, but finishing a run nobody finished fetches + # and scans every lane's files, which the publisher allows 30 minutes. + timeout-minutes: 45 + steps: + - name: Publish the run + env: + REQUEST_ID: ${{ inputs.request_id }} + # One line: the workflow contract forbids backslashes in this file. + run: /opt/queue/bin/queue-publish --out "${RUNNER_TEMP}/results" --request "${REQUEST_ID}" + + - name: Upload lane results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-queue-results + path: ${{ runner.temp }}/results + # The publisher's files only; a lane's hidden files are not published. + include-hidden-files: false + if-no-files-found: warn diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ac3d31ace3..5399e19e9f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: permissions: - contents: write + contents: read env: CARGO_TERM_COLOR: always @@ -42,6 +42,9 @@ env: jobs: nightly: + # Creates, uploads and deletes releases; the workflow default is read-only. + permissions: + contents: write name: Build Linux nightly assets runs-on: ubuntu-24.04 # one LTS: the assets are built in the manylinux container, not on the image outputs: @@ -56,8 +59,10 @@ jobs: ref: ${{ github.ref_name }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. @@ -130,7 +135,7 @@ jobs: -u "$(id -u):$(id -g)" \ -e CARGO_HOME=/host-cargo \ -e RUSTUP_HOME=/host-cargo/rustup \ - -e ROCM_CLI_RELEASE_REF="${{ github.ref_name }}" \ + -e ROCM_CLI_RELEASE_REF="${GITHUB_REF_NAME}" \ -e AWS_LC_SYS_USE_SYSTEM \ -v "$HOME/.cargo":/host-cargo \ -v "$PWD":/work \ @@ -157,9 +162,11 @@ jobs: env: ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} ROCM_CLI_REQUIRE_SIGNATURE: "1" + STEPS_META_OUTPUTS_DATE: ${{ steps.meta.outputs.date }} + STEPS_META_OUTPUTS_SHORT_SHA: ${{ steps.meta.outputs.short_sha }} run: | - DATE="${{ steps.meta.outputs.date }}" - SHORT_SHA="${{ steps.meta.outputs.short_sha }}" + DATE="${STEPS_META_OUTPUTS_DATE}" + SHORT_SHA="${STEPS_META_OUTPUTS_SHORT_SHA}" DIST="rocm-cli-nightly-${DATE}-${SHORT_SHA}-linux-amd64" chmod +x install.sh @@ -190,6 +197,7 @@ jobs: if: steps.check.outputs.skip != 'true' env: GH_TOKEN: ${{ github.token }} + STEPS_CHECK_OUTPUTS_COMMITS: ${{ steps.check.outputs.commits }} run: | STAGING_TAG="nightly-staging-${DATE}-${SHORT_SHA}" gh release delete "${STAGING_TAG}" --yes --cleanup-tag 2>/dev/null || true @@ -202,7 +210,7 @@ jobs: Commit: [\`${SHORT_SHA}\`](https://github.com/${{ github.repository }}/commit/$(git rev-parse HEAD)) - ${{ steps.check.outputs.commits }} commit(s) since the previous nightly. + ${STEPS_CHECK_OUTPUTS_COMMITS} commit(s) since the previous nightly. Install the latest nightly on Linux x86_64: @@ -223,14 +231,19 @@ jobs: echo "tag=${STAGING_TAG}" >> "$GITHUB_OUTPUT" windows-nightly: + # Creates, uploads and deletes releases; the workflow default is read-only. + permissions: + contents: write name: Build and upload Windows nightly runs-on: windows-latest needs: nightly if: needs.nightly.outputs.skip != 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - name: Verify pinned keys match the signing key shell: pwsh @@ -252,9 +265,11 @@ jobs: env: ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} ROCM_CLI_REQUIRE_SIGNATURE: "1" + NEEDS_NIGHTLY_OUTPUTS_DATE: ${{ needs.nightly.outputs.date }} + NEEDS_NIGHTLY_OUTPUTS_SHORT_SHA: ${{ needs.nightly.outputs.short_sha }} run: | - $date = "${{ needs.nightly.outputs.date }}" - $shortSha = "${{ needs.nightly.outputs.short_sha }}" + $date = "$env:NEEDS_NIGHTLY_OUTPUTS_DATE" + $shortSha = "$env:NEEDS_NIGHTLY_OUTPUTS_SHORT_SHA" $dist = "rocm-cli-nightly-$date-$shortSha-windows-amd64" cargo xtask package $dist Copy-Item "dist\$dist.zip" "dist\rocm-cli-nightly-windows-amd64.zip" @@ -277,8 +292,9 @@ jobs: shell: pwsh env: GH_TOKEN: ${{ github.token }} + NEEDS_NIGHTLY_OUTPUTS_STAGING_TAG: ${{ needs.nightly.outputs.staging_tag }} run: | - $stagingTag = "${{ needs.nightly.outputs.staging_tag }}" + $stagingTag = "$env:NEEDS_NIGHTLY_OUTPUTS_STAGING_TAG" if (-not $stagingTag) { throw "nightly staging release tag was not produced" } @@ -289,6 +305,9 @@ jobs: gh release upload $stagingTag @assets --clobber publish-nightly: + # Creates, uploads and deletes releases; the workflow default is read-only. + permissions: + contents: write name: Publish nightly runs-on: ubuntu-24.04 # one LTS: publishing only needs: @@ -297,15 +316,20 @@ jobs: if: needs.nightly.outputs.skip != 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Publish staged nightly env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} + NEEDS_NIGHTLY_OUTPUTS_STAGING_TAG: ${{ needs.nightly.outputs.staging_tag }} + NEEDS_NIGHTLY_OUTPUTS_DATE: ${{ needs.nightly.outputs.date }} + NEEDS_NIGHTLY_OUTPUTS_SHORT_SHA: ${{ needs.nightly.outputs.short_sha }} run: | - STAGING_TAG="${{ needs.nightly.outputs.staging_tag }}" - DATE="${{ needs.nightly.outputs.date }}" - SHORT_SHA="${{ needs.nightly.outputs.short_sha }}" + STAGING_TAG="${NEEDS_NIGHTLY_OUTPUTS_STAGING_TAG}" + DATE="${NEEDS_NIGHTLY_OUTPUTS_DATE}" + SHORT_SHA="${NEEDS_NIGHTLY_OUTPUTS_SHORT_SHA}" ASSET_DIR="$(mktemp -d)" if [ -z "${STAGING_TAG}" ]; then echo "nightly staging release tag was not produced" >&2 @@ -358,6 +382,9 @@ jobs: gh release delete "${STAGING_TAG}" --yes 2>/dev/null || true cleanup-staging-nightly: + # Creates, uploads and deletes releases; the workflow default is read-only. + permissions: + contents: write name: Clean failed nightly staging release runs-on: ubuntu-24.04 # one LTS: release cleanup only needs: @@ -369,7 +396,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} - run: gh release delete "${{ needs.nightly.outputs.staging_tag }}" --yes --cleanup-tag 2>/dev/null || true + NEEDS_NIGHTLY_OUTPUTS_STAGING_TAG: ${{ needs.nightly.outputs.staging_tag }} + run: gh release delete "${NEEDS_NIGHTLY_OUTPUTS_STAGING_TAG}" --yes --cleanup-tag 2>/dev/null || true # Full E2E on app-dev MI300X INCLUDING the expensive `@nightly` scenarios (the # large-model serve), which the per-PR e2e-selfhosted.yml `e2e-gpu` job skips @@ -395,6 +423,8 @@ jobs: E2E_SERVE_TIMEOUT_SECS: "300" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine @@ -422,7 +452,7 @@ jobs: # cache: false — this self-hosted runner persists the build cache itself via # CARGO_TARGET_DIR (below), so the action's rust-cache is pure overhead. - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: cache: false @@ -495,6 +525,8 @@ jobs: E2E_SERVE_TIMEOUT_SECS: "300" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine @@ -519,7 +551,7 @@ jobs: run: | bash .github/scripts/gpu-preflight.sh - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: cache: false @@ -574,6 +606,8 @@ jobs: E2E_SERVE_TIMEOUT_SECS: "300" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine @@ -596,7 +630,7 @@ jobs: run: | bash .github/scripts/gpu-preflight.sh - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: cache: false @@ -637,10 +671,11 @@ jobs: name: e2e-gpu-mi350p-${{ matrix.channel }}-report path: tests/e2e-cucumber/results/ - # W7900 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-w7900 lane - # in e2e-selfhosted.yml (w7900 label, shared uv cache, GPU preflight), with - # @nightly scenarios enabled. Keep the two in step: the report crate asserts - # the nightly lanes publish the same platforms as the per-PR lanes. + # W7900 counterpart to e2e-gpu-nightly. Mirrors the e2e-gpu-w7900 lane (which + # no longer runs on pull_request) in e2e-selfhosted.yml (w7900 label, shared + # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in + # step: the report crate asserts the nightly lanes publish the same platforms + # as the lanes in e2e-selfhosted.yml. e2e-gpu-nightly-w7900: name: E2E tests (W7900, incl. nightly-only) timeout-minutes: 90 @@ -655,6 +690,8 @@ jobs: E2E_SERVE_TIMEOUT_SECS: "300" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Reclaim the GPU before running: a serve leaked by a killed or # timed-out prior run (its Drop teardown never executed) keeps an engine @@ -679,7 +716,7 @@ jobs: run: | bash .github/scripts/gpu-preflight.sh - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: cache: false @@ -726,40 +763,34 @@ jobs: # Strix Ubuntu job. e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) - # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits when - # this host has Docker; on a host without it they skip and cost nothing. - # Whether it has Docker is unconfirmed: nothing in this job installs or checks - # it, so the "Report whether Docker is reachable" step logs the answer. If it - # does not, no nightly lane runs AIM serve and a pool-based one is needed. + # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits; on a + # host without Docker they skip and cost nothing. The "Report whether Docker + # is reachable" step logs the answer each run. serve-27's known failure on + # this pool is an xfail row in expectations.toml (rocm-cli#1058). timeout-minutes: 120 - # `native` disambiguates this runner from any host that might carry a bare - # `wsl` label alongside `strix-halo` (the static WSL runner this repo used - # before the WSL lanes moved to the ephemeral pool) -- the paths below - # exist only on the native host, whether or not that other runner remains - # registered. - runs-on: [self-hosted, linux, strix-halo, native] + # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in + # e2e-selfhosted.yml, which runs on push and dispatch but not on pull_request + # or merge_group (pinned by workflow_contract.rs). This lane used to target + # the static `native` host, which is offline: every nightly sat queued until + # the 12h run cap and the lane never ran. Pool hosts are cold, + # so there are no host-specific HOME or cache-dir overrides. + runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] continue-on-error: true strategy: fail-fast: false matrix: channel: [release, nightly] env: - HOME: /home/ubuntu/actions-runner/e2e-home - CARGO_HOME: /home/ubuntu/actions-runner/.cargo - RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup - TMPDIR: /home/ubuntu/actions-runner/tmp - PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs - run: | - mkdir -p "$HOME" "$TMPDIR" "$PIP_CACHE_DIR" - echo "prepared" - bash scripts/reclaim-gpu.sh + - name: Reclaim GPU from stray E2E processes + run: bash scripts/reclaim-gpu.sh # Bounded wait for a free GPU, then an assertion that no leaked E2E engine # process survives. gfx1151 shares system memory, so the floor is smaller @@ -777,11 +808,11 @@ jobs: - name: Ensure Rust toolchain run: | - if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | sh -s -- -y --no-modify-path --default-toolchain none fi - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" # serve-26/27 (AIM) need Docker reachable on this host; without it they skip # silently. Say which it is, so a nightly that never ran them is visible. @@ -842,6 +873,8 @@ jobs: E2E_TUI_TIMEOUT_SECS: "90" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Reclaim GPU from stray E2E processes shell: powershell @@ -1122,6 +1155,8 @@ jobs: ROCM_CLI_RELEASE_REF: "${{ github.ref_name }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # See e2e-wsl in e2e-selfhosted.yml for the full rationale behind each # of these steps — this job mirrors it exactly, minus the scoped @@ -1553,9 +1588,8 @@ jobs: name: E2E consolidated report (nightly) runs-on: ubuntu-24.04 # one LTS: only merges report.json files timeout-minutes: 15 - # Narrower than the workflow-level `contents: write`, which exists for the - # release-publishing jobs. This one checks out, downloads artifacts, runs an - # xtask and uploads; it has no business being able to write to the repo. + # This one checks out, downloads artifacts, runs an xtask and uploads; it + # has no business being able to write to the repo. # `actions: read` lets the missing-report check list this run's artifacts. permissions: contents: read @@ -1575,8 +1609,10 @@ jobs: if: always() steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. @@ -1758,14 +1794,14 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: # rust-cache keys on "Linux-x64" only; name the image so a target dir # built against one Ubuntu release's glibc is never restored on another. cache-key: ubuntu-24.04 - name: Download consolidated nightly report - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: e2e-consolidated-report-nightly path: consolidated @@ -1788,7 +1824,7 @@ jobs: - name: Upload release-readiness facts + table if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-readiness-nightly path: release-readiness/ @@ -1819,7 +1855,7 @@ jobs: persist-credentials: false - name: Download release-readiness facts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-readiness-nightly path: release-readiness @@ -1859,7 +1895,7 @@ jobs: # `github-models`, not `copilot` (v3 flipped the default), so # `provider: copilot` below is required here, not just # documentation. - uses: actions/ai-inference@a7805884c80886efc241e94a5351df715968a0ad # v2.1.1 + uses: actions/ai-inference@2c43c91ae16266ca159d311430343c67a5ffa222 # v3 env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_PAT }} with: @@ -1882,7 +1918,7 @@ jobs: - name: Upload narrative if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-readiness-narrative path: release-readiness/narrative.md @@ -1908,7 +1944,7 @@ jobs: cancel-in-progress: false steps: - name: Download release-readiness table - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 # release-readiness-nightly may have failed before writing or # uploading anything (bad `git describe`, `gh issue list` auth # failure, xtask crash); the script below handles a missing @@ -1920,7 +1956,7 @@ jobs: path: release-readiness - name: Download narrative (if the narrative job produced one) - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 continue-on-error: true with: name: release-readiness-narrative diff --git a/.github/workflows/pr-security-scan.yml b/.github/workflows/pr-security-scan.yml new file mode 100644 index 0000000000..62560d492e --- /dev/null +++ b/.github/workflows/pr-security-scan.yml @@ -0,0 +1,42 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Pre-commit security gate: fast, PR-side security scanners that run on +# every pull request. Scanner selection and policy are defined by the pinned +# ROCm/rocm-security-gh `security-baseline.yml` reusable workflow. +# +# Requests human-readable reports only: findings are uploaded as a build +# artifact and printed to the job summary, never to the Security tab, so +# fork PRs (which never receive elevated tokens) behave identically to +# same-repo PRs. See the scheduled counterpart (weekly-security-scan.yml) +# for the SARIF / code-scanning upload. + +name: Security scan (PR) + +on: + pull_request: + # The default types plus ready_for_review, so a draft marked ready without + # a new push still gets scanned. + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + security: + name: Security scan + permissions: + actions: read + contents: read + pull-requests: write + uses: ROCm/rocm-security-gh/.github/workflows/security-baseline.yml@35d3060c4b2ed48f950f85bd414a1302a431fc0b # v1.0.8 + with: + report_formats: human + bandit_config_path: .github/scan_tools_configs/bandit.yml + gitleaks_config_path: .github/scan_tools_configs/gitleaks.toml + trivy_config_path: .github/scan_tools_configs/trivy.yml + zizmor_config_path: .github/scan_tools_configs/zizmor.yml diff --git a/.github/workflows/queue-run.yml b/.github/workflows/queue-run.yml new file mode 100644 index 0000000000..deec8fb115 --- /dev/null +++ b/.github/workflows/queue-run.yml @@ -0,0 +1,60 @@ +# GPU E2E on the shared queue — start a run, wait for it, publish it. +# +# This job runs on the queue-report runner, which sits next to the queue. It asks +# for a run of one branch of this repository (main when scheduled), waits for +# every lane to finish (the lanes' limits, and the job's timeout that covers +# them, are in the comment next to `timeout-minutes` below), then +# prints the run's report and lane matrix, uploads the lanes' files, and passes +# or fails with the run's verdict. It needs no credential and no permission: +# the runner asks from where it stands. The runner is set up to run only this +# workflow and nightly-queue.yml as they are on main; that is enforced on the +# runner, outside this repository. +# What that protects is in docs/nightly-on-the-queue.md. +name: E2E on the queue + +on: + workflow_dispatch: + inputs: + branch: + description: "The branch of this repository to test" + type: string + default: main + # Whether this is a required check is a repository setting, not set here. + schedule: + - cron: "17 2 * * *" + +# Nothing here reads the repository. Uploading files uses the per-job runtime +# token GitHub issues to every job regardless of these permissions. +permissions: {} + +# One run at a time; the queue side refuses a second while one is unfinished. +concurrency: + group: queue-run + cancel-in-progress: false + +jobs: + run: + name: Run and publish + runs-on: [queue-report] + # A lane waits at most 15000 s (4 h 10 min) to be started, then runs for at + # most 3 h counted from its start, however long it waited. Finishing and + # publishing the run can take 30 min more, and 20 min are margin for the + # runner's own delays: 480 min. + timeout-minutes: 480 + steps: + - name: Run on the queue and publish + env: + # Empty when scheduled. + BRANCH: ${{ inputs.branch }} + # One line: the workflow contract forbids backslashes in this file. + run: /opt/queue/bin/queue-publish --out "${RUNNER_TEMP}/results" --start "${BRANCH:-main}" + + - name: Upload lane results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: queue-run-results + path: ${{ runner.temp }}/results + # The publisher's files only; a lane's hidden files are not published. + include-hidden-files: false + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 75a2386c1e..cca9730ba1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ on: required: true permissions: - contents: write + contents: read env: CARGO_TERM_COLOR: always @@ -46,18 +46,23 @@ env: jobs: release: + # Creates, uploads and deletes releases; the workflow default is read-only. + permissions: + contents: write name: Build Linux release assets runs-on: ubuntu-24.04 # one LTS: the assets are built in the manylinux container, not on the image outputs: version: ${{ steps.version.outputs.value }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 with: - # rust-cache keys on "Linux-x64" only; name the image so a target dir - # built against one Ubuntu release's glibc is never restored on another. - cache-key: ubuntu-24.04 + # No rust-cache restore/save: this job holds publishing credentials, + # so it builds from a clean tree rather than a restorable cache. + cache: false - name: Verify pinned keys match the signing key run: cargo xtask verify-pinned-keys @@ -65,7 +70,7 @@ jobs: - name: Determine version id: version run: | - VERSION="${{ github.event.inputs.tag }}" + VERSION="${GITHUB_EVENT_INPUTS_TAG}" if [ -z "${VERSION}" ]; then VERSION="${GITHUB_REF_NAME}" fi @@ -82,21 +87,20 @@ jobs: exit 1 ;; esac + # `rocm version` reports a release only for this shape; any other + # tag would ship binaries that call themselves 0.0.0-dev. Checked + # here so it fails before the builds rather than after them. The + # pattern and the length are RELEASE_TAG_PATTERN and + # RELEASE_VERSION_MAX_LEN (+1 for the v) in apps/rocm/build.rs, and + # its test `release_tag_grammar_is_the_same_everywhere` fails if + # they differ. + if ! [[ "${VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.+-]+)?$ ]] || [ "${#VERSION}" -gt 65 ]; then + echo "release version must be vX.Y.Z with an optional -pre or +build suffix of letters, digits, '.', '-' and '+', at most 65 characters, got: ${VERSION}" >&2 + exit 1 + fi echo "value=${VERSION}" >> "$GITHUB_OUTPUT" - - - name: Cache Cargo home and build artifacts - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/.cargo/bin - ~/.cargo/registry/cache - ~/.cargo/registry/index - ~/.cargo/registry/src - ~/.cargo/rustup - target - key: Linux-manylinux-release-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} - restore-keys: | - Linux-manylinux-release- + env: + GITHUB_EVENT_INPUTS_TAG: ${{ github.event.inputs.tag }} - name: Build release binaries run: | @@ -108,8 +112,7 @@ jobs: # Install system build deps as root: yum needs to write to # /var/cache/yum and the RPM db, which a non-root container user # cannot do. These land in the container's ephemeral layer (not the - # bind mount), so this runs every time; only the Cargo/rustup state - # below is cached. + # bind mount), so this runs every time. docker run --rm \ -v "$PWD":/work \ -w /work \ @@ -119,8 +122,8 @@ jobs: yum install -y pkgconfig libcap-devel ' - # Build as the runner user so cached Cargo/rustup files are owned by - # the runner (the host-side cache save step must be able to read them). + # Build as the runner user so Cargo/rustup files written into the bind + # mount are owned by the runner, not root. # # ROCM_CLI_RELEASE_REF is forwarded explicitly: this container sees # none of GitHub's own GITHUB_REF_* env, which `apps/rocm/build.rs` @@ -134,7 +137,7 @@ jobs: -u "$(id -u):$(id -g)" \ -e CARGO_HOME=/host-cargo \ -e RUSTUP_HOME=/host-cargo/rustup \ - -e ROCM_CLI_RELEASE_REF="${{ steps.version.outputs.value }}" \ + -e ROCM_CLI_RELEASE_REF="${STEPS_VERSION_OUTPUTS_VALUE}" \ -e AWS_LC_SYS_USE_SYSTEM \ -v "$HOME/.cargo":/host-cargo \ -v "$PWD":/work \ @@ -148,13 +151,33 @@ jobs: export PATH="/host-cargo/bin:$PATH" cargo build --release -p rocm -p rocmd -p xtask ' + env: + STEPS_VERSION_OUTPUTS_VALUE: ${{ steps.version.outputs.value }} + + # The tag shape of `rocm version` (`X.Y.Z (vX.Y.Z, )`) is otherwise + # unit-tested only: every PR and nightly build resolves a branch, which + # reports `0.0.0-dev`. Run the binary this release ships and check that it + # leads with the tag's own version; "Determine version" has already + # checked that the tag has the `vX.Y.Z` shape that makes it a release. + - name: Check the release binary's version + env: + VERSION: ${{ steps.version.outputs.value }} + run: | + OUTPUT="$(target/release/rocm --version)" + EXPECTED_PREFIX="rocm-cli ${VERSION#v} (${VERSION}, " + if [[ "${OUTPUT}" != "${EXPECTED_PREFIX}"*")" ]]; then + echo "release binary reports '${OUTPUT}', expected '${EXPECTED_PREFIX})'" >&2 + exit 1 + fi + echo "release binary reports: ${OUTPUT}" - name: Package release bundle env: ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} ROCM_CLI_REQUIRE_SIGNATURE: "1" + STEPS_VERSION_OUTPUTS_VALUE: ${{ steps.version.outputs.value }} run: | - VERSION="${{ steps.version.outputs.value }}" + VERSION="${STEPS_VERSION_OUTPUTS_VALUE}" DIST="rocm-cli-${VERSION}-linux-amd64" chmod +x install.sh @@ -179,8 +202,9 @@ jobs: - name: Create or update release env: GH_TOKEN: ${{ github.token }} + STEPS_VERSION_OUTPUTS_VALUE: ${{ steps.version.outputs.value }} run: | - VERSION="${{ steps.version.outputs.value }}" + VERSION="${STEPS_VERSION_OUTPUTS_VALUE}" NOTES="$(cat <)'" + } + Write-Host "release binary reports: $output" + - name: Package Windows bundle shell: pwsh env: ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} ROCM_CLI_REQUIRE_SIGNATURE: "1" + NEEDS_RELEASE_OUTPUTS_VERSION: ${{ needs.release.outputs.version }} run: | - $version = "${{ needs.release.outputs.version }}" + $version = "$env:NEEDS_RELEASE_OUTPUTS_VERSION" $dist = "rocm-cli-$version-windows-amd64" cargo xtask package $dist Copy-Item "dist\$dist.zip" "dist\rocm-cli-windows-amd64.zip" @@ -272,12 +318,16 @@ jobs: shell: pwsh env: GH_TOKEN: ${{ github.token }} + NEEDS_RELEASE_OUTPUTS_VERSION: ${{ needs.release.outputs.version }} run: | - $version = "${{ needs.release.outputs.version }}" + $version = "$env:NEEDS_RELEASE_OUTPUTS_VERSION" $assets = Get-ChildItem dist -File -Include *.zip,*.sha256,*.sig -Recurse | ForEach-Object { $_.FullName } gh release upload $version @assets --clobber publish-release: + # Creates, uploads and deletes releases; the workflow default is read-only. + permissions: + contents: write name: Publish release runs-on: ubuntu-24.04 # one LTS: publishing only needs: @@ -288,7 +338,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} + NEEDS_RELEASE_OUTPUTS_VERSION: ${{ needs.release.outputs.version }} run: | - VERSION="${{ needs.release.outputs.version }}" + VERSION="${NEEDS_RELEASE_OUTPUTS_VERSION}" RELEASE_ID="$(gh release view "${VERSION}" --json databaseId -q .databaseId)" gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" -f draft=false diff --git a/.github/workflows/weekly-security-scan.yml b/.github/workflows/weekly-security-scan.yml new file mode 100644 index 0000000000..a78036ddaa --- /dev/null +++ b/.github/workflows/weekly-security-scan.yml @@ -0,0 +1,35 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# +# Weekly security scan: runs on a fixed cadence and pushes SARIF findings +# to the repository's code-scanning Security tab. Complements +# `pr-security-scan.yml`, which scans only what changed in a PR and +# uploads reviewer-readable artifacts for in-review browsing. + +name: Security scan (Weekly) + +on: + # Run every Saturday at 10:00 UTC. + schedule: + - cron: "0 10 * * 6" + workflow_dispatch: + +permissions: + contents: read + +jobs: + security: + name: Security scan + # Only the job that uploads SARIF gets write access to code scanning. + permissions: + actions: read + contents: read + security-events: write + uses: ROCm/rocm-security-gh/.github/workflows/security-baseline.yml@35d3060c4b2ed48f950f85bd414a1302a431fc0b # v1.0.8 + with: + scan_mode: all + report_formats: sarif + bandit_config_path: .github/scan_tools_configs/bandit.yml + gitleaks_config_path: .github/scan_tools_configs/gitleaks.toml + trivy_config_path: .github/scan_tools_configs/trivy.yml + zizmor_config_path: .github/scan_tools_configs/zizmor.yml diff --git a/.gitignore b/.gitignore index e8ccf9a302..97f24b9b5a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,10 @@ __pycache__/ /plans/ /workspace/ /e2e-consolidated-report.md +# Default outputs of the rocm-dash-tui gen_cast and gen_screenshots examples when +# run from the repository root. +/rocm-dash.cast +/screenshots/ # Sphinx docs: _toc.yml is generated from _toc.yml.in at build time docs/rocm-docs/sphinx/_toc.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa8a4a489b..43120b797d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -119,6 +119,26 @@ repos: stages: [pre-commit] groups: [local-tools] # runs in the dedicated `docs-links` CI job + # Module map: every production source file needs an entry in + # docs/architecture.md. `lychee` above only checks that links already in the + # doc resolve; this is the other direction, so a new file nobody wrote an + # entry for fails. Pre-commit for fast feedback, pre-push for a commit made + # with --no-verify: it reads the working tree, not the pushed commits, so it + # only agrees with CI when the tree is clean, and CI is the gate. It reads the + # whole tree, so it is not narrowed to changed files: a rename or deletion + # anywhere can change the answer, and a delete-only commit has no changed + # files to match a filter. + - repo: local + hooks: + - id: architecture-doc + name: every source file is in docs/architecture.md + entry: cargo xtask check-architecture-doc + language: system + always_run: true + pass_filenames: false + stages: [pre-commit, pre-push] + groups: [local-tools] # runs in the `clippy` CI job (compiles xtask) + # Rust + PowerShell: mirror the CI checks using the local toolchain. - repo: local hooks: @@ -141,6 +161,11 @@ repos: # cargo clippy -p e2e-cucumber --test e2e -- -D warnings # cargo test --workspace --all-targets --exclude e2e-cucumber # and `cargo xtask pre-push-check --base ` runs one by hand. + # A step that already passed on the same tree (same cargo command, toolchain + # and environment, clean working tree) is skipped and says so; a green run + # records the pass under the git directory. `ROCM_PRE_PUSH_RERUN=1` (or + # `--rerun` by hand) ignores recorded passes, and `SKIP=` still + # skips a hook. - id: cargo-clippy name: cargo clippy entry: cargo xtask pre-push-check clippy @@ -222,20 +247,23 @@ repos: stages: [pre-commit] groups: [local-tools] # compiles xtask; CI covers it in the `clippy` job - id: cargo-module-size - name: Source files are within the module-size limit (advisory) - # Check-only: it never rewrites or restages anything. Advisory for now, so - # it prints the offenders and lets the commit through, because a blocking - # hook would fail the very commits that split an over-limit file. The - # advisory behaviour lives in the `-advisory` subcommand, not a flag on the - # check; making the limit blocking means pointing this entry at - # `cargo xtask check-module-size` and deleting the wrapper. Advisory only - # once xtask has built: a WIP commit that leaves xtask or rocm-core not - # compiling still fails here, as it does the other xtask hooks. - entry: cargo xtask check-module-size-advisory + name: Source files are within the module-size limit + # Check-only: it never rewrites or restages anything. Blocking, the local + # mirror of the module-size step in the `clippy` CI job: a source file over + # the limit fails the hook and stops the commit, and the output names the + # file and how to split it. It measures the working tree, so a Rust file under + # a crate's src/ that no `mod` declares fails it too, even an untracked scratch + # file, and the error says to declare or delete it. Only a commit that stages + # Rust source runs it, and only for someone who installed the hooks; CI stays + # the authority. Stage: pre-commit, like the other local checks, not pre-push. + # The consequence is that it also refuses the intermediate commits when an + # over-limit file is split across several commits (CI only sees the final + # state). Do the split in one commit, or commit the intermediate steps with + # `SKIP=cargo-module-size git commit ...`. + entry: cargo xtask check-module-size language: system types: [rust] pass_filenames: false - verbose: true # a passing hook is silent otherwise, and this one passes while it warns stages: [pre-commit] groups: [local-tools] # compiles xtask; CI covers it in the `clippy` job - id: cargo-tpn diff --git a/AGENTS.md b/AGENTS.md index 03a03d330c..6b972da2dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -227,6 +227,10 @@ Keep docs and behavior claims in sync while editing: file outside it), and a link leaving a skill folder in `skills/` (published or embedded elsewhere, §7). This covers citations you add or edit, not converting existing bare paths in an unrelated change +- when you add, rename or remove a production source file, update its entry in + `docs/architecture.md` in the same change; `cargo xtask check-architecture-doc` + fails on a file with no entry (§6), but not on an entry whose description + went stale - when a statement such as "X is not supported" or "there is no Y" becomes false, grep the same surfaces for the old claim before pushing - write the PR body from the final diff, not the first commit, and re-read it after @@ -246,7 +250,7 @@ Shared UI components — reuse rather than hand-rolling new ones. The CLI (`apps Shared (`crates/rocm-core`): -- [`confirm.rs`](crates/rocm-core/src/confirm.rs)'s **`print_prompt`**/**`read_answer`** — the shared `[y/N]` confirmation prompt: `print_prompt` writes and flushes a prompt (callers include their own trailing `[y/N]: `), `read_answer` reads a line and parses it as yes/no. Two fallible calls rather than one, so a caller can choose whether a failed write is fatal: [`fix/consent.rs`](crates/rocm-core/src/fix/consent.rs)'s `confirm()` attempts the read anyway, while [`uninstall_plan.rs`](apps/rocm/src/uninstall_plan.rs)'s `confirm_uninstall` and [`consent.rs`](apps/rocm/src/therock/consent.rs)'s `prompt_yes_no` treat it as fatal. [`uninstall.rs`](apps/rocm/src/uninstall.rs), [`storage.rs`](apps/rocm/src/storage.rs) and [`runtimes_cmd.rs`](apps/rocm/src/runtimes_cmd.rs) reach the prompt only through `confirm_uninstall`, each behind `require_yes_or_interactive` (below); [`fix/consent.rs`](crates/rocm-core/src/fix/consent.rs)'s `confirm()` additionally layers its own non-interactive short-circuit and refusal wording on top +- [`confirm.rs`](crates/rocm-core/src/confirm.rs)'s **`print_prompt`**/**`read_answer`** — the shared `[y/N]` confirmation prompt: `print_prompt` writes and flushes a prompt (callers include their own trailing `[y/N]: `), `read_answer` reads a line and parses it as yes/no, and writes to the stream it is given a newline when the input ended without one and a line saying so when the answer was neither yes nor no (declined either way). Two fallible calls rather than one, so a caller can choose whether a failed write is fatal: [`fix/consent.rs`](crates/rocm-core/src/fix/consent.rs)'s `confirm()` attempts the read anyway, while [`uninstall_plan.rs`](apps/rocm/src/uninstall_plan.rs)'s `confirm_uninstall` and [`consent.rs`](apps/rocm/src/therock/consent.rs)'s `prompt_yes_no` treat it as fatal. [`uninstall.rs`](apps/rocm/src/uninstall.rs), [`storage.rs`](apps/rocm/src/storage.rs) and [`runtimes_cmd.rs`](apps/rocm/src/runtimes_cmd.rs) reach the prompt only through `confirm_uninstall`, each behind `require_yes_or_interactive` (below); [`fix/consent.rs`](crates/rocm-core/src/fix/consent.rs)'s `confirm()` additionally layers its own non-interactive short-circuit and refusal wording on top - [`confirm.rs`](crates/rocm-core/src/confirm.rs)'s **`require_yes_or_interactive(yes, action)`** — the guard in front of a mutating command that confirms with that `[y/N]` prompt. `Ok(true)` means `--yes` already approved it (skip the prompt), `Ok(false)` means there is no `--yes` but a terminal, so the caller still asks, and otherwise it bails with `"{action} requires --yes outside an interactive terminal"`. A new `--yes`-gated command reuses it instead of re-deriving the `yes` / `interactive_terminal()` / `bail!` sequence. Not for a refusal that deliberately differs: [`therock/consent.rs`](apps/rocm/src/therock/consent.rs)'s `sdk_install_approval` names two consents that are not interchangeable, and [`fix/consent.rs`](crates/rocm-core/src/fix/consent.rs)'s `confirm()` has its own exit code and checks stdin only CLI: @@ -263,17 +267,18 @@ dash-TUI, in `crates/rocm-dash-tui/src/ui/`: - **[`spinner.rs`](crates/rocm-dash-tui/src/ui/spinner.rs)**'s `spinner_frame`/`parse_progress_pct` — the braille animation and percentage parsing `job_console.rs` renders with - **[`panel.rs`](crates/rocm-dash-tui/src/ui/panel.rs)**'s `bento`/`popup` — bordered, titled panels; prefer these over an inline `Block::default()`. Popup geometry and framing live here too: `centered_rect`/`centered_width` size and place a popup, `draw_popup_frame` clears and frames it, and `grey_overlay` dims the backdrop - **[`theme.rs`](crates/rocm-dash-tui/src/ui/theme.rs)**'s `StatusTone` and `Theme::tone_color` — the shared status→color mapping. A status enum gets one exhaustive `fn …_tone(status) -> StatusTone` beside them (`job_status_tone`, `instance_status_tone`) and a `Theme::…_color` wrapper (`job_status_color`, `instance_status_color`); screens call the wrapper (or `tone_color(…)` on a tone) instead of an inline `match … => theme.ok/warn/err`. To color a new status enum, add its tone function there rather than a bespoke match per screen -- **[`widgets.rs`](crates/rocm-dash-tui/src/ui/widgets.rs)**'s `apply_char_key`/`apply_char_key_filtered` — the append-only text-field edit (`Char` appends, `Backspace` pops one char); call it from a form's key handler instead of writing another `push`/`pop` match arm. The `_filtered` form takes a predicate to refuse some chars (a digits-only port); there is no cursor or mid-string editing. The Chat tab's input is the one exception: its keys are first mapped to a `KeyAction` by the pure, separately tested table in [`actions.rs`](crates/rocm-dash-tui/src/app/actions.rs), which also applies them +- **[`widgets.rs`](crates/rocm-dash-tui/src/ui/widgets.rs)**'s `apply_char_key`/`apply_char_key_filtered` — the append-only text-field edit (`Char` appends, `Backspace` pops one char); call it from a form's key handler instead of writing another `push`/`pop` match arm. The `_filtered` form takes a predicate to refuse some chars (a digits-only port); there is no cursor or mid-string editing. The Chat tab's input is the one exception: its keys are first mapped to a `KeyAction` by the pure, separately tested table in [`actions.rs`](crates/rocm-dash-tui/src/app/actions.rs), which also applies them. `cargo test -p xtask dash_text_edit_contract` ([`dash_text_edit_contract.rs`](xtask/src/dash_text_edit_contract.rs)) fails on a hand-rolled `push`/`pop` text edit under `crates/rocm-dash-tui/src/` outside `widgets.rs`; a new exception is a reviewed entry in that test's `ALLOWED` list. - **[`scroll.rs`](crates/rocm-dash-tui/src/app/scroll.rs)**'s `ScrollState` (in `app/`, not `ui/`, beside the `AppState` scroll methods) — an offset plus its upper bound, whose `scroll_by(delta)` is the one "add a signed delta, clamp into `[0, max]`, saturate to `u16`" implementation. The job console (both axes), the LOGS dock and the instance Detail modal clamp through it; a new scrollable surface that keeps a `u16` offset with a known bound builds a `ScrollState` from them and stores `pos` back instead of repeating the `i32` clamp arithmetic. The chat transcript and the GPU list clamp with a `min()` against a measured bound (chat also follows the tail) and stay out of it **Two different components are both informally "the spinner"; they are not interchangeable.** The CLI's [`cli_progress.rs`](apps/rocm/src/cli_progress.rs) `Spinner`/`AnimatedSpinner` above is TTY/stderr-gated for a plain terminal and lives in `apps/rocm`. The dash-TUI's [`spinner.rs`](crates/rocm-dash-tui/src/ui/spinner.rs) above is a pure frame/percentage helper rendered through `job_console.rs` inside a ratatui frame loop, and lives in `crates/rocm-dash-tui`. A dash-TUI screen needs the latter. Guardrails: -- new subsystems/subcommands: default their domain implementation to its own file from day one (full domain extraction, e.g. `therock.rs`/`comfyui.rs` — private `mod` in `apps/rocm`, accessed via qualified paths; the clap command enum usually goes in [`cli.rs`](apps/rocm/src/cli.rs) with the rest of the command tree, and its dispatch function usually stays in `main.rs`; `bootstrap.rs` and `remote/mod.rs` are the exceptions that own their enum and dispatch outright — see `docs/architecture.md`'s `bootstrap.rs` note), not growth inside `main.rs`/`lib.rs` awaiting a future extraction pass; see `docs/architecture.md` for the module map and the mechanical-relocation alternative used for dispatch-adjacent clusters +- new subsystems/subcommands: default their domain implementation to its own file from day one (full domain extraction, e.g. `therock.rs`/`comfyui.rs` — private `mod` in `apps/rocm`, accessed via qualified paths; the clap command enum usually goes in [`cli.rs`](apps/rocm/src/cli.rs) with the rest of the command tree, and its dispatch function goes in the domain module (`comfyui::run`) or a `*_cmd.rs` sibling, with only the top-level `dispatch()` match staying in `main.rs`; `bootstrap.rs` and `remote/mod.rs` are the exceptions that own their enum as well — see `docs/architecture.md`'s `bootstrap.rs` note), not growth inside `main.rs`/`lib.rs` awaiting a future extraction pass; see `docs/architecture.md` for the module map and the mechanical-relocation alternative used for dispatch-adjacent clusters - `crates/rocm-engine-protocol` is a contract surface; verify all impacted engines after protocol changes - first-party crate-layering invariants (e.g. `rocmd` must never depend on `rocm`) are enforced by `cargo xtask check-crate-edges` (`xtask/src/crate_edges.rs`); a new first-party dependency edge failing that check means the edge needs review, not a bypass -- source file size is checked by `cargo xtask check-module-size` ([`module_size.rs`](xtask/src/module_size.rs)): no source file may exceed 1,500 production lines (test-gated items excluded), with one limit for every file and no exemption mechanism. A PR that would push a file past it splits the file in the same change; do not raise the limit or opt a file out. Merely editing a file that is already over the limit does not by itself require a split: those files are split under their own tickets, and the check cannot become blocking until none is left. Today the CI step and pre-commit hook only report; see `docs/architecture.md` +- source file size is checked by `cargo xtask check-module-size` ([`module_size.rs`](xtask/src/module_size.rs)): no source file may exceed 1,500 production lines (test-gated items excluded), with one limit for every file and no exemption mechanism. A PR that would push a file past it splits the file in the same change; do not raise the limit or opt a file out. The check blocks merges: the `clippy` CI job, a required check that also runs on merge-queue runs, fails when any source file is over the limit, so a file over it is split before anything else merges. The `cargo-module-size` pre-commit hook is its local mirror: it runs the same check and refuses the commit, for anyone who installed the hooks; see `docs/architecture.md` +- every production source file (a `.rs` file under `apps/*/src`, `crates/*/src` or `engines/*/src`), and every `.rs` file under `xtask/src`, `tests/e2e-cucumber/src` and `tests/e2e-cucumber/tests`, has an entry in `docs/architecture.md`, enforced by `cargo xtask check-architecture-doc` ([architecture_doc.rs](xtask/src/architecture_doc.rs)) in the `clippy` CI job and in the `architecture-doc` prek hook at commit and push. Add the entry in the same change as the file. The check has no exemption list: document the file. The check reads no prose, so keeping an entry's description true is still on you - preserve strict GPU-required behavior; do not introduce silent CPU fallback - respect platform gates (for example, native Windows handling for vLLM) - pin third-party GitHub Actions to a full commit SHA with a trailing `# vX.Y.Z` comment, never a moving tag (`@v2`, `@main`); a retagged or compromised action otherwise enters CI silently. Bump the SHA and comment together when upgrading @@ -319,7 +324,7 @@ Required consistency points: ### `skills/rocm-doctor/` — published from here, and a test fixture `skills/rocm-cli-assistant/SKILL.md` is compiled into the binary -(`include_str!` in `apps/rocm/src/main.rs`). `skills/rocm-doctor/` is different +(`include_str!` in `apps/rocm/src/chat/turn.rs`). `skills/rocm-doctor/` is different on two counts, and both change how you edit it: - **This repo is its source of truth.** The skill is a thin driver over the @@ -394,7 +399,7 @@ Minimum quality gate before upstream-ready status: ```bash cargo test --workspace --all-targets cargo clippy --workspace --all-targets -- -D warnings -python scripts/smoke_local.py +cargo xtask smoke ``` Deleting or weakening a test is a change in its own right: every crate has a committed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2da2536192..3354c2bad5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ prek install # fast checks on every commit prek install -t pre-push # heavier checks on push (clippy + tests of the crates the push affects) ``` -`prek` runs the same checks locally that CI enforces: `cargo fmt`, `clippy`, `cargo test`, `ruff` (Python), `shellcheck` (shell), PowerShell syntax, license headers (`hawkeye`), markdown links (`lychee`), and the generated files (`MANIFEST.md`, `THIRD_PARTY_NOTICES.txt`, and the published Doctor catalog, `doctor-catalog.json`). +`prek` runs the same checks locally that CI enforces: `cargo fmt`, `clippy`, `cargo test`, `ruff` (Python), `shellcheck` (shell), PowerShell syntax, license headers (`hawkeye`), markdown links (`lychee`), the module map (`cargo xtask check-architecture-doc`), and the generated files (`MANIFEST.md`, `THIRD_PARTY_NOTICES.txt`, and the published Doctor catalog, `doctor-catalog.json`). The pre-push hooks (`cargo clippy`, the e2e harness clippy, and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. Each step prints the scope it chose and why. @@ -57,6 +57,8 @@ The hooks check the whole workspace when: The hooks also add the crates whose tests read files from outside their own crate. Any Rust file adds `xtask`, whose contract tests scan every source file. `README.md`, `docs/`, and `skills/` add `rocm` and `xtask`. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). +A step that already passed on this exact content is skipped, and says so: after a green run it records a stamp keyed on the git tree hash of `HEAD`, the exact cargo command (so a pass for a narrow selection never covers a wider one), the toolchain and the `CARGO_*` and `RUST*` environment, under the clone's `.git` directory where every worktree shares it, so a retried push, a manual run before the push or an amend that only rewords the message does not pay for the same checks twice. Stamps are only used and written when the working tree is clean (cargo checks the working tree, not the pushed commit, and an untracked file can hide a missing `git add`), expire after seven days, and do not cover your user-level cargo configuration, installed tools, files git ignores or flaky tests. A skip shows that this tree passed those checks earlier, not that they ran during this push; the `cargo test --workspace` and `cargo clippy --workspace` commands you run by hand are unaffected, and `--rerun` gives a fresh run when you want proof the checks ran for this push. `cargo xtask pre-push-check --rerun`, or `ROCM_PRE_PUSH_RERUN=1` for the hook, ignores them, drops this step's earlier pass so a failed rerun cannot leave it to skip the next run, and records a fresh pass when the step succeeds; `SKIP=` still skips a hook outright, and the commit-signature hook runs on every push. + A green hook is an early warning, not the gate. On pull requests, CI lints the whole workspace, but its Linux test job runs only the affected crates. The whole workspace's tests run in the Windows debug job, in the merge queue, and on `main`. To see what a push would check without compiling anything, run the following command (the steps are `clippy`, `clippy-e2e`, and `test`): ```bash @@ -104,9 +106,11 @@ The workspace contains these top-level components: New subcommands and subsystems default to their own file from day one. Don't let them grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. See [`docs/architecture.md`](https://github.com/ROCm/rocm-cli/blob/main/docs/architecture.md) for the two extraction patterns in use, the current module map, and the module-organization convention in full. -The `docs-links` job below checks the file links in that document, including every concrete file citation in the per-crate inventories. It checks that they resolve, not that the prose is accurate. Its section headings and directory names stay bare and unchecked. +The `docs-links` job below checks the file links in that document, including every concrete file citation in the per-crate inventories. It checks that they resolve, not that the prose is accurate. Every production source file must also appear in it (see below). Its section headings and directory names stay bare and unchecked. + +Crate-layering invariants, such as `rocmd` never depending on `rocm`, are enforced by `cargo xtask check-crate-edges` (`xtask/src/crate_edges.rs`). File size is checked by `cargo xtask check-module-size` (`xtask/src/module_size.rs`): no source file may exceed 1,500 production lines, test code excluded, and a change that would cross the limit splits the file in the same change instead. The `clippy` CI job fails on a file over the limit, and the `cargo-module-size` pre-commit hook runs the same check locally and refuses the commit, so it is the local mirror of the CI step (it only runs if you installed the hooks). It measures the working tree, so it also fails on a Rust file under a crate's `src/` that no `mod` declares, even an untracked scratch file, and it refuses every intermediate commit of a split spread over several commits, where `SKIP=cargo-module-size git commit ...` commits those steps; `docs/architecture.md` has the details. -Crate-layering invariants, such as `rocmd` never depending on `rocm`, are enforced by `cargo xtask check-crate-edges` (`xtask/src/crate_edges.rs`). File size is checked by `cargo xtask check-module-size` (`xtask/src/module_size.rs`): no source file may exceed 1,500 production lines, test code excluded, and a change that would cross the limit splits the file instead. Editing a file that is already over the limit does not by itself require a split; those files are split under their own tickets. For now the CI step and the `cargo-module-size` pre-commit hook report offenders without failing. +Every production source file, meaning a `.rs` file under `apps/*/src`, `crates/*/src` or `engines/*/src`, must have an entry in `docs/architecture.md`: a relative markdown link to it, which has to render as a link (a path inside an HTML comment, a code block or a code span does not count, and neither does a malformed link such as one missing its `)`, or an image). Use an inline `[text](path)` link with its text on one line; reference-style links, a `?query` suffix and root-relative paths are not recognised. `build.rs` files and the `tests/` and `examples/` trees of those crates are out of scope. The same holds for every `.rs` file under `xtask/src`, `tests/e2e-cucumber/src` and `tests/e2e-cucumber/tests`, which also need an entry. `cargo xtask check-architecture-doc` (`xtask/src/architecture_doc.rs`) enforces that in the `clippy` CI job and in the `architecture-doc` prek hook, which runs on every commit and every push, over the whole tree. A file with no entry fails it, and there is no exemption list. This is the opposite direction to the `lychee` check below, which only confirms that links already in the doc still resolve. Test-only helpers (`test_support.rs`, `test_env.rs`, `*_proptests.rs`, `*_props.rs`, `*_properties.rs`) of those product crates need no entry; the `xtask` and `tests/e2e-cucumber` trees have no such exemption. The check does not read the prose of an entry, so keeping a description true when a file's role changes is still up to you. Every local (relative-path) Markdown link and `#anchor` fragment in the repository (outside `docs/rocm-docs/`, which `docs-build` covers instead, and hidden directories such as `.github/`, which neither check walks) is checked by the `docs-links` CI job and by the `lychee` prek hook above, both configured by `lychee.toml` (which lists the current exclusions). Both run offline only, so they don't catch broken external `https://` links. diff --git a/Cargo.lock b/Cargo.lock index 7071a703eb..ed1f160cf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -259,7 +259,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -299,7 +299,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -615,9 +615,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" dependencies = [ "find-msvc-tools", "jobserver", @@ -1046,7 +1046,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.118", + "syn 2.0.119", "synthez", ] @@ -1084,7 +1084,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1095,7 +1095,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1153,7 +1153,7 @@ checksum = "74ef43543e701c01ad77d3a5922755c6a1d71b22d942cb8042be4994b380caff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1175,7 +1175,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", "unicode-xid", ] @@ -1239,7 +1239,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1284,6 +1284,7 @@ dependencies = [ "rocm-core", "serde", "serde_json", + "sha2", "shlex", "tempfile", "tokio", @@ -1348,7 +1349,7 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1686,7 +1687,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 2.0.119", "textwrap", "thiserror 2.0.21", "typed-builder", @@ -2154,7 +2155,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2220,7 +2221,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2239,7 +2240,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2447,7 +2448,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2681,7 +2682,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2895,7 +2896,7 @@ dependencies = [ "by_address", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2999,7 +3000,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3052,7 +3053,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3081,7 +3082,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3204,9 +3205,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -3219,7 +3220,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "version_check", ] @@ -3548,7 +3549,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3716,6 +3717,7 @@ dependencies = [ "keyring-core", "libc", "proptest", + "regex", "rocm-core", "rocm-dash-core", "rocm-dash-daemon", @@ -3741,7 +3743,6 @@ dependencies = [ "unicode-width", "ureq", "windows-native-keyring-store", - "windows-sys 0.61.2", "zbus-secret-service-keyring-store", ] @@ -3763,7 +3764,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "syn 2.0.118", + "syn 3.0.9", "sysinfo", "tar", "tempfile", @@ -3780,7 +3781,6 @@ dependencies = [ "bollard", "chrono", "csv", - "regex", "reqwest 0.12.28", "rocm-core", "rocm-dash-core", @@ -3905,6 +3905,9 @@ dependencies = [ name = "rocm-engine-protocol" version = "0.1.0" dependencies = [ + "anyhow", + "libc", + "proptest", "rocm-core", "serde", "serde_json", @@ -3928,8 +3931,10 @@ name = "rocmd" version = "0.1.0" dependencies = [ "anyhow", + "async-stream", "axum", "clap", + "libc", "rocm-core", "rocm-engine-protocol", "serde", @@ -3937,7 +3942,6 @@ dependencies = [ "sha2", "tokio", "ureq", - "windows-sys 0.61.2", ] [[package]] @@ -4166,7 +4170,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4183,7 +4187,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4296,7 +4300,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4331,14 +4335,14 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "serde_spanned" -version = "1.1.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +checksum = "9c187e0a369471c550af028cba56d4f4fa2a489ad28151ae95a4d340a2a8b782" dependencies = [ "serde_core", ] @@ -4374,7 +4378,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4530,7 +4534,7 @@ checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4601,7 +4605,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4629,9 +4633,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -4666,7 +4670,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4675,7 +4679,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d8a928f38f1bc873f28e0d2ba8298ad65374a6ac2241dabd297271531a736cd" dependencies = [ - "syn 2.0.118", + "syn 2.0.119", "synthez-codegen", "synthez-core", ] @@ -4686,7 +4690,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fb83b8df4238e11746984dfb3819b155cd270de0e25847f45abad56b3671047" dependencies = [ - "syn 2.0.118", + "syn 2.0.119", "synthez-core", ] @@ -4699,7 +4703,7 @@ dependencies = [ "proc-macro2", "quote", "sealed", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4885,7 +4889,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4991,7 +4995,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5036,9 +5040,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.6+spec-1.1.0" +version = "1.1.8+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" +checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" dependencies = [ "indexmap", "serde_core", @@ -5051,9 +5055,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" dependencies = [ "serde_core", ] @@ -5072,18 +5076,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.1.2+spec-1.1.0" +version = "1.2.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +checksum = "009dbb7700f11c55ef83aebbe4c0e9cf1f1ac0844bd8d6f00dc38f302026e13a" [[package]] name = "tower" @@ -5189,7 +5193,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5308,7 +5312,7 @@ checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5383,9 +5387,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" [[package]] name = "unicode-xid" @@ -5579,7 +5583,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -5808,7 +5812,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5819,7 +5823,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6128,12 +6132,13 @@ dependencies = [ "e2e-report", "flate2", "proc-macro2", + "regex", "rocm-core", "semver", "serde", "serde_json", "sha2", - "syn 2.0.118", + "syn 3.0.9", "tar", "tempfile", "toml", @@ -6159,7 +6164,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -6219,7 +6224,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "zbus_names", "zvariant", "zvariant_utils", @@ -6253,7 +6258,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6273,7 +6278,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -6313,19 +6318,20 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "zip" -version = "9.1.2" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2e3161de27091f18a831d9494e9c53378a34833af5c1dda35ad85849da18d6" +checksum = "eaeb7f16d2605b49aef586996ded29e65642fab7259a3077e537351df6d80426" dependencies = [ "crc32fast", "flate2", "getrandom 0.4.3", "indexmap", + "libc", "memchr", "nix 0.31.3", "typed-path", @@ -6380,7 +6386,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "zvariant_utils", ] @@ -6393,6 +6399,6 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.118", + "syn 2.0.119", "winnow", ] diff --git a/Cargo.toml b/Cargo.toml index 5802b57f51..4a9b2d03cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,7 +62,7 @@ sha2 = { version = "0.10", features = ["oid"] } shlex = "2" # xtask's module-size check parses each source file with it. `printing` is what # `Spanned` needs. -syn = { version = "2", default-features = false, features = ["full", "parsing", "printing"] } +syn = { version = "3", default-features = false, features = ["full", "parsing", "printing"] } # `linux-tmpfs` makes tmpfs mounts visible to the free-space preflight, so a # cache or install root under a tmpfs `/tmp` (the Fedora and Arch default) # reports its real size instead of being invisible. `linux-netdevs` is diff --git a/MANIFEST.md b/MANIFEST.md index 1911556f74..ca33e22907 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -93,7 +93,7 @@ repository. | bytes | 1.12.0 | MIT | | castaway | 0.2.4 | MIT | | cbc | 0.1.2 | MIT OR Apache-2.0 | -| cc | 1.10.0 | MIT OR Apache-2.0 | +| cc | 1.11.0 | MIT OR Apache-2.0 | | cfg-if | 1.0.4 | MIT OR Apache-2.0 | | cfg_aliases | 0.1.1 | MIT | | cfg_aliases | 0.2.1 | MIT | @@ -356,7 +356,7 @@ repository. | powerfmt | 0.2.0 | MIT OR Apache-2.0 | | ppv-lite86 | 0.2.21 | MIT OR Apache-2.0 | | proc-macro-crate | 3.5.0 | MIT OR Apache-2.0 | -| proc-macro2 | 1.0.106 | MIT OR Apache-2.0 | +| proc-macro2 | 1.0.107 | MIT OR Apache-2.0 | | proc-macro2-diagnostics | 0.10.1 | MIT/Apache-2.0 | | proptest | 1.12.0 | MIT OR Apache-2.0 | | quick-xml | 0.39.4 | MIT | @@ -429,7 +429,7 @@ repository. | serde_json | 1.0.154 | MIT OR Apache-2.0 | | serde_path_to_error | 0.1.20 | MIT OR Apache-2.0 | | serde_repr | 0.1.20 | MIT OR Apache-2.0 | -| serde_spanned | 1.1.1 | MIT OR Apache-2.0 | +| serde_spanned | 1.1.3 | MIT OR Apache-2.0 | | serde_urlencoded | 0.7.1 | MIT/Apache-2.0 | | serde_with | 3.21.0 | MIT OR Apache-2.0 | | serde_with_macros | 3.21.0 | MIT OR Apache-2.0 | @@ -463,7 +463,7 @@ repository. | subtle | 2.6.1 | BSD-3-Clause | | symlink | 0.1.0 | MIT/Apache-2.0 | | syn | 1.0.109 | MIT OR Apache-2.0 | -| syn | 2.0.118 | MIT OR Apache-2.0 | +| syn | 2.0.119 | MIT OR Apache-2.0 | | syn | 3.0.9 | MIT OR Apache-2.0 | | sync_wrapper | 1.0.2 | Apache-2.0 | | synstructure | 0.13.2 | MIT | @@ -497,11 +497,11 @@ repository. | tokio-rustls | 0.26.4 | MIT OR Apache-2.0 | | tokio-tungstenite | 0.28.0 | MIT | | tokio-util | 0.7.20 | MIT | -| toml | 1.1.6+spec-1.1.0 | MIT OR Apache-2.0 | -| toml_datetime | 1.1.1+spec-1.1.0 | MIT OR Apache-2.0 | +| toml | 1.1.8+spec-1.1.0 | MIT OR Apache-2.0 | +| toml_datetime | 1.1.3+spec-1.1.0 | MIT OR Apache-2.0 | | toml_edit | 0.25.12+spec-1.1.0 | MIT OR Apache-2.0 | -| toml_parser | 1.1.3+spec-1.1.0 | MIT OR Apache-2.0 | -| toml_writer | 1.1.2+spec-1.1.0 | MIT OR Apache-2.0 | +| toml_parser | 1.1.5+spec-1.1.0 | MIT OR Apache-2.0 | +| toml_writer | 1.2.0+spec-1.1.0 | MIT OR Apache-2.0 | | tower | 0.5.3 | MIT | | tower-http | 0.6.11 | MIT | | tower-http | 0.7.1 | MIT | @@ -530,7 +530,7 @@ repository. | unicode-linebreak | 0.1.5 | Apache-2.0 | | unicode-segmentation | 1.13.3 | MIT OR Apache-2.0 | | unicode-truncate | 2.0.1 | MIT OR Apache-2.0 | -| unicode-width | 0.2.2 | MIT OR Apache-2.0 | +| unicode-width | 0.2.3 | MIT OR Apache-2.0 | | unicode-xid | 0.2.6 | MIT OR Apache-2.0 | | untrusted | 0.9.0 | ISC | | ureq | 2.12.1 | MIT OR Apache-2.0 | @@ -624,7 +624,7 @@ repository. | zerotrie | 0.2.4 | Unicode-3.0 | | zerovec | 0.11.6 | Unicode-3.0 | | zerovec-derive | 0.11.3 | Unicode-3.0 | -| zip | 9.1.2 | MIT | +| zip | 9.3.0 | MIT | | zlib-rs | 0.6.7 | Zlib | | zmij | 1.0.21 | MIT | | zopfli | 0.8.3 | Apache-2.0 | @@ -663,12 +663,20 @@ ROCm distribution. Two install formats are supported: - **Wheel format** — Python wheel packages (`rocm`, `torch`, `torchvision`, `torchaudio`) are resolved from AMD-hosted PyPI-compatible indexes and installed via `uv` into a managed virtual environment. Release channel wheels - are served from `https://repo.amd.com/rocm/whl-multi-arch`. Nightly channel - wheels are served from `https://rocm.nightlies.amd.com/whl-multi-arch`. + are served from `https://stable.repo.amd.com/rocm/whl-next`. Nightly channel + wheels are served from `https://nightly.repo.amd.com/rocm/whl-next`. An + earlier ROCm 7 release is served from the earlier stream, + `https://repo.amd.com/rocm/whl-multi-arch`, and a nightly built before + 2026-08-23 from `https://rocm.nightlies.amd.com/whl-multi-arch`; the CLI reads + those only when the install pins such a build. - **Tarball format** — Prebuilt SDK tarballs are downloaded from AMD-hosted artifact storage. Release channel tarballs are served from - `https://repo.amd.com/rocm/tarball/`. Nightly tarballs are served from - `https://rocm.nightlies.amd.com/tarball/`. The metadata index files for + `https://stable.repo.amd.com/rocm/core/tarball/`. Nightly tarballs are served + from `https://nightly.repo.amd.com/rocm/core/tarball/`. The earlier streams, + `https://repo.amd.com/rocm/tarball/` and + `https://rocm.nightlies.amd.com/tarball/`, are read only when the install pins + a build that exists only there, and a tarball install can pin a version only + against the current catalogs. The metadata index files for tarball installs are optionally verified with an RSA/SHA-256 detached signature (controlled by `ROCM_CLI_METADATA_PUBLIC_KEY_PATH` / `ROCM_CLI_METADATA_PUBLIC_KEY_PEM`); the tarball binary itself is not @@ -732,9 +740,9 @@ contributors and is part of the public source distribution. ### `scripts/` Directory -The `scripts/` directory contains Python and shell scripts for local smoke -testing, GPU integration tests, release packaging, and CI support. These are -development tools, not shipped to end users. +The `scripts/` directory contains Python and shell scripts for GPU integration +tests, release packaging, and CI support. These are development tools, not +shipped to end users. The local smoke test is `cargo xtask smoke`. ## Media Assets diff --git a/README.md b/README.md index d3c45ec460..d7461dfee1 100644 --- a/README.md +++ b/README.md @@ -31,16 +31,17 @@ inference engine adapters for Lemonade and vLLM. | Platform | Prebuilt binary | Notes | |---|---|---| | Linux (x86_64) | Yes | Ubuntu 24.04 or newer; full support, including the live dashboard and both inference engines | -| Windows (x86_64) | Yes | CLI and Lemonade serving; live dashboard shows GPU/CPU/memory but not the serving-instance registry; no vLLM | +| Windows (x86_64) | Yes | CLI and Lemonade serving; live dashboard shows GPU/CPU/memory and running managed servers, without per-server throughput; no vLLM | | WSL2 (x86_64) | Yes (Linux binary) | Ubuntu 24.04 or newer; full support, including the live dashboard | | macOS | No | No official installer, release, CI, or QA coverage | For WSL2 setup and diagnosis, see [docs/wsl.md](docs/wsl.md). -Live dashboard telemetry works on Windows too, but shows only GPU/CPU/memory. -The serving-instance registry view needs the `rocmdashd` Unix socket, so it's -Linux/WSL2 only (see [Interactive interfaces](#interactive-interfaces)). vLLM +Live dashboard telemetry works on Windows too: GPU/CPU/memory and the managed +servers `rocm services list` reports. Per-server throughput and Docker-discovered +instances need the `rocmdashd` unix socket, so they're Linux/WSL2 only (see +[Interactive interfaces](#interactive-interfaces)). vLLM serving is Linux or WSL2 only (see [docs/vllm.md](docs/vllm.md)). The minimum supported Linux release, native or under WSL2, is Ubuntu 24.04. On @@ -83,9 +84,20 @@ add a demo. Until the demo-gifs workflow has run once, these links 404. The installer downloads a prebuilt bundle, verifies its SHA-256 checksum, installs the `rocm` and `rocmd` binaries into `~/.local/bin`, and adds that -directory to your shell `PATH`. Rerun it any time to upgrade. `rocm uninstall` -removes the binaries, and undoes the `PATH` change too when no other program -lives in that directory. +directory to your shell `PATH`. On Linux, when fish is installed (`fish` is on +`PATH`, or fish's config folder already exists), the installer also adds that +directory to `PATH` in fish's `config.fish` and installs fish completions, +including when the login shell is bash, zsh, or ksh. Fish's config folder is +`$XDG_CONFIG_HOME/fish` when that variable is an absolute path, else +`~/.config/fish`. A `completions/rocm.fish` that is already there and that the +installer did not write (your own file, or a link) is left as it is: the +installer prints a note that completions were not installed, and +`rocm uninstall` leaves it too. +Setting `ROCM_CLI_UPDATE_SHELL_PATH=0` skips every shell profile, fish's and +its completions included. Rerun the installer any time to upgrade. +`rocm uninstall` removes the binaries and the fish completions the installer +wrote for them, and undoes the `PATH` change too when no other program lives +in that directory. ### Linux and WSL2 (x86_64) @@ -100,6 +112,10 @@ This tracks the default `release` channel. For nightly builds, pass the curl -fsSL https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.sh | sh -s -- nightly ``` +The channel here is the one the `rocm` binary comes from. It is separate from +the ROCm SDK channel: `rocm install sdk --channel release|nightly` picks which +TheRock stream the SDK comes from, whichever channel installed the binary. + ### Windows (x86_64, PowerShell) ```powershell @@ -114,6 +130,9 @@ $env:ROCM_CLI_CHANNEL = "nightly" irm https://raw.githubusercontent.com/ROCm/rocm-cli/main/install.ps1 | iex ``` +As on Linux, this is the binary's channel only. The ROCm SDK channel is chosen +separately with `rocm install sdk --channel`. + ### Verify the installation Open a new terminal so the updated `PATH` takes effect, then run: @@ -123,10 +142,12 @@ rocm version ``` The output shows the ROCm CLI version with its release tag or branch and commit -hash, the ROCm SDK that ROCm CLI would use on this computer, and the GPU driver -version. A successful install prints the CLI version line. The SDK and driver -lines read `not detected` when none is found, which is expected on a computer -where you haven't installed ROCm or a driver yet. +hash (a release build leads with its release version, such as +`0.2.1 (v0.2.1, 1cde824)`; a build from `main` leads with `0.0.0-dev`), the ROCm +SDK that ROCm CLI would use on this computer, and the GPU driver version. A +successful install prints the CLI version line. The SDK and driver lines read +`not detected` when none is found, which is expected on a computer where you +haven't installed ROCm or a driver yet. If your shell can't find the `rocm` command, for example with `rocm: command not found`, confirm that the install directory is on your `PATH`. @@ -324,11 +345,39 @@ five tabs (switch with `Tab`/`Shift+Tab` or number keys `1`–`5`): | **Observe** | Live GPU utilization, instances, and benchmark telemetry | | **Chat** | Assistant chat backed by a local server or configured provider | -Live mode works on Windows too, but shows only GPU/CPU/memory. The -serving-instance registry needs the Unix domain socket telemetry, so that view -is Linux/WSL2 only. Use `rocm dash --demo` for a synthetic session that runs +Live mode works on Windows too, showing GPU/CPU/memory and the managed servers +`rocm services list` reports. Per-server throughput and Docker-discovered +instances need the Unix domain socket telemetry, so they are Linux/WSL2 only. +Use `rocm dash --demo` for a synthetic session that runs anywhere without a GPU or daemon. +On Linux and WSL, the embedded daemon also discovers loaded models from a +separately running local Lemonade server at `127.0.0.1:13305`. Configure +`dashboard.daemon.lemonade_host` and `dashboard.daemon.lemonade_port` in +`config.json` for another endpoint, or set `dashboard.daemon.enable_lemonade` +to `false` to disable external discovery. The file is `~/.rocm/config.json`, or +`config.json` in the directory named by `ROCM_CLI_CONFIG_DIR`. Editing it is the +only way to set these fields. Native Windows has no live dashboard telemetry, so it +has no external discovery. A healthy server with no loaded models +does not count as a running model. Discovery does not register an external server +as a managed service or give ROCm CLI ownership of its process. External models +are visible in Observe, but are excluded from the Services stop/restart controls. +The tok/s figure for a Lemonade model is the rate of the endpoint's most recent +completed request, not live throughput: it stays the same while the server is +idle. The dashboard explains it with a legend (`Lemonade: last req tok/s, not live`) +under the Instances table and the Services overlay rows, and a `last req` prefix in +the instance detail and the quick preview. Because it is not live, the node-wide +totals (the tok/W and tok/s heroes on Home and Observe) and the per-instance tok/W +leave it out, and the agent snapshot marks it with `gen_tps_last_request`. A managed Lemonade server that requires an API key stays listed with its +registry status and throughput unavailable; the dashboard never reads the key. + +The engine pane distinguishes a ROCm CLI-managed Lemonade installation from an +externally running server. When installation detection is unavailable, it shows +an unchecked status ("unknown, run `rocm engines list`") rather than claiming +Lemonade is not installed. Installation status does not depend on GPU telemetry. +It is read once, when the dashboard starts. After you install Lemonade from the +Engines overlay, restart the dashboard to refresh it. + ## Configure ROCm and serve a model Before serving a model, check whether a managed ROCm runtime is already @@ -351,11 +400,14 @@ This downloads TheRock ROCm wheels and a matching PyTorch stack into a managed environment. Running `rocm install sdk` creates a managed runtime alongside any legacy install. -ROCm CLI supports ROCm 7.14, 10.0, and 10.1. Without `--version`, -`rocm install sdk` currently installs ROCm 7.14. To install ROCm 10.0 or 10.1, -add `--version`. If ROCm CLI can't tell which exact GPU architecture this computer has, -or you want a different one, also add `--family` with the raw `gfx` code from -`rocm examine`. For example: `--version 10.1.0 --family gfx1200`. +Without `--version`, `rocm install sdk` installs the newest stable release +(ROCm 10.1 today) from `stable.repo.amd.com`, and `--channel nightly` installs the +newest nightly build from `nightly.repo.amd.com`. Earlier releases stay +installable by pinning them: `--version 7.14.1` installs the ROCm 7.14.1 release, +and `--channel nightly --build-date 2026-08-19` installs a nightly from before +the nightly channel moved on 2026-08-23. If ROCm CLI can't tell which exact GPU +architecture this computer has, or you want a different one, also add `--family` with the +raw `gfx` code from `rocm examine`. For example: `--family gfx1200`. Running `rocm install sdk` when a managed runtime is already the active default asks first, because the new install takes over as the active default; see @@ -385,17 +437,17 @@ form works depends on the engine your GPU selects. | `rocm` | Open the launcher menu (setup, serve, diagnose, chat, dashboard) | | `rocm examine` | Check GPU, ROCm install, engines, and managed folders | | `rocm diagnose` | Match this machine against known ROCm, PyTorch, and llama.cpp failure modes | -| `rocm diagnose --model ` | Say whether a model will run here, before downloading it | +| `rocm diagnose --model ` | Say whether a model will run here, before downloading it; for a GGUF, estimate the largest context that fits | | `rocm diagnose --report` | Show what this machine would contribute to a problem report, and send nothing | | `rocm fix []` | Apply a fix reported by `rocm diagnose` | | `rocm install sdk` | Install TheRock ROCm wheels into a managed Python environment | | `rocm install driver` | Install the AMD kernel driver on Linux; on Windows, report the display driver version and age (installs nothing) | -| `rocm serve ` | Start a local OpenAI-compatible model server | +| `rocm serve ` | Start a local OpenAI-compatible model server (plus an Anthropic `/v1/messages` endpoint, streaming or not) | | `rocm remote serve ` | Serve a model on another GPU machine over your private network | | `rocm dash` | Open the full-screen telemetry dashboard | | `rocm bench load --endpoint ` | Load-test a local OpenAI-compatible endpoint | | `rocm setup status` | Show first-time setup state | -| `rocm version` | Print the ROCm CLI version, release tag or branch, and commit hash, plus the ROCm SDK and GPU driver in use | +| `rocm version` | Print the ROCm CLI version, release tag or branch, and commit hash (a release `vX.Y.Z` reports `X.Y.Z`; a branch, including `main`, reports `0.0.0-dev`), plus the ROCm SDK and GPU driver in use | | `rocm completions ` | Print a shell completion script (bash, zsh, fish, elvish, powershell) | ## Commands @@ -437,8 +489,8 @@ Windows. The Vulkan driver version is not shown. ### Diagnose and fix ``` -rocm diagnose [--symptom TEXT] [--top N] [--json] [--distro [NAME] | --report [--send]] -rocm diagnose --model [--json] +rocm diagnose [--symptom TEXT | --symptom-file PATH] [--top N] [--json] [--distro [NAME] | --report [--send]] +rocm diagnose --model [--context-size N] [--kv-cache-type f16|q8_0|q4_0] [--json] rocm fix [] [--yes] [--dry-run] [--device-index N] [--json] ``` @@ -454,7 +506,49 @@ position. `rocm diagnose` takes these flags: -- `--symptom` takes raw error text to sharpen keyword scoring. +- `--symptom` takes raw error text to sharpen keyword scoring. Your shell + reads that text first: quotes, `$(...)` and a leading `-` in a pasted error + can be changed, run, or taken as a flag. +- `--symptom-file PATH` reads the same text from a file instead, or from + standard input when PATH is `-`, so no quoting is needed (a file named `-` + is passed as `./-`). It cannot be combined with `--symptom`. The text that + arrives is matched as `--symptom` would match it, minus one trailing line + break. A UTF-8 byte-order mark is dropped and a UTF-16 file with a + byte-order mark (what Windows PowerShell's `>` writes) is decoded; anything + else must be UTF-8. A file that cannot be read, input that is not text, or + input larger than 16 MiB is a usage error (exit code 2). With nothing piped + in, paste the error at the terminal and end it with Ctrl-D at the start of + a line (Ctrl-Z then Enter on Windows). + + ```bash + # bash (Linux, WSL): the quoted 'EOF' keeps the shell out of the paste + rocm diagnose --symptom-file - <<'EOF' + + EOF + ``` + + ```powershell + # PowerShell: save the clipboard to a file first. Windows PowerShell 5.1 + # pipes text to a program as ASCII, so `Get-Clipboard | rocm ...` would + # turn every non-ASCII character in the error into "?". + Get-Clipboard -Raw > error.txt + rocm diagnose --symptom-file error.txt + ``` + + ```bat + :: cmd.exe: save the error to a file first + rocm diagnose --symptom-file error.txt + ``` +- With either flag, a finding about the examined host can still outrank the + entry the text added to, so each match says whether the text contributed to + its score or evidence: `symptom_matched` in `--json` (always present, `false` + when neither `--symptom` nor `--symptom-file` was given). Any keyword hit + counts, however weak, so `true` means the text contributed, not that the + match explains the error. In the human-readable output, a `source: the + examined host's state alone` line marks each shown match the text did not + contribute to, once the text contributed to any match in the report. That + includes one past `--top`, so all the rows shown can carry the line when the + only match the text contributed to was cut off. - On Windows, `diagnose` also checks the age of the Visual C++ runtime, not only whether it is there. The runtime in `System32` must be 14.40 or newer (`fix-21-msvc-redist-outdated`). Programs built with a recent toolset, such @@ -474,7 +568,9 @@ position. those. - `--report` shows exactly what this machine would contribute to a problem report, and sends nothing. There is no automatic transport, and there will be - none: a report leaves a machine only by its owner's own action. + none: a report leaves a machine only by its owner's own action. A `--model` + given with it is not assessed (a report carries no model verdict), so a GGUF + reference is not looked up on Hugging Face for it. - `--send`, which requires `--report`, additionally offers a prefilled mail carrying that report. It still sends nothing: the mail opens already filled in with the content `--report` just printed, addressed to `ROCmCLI@amd.com`, @@ -502,7 +598,15 @@ generalized first: - The ROCm release and the inference engine's release are each cut back to a release, so a build number that would narrow toward one machine never appears there. The CLI's own version is the exception, because it names the - tool that wrote the report instead of something read off the machine. + tool that wrote the report instead of something read off the machine. That + value is the version `rocm version` leads with (`X.Y.Z` for a release tag + `vX.Y.Z`, `0.0.0-dev` for any other resolved ref), without the ref and commit + hash that follow it there, since a local branch name is text from the + machine. A release tag's suffix may only use letters, digits, `.`, `-` and + `+`, and the version is at most 64 characters, so a release always reports + its version here. A tag outside that is not treated as a release, and the + release workflow refuses it. Anything else that reaches the report outside + those rules is reported as `unknown`. - The distribution is checked against a list of known names instead of being repeated from the machine. @@ -549,14 +653,28 @@ hardware, not something an install or a setting can put right. `hipInfo.exe` (Windows) query needed to find the index and makes no change. - `--json`, used without a fix id, prints the catalog manifest as JSON. It applies no fix and exits 0. Together with a fix id it is refused, with exit - code 1. + code 1. The manifest names every exit code `rocm fix` returns + (`exit_codes`, name to number) and says what each means + (`exit_code_descriptions`, name to text). + +Exit code 2 has two causes. It is `unknown_id`, a fix id the catalog does not +have, refused with a stderr line starting `Unknown fix-id:`; and it is also what +the argument parser returns for a malformed command line (an unknown option, a +missing value, or a value it cannot read, such as a non-number for +`--device-index`), refused with a stderr line starting `error:` instead. A +one-time `rocm:` notice can come before either line. A script that needs to know +which happened reads the message, not the code. A negative device index depends +on how it is written: `--device-index=-1` is accepted by the parser and the fix +refuses it with exit 3, while `--device-index -1` (with a space) is read as an +unknown option `-1`, a usage error that exits 2. ### ROCm installation ``` rocm install sdk [--channel release|nightly] [--format wheel|tarball] [--version x.y.z | --build-date YYYY-MM-DD] - [--family gfx110X-all] [--prefix PATH] [--devel] [--dry-run] + [--family gfx1100] [--prefix PATH] [--devel] [--dry-run] + [--python-version X.Y] [--approve-replacing-active-default] [--yes] rocm install driver [--dkms] [--yes] [--dry-run] [--reconcile] @@ -566,9 +684,9 @@ rocm update [--apply] [--runtime KEY] [--activate] [--dry-run] ``` `install sdk` downloads TheRock ROCm wheels into a Python environment managed -by ROCm CLI. ROCm CLI supports ROCm 7.14, 10.0, and 10.1. Without `--version`, -it currently installs ROCm 7.14. To install 10.0 or 10.1, follow the steps under -ROCm 10 and newer, below. +by ROCm CLI. Without `--version` it installs the newest stable release, ROCm 10 +or newer, and a pin installs an earlier one: a ROCm 7 release, for example. The +steps under ROCm 10 and newer, below, say what that layout needs. To remove ROCm CLI and what it manages, see [Logs and cleanup](#logs-and-cleanup). @@ -640,24 +758,78 @@ already there no longer runs its own Python, it is removed outright and rebuilt. The approval prompt doesn't cover this, because it asks only about changing the active default runtime, not about what a named prefix loses. +#### Which Python `install sdk` uses + +ROCm CLI picks the Python from the ROCm version it installs. ROCm 7 installs +use Python 3.12. ROCm 10 and newer use Python 3.14 by default, because vLLM, +flash-attn and amd-aiter for ROCm 10 are published for 3.14 only. The ROCm 10 +SDK and PyTorch wheels are also published for 3.10 to 3.14, so an install that +doesn't need vLLM can choose one: + +``` +rocm install sdk --version 10.1.0 --family gfx1200 --python-version 3.12 +``` + +With a Python other than 3.14, the SDK and PyTorch install and run, but vLLM +doesn't: the engine install that normally follows an SDK install on a vLLM GPU +is skipped with a note, and `rocm engines install vllm` refuses that runtime. +Re-run the install without `--python-version` to move it to 3.14. An update +keeps the Python the runtime was installed with. + +`--python-version` applies to ROCm 10 and newer wheel installs only. A ROCm 7 +install is built for Python 3.12 and refuses any other. A Python that doesn't +match the install is refused rather than used, so `ROCM_CLI_PYTHON` (an +interpreter of your own) and `ROCM_CLI_MANAGED_PYTHON_VERSION` (the version +ROCm CLI prepares itself) must name a Python the install accepts. + +If you see Python 3.14 without asking for it, the install resolved to ROCm 10 +or newer. That happens with `--version 10.x.y`, and also with no `--version` at +all, since the newest build of both channels is ROCm 10 or newer. The install +prints which Python it chose, and `--dry-run` shows it without changing anything. + #### ROCm 10 and newer -ROCm 10 and newer ship from a different source layout. You opt in by pinning the -version with `--version`. These versions select their device payload by exact -GPU architecture, so when ROCm CLI can't resolve one for this computer (it detects only -a family of GPUs, or none), or you want to target a different GPU, also pass -`--family` with the raw `gfx` code instead of a family label: +ROCm 10 and newer ship from a different source layout than ROCm 7: stable +releases from `stable.repo.amd.com` and nightlies from `nightly.repo.amd.com`. +That is the layout an install reads unless it asks for something older. A stable +pin of ROCm 10 or newer, a nightly pinned to a build date of 2026-08-23 or later, +and an install with no pin at all all read it. The earlier release stream +(`repo.amd.com`) and the earlier nightly stream (`rocm.nightlies.amd.com`), which +stopped receiving builds on 2026-08-22, are read only when you pin something that +exists only there: a ROCm 7 release, or a nightly from before 2026-08-23. These +versions select their device payload by exact GPU architecture, so when ROCm CLI can't +resolve one for this computer (it detects only a family of GPUs, or none), or you +want to target a different GPU, also pass `--family` with the raw `gfx` code +instead of a family label: ``` rocm install sdk --version 10.1.0 --family gfx1200 --dry-run ``` +The `--dry-run` preview downloads no `uv` and creates no `tools` folder. For ROCm +10 and newer, `uv pip compile` is what chooses the `torch` and `torchvision` +versions, so the preview runs a `uv` that is already installed and gives the +complete plan. With none installed it can't, and it says so in a +`resolution_note:` line: the package specs, runtime key, install folder, manifest +and `command:` (everything made from those versions) read `not resolved by this +preview`, and a real install fetches `uv` and resolves them. + A family label such as `--family gfx120X-all` is rejected for those versions instead of being resolved to a guess, because the ROCm 10 packages publish one payload per exact arch and there is no bucket payload to fall back to. Run `rocm examine` to see the arch this machine reports. If ROCm CLI detects a GPU family but not the exact arch, the error names the command to re-run. If it -detects no GPU, pass the raw `gfx` code yourself. +detects no GPU, pass the raw `gfx` code yourself. The earlier ROCm 7 releases +still publish family payloads, so `--family gfx120X-all --version 7.14.1` +installs one. + +An install with no pin on the nightly channel takes the newest build that has a +complete PyTorch stack. A nightly `rocm` build is published before its PyTorch +wheels, so when the newest one has none yet, the install uses the build before it +and prints a warning naming both. + +A tarball catalog can be pinned only in this layout, so `--format tarball +--version 7.14.1` is refused and points to `--format wheel`, which can. For ROCm 10, `install sdk` pins torchaudio itself, because its metadata names no ROCm build. When a torchaudio release has been verified on hardware for the @@ -672,14 +844,20 @@ always right: the index has published a CPU-only torchaudio under the same requested version publishes no torchaudio for this Python and platform. It then asks `uv` to resolve Torch and torchvision from their published -dependency metadata alongside the pinned torchaudio. Before it creates or +dependency metadata alongside the pinned torchaudio. For a ROCm line the vLLM +ROCm 10.x install also supports (10.0 and 10.1), the request is bounded to the +Torch and torchvision series that install pairs for that line (for 10.1, torch +2.13 and torchvision 0.28), because the newest torch on the index can pair with +a torchvision that fails to import. The `version_resolution_specs:` line in the +plan shows the bound. Before it creates or changes a managed runtime, it validates that every selected framework package carries the same ROCm build identifier. -Nothing about this happens on its own. Without a `--version` of 10 or newer, -`install sdk` resolves the same release and nightly sources as before. It doesn't -quietly retry against the ROCm 10 sources when a lookup finds nothing; it tells -you what it couldn't find instead. +`rocm update` keeps a runtime on the layout it was installed from. A runtime +installed from an earlier stream is not moved to the newest layout, and those +streams no longer publish new builds, so reinstall with `rocm install sdk` to +move it. `install sdk` doesn't quietly retry against the other layout when a +lookup finds nothing; it tells you what it couldn't find instead. #### Driver installation @@ -833,7 +1011,9 @@ exits without prompting or making changes. `adopt` registers an existing TheRock-based Python environment as a managed runtime. It does not work with standard ROCm package installs (for example, -`/opt/rocm`); use `rocm install sdk` instead. +`/opt/rocm`); use `rocm install sdk` instead. Without `--root`, the environment +folder is the one that holds the `--python` interpreter's `bin` (or `Scripts`) +folder; an interpreter anywhere else is refused until you pass `--root`. ### Disk space @@ -871,6 +1051,28 @@ followed. The two archive rows the report prints are tagged `note: can be downloaded again; safe to remove`, so it is visible which rows `remove-downloads` acts on. +The report also lists `Engine installs`: each Lemonade copy on disk and its +size. Lemonade installs into the active ROCm runtime's own folder when one is +active and into the data folder otherwise, so a machine that installed +Lemonade before its first SDK can hold two full copies (about 25 GiB each). The +copy Lemonade runs from is `in use`. A copy it is not using right now is marked +`not used right now`, which is not a promise it stays unused: Lemonade installs +into the active ROCm runtime's folder, so a copy inside a ROCm install is reused +as soon as that install becomes active, and is never marked reclaimable. Only +the data folder's copy, which no ROCm install owns, is marked reclaimable, and +nothing removes it automatically: delete its `runtime`, `downloads` and `extract` +folders to free the space. A copy is only called unused when Lemonade's install +record points at another copy that is really there: if the record names a folder +with no install, or cannot be read, the report says so and marks no copy +reclaimable. A copy inside a runtime folder +also counts toward that runtime's size. `rocm runtimes uninstall` removes it +with the runtime and clears Lemonade's install record, so Lemonade is no longer +recorded as installed; it says so when the record could not be cleared, and when +it could not be read it names `rocm engines install lemonade`, which rewrites +the record and keeps a current install. +For an external or imported runtime, uninstall only unregisters it, so its +Lemonade copy stays and the report does not say otherwise. + The report also lists `local server records`: one JSON record plus the engine's log for each `rocm serve --managed` launch, kept after the server stops. `rocm services list --all` lists them, and no `rocm storage` command removes @@ -900,6 +1102,18 @@ and installs once more; see [Lemonade backend alignment](https://github.com/ROCm/rocm-cli/blob/main/docs/engine-plugins.md#lemonade-backend-alignment-on-engine-install) for the details and for `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT`, which turns both off. +Before Lemonade writes a new install, whether from `rocm engines install +lemonade` or the first `rocm serve` that needs it, it prints the folder it is +installing into, the download (up to about 4.4 GiB) and disk use (up to about +25 GiB) of its llama.cpp ROCm backend, and any Lemonade install it is not +reusing. Those are upper bounds: Lemonade picks the backend after the install +starts, and a smaller one needs less. So it stops before downloading only when +the drive cannot hold Lemonade itself (about 16 MiB), and with less room than +the ROCm backend or the finished install it warns and continues. An install that +replaces the existing one (`--reinstall`, or a new Lemonade version) deletes it +first, so its space counts as free and the notice says so. On Windows the +backend download's own progress appears in the console. + `--yes` approves the system-package installs vLLM needs (OpenMPI, libatomic and libnuma), including a `sudo` password prompt. `engines install` never asks for confirmation. Without `--yes`, a Linux host with root or passwordless `sudo` @@ -913,14 +1127,14 @@ Windows these installs are skipped. Ask before downloading anything: ``` -rocm diagnose --model [--json] +rocm diagnose --model [--context-size N] [--kv-cache-type f16|q8_0|q4_0] [--json] ``` -Answers in seconds, from the curated recipe and this machine's GPU. It -fetches no weights and makes no network call. The verdict is `ready`, -`degraded`, `blocked`, or `undetermined`. A `ready` answer also names the -engine `rocm serve` would use; a `blocked` one names curated models that would -run here instead. +Answers from the curated recipe and this machine's GPU, in seconds. It +downloads no weights. For a curated model it makes no network call. The +verdict is `ready`, `degraded`, `blocked`, or `undetermined`. A `ready` answer +also names the engine `rocm serve` would use; a `blocked` one names curated +models that would run here instead. `undetermined` is a real answer, not a failure. You get it when any of these is true: @@ -933,7 +1147,104 @@ true: None of these says anything about whether the model fits, so none of them is reported as though it did. `rocm serve` still accepts a model outside the -catalog, but `diagnose` cannot tell you in advance how it will go. +catalog, but for anything other than a GGUF, `diagnose` cannot tell you in +advance how it will go. + +#### A GGUF model outside the catalog + +Pass a local `.gguf` path, or `owner/repo:QUANT` for a file on Hugging Face +(for example `owner/repo:Q4_K_M`, or an exact file name ending in `.gguf`, with +its directory when the file is in one and another file shares its name: +`owner/repo:dir/model-Q4_K_M.gguf`; a bare name means the file at the top of the +repository when there is one). On Hugging Face, a multimodal projector (an `mmproj` +file shipped beside a vision model) is never picked as the model, and naming one +by its file name is answered as such, with the repository's model files named +instead. A local projector file is not recognised as one; it is read like any +other GGUF. A file that +exists on this machine is read as that file even when its name looks like +`owner/repo:QUANT`. A bare +`owner/repo`, with no quantization, names a repository rather than a file and +is answered like any other model outside the catalog: undetermined, with +nothing read. +Instead of a recipe's minimum, `diagnose` reads the file's header, which holds +the layer count, the KV head count and head width, and the tensor shapes. For +a remote file that is one listing request for the repository (up to 16 MiB, to +find the file and its size), a `HEAD` request for a file the listing does not +size, then a few megabytes fetched by HTTP range request (set +`HF_ENDPOINT` to use a mirror); the weights are never downloaded. A remote read +has two minutes in all, so on a slow link it can take that long to fail. No +token is sent, so a gated or private repository cannot be read. +It then estimates weights, plus the KV cache at `--context-size` (default 4096) +and `--kv-cache-type` (default `f16`), plus a compute buffer and a runtime +reserve, against this GPU's dedicated VRAM: + +``` +rocm diagnose --model owner/repo:Q4_K_M --context-size 32768 --kv-cache-type q8_0 +``` + +The answer is labelled `ESTIMATED: fits at N tokens` or `ESTIMATED: does not +fit at N tokens`, never `ready` or `blocked`: in `--json` the `verdict` stays +`undetermined` and the figures are under `model.estimate`. It reports the +largest context that fits with that KV cache type. When the file does not fit +at the size you asked for, it lists what would make it fit: a smaller context, +a q8_0 or q4_0 KV cache (llama.cpp needs flash attention for a quantized V +cache, and the estimate assumes it is on), a smaller quantization, or keeping +some layers on the CPU. Those are llama.cpp options (`--ctx-size`, `--cache-type-k`/`-v`, +`--n-gpu-layers`): **`rocm serve` does not take a context size, KV cache type +or GPU layer count yet**, so none of them is printed as a `rocm serve` command. + +The KV cache is exact for a plain full-attention model, and the weights are the +file's own size from its tensor offsets, less the input embedding, which +llama.cpp keeps in host memory -- except for a model with tied embeddings (no +separate output layer), where llama.cpp uses the same tensor for the output +layer on the GPU, so there it is counted. That case was not measured, and +counting it can only overstate the figure. Only the compute buffer and the +runtime reserve are allowances, calibrated against real loads of two dense +full-attention models on one MI300X with llama.cpp b9752, flash attention on: +a Llama-3.1-8B-shaped and a Qwen2.5-1.5B-shaped GGUF, each with its own output +layer. +No mixture-of-experts, state-space, hybrid or sliding-window model was among +them. How close the estimate came to those +loads, and why that describes the fit rather than a bound, is written down once, +in the +[`gguf_fit` module documentation](https://github.com/ROCm/rocm-cli/blob/main/crates/rocm-core/src/gguf_fit.rs). +A model with sliding-window, state-space or mixture-of-experts layers was not +measured. A sliding-window model's KV cache is sized as if every layer attended +over the whole context, which can only overstate it, so it still gets a verdict. +A state-space or hybrid model's recurrent state is not counted, and a +mixture-of-experts model's compute buffer was not measured, so for those the +figures are shown with this GPU's memory but with no fits / does-not-fit verdict +(`cannot_estimate`, reason `unmeasured_layout`): a "fits" that is too low would +be worse than no answer. Many current open models are mixture-of-experts or +hybrid, so for a large share of them this gives the memory figures and leaves +the comparison to the reader. A header without the feed-forward or embedding width, +which size the compute buffer, is treated the same way (reason +`missing_metadata`, with the figures). Another +GPU, engine build or batch size can miss by more, so leave headroom. The figures +are compared with the total dedicated VRAM of the GPU with the most of it that +ROCm can see, the same figure a curated model's minimum is compared with, not +with what other processes have left free. When a device-visibility variable +(`HIP_VISIBLE_DEVICES`, `ROCR_VISIBLE_DEVICES`, `CUDA_VISIBLE_DEVICES`) is set on +a host whose GPUs differ in memory, the report says that the mapping between the +mask and `amd-smi`'s numbering has not been verified on such a host, so the +figure may be for a different card; that mapping has not been checked on a +multi-GPU machine. Where memory cannot be read, the memory the model needs is +shown and nothing is compared. The memory figures are whole MiB, each rounded +up, and the verdict is decided on their sum, so what is printed always adds up. +The largest context is also capped at the context length the model was trained +for. `--context-size` takes 1 to 16777216 tokens. Passing `--context-size` or +`--kv-cache-type` for a curated model changes nothing, because a recipe's memory +minimum is fixed; the report says so. Under `--report` the model is not assessed +at all, so nothing is fetched for it, and `--context-size` and `--kv-cache-type` +are ignored along with `--model`; a note on stderr names the flags that were +ignored. + +On an integrated GPU (APU), where the CLI can't read the memory +pool the engine uses, and on a host with no GPU or an unreadable one, the +answer stays `undetermined` but still shows what the file needs. Layouts the +formula does not describe (a file that is one shard of a split model, latent +attention, a model with no attention layers) are reported as such rather than +estimated. ### Curated models @@ -971,6 +1282,31 @@ rocm serve [--engine lemonade|vllm|aim] [--engine-env KEY=VALUE]... ``` +Alongside the OpenAI-compatible endpoint, a served model gets an +Anthropic-compatible `/v1/messages` endpoint, shown as `anthropic endpoint:` in +`rocm services list` and in the interactive `rocm serve` summary (a fresh launch +waits briefly for it; if it is not there yet, check `rocm services list`). It is +a text-and-tools endpoint that answers both plain and `stream: true` requests. It is a base URL: point an +Anthropic client at it (`ANTHROPIC_BASE_URL=`) and send your endpoint key, +if the service has one, as `x-api-key` (the translation front also accepts a +bearer token). `rocm` runs a +translation front on its own port, inside the serving process, so it starts and +stops with the service; it fronts the engine even when the engine has its own +`/v1/messages` route, because that route only accepts the served model's name. Whatever model name the client sends, the served model +answers, and the reply names it. The translation covers text turns, system +prompts, `max_tokens`, `temperature`, `top_p`, `top_k` and `stop_sequences`, +and tool use: `tools` and `tool_choice` (including `disable_parallel_tool_use`), +`tool_use` and `tool_result` blocks in the conversation, and `tool_use` blocks +with the `tool_use` stop reason in the reply. Whether the model calls a tool +depends on the engine: vLLM needs `--tool-call-parser ` (and auto tool choice) for any +`tool_choice` (`any` and a named tool included), and answers a request that names a +tool with a 400 without one. A streaming request is answered with Anthropic server-sent events built from the +engine's streamed chunks, but streaming a request that defines `tools` is refused. Extended thinking, server tools (web search and +the like) and image or other non-text content blocks are refused with +an Anthropic `invalid_request_error` rather than ignored. `rocm services list` +marks the endpoint `(not responding)` when a ready service's endpoint does not +answer. + `--engine aim` runs the model in a Docker container instead of a host process; it requires a reachable Docker daemon and is Linux/WSL2 only. `ROCM_CLI_AIM_READY_TIMEOUT_SECS` raises the 5-minute readiness wait for a @@ -1101,13 +1437,39 @@ download with `rocm logs --service --follow` from another terminal. Which model form to pass depends on the engine your GPU selects. The Lemonade engine (Ryzen AI or Radeon) serves llama.cpp **GGUF** models. Pass a GGUF repo with an explicit quantization variant, for example, -`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0`. +`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0`. Lemonade also accepts +registered GGUF names such as `Qwen3-0.6B-GGUF`, or the `qwen-smoke` alias. +On Linux, direct serving resolves simple single-GGUF entries from the installed +Lemonade catalog. Entries needing additional files or model-specific options, +well-formed entries of an unrecognized shape, and registered names on Windows use +Lemonade's router instead. That includes a registered checkpoint that is not a +plain `owner/repo:variant`, such as one whose variant is `*`. Explicit `owner/repo:variant` references require the selected GPU +llama-server backend. Direct serving uses that backend, including Vulkan when +supported; it does not select a stale ROCm binary merely because it is on disk. +When Lemonade selects a GPU backend other than ROCm, such as Vulkan, serving names +that backend instead of presenting it as ROCm. Neither route silently substitutes +the default model. + +On Linux, if Lemonade's router fails to load or verify a registered model, `rocm +serve` stops the router and serves that model directly through the llama-server of +the backend Lemonade selected, when the catalog entry resolves to a simple +checkpoint. The serve log names the failure and the backend. This never uses a CPU +backend: with no GPU llama-server for the selected backend, or on Windows, the +router's error is returned and the service is marked failed. A server that did not +become ready, a failed backend selection and a later server exit are never retried. The vLLM engine (Instinct) serves -**safetensors** repos, such as `rocm serve Qwen/Qwen2.5-1.5B-Instruct`. +**safetensors** repos, such as `rocm serve Qwen/Qwen2.5-0.5B-Instruct`. A safetensors-only id has no GGUF build, so serving it through Lemonade fails instead of silently substituting a different model. +The exception is a catalog recipe that lists Lemonade (`rocm model --verbose`): +it records the build each engine it lists should serve, so one name works on +each of them. `rocm serve qwen-small` gets the quantized GGUF build on Lemonade +and the safetensors repo on vLLM, and `resolved model` in the output names +which one it picked. A recipe that lists only vLLM, such as the `Qwen/Qwen2.5-0.5B-Instruct` +above, gives Lemonade nothing to fall back on and still fails there. + Some models (such as Llama) are gated and require Hugging Face authentication. Log in with `huggingface-cli login` or set `HF_TOKEN` in your environment before serving gated models. The exception is `--engine aim`, whose container @@ -1291,7 +1653,7 @@ rocm services list [--all] [--json] rocm services logs [--follow] rocm services stop --yes rocm services restart --yes -rocm services remove --yes +rocm services remove --yes [--force] rocm services prune [--older-than-hours | --any-age] [--dry-run] [--yes] ``` @@ -1300,10 +1662,15 @@ gone. If it cannot confirm that, it prints `Local server stop not confirmed` on stderr and exits with an error. It keeps the record, which still names the processes, and any endpoint key, which the server might still be enforcing. It names each process that, or a process under it, did not exit after it was -signaled; for a process it could not verify as the server's, it signals nothing -and names an operating-system command to check on it. It does not suggest -running the stop again: when only a process under the recorded one survived, a -second stop can report the server stopped while that process still runs. +signaled. On Windows it also names a process that did exit (or was already gone) +when taking the processes under it down did not complete, or found nothing and +no other recorded process is confirmed gone, because one of them may still be +running. For a process it could not verify as the server's (its recorded +start-time can no longer be read back, or, on Windows, `rocm` cannot query it at +all), it signals nothing and names an operating-system command to check on it. +It does not suggest running the stop again: when only a process under the +recorded one survived, a second stop can report the server stopped while that +process still runs. `restart` starts a server again only once its stop has confirmed that every recorded process is gone; if the stop cannot confirm that, it says so, starts @@ -1312,15 +1679,25 @@ nothing, and leaves the stop request recorded. After the stop confirms, `restart` takes the same launch lock `rocm serve` uses and re-checks the port before it respawns: if another live managed service is now recorded on that port, or (on the default `127.0.0.1` host only) the port is -bound by another process, it refuses and the service stays stopped (`rocm services list` shows it). The lock wait is -bounded (5 seconds), unlike `rocm serve`'s: if a launch still holds the lock, +bound by another process, it refuses and the service stays stopped +(`rocm services list` shows it). The lock wait is bounded (5 seconds), unlike +`rocm serve`'s: if a launch still holds the lock, `restart` fails saying the server was stopped but not restarted and names `rocm services restart --yes` to run once that launch finishes. `remove` deletes one record that is no longer running, together with its log, its engine state file, and its endpoint key file; a running server is refused, so stop it first. `prune` does the same in bulk, always leaves running servers -alone, and additionally clears leftover files whose record is already gone. +alone, and additionally clears leftover files whose record is already gone. A +server counts as running while any process it recorded may be running, which on +Windows includes one `rocm` cannot query (a live process the account may not +open): `remove` and `prune` refuse such a record until that process is gone. +`remove --force --yes` is the way out for exactly that case: it +removes the record when a stop was requested, no recorded process is confirmed +running, and at least one could not be queried, says which one, and never +removes a record whose process `rocm` can see running. `prune` has no +`--force`, but when it leaves such a record alone it lists it with the exact +`remove --force --yes` command. `docs/manual-testing.md` walks through it. Removal destroys both the log and the `restart` option for the records it takes, so `prune` only considers records untouched for 24 hours. Age is @@ -1399,9 +1776,10 @@ For `rocm dash`: - `--demo` runs a deterministic synthetic session with no GPU or daemon needed. It works on all platforms. - `--replay ` replays a recorded newline-delimited JSON (NDJSON) session. -- Live mode works on Windows too, but shows only GPU/CPU/memory. There is no - `rocmdashd` Unix socket to connect to there, so the serving-instance - registry view stays empty (Linux and WSL2 get the full live view). +- Live mode works on Windows too, showing GPU/CPU/memory and the managed + servers `rocm services list` reports. There's no `rocmdashd` unix socket + there, so per-server throughput and Docker-discovered instances are missing + (Linux and WSL2 get the full live view). ### Bench @@ -1459,6 +1837,13 @@ standard input, it sends one prompt and prints the reply. - `--temperature`, `--top-p`, and `--max-tokens` are optional sampling controls forwarded to the request. Each is independent, so omit any of them to use the provider's default. +- A dashboard chat message waits up to 2 minutes for the whole reply, including + any tool calls. The first message to a local model can take tens of seconds + while the model reads the system prompt and tool list. Set + `ROCM_DASH_CHAT_TIMEOUT_SECS` to a whole number of seconds from 1 to 3600 to + change the wait; an unset, empty, non-numeric or out-of-range value keeps the + 2 minutes. + The timeout message names the wait that applied and this variable. In the interactive chat session, if no local engine is reachable and no API key or URL is configured, chat falls back to the no-key ChatGPT OAuth sign-in. @@ -1586,6 +1971,27 @@ rocm config set-provider-key rocm config clear-provider-key ``` +Settings live in `config.json` in the config directory and are replaced in one +step, so an interrupted save leaves the previous file rather than a truncated +one. On Windows, a replace that fails part-way can instead leave the new +settings in a `config.json.tmp-*` file beside it, which the error names: rename +it back to `config.json` before running another command. If the file exists +but cannot be read — a bad hand edit, for example — +rocm-cli reports it and does not save over it, so the settings in it are never +replaced with defaults. Unless `ROCM_CLI_DATA_DIR` is set, the file also records +where runtimes are kept; when it is too damaged to read even that, commands stop +and say how to recover (`rocm diagnose` still runs). Repairing the file +restores everything. Setting `ROCM_CLI_DATA_DIR` (the error names the folder +when the file still shows it) points commands back at your runtimes' folder +meanwhile; until the file is repaired, any command that changes settings still +stops on it, and other commands either stop too or run on default settings. +To start over instead, rename the file yourself: the error names the exact +file it read (under `ROCM_CLI_CONFIG_DIR` when that is set) and a name to give +it, `config.json.broken` beside it, and rocm-cli does not rename or move it for +you. That resets every setting to its default, and runtimes in a custom folder +then need `ROCM_CLI_DATA_DIR`, which the error names when the file recorded +that folder. + ### Setup ``` @@ -1625,12 +2031,19 @@ download's progress: `rocm uninstall` removes four categories of files that ROCm CLI manages on this computer: its binaries, configuration, data, and caches. It lists what it will -do and asks `[y/N]` (default: no) before doing it. Use `--dry-run` to only list, -and `--yes` to skip the confirmation. It does not create the folders it reports -on: with `--keep-binaries`, on a computer where ROCm CLI never ran, it says there -is nothing to remove. Outside an interactive terminal, when there is something -to remove, it refuses with exit code 1 and removes nothing unless you pass -`--yes`. Each `--keep-*` flag skips the removal of one category: +do and asks `[y/N]` (default: no) before doing it. Only `y` or `yes` proceeds. +Enter, `n` and `no` cancel: it prints `uninstall cancelled`, removes nothing and +exits with code 0. Any other answer cancels the same way, after a line saying +that the answer was not understood and is treated as no. Use `--dry-run` to only +list, and `--yes` to skip the confirmation. There is no `--force`; `--force-dev-binaries` +is a separate opt-in for development builds. At a terminal, a dry run that finds +something to remove ends by saying to run the same command again without +`--dry-run`; with nothing to remove, or when stdin or stdout is not a terminal +(its output is piped or its input is redirected), it says nothing about a next step. It does not create the folders it reports on: +with `--keep-binaries`, on a computer where ROCm CLI never ran, it says there is +nothing to remove. Outside an interactive terminal, when there is something to +remove, it refuses with exit code 1 and removes nothing unless you pass `--yes`. +Each `--keep-*` flag skips the removal of one category: - `--keep-binaries` keeps the installed ROCm CLI binaries. - `--keep-config` keeps your saved settings. @@ -1661,6 +2074,16 @@ be, nothing is removed, so you can fix the cause and run `rocm uninstall` again. Each step is reported as it finishes, so if a later step fails, the output still shows what was already removed. +On Linux it also removes the fish completion script (`completions/rocm.fish` +in fish's config folder) that the installer recorded writing, even when the +`PATH` change stays for other programs, and keeps it with `--keep-binaries`. +The installer starts that file with a line naming the folder it installed into, +and uninstall removes the file only when that line names its own folder. A +`rocm.fish` you wrote yourself, a link, or one another install folder wrote +since is left in place, and the review says so under `Left alone:`. If the +file cannot be removed, uninstall prints a warning naming it and carries on, +since the completions are only a convenience; delete it yourself then. + On Linux, if a config, data, or cache folder is a link to somewhere else (for example a cache moved to a bigger disk), `rocm uninstall` removes the link and leaves the folder it points to alone, whether or not the setting ends in a `/`. @@ -1725,7 +2148,10 @@ fpath=(~/.zsh/completions $fpath) autoload -Uz compinit && compinit ``` -For fish: +For fish, the Linux installer writes this file when fish is installed (under +`$XDG_CONFIG_HOME/fish` when that is an absolute path). To refresh it, run the +installer again. A file you write with the command below carries no installer +marker line, so `rocm uninstall` leaves it in place: ```bash mkdir -p ~/.config/fish/completions diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 43cfe0d4e7..175271e8c7 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -2571,6 +2571,11 @@ The following component(s) are licensed under Apache License 2.0: - clap_builder 4.6.7 - clap_complete 4.6.11 - clap_derive 4.6.7 + - serde_spanned 1.1.3 + - toml 1.1.8+spec-1.1.0 + - toml_datetime 1.1.3+spec-1.1.0 + - toml_parser 1.1.5+spec-1.1.0 + - toml_writer 1.2.0+spec-1.1.0 Apache License Version 2.0, January 2004 @@ -3215,13 +3220,8 @@ The following component(s) are licensed under Apache License 2.0: - is_terminal_polyfill 1.70.2 - jni-sys 0.4.1 - once_cell_polyfill 1.70.2 - - serde_spanned 1.1.1 - terminal_size 0.4.4 - - toml 1.1.6+spec-1.1.0 - - toml_datetime 1.1.1+spec-1.1.0 - toml_edit 0.25.12+spec-1.1.0 - - toml_parser 1.1.3+spec-1.1.0 - - toml_writer 1.1.2+spec-1.1.0 Apache License Version 2.0, January 2004 @@ -6074,7 +6074,7 @@ The following component(s) are licensed under Apache License 2.0: - unicase 2.9.0 - unicode-segmentation 1.13.3 - unicode-truncate 2.0.1 - - unicode-width 0.2.2 + - unicode-width 0.2.3 - unicode-xid 0.2.6 - url 2.5.8 - uuid 1.23.3 @@ -8384,7 +8384,7 @@ The following component(s) are licensed under Apache License 2.0: - pin-project-lite 0.2.17 - pin-project 1.1.13 - proc-macro2-diagnostics 0.10.1 - - proc-macro2 1.0.106 + - proc-macro2 1.0.107 - quote 1.0.46 - r-efi 6.0.0 - rand 0.10.3 @@ -8407,7 +8407,7 @@ The following component(s) are licensed under Apache License 2.0: - serde_urlencoded 0.7.1 - serial2 0.2.37 - simdutf8 0.1.5 - - syn 2.0.118 + - syn 2.0.119 - syn 3.0.9 - sync_wrapper 1.0.2 - thiserror-impl 1.0.69 @@ -11037,7 +11037,7 @@ MIT License ================================================================================ The following component(s) are licensed under MIT License: - - zip 9.1.2 + - zip 9.3.0 The MIT License (MIT) diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index e48eb7cf8a..9d57b7cb48 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -22,7 +22,7 @@ e2e-test-hooks = [ # hook that simulates a managed vLLM launch which OOM'd and never became ready, # so the positive OOM-guidance summary can be verified on a GPU-less CI host. # See `e2e_simulate_oom_launch` / `simulate_oom_managed_launch` in -# `service_launch.rs`. +# `service_launch/readiness.rs`. e2e-oom-fault-injection = [] [dependencies] @@ -65,7 +65,6 @@ libc.workspace = true [target.'cfg(windows)'.dependencies] self-replace = "1.5" windows-native-keyring-store = "1.1" -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [target.'cfg(target_os = "macos")'.dependencies] apple-native-keyring-store = { version = "1.0", features = ["keychain"] } @@ -74,6 +73,8 @@ apple-native-keyring-store = { version = "1.0", features = ["keychain"] } zbus-secret-service-keyring-store = { version = "1.0", features = ["rt-tokio-crypto-rust"] } [dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } # Property-based tests for the byte-size formatter behind the `download: # approved up to …` line. "A size is printed in its own unit" has to hold at # every unit boundary, and the narrow band where `{:.1}` rounding reaches 1024.0 @@ -84,3 +85,7 @@ zbus-secret-service-keyring-store = { version = "1.0", features = ["rt-tokio-cry proptest = { version = "1", default-features = false, features = ["std"] } tempfile = "3" shlex.workspace = true +# Test-only: build.rs's `release_tag_grammar_is_the_same_everywhere` (run +# through tests/build_metadata.rs) evaluates the release-tag pattern that +# release.yml and xtask/src/smoke.rs carry. Already in the lockfile. +regex = "1" diff --git a/apps/rocm/build.rs b/apps/rocm/build.rs index 623fb3839b..6f66ba744e 100644 --- a/apps/rocm/build.rs +++ b/apps/rocm/build.rs @@ -2,11 +2,31 @@ // // SPDX-License-Identifier: MIT -//! Besides the Windows stack-size workaround below, embeds what `rocm version` -//! adds to `CARGO_PKG_VERSION`: the release tag for a tag build, or the branch -//! name otherwise (feature branches and `main` CI builds aren't tagged, and a -//! bare commit hash isn't enough to trace a build back to its branch), plus -//! the commit hash either way. +//! Besides the Windows stack-size workaround below, embeds the string +//! `rocm version` and `-V`/`--version` print. `format_cli_version` turns the +//! ref this file already resolves (the release tag for a tag build, otherwise +//! the branch name, otherwise `unknown`) and the commit hash into one value, +//! `ROCM_CLI_VERSION`: +//! +//! - exact tag `vX.Y.Z`, optionally with a `-pre` or `+build` suffix of +//! ASCII letters, digits, `.`, `-` and `+` ([`RELEASE_TAG_PATTERN`]) → +//! `X.Y.Z (vX.Y.Z, )`, with a single leading `v` removed +//! - any other resolved ref, including `main` and feature branches → +//! `0.0.0-dev (, )`, which is not a release +//! - unresolved ref (`unknown`: no tag, no branch, no git) → +//! ` (unknown, )` +//! +//! `ROCM_CLI_VERSION_NUMBER` is that leading version on its own, and +//! `ROCM_CLI_VERSION_REF` and `ROCM_CLI_GIT_HASH` stay set to the inputs. +//! +//! A ref that resolved to the literal name `unknown` (a branch or +//! `ROCM_CLI_RELEASE_REF` of that name) is indistinguishable from one that did +//! not resolve, and takes the package-version fallback. +//! The workspace package version is shared by every crate and is not bumped +//! per release, so it is the leading version only when the ref did not +//! resolve. It still reads as a published version (`0.1.0` is also a release +//! tag), so an unresolved build cannot be told apart from that release by its +//! leading version alone; its `(unknown, )` suffix is what marks it. use std::env; use std::path::{Path, PathBuf}; @@ -38,11 +58,127 @@ fn main() { println!("cargo:rerun-if-env-changed=GITHUB_REF_NAME"); println!("cargo:rerun-if-env-changed=GITHUB_HEAD_REF"); + let version_ref = ref_descriptor(); let git_hash = run_git(&["rev-parse", "--short", "HEAD"]).unwrap_or_else(|| "unknown".to_owned()); + // Cargo always sets this for the package whose build script is running. + // An empty value is not a version; `format_cli_version` substitutes one. + let pkg_version = env::var("CARGO_PKG_VERSION").unwrap_or_default(); - println!("cargo:rustc-env=ROCM_CLI_VERSION_REF={}", ref_descriptor()); + println!("cargo:rustc-env=ROCM_CLI_VERSION_REF={version_ref}"); println!("cargo:rustc-env=ROCM_CLI_GIT_HASH={git_hash}"); + // The only string `-V`/`--version` and `rocm version` print. Formatting + // it here, once, is what keeps those surfaces from disagreeing. + println!( + "cargo:rustc-env=ROCM_CLI_VERSION={}", + format_cli_version(&version_ref, &git_hash, &pkg_version) + ); + // Its leading version on its own, for one-token fields (`rocm update`'s + // `cli: installed=`, a problem report's `cli_version`). Exported from the + // same function rather than cut back out of the string above, so no + // consumer depends on how that string is laid out. + println!( + "cargo:rustc-env=ROCM_CLI_VERSION_NUMBER={}", + cli_version_number(&version_ref, &pkg_version) + ); +} + +/// Leading version for a build whose ref resolved and is not a release tag. +/// Not the workspace package version: a `main` or feature-branch binary must +/// not be presented as a release. +const NON_RELEASE_VERSION: &str = "0.0.0-dev"; + +/// `rocm version` / `-V` text after the `rocm-cli ` prefix. +/// +/// See the module docs for the tag, branch, and unknown shapes. `version_ref` +/// and `git_hash` are the values this build script already resolves; an empty +/// input is treated as `unknown` so the result is never an empty version. +fn format_cli_version(version_ref: &str, git_hash: &str, cargo_pkg_version: &str) -> String { + let version = cli_version_number(version_ref, cargo_pkg_version); + format!( + "{version} ({}, {})", + or_unknown(version_ref), + or_unknown(git_hash) + ) +} + +/// The leading version of [`format_cli_version`]'s string, which that +/// function is built from. An empty ref is `unknown`, as there. +fn cli_version_number(version_ref: &str, cargo_pkg_version: &str) -> String { + displayed_version(or_unknown(version_ref), cargo_pkg_version) +} + +/// An empty ref or hash is reported as `unknown`, never as an empty field. +const fn or_unknown(value: &str) -> &str { + if value.is_empty() { "unknown" } else { value } +} + +fn displayed_version(version_ref: &str, cargo_pkg_version: &str) -> String { + if let Some(release) = release_version(version_ref) { + return release.to_owned(); + } + if version_ref == "unknown" { + let package = if cargo_pkg_version.is_empty() { + "0.0.0" + } else { + cargo_pkg_version + }; + return package.to_owned(); + } + NON_RELEASE_VERSION.to_owned() +} + +/// The release-tag grammar, as an anchored regular expression: `vX.Y.Z` +/// with an optional non-empty `-pre` or `+build` suffix of ASCII letters, +/// digits, `.`, `-` and `+`. `release_version` is its implementation, and +/// release.yml's "Determine version" step and `cargo xtask smoke` carry +/// it verbatim; `release_tag_grammar_is_the_same_everywhere` holds all three +/// together. The suffix characters are the ones a problem report publishes +/// (`cli_version` in `crates/rocm-core/src/report.rs`), so a release never +/// reports itself as `unknown` there. +// Only the test reads it: the build itself runs `release_version`, which +// needs no regex engine. +#[cfg(test)] +const RELEASE_TAG_PATTERN: &str = r"^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.+-]+)?$"; + +/// Longest release version (the tag without its `v`), the same bound a +/// problem report puts on `cli_version`. release.yml checks the tag against +/// this plus one, for the `v`. +const RELEASE_VERSION_MAX_LEN: usize = 64; + +/// [`RELEASE_TAG_PATTERN`], with the version at most +/// [`RELEASE_VERSION_MAX_LEN`] long. Returns the version with that one leading +/// `v` removed. A ref that merely starts with `v` (`vllm`) is not a release +/// tag. +fn release_version(version_ref: &str) -> Option<&str> { + let rest = version_ref.strip_prefix('v')?; + if rest.len() > RELEASE_VERSION_MAX_LEN { + return None; + } + let core = match rest.split_once(['-', '+']) { + Some((core, suffix)) + if !suffix.is_empty() + && suffix + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'+')) => + { + core + } + Some(_) => return None, + None => rest, + }; + let mut components = core.split('.'); + let has_major_minor_patch = (0..3).all(|_| components.next().is_some_and(is_ascii_digits)); + // Exactly three components: `v1.2` and `v1.2.3.4` are not `vX.Y.Z`. + if has_major_minor_patch && components.next().is_none() { + Some(rest) + } else { + None + } +} + +fn is_ascii_digits(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) } /// The release tag for a tag build, else the branch name, else "unknown". @@ -165,7 +301,10 @@ fn run_git_at(cwd: &Path, args: &[&str]) -> Option { #[cfg(test)] mod tests { - use super::git_watch_paths; + use super::{ + RELEASE_TAG_PATTERN, RELEASE_VERSION_MAX_LEN, cli_version_number, format_cli_version, + git_watch_paths, release_version, + }; use std::fs; use std::path::Path; use std::process::Command; @@ -349,4 +488,200 @@ mod tests { "the packed branch ref has no loose file anymore and must not be watched: {paths:?}" ); } + + #[test] + fn format_cli_version_reports_a_release_tag_without_its_leading_v() { + assert_eq!( + format_cli_version("v0.2.1", "1cde824", "0.1.0"), + "0.2.1 (v0.2.1, 1cde824)" + ); + // A pre-release or build suffix is still that release's version. + assert_eq!( + format_cli_version("v0.2.1-rc.1", "1cde824", "0.1.0"), + "0.2.1-rc.1 (v0.2.1-rc.1, 1cde824)" + ); + assert_eq!( + format_cli_version("v0.2.1+build.1", "1cde824", "0.1.0"), + "0.2.1+build.1 (v0.2.1+build.1, 1cde824)" + ); + } + + #[test] + fn format_cli_version_reports_main_as_a_dev_build() { + assert_eq!( + format_cli_version("main", "abcdef1", "0.1.0"), + "0.0.0-dev (main, abcdef1)" + ); + // Same class as `main`: a feature branch, and a name that starts with + // `v` but is not `vX.Y.Z`, are not releases either. + assert_eq!( + format_cli_version("fix/cli-version-string", "abcdef1", "0.1.0"), + "0.0.0-dev (fix/cli-version-string, abcdef1)" + ); + assert_eq!( + format_cli_version("vllm", "abcdef1", "0.1.0"), + "0.0.0-dev (vllm, abcdef1)" + ); + // Not an exact `vX.Y.Z`: too few components, or a separator with no + // pre-release or build suffix. + assert_eq!( + format_cli_version("v0.2", "abcdef1", "0.1.0"), + "0.0.0-dev (v0.2, abcdef1)" + ); + assert_eq!( + format_cli_version("v1.2.3-", "abcdef1", "0.1.0"), + "0.0.0-dev (v1.2.3-, abcdef1)" + ); + } + + #[test] + fn format_cli_version_falls_back_to_the_package_version_only_when_the_ref_is_unknown() { + assert_eq!( + format_cli_version("unknown", "unknown", "0.1.0"), + "0.1.0 (unknown, unknown)" + ); + // The hash can still resolve when the ref cannot (detached HEAD, no + // exact tag). The package version is the fallback for that case only, + // and an empty input does not produce an empty version. + assert_eq!( + format_cli_version("unknown", "abcdef1", "0.1.0"), + "0.1.0 (unknown, abcdef1)" + ); + assert_eq!(format_cli_version("", "", ""), "0.0.0 (unknown, unknown)"); + } + + #[test] + fn format_cli_version_rejects_other_near_misses_of_a_release_tag() { + // Too many components, as the parser's own comment says. + assert_eq!( + format_cli_version("v1.2.3.4", "abcdef1", "0.1.0"), + "0.0.0-dev (v1.2.3.4, abcdef1)" + ); + // A build separator with nothing after it, the `+` twin of `v1.2.3-`. + assert_eq!( + format_cli_version("v1.2.3+", "abcdef1", "0.1.0"), + "0.0.0-dev (v1.2.3+, abcdef1)" + ); + // Three components, but one is not all digits or is empty. + for near_miss in ["v1.x.3", "v1..3", "vfoo.bar.baz", "v1.2.3a"] { + assert_eq!( + format_cli_version(near_miss, "abcdef1", "0.1.0"), + format!("0.0.0-dev ({near_miss}, abcdef1)") + ); + } + // A pre-release and a build suffix together are still a release. + assert_eq!( + format_cli_version("v1.2.3-rc.1+build.5", "abcdef1", "0.1.0"), + "1.2.3-rc.1+build.5 (v1.2.3-rc.1+build.5, abcdef1)" + ); + } + + #[test] + fn cli_version_number_is_the_leading_version_of_format_cli_version() { + for (version_ref, hash, package) in [ + ("v0.2.1", "1cde824", "0.1.0"), + ("v0.2.1-rc.1", "1cde824", "0.1.0"), + ("main", "abcdef1", "0.1.0"), + ("unknown", "abcdef1", "0.1.0"), + ("", "", ""), + ] { + let number = cli_version_number(version_ref, package); + let full = format_cli_version(version_ref, hash, package); + assert!( + full.starts_with(&format!("{number} (")), + "{number:?} does not lead {full:?}" + ); + } + } + + #[test] + fn embedded_cli_version_uses_format_cli_version() { + // The build script exports one string. This ties that export to the + // formatter, using the ref and hash this checkout actually resolved. + assert_eq!( + env!("ROCM_CLI_VERSION"), + format_cli_version( + env!("ROCM_CLI_VERSION_REF"), + env!("ROCM_CLI_GIT_HASH"), + env!("CARGO_PKG_VERSION"), + ) + ); + assert_eq!( + env!("ROCM_CLI_VERSION_NUMBER"), + cli_version_number(env!("ROCM_CLI_VERSION_REF"), env!("CARGO_PKG_VERSION")) + ); + } + + /// The release-tag grammar lives in three places: `release_version` here, + /// the "Determine version" check in release.yml and `is_release_tag` in + /// `xtask/src/smoke.rs` (`cargo xtask smoke`). The two copies must carry + /// [`RELEASE_TAG_PATTERN`] and the [`RELEASE_VERSION_MAX_LEN`] bound + /// verbatim, and the pattern must accept exactly what `release_version` + /// accepts. + #[test] + fn release_tag_grammar_is_the_same_everywhere() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let read = |rel: &str| { + fs::read_to_string(root.join(rel)) + .unwrap_or_else(|err| panic!("reading {rel}: {err}")) + .replace("\r\n", "\n") + }; + let tag_max = RELEASE_VERSION_MAX_LEN + 1; + + let workflow = read(".github/workflows/release.yml"); + let check = format!( + r#"[[ "${{VERSION}}" =~ {RELEASE_TAG_PATTERN} ]] || [ "${{#VERSION}}" -gt {tag_max} ]"# + ); + assert!( + workflow.contains(&check), + "release.yml's tag check is not `{check}`" + ); + let smoke = read("xtask/src/smoke.rs"); + for needle in [ + format!(r#"const RELEASE_TAG_PATTERN: &str = r"{RELEASE_TAG_PATTERN}";"#), + format!("const RELEASE_TAG_MAX_LEN: usize = {tag_max};"), + ] { + assert!( + smoke.contains(&needle), + "xtask/src/smoke.rs lacks `{needle}`" + ); + } + + let pattern = regex::Regex::new(RELEASE_TAG_PATTERN).expect("pattern compiles"); + let at_limit = format!("v1.2.3-{}", "a".repeat(RELEASE_VERSION_MAX_LEN - 6)); + let over_limit = format!("{at_limit}a"); + assert_eq!(at_limit.len(), tag_max); + for tag in [ + "v0.2.1", + "v1.2.3-rc.1", + "v1.2.3+build.5", + "v1.2.3-rc.1+build.5", + "v0.1.0-experimental.1", + "v1.2.3-rc_1", + "v1.2.3-rc 1", + "v1.2.3-", + "v1.2.3+", + "v1.2", + "v1.2.3.4", + "v1.x.3", + "vllm", + "main", + at_limit.as_str(), + over_limit.as_str(), + ] { + let by_pattern = pattern.is_match(tag) && tag.len() <= tag_max; + assert_eq!( + release_version(tag).is_some(), + by_pattern, + "release_version and RELEASE_TAG_PATTERN disagree on {tag:?}" + ); + } + // The suffix rule is the problem report's: a release must never be + // published as `unknown`. + assert!(release_version("v1.2.3-rc_1").is_none()); + assert_eq!( + release_version(&at_limit).map(str::len), + Some(RELEASE_VERSION_MAX_LEN) + ); + } } diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index 8cca54df38..84a5ee852e 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -640,6 +640,13 @@ fn split_shell_list(text: &str) -> Vec { } match ch { '"' | '\'' => quote = Some(ch), + // `<<` starts a heredoc, a redirection, not a ``: + // taken whole, so neither `<` opens one. + '<' if next == Some('<') => { + current.push_str("<<"); + index += 2; + continue; + } // `<` opens a placeholder only at a word start and before a // non-space: ` < file` is a redirection. '<' if at_word_start && next.is_some_and(|c| !c.is_whitespace()) => closers.push('>'), @@ -677,7 +684,7 @@ pub(crate) fn command_parts(raw: &str) -> Vec { } let mut command = command.to_owned(); for marker in [ - " >> ", " > ", " < ", " 2>", " (", " —", " –", " → ", " before ", " then ", + " >> ", " > ", " <<", " < ", " 2>", " (", " —", " –", " → ", " before ", " then ", ] { if let Some(position) = command.find(marker) { command.truncate(position); @@ -1159,7 +1166,7 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "heading of the non-interactive launcher report", ), ( - "apps/rocm/src/chat.rs", + "apps/rocm/src/chat/turn.rs", "rocm tools: {}", "`rocm tools: enabled` status row", ), @@ -1169,32 +1176,32 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "heading of the model-readiness report, `: `, naming what was run", ), ( - "apps/rocm/src/chat.rs", + "apps/rocm/src/chat/intent.rs", "rocm install folder", "natural-language phrase the planner matches in a request", ), ( - "apps/rocm/src/chat.rs", + "apps/rocm/src/chat/intent.rs", "rocm installed at", "natural-language phrase the planner matches in a request", ), ( - "apps/rocm/src/chat.rs", + "apps/rocm/src/chat/intent.rs", "rocm installed", "natural-language phrase the planner matches in a request", ), ( - "apps/rocm/src/chat.rs", + "apps/rocm/src/chat/intent.rs", "rocm please", "natural-language phrase the planner matches in a request", ), ( - "apps/rocm/src/chat.rs", + "apps/rocm/src/chat/mcp_bridge.rs", "rocm command requires at least one argument", "validation error naming the chat `rocm` tool", ), ( - "apps/rocm/src/chat.rs", + "apps/rocm/src/chat/turn.rs", "rocm command", "fallback label for an unrenderable chat tool call", ), @@ -1275,6 +1282,11 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "rocm debug: command capture {stage} failed for {}: {detail}", "debug log line", ), + ( + "crates/rocm-core/src/host_gpu/capture.rs", + "rocm debug: command capture output from {} may be incomplete: {note}", + "debug log line", + ), ( "crates/rocm-dash-tui/src/ui/command_screen.rs", "rocm {}", @@ -1286,6 +1298,12 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "rocm services prune --any-age --older-than-hours 0", "the doc's expected result is that the parser rejects this combination", ), + ( + "docs/manual-testing.md", + "rocm fix --no-such-option; echo $?", + "the doc's expected result is that the parser rejects the unknown option \ + with exit 2, the code the catalog publishes for an unknown fix-id", + ), ]; /// Natural-language examples must reach the planner with a multi-word request: @@ -2104,6 +2122,50 @@ fn every_command_in_a_shell_list_is_checked() { ); } +/// A heredoc is a redirection like ` < file`: `rocm diagnose --symptom-file - +/// <<'EOF'` hands the command a paste on standard input, so the `<<` and its +/// delimiter are not arguments. +#[test] +fn a_heredoc_ends_the_command() { + for raw in [ + "rocm diagnose --symptom-file - <<'EOF'", + "rocm diagnose --symptom-file - <<\"EOF\"", + "rocm diagnose --symptom-file - << EOF", + "rocm diagnose --symptom-file - <`, so a command after it in the + // same list is still split off and checked. + for raw in [ + "rocm diagnose --symptom-file - <<'EOF' && rocm frobnicate --x", + "rocm diagnose --symptom-file - < Result { + Ok(AutomationRuntimeState::load(paths)? + .is_some_and(|state| state.running && rocm_core::process_is_running(state.daemon_pid))) +} + +/// What the user should be told about the background automation helper +/// (`rocm daemon`) after stopping a service or when planning an uninstall. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HelperNote { + /// The sentence to print. + pub(crate) text: String, + /// The helper is running right now. + pub(crate) running: bool, + /// An automation watcher is enabled, so the helper stays up on its own. + pub(crate) outlives: bool, +} + +/// The note for one combination of the two facts the helper itself acts on: +/// whether a watcher is enabled +/// ([`rocm_core::RocmCliConfig::automation_daemon_enabled`] gates both the spawn +/// and the helper's exit) and whether it is running. `None` for neither. +fn helper_note_for(enabled: &[&str], running: bool) -> Option { + let text = match (enabled.is_empty(), running) { + (true, false) => return None, + (true, true) => "the background automation helper (`rocm daemon`) is still running; no automation is enabled, so it exits on its own within a few seconds".to_owned(), + (false, true) => { + let disable = enabled + .iter() + .map(|id| format!("`rocm automations disable {id}`")) + .collect::>() + .join(", "); + format!( + "the background automation helper (`rocm daemon`) keeps running while an automation is enabled ({}); to stop it, run {disable}, after which it exits on its own within a few seconds", + enabled.join(", ") + ) + } + (false, false) => format!( + "an automation is enabled ({}) but the background automation helper (`rocm daemon`) is not running; `rocm serve` or `rocm automations enable` starts it", + enabled.join(", ") + ), + }; + Some(HelperNote { + text, + running, + outlives: !enabled.is_empty() && running, + }) +} + +/// [`helper_note_for`] over the facts on disk. An unreadable config counts as +/// "no watcher enabled". +pub(crate) fn background_helper_note(paths: &AppPaths) -> Option { + let config = rocm_core::RocmCliConfig::load(paths).unwrap_or_default(); + let enabled: Vec<&str> = config + .automations + .watchers + .iter() + .filter(|(_, watcher)| watcher.enabled) + .map(|(id, _)| id.as_str()) + .collect(); + let running = background_helper_already_running(paths).unwrap_or(false); + helper_note_for(&enabled, running) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + use crate::test_support::{runtime_state, set_helper_facts}; + use crate::tests::test_paths; + + #[test] + fn background_helper_already_running_true_for_live_pid() { + // Phase-10 daemon no-double-spawn: a runtime-state.json with running=true + // and a LIVE daemon_pid (this very test process) means the helper is + // already up — the "should spawn?" decision must say NO (true ⇒ skip). + // Hermetic + offline: no spawn, just the file-based liveness check. + let (root, paths) = test_paths("helper-live-pid"); + runtime_state(true, std::process::id()) + .write(&paths) + .expect("write runtime state"); + assert!( + background_helper_already_running(&paths).expect("liveness check ok"), + "live recorded pid + running=true ⇒ do not spawn a second daemon" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn background_helper_already_running_false_for_dead_or_missing() { + // The inverse guard cases — each must report NOT running (false ⇒ spawn): + // (1) no state file at all, (2) running=true but a dead/zero pid, + // (3) a live pid but running=false. None must spawn from this decision. + let (root, paths) = test_paths("helper-dead-or-missing"); + // (1) No state file yet. + assert!( + !background_helper_already_running(&paths).expect("missing state ⇒ ok"), + "no runtime state ⇒ not running" + ); + // (2) running=true but pid 0 is never a live process. + runtime_state(true, 0).write(&paths).expect("write state"); + assert!( + !background_helper_already_running(&paths).expect("dead pid ⇒ ok"), + "running=true + dead pid ⇒ not running (spawn)" + ); + // (3) live pid but running flag is false. + runtime_state(false, std::process::id()) + .write(&paths) + .expect("write state"); + assert!( + !background_helper_already_running(&paths).expect("not-running flag ⇒ ok"), + "running=false ⇒ not running even with a live pid" + ); + let _ = fs::remove_dir_all(&root); + } + + /// ROCMAI-827: the helper note is derived from both facts, so it cannot + /// disagree with the state it describes. Each combination: the exact wording, + /// and the running/outlives flags the uninstall plan acts on. + #[test] + fn helper_note_follows_watcher_enabled_and_helper_running() { + let (root, paths) = test_paths("helper-note-combos"); + paths.ensure().expect("create app dirs"); + + set_helper_facts(&paths, false, false); + let neither = background_helper_note(&paths); + set_helper_facts(&paths, false, true); + let idle_running = background_helper_note(&paths).expect("note"); + set_helper_facts(&paths, true, false); + let enabled_dead = background_helper_note(&paths).expect("note"); + set_helper_facts(&paths, true, true); + let enabled_running = background_helper_note(&paths).expect("note"); + let _ = fs::remove_dir_all(&root); + + assert_eq!(neither, None); + assert_eq!( + idle_running.text, + "the background automation helper (`rocm daemon`) is still running; no automation is enabled, so it exits on its own within a few seconds" + ); + assert!(idle_running.running && !idle_running.outlives); + assert_eq!( + enabled_dead.text, + "an automation is enabled (server-recover) but the background automation helper (`rocm daemon`) is not running; `rocm serve` or `rocm automations enable` starts it" + ); + assert!(!enabled_dead.running && !enabled_dead.outlives); + assert_eq!( + enabled_running.text, + "the background automation helper (`rocm daemon`) keeps running while an automation is enabled (server-recover); to stop it, run `rocm automations disable server-recover`, after which it exits on its own within a few seconds" + ); + assert!(enabled_running.running && enabled_running.outlives); + } + + /// The command the note names must exist and accept exactly those arguments, + /// and its effect (the helper exiting once the last watcher is off) is + /// `automations-11` in `features/automations.feature` plus the rocmd + /// `automation_loop_` tests. + #[test] + fn the_command_named_by_the_helper_note_parses() { + use clap::Parser as _; + + let (root, paths) = test_paths("helper-note-command"); + paths.ensure().expect("create app dirs"); + set_helper_facts(&paths, true, true); + let note = background_helper_note(&paths).expect("a note"); + let _ = fs::remove_dir_all(&root); + + let command = note + .text + .split('`') + .find(|part| part.starts_with("rocm automations disable")) + .expect("the note names the disable command"); + let argv: Vec<&str> = command.split_whitespace().collect(); + assert!(crate::Cli::try_parse_from(argv).is_ok(), "{command}"); + } +} diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 1d00092627..62e353f4b6 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -14,7 +14,7 @@ //! Mechanically relocated from `main.rs` with no behavior change; see //! `docs/architecture.md`. `dispatch()`'s `InternalBridgeSnapshot` arm calls //! `build_codex_bridge_snapshot` through a re-import in `main.rs`, and -//! `chat.rs`'s internal MCP tools (`bridge_snapshot`, `gpu_snapshot` and +//! `chat/mcp_bridge.rs`'s internal MCP tools (`bridge_snapshot`, `gpu_snapshot` and //! `engines`) reach `build_codex_bridge_snapshot`, //! `build_codex_bridge_gpu_snapshot` and `builtin_codex_bridge_engine_inventory` //! as `crate::` through the same re-import, so those three are diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 4ddcaa6893..0b95d736b1 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2,427 +2,68 @@ // // SPDX-License-Identifier: MIT -//! Chat/assistant tool-call support: the chat/MCP-bridge cluster's types -//! (`DeterministicModelRecipe`, `DeterministicServiceRow`, -//! `DeterministicEngineRow`, and others — all module-private), its -//! tool-call execution logic, deterministic tool-output summarization, and -//! the MCP tool-bridge entry points (`run_internal_mcp_call` and its -//! helpers). The tool-call validation and approval policy -//! (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, -//! `ChatRocmCommandAction` and `ChatToolApprovalRequest`) lives in -//! `tool_policy.rs`, which this module calls into and re-exports from. -//! Callers in other modules, production and test-only, reach this module's -//! items through the `chat::` module boundary. +//! The chat assistant's entry points and the module that ties its pieces +//! together: reading a piped prompt (`read_piped_prompt`), the status screen +//! (`render_chat_text`), the one-shot reply text (`render_chat_prompt_text`) and +//! the clap value parsers for the sampling flags. The work behind them is split +//! into child modules, each a leaf of the call graph below the turn loop: `turn` +//! (the turn loop, the system prompt and the transcript), `intent` (the +//! prompt-intent heuristics), `mcp_bridge` (the MCP tool bridge and in-process +//! command execution) and `summaries` (the deterministic tool-output +//! summaries). The tool-call validation and approval policy lives in +//! `tool_policy.rs`. //! -//! Full domain extraction, mirroring `providers.rs`. Phase 6d (ROCMAI-88) of -//! the `main.rs` modularization (see `docs/architecture.md`), split across -//! three PRs given the cluster's size: this module's contents are the -//! combined slice of all three (types and leaf helpers, the orchestration -//! bodies, and the cluster's regression tests moved out of `main.rs`'s -//! shared `mod tests` block), plus the third PR's own visibility audit — -//! every function still `pub(crate)` has a confirmed caller outside this -//! module, production or test-only; none of the module's own types need to -//! be. - -use crate::automations::{render_automations_text, render_daemon_text}; -use crate::engine_runtime_select::{app_path_env_vars, service_model_names_match}; -use crate::freeform_planner::{format_structured_tool_call, plan_freeform_request_with_context}; -use crate::managed_service_status::{ - friendly_engine_label, load_managed_services, managed_service_is_live, -}; +//! Callers in other modules, production and test-only, reach the moved items +//! through the `chat::` module boundary, so this module re-exports each one +//! another module reads (`run_internal_mcp_call`, +//! `rocm_chat_tool_system_prompt_for_host`, the tool-policy items, `any_substring` +//! and `ensure_flag`, and the test-only ones under `#[cfg(test)]`). + +mod intent; +mod mcp_bridge; +mod summaries; +mod turn; + +// `deterministic_comfyui_tool_summary` lives in `summaries.rs` and stays +// reachable here as `chat::deterministic_comfyui_tool_summary` for +// `comfyui.rs`'s tests, which are its only readers outside `summaries.rs`. +// Those tests are all `#[cfg(unix)]`, so the re-export is gated the same way: +// on Windows it would be an unused import. +#[cfg(all(test, unix))] +pub(crate) use self::summaries::deterministic_comfyui_tool_summary; +// `run_internal_mcp_call` lives in `mcp_bridge.rs` and stays reachable here as +// `chat::run_internal_mcp_call` for `main.rs` and `dash_seam.rs`. +pub(crate) use self::mcp_bridge::run_internal_mcp_call; +// `rocm_chat_tool_system_prompt_for_host` lives in `turn.rs` and stays +// reachable here as `chat::rocm_chat_tool_system_prompt_for_host` for `dash.rs`. +use self::turn::render_chat_prompt_result_with_progress; +pub(crate) use self::turn::rocm_chat_tool_system_prompt_for_host; // `any_substring` lives in `prompt_text.rs` and stays reachable here as // `chat::any_substring` for `freeform_answer.rs`. pub(crate) use crate::prompt_text::any_substring; -use crate::prompt_text::{ - infer_engine_from_request, requested_install_prefix_from_prompt, - requested_therock_build_date_from_prompt, requested_therock_version_from_prompt, -}; // `ensure_flag` lives in `rocm_argv.rs` and stays reachable here as // `chat::ensure_flag` for `freeform_execution.rs`. pub(crate) use crate::rocm_argv::ensure_flag; -use crate::rocm_argv::{chat_cli_arg_value, chat_cli_has_flag}; -use crate::runtimes_cmd::render_runtimes_text; -use crate::services_text::{ - read_tail_lines, render_logs_browser_text, render_logs_text, render_service_logs_text, - render_services_text, -}; // The tool-call policy lives in `tool_policy.rs`. `chat_tool_approval_request`, // `chat_tool_call_is_read_only` and `validate_chat_tool_call` stay reachable -// here as `chat::` for `dash_seam.rs`. `ChatRocmCommandAction` and -// `rocm_chat_tool_requested_args` are used by this module's own code, so the -// import cannot be `#[cfg(test)]`; it is also `pub(crate)` because other -// modules' tests (`driver_plan.rs`, `system_packages.rs`) read both through -// `chat::`. `chat_install_driver_action` and `chat_rocm_command_action_from_args` -// are read through `chat::` only by those tests and this module does not call -// them itself, so their re-export below is `#[cfg(test)]`. -pub(crate) use crate::tool_policy::{ - ChatRocmCommandAction, chat_tool_approval_request, chat_tool_call_is_read_only, - rocm_chat_tool_requested_args, validate_chat_tool_call, -}; -use crate::tool_policy::{ - ChatToolApprovalRequest, chat_install_prefix_is_system, chat_rocm_command_action, json_string, - normalized_chat_rocm_command_args, rocm_chat_tool_requested_command, -}; +// here as `chat::` for `dash_seam.rs`. `ChatRocmCommandAction`, +// `rocm_chat_tool_requested_args`, `chat_install_driver_action` and +// `chat_rocm_command_action_from_args` are read through `chat::` only by other +// modules' tests (`driver_plan.rs`, `system_packages.rs`), and this module does +// not call them itself, so their re-export below is `#[cfg(test)]`. #[cfg(test)] pub(crate) use crate::tool_policy::{ - chat_install_driver_action, chat_rocm_command_action_from_args, -}; -use crate::uninstall_plan::render_uninstall_dry_run; -use crate::update_cmd::render_update_text; -use crate::{ - ChatInferenceParams, build_codex_bridge_gpu_snapshot, build_codex_bridge_snapshot, - builtin_codex_bridge_engine_inventory, cli_version_string, record_cli_audit_event, - render_config_text, render_engine_inventory_text_with_paths, render_examine_text, - render_examine_text_with_paths, render_model_registry_verbose_text_with_context_and_host, - render_setup_status_text, therock_install_version_selector, + ChatRocmCommandAction, chat_install_driver_action, chat_rocm_command_action_from_args, + rocm_chat_tool_requested_args, }; -use crate::{chat_host_facts, comfyui, providers, therock}; -use anyhow::{Context, Result, bail}; -use rocm_core::placeholder::empty_as_unknown; -use rocm_core::{ - AppPaths, AutomationRuntimeState, CommandCapture, DEFAULT_LOCAL_HOST, ExamineSummary, - ManagedServiceRecord, RocmCliConfig, command_capture_text, load_recent_automation_events, +pub(crate) use crate::tool_policy::{ + chat_tool_approval_request, chat_tool_call_is_read_only, validate_chat_tool_call, }; -use rocm_engine_protocol::DEFAULT_LOG_TAIL_LINES; +use crate::{ChatInferenceParams, providers}; +use anyhow::{Context, Result}; +use rocm_core::AppPaths; use std::fmt::Write as _; use std::io::Read as _; -use std::net::{TcpStream, ToSocketAddrs}; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -fn mcp_tool_result_text(value: &serde_json::Value) -> String { - value - .get("content") - .and_then(serde_json::Value::as_array) - .map(|items| { - items - .iter() - .filter_map(|item| { - (item.get("type").and_then(serde_json::Value::as_str) == Some("text")) - .then(|| item.get("text").and_then(serde_json::Value::as_str)) - .flatten() - }) - .collect::>() - .join("\n") - }) - .filter(|text| !text.trim().is_empty()) - .unwrap_or_else(|| value.to_string()) -} - -fn mcp_tool_result_is_error(value: &serde_json::Value) -> bool { - value - .get("isError") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) -} - -fn chat_tool_display_label(name: &str) -> String { - match name { - "examine" => "Checked this computer".to_owned(), - "gpu_snapshot" => "Checked GPU status".to_owned(), - "engines" => "Checked local engines".to_owned(), - "services" => "Checked model servers".to_owned(), - "bridge_snapshot" => "Checked ROCm state".to_owned(), - "service_logs" => "Read server logs".to_owned(), - "automations" => "Checked automations".to_owned(), - "natural_language_plan" => "Planned ROCm request".to_owned(), - "path_exists" => "Checked folder path".to_owned(), - "port_status" => "Checked local port".to_owned(), - "update_check" => "Checked for ROCm updates".to_owned(), - "install_sdk_dry_run" => "Previewed ROCm install".to_owned(), - "install_sdk" => "Install ROCm".to_owned(), - "install_engine" => "Install engine".to_owned(), - "launch_server" => "Start local model server".to_owned(), - "stop_server" => "Stop local model server".to_owned(), - "watcher_enable" => "Enable automation".to_owned(), - "watcher_disable" => "Disable automation".to_owned(), - other => other.replace('_', " "), - } -} - -fn chat_tool_call_display_label(call: &providers::ChatToolCall) -> String { - if call.name != "rocm_command" { - return chat_tool_display_label(&call.name); - } - let Some(args) = normalized_chat_rocm_command_args(call).ok() else { - return "rocm command".to_owned(); - }; - match args.as_slice() { - [command] if command.eq_ignore_ascii_case("examine") => "Checked this computer".to_owned(), - [command] - if command.eq_ignore_ascii_case("model") || command.eq_ignore_ascii_case("models") => - { - "Checked model recipes".to_owned() - } - [command, subcommand] - if command.eq_ignore_ascii_case("engines") - && subcommand.eq_ignore_ascii_case("list") => - { - "Checked local engines".to_owned() - } - [command] if command.eq_ignore_ascii_case("services") => "Checked model servers".to_owned(), - [command, subcommand, rest @ ..] - if command.eq_ignore_ascii_case("services") - && subcommand.eq_ignore_ascii_case("list") - && rest - .iter() - .all(|arg| matches!(arg.as_str(), "-a" | "--all")) => - { - "Checked model servers".to_owned() - } - [command] if command.eq_ignore_ascii_case("comfyui") => "Checked ComfyUI".to_owned(), - [command, subcommand] - if command.eq_ignore_ascii_case("comfyui") - && subcommand.eq_ignore_ascii_case("status") => - { - "Checked ComfyUI".to_owned() - } - [command, subcommand, ..] - if command.eq_ignore_ascii_case("comfyui") - && matches!(subcommand.to_ascii_lowercase().as_str(), "logs" | "log") => - { - "Read ComfyUI logs".to_owned() - } - [command, subcommand] - if command.eq_ignore_ascii_case("config") - && subcommand.eq_ignore_ascii_case("show") => - { - "Checked ROCm config".to_owned() - } - _ => "rocm command".to_owned(), - } -} - -fn loopback_tcp_port_is_reachable(host: &str, port: u16) -> bool { - let Ok(addresses) = (host, port).to_socket_addrs() else { - return false; - }; - addresses - .into_iter() - .any(|address| TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok()) -} - -fn parse_optional_query(args: &[String]) -> Result> { - let mut index = 0; - while index < args.len() { - match args[index].as_str() { - "--query" | "-q" => { - let value = args - .get(index + 1) - .context("logs query flag requires a value")?; - return Ok(Some(value.to_owned())); - } - value if value.starts_with("--") => {} - value => return Ok(Some(value.to_owned())), - } - index += 1; - } - Ok(None) -} - -fn parse_optional_lines(args: &[String]) -> Result { - let mut index = 0; - while index < args.len() { - if matches!(args[index].as_str(), "--lines" | "-n") { - let value = args.get(index + 1).context("lines flag requires a value")?; - return value - .parse::() - .context("lines flag must be a positive number"); - } - index += 1; - } - Ok(DEFAULT_LOG_TAIL_LINES) -} - -#[derive(Debug, Default)] -struct DeterministicModelRecipe { - pub(crate) canonical_id: String, - pub(crate) min_gpu_mem_gib: Option, - pub(crate) engines: Vec, - pub(crate) engine_statuses: Vec, - pub(crate) warnings: Vec, -} - -fn parse_deterministic_model_recipes(tool_text: &str) -> Vec { - let mut recipes = Vec::new(); - let mut current: Option = None; - for line in tool_text.lines() { - let trimmed = line.trim(); - if let Some(recipe) = parse_model_recipe_header(trimmed) { - if let Some(previous) = current.replace(recipe) { - recipes.push(previous); - } - continue; - } - let Some(recipe) = current.as_mut() else { - continue; - }; - if is_engine_status_line(trimmed) { - recipe.engine_statuses.push(trimmed.to_owned()); - } else if let Some(warning) = trimmed.strip_prefix("warning:") { - recipe.warnings.push(warning.trim().to_owned()); - } - } - if let Some(recipe) = current { - recipes.push(recipe); - } - recipes -} - -fn parse_model_recipe_header(line: &str) -> Option { - if !line.contains(" aliases=[") || !line.contains("min_gpu_mem=") { - return None; - } - let canonical_id = line.split_whitespace().next()?.to_owned(); - let engines = parse_bracketed_csv(line, "engines=[", "]"); - let min_gpu_mem_gib = line - .split_once("min_gpu_mem=") - .and_then(|(_, rest)| rest.split_whitespace().next()) - .and_then(|value| value.parse::().ok()); - Some(DeterministicModelRecipe { - canonical_id, - min_gpu_mem_gib, - engines, - engine_statuses: Vec::new(), - warnings: Vec::new(), - }) -} - -fn parse_bracketed_csv(line: &str, start: &str, end: &str) -> Vec { - line.split_once(start) - .and_then(|(_, rest)| rest.split_once(end)) - .map(|(values, _)| { - values - .split(',') - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default() -} - -fn is_engine_status_line(line: &str) -> bool { - matches!( - line.split_once(':').map(|(engine, _)| engine), - Some("lemonade" | "vllm" | "aim") - ) -} - -#[derive(Debug, Default)] -struct DeterministicServiceRow { - pub(crate) service_id: String, - pub(crate) engine: String, - pub(crate) model: String, - pub(crate) status: String, - pub(crate) running_state: String, - pub(crate) endpoint: String, -} - -fn parse_deterministic_service_rows(tool_text: &str) -> Vec { - let mut rows = Vec::new(); - let mut current: Option = None; - for line in tool_text.lines() { - let trimmed = line.trim(); - if let Some(rest) = trimmed.strip_prefix("- ") { - if rest.starts_with("service_id=") { - if let Some(row) = current.take() { - rows.push(row); - } - let mut row = DeterministicServiceRow::default(); - for token in rest.split_whitespace() { - if let Some((key, value)) = token.split_once('=') { - match key { - "service_id" => row.service_id = value.to_owned(), - "engine" => row.engine = value.to_owned(), - "model" => row.model = value.to_owned(), - "status" => row.status = value.to_owned(), - "running_state" => row.running_state = value.to_owned(), - "endpoint" => row.endpoint = value.to_owned(), - _ => {} - } - } - } - if row.running_state.is_empty() { - row.running_state = managed_service_running_state(&row.status).to_owned(); - } - current = Some(row); - continue; - } - if let Some(row) = current.take() { - rows.push(row); - } - current = Some(DeterministicServiceRow { - service_id: rest.to_owned(), - ..Default::default() - }); - continue; - } - let Some(row) = current.as_mut() else { - continue; - }; - if let Some(value) = trimmed.strip_prefix("status:") { - row.status = value.trim().to_owned(); - row.running_state = managed_service_running_state(&row.status).to_owned(); - } else if let Some(value) = trimmed.strip_prefix("engine:") { - row.engine = value.trim().to_owned(); - } else if let Some(value) = trimmed.strip_prefix("model:") { - row.model = value.trim().to_owned(); - } else if let Some(value) = trimmed.strip_prefix("endpoint:") { - row.endpoint = value.trim().to_owned(); - } - } - if let Some(row) = current { - rows.push(row); - } - rows -} - -#[derive(Debug, Default)] -struct DeterministicEngineRow { - pub(crate) engine: String, - pub(crate) runtime: String, - pub(crate) note: String, -} - -fn parse_deterministic_engine_rows(tool_text: &str) -> Vec { - let mut rows = Vec::new(); - let mut current: Option = None; - for line in tool_text.lines() { - let trimmed = line.trim().trim_start_matches("* ").trim_start(); - if let Some(engine) = engine_name_from_inventory_line(trimmed) { - if let Some(row) = current.take() { - rows.push(row); - } - current = Some(DeterministicEngineRow { - engine: engine.to_owned(), - ..Default::default() - }); - continue; - } - let Some(row) = current.as_mut() else { - continue; - }; - if let Some(runtime) = trimmed.strip_prefix("runtime:") { - row.runtime = runtime.trim().to_owned(); - } else if let Some(note) = trimmed.strip_prefix("note:") { - row.note = note.trim().to_owned(); - } - } - if let Some(row) = current { - rows.push(row); - } - rows -} - -fn engine_name_from_inventory_line(line: &str) -> Option<&'static str> { - ["lemonade", "vllm", "aim"].into_iter().find(|engine| { - line == *engine - || line - .strip_prefix(*engine) - .is_some_and(|rest| rest.starts_with(char::is_whitespace)) - }) -} /// Read a one-shot chat prompt from standard input when it is piped in. /// @@ -437,7 +78,7 @@ fn engine_name_from_inventory_line(line: &str) -> Option<&'static str> { /// it never writes to and never closes therefore waits, exactly as `cat` would; /// the TTY guard is what keeps that off an interactive user, and a /// non-interactive caller with no prompt to send should pass `/dev/null` (as -/// `scripts/smoke_local.py` does) rather than an idle pipe. A read error — a +/// `cargo xtask smoke` does) rather than an idle pipe. A read error — a /// closed or non-UTF-8 fd 0 — is reported instead of being folded into "no /// prompt", so text that was piped but could not be decoded fails loudly rather /// than silently becoming a status screen and a zero exit. @@ -585,4086 +226,9 @@ pub(crate) fn parse_positive_u32(value: &str) -> Result { } Ok(parsed) } - -#[derive(Debug, Clone, Eq, PartialEq)] -struct ChatPromptResult { - pub rendered: String, - pub approval: Option, -} - -struct ChatToolRunResult { - pub(crate) approval: Option, - pub(crate) follow_up_text: String, - pub(crate) ran_read_only_tool: bool, - pub(crate) read_only_tool_error: bool, - pub(crate) needs_install_folder: bool, -} - -/// Convenience wrapper used by tests that do not exercise sampling controls. #[cfg(test)] -fn render_chat_prompt_result( - paths: &AppPaths, - provider: &str, - model: Option<&str>, - prompt: &str, - rocm_tools: bool, -) -> Result { - render_chat_prompt_result_with_progress( - paths, - provider, - model, - prompt, - rocm_tools, - ChatInferenceParams::default(), - None, - ) -} - -fn render_chat_prompt_result_with_progress( - paths: &AppPaths, - provider: &str, - model: Option<&str>, - prompt: &str, - rocm_tools: bool, - params: ChatInferenceParams, - progress: Option<&mut dyn FnMut(String)>, -) -> Result { - let mut progress = progress; - let user_prompt = latest_user_chat_message(prompt); - let assistant_model = local_rocm_tools_assistant_model(provider, rocm_tools).or(model); - let service_needed_model = if local_rocm_tools_assistant_model(provider, rocm_tools).is_some() { - None - } else { - model - }; - if rocm_tools && let Some(approval) = install_sdk_without_prefix_chat_approval(user_prompt) { - report_chat_tool_progress(&mut progress, "Asking the assistant."); - report_chat_tool_progress(&mut progress, "Review needed: Install ROCm"); - report_chat_tool_progress(&mut progress, "Waiting for the ROCm install folder."); - return Ok(ChatPromptResult { - rendered: render_install_sdk_folder_needed_chat_text(user_prompt, &approval.args), - approval: Some(approval), - }); - } - let mut messages = Vec::new(); - if rocm_tools { - messages.push(providers::ChatMessage { - role: "system".to_owned(), - content: rocm_chat_tool_system_prompt_for_host(Some(paths)), - }); - } - messages.push(providers::ChatMessage { - role: "user".to_owned(), - content: prompt.to_owned(), - }); - let mut response = if rocm_tools { - report_chat_tool_progress(&mut progress, "Asking the assistant."); - if let Some(call) = deterministic_mutating_tool_call_for_prompt(user_prompt)? { - report_chat_tool_progress(&mut progress, "Preparing a review card."); - providers::ChatResponse { - provider: provider.to_owned(), - model: assistant_model.unwrap_or("local").to_owned(), - content: deterministic_mutating_tool_intro(&call), - tool_calls: vec![call], - } - } else if !local_assistant_service_ready_for_chat(paths, assistant_model) - && let Some(response) = - local_read_only_fallback_response_for_prompt(provider, assistant_model, user_prompt) - { - report_chat_tool_progress(&mut progress, "Running ROCm status check."); - response - } else { - match providers::provider_chat( - paths, - provider, - &providers::ChatRequest { - model: assistant_model.map(str::to_owned), - messages: messages.clone(), - max_tokens: params.max_tokens, - temperature: params.temperature, - top_p: params.top_p, - rocm_tools, - }, - ) { - Ok(response) => response, - Err(error) - if provider == "local" && local_provider_missing_service_error(&error) => - { - if let Some(response) = local_read_only_fallback_response_for_prompt( - provider, - assistant_model, - user_prompt, - ) { - report_chat_tool_progress(&mut progress, "Running ROCm status check."); - response - } else { - return Ok(ChatPromptResult { - rendered: local_chat_service_needed_text( - service_needed_model, - user_prompt, - rocm_tools, - ), - approval: None, - }); - } - } - Err(error) => return Err(error), - } - } - } else { - report_chat_tool_progress(&mut progress, "Asking the assistant."); - match providers::provider_chat( - paths, - provider, - &providers::ChatRequest { - model: assistant_model.map(str::to_owned), - messages: messages.clone(), - max_tokens: params.max_tokens, - temperature: params.temperature, - top_p: params.top_p, - rocm_tools, - }, - ) { - Ok(response) => response, - Err(error) if provider == "local" && local_provider_missing_service_error(&error) => { - return Ok(ChatPromptResult { - rendered: local_chat_service_needed_text( - service_needed_model, - user_prompt, - rocm_tools, - ), - approval: None, - }); - } - Err(error) => return Err(error), - } - }; - let fallback_tool_call = if rocm_tools && response.tool_calls.is_empty() { - fallback_rocm_tool_call_for_prompt(user_prompt) - } else { - None - }; - let fallback_tool_call_used = fallback_tool_call.is_some(); - if let Some(call) = fallback_tool_call { - response.tool_calls.push(call); - } - if rocm_tools - && let Some(call) = - supplemental_read_only_tool_call_for_prompt(user_prompt, &response.tool_calls) - { - response.tool_calls.push(call); - } - let mut output = String::new(); - let _ = writeln!(output, "chat response"); - let _ = writeln!(output, " provider: {}", response.provider); - let _ = writeln!(output, " model: {}", response.model); - let _ = writeln!( - output, - " rocm tools: {}", - if rocm_tools { "enabled" } else { "off" } - ); - let _ = writeln!(output); - let initial_content_is_intermediate = fallback_tool_call_used - || local_tool_call_content_is_intermediate(provider, rocm_tools, &response); - let initial_content = visible_chat_content(&response.content); - if initial_content_is_intermediate { - let _ = writeln!(output, "Assistant is preparing the next step."); - } else if !initial_content.trim().is_empty() { - let _ = writeln!(output, "{initial_content}"); - } - let tool_result = if rocm_tools { - let explanation = (!fallback_tool_call_used && !initial_content.trim().is_empty()) - .then_some(initial_content.as_str()); - append_chat_tool_results(paths, &response, &mut output, explanation, &mut progress)? - } else { - ChatToolRunResult { - approval: None, - follow_up_text: String::new(), - ran_read_only_tool: false, - read_only_tool_error: false, - needs_install_folder: false, - } - }; - let deterministic_summary = deterministic_chat_tool_summary(&tool_result.follow_up_text); - if let Some(summary) = deterministic_summary.as_deref() { - let _ = writeln!(output); - let _ = writeln!(output, "ROCm CLI summary"); - let _ = writeln!(output, "{summary}"); - } - let follow_up_blocking_summary = deterministic_summary - .as_deref() - .filter(|_| deterministic_summary_can_stand_alone(&tool_result.follow_up_text)); - if should_request_local_tool_follow_up(provider, &tool_result, follow_up_blocking_summary) { - let follow_up_messages = vec![ - providers::ChatMessage { - role: "system".to_owned(), - content: "You are ROCm CLI's local assistant. ROCm tools have already been checked for this turn. Use the supplied tool results for local facts about this machine. For service results, ready/running means running, starting/recovering means starting, failed/stopped means not running, and no matching service row means ROCm CLI is not managing that service as running. If the tool results do not contain enough information to answer the whole question, say what is known from the results and then answer the rest normally from your own model knowledge. Do not request another tool call. Do not invent local paths, GPU names, versions, or install state that are not shown in the tool results. Keep the answer concise.".to_owned(), - }, - providers::ChatMessage { - role: "user".to_owned(), - content: format!( - "Original request:\n{}\n\nROCm tool results:\n{}\n\nAnswer the original request. Use the ROCm tool results for this machine's facts; if they are incomplete, answer the remaining part generally.", - user_prompt.trim(), - tool_result.follow_up_text.trim() - ), - }, - ]; - let follow_up = providers::provider_chat( - paths, - provider, - &providers::ChatRequest { - model: assistant_model.map(str::to_owned), - messages: follow_up_messages, - max_tokens: params.max_tokens.or(Some(256)), - temperature: params.temperature, - top_p: params.top_p, - rocm_tools: false, - }, - )?; - if follow_up.tool_calls.is_empty() { - let follow_up_content = visible_chat_content(&follow_up.content); - if local_follow_up_content_is_final(&follow_up) && !follow_up_content.trim().is_empty() - { - let _ = writeln!(output); - let _ = writeln!(output, "Assistant after ROCm checks"); - let _ = writeln!(output, "{}", follow_up_content.trim()); - } - } else { - let _ = writeln!(output); - let _ = writeln!( - output, - "Assistant asked for another ROCm check after the first tool results." - ); - let _ = writeln!( - output, - "rocm-cli stopped there so the answer is not based on another guess. Nothing was changed." - ); - } - } else if tool_result.read_only_tool_error { - let _ = writeln!(output); - let _ = writeln!( - output, - "Assistant did not answer from this result because a ROCm check reported an error." - ); - let _ = writeln!(output, "Nothing was changed."); - } else if tool_result.needs_install_folder { - let _ = writeln!(output); - let _ = writeln!(output, "Assistant after ROCm checks"); - let _ = writeln!( - output, - "I can install ROCm/TheRock, but I need the install folder first. Type the folder you want to use, for example D:\\ROCm\\therock_venvs. Nothing will download until ROCm CLI shows a review card and you approve it." - ); - } - if tool_result.approval.is_none() - && tool_result.ran_read_only_tool - && !output.contains("Nothing was changed.") - { - let _ = writeln!(output); - let _ = writeln!(output, "Nothing was changed."); - } - Ok(ChatPromptResult { - rendered: output, - approval: tool_result.approval, - }) -} - -fn report_chat_tool_progress(progress: &mut Option<&mut dyn FnMut(String)>, message: &str) { - if let Some(progress) = progress.as_deref_mut() { - progress(message.to_owned()); - } -} - -fn latest_user_chat_message(prompt: &str) -> &str { - const MARKER: &str = "\nNew message:\n"; - prompt - .rfind(MARKER) - .map_or(prompt, |index| &prompt[index + MARKER.len()..]) - .trim() -} - -fn deterministic_mutating_tool_call_for_prompt( - prompt: &str, -) -> Result> { - let Some(call) = fallback_rocm_tool_call_for_prompt(prompt) else { - return Ok(None); - }; - validate_chat_tool_call(&call)?; - if chat_tool_call_is_read_only(&call) { - Ok(None) - } else { - Ok(Some(call)) - } -} - -fn supplemental_read_only_tool_call_for_prompt( - prompt: &str, - existing_calls: &[providers::ChatToolCall], -) -> Option { - let normalized = latest_user_chat_message(prompt).to_ascii_lowercase(); - if prompt_mentions_serving_engine_or_service(&normalized) - && prompt_asks_running_or_status(&normalized) - && prompt_asks_engine_install_state(&normalized) - { - for call in [ - fallback_services_list_tool_call(), - fallback_engine_list_tool_call(), - ] { - if !chat_tool_calls_include_equivalent(existing_calls, &call) { - return Some(call); - } - } - return None; - } - let call = fallback_rocm_tool_call_for_prompt(prompt)?; - if !chat_tool_call_is_read_only(&call) - || chat_tool_calls_include_equivalent(existing_calls, &call) - { - return None; - } - Some(call) -} - -fn chat_tool_calls_include_equivalent( - existing_calls: &[providers::ChatToolCall], - required: &providers::ChatToolCall, -) -> bool { - existing_calls.iter().any(|existing| { - if existing.name != required.name { - return false; - } - if required.name == "rocm_command" { - return normalized_chat_rocm_command_args(existing).ok() - == normalized_chat_rocm_command_args(required).ok(); - } - if required.name == "port_status" { - return chat_port_status_tool_calls_equivalent(existing, required); - } - existing.arguments == required.arguments - }) -} - -fn chat_port_status_tool_calls_equivalent( - left: &providers::ChatToolCall, - right: &providers::ChatToolCall, -) -> bool { - let Some(left_object) = left.arguments.as_object() else { - return false; - }; - let Some(right_object) = right.arguments.as_object() else { - return false; - }; - let left_port = left_object.get("port").and_then(serde_json::Value::as_u64); - let right_port = right_object.get("port").and_then(serde_json::Value::as_u64); - if left_port != right_port { - return false; - } - let left_host = - json_string(left_object, "host").unwrap_or_else(|| DEFAULT_LOCAL_HOST.to_owned()); - let right_host = - json_string(right_object, "host").unwrap_or_else(|| DEFAULT_LOCAL_HOST.to_owned()); - loopback_host_key(&left_host) == loopback_host_key(&right_host) -} - -fn loopback_host_key(host: &str) -> String { - match host.trim().to_ascii_lowercase().as_str() { - "localhost" | "127.0.0.1" => "127.0.0.1".to_owned(), - "::1" | "[::1]" => "::1".to_owned(), - other => other.to_owned(), - } -} - -fn deterministic_mutating_tool_intro(call: &providers::ChatToolCall) -> String { - match chat_tool_approval_request(call, None) { - Ok(approval) if approval.pending_title == "Install ROCm" => { - "I can install ROCm/TheRock into the folder you chose. Review the card before anything downloads or changes." - .to_owned() - } - Ok(approval) if approval.pending_title == "Install ComfyUI" => { - "I can install ComfyUI into ROCm CLI's managed app folder. Review the card before anything downloads or changes." - .to_owned() - } - Ok(approval) if approval.pending_title == "Start ComfyUI" => { - "I can start ComfyUI for you. Review the card before ROCm CLI launches it." - .to_owned() - } - Ok(approval) if approval.pending_title == "Start local model server" => { - "I can start the recommended local model server on the GPU. Review the card before ROCm CLI launches it." - .to_owned() - } - Ok(approval) => format!( - "I can prepare this ROCm change: {}. Review the card before anything runs.", - approval.pending_title - ), - Err(_) => { - "I can prepare this ROCm change. Review the card before anything runs.".to_owned() - } - } -} - -fn local_tool_call_content_is_intermediate( - provider: &str, - rocm_tools: bool, - response: &providers::ChatResponse, -) -> bool { - provider == "local" - && rocm_tools - && !response.tool_calls.is_empty() - && !response.content.trim().is_empty() -} - -fn local_follow_up_content_is_final(response: &providers::ChatResponse) -> bool { - response.tool_calls.is_empty() && !response.content.trim().is_empty() -} - -fn local_read_only_fallback_response_for_prompt( - provider: &str, - assistant_model: Option<&str>, - prompt: &str, -) -> Option { - if provider != "local" || !prompt_can_use_read_only_without_local_assistant(prompt) { - return None; - } - let call = fallback_rocm_tool_call_for_prompt(prompt)?; - if !chat_tool_call_is_read_only(&call) { - return None; - } - Some(providers::ChatResponse { - provider: provider.to_owned(), - model: assistant_model.unwrap_or("local").to_owned(), - content: String::new(), - tool_calls: vec![call], - }) -} - -fn local_assistant_service_ready_for_chat(paths: &AppPaths, assistant_model: Option<&str>) -> bool { - let model = assistant_model.unwrap_or(providers::BUILTIN_ASSISTANT_MODEL_ID); - load_managed_services(paths).is_ok_and(|records| { - records.iter().any(|record| { - matches!(record.status.as_str(), "ready" | "running") - && (service_model_names_match(&record.canonical_model_id, model) - || service_model_names_match(&record.model_ref, model)) - }) - }) -} - -fn prompt_can_use_read_only_without_local_assistant(prompt: &str) -> bool { - let normalized = latest_user_chat_message(prompt).to_ascii_lowercase(); - let mentions_status_subject = any_substring( - &normalized, - &[ - "comfyui", - "comfy ui", - "comfy", - "vllm", - "lemonade", - "llama.cpp", - "llama cpp", - "qwen", - "model server", - "local server", - "local model server", - "assistant server", - "port", - "8188", - "therock", - "rocm", - ], - ); - mentions_status_subject - && (prompt_asks_running_or_status(&normalized) - || any_substring( - &normalized, - &["installed", "available", "detected", "engine status"], - )) -} - -fn prompt_mentions_serving_engine_or_service(normalized_prompt: &str) -> bool { - any_substring( - normalized_prompt, - &[ - "vllm", - "lemonade", - "llama.cpp", - "llama cpp", - "qwen", - "model server", - "local server", - "local model server", - "assistant server", - ], - ) -} - -fn prompt_asks_engine_install_state(normalized_prompt: &str) -> bool { - any_substring( - normalized_prompt, - &["installed", "available", "detected", "engine status"], - ) -} - -fn fallback_engine_list_tool_call() -> providers::ChatToolCall { - providers::ChatToolCall { - id: Some("fallback-engine-list".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["engines", "list"] }), - } -} - -fn fallback_services_list_tool_call() -> providers::ChatToolCall { - providers::ChatToolCall { - id: Some("fallback-services-list".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["services", "list", "--all"] }), - } -} - -fn fallback_rocm_tool_call_for_prompt(prompt: &str) -> Option { - let prompt = latest_user_chat_message(prompt); - let normalized = prompt.to_ascii_lowercase(); - let asks_running_or_status = prompt_asks_running_or_status(&normalized); - let mentions_comfyui = any_substring(&normalized, &["comfyui", "comfy ui", "comfy"]); - if asks_running_or_status - && !mentions_comfyui - && any_substring(&normalized, &["8188", "port 8188"]) - { - return Some(providers::ChatToolCall { - id: Some("fallback-port-8188-status".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": DEFAULT_LOCAL_HOST, "port": 8188 }), - }); - } - let mentions_serving_engine_or_service = prompt_mentions_serving_engine_or_service(&normalized); - if mentions_serving_engine_or_service && prompt_asks_engine_install_state(&normalized) { - return Some(fallback_engine_list_tool_call()); - } - if mentions_serving_engine_or_service && asks_running_or_status { - return Some(fallback_services_list_tool_call()); - } - let mentions_llm_or_model = any_substring( - &normalized, - &["llm", "llms", "model", "models", "assistant"], - ); - let asks_support_or_fit = any_substring( - &normalized, - &[ - "support", - "supported", - "run", - "runs", - "fit", - "fits", - "can my machine", - "can this machine", - ], - ); - if mentions_llm_or_model && asks_support_or_fit { - return Some(providers::ChatToolCall { - id: Some("fallback-rocm-model".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["model"] }), - }); - } - - if mentions_comfyui { - if any_substring(&normalized, &["log", "logs"]) { - return Some(providers::ChatToolCall { - id: Some("fallback-comfyui-logs".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["comfyui", "logs"] }), - }); - } - if asks_running_or_status { - return Some(providers::ChatToolCall { - id: Some("fallback-comfyui-status".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["comfyui", "status"] }), - }); - } - if any_substring(&normalized, &["start", "run", "launch", "open"]) { - return Some(providers::ChatToolCall { - id: Some("fallback-comfyui-start".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["comfyui", "start"], - "reason": "Start ComfyUI locally after the user approves it." - }), - }); - } - if any_substring( - &normalized, - &[ - "can you setup", - "can you set up", - "please setup", - "please set up", - "setup comfyui for me", - "set up comfyui for me", - "install comfyui", - "download comfyui", - ], - ) { - return Some(providers::ChatToolCall { - id: Some("fallback-comfyui-install".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["comfyui", "install"], - "reason": "Install ComfyUI into ROCm CLI's managed app folder after the user approves it." - }), - }); - } - return Some(providers::ChatToolCall { - id: Some("fallback-comfyui-status".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["comfyui", "status"] }), - }); - } - - if let Some(call) = fallback_config_tool_call_for_prompt(&normalized) { - return Some(call); - } - - if mentions_llm_or_model - && any_substring( - &normalized, - &[ - "serve", - "server", - "start", - "setup and serve", - "set up and serve", - "run locally", - "local model", - "local assistant", - ], - ) - { - return Some(providers::ChatToolCall { - id: Some("fallback-serve-qwen".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["serve", "qwen", "--engine", "lemonade", "--device", "gpu_required", "--managed"], - "reason": "Start the recommended local assistant after the user approves it." - }), - }); - } - - let mentions_setup = any_substring(&normalized, &["setup", "set up", "install"]); - let asks_how = any_substring(&normalized, &["how", "help", "what do i need"]); - let mentions_rocm_or_therock = any_substring(&normalized, &["therock", "rocm"]); - let requested_install_prefix = requested_install_prefix_from_prompt(prompt); - if requested_install_prefix.is_none() - && let Some(approval) = install_sdk_without_prefix_chat_approval(prompt) - { - return Some(providers::ChatToolCall { - id: Some("fallback-therock-install-folder".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": approval.args, - "reason": "Ask the user to choose the ROCm/TheRock install folder before installing." - }), - }); - } - if mentions_rocm_or_therock - && prompt_requests_install_action(&normalized) - && let Some(prefix) = requested_install_prefix - { - let mut args = vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - prefix, - ]; - if let Some(build_date) = requested_therock_build_date_from_prompt(&normalized) { - args.push("--build-date".to_owned()); - args.push(build_date); - } else if let Some(version) = requested_therock_version_from_prompt(&normalized) { - args.push("--version".to_owned()); - args.push(version); - } - return Some(providers::ChatToolCall { - id: Some("fallback-therock-install".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": args, - "reason": "Install TheRock ROCm into the user-selected folder after the user approves it." - }), - }); - } - if asks_how && mentions_setup && any_substring(&normalized, &["therock", "rocm"]) { - return Some(providers::ChatToolCall { - id: Some("fallback-therock-examine".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }); - } - - let asks_where_installed = any_substring( - &normalized, - &[ - "where is rocm", - "where's rocm", - "where is therock", - "where's therock", - "where did rocm", - "where did therock", - "where rocm is installed", - "where therock is installed", - "rocm install folder", - "therock install folder", - "rocm installed at", - "therock installed at", - ], - ); - let asks_status = asks_where_installed - || any_substring( - &normalized, - &[ - "is rocm installed", - "is therock installed", - "is therock setup", - "is therock set up", - "rocm installed", - "therock installed", - "check this rocm setup", - "which gpu", - "what gpu", - "gpu is on", - "gpu do i have", - "my machine", - ], - ); - if asks_status && any_substring(&normalized, &["gpu", "rocm", "therock", "setup"]) { - return Some(providers::ChatToolCall { - id: Some("fallback-examine".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }); - } - - None -} - -fn prompt_asks_running_or_status(normalized: &str) -> bool { - any_substring( - normalized, - &[ - "running", - "is it up", - "is this up", - "is there", - "are there", - "status", - "started", - "listening", - "on port", - "port ", - ], - ) -} - -fn fallback_config_tool_call_for_prompt(normalized: &str) -> Option { - if !any_substring( - normalized, - &[ - "config", - "setting", - "settings", - "default engine", - "telemetry", - ], - ) { - return None; - } - if any_substring(normalized, &["show", "check", "what", "current", "list"]) { - return Some(providers::ChatToolCall { - id: Some("fallback-config-show".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["config", "show"] }), - }); - } - if any_substring(normalized, &["default engine", "set engine", "use engine"]) { - // Word-boundary inference: a bare `contains("aim")` matched "aiming". - if let Some(engine) = infer_engine_from_request(normalized) { - return Some(providers::ChatToolCall { - id: Some("fallback-config-default-engine".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["config", "set-default-engine", engine], - "reason": "Change ROCm CLI's default engine after the user approves it." - }), - }); - } - } - if normalized.contains("telemetry") { - if any_substring(normalized, &["off", "disable", "disabled"]) { - return Some(providers::ChatToolCall { - id: Some("fallback-config-telemetry-off".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["config", "set-telemetry", "off"], - "reason": "Turn ROCm CLI telemetry off after the user approves it." - }), - }); - } - if any_substring(normalized, &["local", "on", "enable", "enabled"]) { - return Some(providers::ChatToolCall { - id: Some("fallback-config-telemetry-local".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["config", "set-telemetry", "local"], - "reason": "Enable local-only ROCm inspection after the user approves it." - }), - }); - } - } - None -} - -fn install_sdk_without_prefix_chat_approval(prompt: &str) -> Option { - install_sdk_chat_approval_for_prompt(prompt).and_then(|approval| { - chat_cli_arg_value(&approval.args, "--prefix") - .is_none() - .then_some(approval) - }) -} - -fn install_sdk_chat_approval_for_prompt(prompt: &str) -> Option { - let prompt = latest_user_chat_message(prompt); - let normalized = prompt.to_ascii_lowercase(); - if !prompt_requests_rocm_install_or_setup(&normalized) { - return None; - } - let mut args = vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - if normalized.contains("nightly") { - "nightly".to_owned() - } else { - "release".to_owned() - }, - "--format".to_owned(), - "wheel".to_owned(), - ]; - if let Some(prefix) = requested_install_prefix_from_prompt(prompt) { - args.push("--prefix".to_owned()); - args.push(prefix); - } - if let Some(build_date) = requested_therock_build_date_from_prompt(&normalized) { - args.push("--build-date".to_owned()); - args.push(build_date); - } else if let Some(version) = requested_therock_version_from_prompt(&normalized) { - args.push("--version".to_owned()); - args.push(version); - } - Some(ChatToolApprovalRequest { - pending_title: "Install ROCm".to_owned(), - command_title: "Install".to_owned(), - display_command: Some(format_structured_tool_call("rocm", &args)), - args, - explanation: Some( - "Install ROCm/TheRock after the user chooses the install folder.".to_owned(), - ), - }) -} - -fn render_install_sdk_folder_needed_chat_text(prompt: &str, args: &[String]) -> String { - let mut output = String::new(); - let _ = writeln!(output, "ROCm install"); - let _ = writeln!(output); - let _ = writeln!(output, "I can install ROCm/TheRock for you."); - let _ = writeln!( - output, - "First choose the folder where ROCm CLI should put the Python environment." - ); - let _ = writeln!( - output, - "Nothing will download or change until you review and approve the install." - ); - if let Some(date) = chat_cli_arg_value(args, "--build-date") { - let _ = writeln!(output); - let _ = writeln!(output, "Requested build date: {date}"); - } - if let Some(version) = chat_cli_arg_value(args, "--version") { - let _ = writeln!(output); - let _ = writeln!(output, "Requested version: {version}"); - } - let _ = writeln!(output); - let _ = writeln!(output, "Your request"); - let _ = writeln!(output, " {}", prompt.trim()); - output.trim_end().to_owned() -} - -fn prompt_requests_rocm_install_or_setup(normalized: &str) -> bool { - let mentions_rocm_stack = any_substring( - normalized, - &[ - "rocm", - "therock", - "the rock", - "amd gpu package", - "amd gpu ready", - "local ai", - ], - ); - if !mentions_rocm_stack { - return false; - } - prompt_requests_install_action(normalized) - || any_substring( - normalized, - &[ - "need rocm", - "need therock", - "need the rock", - "need to get rocm", - "need to get therock", - "get rocm installed", - "get therock installed", - "get the rock installed", - "make rocm work", - "make therock work", - "make the rock work", - "prepare rocm", - "prepare therock", - "prepare the rock", - "set up my amd gpu", - "setup my amd gpu", - "set up local ai", - "setup local ai", - "get local ai working", - "make local ai work", - "rocm please", - "make my amd gpu ready", - "make my gpu ready", - "get my amd gpu ready", - "get my gpu ready", - ], - ) -} - -fn prompt_requests_install_action(normalized: &str) -> bool { - any_substring( - normalized, - &[ - "install this", - "install specific", - "install the", - "install rocm", - "install therock", - "install the rock", - "install amd gpu package", - "install local ai", - "can you install", - "please install", - "rocm please", - "therock please", - "the rock please", - "need to install", - "want to install", - "i need rocm", - "i need therock", - "i need the rock", - "get me rocm", - "get me therock", - "get my gpu ready", - "make my gpu ready", - "make my amd gpu ready", - "get installed", - "setup for me", - "set up for me", - "setup rocm", - "set up rocm", - "setup therock", - "set up therock", - "setup the rock", - "set up the rock", - "setup my gpu", - "set up my gpu", - "setup amd gpu", - "set up amd gpu", - ], - ) -} - -fn visible_chat_content(content: &str) -> String { - let mut output = String::new(); - let mut remaining = content; - while let Some(start) = find_ascii_case_insensitive(remaining, "") { - output.push_str(&remaining[..start]); - let after_start = &remaining[start + "".len()..]; - let Some(end) = find_ascii_case_insensitive(after_start, "") else { - remaining = ""; - break; - }; - remaining = &after_start[end + "".len()..]; - } - output.push_str(remaining); - output.trim().to_owned() -} - -fn find_ascii_case_insensitive(haystack: &str, needle: &str) -> Option { - let needle = needle.as_bytes(); - haystack - .as_bytes() - .windows(needle.len()) - .position(|window| window.eq_ignore_ascii_case(needle)) -} - -/// The host-independent half of the assistant prompt. -/// -/// Statements that are only true on *some* hosts do not belong here — they used -/// to, and a WSL user was told "on native Windows, vLLM is skipped" while vLLM -/// was in fact their supported path. Anything host-dependent now comes from -/// [`chat_host_facts::HostFacts`], which knows which machine it is describing. -const ROCM_CHAT_TOOL_SYSTEM_PROMPT: &str = "You are ROCm CLI's local assistant. Speak in simple English for non-technical users. Use the provided ROCm tools when you need to inspect this machine, preview setup, read service logs, check updates, inspect automations, install or start ROCm-managed apps, or request ROCm/TheRock, config, engine, app, and local model server changes. For simple greetings or thanks like hello, hi, hey, ok, or thank you, reply normally; do not inspect ROCm, do not call tools, and do not launch or propose a model server. Tool-use rules: inspect first with read-only tools; call rocm_command only with argv-style args and no shell text; use natural_language_plan for ROCm requests that do not fit another read-only tool; ask for a mutating tool call only after explaining why it is needed; summarize tool results after they are returned. Read-only tools may run immediately. Tools that install, launch, stop, delete, or change state require user approval; request rocm_command and explain why. For 'is X running?', 'what is running?', status, or port questions, inspect before answering and do not start, stop, install, or serve anything. For ComfyUI or port 8188 use [\"comfyui\",\"status\"] or port_status. For vLLM, Lemonade, AIM, qwen, or local model servers use [\"services\",\"list\",\"--all\"] for running state and [\"engines\",\"list\"] for installed/available engine state. Treat ready/running as running, starting/recovering as starting, failed/stopped as not running, and no matching record as unknown or not managed by ROCm CLI. Interpret Examine carefully: active_runtime_status=ready means ROCm CLI has an active managed TheRock/ROCm runtime; legacy_rocm_status=not_detected only means no global system ROCm install was found. If active_runtime_status=ready, tell the user ROCm/TheRock is installed and active for ROCm CLI. For 'is TheRock installed', 'is ROCm installed', or 'which GPU is on this machine', use examine or gpu_snapshot before answering. For 'how do I setup TheRock' or install/setup requests, guide the user to choose an install folder first; do not answer with only a status check. For 'which LLMs can this machine support', use rocm_command args [\"model\"] or natural_language_plan before answering. For TheRock installs, always let the user choose the install folder. If the user names a folder or prefix, preserve that exact folder with [\"--prefix\",\"PATH\"]; you may call path_exists first to check whether that user-provided folder or its parent exists. If the user asks you to install TheRock/ROCm but has not named a folder, ask for the folder or let the guided setup folder picker collect it; do not invent a hidden default folder and do not request an install command without --prefix. Use rocm_command args [\"install\",\"sdk\",\"--channel\",\"release\",\"--format\",\"wheel\",\"--prefix\",\"PATH\"] only when the user asks you to install it and a folder is known; for a requested build date add [\"--build-date\",\"YYYY-MM-DD\"] and for a requested exact version add [\"--version\",\"VERSION\"]. For config changes, inspect with [\"config\",\"show\"] first when useful, then request config subcommands such as [\"config\",\"set-default-engine\",\"lemonade\"], [\"config\",\"set-default-runtime\",\"RUNTIME_KEY\"], or [\"config\",\"set-telemetry\",\"local\"] only after explaining why. For ComfyUI, use rocm_command with args like [\"comfyui\",\"status\"], [\"comfyui\",\"logs\"], [\"comfyui\",\"install\"], [\"comfyui\",\"start\"], or [\"comfyui\",\"stop\"]. First-time setup is the same thing as bootstrap in ROCm CLI; it is a deterministic ROCm setup flow, not a separate model chat. The built-in local assistant is fixed to qwen, which maps to Qwen3-4B-Instruct-2507-GGUF served by Lemonade with gpu_required. vLLM, Lemonade, and AIM (Docker-based) are the general serving engines; inspect or manage them when the user asks about general model serving, but do not switch the built-in assistant away from Lemonade. Use qwen-smoke only for a quick server smoke test. For vLLM management, inspect engines first and use [\"engines\",\"install\",\"vllm\"] or [\"serve\",\"MODEL\",\"--engine\",\"vllm\",\"--device\",\"gpu_required\",\"--managed\"] only where the host supports it. Do not invent shell commands and do not request CPU fallback."; -const ROCM_CHAT_TOOL_SKILL: &str = include_str!("../../../skills/rocm-cli-assistant/SKILL.md"); - -fn rocm_chat_tool_system_prompt() -> String { - format!("{ROCM_CHAT_TOOL_SYSTEM_PROMPT}\n\nROCm CLI assistant skill:\n{ROCM_CHAT_TOOL_SKILL}") -} - -/// The assistant prompt grounded in the machine it will answer for. -/// -/// The single composition point: both chat surfaces (`rocm chat --prompt -/// --tools` in this bin, and the dashboard chat via -/// [`crate::dash::resolved_args`]) send this, so neither can drift into -/// answering platform questions from pretraining alone. -pub(crate) fn rocm_chat_tool_system_prompt_for_host(paths: Option<&AppPaths>) -> String { - let facts = chat_host_facts::HostFacts::detect(paths); - format!("{}\n\n{}", rocm_chat_tool_system_prompt(), facts.render()) -} - -fn local_provider_missing_service_error(error: &anyhow::Error) -> bool { - error.chain().any(|cause| { - cause - .to_string() - .contains("local provider has no ready managed service") - }) -} - -fn local_rocm_tools_assistant_model(provider: &str, rocm_tools: bool) -> Option<&'static str> { - (provider == "local" && rocm_tools).then_some(providers::BUILTIN_ASSISTANT_MODEL_ID) -} - -fn local_chat_service_needed_text(model: Option<&str>, prompt: &str, rocm_tools: bool) -> String { - let mut output = String::new(); - let _ = writeln!(output, "No local assistant is running yet."); - let _ = writeln!(output); - let _ = writeln!( - output, - "To use LLM-assisted ROCm commands, start a ROCm GPU local model server first." - ); - let _ = writeln!( - output, - "First-time ROCm setup does not need an LLM; run `rocm` and use Set Up ROCm for that." - ); - let _ = writeln!(output); - let _ = writeln!( - output, - "Recommended path:\n run `rocm`, choose Start a local model, use the recommended assistant model, then start it" - ); - let _ = writeln!(output); - if let Some(model) = model.filter(|value| !value.trim().is_empty()) { - let args = vec![ - "serve".to_owned(), - model.to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ]; - let _ = writeln!( - output, - "Advanced manual command for the selected model:\n {}", - format_structured_tool_call("rocm", &args) - ); - } else { - let example_args = vec![ - "serve".to_owned(), - providers::LEMONADE_ASSISTANT_MODEL_ID.to_owned(), - "--engine".to_owned(), - "lemonade".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ]; - let _ = writeln!( - output, - "Advanced manual command for the recommended assistant model:\n {}", - format_structured_tool_call("rocm", &example_args) - ); - } - let mut chat_args = vec!["chat".to_owned()]; - if rocm_tools { - chat_args.push("--tools".to_owned()); - } - chat_args.push("--provider".to_owned()); - chat_args.push("local".to_owned()); - if let Some(model) = model.filter(|value| !value.trim().is_empty()) { - chat_args.push("--model".to_owned()); - chat_args.push(model.to_owned()); - } - if !prompt.trim().is_empty() { - chat_args.push("--prompt".to_owned()); - chat_args.push(prompt.to_owned()); - } - let _ = writeln!(output); - let _ = writeln!( - output, - "After the server is ready, run:\n {}", - format_structured_tool_call("rocm", &chat_args) - ); - let _ = writeln!(output); - let _ = writeln!( - output, - "For the guided flow, run `rocm`, choose Start a local model, start a model, then choose Chat." - ); - let _ = writeln!(output, "Nothing was changed."); - output.trim_end().to_owned() -} - -fn append_chat_tool_results( - paths: &AppPaths, - response: &providers::ChatResponse, - output: &mut String, - assistant_explanation: Option<&str>, - progress: &mut Option<&mut dyn FnMut(String)>, -) -> Result { - if response.tool_calls.is_empty() { - let _ = writeln!(output, "ROCm checks used"); - let _ = writeln!(output, " none requested"); - return Ok(ChatToolRunResult { - approval: None, - follow_up_text: String::new(), - ran_read_only_tool: false, - read_only_tool_error: false, - needs_install_folder: false, - }); - } - - let _ = writeln!(output, "ROCm checks used"); - let mut approval = None; - let mut follow_up_text = String::new(); - let mut ran_read_only_tool = false; - let mut read_only_tool_error = false; - let mut needs_install_folder = false; - for call in &response.tool_calls { - if chat_tool_call_requests_therock_install_without_prefix(call) { - needs_install_folder = true; - report_chat_tool_progress(progress, "Waiting for the ROCm install folder."); - let _ = writeln!(output, " Install ROCm: needs install folder"); - let _ = writeln!( - output, - " not run: choose an install folder before the review card" - ); - continue; - } - validate_chat_tool_call(call)?; - let label = chat_tool_call_display_label(call); - if chat_tool_call_is_read_only(call) { - report_chat_tool_progress(progress, &format!("Running ROCm check: {label}.")); - let result = run_chat_read_only_tool(paths, call)?; - ran_read_only_tool = true; - let is_error = mcp_tool_result_is_error(&result); - read_only_tool_error |= is_error; - report_chat_tool_progress( - progress, - &format!( - "ROCm check finished: {label} ({})", - chat_read_only_tool_status_label(is_error) - ), - ); - let _ = writeln!( - output, - " {}: {}", - label, - chat_read_only_tool_status_label(is_error) - ); - let result_text = mcp_tool_result_text(&result); - let _ = writeln!(follow_up_text, "{}:", call.name); - let _ = writeln!(follow_up_text, "{result_text}"); - for line in result_text.lines() { - let _ = writeln!(output, " {line}"); - } - } else { - report_chat_tool_progress(progress, &format!("Review needed: {label}.")); - let _ = writeln!(output, " {label}: needs your review"); - let _ = writeln!( - output, - " not run: review the approval card before anything runs" - ); - if let Some(command) = rocm_chat_tool_requested_command(call) { - let _ = writeln!(output, " advanced manual command: {command}"); - } - if approval.is_none() { - approval = Some(chat_tool_approval_request(call, assistant_explanation)?); - } - } - } - Ok(ChatToolRunResult { - approval, - follow_up_text, - ran_read_only_tool, - read_only_tool_error, - needs_install_folder, - }) -} - -fn chat_tool_call_requests_therock_install_without_prefix(call: &providers::ChatToolCall) -> bool { - let Some(object) = call.arguments.as_object() else { - return false; - }; - match call.name.as_str() { - "install_sdk" => json_string(object, "prefix").is_none(), - "rocm_command" => rocm_command_args_install_sdk_without_prefix(object), - _ => false, - } -} - -fn rocm_command_args_install_sdk_without_prefix( - object: &serde_json::Map, -) -> bool { - let Some(args) = object.get("args").and_then(serde_json::Value::as_array) else { - return false; - }; - let args = args - .iter() - .filter_map(serde_json::Value::as_str) - .map(str::to_owned) - .collect::>(); - args.first() - .is_some_and(|arg| arg.eq_ignore_ascii_case("install")) - && args - .get(1) - .is_some_and(|arg| arg.eq_ignore_ascii_case("sdk")) - && chat_cli_arg_value(&args, "--prefix").is_none() -} - -fn managed_service_running_state(status: &str) -> &'static str { - match status { - "ready" | "running" => "running", - "starting" | "recovering" => "starting", - "failed" | "stopped" => "not_running", - _ => "unknown", - } -} - -fn render_services_tool_result_text(records: &[ManagedServiceRecord]) -> String { - let mut output = String::new(); - let _ = writeln!(output, "managed_services: {}", records.len()); - let _ = writeln!( - output, - "status_meaning: ready/running = running; starting/recovering = starting; failed/stopped = not running; no matching row = not managed by ROCm CLI" - ); - if records.is_empty() { - let _ = writeln!(output, "services: none"); - return output; - } - let _ = writeln!(output, "services:"); - for record in records { - let _ = writeln!( - output, - " - service_id={} engine={} model={} canonical_model={} status={} running_state={} endpoint={}", - record.service_id, - record.engine, - record.model_ref, - record.canonical_model_id, - record.status, - managed_service_running_state(&record.status), - record.endpoint_url - ); - } - output -} - -fn deterministic_rocm_tool_summary(tool_text: &str) -> Option { - if !tool_text.contains("examine:") { - return None; - } - let mut lines = Vec::new(); - let gpu = chat_tool_value(tool_text, "driver_detail") - .filter(|value| value != "") - .or_else(|| { - let target = chat_tool_value(tool_text, "detected_gfx_target")?; - Some(format!("AMD GPU target {target}")) - }); - if let Some(gpu) = gpu { - lines.push(format!(" GPU: {gpu}")); - } - - let active_status = chat_tool_value(tool_text, "active_runtime_status"); - if active_status.as_deref() == Some("ready") { - let family = chat_tool_value(tool_text, "active_runtime_family") - .filter(|value| value != "" && value != ""); - let version = chat_tool_value(tool_text, "active_runtime_version") - .filter(|value| value != "" && value != ""); - let mut detail = "installed and active for ROCm CLI".to_owned(); - if let Some(family) = family { - let _ = write!(detail, " ({family})"); - } - if let Some(version) = version { - let _ = write!(detail, ", {version}"); - } - lines.push(format!(" ROCm/TheRock: {detail}")); - if let Some(root) = chat_tool_value(tool_text, "active_runtime_root") - .or_else(|| chat_tool_value(tool_text, "setup_runtime_root")) - .filter(|value| value != "" && value != "" && value != "") - { - lines.push(format!(" Install folder: {root}")); - } - if let Some(cache) = chat_tool_value(tool_text, "active_runtime_pip_cache_dir") - .or_else(|| chat_tool_value(tool_text, "setup_runtime_pip_cache_dir")) - .filter(|value| value != "" && value != "" && value != "") - { - lines.push(format!(" Downloads/cache: {cache}")); - } - } else if let Some(status) = active_status.as_deref() { - lines.push(format!(" ROCm/TheRock: active runtime status is {status}")); - if let Some(root) = chat_tool_value(tool_text, "setup_runtime_root") - .filter(|value| value != "" && value != "" && value != "") - { - lines.push(format!(" Selected setup folder: {root}")); - } - if let Some(cache) = chat_tool_value(tool_text, "setup_runtime_pip_cache_dir") - .filter(|value| value != "" && value != "" && value != "") - { - lines.push(format!(" Downloads/cache: {cache}")); - } - } - - if chat_tool_value(tool_text, "legacy_rocm_status").as_deref() == Some("not_detected") - && active_status.as_deref() == Some("ready") - { - lines.push( - " Note: no global legacy ROCm install was found; ROCm CLI is using its managed TheRock runtime." - .to_owned(), - ); - } - - (!lines.is_empty()).then(|| lines.join("\n")) -} - -fn deterministic_chat_tool_summary(tool_text: &str) -> Option { - deterministic_rocm_tool_summary(tool_text) - .or_else(|| deterministic_model_tool_summary(tool_text)) - .or_else(|| deterministic_combined_status_tool_summary(tool_text)) -} - -fn deterministic_combined_status_tool_summary(tool_text: &str) -> Option { - let summaries = [ - deterministic_comfyui_tool_summary(tool_text), - deterministic_port_status_tool_summary(tool_text), - deterministic_engine_inventory_tool_summary(tool_text), - deterministic_services_tool_summary(tool_text), - ] - .into_iter() - .flatten() - .collect::>(); - (!summaries.is_empty()).then(|| summaries.join("\n")) -} - -fn deterministic_summary_can_stand_alone(tool_text: &str) -> bool { - !tool_text.contains("model recipes") -} - -fn deterministic_model_tool_summary(tool_text: &str) -> Option { - if !tool_text.contains("model recipes") { - return None; - } - let recipes = parse_deterministic_model_recipes(tool_text); - if recipes.is_empty() { - return None; - } - - let mut lines = Vec::new(); - if let Some(recipe) = recipes - .iter() - .find(|recipe| recipe.canonical_id == providers::LEMONADE_ASSISTANT_MODEL_ID) - { - lines.push(format!( - " Recommended local assistant: qwen ({})", - recipe.canonical_id - )); - lines.push(format!( - " Fit: needs about {} GPU memory.", - recipe - .min_gpu_mem_gib - .map_or_else(|| "unknown".to_owned(), |value| format!("{value} GiB")) - )); - lines.push(format!( - " Engine: {}", - deterministic_engine_fit_summary(recipe, "lemonade") - )); - } - - if let Some(recipe) = recipes - .iter() - .find(|recipe| recipe.canonical_id == "Qwen3-0.6B-GGUF") - { - lines.push(format!( - " Tiny smoke test: qwen-smoke ({})", - recipe.canonical_id - )); - lines.push( - " Use this only to check that GPU serving starts; it is not the default assistant." - .to_owned(), - ); - } - - if let Some(recipe) = recipes - .iter() - .find(|recipe| recipe.canonical_id == "meta-llama/Llama-3.2-3B-Instruct") - { - lines.push(format!( - " 8 GiB-class option: llama ({})", - recipe.canonical_id - )); - lines.push(format!( - " Fit: asks for {}; this can be tight on APUs and depends on available shared GPU memory.", - recipe - .min_gpu_mem_gib.map_or_else(|| "unknown GPU memory".to_owned(), |value| format!("{value} GiB")) - )); - lines.push(format!( - " Engines: {}", - deterministic_available_engine_names(recipe).join(", ") - )); - } - - let larger = recipes - .iter() - .filter(|recipe| recipe.min_gpu_mem_gib.is_some_and(|value| value > 8)) - .filter(|recipe| recipe.canonical_id != providers::BUILTIN_ASSISTANT_MODEL_ID) - .map(|recipe| { - format!( - "{} asks for {}", - recipe.canonical_id, - recipe.min_gpu_mem_gib.map_or_else( - || "more GPU memory".to_owned(), - |value| format!("{value} GiB") - ) - ) - }) - .collect::>(); - if !larger.is_empty() { - lines.push(format!( - " Larger recipes are not low-VRAM defaults: {}.", - larger.join("; ") - )); - } - - let linux_wsl_only = recipes - .iter() - .flat_map(|recipe| { - recipe - .engine_statuses - .iter() - .filter(|status| { - status.contains("unsupported_native_windows") - || status.to_ascii_lowercase().contains("wsl/linux") - || status.to_ascii_lowercase().contains("linux/wsl") - }) - .filter_map(|status| { - let (engine, _) = status.split_once(':')?; - Some(format!( - "{} uses {} through WSL/Linux on Windows", - recipe.canonical_id, engine - )) - }) - }) - .collect::>(); - if !linux_wsl_only.is_empty() { - lines.push(format!( - " Native Windows note: {}.", - linux_wsl_only.join("; ") - )); - } - - if !lines.is_empty() { - lines.push( - " Run `rocm examine` to refresh GPU memory details before starting anything large." - .to_owned(), - ); - } - (!lines.is_empty()).then(|| lines.join("\n")) -} - -fn deterministic_engine_fit_summary(recipe: &DeterministicModelRecipe, engine: &str) -> String { - let status = recipe - .engine_statuses - .iter() - .find(|status| status.starts_with(&format!("{engine}:"))) - .map(std::string::String::as_str); - match status { - Some(status) if status.contains("unsupported_native_windows") => { - format!("{engine}: WSL/Linux only on Windows") - } - Some(status) - if status.split_once(':').is_some_and(|(_, rest)| { - rest.trim_start().starts_with("available") - || rest.trim_start().starts_with("adapter_available") - }) => - { - format!("{engine}: available") - } - Some(status) => status.to_owned(), - None if recipe.engines.iter().any(|candidate| candidate == engine) => { - format!("{engine}: listed") - } - None => format!("{engine}: not listed"), - } -} - -fn deterministic_available_engine_names(recipe: &DeterministicModelRecipe) -> Vec { - let engines = recipe - .engine_statuses - .iter() - .filter_map(|status| { - let (engine, rest) = status.split_once(':')?; - (rest.trim_start().starts_with("available") - || rest.trim_start().starts_with("adapter_available")) - .then_some(engine.to_owned()) - }) - .collect::>(); - if engines.is_empty() { - recipe.engines.clone() - } else { - engines - } -} - -pub(crate) fn deterministic_comfyui_tool_summary(tool_text: &str) -> Option { - if !tool_text.contains("ComfyUI") || !tool_text.contains("Running") { - return None; - } - let mut lines = Vec::new(); - match chat_tool_value(tool_text, "installed").as_deref() { - Some("yes") => lines.push(" ComfyUI: installed.".to_owned()), - Some("no") => lines.push(" ComfyUI: not installed.".to_owned()), - Some(value) => lines.push(format!(" ComfyUI: installed status is {value}.")), - None => {} - } - if let Some(status) = chat_tool_value(tool_text, "status") { - let running_text = if status.eq_ignore_ascii_case("running") { - "running".to_owned() - } else if status.eq_ignore_ascii_case("starting") { - "starting".to_owned() - } else { - format!("not running ({status})") - }; - lines.push(format!(" Running: {running_text}.")); - } - if let Some(models_path) = chat_tool_value(tool_text, "models path") { - lines.push(format!(" Models folder: {models_path}.")); - } - // A note about the install itself (an interrupted reinstall, which names - // the command that finishes it) sits between `status`'s `ComfyUI` heading - // and its `Running` section, whose own notes the `Running:` line sums up. - for line in tool_text - .lines() - .skip_while(|line| line.trim() != "ComfyUI") - .take_while(|line| line.trim() != "Running") - { - if let Some(note) = line.trim().strip_prefix("note:").map(str::trim) - && !note.is_empty() - { - lines.push(format!(" Note: {note}.")); - } - } - (!lines.is_empty()).then(|| lines.join("\n")) -} - -fn deterministic_services_tool_summary(tool_text: &str) -> Option { - if !tool_text.contains("Local Servers") - && !tool_text.contains("managed_services:") - && !tool_text.contains("services:") - { - return None; - } - let rows = parse_deterministic_service_rows(tool_text); - let mut lines = Vec::new(); - let live = rows - .iter() - .filter(|row| matches!(row.running_state.as_str(), "running" | "starting")) - .collect::>(); - if live.is_empty() { - lines.push(" Local model servers: none running under ROCm CLI.".to_owned()); - } else { - lines.push(format!( - " Local model servers running/starting: {}.", - live.len() - )); - for row in live.iter().take(4) { - lines.push(format!( - " {} {} ({}) at {}.", - row.engine, - row.model, - row.running_state, - empty_as_unknown(&row.endpoint) - )); - } - } - let past = rows - .iter() - .filter(|row| !matches!(row.running_state.as_str(), "running" | "starting")) - .take(3) - .map(|row| { - format!( - "{} {} {}", - empty_as_unknown(&row.engine), - empty_as_unknown(&row.model), - empty_as_unknown(&row.status) - ) - }) - .collect::>(); - if !past.is_empty() { - lines.push(format!(" Past/non-running records: {}.", past.join("; "))); - } - (!lines.is_empty()).then(|| lines.join("\n")) -} - -fn deterministic_port_status_tool_summary(tool_text: &str) -> Option { - if !tool_text.contains("port_status:") && !tool_text.contains("listening:") { - return None; - } - let port = chat_tool_value(tool_text, "port")?; - let listening = chat_tool_value(tool_text, "listening").unwrap_or_else(|| "unknown".to_owned()); - let mut lines = Vec::new(); - let state = match listening.as_str() { - "true" => "listening", - "false" => "not listening", - _ => "unknown", - }; - lines.push(format!(" Port {port}: {state}.")); - if let Some(hint) = chat_tool_value(tool_text, "hint") { - lines.push(format!(" Hint: {hint}.")); - } - if chat_tool_value(tool_text, "managed_service").as_deref() == Some("none") { - lines.push(" Managed service: none on that endpoint.".to_owned()); - } - Some(lines.join("\n")) -} - -fn deterministic_engine_inventory_tool_summary(tool_text: &str) -> Option { - if !tool_text.contains("Local model engines") { - return None; - } - let rows = parse_deterministic_engine_rows(tool_text); - if rows.is_empty() { - return None; - } - let mut lines = vec![" Engine runtimes:".to_owned()]; - for row in rows { - let mut line = format!( - " {}: {}", - friendly_engine_label(&row.engine), - empty_as_unknown(&row.runtime) - ); - if !row.note.trim().is_empty() { - let _ = write!(line, " ({})", row.note.trim()); - } - line.push('.'); - lines.push(line); - } - Some(lines.join("\n")) -} - -fn should_request_local_tool_follow_up( - provider: &str, - tool_result: &ChatToolRunResult, - deterministic_summary: Option<&str>, -) -> bool { - provider == "local" - && deterministic_summary.is_none() - && tool_result.approval.is_none() - && tool_result.ran_read_only_tool - && !tool_result.read_only_tool_error - && !tool_result.follow_up_text.trim().is_empty() -} - -fn chat_tool_value(tool_text: &str, key: &str) -> Option { - let prefix = format!("{key}:"); - tool_text.lines().find_map(|line| { - line.trim() - .strip_prefix(&prefix) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) - }) -} - -// pub(crate): called from main.rs's own MCP dispatch and from the dash -// execution seam (dash_seam.rs) -pub(crate) fn run_internal_mcp_call( - paths: &AppPaths, - name: &str, - arguments: serde_json::Value, - allow_mutation: bool, -) -> Result { - let arguments = internal_mcp_arguments(arguments); - let call = providers::ChatToolCall { - id: None, - name: name.to_owned(), - arguments: serde_json::Value::Object(arguments.clone()), - }; - validate_chat_tool_call(&call)?; - if !chat_tool_call_is_read_only(&call) && !allow_mutation { - bail!( - "MCP tool `{name}` changes local ROCm state; rerun `rocm mcp-call {name}` with --allow-mutation only after explicit user approval" - ); - } - - match name { - "examine" | "doctor" => { - let examine = ExamineSummary::gather()?; - let text = render_examine_text()?; - Ok(internal_mcp_tool_success(text, serde_json::json!(examine))) - } - "bridge_snapshot" => { - let snapshot = build_codex_bridge_snapshot(paths)?; - Ok(internal_mcp_tool_success( - format!( - "Captured bridge snapshot for {} / {} with default engine `{}`.", - snapshot.examine.os, snapshot.examine.arch, snapshot.examine.default_engine - ), - serde_json::json!(snapshot), - )) - } - "gpu_snapshot" => { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - let gpu = build_codex_bridge_gpu_snapshot(&config); - let status = if !config.telemetry.local_inspection_enabled() { - "GPU telemetry is disabled by rocm-cli config." - } else if gpu.amd_smi_available { - "Captured amd-smi GPU snapshot." - } else { - "Use `rocm examine` for the current local AMD GPU summary." - }; - Ok(internal_mcp_tool_success( - status.to_owned(), - serde_json::json!(gpu), - )) - } - "engines" => { - let engines = builtin_codex_bridge_engine_inventory(); - Ok(internal_mcp_tool_success( - format!("Found {} engine entries.", engines.len()), - serde_json::json!({ "engines": engines }), - )) - } - "services" => { - let services = load_managed_services(paths)?; - Ok(internal_mcp_tool_success( - render_services_tool_result_text(&services), - serde_json::json!({ "services": services }), - )) - } - "port_status" => run_chat_port_status_tool(paths, &call), - "service_logs" => { - let service_id = json_string(&arguments, "service_id") - .context("service_logs requires `service_id`")?; - let lines = arguments - .get("lines") - .and_then(serde_json::Value::as_u64) - .unwrap_or(80) - .clamp(1, 500) as usize; - let record = load_managed_services(paths)? - .into_iter() - .find(|service| service.service_id == service_id) - .with_context(|| format!("managed service `{service_id}` not found"))?; - let tail = read_tail_lines(&record.log_path, lines, "service log")?.join("\n"); - Ok(internal_mcp_tool_success( - format!( - "Read the last {} line(s) from service `{}`.", - lines, record.service_id - ), - serde_json::json!({ - "service": record, - "lines": lines, - "tail": tail, - }), - )) - } - "automations" => { - let event_limit = arguments - .get("event_limit") - .and_then(serde_json::Value::as_u64) - .unwrap_or(10) - .clamp(1, 64) as usize; - let runtime = AutomationRuntimeState::load(paths)?; - let events = load_recent_automation_events(paths, event_limit)?; - Ok(internal_mcp_tool_success( - format!( - "Loaded automation runtime and {} recent events.", - events.len() - ), - serde_json::json!({ - "runtime": runtime, - "recent_events": events, - }), - )) - } - "natural_language_plan" => { - let request = json_string(&arguments, "request") - .context("natural_language_plan requires `request`")?; - let config = RocmCliConfig::load(paths).unwrap_or_default(); - // Built exactly once: the text shown to the operator and the - // `action` reported as what would run next both come from the - // same `StructuredRequestPlan`, so a provider-assisted plan (not - // guaranteed deterministic across calls) cannot describe one - // command in `text` and a different one in `action`. - let (text, action) = plan_freeform_request_with_context(&request, paths, &config); - let action = action.map(|action| { - serde_json::json!({ - "args": action.args, - "approval_required": action.approval_required, - "has_placeholders": action.has_placeholders, - "provider_assisted": action.provider_assisted, - }) - }); - Ok(internal_mcp_tool_success( - "Planned the ROCm request.".to_owned(), - serde_json::json!({ - "request": request, - "text": text, - "action": action, - }), - )) - } - "rocm_command" => { - let action = chat_rocm_command_action(&call)?; - let output = match action { - ChatRocmCommandAction::ReadOnly(args) => { - run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))? - } - ChatRocmCommandAction::Approval { args, .. } if allow_mutation => { - let refs = args.iter().map(String::as_str).collect::>(); - run_rocm_capture_for_paths(paths, &refs, Duration::from_mins(2))? - } - ChatRocmCommandAction::Approval { .. } => { - bail!("rocm_command changes local ROCm state and needs approval") - } - }; - Ok(internal_mcp_tool_result_from_command( - "Ran `rocm` command.", - output, - false, - )) - } - "update_check" => { - let output = - run_rocm_command_for_paths(paths, &["update".to_owned()], Duration::from_mins(1))?; - Ok(internal_mcp_tool_result_from_command( - "Ran `rocm update`.", - output, - false, - )) - } - "install_sdk_dry_run" => { - let args = internal_mcp_install_sdk_args(&arguments, true)?; - let output = run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))?; - Ok(internal_mcp_tool_result_from_command( - "Ran `rocm install sdk --dry-run`.", - output, - false, - )) - } - "path_exists" => run_chat_path_exists_tool(&call), - "proposal_action" => { - // Executes IN-PROCESS (no CLI subprocess): show loads a proposal; - // approve/reject mutate its status (allow_mutation already enforced - // by the read-only split above). proposal_action executes in-process - // (via update_automation_proposal_status); it never delegates to the - // subprocess arm. - let proposal_id = json_string(&arguments, "proposal_id") - .context("proposal_action requires `proposal_id`")?; - let action = - json_string(&arguments, "action").context("proposal_action requires `action`")?; - match action.as_str() { - "show" => { - let proposal = rocm_core::find_automation_proposal(paths, &proposal_id) - .with_context(|| { - format!("automation proposal `{proposal_id}` not found") - })?; - Ok(internal_mcp_tool_success( - format!( - "Proposal {} ({}): {}", - proposal.proposal_id, proposal.status, proposal.title - ), - serde_json::json!({ - "id": proposal.proposal_id, - "status": proposal.status, - "summary": proposal.title, - "reason": proposal.message, - }), - )) - } - "approve" | "reject" => { - let status = if action == "approve" { - "approved" - } else { - "rejected" - }; - let updated = - rocm_core::update_automation_proposal_status(paths, &proposal_id, status)?; - record_cli_audit_event( - paths, - "automations", - if action == "approve" { - "proposal_approved" - } else { - "proposal_rejected" - }, - "info", - format!("proposal {proposal_id} {status}"), - None, - ); - Ok(internal_mcp_tool_success( - format!("Proposal {proposal_id} {status}."), - serde_json::json!({ - "id": updated.proposal_id, - "status": updated.status, - "summary": updated.title, - "reason": updated.message, - }), - )) - } - other => bail!("proposal_action `{other}` is not supported"), - } - } - "install_sdk" | "install_engine" | "launch_server" | "stop_server" | "watcher_enable" - | "watcher_disable" => { - let args = rocm_chat_tool_requested_args(&call) - .with_context(|| format!("MCP tool `{name}` is missing required arguments"))?; - let refs = args.iter().map(String::as_str).collect::>(); - let output = run_rocm_capture_for_paths(paths, &refs, Duration::from_mins(2))?; - Ok(internal_mcp_tool_result_from_command( - "Ran approved `rocm` command.", - output, - false, - )) - } - other => bail!("unsupported MCP tool `{other}`"), - } -} - -fn internal_mcp_arguments(value: serde_json::Value) -> serde_json::Map { - if let Some(arguments) = value - .get("arguments") - .and_then(serde_json::Value::as_object) - { - return arguments.clone(); - } - value.as_object().cloned().unwrap_or_default() -} - -fn internal_mcp_tool_success(text: String, structured: serde_json::Value) -> serde_json::Value { - serde_json::json!({ - "content": [ - { - "type": "text", - "text": text, - } - ], - "structuredContent": structured, - "isError": false, - }) -} - -fn internal_mcp_tool_result_from_command( - prefix: &str, - output: CommandCapture, - is_error: bool, -) -> serde_json::Value { - let text = format!("{prefix}\n\n{}", command_capture_text(&output)); - serde_json::json!({ - "content": [ - { - "type": "text", - "text": text, - } - ], - "structuredContent": { - "argv": output.argv, - "exit_status": output.exit_status, - "stdout": output.stdout, - "stderr": output.stderr, - }, - "isError": is_error || output.exit_status != 0, - }) -} - -fn run_rocm_capture_for_paths( - paths: &AppPaths, - args: &[&str], - timeout: Duration, -) -> Result { - rocm_core::run_rocm_capture_with_env(args, &app_path_env_vars(paths), timeout) -} - -fn run_rocm_command_for_paths( - paths: &AppPaths, - args: &[String], - _timeout: Duration, -) -> Result { - match run_rocm_read_only_in_process(paths, args) { - Ok(stdout) => Ok(CommandCapture { - argv: std::iter::once("rocm".to_owned()) - .chain(args.iter().cloned()) - .collect(), - exit_status: 0, - stdout, - stderr: String::new(), - }), - Err(error) => Err(error).with_context(|| { - format!( - "read-only assistant command is not implemented in-process: {}", - format_structured_tool_call("rocm", args) - ) - }), - } -} - -fn run_rocm_read_only_in_process(paths: &AppPaths, args: &[String]) -> Result { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - match args { - [] => bail!("rocm command requires at least one argument"), - [command] if command.eq_ignore_ascii_case("examine") => { - render_examine_text_with_paths(paths, &config) - } - [command] - if command.eq_ignore_ascii_case("version") - || command == "--version" - || command == "-V" => - { - Ok(format!("rocm-cli {}\n", cli_version_string())) - } - [command] - if command.eq_ignore_ascii_case("model") || command.eq_ignore_ascii_case("models") => - { - Ok(render_model_registry_verbose_text_with_context_and_host( - Some(paths), - None, - None, - )) - } - [command] if command.eq_ignore_ascii_case("daemon") => { - Ok(render_daemon_text(paths, &config)) - } - [command] if command.eq_ignore_ascii_case("logs") => Ok(render_logs_text(paths)), - [command, rest @ ..] if command.eq_ignore_ascii_case("logs") => { - let query = parse_optional_query(rest)?; - Ok(render_logs_browser_text(paths, query.as_deref())) - } - [command] if command.eq_ignore_ascii_case("runtimes") => { - render_runtimes_text(paths, &config) - } - [command, subcommand] - if command.eq_ignore_ascii_case("runtimes") - && subcommand.eq_ignore_ascii_case("list") => - { - render_runtimes_text(paths, &config) - } - [command, subcommand] - if command.eq_ignore_ascii_case("engines") - && subcommand.eq_ignore_ascii_case("list") => - { - Ok(render_engine_inventory_text_with_paths(Some(paths))) - } - [command] if command.eq_ignore_ascii_case("services") => render_services_text(paths, false), - [command, subcommand] - if command.eq_ignore_ascii_case("services") - && subcommand.eq_ignore_ascii_case("list") => - { - render_services_text(paths, false) - } - [command, subcommand, flag] - if command.eq_ignore_ascii_case("services") - && subcommand.eq_ignore_ascii_case("list") - && matches!(flag.as_str(), "-a" | "--all") => - { - render_services_text(paths, true) - } - [command, subcommand, service_id] - if command.eq_ignore_ascii_case("services") - && matches!(subcommand.to_ascii_lowercase().as_str(), "logs" | "log") => - { - render_service_logs_text(paths, service_id) - } - [command] if command.eq_ignore_ascii_case("automations") => { - render_automations_text(paths, &config) - } - [command, subcommand] - if command.eq_ignore_ascii_case("automations") - && subcommand.eq_ignore_ascii_case("list") => - { - render_automations_text(paths, &config) - } - [command, subcommand] - if command.eq_ignore_ascii_case("config") - && subcommand.eq_ignore_ascii_case("show") => - { - Ok(render_config_text(paths, &config)) - } - [command] if command.eq_ignore_ascii_case("comfyui") => { - comfyui::render_status(paths, &config) - } - [command, subcommand] - if command.eq_ignore_ascii_case("comfyui") - && subcommand.eq_ignore_ascii_case("status") => - { - comfyui::render_status(paths, &config) - } - [command, subcommand, rest @ ..] - if command.eq_ignore_ascii_case("comfyui") - && matches!(subcommand.to_ascii_lowercase().as_str(), "logs" | "log") => - { - comfyui::render_logs( - paths, - parse_optional_lines(rest).unwrap_or(DEFAULT_LOG_TAIL_LINES), - ) - } - [command, rest @ ..] - if command.eq_ignore_ascii_case("update") - && !rest.iter().any(|arg| arg.eq_ignore_ascii_case("--apply")) => - { - render_update_text(paths) - } - [command, subcommand, rest @ ..] - if command.eq_ignore_ascii_case("install") - && subcommand.eq_ignore_ascii_case("sdk") - && rest.iter().any(|arg| arg.eq_ignore_ascii_case("--dry-run")) => - { - render_install_sdk_dry_run_for_args(paths, rest) - } - [command, rest @ ..] - if command.eq_ignore_ascii_case("uninstall") - && rest.iter().any(|arg| arg.eq_ignore_ascii_case("--dry-run")) => - { - render_uninstall_dry_run(paths) - } - [command] if command.eq_ignore_ascii_case("setup") => { - render_setup_status_text(paths, &config) - } - [command, subcommand] - if command.eq_ignore_ascii_case("setup") - && subcommand.eq_ignore_ascii_case("status") => - { - render_setup_status_text(paths, &config) - } - // `install driver` is a read-only call where the command only reports - // (`chat_install_driver_action`). It runs as a dry run that never - // executes anything, so a call that reaches it elsewhere cannot change - // the host either. `--dry-run` is accepted because it is what this arm - // already does; `--reconcile` is not, since it rewrites the saved driver - // install state and the command refuses to combine it with the other - // flags. - [command, subcommand, rest @ ..] - if command.eq_ignore_ascii_case("install") - && subcommand.eq_ignore_ascii_case("driver") - && rest.iter().all(|arg| { - arg.eq_ignore_ascii_case("--dkms") || arg.eq_ignore_ascii_case("--dry-run") - }) => - { - let dkms = rest.iter().any(|arg| arg.eq_ignore_ascii_case("--dkms")); - crate::driver_install::install_driver(paths, dkms, false, true) - .map(|result| result.output) - .map_err(|error| error.source) - } - _ => bail!( - "unsupported in-process read-only rocm command: {}", - format_structured_tool_call("rocm", args) - ), - } -} - -fn render_install_sdk_dry_run_for_args(paths: &AppPaths, args: &[String]) -> Result { - let channel = chat_cli_arg_value(args, "--channel").unwrap_or("release"); - let format = chat_cli_arg_value(args, "--format").unwrap_or("wheel"); - let prefix = chat_cli_arg_value(args, "--prefix").map(PathBuf::from); - let version = chat_cli_arg_value(args, "--version").map(str::to_owned); - let build_date = chat_cli_arg_value(args, "--build-date").map(str::to_owned); - let selector = therock_install_version_selector(version, build_date)?; - let devel = chat_cli_has_flag(args, "--devel"); - // Dry run, so nothing is displaced and the consent gate is never reached; - // the narrow consent is what this chat surface would pass for a real - // install, and passing `--yes`'s source here would be a lie waiting to be - // printed if the preview ever grew a gate. - Ok(therock::install_sdk( - paths, - therock::SdkInstallRequest { - channel, - format, - prefix, - version_selector: selector, - dry_run: true, - include_devel: devel, - consent: therock::SdkInstallConsent::Preapproved( - therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault, - ), - ..therock::SdkInstallRequest::default() - }, - )? - .output) -} - -fn internal_mcp_install_sdk_args( - arguments: &serde_json::Map, - dry_run: bool, -) -> Result> { - let channel = json_string(arguments, "channel").unwrap_or_else(|| "release".to_owned()); - let format = json_string(arguments, "format").unwrap_or_else(|| "wheel".to_owned()); - let mut argv = vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - channel, - "--format".to_owned(), - format, - ]; - if let Some(prefix) = json_string(arguments, "prefix") { - let prefix_path = Path::new(&prefix); - if chat_install_prefix_is_system(prefix_path) { - bail!( - "install_sdk prefix `{}` is a system folder; choose a user folder instead", - prefix_path.display() - ); - } - argv.push("--prefix".to_owned()); - argv.push(prefix); - } - if let Some(version) = json_string(arguments, "version") { - argv.push("--version".to_owned()); - argv.push(version); - } - if let Some(build_date) = json_string(arguments, "build_date") { - argv.push("--build-date".to_owned()); - argv.push(build_date); - } - if dry_run { - argv.push("--dry-run".to_owned()); - } - Ok(argv) -} - -fn run_chat_read_only_tool( - paths: &AppPaths, - call: &providers::ChatToolCall, -) -> Result { - match call.name.as_str() { - "path_exists" => run_chat_path_exists_tool(call), - "port_status" => run_chat_port_status_tool(paths, call), - "rocm_command" => { - let action = chat_rocm_command_action(call)?; - let ChatRocmCommandAction::ReadOnly(args) = action else { - bail!("assistant read-only path cannot run mutating rocm_command"); - }; - let output = run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))?; - Ok(internal_mcp_tool_result_from_command( - "Ran `rocm` command.", - output, - false, - )) - } - "update_check" => { - let output = - run_rocm_command_for_paths(paths, &["update".to_owned()], Duration::from_mins(1))?; - Ok(internal_mcp_tool_result_from_command( - "Ran `rocm update`.", - output, - false, - )) - } - "install_sdk_dry_run" => { - let arguments = internal_mcp_arguments(call.arguments.clone()); - let args = internal_mcp_install_sdk_args(&arguments, true)?; - let output = run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))?; - Ok(internal_mcp_tool_result_from_command( - "Ran `rocm install sdk --dry-run`.", - output, - false, - )) - } - _ => run_internal_mcp_call(paths, &call.name, call.arguments.clone(), false), - } -} - -fn run_chat_path_exists_tool(call: &providers::ChatToolCall) -> Result { - let object = call - .arguments - .as_object() - .context("path_exists arguments must be a JSON object")?; - let path = json_string(object, "path").context("path_exists requires path")?; - let path = Path::new(&path); - let metadata = path.metadata().ok(); - let path_kind = metadata.as_ref().map_or("missing", |metadata| { - if metadata.is_dir() { - "directory" - } else if metadata.is_file() { - "file" - } else { - "other" - } - }); - let parent = path.parent(); - let parent_exists = parent.is_some_and(Path::exists); - let parent_display = parent.map_or_else( - || "".to_owned(), - |parent| parent.display().to_string(), - ); - let text = format!( - "path: {}\nexists: {}\nkind: {}\nparent: {}\nparent_exists: {}", - path.display(), - metadata.is_some(), - path_kind, - parent_display, - parent_exists - ); - Ok(serde_json::json!({ - "content": [{ - "type": "text", - "text": text, - }] - })) -} - -fn run_chat_port_status_tool( - paths: &AppPaths, - call: &providers::ChatToolCall, -) -> Result { - let object = call - .arguments - .as_object() - .context("port_status arguments must be a JSON object")?; - let host = json_string(object, "host").unwrap_or_else(|| DEFAULT_LOCAL_HOST.to_owned()); - let port = object - .get("port") - .and_then(serde_json::Value::as_u64) - .context("port_status requires port")? as u16; - let reachable = loopback_tcp_port_is_reachable(&host, port); - let host_key = loopback_host_key(&host); - let matching_services = load_managed_services(paths)? - .into_iter() - .filter(|record| { - record.port == port - && loopback_host_key(&record.host) == host_key - && managed_service_is_live(record) - }) - .collect::>(); - let app_hint = if port == comfyui::default_port() { - Some("ComfyUI default port") - } else if port == rocm_core::DEFAULT_LOCAL_PORT { - Some("ROCm local model server default port") - } else { - None - }; - let mut text = String::new(); - let _ = writeln!(text, "host: {host}"); - let _ = writeln!(text, "port: {port}"); - let _ = writeln!(text, "listening: {reachable}"); - if let Some(app_hint) = app_hint { - let _ = writeln!(text, "hint: {app_hint}"); - } - if matching_services.is_empty() { - let _ = writeln!(text, "managed_service: none"); - } else { - let _ = writeln!(text, "managed_services:"); - for service in &matching_services { - let _ = writeln!( - text, - " - service_id={} engine={} model={} status={} running_state={} endpoint={}", - service.service_id, - service.engine, - service.model_ref, - service.status, - managed_service_running_state(&service.status), - service.endpoint_url - ); - } - } - Ok(serde_json::json!({ - "content": [{ - "type": "text", - "text": text, - }], - "structuredContent": { - "host": host, - "port": port, - "listening": reachable, - "hint": app_hint, - "managed_services": matching_services, - }, - "isError": false, - })) -} - -const fn chat_read_only_tool_status_label(is_error: bool) -> &'static str { - if is_error { - "reported an error" - } else { - "done" - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::tests::test_paths; - use rocm_core::ManagedServiceRecord; - use std::fs; - use std::net::TcpListener; - - fn rocm_command_call(args: &[&str]) -> providers::ChatToolCall { - providers::ChatToolCall { - id: Some("test".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": args }), - } - } - - #[test] - fn mcp_tool_result_text_joins_text_content_items() { - let value = serde_json::json!({ - "content": [ - {"type": "text", "text": "line one"}, - {"type": "image", "text": "ignored"}, - {"type": "text", "text": "line two"}, - ] - }); - assert_eq!(mcp_tool_result_text(&value), "line one\nline two"); - } - - #[test] - fn mcp_tool_result_text_falls_back_to_raw_value_without_text_content() { - let value = serde_json::json!({"content": []}); - assert_eq!(mcp_tool_result_text(&value), value.to_string()); - } - - #[test] - fn mcp_tool_result_is_error_reads_the_is_error_flag() { - assert!(mcp_tool_result_is_error( - &serde_json::json!({"isError": true}) - )); - assert!(!mcp_tool_result_is_error( - &serde_json::json!({"isError": false}) - )); - assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); - } - - #[test] - fn chat_tool_display_label_maps_known_tools_and_humanizes_others() { - assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); - assert_eq!( - chat_tool_display_label("gpu_snapshot"), - "Checked GPU status" - ); - assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); - assert_eq!( - chat_tool_display_label("some_unmapped_tool"), - "some unmapped tool" - ); - } - - #[test] - fn chat_tool_call_display_label_delegates_for_non_rocm_command_tools() { - let call = providers::ChatToolCall { - id: None, - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }; - assert_eq!(chat_tool_call_display_label(&call), "Checked this computer"); - } - - #[test] - fn chat_tool_call_display_label_recognizes_rocm_command_shapes() { - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["examine"])), - "Checked this computer" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["models"])), - "Checked model recipes" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["model"])), - "Checked model recipes" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["ENGINES", "List"])), - "Checked local engines" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["services"])), - "Checked model servers" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["services", "list", "-a"])), - "Checked model servers" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["services", "list", "--all"])), - "Checked model servers" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["comfyui"])), - "Checked ComfyUI" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["comfyui", "status"])), - "Checked ComfyUI" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["comfyui", "logs", "-n", "50"])), - "Read ComfyUI logs" - ); - assert_eq!( - chat_tool_call_display_label(&rocm_command_call(&["config", "show"])), - "Checked ROCm config" - ); - } - - #[test] - fn chat_tool_call_display_label_falls_back_for_unparseable_args() { - let call = providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({"args": []}), - }; - assert_eq!(chat_tool_call_display_label(&call), "rocm command"); - } - - #[test] - fn loopback_tcp_port_is_reachable_detects_listening_and_closed_ports() { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback listener"); - let port = listener.local_addr().expect("local addr").port(); - assert!(loopback_tcp_port_is_reachable("127.0.0.1", port)); - drop(listener); - - // Port 0 can never be connected to; stands in for "closed" without - // depending on a specific unused port staying unused. - assert!(!loopback_tcp_port_is_reachable("127.0.0.1", 0)); - } - - #[test] - fn parse_optional_query_reads_flag_and_positional_forms() { - assert_eq!( - parse_optional_query(&["--query".to_owned(), "gpu".to_owned()]).unwrap(), - Some("gpu".to_owned()) - ); - assert_eq!( - parse_optional_query(&["-q".to_owned(), "gpu".to_owned()]).unwrap(), - Some("gpu".to_owned()) - ); - assert_eq!( - parse_optional_query(&["gpu".to_owned()]).unwrap(), - Some("gpu".to_owned()) - ); - assert_eq!(parse_optional_query(&[]).unwrap(), None); - } - - #[test] - fn parse_optional_query_rejects_a_dangling_flag() { - assert!(parse_optional_query(&["--query".to_owned()]).is_err()); - } - - #[test] - fn parse_optional_query_skips_unknown_flags_to_find_the_positional_value() { - assert_eq!( - parse_optional_query(&["--unknown-flag".to_owned(), "gpu".to_owned()]).unwrap(), - Some("gpu".to_owned()) - ); - } - - #[test] - fn parse_optional_lines_reads_flag_or_defaults() { - assert_eq!( - parse_optional_lines(&["--lines".to_owned(), "25".to_owned()]).unwrap(), - 25 - ); - assert_eq!( - parse_optional_lines(&["-n".to_owned(), "25".to_owned()]).unwrap(), - 25 - ); - assert_eq!(parse_optional_lines(&[]).unwrap(), DEFAULT_LOG_TAIL_LINES); - } - - #[test] - fn parse_optional_lines_rejects_a_non_numeric_value() { - assert!(parse_optional_lines(&["--lines".to_owned(), "nope".to_owned()]).is_err()); - } - - #[test] - fn parse_deterministic_model_recipes_reads_header_fields_and_notes() { - let recipes = parse_deterministic_model_recipes( - "\ - Qwen3-4B-Instruct-2507-GGUF aliases=[qwen, lemonade-qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] -lemonade: ready -warning: slow on this GPU - Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] -vllm: not_installed -", - ); - assert_eq!(recipes.len(), 2); - assert_eq!(recipes[0].canonical_id, "Qwen3-4B-Instruct-2507-GGUF"); - assert_eq!(recipes[0].min_gpu_mem_gib, Some(4)); - assert_eq!(recipes[0].engines, vec!["lemonade".to_owned()]); - assert_eq!( - recipes[0].engine_statuses, - vec!["lemonade: ready".to_owned()] - ); - assert_eq!(recipes[0].warnings, vec!["slow on this GPU".to_owned()]); - assert_eq!(recipes[1].canonical_id, "Qwen/Qwen3.5-4B"); - assert_eq!(recipes[1].min_gpu_mem_gib, Some(12)); - } - - #[test] - fn parse_deterministic_model_recipes_requires_aliases_bracket_in_header() { - let recipes = parse_deterministic_model_recipes( - "\ - NotAHeader task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] - Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] -", - ); - assert_eq!(recipes.len(), 1); - assert_eq!(recipes[0].canonical_id, "Qwen/Qwen3.5-4B"); - } - - #[test] - fn parse_deterministic_service_rows_reads_inline_key_value_form() { - let rows = parse_deterministic_service_rows( - " - service_id=svc-1 engine=vllm model=qwen status=ready running_state=running endpoint=127.0.0.1:8000", - ); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].service_id, "svc-1"); - assert_eq!(rows[0].engine, "vllm"); - assert_eq!(rows[0].status, "ready"); - assert_eq!(rows[0].running_state, "running"); - assert_eq!(rows[0].endpoint, "127.0.0.1:8000"); - } - - #[test] - fn parse_deterministic_service_rows_derives_running_state_when_absent() { - let rows = parse_deterministic_service_rows( - "\ -- svc-2 -status: starting -engine: lemonade -model: qwen -endpoint: 127.0.0.1:8001 -", - ); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].service_id, "svc-2"); - assert_eq!(rows[0].running_state, "starting"); - } - - #[test] - fn parse_deterministic_service_rows_derives_running_state_for_inline_form_when_absent() { - let rows = parse_deterministic_service_rows( - " - service_id=svc-3 engine=vllm model=qwen status=starting endpoint=127.0.0.1:8002", - ); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].running_state, "starting"); - } - - #[test] - fn parse_deterministic_engine_rows_reads_runtime_and_note() { - let rows = parse_deterministic_engine_rows( - "\ -* lemonade - runtime: gfx120X-all - note: default engine -* vllm - runtime: gfx110X-all -", - ); - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].engine, "lemonade"); - assert_eq!(rows[0].runtime, "gfx120X-all"); - assert_eq!(rows[0].note, "default engine"); - assert_eq!(rows[1].engine, "vllm"); - assert_eq!(rows[1].note, ""); - } - - #[test] - fn managed_service_running_state_maps_every_known_status() { - assert_eq!(managed_service_running_state("ready"), "running"); - assert_eq!(managed_service_running_state("running"), "running"); - assert_eq!(managed_service_running_state("starting"), "starting"); - assert_eq!(managed_service_running_state("recovering"), "starting"); - assert_eq!(managed_service_running_state("failed"), "not_running"); - assert_eq!(managed_service_running_state("stopped"), "not_running"); - assert_eq!(managed_service_running_state("anything-else"), "unknown"); - } - - #[test] - fn services_tool_result_text_includes_running_interpretation() { - let (_root, paths) = test_paths("services-tool-text"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-vllm", - "vllm", - "Qwen/Qwen3.5", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - std::process::id(), - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = "ready".to_owned(); - - let rendered = render_services_tool_result_text(&[record]); - - assert!(rendered.contains("status_meaning: ready/running = running")); - assert!(rendered.contains("engine=vllm")); - assert!(rendered.contains("running_state=running")); - } - - #[test] - fn local_assistant_prompt_instructions_cover_core_support_questions() { - let prompt = rocm_chat_tool_system_prompt(); - for expected in [ - "is TheRock installed", - "which GPU is on this machine", - "active_runtime_status=ready", - "legacy_rocm_status=not_detected", - "[\"model\"]", - "--build-date", - "always let the user choose the install folder", - "--prefix", - "do not invent a hidden default folder", - "config", - "comfyui", - "First-time setup is the same thing as bootstrap", - "vllm", - "Qwen3-4B-Instruct-2507-GGUF", - "fixed to qwen", - "served by Lemonade", - "port_status", - "[\"services\",\"list\",\"--all\"]", - "qwen-smoke", - "Do not invent shell commands", - "ROCm CLI Assistant Skill", - "Treat `localhost` and `127.0.0.1` as the same loopback endpoint", - ] { - assert!( - prompt.contains(expected), - "system prompt should mention {expected}" - ); - } - } - - /// The reported bug: asked "What can ROCm do on Windows?", the assistant - /// answered that ROCm is Windows-incompatible and suggested CUDA/DirectX — - /// because nothing ever told it which machine it was on. The prompt the CLI - /// actually sends must carry the host. - #[test] - fn assistant_prompt_states_the_host_it_is_answering_for() { - let prompt = rocm_chat_tool_system_prompt_for_host(None); - - // The tool-use rules survive the composition (this is the same prompt, - // grounded — not a replacement for it). - assert!( - prompt.contains("You are ROCm CLI's local assistant"), - "the ROCm tool-use prompt must still be there:\n{prompt}" - ); - - // …and it now names this machine's OS and GPU state. - let expected_os = if cfg!(windows) { - "- Operating system: Windows" - } else { - "- Operating system: Linux" - }; - assert!( - prompt.contains(expected_os), - "the prompt must state this machine's operating system:\n{prompt}" - ); - assert!( - prompt.contains("- AMD GPU: "), - "the prompt must state what GPU was detected (or that none was):\n{prompt}" - ); - assert!( - prompt.contains("Never tell the user ROCm is unavailable on their platform"), - "the prompt must refuse the reported answer:\n{prompt}" - ); - } - - /// The static prompt asserted two Windows-only facts at every host, which is - /// how a WSL user — where vLLM IS the supported path — was told to go use - /// WSL. Platform claims now come from the detected facts instead. - #[test] - fn assistant_prompt_makes_no_unconditional_windows_claims() { - let prompt = rocm_chat_tool_system_prompt(); - assert!( - !prompt.contains("non-technical Windows users"), - "the audience is not assumed to be on Windows:\n{prompt}" - ); - assert!( - !prompt.contains("On native Windows, vLLM is skipped"), - "the vLLM caveat belongs in the host facts, not the static prompt:\n{prompt}" - ); - - // Stated only where it is true: present on Windows, absent elsewhere. - let grounded = rocm_chat_tool_system_prompt_for_host(None); - assert_eq!( - grounded.contains("vLLM and AIM are skipped on native Windows"), - cfg!(windows), - "the vLLM caveat must track the host:\n{grounded}" - ); - } - - /// The prompt tells the model to "use examine … before answering". The dash - /// registers its machine check as `doctor`, so before the alias that - /// sentence named a tool absent from the dash's schema. - #[test] - fn every_tool_the_prompt_names_exists_in_the_dash_schema() { - let prompt = rocm_chat_tool_system_prompt(); - for named in [ - "examine", - "gpu_snapshot", - "port_status", - "natural_language_plan", - ] { - assert!(prompt.contains(named), "prompt should mention {named}"); - assert!( - rocm_dash_tui::agent::ROCM_READ_TOOL_NAMES.contains(&named), - "the prompt names `{named}` but the dash never registers it, so a \ - model that obeys the prompt calls a tool the schema does not offer" - ); - } - } - - #[test] - fn deterministic_rocm_tool_summary_interprets_managed_runtime_as_installed() { - let summary = deterministic_rocm_tool_summary( - "\ -examine: - driver_detail: AMD Radeon RX 9070 XT driver 32.0.23033.1002 - legacy_rocm_status: not_detected -runtime_state: - active_runtime_status: ready - active_runtime_root: D:\\ROCm\\therock_venvs - active_runtime_pip_cache_dir: D:\\ROCm\\therock_venvs\\pip-cache - active_runtime_version: 7.13.0a20260511 (build 2026-05-11) - active_runtime_family: gfx120X-all -", - ) - .expect("examine output should summarize"); - - assert!(summary.contains("GPU: AMD Radeon RX 9070 XT driver 32.0.23033.1002")); - assert!(summary.contains("ROCm/TheRock: installed and active for ROCm CLI")); - assert!(summary.contains("gfx120X-all")); - assert!(summary.contains(r"Install folder: D:\ROCm\therock_venvs")); - assert!(summary.contains(r"Downloads/cache: D:\ROCm\therock_venvs\pip-cache")); - assert!(summary.contains("no global legacy ROCm install was found")); - } - - #[test] - fn fallback_tool_call_routes_where_installed_to_read_only_examine() { - for prompt in [ - "where is rocm installed?", - "where is TheRock installed?", - "what is the ROCm install folder?", - "where did rocm install to?", - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "examine", "{prompt}"); - assert!(chat_tool_call_is_read_only(&call), "{prompt}"); - } - } - - #[test] - fn deterministic_rocm_tool_summary_suppresses_extra_local_model_follow_up() { - let tool_result = ChatToolRunResult { - approval: None, - follow_up_text: "\ -examine: - legacy_rocm_status: not_detected -runtime_state: - active_runtime_status: ready -" - .to_owned(), - ran_read_only_tool: true, - read_only_tool_error: false, - needs_install_folder: false, - }; - let summary = deterministic_rocm_tool_summary(&tool_result.follow_up_text); - - assert!(summary.is_some()); - assert!(!should_request_local_tool_follow_up( - "local", - &tool_result, - summary.as_deref() - )); - - let mut model_list_result = tool_result; - model_list_result.follow_up_text = "rocm_command:\nmodel recipes\n qwen\n".to_owned(); - assert!(should_request_local_tool_follow_up( - "local", - &model_list_result, - None - )); - } - - #[test] - fn deterministic_model_tool_summary_identifies_low_vram_assistant() { - let summary = deterministic_model_tool_summary( - "\ -rocm_command: -model recipes - Qwen3-4B-Instruct-2507-GGUF aliases=[qwen, lemonade-qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - warning: recommended Lemonade GGUF assistant for ROCm machines - Qwen3-0.6B-GGUF aliases=[qwen-smoke, lemonade-tiny] task=chat dtype=gguf device=gpu_required min_gpu_mem=2 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - warning: tiny Lemonade GGUF smoke-test model; not the default assistant - Qwen/Qwen2.5-0.5B-Instruct aliases=[qwen-tiny] task=chat dtype=float16 device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] - engine_support: - vllm: adapter_available path=D:\\rocm\\rocm-engine-vllm.exe runtime_status=unsupported_native_windows reason=native Windows skipped; use WSL/Linux vLLM ROCm - meta-llama/Llama-3.2-3B-Instruct aliases=[llama] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=8 GiB engines=[lemonade, vllm] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - vllm: available path=D:\\rocm\\rocm-engine-vllm.exe -", - ) - .expect("model output should summarize"); - - assert!(summary.contains("Recommended local assistant: qwen")); - assert!(summary.contains("Qwen3-4B-Instruct-2507-GGUF")); - assert!(summary.contains("4 GiB")); - assert!(summary.contains("Tiny smoke test: qwen-smoke")); - assert!(summary.contains("Qwen3-0.6B-GGUF")); - assert!(summary.contains("8 GiB-class option: llama")); - assert!(summary.contains("lemonade, vllm")); - assert!(summary.contains("Qwen/Qwen3.5-4B asks for 12 GiB")); - assert!(summary.contains("Native Windows note")); - assert!(summary.contains("Run `rocm examine`")); - } - - #[test] - fn deterministic_model_tool_summary_suppresses_extra_local_model_follow_up() { - let tool_result = ChatToolRunResult { - approval: None, - follow_up_text: "\ -rocm_command: -model recipes - Qwen3-4B-Instruct-2507-GGUF aliases=[qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe -" - .to_owned(), - ran_read_only_tool: true, - read_only_tool_error: false, - needs_install_folder: false, - }; - let summary = deterministic_chat_tool_summary(&tool_result.follow_up_text); - - assert!(summary.is_some()); - assert!(!should_request_local_tool_follow_up( - "local", - &tool_result, - summary.as_deref() - )); - } - - #[test] - fn default_engine_fallback_matches_aim_only_as_a_word() { - let proposed = |prompt: &str| { - fallback_config_tool_call_for_prompt(prompt).map(|call| { - call.arguments["args"][2] - .as_str() - .expect("engine arg") - .to_owned() - }) - }; - assert_eq!( - proposed("change the default engine to aim").as_deref(), - Some("aim") - ); - assert_eq!( - proposed("change the default engine to vllm, aiming for low latency").as_deref(), - Some("vllm") - ); - assert_eq!( - proposed( - "change the default engine so I can maintain my claim, aiming for low latency" - ), - None - ); - } - - #[test] - fn proposal_action_approve_updates_status() { - let (root, paths) = test_paths("proposal-approve"); - // Seed a pending proposal. - let proposal = rocm_core::AutomationProposalRecord { - at_unix_ms: rocm_core::unix_time_millis(), - proposal_id: "prop-approve-1".to_owned(), - watcher_id: "therock-update".to_owned(), - action: "prepare_driver_plan".to_owned(), - title: "Apply driver plan".to_owned(), - message: "A reviewed driver plan is ready.".to_owned(), - status: "pending".to_owned(), - service_id: None, - tool: None, - arguments: serde_json::Value::Null, - reviewed_at_unix_ms: None, - }; - rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); - - // show is read-only and returns the proposal. - let shown = run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-approve-1", "action": "show" }), - false, - ) - .expect("show ok"); - assert_eq!(shown["structuredContent"]["status"], "pending"); - - // approve requires allow_mutation. - assert!( - run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), - false, - ) - .is_err(), - "approve without allow_mutation must bail" - ); - - // approve with allow_mutation sets status to approved. - let approved = run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), - true, - ) - .expect("approve ok"); - assert_eq!(approved["structuredContent"]["status"], "approved"); - let stored = rocm_core::find_automation_proposal(&paths, "prop-approve-1") - .expect("proposal still present"); - assert_eq!(stored.status, "approved"); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn proposal_action_reject_updates_status() { - let (root, paths) = test_paths("proposal-reject"); - let proposal = rocm_core::AutomationProposalRecord { - at_unix_ms: rocm_core::unix_time_millis(), - proposal_id: "prop-reject-1".to_owned(), - watcher_id: "server-recover".to_owned(), - action: "queue_stop_server_proposal".to_owned(), - title: "Stop overheating server".to_owned(), - message: "GPU thermal pressure detected.".to_owned(), - status: "pending".to_owned(), - service_id: None, - tool: None, - arguments: serde_json::Value::Null, - reviewed_at_unix_ms: None, - }; - rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); - - let rejected = run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-reject-1", "action": "reject" }), - true, - ) - .expect("reject ok"); - assert_eq!(rejected["structuredContent"]["status"], "rejected"); - let stored = rocm_core::find_automation_proposal(&paths, "prop-reject-1") - .expect("proposal still present"); - assert_eq!(stored.status, "rejected"); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn proposal_action_show_missing_proposal_errors() { - let (root, paths) = test_paths("proposal-missing"); - assert!( - run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "nope", "action": "show" }), - false, - ) - .is_err(), - "showing a missing proposal must error" - ); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn assistant_read_only_rocm_commands_do_not_fallback_to_child_process() { - let (_root, paths) = test_paths("readonly-rocm-in-process-only"); - let args = vec!["services".to_owned(), "status".to_owned()]; - - let error = run_rocm_command_for_paths(&paths, &args, Duration::from_secs(1)) - .unwrap_err() - .to_string(); - - assert!(error.contains("read-only assistant command is not implemented in-process")); - assert!(error.contains("rocm services status")); - assert!(!paths.data_dir.join("logs").exists()); - } - - #[test] - fn internal_mcp_read_only_rocm_command_runs_in_process() { - let (_root, paths) = test_paths("mcp-readonly-rocm-in-process"); - - let result = run_internal_mcp_call( - &paths, - "rocm_command", - serde_json::json!({ "args": ["version"] }), - false, - ) - .expect("read-only rocm mcp-call should run"); - - assert_eq!( - result.get("isError").and_then(serde_json::Value::as_bool), - Some(false) - ); - assert_eq!( - result - .pointer("/structuredContent/argv/0") - .and_then(serde_json::Value::as_str), - Some("rocm") - ); - assert!(mcp_tool_result_text(&result).contains(env!("CARGO_PKG_VERSION"))); - } - - #[test] - fn natural_language_plan_returns_structured_mutating_action() { - let (_root, paths) = test_paths("nl-plan-structured-mutating"); - - let result = run_internal_mcp_call( - &paths, - "natural_language_plan", - serde_json::json!({ - "request": "install TheRock into /opt/rocm-target" - }), - false, - ) - .expect("natural_language_plan should plan the request"); - - assert_eq!( - result.get("isError").and_then(serde_json::Value::as_bool), - Some(false) - ); - // text (rendered plan) preserved. - assert!( - !result - .pointer("/structuredContent/text") - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .trim() - .is_empty() - ); - // A known install folder yields a mutating, placeholder-free action. - assert_eq!( - result - .pointer("/structuredContent/action/approval_required") - .and_then(serde_json::Value::as_bool), - Some(true) - ); - assert_eq!( - result - .pointer("/structuredContent/action/has_placeholders") - .and_then(serde_json::Value::as_bool), - Some(false) - ); - let args = result - .pointer("/structuredContent/action/args") - .and_then(serde_json::Value::as_array) - .expect("action args should be present"); - assert!(args.iter().any(|arg| arg.as_str() == Some("install"))); - assert!(args.iter().any(|arg| arg.as_str() == Some("--prefix"))); - } - - #[test] - fn natural_language_plan_returns_placeholder_action_when_incomplete() { - let (_root, paths) = test_paths("nl-plan-structured-placeholder"); - - let result = run_internal_mcp_call( - &paths, - "natural_language_plan", - serde_json::json!({ "request": "serve" }), - false, - ) - .expect("natural_language_plan should plan the request"); - - assert_eq!( - result - .pointer("/structuredContent/action/has_placeholders") - .and_then(serde_json::Value::as_bool), - Some(true) - ); - } - - /// On Windows `rocm install driver` only reports, and chat plans it as a - /// read-only call. That call must run in process: before this arm existed it - /// failed with "unsupported in-process read-only rocm command". It is a dry - /// run, so it never executes anything on any platform. - #[test] - fn install_driver_read_only_call_runs_in_process() { - let (_root, paths) = test_paths("install-driver-in-process"); - let args = vec!["install".to_owned(), "driver".to_owned()]; - let text = run_rocm_read_only_in_process(&paths, &args) - .expect("the read-only install driver call should run in process"); - assert!(text.contains("approval:"), "{text}"); - assert!(!text.contains("driver install completed"), "{text}"); - assert!( - run_rocm_read_only_in_process( - &paths, - &[ - "install".to_owned(), - "driver".to_owned(), - "--yes".to_owned() - ] - ) - .is_err(), - "a flag the read-only path does not know is still refused" - ); - } - - /// `--dry-run` is what the in-process arm already does, so it is accepted - /// and the call reports a dry run. `--reconcile` writes driver install state - /// and stays refused, as does any flag the arm does not know. - #[test] - fn install_driver_read_only_call_accepts_dry_run_and_refuses_reconcile() { - let (_root, paths) = test_paths("install-driver-in-process-dry-run"); - let run = |extra: &[&str]| { - let mut args = vec!["install".to_owned(), "driver".to_owned()]; - args.extend(extra.iter().map(|arg| (*arg).to_owned())); - run_rocm_read_only_in_process(&paths, &args) - }; - for extra in [&["--dry-run"][..], &["--dkms", "--dry-run"], &["--DRY-RUN"]] { - let text = run(extra).unwrap_or_else(|error| panic!("{extra:?} must run: {error}")); - assert!(text.contains("dry_run: true"), "{extra:?}: {text}"); - assert!( - !text.contains("driver install completed"), - "{extra:?}: {text}" - ); - } - for extra in [&["--reconcile"][..], &["--dry-run", "--reconcile"]] { - assert!(run(extra).is_err(), "{extra:?} must stay refused"); - } - // Refusing `--reconcile` changed nothing on disk: the state file is - // written only by a reconcile or an executed install. - assert!( - !crate::driver_install::driver_install_state_path(&paths).exists(), - "a refused call must not create driver install state" - ); - } - - #[test] - fn setup_status_renders_in_process() { - let (_root, paths) = test_paths("setup-status-in-process"); - - for args in [ - vec!["setup".to_owned()], - vec!["setup".to_owned(), "status".to_owned()], - ] { - let text = run_rocm_read_only_in_process(&paths, &args) - .expect("setup status read path should render in-process"); - assert!( - !text.trim().is_empty(), - "setup status text should be non-empty for {args:?}" - ); - assert!( - text.contains("ROCm setup"), - "setup status text should be reached for {args:?}: {text}" - ); - } - } - - #[test] - fn chat_tool_call_refusal_prioritizes_safe_review_wording() -> Result<()> { - let mut output = String::new(); - let (_root, paths) = test_paths("chat-tool-refusal"); - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "tiny.gguf".to_owned(), - content: "I can install ROCm.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: None, - name: "install_sdk".to_owned(), - arguments: serde_json::json!({ - "channel": "release", - "format": "wheel", - "prefix": "D:\\ROCm\\therock_venvs" - }), - }], - }; - - let mut progress = None; - append_chat_tool_results( - &paths, - &response, - &mut output, - Some("I can install ROCm."), - &mut progress, - )?; - - assert!(output.contains("Install ROCm: needs your review")); - assert!(!output.contains("install_sdk: approval required")); - assert!(output.contains("not run: review the approval card before anything runs")); - assert!(output.contains("advanced manual command: rocm install sdk")); - assert!(!output.contains("run the shown ROCm command")); - Ok(()) - } - - #[test] - fn fallback_tool_call_runs_model_support_checks() { - let call = - fallback_rocm_tool_call_for_prompt("Which LLMs can this machine support?").unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["model".to_owned()] - ); - - assert!(fallback_rocm_tool_call_for_prompt("Which LLM are you using?").is_none()); - } - - #[test] - fn local_rocm_tools_chat_uses_fixed_lemonade_qwen_assistant() { - assert_eq!( - local_rocm_tools_assistant_model("local", true), - Some(providers::BUILTIN_ASSISTANT_MODEL_ID) - ); - assert_eq!(local_rocm_tools_assistant_model("local", false), None); - assert_eq!(local_rocm_tools_assistant_model("openai", true), None); - } - - #[test] - fn fallback_tool_call_routes_running_questions_to_status_tools() { - let comfy = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); - assert_eq!(comfy.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&comfy).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&comfy)); - - let comfy_port = - fallback_rocm_tool_call_for_prompt("Is ComfyUI running on port 8188?").unwrap(); - assert_eq!(comfy_port.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&comfy_port).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&comfy_port)); - - for prompt in [ - "Is vLLM running?", - "is lemonade running?", - "is the model server running?", - "is qwen running?", - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command", "{prompt}"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["services".to_owned(), "list".to_owned(), "--all".to_owned(),], - "{prompt}" - ); - assert!(chat_tool_call_is_read_only(&call), "{prompt}"); - } - - let port = fallback_rocm_tool_call_for_prompt("what is running on port 8188?").unwrap(); - assert_eq!(port.name, "port_status"); - assert_eq!( - port.arguments, - serde_json::json!({ "host": DEFAULT_LOCAL_HOST, "port": 8188 }) - ); - assert!(chat_tool_call_is_read_only(&port)); - } - - #[test] - fn fallback_tool_call_routes_engine_install_state_to_engines_list() { - for prompt in ["is vLLM installed?", "is Lemonade available?"] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command", "{prompt}"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["engines".to_owned(), "list".to_owned()], - "{prompt}" - ); - assert!(chat_tool_call_is_read_only(&call), "{prompt}"); - } - } - - #[test] - fn supplemental_tool_call_adds_missing_specific_status_check() { - let generic_services = providers::ChatToolCall { - id: Some("model-picked-services".to_owned()), - name: "services".to_owned(), - arguments: serde_json::json!({}), - }; - - let call = - supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[generic_services]) - .unwrap(); - - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&call)); - } - - #[test] - fn supplemental_tool_call_does_not_duplicate_equivalent_status_check() { - let comfy_status = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); - - assert!( - supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[comfy_status]) - .is_none() - ); - } - - #[test] - fn supplemental_tool_call_adds_running_state_for_engine_install_question() { - let engine_inventory = - fallback_rocm_tool_call_for_prompt("Is vLLM installed and is it running?").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&engine_inventory).unwrap(), - vec!["engines".to_owned(), "list".to_owned()] - ); - - let services = supplemental_read_only_tool_call_for_prompt( - "Is vLLM installed and is it running?", - &[engine_inventory], - ) - .unwrap(); - - assert_eq!(services.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&services).unwrap(), - vec!["services".to_owned(), "list".to_owned(), "--all".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&services)); - } - - #[test] - fn supplemental_tool_call_treats_loopback_port_checks_as_equivalent() { - let model_port_check = providers::ChatToolCall { - id: Some("model-picked-port".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "localhost", "port": 8188 }), - }; - - assert!( - supplemental_read_only_tool_call_for_prompt( - "what is running on port 8188?", - &[model_port_check] - ) - .is_none() - ); - } - - #[test] - fn fallback_tool_call_runs_status_checks() { - for prompt in [ - "Which GPU is on my machine, and is ROCm installed?", - "Is TheRock setup on my machine?", - "Check this ROCm setup.", - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "examine"); - assert_eq!(call.arguments, serde_json::json!({})); - } - } - - #[test] - fn fallback_tool_call_routes_therock_setup_to_install_folder_flow() { - let call = fallback_rocm_tool_call_for_prompt("How do I setup TheRock?").unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&call)); - } - - #[test] - fn fallback_tool_call_preserves_requested_therock_install_prefix() { - for (prompt, expected_prefix) in [ - ( - "install TheRock for me in D:\\ROCm\\therock_venvs", - "D:\\ROCm\\therock_venvs", - ), - ("install ROCm to D:\\ROCm\\temp", "D:\\ROCm\\temp"), - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - expected_prefix.to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&call)); - } - } - - #[test] - fn fallback_tool_call_routes_requested_therock_build_date_install() { - for (prompt, expected_prefix) in [ - ( - "Install this specific TheRock wheel from date 06052026 into D:\\ROCm\\therock_venvs", - "D:\\ROCm\\therock_venvs", - ), - ( - "install ROCm at D:\\ROCm\\temp with build date 2026-06-05", - "D:\\ROCm\\temp", - ), - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - expected_prefix.to_owned(), - "--build-date".to_owned(), - "2026-06-05".to_owned(), - ], - "{prompt}" - ); - assert!(!chat_tool_call_is_read_only(&call)); - } - } - - #[test] - fn fallback_tool_call_does_not_install_therock_without_folder() { - let call = fallback_rocm_tool_call_for_prompt( - "Install this specific TheRock wheel from date 06052026", - ) - .unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--build-date".to_owned(), - "2026-06-05".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&call)); - } - - #[test] - fn chat_install_intent_without_folder_uses_folder_picker_not_rocm_check() -> Result<()> { - let (_root, paths) = test_paths("chat-install-intent-folder-picker"); - for prompt in [ - "i need to install rocm", - "How do I setup TheRock?", - "install therock", - "How do I get it to install therock?", - "rocm please", - "make my AMD GPU ready", - "get TheRock installed for local AI", - "set up my AMD GPU for local AI", - "install this specific TheRock wheel from date 06052026", - ] { - let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; - let approval = result - .approval - .as_ref() - .expect("install should need folder"); - assert_eq!(approval.pending_title, "Install ROCm", "{prompt}"); - assert_eq!( - approval.args[..6], - [ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - ], - "{prompt}" - ); - assert!( - !approval.args.iter().any(|arg| arg == "--prefix"), - "{prompt}" - ); - assert!( - result.rendered.contains("First choose the folder"), - "{prompt}: {}", - result.rendered - ); - assert!( - !result.rendered.contains("I checked ROCm"), - "{prompt}: {}", - result.rendered - ); - assert!( - !result.rendered.contains("ROCm CLI summary"), - "{prompt}: {}", - result.rendered - ); - if prompt.contains("06052026") { - assert!(approval.args.contains(&"--build-date".to_owned())); - assert!(approval.args.contains(&"2026-06-05".to_owned())); - } - } - Ok(()) - } - - #[test] - fn chat_install_intent_ignores_old_conversation_words() -> Result<()> { - let (_root, paths) = test_paths("chat-install-intent-latest-message"); - let prompt = "\ -Conversation so far: -Assistant: Use /examine to refresh actual GPU memory fit before starting anything large. -Assistant: Native Windows note: models may use WSL/Linux through Windows. - -New message: -install therock"; - - let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; - let approval = result - .approval - .as_ref() - .expect("direct latest install request should need a folder"); - - assert_eq!(approval.pending_title, "Install ROCm"); - assert!(!approval.args.iter().any(|arg| arg == "--prefix")); - assert!(result.rendered.contains("I can install ROCm/TheRock")); - assert!(result.rendered.contains("First choose the folder")); - assert!(!result.rendered.contains("I checked ROCm")); - assert!(!result.rendered.contains("ROCm CLI summary")); - Ok(()) - } - - #[test] - fn chat_how_to_setup_question_opens_install_folder_flow() { - assert!( - install_sdk_without_prefix_chat_approval("How do I setup TheRock?").is_some(), - "a setup question should ask for the install folder" - ); - assert!( - install_sdk_without_prefix_chat_approval("install therock").is_some(), - "a direct install command should open the folder picker" - ); - } - - #[test] - fn chat_install_intent_preserves_bare_folder_path() { - let approval = - install_sdk_chat_approval_for_prompt("install therock D:\\ROCm\\therock_venvs") - .expect("direct install prompt should be recognized"); - - assert_eq!( - approval.args, - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - ] - ); - } - - #[test] - fn fallback_tool_call_routes_requested_therock_exact_version_install() { - let call = fallback_rocm_tool_call_for_prompt( - "Install the TheRock ROCm wheel version 7.13.0a20260605 into D:\\ROCm\\therock_venvs", - ) - .unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - "--version".to_owned(), - "7.13.0a20260605".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&call)); - } - - #[test] - fn port_status_matches_loopback_managed_services() -> Result<()> { - let (root, paths) = test_paths("port-status-loopback"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-comfyui", - "comfyui", - "ComfyUI", - "ComfyUI", - "127.0.0.1", - 18188, - "managed", - std::process::id(), - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = "ready".to_owned(); - record.write()?; - - let call = providers::ChatToolCall { - id: Some("port-check".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "localhost", "port": 18188 }), - }; - let result = run_chat_port_status_tool(&paths, &call)?; - let text = mcp_tool_result_text(&result); - let managed_service_count = result - .get("structuredContent") - .and_then(|content| content.get("managed_services")) - .and_then(serde_json::Value::as_array) - .map_or(0, Vec::len); - let _ = fs::remove_dir_all(root); - - assert_eq!(managed_service_count, 1); - assert!(text.contains("managed_services:"), "{text}"); - assert!(text.contains("service_id=svc-comfyui"), "{text}"); - assert!(text.contains("running_state=starting"), "{text}"); - Ok(()) - } - - #[test] - fn fallback_tool_call_routes_simple_config_changes() { - let show = fallback_rocm_tool_call_for_prompt("Show current ROCm CLI config").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&show).unwrap(), - vec!["config".to_owned(), "show".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&show)); - - let engine = fallback_rocm_tool_call_for_prompt("Set the default engine to vllm").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&engine).unwrap(), - vec![ - "config".to_owned(), - "set-default-engine".to_owned(), - "vllm".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&engine)); - - let telemetry = - fallback_rocm_tool_call_for_prompt("Disable telemetry in settings").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&telemetry).unwrap(), - vec![ - "config".to_owned(), - "set-telemetry".to_owned(), - "off".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&telemetry)); - } - - #[test] - fn fallback_tool_call_routes_comfyui_support_and_actions() { - let status = fallback_rocm_tool_call_for_prompt("How do I setup ComfyUI?").unwrap(); - assert_eq!(status.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&status).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&status)); - - let install = fallback_rocm_tool_call_for_prompt("Can you setup ComfyUI for me?").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&install).unwrap(), - vec!["comfyui".to_owned(), "install".to_owned()] - ); - assert!(!chat_tool_call_is_read_only(&install)); - let approval = - chat_tool_approval_request(&install, Some("Install ComfyUI after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Install ComfyUI"); - - let start = fallback_rocm_tool_call_for_prompt("Can you start ComfyUI?").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&start).unwrap(), - vec!["comfyui".to_owned(), "start".to_owned()] - ); - assert!(!chat_tool_call_is_read_only(&start)); - let approval = - chat_tool_approval_request(&start, Some("Start ComfyUI after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Start ComfyUI"); - } - - #[test] - fn fallback_tool_call_routes_local_llm_serve_requests() { - let call = - fallback_rocm_tool_call_for_prompt("Can you setup and serve an LLM for me?").unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "serve".to_owned(), - "qwen".to_owned(), - "--engine".to_owned(), - "lemonade".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&call)); - let approval = - chat_tool_approval_request(&call, Some("Start qwen after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Start local model server"); - assert_eq!( - rocm_chat_tool_requested_command(&call).as_deref(), - Some("rocm serve qwen --engine lemonade --device gpu_required --managed") - ); - } - - #[test] - fn local_chat_tool_call_content_is_treated_as_intermediate() { - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "Qwen/Qwen3-0.6B".to_owned(), - content: "The active runtime root is /opt/rocm.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: Some("call-1".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }], - }; - - assert!(local_tool_call_content_is_intermediate( - "local", true, &response - )); - assert!(!local_tool_call_content_is_intermediate( - "openai", true, &response - )); - assert!(!local_tool_call_content_is_intermediate( - "local", false, &response - )); - - let without_tools = providers::ChatResponse { - tool_calls: Vec::new(), - ..response - }; - assert!(!local_tool_call_content_is_intermediate( - "local", - true, - &without_tools - )); - } - - #[test] - fn local_chat_follow_up_with_tool_call_is_not_final_answer() { - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "Qwen/Qwen3-0.6B".to_owned(), - content: "The runtime root is /opt/rocml.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: Some("call-2".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }], - }; - - assert!(!local_follow_up_content_is_final(&response)); - - let final_answer = providers::ChatResponse { - tool_calls: Vec::new(), - content: "The runtime root is D:\\ROCm\\therock_venvs.".to_owned(), - ..response - }; - assert!(local_follow_up_content_is_final(&final_answer)); - } - - #[test] - fn visible_chat_content_removes_reasoning_blocks() { - assert_eq!( - visible_chat_content( - "\nchecking the tool output\n\nThe runtime root is D:\\ROCm\\temp." - ), - "The runtime root is D:\\ROCm\\temp." - ); - assert_eq!( - visible_chat_content("Before\nhidden\nAfter"), - "Before\n\nAfter" - ); - assert_eq!(visible_chat_content("unfinished"), ""); - } - - #[test] - fn chat_tool_result_errors_use_plain_failure_wording() { - assert_eq!(chat_read_only_tool_status_label(false), "done"); - assert_eq!(chat_read_only_tool_status_label(true), "reported an error"); - assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); - assert_eq!( - chat_tool_display_label("gpu_snapshot"), - "Checked GPU status" - ); - assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); - assert!(mcp_tool_result_is_error(&serde_json::json!({ - "isError": true - }))); - assert!(!mcp_tool_result_is_error(&serde_json::json!({ - "isError": false - }))); - assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); - } - - #[test] - fn local_chat_without_service_explains_serve_before_chat_without_llm_setup() { - let (_root, paths) = test_paths("local-chat-no-service-guidance"); - let result = - render_chat_prompt_result(&paths, "local", None, "Check this ROCm setup", true) - .expect("missing local assistant should render guidance"); - assert!(result.approval.is_none()); - let rendered = result.rendered; - - assert!(rendered.contains("No local assistant is running yet.")); - assert!(rendered.contains("First-time ROCm setup does not need an LLM")); - assert!(rendered.contains("Recommended path:")); - assert!(rendered.contains("Advanced manual command")); - assert!(rendered.contains( - "rocm serve Qwen3-4B-Instruct-2507-GGUF --engine lemonade --device gpu_required --managed" - )); - assert!(!rendered.contains("sshleifer/tiny-gpt2")); - assert!(rendered.contains("rocm chat --tools --provider local --prompt")); - assert!(rendered.contains("Nothing was changed.")); - assert!(!rendered.contains("install sdk")); - assert!(!rendered.contains("setup TheRock with an LLM")); - } - - #[test] - fn local_chat_status_prompts_use_read_only_tools_without_assistant() -> Result<()> { - let (_root, paths) = test_paths("local-chat-status-fallback"); - - let running = - render_chat_prompt_result(&paths, "local", None, "Is vLLM running?", true)?.rendered; - assert!(!running.contains("No local assistant is running yet.")); - assert!(running.contains("Checked model servers: done"), "{running}"); - assert!(running.contains("ROCm CLI summary"), "{running}"); - assert!( - running.contains("Local model servers: none running under ROCm CLI."), - "{running}" - ); - assert!(running.contains("Nothing was changed.")); - - let installed = - render_chat_prompt_result(&paths, "local", None, "Is vLLM installed?", true)?.rendered; - assert!(!installed.contains("No local assistant is running yet.")); - assert!(installed.contains("Engine runtimes:"), "{installed}"); - assert!(installed.contains("vLLM:"), "{installed}"); - - let installed_and_running = render_chat_prompt_result( - &paths, - "local", - None, - "Is vLLM installed and is it running?", - true, - )? - .rendered; - assert!( - installed_and_running.contains("Checked local engines: done"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Checked model servers: done"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Engine runtimes:"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Local model servers: none running under ROCm CLI."), - "{installed_and_running}" - ); - - let port = render_chat_prompt_result( - &paths, - "local", - None, - "What is running on port 8188?", - true, - )? - .rendered; - assert!(!port.contains("No local assistant is running yet.")); - assert!(port.contains("Checked local port: done"), "{port}"); - assert!(port.contains("Port 8188:"), "{port}"); - Ok(()) - } - - #[test] - fn chat_tools_anthropic_reaches_provider_opt_in_boundary() { - let (_root, paths) = test_paths("anthropic-chat-tools-opt-in"); - - let error = render_chat_prompt_result( - &paths, - "anthropic", - Some("claude-test"), - "Check this ROCm setup", - true, - ) - .unwrap_err() - .to_string(); - - assert!(error.contains("cloud provider `anthropic` is disabled")); - assert!(error.contains("rocm config enable-provider anthropic")); - assert!(!error.contains("OpenAI-compatible provider")); - } +mod tests { + use super::*; #[test] fn piped_prompt_keeps_everything_but_the_trailing_line_ending() { @@ -4734,11 +298,13 @@ install therock"; /// mostly metacharacters. #[cfg(test)] mod command_preview_properties { + use super::mcp_bridge::{mcp_tool_result_text, run_chat_path_exists_tool}; use super::*; use crate::freeform_planner::{ - ProviderPlannerToolCall, planner_failure_note, provider_planner_response_to_plan, - validate_provider_planner_tool_call, + ProviderPlannerToolCall, format_structured_tool_call, planner_failure_note, + provider_planner_response_to_plan, validate_provider_planner_tool_call, }; + use crate::tool_policy::rocm_chat_tool_requested_command; // The one generator, shared with `shell_quote`'s own suite so both are // covered by its reach guard. See `crate::shell_quote::hostile`. use crate::shell_quote::has_terminal_control_character; diff --git a/apps/rocm/src/chat/intent.rs b/apps/rocm/src/chat/intent.rs new file mode 100644 index 0000000000..e665f666d4 --- /dev/null +++ b/apps/rocm/src/chat/intent.rs @@ -0,0 +1,949 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The chat assistant's prompt-intent heuristics: what a user's latest message +//! asks for, decided from its words alone. `fallback_rocm_tool_call_for_prompt` +//! maps a prompt to the read-only or approval-gated tool call the assistant +//! proposes when it does not need the local model to choose one (with +//! `fallback_config_tool_call_for_prompt`, `fallback_engine_list_tool_call` and +//! `fallback_services_list_tool_call` behind it), +//! `prompt_can_use_read_only_without_local_assistant` says whether a status +//! question can be answered before any model server is up, and `install_sdk_chat_approval_for_prompt` (with +//! `install_sdk_without_prefix_chat_approval`) turns an install or setup +//! request into the `rocm install sdk` approval card. The `prompt_asks_*`, +//! `prompt_mentions_*` and `prompt_requests_*` predicates hold the phrase +//! lists, and `latest_user_chat_message` cuts the newest message out of the +//! conversation text the chat builds around it, so older words in the +//! conversation never steer these heuristics. +//! +//! A leaf of the `chat` module: nothing here calls back into `chat.rs`. The +//! edges out are `prompt_text.rs` (the phrase and install-detail extraction), +//! `rocm_argv.rs`, `tool_policy.rs` (`ChatToolApprovalRequest`), +//! `freeform_planner.rs` (`format_structured_tool_call`) and `providers` (the +//! tool-call type). The turn loop in `chat/turn.rs` reads this module's items +//! through `super::intent`. + +use crate::freeform_planner::format_structured_tool_call; +use crate::prompt_text::{ + any_substring, infer_engine_from_request, requested_install_prefix_from_prompt, + requested_therock_build_date_from_prompt, requested_therock_version_from_prompt, +}; +use crate::providers; +use crate::rocm_argv::chat_cli_arg_value; +use crate::tool_policy::ChatToolApprovalRequest; +use rocm_core::DEFAULT_LOCAL_HOST; + +pub(super) fn latest_user_chat_message(prompt: &str) -> &str { + const MARKER: &str = "\nNew message:\n"; + prompt + .rfind(MARKER) + .map_or(prompt, |index| &prompt[index + MARKER.len()..]) + .trim() +} + +pub(super) fn prompt_can_use_read_only_without_local_assistant(prompt: &str) -> bool { + let normalized = latest_user_chat_message(prompt).to_ascii_lowercase(); + let mentions_status_subject = any_substring( + &normalized, + &[ + "comfyui", + "comfy ui", + "comfy", + "vllm", + "lemonade", + "llama.cpp", + "llama cpp", + "qwen", + "model server", + "local server", + "local model server", + "assistant server", + "port", + "8188", + "therock", + "rocm", + ], + ); + mentions_status_subject + && (prompt_asks_running_or_status(&normalized) + || any_substring( + &normalized, + &["installed", "available", "detected", "engine status"], + )) +} + +pub(super) fn prompt_mentions_serving_engine_or_service(normalized_prompt: &str) -> bool { + any_substring( + normalized_prompt, + &[ + "vllm", + "lemonade", + "llama.cpp", + "llama cpp", + "qwen", + "model server", + "local server", + "local model server", + "assistant server", + ], + ) +} + +pub(super) fn prompt_asks_engine_install_state(normalized_prompt: &str) -> bool { + any_substring( + normalized_prompt, + &["installed", "available", "detected", "engine status"], + ) +} + +pub(super) fn fallback_engine_list_tool_call() -> providers::ChatToolCall { + providers::ChatToolCall { + id: Some("fallback-engine-list".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["engines", "list"] }), + } +} + +pub(super) fn fallback_services_list_tool_call() -> providers::ChatToolCall { + providers::ChatToolCall { + id: Some("fallback-services-list".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["services", "list", "--all"] }), + } +} + +pub(super) fn fallback_rocm_tool_call_for_prompt(prompt: &str) -> Option { + let prompt = latest_user_chat_message(prompt); + let normalized = prompt.to_ascii_lowercase(); + let asks_running_or_status = prompt_asks_running_or_status(&normalized); + let mentions_comfyui = any_substring(&normalized, &["comfyui", "comfy ui", "comfy"]); + if asks_running_or_status + && !mentions_comfyui + && any_substring(&normalized, &["8188", "port 8188"]) + { + return Some(providers::ChatToolCall { + id: Some("fallback-port-8188-status".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": DEFAULT_LOCAL_HOST, "port": 8188 }), + }); + } + let mentions_serving_engine_or_service = prompt_mentions_serving_engine_or_service(&normalized); + if mentions_serving_engine_or_service && prompt_asks_engine_install_state(&normalized) { + return Some(fallback_engine_list_tool_call()); + } + if mentions_serving_engine_or_service && asks_running_or_status { + return Some(fallback_services_list_tool_call()); + } + let mentions_llm_or_model = any_substring( + &normalized, + &["llm", "llms", "model", "models", "assistant"], + ); + let asks_support_or_fit = any_substring( + &normalized, + &[ + "support", + "supported", + "run", + "runs", + "fit", + "fits", + "can my machine", + "can this machine", + ], + ); + if mentions_llm_or_model && asks_support_or_fit { + return Some(providers::ChatToolCall { + id: Some("fallback-rocm-model".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["model"] }), + }); + } + + if mentions_comfyui { + if any_substring(&normalized, &["log", "logs"]) { + return Some(providers::ChatToolCall { + id: Some("fallback-comfyui-logs".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["comfyui", "logs"] }), + }); + } + if asks_running_or_status { + return Some(providers::ChatToolCall { + id: Some("fallback-comfyui-status".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["comfyui", "status"] }), + }); + } + if any_substring(&normalized, &["start", "run", "launch", "open"]) { + return Some(providers::ChatToolCall { + id: Some("fallback-comfyui-start".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["comfyui", "start"], + "reason": "Start ComfyUI locally after the user approves it." + }), + }); + } + if any_substring( + &normalized, + &[ + "can you setup", + "can you set up", + "please setup", + "please set up", + "setup comfyui for me", + "set up comfyui for me", + "install comfyui", + "download comfyui", + ], + ) { + return Some(providers::ChatToolCall { + id: Some("fallback-comfyui-install".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["comfyui", "install"], + "reason": "Install ComfyUI into ROCm CLI's managed app folder after the user approves it." + }), + }); + } + return Some(providers::ChatToolCall { + id: Some("fallback-comfyui-status".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["comfyui", "status"] }), + }); + } + + if let Some(call) = fallback_config_tool_call_for_prompt(&normalized) { + return Some(call); + } + + if mentions_llm_or_model + && any_substring( + &normalized, + &[ + "serve", + "server", + "start", + "setup and serve", + "set up and serve", + "run locally", + "local model", + "local assistant", + ], + ) + { + return Some(providers::ChatToolCall { + id: Some("fallback-serve-qwen".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "qwen", "--engine", "lemonade", "--device", "gpu_required", "--managed"], + "reason": "Start the recommended local assistant after the user approves it." + }), + }); + } + + let mentions_setup = any_substring(&normalized, &["setup", "set up", "install"]); + let asks_how = any_substring(&normalized, &["how", "help", "what do i need"]); + let mentions_rocm_or_therock = any_substring(&normalized, &["therock", "rocm"]); + let requested_install_prefix = requested_install_prefix_from_prompt(prompt); + if requested_install_prefix.is_none() + && let Some(approval) = install_sdk_without_prefix_chat_approval(prompt) + { + return Some(providers::ChatToolCall { + id: Some("fallback-therock-install-folder".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": approval.args, + "reason": "Ask the user to choose the ROCm/TheRock install folder before installing." + }), + }); + } + if mentions_rocm_or_therock + && prompt_requests_install_action(&normalized) + && let Some(prefix) = requested_install_prefix + { + let mut args = vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + prefix, + ]; + if let Some(build_date) = requested_therock_build_date_from_prompt(&normalized) { + args.push("--build-date".to_owned()); + args.push(build_date); + } else if let Some(version) = requested_therock_version_from_prompt(&normalized) { + args.push("--version".to_owned()); + args.push(version); + } + return Some(providers::ChatToolCall { + id: Some("fallback-therock-install".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": args, + "reason": "Install TheRock ROCm into the user-selected folder after the user approves it." + }), + }); + } + if asks_how && mentions_setup && any_substring(&normalized, &["therock", "rocm"]) { + return Some(providers::ChatToolCall { + id: Some("fallback-therock-examine".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }); + } + + let asks_where_installed = any_substring( + &normalized, + &[ + "where is rocm", + "where's rocm", + "where is therock", + "where's therock", + "where did rocm", + "where did therock", + "where rocm is installed", + "where therock is installed", + "rocm install folder", + "therock install folder", + "rocm installed at", + "therock installed at", + ], + ); + let asks_status = asks_where_installed + || any_substring( + &normalized, + &[ + "is rocm installed", + "is therock installed", + "is therock setup", + "is therock set up", + "rocm installed", + "therock installed", + "check this rocm setup", + "which gpu", + "what gpu", + "gpu is on", + "gpu do i have", + "my machine", + ], + ); + if asks_status && any_substring(&normalized, &["gpu", "rocm", "therock", "setup"]) { + return Some(providers::ChatToolCall { + id: Some("fallback-examine".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }); + } + + None +} + +pub(super) fn prompt_asks_running_or_status(normalized: &str) -> bool { + any_substring( + normalized, + &[ + "running", + "is it up", + "is this up", + "is there", + "are there", + "status", + "started", + "listening", + "on port", + "port ", + ], + ) +} + +fn fallback_config_tool_call_for_prompt(normalized: &str) -> Option { + if !any_substring( + normalized, + &[ + "config", + "setting", + "settings", + "default engine", + "telemetry", + ], + ) { + return None; + } + if any_substring(normalized, &["show", "check", "what", "current", "list"]) { + return Some(providers::ChatToolCall { + id: Some("fallback-config-show".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["config", "show"] }), + }); + } + if any_substring(normalized, &["default engine", "set engine", "use engine"]) { + // Word-boundary inference: a bare `contains("aim")` matched "aiming". + if let Some(engine) = infer_engine_from_request(normalized) { + return Some(providers::ChatToolCall { + id: Some("fallback-config-default-engine".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["config", "set-default-engine", engine], + "reason": "Change ROCm CLI's default engine after the user approves it." + }), + }); + } + } + if normalized.contains("telemetry") { + if any_substring(normalized, &["off", "disable", "disabled"]) { + return Some(providers::ChatToolCall { + id: Some("fallback-config-telemetry-off".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["config", "set-telemetry", "off"], + "reason": "Turn ROCm CLI telemetry off after the user approves it." + }), + }); + } + if any_substring(normalized, &["local", "on", "enable", "enabled"]) { + return Some(providers::ChatToolCall { + id: Some("fallback-config-telemetry-local".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["config", "set-telemetry", "local"], + "reason": "Enable local-only ROCm inspection after the user approves it." + }), + }); + } + } + None +} + +pub(super) fn install_sdk_without_prefix_chat_approval( + prompt: &str, +) -> Option { + install_sdk_chat_approval_for_prompt(prompt).and_then(|approval| { + chat_cli_arg_value(&approval.args, "--prefix") + .is_none() + .then_some(approval) + }) +} + +pub(super) fn install_sdk_chat_approval_for_prompt( + prompt: &str, +) -> Option { + let prompt = latest_user_chat_message(prompt); + let normalized = prompt.to_ascii_lowercase(); + if !prompt_requests_rocm_install_or_setup(&normalized) { + return None; + } + let mut args = vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + if normalized.contains("nightly") { + "nightly".to_owned() + } else { + "release".to_owned() + }, + "--format".to_owned(), + "wheel".to_owned(), + ]; + if let Some(prefix) = requested_install_prefix_from_prompt(prompt) { + args.push("--prefix".to_owned()); + args.push(prefix); + } + if let Some(build_date) = requested_therock_build_date_from_prompt(&normalized) { + args.push("--build-date".to_owned()); + args.push(build_date); + } else if let Some(version) = requested_therock_version_from_prompt(&normalized) { + args.push("--version".to_owned()); + args.push(version); + } + Some(ChatToolApprovalRequest { + pending_title: "Install ROCm".to_owned(), + command_title: "Install".to_owned(), + display_command: Some(format_structured_tool_call("rocm", &args)), + args, + explanation: Some( + "Install ROCm/TheRock after the user chooses the install folder.".to_owned(), + ), + }) +} + +fn prompt_requests_rocm_install_or_setup(normalized: &str) -> bool { + let mentions_rocm_stack = any_substring( + normalized, + &[ + "rocm", + "therock", + "the rock", + "amd gpu package", + "amd gpu ready", + "local ai", + ], + ); + if !mentions_rocm_stack { + return false; + } + prompt_requests_install_action(normalized) + || any_substring( + normalized, + &[ + "need rocm", + "need therock", + "need the rock", + "need to get rocm", + "need to get therock", + "get rocm installed", + "get therock installed", + "get the rock installed", + "make rocm work", + "make therock work", + "make the rock work", + "prepare rocm", + "prepare therock", + "prepare the rock", + "set up my amd gpu", + "setup my amd gpu", + "set up local ai", + "setup local ai", + "get local ai working", + "make local ai work", + "rocm please", + "make my amd gpu ready", + "make my gpu ready", + "get my amd gpu ready", + "get my gpu ready", + ], + ) +} + +fn prompt_requests_install_action(normalized: &str) -> bool { + any_substring( + normalized, + &[ + "install this", + "install specific", + "install the", + "install rocm", + "install therock", + "install the rock", + "install amd gpu package", + "install local ai", + "can you install", + "please install", + "rocm please", + "therock please", + "the rock please", + "need to install", + "want to install", + "i need rocm", + "i need therock", + "i need the rock", + "get me rocm", + "get me therock", + "get my gpu ready", + "make my gpu ready", + "make my amd gpu ready", + "get installed", + "setup for me", + "set up for me", + "setup rocm", + "set up rocm", + "setup therock", + "set up therock", + "setup the rock", + "set up the rock", + "setup my gpu", + "set up my gpu", + "setup amd gpu", + "set up amd gpu", + ], + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tool_policy::{ + chat_tool_approval_request, chat_tool_call_is_read_only, normalized_chat_rocm_command_args, + rocm_chat_tool_requested_command, + }; + + #[test] + fn fallback_tool_call_routes_where_installed_to_read_only_examine() { + for prompt in [ + "where is rocm installed?", + "where is TheRock installed?", + "what is the ROCm install folder?", + "where did rocm install to?", + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "examine", "{prompt}"); + assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + } + } + + #[test] + fn default_engine_fallback_matches_aim_only_as_a_word() { + let proposed = |prompt: &str| { + fallback_config_tool_call_for_prompt(prompt).map(|call| { + call.arguments["args"][2] + .as_str() + .expect("engine arg") + .to_owned() + }) + }; + assert_eq!( + proposed("change the default engine to aim").as_deref(), + Some("aim") + ); + assert_eq!( + proposed("change the default engine to vllm, aiming for low latency").as_deref(), + Some("vllm") + ); + assert_eq!( + proposed( + "change the default engine so I can maintain my claim, aiming for low latency" + ), + None + ); + } + + #[test] + fn fallback_tool_call_runs_model_support_checks() { + let call = + fallback_rocm_tool_call_for_prompt("Which LLMs can this machine support?").unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["model".to_owned()] + ); + + assert!(fallback_rocm_tool_call_for_prompt("Which LLM are you using?").is_none()); + } + + #[test] + fn fallback_tool_call_routes_running_questions_to_status_tools() { + let comfy = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); + assert_eq!(comfy.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&comfy).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&comfy)); + + let comfy_port = + fallback_rocm_tool_call_for_prompt("Is ComfyUI running on port 8188?").unwrap(); + assert_eq!(comfy_port.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&comfy_port).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&comfy_port)); + + for prompt in [ + "Is vLLM running?", + "is lemonade running?", + "is the model server running?", + "is qwen running?", + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command", "{prompt}"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["services".to_owned(), "list".to_owned(), "--all".to_owned(),], + "{prompt}" + ); + assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + } + + let port = fallback_rocm_tool_call_for_prompt("what is running on port 8188?").unwrap(); + assert_eq!(port.name, "port_status"); + assert_eq!( + port.arguments, + serde_json::json!({ "host": DEFAULT_LOCAL_HOST, "port": 8188 }) + ); + assert!(chat_tool_call_is_read_only(&port)); + } + + #[test] + fn fallback_tool_call_routes_engine_install_state_to_engines_list() { + for prompt in ["is vLLM installed?", "is Lemonade available?"] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command", "{prompt}"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["engines".to_owned(), "list".to_owned()], + "{prompt}" + ); + assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + } + } + + #[test] + fn fallback_tool_call_runs_status_checks() { + for prompt in [ + "Which GPU is on my machine, and is ROCm installed?", + "Is TheRock setup on my machine?", + "Check this ROCm setup.", + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "examine"); + assert_eq!(call.arguments, serde_json::json!({})); + } + } + + #[test] + fn fallback_tool_call_routes_therock_setup_to_install_folder_flow() { + let call = fallback_rocm_tool_call_for_prompt("How do I setup TheRock?").unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + + #[test] + fn fallback_tool_call_preserves_requested_therock_install_prefix() { + for (prompt, expected_prefix) in [ + ( + "install TheRock for me in D:\\ROCm\\therock_venvs", + "D:\\ROCm\\therock_venvs", + ), + ("install ROCm to D:\\ROCm\\temp", "D:\\ROCm\\temp"), + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + expected_prefix.to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + } + + #[test] + fn fallback_tool_call_routes_requested_therock_build_date_install() { + for (prompt, expected_prefix) in [ + ( + "Install this specific TheRock wheel from date 06052026 into D:\\ROCm\\therock_venvs", + "D:\\ROCm\\therock_venvs", + ), + ( + "install ROCm at D:\\ROCm\\temp with build date 2026-06-05", + "D:\\ROCm\\temp", + ), + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + expected_prefix.to_owned(), + "--build-date".to_owned(), + "2026-06-05".to_owned(), + ], + "{prompt}" + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + } + + #[test] + fn fallback_tool_call_does_not_install_therock_without_folder() { + let call = fallback_rocm_tool_call_for_prompt( + "Install this specific TheRock wheel from date 06052026", + ) + .unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--build-date".to_owned(), + "2026-06-05".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + + #[test] + fn chat_how_to_setup_question_opens_install_folder_flow() { + assert!( + install_sdk_without_prefix_chat_approval("How do I setup TheRock?").is_some(), + "a setup question should ask for the install folder" + ); + assert!( + install_sdk_without_prefix_chat_approval("install therock").is_some(), + "a direct install command should open the folder picker" + ); + } + + #[test] + fn chat_install_intent_preserves_bare_folder_path() { + let approval = + install_sdk_chat_approval_for_prompt("install therock D:\\ROCm\\therock_venvs") + .expect("direct install prompt should be recognized"); + + assert_eq!( + approval.args, + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + ] + ); + } + + #[test] + fn fallback_tool_call_routes_requested_therock_exact_version_install() { + let call = fallback_rocm_tool_call_for_prompt( + "Install the TheRock ROCm wheel version 7.13.0a20260605 into D:\\ROCm\\therock_venvs", + ) + .unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + "--version".to_owned(), + "7.13.0a20260605".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + + #[test] + fn fallback_tool_call_routes_simple_config_changes() { + let show = fallback_rocm_tool_call_for_prompt("Show current ROCm CLI config").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&show).unwrap(), + vec!["config".to_owned(), "show".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&show)); + + let engine = fallback_rocm_tool_call_for_prompt("Set the default engine to vllm").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&engine).unwrap(), + vec![ + "config".to_owned(), + "set-default-engine".to_owned(), + "vllm".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&engine)); + + let telemetry = + fallback_rocm_tool_call_for_prompt("Disable telemetry in settings").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&telemetry).unwrap(), + vec![ + "config".to_owned(), + "set-telemetry".to_owned(), + "off".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&telemetry)); + } + + #[test] + fn fallback_tool_call_routes_comfyui_support_and_actions() { + let status = fallback_rocm_tool_call_for_prompt("How do I setup ComfyUI?").unwrap(); + assert_eq!(status.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&status).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&status)); + + let install = fallback_rocm_tool_call_for_prompt("Can you setup ComfyUI for me?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&install).unwrap(), + vec!["comfyui".to_owned(), "install".to_owned()] + ); + assert!(!chat_tool_call_is_read_only(&install)); + let approval = + chat_tool_approval_request(&install, Some("Install ComfyUI after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Install ComfyUI"); + + let start = fallback_rocm_tool_call_for_prompt("Can you start ComfyUI?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&start).unwrap(), + vec!["comfyui".to_owned(), "start".to_owned()] + ); + assert!(!chat_tool_call_is_read_only(&start)); + let approval = + chat_tool_approval_request(&start, Some("Start ComfyUI after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Start ComfyUI"); + } + + #[test] + fn fallback_tool_call_routes_local_llm_serve_requests() { + let call = + fallback_rocm_tool_call_for_prompt("Can you setup and serve an LLM for me?").unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "serve".to_owned(), + "qwen".to_owned(), + "--engine".to_owned(), + "lemonade".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + let approval = + chat_tool_approval_request(&call, Some("Start qwen after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Start local model server"); + assert_eq!( + rocm_chat_tool_requested_command(&call).as_deref(), + Some("rocm serve qwen --engine lemonade --device gpu_required --managed") + ); + } +} diff --git a/apps/rocm/src/chat/mcp_bridge.rs b/apps/rocm/src/chat/mcp_bridge.rs new file mode 100644 index 0000000000..34f3d6e052 --- /dev/null +++ b/apps/rocm/src/chat/mcp_bridge.rs @@ -0,0 +1,1462 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The chat assistant's MCP bridge and in-process command execution: the +//! entry point `run_internal_mcp_call` (the tool dispatch behind `rocm +//! mcp-call`, the dash execution seam and the assistant's read-only tool +//! path), the in-process runner for the read-only `rocm` commands the assistant +//! may ask for (`run_rocm_read_only_in_process`), the helpers that wrap a +//! command's output as an MCP tool result, and the `path_exists` and +//! `port_status` tools. `render_services_tool_result_text`, the text of the +//! `services` tool, lives here too, as does `mcp_tool_result_text`, which reads +//! the text back out of an MCP tool result for the turn loop. +//! +//! A child of the `chat` module. It calls into `tool_policy.rs` (the validators +//! and the approval split), `rocm_argv.rs`, `chat/summaries.rs` +//! (`managed_service_running_state`) and the crate-root renderers; nothing here +//! calls back into the turn loop in `chat/turn.rs`, tests included; the turn +//! loop reads this module's items through `super::mcp_bridge`. +//! `run_internal_mcp_call` stays reachable as `chat::run_internal_mcp_call` for +//! `main.rs` and `dash_seam.rs`. + +use super::summaries::managed_service_running_state; +use crate::automations::{render_automations_text, render_daemon_text}; +use crate::freeform_planner::{format_structured_tool_call, plan_freeform_request_with_context}; +use crate::managed_service_status::{load_managed_services, managed_service_is_live}; +use crate::rocm_argv::{chat_cli_arg_value, chat_cli_has_flag}; +use crate::runtimes_cmd::render_runtimes_text; +use crate::services_text::{ + read_log_tail, render_logs_browser_text, render_logs_text, render_service_logs_text, + render_services_text, +}; +use crate::tool_policy::{ + ChatRocmCommandAction, chat_install_prefix_is_system, chat_rocm_command_action, + chat_tool_call_is_read_only, json_string, rocm_chat_tool_requested_args, + validate_chat_tool_call, +}; +use crate::uninstall_plan::render_uninstall_dry_run; +use crate::update_cmd::render_update_text; +use crate::{ + build_codex_bridge_gpu_snapshot, build_codex_bridge_snapshot, + builtin_codex_bridge_engine_inventory, cli_version_string, record_cli_audit_event, + render_config_text, render_engine_inventory_text_with_paths, render_examine_text, + render_examine_text_with_paths, render_model_registry_text_with_context_and_host, + render_model_registry_verbose_text_with_context_and_host, render_setup_status_text, + therock_install_version_selector, +}; +use crate::{comfyui, providers, therock}; +use anyhow::{Context, Result, bail}; +use rocm_core::{ + AppPaths, AutomationRuntimeState, CommandCapture, DEFAULT_LOCAL_HOST, ExamineSummary, + ManagedServiceRecord, RocmCliConfig, command_capture_text, load_recent_automation_events, +}; +use rocm_engine_protocol::DEFAULT_LOG_TAIL_LINES; +use std::fmt::Write as _; +use std::net::{TcpStream, ToSocketAddrs}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +fn loopback_tcp_port_is_reachable(host: &str, port: u16) -> bool { + let Ok(addresses) = (host, port).to_socket_addrs() else { + return false; + }; + addresses + .into_iter() + .any(|address| TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok()) +} + +fn parse_optional_query(args: &[String]) -> Result> { + let mut index = 0; + while index < args.len() { + match args[index].as_str() { + "--query" | "-q" => { + let value = args + .get(index + 1) + .context("logs query flag requires a value")?; + return Ok(Some(value.to_owned())); + } + value if value.starts_with("--") => {} + value => return Ok(Some(value.to_owned())), + } + index += 1; + } + Ok(None) +} + +fn parse_optional_lines(args: &[String]) -> Result { + let mut index = 0; + while index < args.len() { + if matches!(args[index].as_str(), "--lines" | "-n") { + let value = args.get(index + 1).context("lines flag requires a value")?; + return value + .parse::() + .context("lines flag must be a positive number"); + } + index += 1; + } + Ok(DEFAULT_LOG_TAIL_LINES) +} + +pub(super) fn loopback_host_key(host: &str) -> String { + match host.trim().to_ascii_lowercase().as_str() { + "localhost" | "127.0.0.1" => "127.0.0.1".to_owned(), + "::1" | "[::1]" => "::1".to_owned(), + other => other.to_owned(), + } +} + +fn render_services_tool_result_text(records: &[ManagedServiceRecord]) -> String { + let mut output = String::new(); + let _ = writeln!(output, "managed_services: {}", records.len()); + let _ = writeln!( + output, + "status_meaning: ready/running = running; starting/recovering = starting; failed/stopped = not running; no matching row = not managed by ROCm CLI" + ); + if records.is_empty() { + let _ = writeln!(output, "services: none"); + return output; + } + let _ = writeln!(output, "services:"); + for record in records { + let _ = writeln!( + output, + " - service_id={} engine={} model={} canonical_model={} status={} running_state={} endpoint={}", + record.service_id, + record.engine, + record.model_ref, + record.canonical_model_id, + record.status, + managed_service_running_state(&record.status), + record.endpoint_url + ); + } + output +} + +// pub(crate): called from main.rs's own MCP dispatch and from the dash +// execution seam (dash_seam.rs) +pub(crate) fn run_internal_mcp_call( + paths: &AppPaths, + name: &str, + arguments: serde_json::Value, + allow_mutation: bool, +) -> Result { + let arguments = internal_mcp_arguments(arguments); + let call = providers::ChatToolCall { + id: None, + name: name.to_owned(), + arguments: serde_json::Value::Object(arguments.clone()), + }; + validate_chat_tool_call(&call)?; + if !chat_tool_call_is_read_only(&call) && !allow_mutation { + bail!( + "MCP tool `{name}` changes local ROCm state; rerun `rocm mcp-call {name}` with --allow-mutation only after explicit user approval" + ); + } + + match name { + "examine" | "doctor" => { + let examine = ExamineSummary::gather()?; + let text = render_examine_text()?; + Ok(internal_mcp_tool_success(text, serde_json::json!(examine))) + } + "bridge_snapshot" => { + let snapshot = build_codex_bridge_snapshot(paths)?; + Ok(internal_mcp_tool_success( + format!( + "Captured bridge snapshot for {} / {} with default engine `{}`.", + snapshot.examine.os, snapshot.examine.arch, snapshot.examine.default_engine + ), + serde_json::json!(snapshot), + )) + } + "gpu_snapshot" => { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + let gpu = build_codex_bridge_gpu_snapshot(&config); + let status = if !config.telemetry.local_inspection_enabled() { + "GPU telemetry is disabled by rocm-cli config." + } else if gpu.amd_smi_available { + "Captured amd-smi GPU snapshot." + } else { + "Use `rocm examine` for the current local AMD GPU summary." + }; + Ok(internal_mcp_tool_success( + status.to_owned(), + serde_json::json!(gpu), + )) + } + "engines" => { + let engines = builtin_codex_bridge_engine_inventory(); + Ok(internal_mcp_tool_success( + format!("Found {} engine entries.", engines.len()), + serde_json::json!({ "engines": engines }), + )) + } + "services" => { + let services = load_managed_services(paths)?; + Ok(internal_mcp_tool_success( + render_services_tool_result_text(&services), + serde_json::json!({ "services": services }), + )) + } + "port_status" => run_chat_port_status_tool(paths, &call), + "service_logs" => { + let service_id = json_string(&arguments, "service_id") + .context("service_logs requires `service_id`")?; + let lines = arguments + .get("lines") + .and_then(serde_json::Value::as_u64) + .unwrap_or(80) + .clamp(1, 500) as usize; + let record = load_managed_services(paths)? + .into_iter() + .find(|service| service.service_id == service_id) + .with_context(|| format!("managed service `{service_id}` not found"))?; + let log_tail = read_log_tail(&record.log_path, lines, "service log")?; + Ok(internal_mcp_tool_success( + rocm_core::service_log_tool_summary(&record.service_id, &log_tail), + serde_json::json!({ + "service": record, + "lines": log_tail.lines.len(), + "lines_requested": lines, + "newest_line_exceeds_read_bound": log_tail.newest_line_exceeds_the_read_bound(), + "tail": log_tail.lines.join("\n"), + }), + )) + } + "automations" => { + let event_limit = arguments + .get("event_limit") + .and_then(serde_json::Value::as_u64) + .unwrap_or(10) + .clamp(1, 64) as usize; + let runtime = AutomationRuntimeState::load(paths)?; + let events = load_recent_automation_events(paths, event_limit)?; + Ok(internal_mcp_tool_success( + format!( + "Loaded automation runtime and {} recent events.", + events.len() + ), + serde_json::json!({ + "runtime": runtime, + "recent_events": events, + }), + )) + } + "natural_language_plan" => { + let request = json_string(&arguments, "request") + .context("natural_language_plan requires `request`")?; + let config = RocmCliConfig::load(paths).unwrap_or_default(); + // Built exactly once: the text shown to the operator and the + // `action` reported as what would run next both come from the + // same `StructuredRequestPlan`, so a provider-assisted plan (not + // guaranteed deterministic across calls) cannot describe one + // command in `text` and a different one in `action`. + let (text, action) = plan_freeform_request_with_context(&request, paths, &config); + let action = action.map(|action| { + serde_json::json!({ + "args": action.args, + "approval_required": action.approval_required, + "has_placeholders": action.has_placeholders, + "provider_assisted": action.provider_assisted, + }) + }); + Ok(internal_mcp_tool_success( + "Planned the ROCm request.".to_owned(), + serde_json::json!({ + "request": request, + "text": text, + "action": action, + }), + )) + } + "rocm_command" => { + let action = chat_rocm_command_action(&call)?; + let output = match action { + ChatRocmCommandAction::ReadOnly(args) => { + run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))? + } + ChatRocmCommandAction::Approval { args, .. } if allow_mutation => { + let refs = args.iter().map(String::as_str).collect::>(); + run_rocm_capture_for_paths(paths, &refs, Duration::from_mins(2))? + } + ChatRocmCommandAction::Approval { .. } => { + bail!("rocm_command changes local ROCm state and needs approval") + } + }; + Ok(internal_mcp_tool_result_from_command( + "Ran `rocm` command.", + output, + false, + )) + } + "update_check" => { + let output = + run_rocm_command_for_paths(paths, &["update".to_owned()], Duration::from_mins(1))?; + Ok(internal_mcp_tool_result_from_command( + "Ran `rocm update`.", + output, + false, + )) + } + "install_sdk_dry_run" => { + let args = internal_mcp_install_sdk_args(&arguments, true)?; + let output = run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))?; + Ok(internal_mcp_tool_result_from_command( + "Ran `rocm install sdk --dry-run`.", + output, + false, + )) + } + "path_exists" => run_chat_path_exists_tool(&call), + "proposal_action" => { + // Executes IN-PROCESS (no CLI subprocess): show loads a proposal; + // approve/reject mutate its status (allow_mutation already enforced + // by the read-only split above). proposal_action executes in-process + // (via update_automation_proposal_status); it never delegates to the + // subprocess arm. + let proposal_id = json_string(&arguments, "proposal_id") + .context("proposal_action requires `proposal_id`")?; + let action = + json_string(&arguments, "action").context("proposal_action requires `action`")?; + match action.as_str() { + "show" => { + let proposal = rocm_core::find_automation_proposal(paths, &proposal_id) + .with_context(|| { + format!("automation proposal `{proposal_id}` not found") + })?; + Ok(internal_mcp_tool_success( + format!( + "Proposal {} ({}): {}", + proposal.proposal_id, proposal.status, proposal.title + ), + serde_json::json!({ + "id": proposal.proposal_id, + "status": proposal.status, + "summary": proposal.title, + "reason": proposal.message, + }), + )) + } + "approve" | "reject" => { + let status = if action == "approve" { + "approved" + } else { + "rejected" + }; + let updated = + rocm_core::update_automation_proposal_status(paths, &proposal_id, status)?; + record_cli_audit_event( + paths, + "automations", + if action == "approve" { + "proposal_approved" + } else { + "proposal_rejected" + }, + "info", + format!("proposal {proposal_id} {status}"), + None, + ); + Ok(internal_mcp_tool_success( + format!("Proposal {proposal_id} {status}."), + serde_json::json!({ + "id": updated.proposal_id, + "status": updated.status, + "summary": updated.title, + "reason": updated.message, + }), + )) + } + other => bail!("proposal_action `{other}` is not supported"), + } + } + "install_sdk" | "install_engine" | "launch_server" | "stop_server" | "watcher_enable" + | "watcher_disable" => { + let args = rocm_chat_tool_requested_args(&call) + .with_context(|| format!("MCP tool `{name}` is missing required arguments"))?; + let refs = args.iter().map(String::as_str).collect::>(); + let output = run_rocm_capture_for_paths(paths, &refs, Duration::from_mins(2))?; + Ok(internal_mcp_tool_result_from_command( + "Ran approved `rocm` command.", + output, + false, + )) + } + other => bail!("unsupported MCP tool `{other}`"), + } +} + +fn internal_mcp_arguments(value: serde_json::Value) -> serde_json::Map { + if let Some(arguments) = value + .get("arguments") + .and_then(serde_json::Value::as_object) + { + return arguments.clone(); + } + value.as_object().cloned().unwrap_or_default() +} + +fn internal_mcp_tool_success(text: String, structured: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "content": [ + { + "type": "text", + "text": text, + } + ], + "structuredContent": structured, + "isError": false, + }) +} + +fn internal_mcp_tool_result_from_command( + prefix: &str, + output: CommandCapture, + is_error: bool, +) -> serde_json::Value { + let text = format!("{prefix}\n\n{}", command_capture_text(&output)); + serde_json::json!({ + "content": [ + { + "type": "text", + "text": text, + } + ], + "structuredContent": { + "argv": output.argv, + "exit_status": output.exit_status, + "stdout": output.stdout, + "stderr": output.stderr, + }, + "isError": is_error || output.exit_status != 0, + }) +} + +pub(super) fn mcp_tool_result_text(value: &serde_json::Value) -> String { + value + .get("content") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(|item| { + (item.get("type").and_then(serde_json::Value::as_str) == Some("text")) + .then(|| item.get("text").and_then(serde_json::Value::as_str)) + .flatten() + }) + .collect::>() + .join("\n") + }) + .filter(|text| !text.trim().is_empty()) + .unwrap_or_else(|| value.to_string()) +} + +fn run_rocm_capture_for_paths( + paths: &AppPaths, + args: &[&str], + timeout: Duration, +) -> Result { + rocm_core::run_rocm_capture_with_env(args, &paths.env_vars(), timeout) +} + +fn run_rocm_command_for_paths( + paths: &AppPaths, + args: &[String], + _timeout: Duration, +) -> Result { + match run_rocm_read_only_in_process(paths, args) { + Ok(stdout) => Ok(CommandCapture::from_text( + std::iter::once("rocm".to_owned()) + .chain(args.iter().cloned()) + .collect(), + 0, + stdout, + String::new(), + )), + Err(error) => Err(error).with_context(|| { + format!( + "read-only assistant command is not implemented in-process: {}", + format_structured_tool_call("rocm", args) + ) + }), + } +} + +fn run_rocm_read_only_in_process(paths: &AppPaths, args: &[String]) -> Result { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + match args { + [] => bail!("rocm command requires at least one argument"), + [command] if command.eq_ignore_ascii_case("examine") => { + render_examine_text_with_paths(paths, &config) + } + [command] + if command.eq_ignore_ascii_case("version") + || command == "--version" + || command == "-V" => + { + Ok(format!("rocm-cli {}\n", cli_version_string())) + } + // `model` / `models` render what `rocm model` prints in a terminal — + // the curated recommendation list. They used to render the + // `--verbose` diagnostics instead: every recipe with its aliases, + // engine recipes, flags and policy notes, measured at 204,470 + // characters (~58k tokens) on a developer host against 864 characters + // for plain `rocm model`. That is more than a locally served model's + // whole context, so the one tool call a small model reaches for when + // asked what it can run ended the turn with a provider error. The + // verbose dump is still reachable, but only when it was asked for. + [command] + if command.eq_ignore_ascii_case("model") || command.eq_ignore_ascii_case("models") => + { + Ok(render_model_registry_text_with_context_and_host( + Some(paths), + None, + None, + )) + } + [command, flag] + if (command.eq_ignore_ascii_case("model") + || command.eq_ignore_ascii_case("models")) + && flag.eq_ignore_ascii_case("--verbose") => + { + Ok(render_model_registry_verbose_text_with_context_and_host( + Some(paths), + None, + None, + )) + } + [command] if command.eq_ignore_ascii_case("daemon") => { + Ok(render_daemon_text(paths, &config)) + } + [command] if command.eq_ignore_ascii_case("logs") => Ok(render_logs_text(paths)), + [command, rest @ ..] if command.eq_ignore_ascii_case("logs") => { + let query = parse_optional_query(rest)?; + Ok(render_logs_browser_text(paths, query.as_deref())) + } + [command] if command.eq_ignore_ascii_case("runtimes") => { + render_runtimes_text(paths, &config) + } + [command, subcommand] + if command.eq_ignore_ascii_case("runtimes") + && subcommand.eq_ignore_ascii_case("list") => + { + render_runtimes_text(paths, &config) + } + [command, subcommand] + if command.eq_ignore_ascii_case("engines") + && subcommand.eq_ignore_ascii_case("list") => + { + Ok(render_engine_inventory_text_with_paths(Some(paths))) + } + [command] if command.eq_ignore_ascii_case("services") => render_services_text(paths, false), + [command, subcommand] + if command.eq_ignore_ascii_case("services") + && subcommand.eq_ignore_ascii_case("list") => + { + render_services_text(paths, false) + } + [command, subcommand, flag] + if command.eq_ignore_ascii_case("services") + && subcommand.eq_ignore_ascii_case("list") + && matches!(flag.as_str(), "-a" | "--all") => + { + render_services_text(paths, true) + } + [command, subcommand, service_id] + if command.eq_ignore_ascii_case("services") + && matches!(subcommand.to_ascii_lowercase().as_str(), "logs" | "log") => + { + render_service_logs_text(paths, service_id) + } + [command] if command.eq_ignore_ascii_case("automations") => { + render_automations_text(paths, &config) + } + [command, subcommand] + if command.eq_ignore_ascii_case("automations") + && subcommand.eq_ignore_ascii_case("list") => + { + render_automations_text(paths, &config) + } + [command, subcommand] + if command.eq_ignore_ascii_case("config") + && subcommand.eq_ignore_ascii_case("show") => + { + Ok(render_config_text(paths, &config)) + } + [command] if command.eq_ignore_ascii_case("comfyui") => { + comfyui::render_status(paths, &config) + } + [command, subcommand] + if command.eq_ignore_ascii_case("comfyui") + && subcommand.eq_ignore_ascii_case("status") => + { + comfyui::render_status(paths, &config) + } + [command, subcommand, rest @ ..] + if command.eq_ignore_ascii_case("comfyui") + && matches!(subcommand.to_ascii_lowercase().as_str(), "logs" | "log") => + { + comfyui::render_logs( + paths, + parse_optional_lines(rest).unwrap_or(DEFAULT_LOG_TAIL_LINES), + ) + } + [command, rest @ ..] + if command.eq_ignore_ascii_case("update") + && !rest.iter().any(|arg| arg.eq_ignore_ascii_case("--apply")) => + { + render_update_text(paths) + } + [command, subcommand, rest @ ..] + if command.eq_ignore_ascii_case("install") + && subcommand.eq_ignore_ascii_case("sdk") + && rest.iter().any(|arg| arg.eq_ignore_ascii_case("--dry-run")) => + { + render_install_sdk_dry_run_for_args(paths, rest) + } + [command, rest @ ..] + if command.eq_ignore_ascii_case("uninstall") + && rest.iter().any(|arg| arg.eq_ignore_ascii_case("--dry-run")) => + { + render_uninstall_dry_run(paths) + } + [command] if command.eq_ignore_ascii_case("setup") => { + render_setup_status_text(paths, &config) + } + [command, subcommand] + if command.eq_ignore_ascii_case("setup") + && subcommand.eq_ignore_ascii_case("status") => + { + render_setup_status_text(paths, &config) + } + // `install driver` is a read-only call where the command only reports + // (`chat_install_driver_action`). It runs as a dry run that never + // executes anything, so a call that reaches it elsewhere cannot change + // the host either. `--dry-run` is accepted because it is what this arm + // already does; `--reconcile` is not, since it rewrites the saved driver + // install state and the command refuses to combine it with the other + // flags. + [command, subcommand, rest @ ..] + if command.eq_ignore_ascii_case("install") + && subcommand.eq_ignore_ascii_case("driver") + && rest.iter().all(|arg| { + arg.eq_ignore_ascii_case("--dkms") || arg.eq_ignore_ascii_case("--dry-run") + }) => + { + let dkms = rest.iter().any(|arg| arg.eq_ignore_ascii_case("--dkms")); + crate::driver_install::install_driver(paths, dkms, false, true) + .map(|result| result.output) + .map_err(|error| error.source) + } + _ => bail!( + "unsupported in-process read-only rocm command: {}", + format_structured_tool_call("rocm", args) + ), + } +} + +fn render_install_sdk_dry_run_for_args(paths: &AppPaths, args: &[String]) -> Result { + let channel = chat_cli_arg_value(args, "--channel").unwrap_or("release"); + let format = chat_cli_arg_value(args, "--format").unwrap_or("wheel"); + let prefix = chat_cli_arg_value(args, "--prefix").map(PathBuf::from); + let version = chat_cli_arg_value(args, "--version").map(str::to_owned); + let build_date = chat_cli_arg_value(args, "--build-date").map(str::to_owned); + let selector = therock_install_version_selector(version, build_date)?; + let devel = chat_cli_has_flag(args, "--devel"); + // Dry run, so nothing is displaced and the consent gate is never reached; + // the narrow consent is what this chat surface would pass for a real + // install, and passing `--yes`'s source here would be a lie waiting to be + // printed if the preview ever grew a gate. + Ok(therock::install_sdk( + paths, + therock::SdkInstallRequest { + channel, + format, + prefix, + version_selector: selector, + dry_run: true, + include_devel: devel, + consent: therock::SdkInstallConsent::Preapproved( + therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault, + ), + ..therock::SdkInstallRequest::default() + }, + )? + .output) +} + +fn internal_mcp_install_sdk_args( + arguments: &serde_json::Map, + dry_run: bool, +) -> Result> { + let channel = json_string(arguments, "channel").unwrap_or_else(|| "release".to_owned()); + let format = json_string(arguments, "format").unwrap_or_else(|| "wheel".to_owned()); + let mut argv = vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + channel, + "--format".to_owned(), + format, + ]; + if let Some(prefix) = json_string(arguments, "prefix") { + let prefix_path = Path::new(&prefix); + if chat_install_prefix_is_system(prefix_path) { + bail!( + "install_sdk prefix `{}` is a system folder; choose a user folder instead", + prefix_path.display() + ); + } + argv.push("--prefix".to_owned()); + argv.push(prefix); + } + if let Some(version) = json_string(arguments, "version") { + argv.push("--version".to_owned()); + argv.push(version); + } + if let Some(build_date) = json_string(arguments, "build_date") { + argv.push("--build-date".to_owned()); + argv.push(build_date); + } + if dry_run { + argv.push("--dry-run".to_owned()); + } + Ok(argv) +} + +pub(super) fn run_chat_read_only_tool( + paths: &AppPaths, + call: &providers::ChatToolCall, +) -> Result { + match call.name.as_str() { + "path_exists" => run_chat_path_exists_tool(call), + "port_status" => run_chat_port_status_tool(paths, call), + "rocm_command" => { + let action = chat_rocm_command_action(call)?; + let ChatRocmCommandAction::ReadOnly(args) = action else { + bail!("assistant read-only path cannot run mutating rocm_command"); + }; + let output = run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))?; + Ok(internal_mcp_tool_result_from_command( + "Ran `rocm` command.", + output, + false, + )) + } + "update_check" => { + let output = + run_rocm_command_for_paths(paths, &["update".to_owned()], Duration::from_mins(1))?; + Ok(internal_mcp_tool_result_from_command( + "Ran `rocm update`.", + output, + false, + )) + } + "install_sdk_dry_run" => { + let arguments = internal_mcp_arguments(call.arguments.clone()); + let args = internal_mcp_install_sdk_args(&arguments, true)?; + let output = run_rocm_command_for_paths(paths, &args, Duration::from_mins(2))?; + Ok(internal_mcp_tool_result_from_command( + "Ran `rocm install sdk --dry-run`.", + output, + false, + )) + } + _ => run_internal_mcp_call(paths, &call.name, call.arguments.clone(), false), + } +} + +pub(super) fn run_chat_path_exists_tool( + call: &providers::ChatToolCall, +) -> Result { + let object = call + .arguments + .as_object() + .context("path_exists arguments must be a JSON object")?; + let path = json_string(object, "path").context("path_exists requires path")?; + let path = Path::new(&path); + let metadata = path.metadata().ok(); + let path_kind = metadata.as_ref().map_or("missing", |metadata| { + if metadata.is_dir() { + "directory" + } else if metadata.is_file() { + "file" + } else { + "other" + } + }); + let parent = path.parent(); + let parent_exists = parent.is_some_and(Path::exists); + let parent_display = parent.map_or_else( + || "".to_owned(), + |parent| parent.display().to_string(), + ); + let text = format!( + "path: {}\nexists: {}\nkind: {}\nparent: {}\nparent_exists: {}", + path.display(), + metadata.is_some(), + path_kind, + parent_display, + parent_exists + ); + Ok(serde_json::json!({ + "content": [{ + "type": "text", + "text": text, + }] + })) +} + +fn run_chat_port_status_tool( + paths: &AppPaths, + call: &providers::ChatToolCall, +) -> Result { + let object = call + .arguments + .as_object() + .context("port_status arguments must be a JSON object")?; + let host = json_string(object, "host").unwrap_or_else(|| DEFAULT_LOCAL_HOST.to_owned()); + let port = object + .get("port") + .and_then(serde_json::Value::as_u64) + .context("port_status requires port")? as u16; + let reachable = loopback_tcp_port_is_reachable(&host, port); + let host_key = loopback_host_key(&host); + let matching_services = load_managed_services(paths)? + .into_iter() + .filter(|record| { + record.port == port + && loopback_host_key(&record.host) == host_key + && managed_service_is_live(record) + }) + .collect::>(); + let app_hint = if port == comfyui::default_port() { + Some("ComfyUI default port") + } else if port == rocm_core::DEFAULT_LOCAL_PORT { + Some("ROCm local model server default port") + } else { + None + }; + let mut text = String::new(); + let _ = writeln!(text, "host: {host}"); + let _ = writeln!(text, "port: {port}"); + let _ = writeln!(text, "listening: {reachable}"); + if let Some(app_hint) = app_hint { + let _ = writeln!(text, "hint: {app_hint}"); + } + if matching_services.is_empty() { + let _ = writeln!(text, "managed_service: none"); + } else { + let _ = writeln!(text, "managed_services:"); + for service in &matching_services { + let _ = writeln!( + text, + " - service_id={} engine={} model={} status={} running_state={} endpoint={}", + service.service_id, + service.engine, + service.model_ref, + service.status, + managed_service_running_state(&service.status), + service.endpoint_url + ); + } + } + Ok(serde_json::json!({ + "content": [{ + "type": "text", + "text": text, + }], + "structuredContent": { + "host": host, + "port": port, + "listening": reachable, + "hint": app_hint, + "managed_services": matching_services, + }, + "isError": false, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::test_paths; + use std::fs; + use std::net::TcpListener; + + /// The structured reply of the CLI's own `service_logs` tool for a service + /// whose log holds `log`, asking for `lines` lines. + fn service_logs_structured(label: &str, log: &[u8], lines: u64) -> serde_json::Value { + let (root, paths) = test_paths(label); + paths.ensure().expect("paths"); + let record = ManagedServiceRecord::new( + &paths, + "svc-tail-count", + "llamacpp", + "a-model", + "a-model", + "127.0.0.1", + 18080, + "managed", + std::process::id(), + None, + None, + None, + ); + record.write().expect("record"); + fs::write(&record.log_path, log).expect("log"); + let reply = run_internal_mcp_call( + &paths, + "service_logs", + serde_json::json!({ "service_id": "svc-tail-count", "lines": lines }), + false, + ) + .expect("service_logs ok"); + let _ = fs::remove_dir_all(root); + reply["structuredContent"].clone() + } + + #[test] + fn service_logs_report_the_lines_returned_and_the_lines_asked_for_apart() { + // Asking for 50 lines of a four-line log returns four. + let content = service_logs_structured("service-logs-count", b"a\nb\nc\nd\n", 50); + assert_eq!(content["lines"], 4, "{content}"); + assert_eq!(content["lines_requested"], 50, "{content}"); + assert_eq!(content["tail"], "a\nb\nc\nd", "{content}"); + // A log with more lines than asked for returns exactly what was asked. + let content = service_logs_structured("service-logs-count-cut", b"a\nb\nc\nd\n", 2); + assert_eq!(content["lines"], 2, "{content}"); + assert_eq!(content["lines_requested"], 2, "{content}"); + // An empty log returned none, whatever was asked for. + let content = service_logs_structured("service-logs-count-empty", b"", 50); + assert_eq!(content["lines"], 0, "{content}"); + assert_eq!(content["lines_requested"], 50, "{content}"); + } + + #[test] + fn mcp_tool_result_text_joins_text_content_items() { + let value = serde_json::json!({ + "content": [ + {"type": "text", "text": "line one"}, + {"type": "image", "text": "ignored"}, + {"type": "text", "text": "line two"}, + ] + }); + assert_eq!(mcp_tool_result_text(&value), "line one\nline two"); + } + + #[test] + fn mcp_tool_result_text_falls_back_to_raw_value_without_text_content() { + let value = serde_json::json!({"content": []}); + assert_eq!(mcp_tool_result_text(&value), value.to_string()); + } + + #[test] + fn loopback_tcp_port_is_reachable_detects_listening_and_closed_ports() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback listener"); + let port = listener.local_addr().expect("local addr").port(); + assert!(loopback_tcp_port_is_reachable("127.0.0.1", port)); + drop(listener); + + // Port 0 can never be connected to; stands in for "closed" without + // depending on a specific unused port staying unused. + assert!(!loopback_tcp_port_is_reachable("127.0.0.1", 0)); + } + + #[test] + fn parse_optional_query_reads_flag_and_positional_forms() { + assert_eq!( + parse_optional_query(&["--query".to_owned(), "gpu".to_owned()]).unwrap(), + Some("gpu".to_owned()) + ); + assert_eq!( + parse_optional_query(&["-q".to_owned(), "gpu".to_owned()]).unwrap(), + Some("gpu".to_owned()) + ); + assert_eq!( + parse_optional_query(&["gpu".to_owned()]).unwrap(), + Some("gpu".to_owned()) + ); + assert_eq!(parse_optional_query(&[]).unwrap(), None); + } + + #[test] + fn parse_optional_query_rejects_a_dangling_flag() { + assert!(parse_optional_query(&["--query".to_owned()]).is_err()); + } + + #[test] + fn parse_optional_query_skips_unknown_flags_to_find_the_positional_value() { + assert_eq!( + parse_optional_query(&["--unknown-flag".to_owned(), "gpu".to_owned()]).unwrap(), + Some("gpu".to_owned()) + ); + } + + #[test] + fn parse_optional_lines_reads_flag_or_defaults() { + assert_eq!( + parse_optional_lines(&["--lines".to_owned(), "25".to_owned()]).unwrap(), + 25 + ); + assert_eq!( + parse_optional_lines(&["-n".to_owned(), "25".to_owned()]).unwrap(), + 25 + ); + assert_eq!(parse_optional_lines(&[]).unwrap(), DEFAULT_LOG_TAIL_LINES); + } + + #[test] + fn parse_optional_lines_rejects_a_non_numeric_value() { + assert!(parse_optional_lines(&["--lines".to_owned(), "nope".to_owned()]).is_err()); + } + + #[test] + fn services_tool_result_text_includes_running_interpretation() { + let (_root, paths) = test_paths("services-tool-text"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-vllm", + "vllm", + "Qwen/Qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); + + let rendered = render_services_tool_result_text(&[record]); + + assert!(rendered.contains("status_meaning: ready/running = running")); + assert!(rendered.contains("engine=vllm")); + assert!(rendered.contains("running_state=running")); + } + + #[test] + fn proposal_action_approve_updates_status() { + let (root, paths) = test_paths("proposal-approve"); + // Seed a pending proposal. + let proposal = rocm_core::AutomationProposalRecord { + at_unix_ms: rocm_core::unix_time_millis(), + proposal_id: "prop-approve-1".to_owned(), + watcher_id: "therock-update".to_owned(), + action: "prepare_driver_plan".to_owned(), + title: "Apply driver plan".to_owned(), + message: "A reviewed driver plan is ready.".to_owned(), + status: "pending".to_owned(), + service_id: None, + tool: None, + arguments: serde_json::Value::Null, + reviewed_at_unix_ms: None, + }; + rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); + + // show is read-only and returns the proposal. + let shown = run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "prop-approve-1", "action": "show" }), + false, + ) + .expect("show ok"); + assert_eq!(shown["structuredContent"]["status"], "pending"); + + // approve requires allow_mutation. + assert!( + run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), + false, + ) + .is_err(), + "approve without allow_mutation must bail" + ); + + // approve with allow_mutation sets status to approved. + let approved = run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), + true, + ) + .expect("approve ok"); + assert_eq!(approved["structuredContent"]["status"], "approved"); + let stored = rocm_core::find_automation_proposal(&paths, "prop-approve-1") + .expect("proposal still present"); + assert_eq!(stored.status, "approved"); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn proposal_action_reject_updates_status() { + let (root, paths) = test_paths("proposal-reject"); + let proposal = rocm_core::AutomationProposalRecord { + at_unix_ms: rocm_core::unix_time_millis(), + proposal_id: "prop-reject-1".to_owned(), + watcher_id: "server-recover".to_owned(), + action: "queue_stop_server_proposal".to_owned(), + title: "Stop overheating server".to_owned(), + message: "GPU thermal pressure detected.".to_owned(), + status: "pending".to_owned(), + service_id: None, + tool: None, + arguments: serde_json::Value::Null, + reviewed_at_unix_ms: None, + }; + rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); + + let rejected = run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "prop-reject-1", "action": "reject" }), + true, + ) + .expect("reject ok"); + assert_eq!(rejected["structuredContent"]["status"], "rejected"); + let stored = rocm_core::find_automation_proposal(&paths, "prop-reject-1") + .expect("proposal still present"); + assert_eq!(stored.status, "rejected"); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn proposal_action_show_missing_proposal_errors() { + let (root, paths) = test_paths("proposal-missing"); + assert!( + run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "nope", "action": "show" }), + false, + ) + .is_err(), + "showing a missing proposal must error" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn assistant_read_only_rocm_commands_do_not_fallback_to_child_process() { + let (_root, paths) = test_paths("readonly-rocm-in-process-only"); + let args = vec!["services".to_owned(), "status".to_owned()]; + + let error = run_rocm_command_for_paths(&paths, &args, Duration::from_secs(1)) + .unwrap_err() + .to_string(); + + assert!(error.contains("read-only assistant command is not implemented in-process")); + assert!(error.contains("rocm services status")); + assert!(!paths.data_dir.join("logs").exists()); + } + + #[test] + fn internal_mcp_read_only_rocm_command_runs_in_process() { + let (_root, paths) = test_paths("mcp-readonly-rocm-in-process"); + + let result = run_internal_mcp_call( + &paths, + "rocm_command", + serde_json::json!({ "args": ["version"] }), + false, + ) + .expect("read-only rocm mcp-call should run"); + + assert_eq!( + result.get("isError").and_then(serde_json::Value::as_bool), + Some(false) + ); + assert_eq!( + result + .pointer("/structuredContent/argv/0") + .and_then(serde_json::Value::as_str), + Some("rocm") + ); + // The MCP summary prefixes the command output. The expected line is + // assembled from the build script's separate exports rather than from + // `cli_version_string()`, the function the handler itself calls, so a + // regression in that function's content fails here too. Which leading + // version a tag, `main`, or an unknown ref produces is the + // `format_cli_version_*` tests in `build.rs` (run via + // `tests/build_metadata.rs`). + let text = mcp_tool_result_text(&result); + let version_line = format!( + "rocm-cli {} ({}, {})", + env!("ROCM_CLI_VERSION_NUMBER"), + env!("ROCM_CLI_VERSION_REF"), + env!("ROCM_CLI_GIT_HASH") + ); + assert!(text.lines().any(|line| line == version_line), "{text}"); + } + + /// What the chat `model` tool hands back has to be what `rocm model` + /// prints, not the `--verbose` diagnostics. The verbose dump is every + /// recipe with its aliases, engine recipes, flags and policy notes; on a + /// developer host it measured 204,470 characters against 864 for plain + /// `rocm model`, which is more than a locally served model's whole + /// context. Asking a small model "which LLMs can this machine support" + /// makes it call exactly this, so the one question the tool exists for + /// used to end in a provider context-overflow error. + #[test] + fn chat_model_tool_returns_the_curated_list_rocm_model_prints() { + let (_root, paths) = test_paths("mcp-model-curated"); + + let result = run_internal_mcp_call( + &paths, + "rocm_command", + serde_json::json!({ "args": ["model"] }), + false, + ) + .expect("read-only rocm_command model should run"); + let text = mcp_tool_result_text(&result); + + // The same recipes, from the same renderer `rocm model` calls. + let curated = render_model_registry_text_with_context_and_host(Some(&paths), None, None); + assert!( + text.contains(curated.trim()), + "the tool must return what `rocm model` prints:\n{text}" + ); + // And not the per-recipe diagnostics, which are what made it huge. + assert!( + !text.contains("aliases=["), + "the verbose dump must not be the default:\n{text}" + ); + // Comfortably inside the dash's 16 KiB per-result budget + // (`TOOL_RESULT_CHAR_BUDGET` in `crates/rocm-dash-tui/src/agent/tools.rs`), + // so a chat turn carrying it is never truncated on the built-in + // catalog. Asserted here rather than imported so the bin does not + // depend on a dash constant. + assert!( + text.len() < 16 * 1024, + "the curated list is {} characters, past the dash budget", + text.len() + ); + } + + /// The diagnostics are still reachable — the model has to ask for them, + /// the same way a person types `rocm model --verbose`. + #[test] + fn chat_model_tool_still_renders_the_verbose_dump_on_request() { + let (_root, paths) = test_paths("mcp-model-verbose"); + + let result = run_internal_mcp_call( + &paths, + "rocm_command", + serde_json::json!({ "args": ["model", "--verbose"] }), + false, + ) + .expect("read-only rocm_command model --verbose should run"); + let text = mcp_tool_result_text(&result); + + assert!(text.contains("model recipes"), "{text}"); + assert!( + text.contains("aliases=["), + "--verbose must still render the per-recipe diagnostics:\n{text}" + ); + // The two really are different sizes; without this the test above + // could pass against a renderer that lost its diagnostics entirely. + let curated = render_model_registry_text_with_context_and_host(Some(&paths), None, None); + assert!( + text.len() > curated.len(), + "the verbose dump ({}) should be longer than the curated list ({})", + text.len(), + curated.len() + ); + } + + #[test] + fn natural_language_plan_returns_structured_mutating_action() { + let (_root, paths) = test_paths("nl-plan-structured-mutating"); + + let result = run_internal_mcp_call( + &paths, + "natural_language_plan", + serde_json::json!({ + "request": "install TheRock into /opt/rocm-target" + }), + false, + ) + .expect("natural_language_plan should plan the request"); + + assert_eq!( + result.get("isError").and_then(serde_json::Value::as_bool), + Some(false) + ); + // text (rendered plan) preserved. + assert!( + !result + .pointer("/structuredContent/text") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .trim() + .is_empty() + ); + // A known install folder yields a mutating, placeholder-free action. + assert_eq!( + result + .pointer("/structuredContent/action/approval_required") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + result + .pointer("/structuredContent/action/has_placeholders") + .and_then(serde_json::Value::as_bool), + Some(false) + ); + let args = result + .pointer("/structuredContent/action/args") + .and_then(serde_json::Value::as_array) + .expect("action args should be present"); + assert!(args.iter().any(|arg| arg.as_str() == Some("install"))); + assert!(args.iter().any(|arg| arg.as_str() == Some("--prefix"))); + } + + #[test] + fn natural_language_plan_returns_placeholder_action_when_incomplete() { + let (_root, paths) = test_paths("nl-plan-structured-placeholder"); + + let result = run_internal_mcp_call( + &paths, + "natural_language_plan", + serde_json::json!({ "request": "serve" }), + false, + ) + .expect("natural_language_plan should plan the request"); + + assert_eq!( + result + .pointer("/structuredContent/action/has_placeholders") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + } + + /// On Windows `rocm install driver` only reports, and chat plans it as a + /// read-only call. That call must run in process: before this arm existed it + /// failed with "unsupported in-process read-only rocm command". It is a dry + /// run, so it never executes anything on any platform. + #[test] + fn install_driver_read_only_call_runs_in_process() { + let (_root, paths) = test_paths("install-driver-in-process"); + let args = vec!["install".to_owned(), "driver".to_owned()]; + let text = run_rocm_read_only_in_process(&paths, &args) + .expect("the read-only install driver call should run in process"); + assert!(text.contains("approval:"), "{text}"); + assert!(!text.contains("driver install completed"), "{text}"); + assert!( + run_rocm_read_only_in_process( + &paths, + &[ + "install".to_owned(), + "driver".to_owned(), + "--yes".to_owned() + ] + ) + .is_err(), + "a flag the read-only path does not know is still refused" + ); + } + + /// `--dry-run` is what the in-process arm already does, so it is accepted + /// and the call reports a dry run. `--reconcile` writes driver install state + /// and stays refused, as does any flag the arm does not know. + #[test] + fn install_driver_read_only_call_accepts_dry_run_and_refuses_reconcile() { + let (_root, paths) = test_paths("install-driver-in-process-dry-run"); + let run = |extra: &[&str]| { + let mut args = vec!["install".to_owned(), "driver".to_owned()]; + args.extend(extra.iter().map(|arg| (*arg).to_owned())); + run_rocm_read_only_in_process(&paths, &args) + }; + for extra in [&["--dry-run"][..], &["--dkms", "--dry-run"], &["--DRY-RUN"]] { + let text = run(extra).unwrap_or_else(|error| panic!("{extra:?} must run: {error}")); + assert!(text.contains("dry_run: true"), "{extra:?}: {text}"); + assert!( + !text.contains("driver install completed"), + "{extra:?}: {text}" + ); + } + for extra in [&["--reconcile"][..], &["--dry-run", "--reconcile"]] { + assert!(run(extra).is_err(), "{extra:?} must stay refused"); + } + // Refusing `--reconcile` changed nothing on disk: the state file is + // written only by a reconcile or an executed install. + assert!( + !crate::driver_install::driver_install_state_path(&paths).exists(), + "a refused call must not create driver install state" + ); + } + + #[test] + fn setup_status_renders_in_process() { + let (_root, paths) = test_paths("setup-status-in-process"); + + for args in [ + vec!["setup".to_owned()], + vec!["setup".to_owned(), "status".to_owned()], + ] { + let text = run_rocm_read_only_in_process(&paths, &args) + .expect("setup status read path should render in-process"); + assert!( + !text.trim().is_empty(), + "setup status text should be non-empty for {args:?}" + ); + assert!( + text.contains("ROCm setup"), + "setup status text should be reached for {args:?}: {text}" + ); + } + } + + #[test] + fn port_status_matches_loopback_managed_services() -> Result<()> { + let (root, paths) = test_paths("port-status-loopback"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-comfyui", + "comfyui", + "ComfyUI", + "ComfyUI", + "127.0.0.1", + 18188, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); + record.write()?; + + let call = providers::ChatToolCall { + id: Some("port-check".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "localhost", "port": 18188 }), + }; + let result = run_chat_port_status_tool(&paths, &call)?; + let text = mcp_tool_result_text(&result); + let managed_service_count = result + .get("structuredContent") + .and_then(|content| content.get("managed_services")) + .and_then(serde_json::Value::as_array) + .map_or(0, Vec::len); + let _ = fs::remove_dir_all(root); + + assert_eq!(managed_service_count, 1); + assert!(text.contains("managed_services:"), "{text}"); + assert!(text.contains("service_id=svc-comfyui"), "{text}"); + assert!(text.contains("running_state=starting"), "{text}"); + Ok(()) + } +} diff --git a/apps/rocm/src/chat/summaries.rs b/apps/rocm/src/chat/summaries.rs new file mode 100644 index 0000000000..7415ec0ba9 --- /dev/null +++ b/apps/rocm/src/chat/summaries.rs @@ -0,0 +1,804 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Deterministic summaries of the read-only tool output the chat assistant +//! reads back: `examine`, the model recipes, ComfyUI status, the managed +//! services, a port check and the engine inventory. Each +//! `deterministic_*_tool_summary` function recognises one tool's text, reads +//! the values it needs out of it and returns a short plain-language summary, +//! or `None` when the text is not that tool's. `deterministic_chat_tool_summary` +//! is the entry point the turn loop calls; when it returns a summary the chat +//! shows it instead of asking the local model to restate the output. +//! +//! The parsers that turn tool text back into rows (`DeterministicModelRecipe`, +//! `DeterministicServiceRow`, `DeterministicEngineRow`) and +//! `managed_service_running_state`, the mapping from a managed service's status +//! to `running`, `starting`, `not_running` or `unknown`, live here too. The +//! services tool text itself is rendered by `chat/mcp_bridge.rs` +//! (`render_services_tool_result_text`), which reads +//! `managed_service_running_state` from this module. +//! +//! A leaf of the `chat` module: nothing here calls back into `chat.rs`. The +//! only edges out are `providers` (the assistant model ids) and +//! `managed_service_status::friendly_engine_label`. + +use crate::managed_service_status::friendly_engine_label; +use crate::providers; +use rocm_core::placeholder::empty_as_unknown; +use std::fmt::Write as _; + +#[derive(Debug, Default)] +struct DeterministicModelRecipe { + pub(crate) canonical_id: String, + pub(crate) min_gpu_mem_gib: Option, + pub(crate) engines: Vec, + pub(crate) engine_statuses: Vec, + pub(crate) warnings: Vec, +} + +fn parse_deterministic_model_recipes(tool_text: &str) -> Vec { + let mut recipes = Vec::new(); + let mut current: Option = None; + for line in tool_text.lines() { + let trimmed = line.trim(); + if let Some(recipe) = parse_model_recipe_header(trimmed) { + if let Some(previous) = current.replace(recipe) { + recipes.push(previous); + } + continue; + } + let Some(recipe) = current.as_mut() else { + continue; + }; + if is_engine_status_line(trimmed) { + recipe.engine_statuses.push(trimmed.to_owned()); + } else if let Some(warning) = trimmed.strip_prefix("warning:") { + recipe.warnings.push(warning.trim().to_owned()); + } + } + if let Some(recipe) = current { + recipes.push(recipe); + } + recipes +} + +fn parse_model_recipe_header(line: &str) -> Option { + if !line.contains(" aliases=[") || !line.contains("min_gpu_mem=") { + return None; + } + let canonical_id = line.split_whitespace().next()?.to_owned(); + let engines = parse_bracketed_csv(line, "engines=[", "]"); + let min_gpu_mem_gib = line + .split_once("min_gpu_mem=") + .and_then(|(_, rest)| rest.split_whitespace().next()) + .and_then(|value| value.parse::().ok()); + Some(DeterministicModelRecipe { + canonical_id, + min_gpu_mem_gib, + engines, + engine_statuses: Vec::new(), + warnings: Vec::new(), + }) +} + +fn parse_bracketed_csv(line: &str, start: &str, end: &str) -> Vec { + line.split_once(start) + .and_then(|(_, rest)| rest.split_once(end)) + .map(|(values, _)| { + values + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() +} + +fn is_engine_status_line(line: &str) -> bool { + matches!( + line.split_once(':').map(|(engine, _)| engine), + Some("lemonade" | "vllm" | "aim") + ) +} + +#[derive(Debug, Default)] +struct DeterministicServiceRow { + pub(crate) service_id: String, + pub(crate) engine: String, + pub(crate) model: String, + pub(crate) status: String, + pub(crate) running_state: String, + pub(crate) endpoint: String, +} + +fn parse_deterministic_service_rows(tool_text: &str) -> Vec { + let mut rows = Vec::new(); + let mut current: Option = None; + for line in tool_text.lines() { + let trimmed = line.trim(); + if let Some(rest) = trimmed.strip_prefix("- ") { + if rest.starts_with("service_id=") { + if let Some(row) = current.take() { + rows.push(row); + } + let mut row = DeterministicServiceRow::default(); + for token in rest.split_whitespace() { + if let Some((key, value)) = token.split_once('=') { + match key { + "service_id" => row.service_id = value.to_owned(), + "engine" => row.engine = value.to_owned(), + "model" => row.model = value.to_owned(), + "status" => row.status = value.to_owned(), + "running_state" => row.running_state = value.to_owned(), + "endpoint" => row.endpoint = value.to_owned(), + _ => {} + } + } + } + if row.running_state.is_empty() { + row.running_state = managed_service_running_state(&row.status).to_owned(); + } + current = Some(row); + continue; + } + if let Some(row) = current.take() { + rows.push(row); + } + current = Some(DeterministicServiceRow { + service_id: rest.to_owned(), + ..Default::default() + }); + continue; + } + let Some(row) = current.as_mut() else { + continue; + }; + if let Some(value) = trimmed.strip_prefix("status:") { + row.status = value.trim().to_owned(); + row.running_state = managed_service_running_state(&row.status).to_owned(); + } else if let Some(value) = trimmed.strip_prefix("engine:") { + row.engine = value.trim().to_owned(); + } else if let Some(value) = trimmed.strip_prefix("model:") { + row.model = value.trim().to_owned(); + } else if let Some(value) = trimmed.strip_prefix("endpoint:") { + row.endpoint = value.trim().to_owned(); + } + } + if let Some(row) = current { + rows.push(row); + } + rows +} + +#[derive(Debug, Default)] +struct DeterministicEngineRow { + pub(crate) engine: String, + pub(crate) runtime: String, + pub(crate) note: String, +} + +fn parse_deterministic_engine_rows(tool_text: &str) -> Vec { + let mut rows = Vec::new(); + let mut current: Option = None; + for line in tool_text.lines() { + let trimmed = line.trim().trim_start_matches("* ").trim_start(); + if let Some(engine) = engine_name_from_inventory_line(trimmed) { + if let Some(row) = current.take() { + rows.push(row); + } + current = Some(DeterministicEngineRow { + engine: engine.to_owned(), + ..Default::default() + }); + continue; + } + let Some(row) = current.as_mut() else { + continue; + }; + if let Some(runtime) = trimmed.strip_prefix("runtime:") { + row.runtime = runtime.trim().to_owned(); + } else if let Some(note) = trimmed.strip_prefix("note:") { + row.note = note.trim().to_owned(); + } + } + if let Some(row) = current { + rows.push(row); + } + rows +} + +fn engine_name_from_inventory_line(line: &str) -> Option<&'static str> { + ["lemonade", "vllm", "aim"].into_iter().find(|engine| { + line == *engine + || line + .strip_prefix(*engine) + .is_some_and(|rest| rest.starts_with(char::is_whitespace)) + }) +} + +pub(super) fn managed_service_running_state(status: &str) -> &'static str { + match status { + "ready" | "running" => "running", + "starting" | "recovering" => "starting", + "failed" | "stopped" => "not_running", + _ => "unknown", + } +} + +pub(super) fn deterministic_rocm_tool_summary(tool_text: &str) -> Option { + if !tool_text.contains("examine:") { + return None; + } + let mut lines = Vec::new(); + let gpu = chat_tool_value(tool_text, "driver_detail") + .filter(|value| value != "") + .or_else(|| { + let target = chat_tool_value(tool_text, "detected_gfx_target")?; + Some(format!("AMD GPU target {target}")) + }); + if let Some(gpu) = gpu { + lines.push(format!(" GPU: {gpu}")); + } + + let active_status = chat_tool_value(tool_text, "active_runtime_status"); + if active_status.as_deref() == Some("ready") { + let family = chat_tool_value(tool_text, "active_runtime_family") + .filter(|value| value != "" && value != ""); + let version = chat_tool_value(tool_text, "active_runtime_version") + .filter(|value| value != "" && value != ""); + let mut detail = "installed and active for ROCm CLI".to_owned(); + if let Some(family) = family { + let _ = write!(detail, " ({family})"); + } + if let Some(version) = version { + let _ = write!(detail, ", {version}"); + } + lines.push(format!(" ROCm/TheRock: {detail}")); + if let Some(root) = chat_tool_value(tool_text, "active_runtime_root") + .or_else(|| chat_tool_value(tool_text, "setup_runtime_root")) + .filter(|value| value != "" && value != "" && value != "") + { + lines.push(format!(" Install folder: {root}")); + } + if let Some(cache) = chat_tool_value(tool_text, "active_runtime_pip_cache_dir") + .or_else(|| chat_tool_value(tool_text, "setup_runtime_pip_cache_dir")) + .filter(|value| value != "" && value != "" && value != "") + { + lines.push(format!(" Downloads/cache: {cache}")); + } + } else if let Some(status) = active_status.as_deref() { + lines.push(format!(" ROCm/TheRock: active runtime status is {status}")); + if let Some(root) = chat_tool_value(tool_text, "setup_runtime_root") + .filter(|value| value != "" && value != "" && value != "") + { + lines.push(format!(" Selected setup folder: {root}")); + } + if let Some(cache) = chat_tool_value(tool_text, "setup_runtime_pip_cache_dir") + .filter(|value| value != "" && value != "" && value != "") + { + lines.push(format!(" Downloads/cache: {cache}")); + } + } + + if chat_tool_value(tool_text, "legacy_rocm_status").as_deref() == Some("not_detected") + && active_status.as_deref() == Some("ready") + { + lines.push( + " Note: no global legacy ROCm install was found; ROCm CLI is using its managed TheRock runtime." + .to_owned(), + ); + } + + (!lines.is_empty()).then(|| lines.join("\n")) +} + +pub(super) fn deterministic_chat_tool_summary(tool_text: &str) -> Option { + deterministic_rocm_tool_summary(tool_text) + .or_else(|| deterministic_model_tool_summary(tool_text)) + .or_else(|| deterministic_combined_status_tool_summary(tool_text)) +} + +fn deterministic_combined_status_tool_summary(tool_text: &str) -> Option { + let summaries = [ + deterministic_comfyui_tool_summary(tool_text), + deterministic_port_status_tool_summary(tool_text), + deterministic_engine_inventory_tool_summary(tool_text), + deterministic_services_tool_summary(tool_text), + ] + .into_iter() + .flatten() + .collect::>(); + (!summaries.is_empty()).then(|| summaries.join("\n")) +} + +pub(super) fn deterministic_summary_can_stand_alone(tool_text: &str) -> bool { + !tool_text.contains("model recipes") +} + +fn deterministic_model_tool_summary(tool_text: &str) -> Option { + if !tool_text.contains("model recipes") { + return None; + } + let recipes = parse_deterministic_model_recipes(tool_text); + if recipes.is_empty() { + return None; + } + + let mut lines = Vec::new(); + if let Some(recipe) = recipes + .iter() + .find(|recipe| recipe.canonical_id == providers::LEMONADE_ASSISTANT_MODEL_ID) + { + lines.push(format!( + " Recommended local assistant: qwen ({})", + recipe.canonical_id + )); + lines.push(format!( + " Fit: needs about {} GPU memory.", + recipe + .min_gpu_mem_gib + .map_or_else(|| "unknown".to_owned(), |value| format!("{value} GiB")) + )); + lines.push(format!( + " Engine: {}", + deterministic_engine_fit_summary(recipe, "lemonade") + )); + } + + if let Some(recipe) = recipes + .iter() + .find(|recipe| recipe.canonical_id == "Qwen3-0.6B-GGUF") + { + lines.push(format!( + " Tiny smoke test: qwen-smoke ({})", + recipe.canonical_id + )); + lines.push( + " Use this only to check that GPU serving starts; it is not the default assistant." + .to_owned(), + ); + } + + if let Some(recipe) = recipes + .iter() + .find(|recipe| recipe.canonical_id == "meta-llama/Llama-3.2-3B-Instruct") + { + lines.push(format!( + " 8 GiB-class option: llama ({})", + recipe.canonical_id + )); + lines.push(format!( + " Fit: asks for {}; this can be tight on APUs and depends on available shared GPU memory.", + recipe + .min_gpu_mem_gib.map_or_else(|| "unknown GPU memory".to_owned(), |value| format!("{value} GiB")) + )); + lines.push(format!( + " Engines: {}", + deterministic_available_engine_names(recipe).join(", ") + )); + } + + let larger = recipes + .iter() + .filter(|recipe| recipe.min_gpu_mem_gib.is_some_and(|value| value > 8)) + .filter(|recipe| recipe.canonical_id != providers::BUILTIN_ASSISTANT_MODEL_ID) + .map(|recipe| { + format!( + "{} asks for {}", + recipe.canonical_id, + recipe.min_gpu_mem_gib.map_or_else( + || "more GPU memory".to_owned(), + |value| format!("{value} GiB") + ) + ) + }) + .collect::>(); + if !larger.is_empty() { + lines.push(format!( + " Larger recipes are not low-VRAM defaults: {}.", + larger.join("; ") + )); + } + + let linux_wsl_only = recipes + .iter() + .flat_map(|recipe| { + recipe + .engine_statuses + .iter() + .filter(|status| { + status.contains("unsupported_native_windows") + || status.to_ascii_lowercase().contains("wsl/linux") + || status.to_ascii_lowercase().contains("linux/wsl") + }) + .filter_map(|status| { + let (engine, _) = status.split_once(':')?; + Some(format!( + "{} uses {} through WSL/Linux on Windows", + recipe.canonical_id, engine + )) + }) + }) + .collect::>(); + if !linux_wsl_only.is_empty() { + lines.push(format!( + " Native Windows note: {}.", + linux_wsl_only.join("; ") + )); + } + + if !lines.is_empty() { + lines.push( + " Run `rocm examine` to refresh GPU memory details before starting anything large." + .to_owned(), + ); + } + (!lines.is_empty()).then(|| lines.join("\n")) +} + +fn deterministic_engine_fit_summary(recipe: &DeterministicModelRecipe, engine: &str) -> String { + let status = recipe + .engine_statuses + .iter() + .find(|status| status.starts_with(&format!("{engine}:"))) + .map(std::string::String::as_str); + match status { + Some(status) if status.contains("unsupported_native_windows") => { + format!("{engine}: WSL/Linux only on Windows") + } + Some(status) + if status.split_once(':').is_some_and(|(_, rest)| { + rest.trim_start().starts_with("available") + || rest.trim_start().starts_with("adapter_available") + }) => + { + format!("{engine}: available") + } + Some(status) => status.to_owned(), + None if recipe.engines.iter().any(|candidate| candidate == engine) => { + format!("{engine}: listed") + } + None => format!("{engine}: not listed"), + } +} + +fn deterministic_available_engine_names(recipe: &DeterministicModelRecipe) -> Vec { + let engines = recipe + .engine_statuses + .iter() + .filter_map(|status| { + let (engine, rest) = status.split_once(':')?; + (rest.trim_start().starts_with("available") + || rest.trim_start().starts_with("adapter_available")) + .then_some(engine.to_owned()) + }) + .collect::>(); + if engines.is_empty() { + recipe.engines.clone() + } else { + engines + } +} + +pub(crate) fn deterministic_comfyui_tool_summary(tool_text: &str) -> Option { + if !tool_text.contains("ComfyUI") || !tool_text.contains("Running") { + return None; + } + let mut lines = Vec::new(); + match chat_tool_value(tool_text, "installed").as_deref() { + Some("yes") => lines.push(" ComfyUI: installed.".to_owned()), + Some("no") => lines.push(" ComfyUI: not installed.".to_owned()), + Some(value) => lines.push(format!(" ComfyUI: installed status is {value}.")), + None => {} + } + if let Some(status) = chat_tool_value(tool_text, "status") { + let running_text = if status.eq_ignore_ascii_case("running") { + "running".to_owned() + } else if status.eq_ignore_ascii_case("starting") { + "starting".to_owned() + } else { + format!("not running ({status})") + }; + lines.push(format!(" Running: {running_text}.")); + } + if let Some(models_path) = chat_tool_value(tool_text, "models path") { + lines.push(format!(" Models folder: {models_path}.")); + } + // A note about the install itself (an interrupted reinstall, which names + // the command that finishes it) sits between `status`'s `ComfyUI` heading + // and its `Running` section, whose own notes the `Running:` line sums up. + for line in tool_text + .lines() + .skip_while(|line| line.trim() != "ComfyUI") + .take_while(|line| line.trim() != "Running") + { + if let Some(note) = line.trim().strip_prefix("note:").map(str::trim) + && !note.is_empty() + { + lines.push(format!(" Note: {note}.")); + } + } + (!lines.is_empty()).then(|| lines.join("\n")) +} + +fn deterministic_services_tool_summary(tool_text: &str) -> Option { + if !tool_text.contains("Local Servers") + && !tool_text.contains("managed_services:") + && !tool_text.contains("services:") + { + return None; + } + let rows = parse_deterministic_service_rows(tool_text); + let mut lines = Vec::new(); + let live = rows + .iter() + .filter(|row| matches!(row.running_state.as_str(), "running" | "starting")) + .collect::>(); + if live.is_empty() { + lines.push(" Local model servers: none running under ROCm CLI.".to_owned()); + } else { + lines.push(format!( + " Local model servers running/starting: {}.", + live.len() + )); + for row in live.iter().take(4) { + lines.push(format!( + " {} {} ({}) at {}.", + row.engine, + row.model, + row.running_state, + empty_as_unknown(&row.endpoint) + )); + } + } + let past = rows + .iter() + .filter(|row| !matches!(row.running_state.as_str(), "running" | "starting")) + .take(3) + .map(|row| { + format!( + "{} {} {}", + empty_as_unknown(&row.engine), + empty_as_unknown(&row.model), + empty_as_unknown(&row.status) + ) + }) + .collect::>(); + if !past.is_empty() { + lines.push(format!(" Past/non-running records: {}.", past.join("; "))); + } + (!lines.is_empty()).then(|| lines.join("\n")) +} + +fn deterministic_port_status_tool_summary(tool_text: &str) -> Option { + if !tool_text.contains("port_status:") && !tool_text.contains("listening:") { + return None; + } + let port = chat_tool_value(tool_text, "port")?; + let listening = chat_tool_value(tool_text, "listening").unwrap_or_else(|| "unknown".to_owned()); + let mut lines = Vec::new(); + let state = match listening.as_str() { + "true" => "listening", + "false" => "not listening", + _ => "unknown", + }; + lines.push(format!(" Port {port}: {state}.")); + if let Some(hint) = chat_tool_value(tool_text, "hint") { + lines.push(format!(" Hint: {hint}.")); + } + if chat_tool_value(tool_text, "managed_service").as_deref() == Some("none") { + lines.push(" Managed service: none on that endpoint.".to_owned()); + } + Some(lines.join("\n")) +} + +fn deterministic_engine_inventory_tool_summary(tool_text: &str) -> Option { + if !tool_text.contains("Local model engines") { + return None; + } + let rows = parse_deterministic_engine_rows(tool_text); + if rows.is_empty() { + return None; + } + let mut lines = vec![" Engine runtimes:".to_owned()]; + for row in rows { + let mut line = format!( + " {}: {}", + friendly_engine_label(&row.engine), + empty_as_unknown(&row.runtime) + ); + if !row.note.trim().is_empty() { + let _ = write!(line, " ({})", row.note.trim()); + } + line.push('.'); + lines.push(line); + } + Some(lines.join("\n")) +} + +fn chat_tool_value(tool_text: &str, key: &str) -> Option { + let prefix = format!("{key}:"); + tool_text.lines().find_map(|line| { + line.trim() + .strip_prefix(&prefix) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_deterministic_model_recipes_reads_header_fields_and_notes() { + let recipes = parse_deterministic_model_recipes( + "\ + Qwen3-4B-Instruct-2507-GGUF aliases=[qwen, lemonade-qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] +lemonade: ready +warning: slow on this GPU + Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] +vllm: not_installed +", + ); + assert_eq!(recipes.len(), 2); + assert_eq!(recipes[0].canonical_id, "Qwen3-4B-Instruct-2507-GGUF"); + assert_eq!(recipes[0].min_gpu_mem_gib, Some(4)); + assert_eq!(recipes[0].engines, vec!["lemonade".to_owned()]); + assert_eq!( + recipes[0].engine_statuses, + vec!["lemonade: ready".to_owned()] + ); + assert_eq!(recipes[0].warnings, vec!["slow on this GPU".to_owned()]); + assert_eq!(recipes[1].canonical_id, "Qwen/Qwen3.5-4B"); + assert_eq!(recipes[1].min_gpu_mem_gib, Some(12)); + } + + #[test] + fn parse_deterministic_model_recipes_requires_aliases_bracket_in_header() { + let recipes = parse_deterministic_model_recipes( + "\ + NotAHeader task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] + Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] +", + ); + assert_eq!(recipes.len(), 1); + assert_eq!(recipes[0].canonical_id, "Qwen/Qwen3.5-4B"); + } + + #[test] + fn parse_deterministic_service_rows_reads_inline_key_value_form() { + let rows = parse_deterministic_service_rows( + " - service_id=svc-1 engine=vllm model=qwen status=ready running_state=running endpoint=127.0.0.1:8000", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].service_id, "svc-1"); + assert_eq!(rows[0].engine, "vllm"); + assert_eq!(rows[0].status, "ready"); + assert_eq!(rows[0].running_state, "running"); + assert_eq!(rows[0].endpoint, "127.0.0.1:8000"); + } + + #[test] + fn parse_deterministic_service_rows_derives_running_state_when_absent() { + let rows = parse_deterministic_service_rows( + "\ +- svc-2 +status: starting +engine: lemonade +model: qwen +endpoint: 127.0.0.1:8001 +", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].service_id, "svc-2"); + assert_eq!(rows[0].running_state, "starting"); + } + + #[test] + fn parse_deterministic_service_rows_derives_running_state_for_inline_form_when_absent() { + let rows = parse_deterministic_service_rows( + " - service_id=svc-3 engine=vllm model=qwen status=starting endpoint=127.0.0.1:8002", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].running_state, "starting"); + } + + #[test] + fn parse_deterministic_engine_rows_reads_runtime_and_note() { + let rows = parse_deterministic_engine_rows( + "\ +* lemonade + runtime: gfx120X-all + note: default engine +* vllm + runtime: gfx110X-all +", + ); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].engine, "lemonade"); + assert_eq!(rows[0].runtime, "gfx120X-all"); + assert_eq!(rows[0].note, "default engine"); + assert_eq!(rows[1].engine, "vllm"); + assert_eq!(rows[1].note, ""); + } + + #[test] + fn managed_service_running_state_maps_every_known_status() { + assert_eq!(managed_service_running_state("ready"), "running"); + assert_eq!(managed_service_running_state("running"), "running"); + assert_eq!(managed_service_running_state("starting"), "starting"); + assert_eq!(managed_service_running_state("recovering"), "starting"); + assert_eq!(managed_service_running_state("failed"), "not_running"); + assert_eq!(managed_service_running_state("stopped"), "not_running"); + assert_eq!(managed_service_running_state("anything-else"), "unknown"); + } + + #[test] + fn deterministic_rocm_tool_summary_interprets_managed_runtime_as_installed() { + let summary = deterministic_rocm_tool_summary( + "\ +examine: + driver_detail: AMD Radeon RX 9070 XT driver 32.0.23033.1002 + legacy_rocm_status: not_detected +runtime_state: + active_runtime_status: ready + active_runtime_root: D:\\ROCm\\therock_venvs + active_runtime_pip_cache_dir: D:\\ROCm\\therock_venvs\\pip-cache + active_runtime_version: 7.13.0a20260511 (build 2026-05-11) + active_runtime_family: gfx120X-all +", + ) + .expect("examine output should summarize"); + + assert!(summary.contains("GPU: AMD Radeon RX 9070 XT driver 32.0.23033.1002")); + assert!(summary.contains("ROCm/TheRock: installed and active for ROCm CLI")); + assert!(summary.contains("gfx120X-all")); + assert!(summary.contains(r"Install folder: D:\ROCm\therock_venvs")); + assert!(summary.contains(r"Downloads/cache: D:\ROCm\therock_venvs\pip-cache")); + assert!(summary.contains("no global legacy ROCm install was found")); + } + + #[test] + fn deterministic_model_tool_summary_identifies_low_vram_assistant() { + let summary = deterministic_model_tool_summary( + "\ +rocm_command: +model recipes + Qwen3-4B-Instruct-2507-GGUF aliases=[qwen, lemonade-qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + warning: recommended Lemonade GGUF assistant for ROCm machines + Qwen3-0.6B-GGUF aliases=[qwen-smoke, lemonade-tiny] task=chat dtype=gguf device=gpu_required min_gpu_mem=2 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + warning: tiny Lemonade GGUF smoke-test model; not the default assistant + Qwen/Qwen2.5-0.5B-Instruct aliases=[qwen-tiny] task=chat dtype=float16 device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] + engine_support: + vllm: adapter_available path=D:\\rocm\\rocm-engine-vllm.exe runtime_status=unsupported_native_windows reason=native Windows skipped; use WSL/Linux vLLM ROCm + meta-llama/Llama-3.2-3B-Instruct aliases=[llama] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=8 GiB engines=[lemonade, vllm] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + vllm: available path=D:\\rocm\\rocm-engine-vllm.exe +", + ) + .expect("model output should summarize"); + + assert!(summary.contains("Recommended local assistant: qwen")); + assert!(summary.contains("Qwen3-4B-Instruct-2507-GGUF")); + assert!(summary.contains("4 GiB")); + assert!(summary.contains("Tiny smoke test: qwen-smoke")); + assert!(summary.contains("Qwen3-0.6B-GGUF")); + assert!(summary.contains("8 GiB-class option: llama")); + assert!(summary.contains("lemonade, vllm")); + assert!(summary.contains("Qwen/Qwen3.5-4B asks for 12 GiB")); + assert!(summary.contains("Native Windows note")); + assert!(summary.contains("Run `rocm examine`")); + } +} diff --git a/apps/rocm/src/chat/turn.rs b/apps/rocm/src/chat/turn.rs new file mode 100644 index 0000000000..fcef7a8b39 --- /dev/null +++ b/apps/rocm/src/chat/turn.rs @@ -0,0 +1,1547 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The chat assistant's turn loop and transcript: one prompt in, the rendered +//! reply (and the approval request, if the assistant asked for a change) out. +//! `render_chat_prompt_result_with_progress` is the entry point. It builds the +//! assistant's system prompt (`ROCM_CHAT_TOOL_SYSTEM_PROMPT` with the compiled-in +//! assistant skill, grounded in the host by `rocm_chat_tool_system_prompt_for_host`), +//! asks the provider, runs the read-only tool calls the reply requests through +//! the bridge, summarizes their output, and renders the transcript +//! (`append_chat_tool_results`, the `local_*` and display-label helpers). It +//! owns the turn's result types, `ChatPromptResult` and `ChatToolRunResult`. +//! +//! A child of the `chat` module and the top of its call graph: it calls into +//! the prompt-intent heuristics (`chat/intent.rs`), the MCP bridge +//! (`chat/mcp_bridge.rs`), the summaries (`chat/summaries.rs`) and +//! `tool_policy.rs`, and none of them calls back into it. `chat.rs` reads +//! `render_chat_prompt_result_with_progress` from here, and re-exports +//! `rocm_chat_tool_system_prompt_for_host` for `dash.rs`. + +use super::intent::{ + fallback_engine_list_tool_call, fallback_rocm_tool_call_for_prompt, + fallback_services_list_tool_call, install_sdk_without_prefix_chat_approval, + latest_user_chat_message, prompt_asks_engine_install_state, prompt_asks_running_or_status, + prompt_can_use_read_only_without_local_assistant, prompt_mentions_serving_engine_or_service, +}; +use super::mcp_bridge::{loopback_host_key, mcp_tool_result_text, run_chat_read_only_tool}; +use super::summaries::{deterministic_chat_tool_summary, deterministic_summary_can_stand_alone}; +use crate::engine_runtime_select::service_model_names_match; +use crate::freeform_planner::format_structured_tool_call; +use crate::managed_service_status::load_managed_services; +use crate::rocm_argv::chat_cli_arg_value; +use crate::tool_policy::{ + ChatToolApprovalRequest, chat_tool_approval_request, chat_tool_call_is_read_only, json_string, + normalized_chat_rocm_command_args, rocm_chat_tool_requested_command, validate_chat_tool_call, +}; +use crate::{ChatInferenceParams, chat_host_facts, providers}; +use anyhow::Result; +use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST}; +use std::fmt::Write as _; + +fn mcp_tool_result_is_error(value: &serde_json::Value) -> bool { + value + .get("isError") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) +} + +fn chat_tool_display_label(name: &str) -> String { + match name { + "examine" => "Checked this computer".to_owned(), + "gpu_snapshot" => "Checked GPU status".to_owned(), + "engines" => "Checked local engines".to_owned(), + "services" => "Checked model servers".to_owned(), + "bridge_snapshot" => "Checked ROCm state".to_owned(), + "service_logs" => "Read server logs".to_owned(), + "automations" => "Checked automations".to_owned(), + "natural_language_plan" => "Planned ROCm request".to_owned(), + "path_exists" => "Checked folder path".to_owned(), + "port_status" => "Checked local port".to_owned(), + "update_check" => "Checked for ROCm updates".to_owned(), + "install_sdk_dry_run" => "Previewed ROCm install".to_owned(), + "install_sdk" => "Install ROCm".to_owned(), + "install_engine" => "Install engine".to_owned(), + "launch_server" => "Start local model server".to_owned(), + "stop_server" => "Stop local model server".to_owned(), + "watcher_enable" => "Enable automation".to_owned(), + "watcher_disable" => "Disable automation".to_owned(), + other => other.replace('_', " "), + } +} + +fn chat_tool_call_display_label(call: &providers::ChatToolCall) -> String { + if call.name != "rocm_command" { + return chat_tool_display_label(&call.name); + } + let Some(args) = normalized_chat_rocm_command_args(call).ok() else { + return "rocm command".to_owned(); + }; + match args.as_slice() { + [command] if command.eq_ignore_ascii_case("examine") => "Checked this computer".to_owned(), + [command] + if command.eq_ignore_ascii_case("model") || command.eq_ignore_ascii_case("models") => + { + "Checked model recipes".to_owned() + } + [command, subcommand] + if command.eq_ignore_ascii_case("engines") + && subcommand.eq_ignore_ascii_case("list") => + { + "Checked local engines".to_owned() + } + [command] if command.eq_ignore_ascii_case("services") => "Checked model servers".to_owned(), + [command, subcommand, rest @ ..] + if command.eq_ignore_ascii_case("services") + && subcommand.eq_ignore_ascii_case("list") + && rest + .iter() + .all(|arg| matches!(arg.as_str(), "-a" | "--all")) => + { + "Checked model servers".to_owned() + } + [command] if command.eq_ignore_ascii_case("comfyui") => "Checked ComfyUI".to_owned(), + [command, subcommand] + if command.eq_ignore_ascii_case("comfyui") + && subcommand.eq_ignore_ascii_case("status") => + { + "Checked ComfyUI".to_owned() + } + [command, subcommand, ..] + if command.eq_ignore_ascii_case("comfyui") + && matches!(subcommand.to_ascii_lowercase().as_str(), "logs" | "log") => + { + "Read ComfyUI logs".to_owned() + } + [command, subcommand] + if command.eq_ignore_ascii_case("config") + && subcommand.eq_ignore_ascii_case("show") => + { + "Checked ROCm config".to_owned() + } + _ => "rocm command".to_owned(), + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) struct ChatPromptResult { + pub rendered: String, + pub approval: Option, +} + +struct ChatToolRunResult { + pub(crate) approval: Option, + pub(crate) follow_up_text: String, + pub(crate) ran_read_only_tool: bool, + pub(crate) read_only_tool_error: bool, + pub(crate) needs_install_folder: bool, +} + +/// Convenience wrapper used by tests that do not exercise sampling controls. +#[cfg(test)] +fn render_chat_prompt_result( + paths: &AppPaths, + provider: &str, + model: Option<&str>, + prompt: &str, + rocm_tools: bool, +) -> Result { + render_chat_prompt_result_with_progress( + paths, + provider, + model, + prompt, + rocm_tools, + ChatInferenceParams::default(), + None, + ) +} + +pub(super) fn render_chat_prompt_result_with_progress( + paths: &AppPaths, + provider: &str, + model: Option<&str>, + prompt: &str, + rocm_tools: bool, + params: ChatInferenceParams, + progress: Option<&mut dyn FnMut(String)>, +) -> Result { + let mut progress = progress; + let user_prompt = latest_user_chat_message(prompt); + let assistant_model = local_rocm_tools_assistant_model(provider, rocm_tools).or(model); + let service_needed_model = if local_rocm_tools_assistant_model(provider, rocm_tools).is_some() { + None + } else { + model + }; + if rocm_tools && let Some(approval) = install_sdk_without_prefix_chat_approval(user_prompt) { + report_chat_tool_progress(&mut progress, "Asking the assistant."); + report_chat_tool_progress(&mut progress, "Review needed: Install ROCm"); + report_chat_tool_progress(&mut progress, "Waiting for the ROCm install folder."); + return Ok(ChatPromptResult { + rendered: render_install_sdk_folder_needed_chat_text(user_prompt, &approval.args), + approval: Some(approval), + }); + } + let mut messages = Vec::new(); + if rocm_tools { + messages.push(providers::ChatMessage { + role: "system".to_owned(), + content: rocm_chat_tool_system_prompt_for_host(Some(paths)), + }); + } + messages.push(providers::ChatMessage { + role: "user".to_owned(), + content: prompt.to_owned(), + }); + let mut response = if rocm_tools { + report_chat_tool_progress(&mut progress, "Asking the assistant."); + if let Some(call) = deterministic_mutating_tool_call_for_prompt(user_prompt)? { + report_chat_tool_progress(&mut progress, "Preparing a review card."); + providers::ChatResponse { + provider: provider.to_owned(), + model: assistant_model.unwrap_or("local").to_owned(), + content: deterministic_mutating_tool_intro(&call), + tool_calls: vec![call], + } + } else if !local_assistant_service_ready_for_chat(paths, assistant_model) + && let Some(response) = + local_read_only_fallback_response_for_prompt(provider, assistant_model, user_prompt) + { + report_chat_tool_progress(&mut progress, "Running ROCm status check."); + response + } else { + match providers::provider_chat( + paths, + provider, + &providers::ChatRequest { + model: assistant_model.map(str::to_owned), + messages: messages.clone(), + max_tokens: params.max_tokens, + temperature: params.temperature, + top_p: params.top_p, + rocm_tools, + }, + ) { + Ok(response) => response, + Err(error) + if provider == "local" && local_provider_missing_service_error(&error) => + { + if let Some(response) = local_read_only_fallback_response_for_prompt( + provider, + assistant_model, + user_prompt, + ) { + report_chat_tool_progress(&mut progress, "Running ROCm status check."); + response + } else { + return Ok(ChatPromptResult { + rendered: local_chat_service_needed_text( + service_needed_model, + user_prompt, + rocm_tools, + ), + approval: None, + }); + } + } + Err(error) => return Err(error), + } + } + } else { + report_chat_tool_progress(&mut progress, "Asking the assistant."); + match providers::provider_chat( + paths, + provider, + &providers::ChatRequest { + model: assistant_model.map(str::to_owned), + messages: messages.clone(), + max_tokens: params.max_tokens, + temperature: params.temperature, + top_p: params.top_p, + rocm_tools, + }, + ) { + Ok(response) => response, + Err(error) if provider == "local" && local_provider_missing_service_error(&error) => { + return Ok(ChatPromptResult { + rendered: local_chat_service_needed_text( + service_needed_model, + user_prompt, + rocm_tools, + ), + approval: None, + }); + } + Err(error) => return Err(error), + } + }; + let fallback_tool_call = if rocm_tools && response.tool_calls.is_empty() { + fallback_rocm_tool_call_for_prompt(user_prompt) + } else { + None + }; + let fallback_tool_call_used = fallback_tool_call.is_some(); + if let Some(call) = fallback_tool_call { + response.tool_calls.push(call); + } + if rocm_tools + && let Some(call) = + supplemental_read_only_tool_call_for_prompt(user_prompt, &response.tool_calls) + { + response.tool_calls.push(call); + } + let mut output = String::new(); + let _ = writeln!(output, "chat response"); + let _ = writeln!(output, " provider: {}", response.provider); + let _ = writeln!(output, " model: {}", response.model); + let _ = writeln!( + output, + " rocm tools: {}", + if rocm_tools { "enabled" } else { "off" } + ); + let _ = writeln!(output); + let initial_content_is_intermediate = fallback_tool_call_used + || local_tool_call_content_is_intermediate(provider, rocm_tools, &response); + let initial_content = visible_chat_content(&response.content); + if initial_content_is_intermediate { + let _ = writeln!(output, "Assistant is preparing the next step."); + } else if !initial_content.trim().is_empty() { + let _ = writeln!(output, "{initial_content}"); + } + let tool_result = if rocm_tools { + let explanation = (!fallback_tool_call_used && !initial_content.trim().is_empty()) + .then_some(initial_content.as_str()); + append_chat_tool_results(paths, &response, &mut output, explanation, &mut progress)? + } else { + ChatToolRunResult { + approval: None, + follow_up_text: String::new(), + ran_read_only_tool: false, + read_only_tool_error: false, + needs_install_folder: false, + } + }; + let deterministic_summary = deterministic_chat_tool_summary(&tool_result.follow_up_text); + if let Some(summary) = deterministic_summary.as_deref() { + let _ = writeln!(output); + let _ = writeln!(output, "ROCm CLI summary"); + let _ = writeln!(output, "{summary}"); + } + let follow_up_blocking_summary = deterministic_summary + .as_deref() + .filter(|_| deterministic_summary_can_stand_alone(&tool_result.follow_up_text)); + if should_request_local_tool_follow_up(provider, &tool_result, follow_up_blocking_summary) { + let follow_up_messages = vec![ + providers::ChatMessage { + role: "system".to_owned(), + content: "You are ROCm CLI's local assistant. ROCm tools have already been checked for this turn. Use the supplied tool results for local facts about this machine. For service results, ready/running means running, starting/recovering means starting, failed/stopped means not running, and no matching service row means ROCm CLI is not managing that service as running. If the tool results do not contain enough information to answer the whole question, say what is known from the results and then answer the rest normally from your own model knowledge. Do not request another tool call. Do not invent local paths, GPU names, versions, or install state that are not shown in the tool results. Keep the answer concise.".to_owned(), + }, + providers::ChatMessage { + role: "user".to_owned(), + content: format!( + "Original request:\n{}\n\nROCm tool results:\n{}\n\nAnswer the original request. Use the ROCm tool results for this machine's facts; if they are incomplete, answer the remaining part generally.", + user_prompt.trim(), + tool_result.follow_up_text.trim() + ), + }, + ]; + let follow_up = providers::provider_chat( + paths, + provider, + &providers::ChatRequest { + model: assistant_model.map(str::to_owned), + messages: follow_up_messages, + max_tokens: params.max_tokens.or(Some(256)), + temperature: params.temperature, + top_p: params.top_p, + rocm_tools: false, + }, + )?; + if follow_up.tool_calls.is_empty() { + let follow_up_content = visible_chat_content(&follow_up.content); + if local_follow_up_content_is_final(&follow_up) && !follow_up_content.trim().is_empty() + { + let _ = writeln!(output); + let _ = writeln!(output, "Assistant after ROCm checks"); + let _ = writeln!(output, "{}", follow_up_content.trim()); + } + } else { + let _ = writeln!(output); + let _ = writeln!( + output, + "Assistant asked for another ROCm check after the first tool results." + ); + let _ = writeln!( + output, + "rocm-cli stopped there so the answer is not based on another guess. Nothing was changed." + ); + } + } else if tool_result.read_only_tool_error { + let _ = writeln!(output); + let _ = writeln!( + output, + "Assistant did not answer from this result because a ROCm check reported an error." + ); + let _ = writeln!(output, "Nothing was changed."); + } else if tool_result.needs_install_folder { + let _ = writeln!(output); + let _ = writeln!(output, "Assistant after ROCm checks"); + let _ = writeln!( + output, + "I can install ROCm/TheRock, but I need the install folder first. Type the folder you want to use, for example D:\\ROCm\\therock_venvs. Nothing will download until ROCm CLI shows a review card and you approve it." + ); + } + if tool_result.approval.is_none() + && tool_result.ran_read_only_tool + && !output.contains("Nothing was changed.") + { + let _ = writeln!(output); + let _ = writeln!(output, "Nothing was changed."); + } + Ok(ChatPromptResult { + rendered: output, + approval: tool_result.approval, + }) +} + +fn report_chat_tool_progress(progress: &mut Option<&mut dyn FnMut(String)>, message: &str) { + if let Some(progress) = progress.as_deref_mut() { + progress(message.to_owned()); + } +} + +fn deterministic_mutating_tool_call_for_prompt( + prompt: &str, +) -> Result> { + let Some(call) = fallback_rocm_tool_call_for_prompt(prompt) else { + return Ok(None); + }; + validate_chat_tool_call(&call)?; + if chat_tool_call_is_read_only(&call) { + Ok(None) + } else { + Ok(Some(call)) + } +} + +fn supplemental_read_only_tool_call_for_prompt( + prompt: &str, + existing_calls: &[providers::ChatToolCall], +) -> Option { + let normalized = latest_user_chat_message(prompt).to_ascii_lowercase(); + if prompt_mentions_serving_engine_or_service(&normalized) + && prompt_asks_running_or_status(&normalized) + && prompt_asks_engine_install_state(&normalized) + { + for call in [ + fallback_services_list_tool_call(), + fallback_engine_list_tool_call(), + ] { + if !chat_tool_calls_include_equivalent(existing_calls, &call) { + return Some(call); + } + } + return None; + } + let call = fallback_rocm_tool_call_for_prompt(prompt)?; + if !chat_tool_call_is_read_only(&call) + || chat_tool_calls_include_equivalent(existing_calls, &call) + { + return None; + } + Some(call) +} + +fn chat_tool_calls_include_equivalent( + existing_calls: &[providers::ChatToolCall], + required: &providers::ChatToolCall, +) -> bool { + existing_calls.iter().any(|existing| { + if existing.name != required.name { + return false; + } + if required.name == "rocm_command" { + return normalized_chat_rocm_command_args(existing).ok() + == normalized_chat_rocm_command_args(required).ok(); + } + if required.name == "port_status" { + return chat_port_status_tool_calls_equivalent(existing, required); + } + existing.arguments == required.arguments + }) +} + +fn chat_port_status_tool_calls_equivalent( + left: &providers::ChatToolCall, + right: &providers::ChatToolCall, +) -> bool { + let Some(left_object) = left.arguments.as_object() else { + return false; + }; + let Some(right_object) = right.arguments.as_object() else { + return false; + }; + let left_port = left_object.get("port").and_then(serde_json::Value::as_u64); + let right_port = right_object.get("port").and_then(serde_json::Value::as_u64); + if left_port != right_port { + return false; + } + let left_host = + json_string(left_object, "host").unwrap_or_else(|| DEFAULT_LOCAL_HOST.to_owned()); + let right_host = + json_string(right_object, "host").unwrap_or_else(|| DEFAULT_LOCAL_HOST.to_owned()); + loopback_host_key(&left_host) == loopback_host_key(&right_host) +} + +fn deterministic_mutating_tool_intro(call: &providers::ChatToolCall) -> String { + match chat_tool_approval_request(call, None) { + Ok(approval) if approval.pending_title == "Install ROCm" => { + "I can install ROCm/TheRock into the folder you chose. Review the card before anything downloads or changes." + .to_owned() + } + Ok(approval) if approval.pending_title == "Install ComfyUI" => { + "I can install ComfyUI into ROCm CLI's managed app folder. Review the card before anything downloads or changes." + .to_owned() + } + Ok(approval) if approval.pending_title == "Start ComfyUI" => { + "I can start ComfyUI for you. Review the card before ROCm CLI launches it." + .to_owned() + } + Ok(approval) if approval.pending_title == "Start local model server" => { + "I can start the recommended local model server on the GPU. Review the card before ROCm CLI launches it." + .to_owned() + } + Ok(approval) => format!( + "I can prepare this ROCm change: {}. Review the card before anything runs.", + approval.pending_title + ), + Err(_) => { + "I can prepare this ROCm change. Review the card before anything runs.".to_owned() + } + } +} + +fn local_tool_call_content_is_intermediate( + provider: &str, + rocm_tools: bool, + response: &providers::ChatResponse, +) -> bool { + provider == "local" + && rocm_tools + && !response.tool_calls.is_empty() + && !response.content.trim().is_empty() +} + +fn local_follow_up_content_is_final(response: &providers::ChatResponse) -> bool { + response.tool_calls.is_empty() && !response.content.trim().is_empty() +} + +fn local_read_only_fallback_response_for_prompt( + provider: &str, + assistant_model: Option<&str>, + prompt: &str, +) -> Option { + if provider != "local" || !prompt_can_use_read_only_without_local_assistant(prompt) { + return None; + } + let call = fallback_rocm_tool_call_for_prompt(prompt)?; + if !chat_tool_call_is_read_only(&call) { + return None; + } + Some(providers::ChatResponse { + provider: provider.to_owned(), + model: assistant_model.unwrap_or("local").to_owned(), + content: String::new(), + tool_calls: vec![call], + }) +} + +fn local_assistant_service_ready_for_chat(paths: &AppPaths, assistant_model: Option<&str>) -> bool { + let model = assistant_model.unwrap_or(providers::BUILTIN_ASSISTANT_MODEL_ID); + load_managed_services(paths).is_ok_and(|records| { + records.iter().any(|record| { + matches!(record.status.as_str(), "ready" | "running") + && (service_model_names_match(&record.canonical_model_id, model) + || service_model_names_match(&record.model_ref, model)) + }) + }) +} + +fn render_install_sdk_folder_needed_chat_text(prompt: &str, args: &[String]) -> String { + let mut output = String::new(); + let _ = writeln!(output, "ROCm install"); + let _ = writeln!(output); + let _ = writeln!(output, "I can install ROCm/TheRock for you."); + let _ = writeln!( + output, + "First choose the folder where ROCm CLI should put the Python environment." + ); + let _ = writeln!( + output, + "Nothing will download or change until you review and approve the install." + ); + if let Some(date) = chat_cli_arg_value(args, "--build-date") { + let _ = writeln!(output); + let _ = writeln!(output, "Requested build date: {date}"); + } + if let Some(version) = chat_cli_arg_value(args, "--version") { + let _ = writeln!(output); + let _ = writeln!(output, "Requested version: {version}"); + } + let _ = writeln!(output); + let _ = writeln!(output, "Your request"); + let _ = writeln!(output, " {}", prompt.trim()); + output.trim_end().to_owned() +} + +fn visible_chat_content(content: &str) -> String { + let mut output = String::new(); + let mut remaining = content; + while let Some(start) = find_ascii_case_insensitive(remaining, "") { + output.push_str(&remaining[..start]); + let after_start = &remaining[start + "".len()..]; + let Some(end) = find_ascii_case_insensitive(after_start, "") else { + remaining = ""; + break; + }; + remaining = &after_start[end + "".len()..]; + } + output.push_str(remaining); + output.trim().to_owned() +} + +fn find_ascii_case_insensitive(haystack: &str, needle: &str) -> Option { + let needle = needle.as_bytes(); + haystack + .as_bytes() + .windows(needle.len()) + .position(|window| window.eq_ignore_ascii_case(needle)) +} + +/// The host-independent half of the assistant prompt. +/// +/// Statements that are only true on *some* hosts do not belong here — they used +/// to, and a WSL user was told "on native Windows, vLLM is skipped" while vLLM +/// was in fact their supported path. Anything host-dependent now comes from +/// [`chat_host_facts::HostFacts`], which knows which machine it is describing. +const ROCM_CHAT_TOOL_SYSTEM_PROMPT: &str = "You are ROCm CLI's local assistant. Speak in simple English for non-technical users. Use the provided ROCm tools when you need to inspect this machine, preview setup, read service logs, check updates, inspect automations, install or start ROCm-managed apps, or request ROCm/TheRock, config, engine, app, and local model server changes. For simple greetings or thanks like hello, hi, hey, ok, or thank you, reply normally; do not inspect ROCm, do not call tools, and do not launch or propose a model server. Tool-use rules: inspect first with read-only tools; call rocm_command only with argv-style args and no shell text; use natural_language_plan for ROCm requests that do not fit another read-only tool; ask for a mutating tool call only after explaining why it is needed; summarize tool results after they are returned. Read-only tools may run immediately. Tools that install, launch, stop, delete, or change state require user approval; request rocm_command and explain why. For 'is X running?', 'what is running?', status, or port questions, inspect before answering and do not start, stop, install, or serve anything. For ComfyUI or port 8188 use [\"comfyui\",\"status\"] or port_status. For vLLM, Lemonade, AIM, qwen, or local model servers use [\"services\",\"list\",\"--all\"] for running state and [\"engines\",\"list\"] for installed/available engine state. Treat ready/running as running, starting/recovering as starting, failed/stopped as not running, and no matching record as unknown or not managed by ROCm CLI. Interpret Examine carefully: active_runtime_status=ready means ROCm CLI has an active managed TheRock/ROCm runtime; legacy_rocm_status=not_detected only means no global system ROCm install was found. If active_runtime_status=ready, tell the user ROCm/TheRock is installed and active for ROCm CLI. For 'is TheRock installed', 'is ROCm installed', or 'which GPU is on this machine', use examine or gpu_snapshot before answering. For 'how do I setup TheRock' or install/setup requests, guide the user to choose an install folder first; do not answer with only a status check. For 'which LLMs can this machine support', use rocm_command args [\"model\"] or natural_language_plan before answering. For TheRock installs, always let the user choose the install folder. If the user names a folder or prefix, preserve that exact folder with [\"--prefix\",\"PATH\"]; you may call path_exists first to check whether that user-provided folder or its parent exists. If the user asks you to install TheRock/ROCm but has not named a folder, ask for the folder or let the guided setup folder picker collect it; do not invent a hidden default folder and do not request an install command without --prefix. Use rocm_command args [\"install\",\"sdk\",\"--channel\",\"release\",\"--format\",\"wheel\",\"--prefix\",\"PATH\"] only when the user asks you to install it and a folder is known; for a requested build date add [\"--build-date\",\"YYYY-MM-DD\"] and for a requested exact version add [\"--version\",\"VERSION\"]. For config changes, inspect with [\"config\",\"show\"] first when useful, then request config subcommands such as [\"config\",\"set-default-engine\",\"lemonade\"], [\"config\",\"set-default-runtime\",\"RUNTIME_KEY\"], or [\"config\",\"set-telemetry\",\"local\"] only after explaining why. For ComfyUI, use rocm_command with args like [\"comfyui\",\"status\"], [\"comfyui\",\"logs\"], [\"comfyui\",\"install\"], [\"comfyui\",\"start\"], or [\"comfyui\",\"stop\"]. First-time setup is the same thing as bootstrap in ROCm CLI; it is a deterministic ROCm setup flow, not a separate model chat. The built-in local assistant is fixed to qwen, which maps to Qwen3-4B-Instruct-2507-GGUF served by Lemonade with gpu_required. vLLM, Lemonade, and AIM (Docker-based) are the general serving engines; inspect or manage them when the user asks about general model serving, but do not switch the built-in assistant away from Lemonade. Use qwen-smoke only for a quick server smoke test. For vLLM management, inspect engines first and use [\"engines\",\"install\",\"vllm\"] or [\"serve\",\"MODEL\",\"--engine\",\"vllm\",\"--device\",\"gpu_required\",\"--managed\"] only where the host supports it. Do not invent shell commands and do not request CPU fallback."; + +const ROCM_CHAT_TOOL_SKILL: &str = include_str!("../../../../skills/rocm-cli-assistant/SKILL.md"); + +fn rocm_chat_tool_system_prompt() -> String { + format!("{ROCM_CHAT_TOOL_SYSTEM_PROMPT}\n\nROCm CLI assistant skill:\n{ROCM_CHAT_TOOL_SKILL}") +} + +/// The assistant prompt grounded in the machine it will answer for. +/// +/// The single composition point: both chat surfaces (`rocm chat --prompt +/// --tools` in this bin, and the dashboard chat via +/// [`crate::dash::resolved_args`]) send this, so neither can drift into +/// answering platform questions from pretraining alone. +pub(crate) fn rocm_chat_tool_system_prompt_for_host(paths: Option<&AppPaths>) -> String { + let facts = chat_host_facts::HostFacts::detect(paths); + format!("{}\n\n{}", rocm_chat_tool_system_prompt(), facts.render()) +} + +fn local_provider_missing_service_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .to_string() + .contains("local provider has no ready managed service") + }) +} + +fn local_rocm_tools_assistant_model(provider: &str, rocm_tools: bool) -> Option<&'static str> { + (provider == "local" && rocm_tools).then_some(providers::BUILTIN_ASSISTANT_MODEL_ID) +} + +fn local_chat_service_needed_text(model: Option<&str>, prompt: &str, rocm_tools: bool) -> String { + let mut output = String::new(); + let _ = writeln!(output, "No local assistant is running yet."); + let _ = writeln!(output); + let _ = writeln!( + output, + "To use LLM-assisted ROCm commands, start a ROCm GPU local model server first." + ); + let _ = writeln!( + output, + "First-time ROCm setup does not need an LLM; run `rocm` and use Set Up ROCm for that." + ); + let _ = writeln!(output); + let _ = writeln!( + output, + "Recommended path:\n run `rocm`, choose Start a local model, use the recommended assistant model, then start it" + ); + let _ = writeln!(output); + if let Some(model) = model.filter(|value| !value.trim().is_empty()) { + let args = vec![ + "serve".to_owned(), + model.to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ]; + let _ = writeln!( + output, + "Advanced manual command for the selected model:\n {}", + format_structured_tool_call("rocm", &args) + ); + } else { + let example_args = vec![ + "serve".to_owned(), + providers::LEMONADE_ASSISTANT_MODEL_ID.to_owned(), + "--engine".to_owned(), + "lemonade".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ]; + let _ = writeln!( + output, + "Advanced manual command for the recommended assistant model:\n {}", + format_structured_tool_call("rocm", &example_args) + ); + } + let mut chat_args = vec!["chat".to_owned()]; + if rocm_tools { + chat_args.push("--tools".to_owned()); + } + chat_args.push("--provider".to_owned()); + chat_args.push("local".to_owned()); + if let Some(model) = model.filter(|value| !value.trim().is_empty()) { + chat_args.push("--model".to_owned()); + chat_args.push(model.to_owned()); + } + if !prompt.trim().is_empty() { + chat_args.push("--prompt".to_owned()); + chat_args.push(prompt.to_owned()); + } + let _ = writeln!(output); + let _ = writeln!( + output, + "After the server is ready, run:\n {}", + format_structured_tool_call("rocm", &chat_args) + ); + let _ = writeln!(output); + let _ = writeln!( + output, + "For the guided flow, run `rocm`, choose Start a local model, start a model, then choose Chat." + ); + let _ = writeln!(output, "Nothing was changed."); + output.trim_end().to_owned() +} + +fn append_chat_tool_results( + paths: &AppPaths, + response: &providers::ChatResponse, + output: &mut String, + assistant_explanation: Option<&str>, + progress: &mut Option<&mut dyn FnMut(String)>, +) -> Result { + if response.tool_calls.is_empty() { + let _ = writeln!(output, "ROCm checks used"); + let _ = writeln!(output, " none requested"); + return Ok(ChatToolRunResult { + approval: None, + follow_up_text: String::new(), + ran_read_only_tool: false, + read_only_tool_error: false, + needs_install_folder: false, + }); + } + + let _ = writeln!(output, "ROCm checks used"); + let mut approval = None; + let mut follow_up_text = String::new(); + let mut ran_read_only_tool = false; + let mut read_only_tool_error = false; + let mut needs_install_folder = false; + for call in &response.tool_calls { + if chat_tool_call_requests_therock_install_without_prefix(call) { + needs_install_folder = true; + report_chat_tool_progress(progress, "Waiting for the ROCm install folder."); + let _ = writeln!(output, " Install ROCm: needs install folder"); + let _ = writeln!( + output, + " not run: choose an install folder before the review card" + ); + continue; + } + validate_chat_tool_call(call)?; + let label = chat_tool_call_display_label(call); + if chat_tool_call_is_read_only(call) { + report_chat_tool_progress(progress, &format!("Running ROCm check: {label}.")); + let result = run_chat_read_only_tool(paths, call)?; + ran_read_only_tool = true; + let is_error = mcp_tool_result_is_error(&result); + read_only_tool_error |= is_error; + report_chat_tool_progress( + progress, + &format!( + "ROCm check finished: {label} ({})", + chat_read_only_tool_status_label(is_error) + ), + ); + let _ = writeln!( + output, + " {}: {}", + label, + chat_read_only_tool_status_label(is_error) + ); + let result_text = mcp_tool_result_text(&result); + let _ = writeln!(follow_up_text, "{}:", call.name); + let _ = writeln!(follow_up_text, "{result_text}"); + for line in result_text.lines() { + let _ = writeln!(output, " {line}"); + } + } else { + report_chat_tool_progress(progress, &format!("Review needed: {label}.")); + let _ = writeln!(output, " {label}: needs your review"); + let _ = writeln!( + output, + " not run: review the approval card before anything runs" + ); + if let Some(command) = rocm_chat_tool_requested_command(call) { + let _ = writeln!(output, " advanced manual command: {command}"); + } + if approval.is_none() { + approval = Some(chat_tool_approval_request(call, assistant_explanation)?); + } + } + } + Ok(ChatToolRunResult { + approval, + follow_up_text, + ran_read_only_tool, + read_only_tool_error, + needs_install_folder, + }) +} + +fn chat_tool_call_requests_therock_install_without_prefix(call: &providers::ChatToolCall) -> bool { + let Some(object) = call.arguments.as_object() else { + return false; + }; + match call.name.as_str() { + "install_sdk" => json_string(object, "prefix").is_none(), + "rocm_command" => rocm_command_args_install_sdk_without_prefix(object), + _ => false, + } +} + +fn rocm_command_args_install_sdk_without_prefix( + object: &serde_json::Map, +) -> bool { + let Some(args) = object.get("args").and_then(serde_json::Value::as_array) else { + return false; + }; + let args = args + .iter() + .filter_map(serde_json::Value::as_str) + .map(str::to_owned) + .collect::>(); + args.first() + .is_some_and(|arg| arg.eq_ignore_ascii_case("install")) + && args + .get(1) + .is_some_and(|arg| arg.eq_ignore_ascii_case("sdk")) + && chat_cli_arg_value(&args, "--prefix").is_none() +} + +fn should_request_local_tool_follow_up( + provider: &str, + tool_result: &ChatToolRunResult, + deterministic_summary: Option<&str>, +) -> bool { + provider == "local" + && deterministic_summary.is_none() + && tool_result.approval.is_none() + && tool_result.ran_read_only_tool + && !tool_result.read_only_tool_error + && !tool_result.follow_up_text.trim().is_empty() +} + +const fn chat_read_only_tool_status_label(is_error: bool) -> &'static str { + if is_error { + "reported an error" + } else { + "done" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chat::summaries::deterministic_rocm_tool_summary; + use crate::tests::test_paths; + + fn rocm_command_call(args: &[&str]) -> providers::ChatToolCall { + providers::ChatToolCall { + id: Some("test".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": args }), + } + } + + #[test] + fn mcp_tool_result_is_error_reads_the_is_error_flag() { + assert!(mcp_tool_result_is_error( + &serde_json::json!({"isError": true}) + )); + assert!(!mcp_tool_result_is_error( + &serde_json::json!({"isError": false}) + )); + assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); + } + + #[test] + fn chat_tool_display_label_maps_known_tools_and_humanizes_others() { + assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); + assert_eq!( + chat_tool_display_label("gpu_snapshot"), + "Checked GPU status" + ); + assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); + assert_eq!( + chat_tool_display_label("some_unmapped_tool"), + "some unmapped tool" + ); + } + + #[test] + fn chat_tool_call_display_label_delegates_for_non_rocm_command_tools() { + let call = providers::ChatToolCall { + id: None, + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }; + assert_eq!(chat_tool_call_display_label(&call), "Checked this computer"); + } + + #[test] + fn chat_tool_call_display_label_recognizes_rocm_command_shapes() { + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["examine"])), + "Checked this computer" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["models"])), + "Checked model recipes" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["model"])), + "Checked model recipes" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["ENGINES", "List"])), + "Checked local engines" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["services"])), + "Checked model servers" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["services", "list", "-a"])), + "Checked model servers" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["services", "list", "--all"])), + "Checked model servers" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["comfyui"])), + "Checked ComfyUI" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["comfyui", "status"])), + "Checked ComfyUI" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["comfyui", "logs", "-n", "50"])), + "Read ComfyUI logs" + ); + assert_eq!( + chat_tool_call_display_label(&rocm_command_call(&["config", "show"])), + "Checked ROCm config" + ); + } + + #[test] + fn chat_tool_call_display_label_falls_back_for_unparseable_args() { + let call = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({"args": []}), + }; + assert_eq!(chat_tool_call_display_label(&call), "rocm command"); + } + + #[test] + fn local_assistant_prompt_instructions_cover_core_support_questions() { + let prompt = rocm_chat_tool_system_prompt(); + for expected in [ + "is TheRock installed", + "which GPU is on this machine", + "active_runtime_status=ready", + "legacy_rocm_status=not_detected", + "[\"model\"]", + "--build-date", + "always let the user choose the install folder", + "--prefix", + "do not invent a hidden default folder", + "config", + "comfyui", + "First-time setup is the same thing as bootstrap", + "vllm", + "Qwen3-4B-Instruct-2507-GGUF", + "fixed to qwen", + "served by Lemonade", + "port_status", + "[\"services\",\"list\",\"--all\"]", + "qwen-smoke", + "Do not invent shell commands", + "ROCm CLI Assistant Skill", + "Treat `localhost` and `127.0.0.1` as the same loopback endpoint", + ] { + assert!( + prompt.contains(expected), + "system prompt should mention {expected}" + ); + } + } + + /// The reported bug: asked "What can ROCm do on Windows?", the assistant + /// answered that ROCm is Windows-incompatible and suggested CUDA/DirectX — + /// because nothing ever told it which machine it was on. The prompt the CLI + /// actually sends must carry the host. + #[test] + fn assistant_prompt_states_the_host_it_is_answering_for() { + let prompt = rocm_chat_tool_system_prompt_for_host(None); + + // The tool-use rules survive the composition (this is the same prompt, + // grounded — not a replacement for it). + assert!( + prompt.contains("You are ROCm CLI's local assistant"), + "the ROCm tool-use prompt must still be there:\n{prompt}" + ); + + // …and it now names this machine's OS and GPU state. + let expected_os = if cfg!(windows) { + "- Operating system: Windows" + } else { + "- Operating system: Linux" + }; + assert!( + prompt.contains(expected_os), + "the prompt must state this machine's operating system:\n{prompt}" + ); + assert!( + prompt.contains("- AMD GPU: "), + "the prompt must state what GPU was detected (or that none was):\n{prompt}" + ); + assert!( + prompt.contains("Never tell the user ROCm is unavailable on their platform"), + "the prompt must refuse the reported answer:\n{prompt}" + ); + } + + /// The static prompt asserted two Windows-only facts at every host, which is + /// how a WSL user — where vLLM IS the supported path — was told to go use + /// WSL. Platform claims now come from the detected facts instead. + #[test] + fn assistant_prompt_makes_no_unconditional_windows_claims() { + let prompt = rocm_chat_tool_system_prompt(); + assert!( + !prompt.contains("non-technical Windows users"), + "the audience is not assumed to be on Windows:\n{prompt}" + ); + assert!( + !prompt.contains("On native Windows, vLLM is skipped"), + "the vLLM caveat belongs in the host facts, not the static prompt:\n{prompt}" + ); + + // Stated only where it is true: present on Windows, absent elsewhere. + let grounded = rocm_chat_tool_system_prompt_for_host(None); + assert_eq!( + grounded.contains("vLLM and AIM are skipped on native Windows"), + cfg!(windows), + "the vLLM caveat must track the host:\n{grounded}" + ); + } + + /// The prompt tells the model to "use examine … before answering". The dash + /// registers its machine check as `doctor`, so before the alias that + /// sentence named a tool absent from the dash's schema. + #[test] + fn every_tool_the_prompt_names_exists_in_the_dash_schema() { + let prompt = rocm_chat_tool_system_prompt(); + for named in [ + "examine", + "gpu_snapshot", + "port_status", + "natural_language_plan", + ] { + assert!(prompt.contains(named), "prompt should mention {named}"); + assert!( + rocm_dash_tui::agent::ROCM_READ_TOOL_NAMES.contains(&named), + "the prompt names `{named}` but the dash never registers it, so a \ + model that obeys the prompt calls a tool the schema does not offer" + ); + } + } + + #[test] + fn deterministic_rocm_tool_summary_suppresses_extra_local_model_follow_up() { + let tool_result = ChatToolRunResult { + approval: None, + follow_up_text: "\ +examine: + legacy_rocm_status: not_detected +runtime_state: + active_runtime_status: ready +" + .to_owned(), + ran_read_only_tool: true, + read_only_tool_error: false, + needs_install_folder: false, + }; + let summary = deterministic_rocm_tool_summary(&tool_result.follow_up_text); + + assert!(summary.is_some()); + assert!(!should_request_local_tool_follow_up( + "local", + &tool_result, + summary.as_deref() + )); + + let mut model_list_result = tool_result; + model_list_result.follow_up_text = "rocm_command:\nmodel recipes\n qwen\n".to_owned(); + assert!(should_request_local_tool_follow_up( + "local", + &model_list_result, + None + )); + } + + #[test] + fn deterministic_model_tool_summary_suppresses_extra_local_model_follow_up() { + let tool_result = ChatToolRunResult { + approval: None, + follow_up_text: "\ +rocm_command: +model recipes + Qwen3-4B-Instruct-2507-GGUF aliases=[qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe +" + .to_owned(), + ran_read_only_tool: true, + read_only_tool_error: false, + needs_install_folder: false, + }; + let summary = deterministic_chat_tool_summary(&tool_result.follow_up_text); + + assert!(summary.is_some()); + assert!(!should_request_local_tool_follow_up( + "local", + &tool_result, + summary.as_deref() + )); + } + + #[test] + fn chat_tool_call_refusal_prioritizes_safe_review_wording() -> Result<()> { + let mut output = String::new(); + let (_root, paths) = test_paths("chat-tool-refusal"); + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "tiny.gguf".to_owned(), + content: "I can install ROCm.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: None, + name: "install_sdk".to_owned(), + arguments: serde_json::json!({ + "channel": "release", + "format": "wheel", + "prefix": "D:\\ROCm\\therock_venvs" + }), + }], + }; + + let mut progress = None; + append_chat_tool_results( + &paths, + &response, + &mut output, + Some("I can install ROCm."), + &mut progress, + )?; + + assert!(output.contains("Install ROCm: needs your review")); + assert!(!output.contains("install_sdk: approval required")); + assert!(output.contains("not run: review the approval card before anything runs")); + assert!(output.contains("advanced manual command: rocm install sdk")); + assert!(!output.contains("run the shown ROCm command")); + Ok(()) + } + + #[test] + fn local_rocm_tools_chat_uses_fixed_lemonade_qwen_assistant() { + assert_eq!( + local_rocm_tools_assistant_model("local", true), + Some(providers::BUILTIN_ASSISTANT_MODEL_ID) + ); + assert_eq!(local_rocm_tools_assistant_model("local", false), None); + assert_eq!(local_rocm_tools_assistant_model("openai", true), None); + } + + #[test] + fn supplemental_tool_call_adds_missing_specific_status_check() { + let generic_services = providers::ChatToolCall { + id: Some("model-picked-services".to_owned()), + name: "services".to_owned(), + arguments: serde_json::json!({}), + }; + + let call = + supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[generic_services]) + .unwrap(); + + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&call)); + } + + #[test] + fn supplemental_tool_call_does_not_duplicate_equivalent_status_check() { + let comfy_status = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); + + assert!( + supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[comfy_status]) + .is_none() + ); + } + + #[test] + fn supplemental_tool_call_adds_running_state_for_engine_install_question() { + let engine_inventory = + fallback_rocm_tool_call_for_prompt("Is vLLM installed and is it running?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&engine_inventory).unwrap(), + vec!["engines".to_owned(), "list".to_owned()] + ); + + let services = supplemental_read_only_tool_call_for_prompt( + "Is vLLM installed and is it running?", + &[engine_inventory], + ) + .unwrap(); + + assert_eq!(services.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&services).unwrap(), + vec!["services".to_owned(), "list".to_owned(), "--all".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&services)); + } + + #[test] + fn supplemental_tool_call_treats_loopback_port_checks_as_equivalent() { + let model_port_check = providers::ChatToolCall { + id: Some("model-picked-port".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "localhost", "port": 8188 }), + }; + + assert!( + supplemental_read_only_tool_call_for_prompt( + "what is running on port 8188?", + &[model_port_check] + ) + .is_none() + ); + } + + #[test] + fn chat_install_intent_without_folder_uses_folder_picker_not_rocm_check() -> Result<()> { + let (_root, paths) = test_paths("chat-install-intent-folder-picker"); + for prompt in [ + "i need to install rocm", + "How do I setup TheRock?", + "install therock", + "How do I get it to install therock?", + "rocm please", + "make my AMD GPU ready", + "get TheRock installed for local AI", + "set up my AMD GPU for local AI", + "install this specific TheRock wheel from date 06052026", + ] { + let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; + let approval = result + .approval + .as_ref() + .expect("install should need folder"); + assert_eq!(approval.pending_title, "Install ROCm", "{prompt}"); + assert_eq!( + approval.args[..6], + [ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + ], + "{prompt}" + ); + assert!( + !approval.args.iter().any(|arg| arg == "--prefix"), + "{prompt}" + ); + assert!( + result.rendered.contains("First choose the folder"), + "{prompt}: {}", + result.rendered + ); + assert!( + !result.rendered.contains("I checked ROCm"), + "{prompt}: {}", + result.rendered + ); + assert!( + !result.rendered.contains("ROCm CLI summary"), + "{prompt}: {}", + result.rendered + ); + if prompt.contains("06052026") { + assert!(approval.args.contains(&"--build-date".to_owned())); + assert!(approval.args.contains(&"2026-06-05".to_owned())); + } + } + Ok(()) + } + + #[test] + fn chat_install_intent_ignores_old_conversation_words() -> Result<()> { + let (_root, paths) = test_paths("chat-install-intent-latest-message"); + let prompt = "\ +Conversation so far: +Assistant: Use /examine to refresh actual GPU memory fit before starting anything large. +Assistant: Native Windows note: models may use WSL/Linux through Windows. + +New message: +install therock"; + + let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; + let approval = result + .approval + .as_ref() + .expect("direct latest install request should need a folder"); + + assert_eq!(approval.pending_title, "Install ROCm"); + assert!(!approval.args.iter().any(|arg| arg == "--prefix")); + assert!(result.rendered.contains("I can install ROCm/TheRock")); + assert!(result.rendered.contains("First choose the folder")); + assert!(!result.rendered.contains("I checked ROCm")); + assert!(!result.rendered.contains("ROCm CLI summary")); + Ok(()) + } + + #[test] + fn local_chat_tool_call_content_is_treated_as_intermediate() { + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "Qwen/Qwen3-0.6B".to_owned(), + content: "The active runtime root is /opt/rocm.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: Some("call-1".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }], + }; + + assert!(local_tool_call_content_is_intermediate( + "local", true, &response + )); + assert!(!local_tool_call_content_is_intermediate( + "openai", true, &response + )); + assert!(!local_tool_call_content_is_intermediate( + "local", false, &response + )); + + let without_tools = providers::ChatResponse { + tool_calls: Vec::new(), + ..response + }; + assert!(!local_tool_call_content_is_intermediate( + "local", + true, + &without_tools + )); + } + + #[test] + fn local_chat_follow_up_with_tool_call_is_not_final_answer() { + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "Qwen/Qwen3-0.6B".to_owned(), + content: "The runtime root is /opt/rocml.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: Some("call-2".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }], + }; + + assert!(!local_follow_up_content_is_final(&response)); + + let final_answer = providers::ChatResponse { + tool_calls: Vec::new(), + content: "The runtime root is D:\\ROCm\\therock_venvs.".to_owned(), + ..response + }; + assert!(local_follow_up_content_is_final(&final_answer)); + } + + #[test] + fn visible_chat_content_removes_reasoning_blocks() { + assert_eq!( + visible_chat_content( + "\nchecking the tool output\n\nThe runtime root is D:\\ROCm\\temp." + ), + "The runtime root is D:\\ROCm\\temp." + ); + assert_eq!( + visible_chat_content("Before\nhidden\nAfter"), + "Before\n\nAfter" + ); + assert_eq!(visible_chat_content("unfinished"), ""); + } + + #[test] + fn chat_tool_result_errors_use_plain_failure_wording() { + assert_eq!(chat_read_only_tool_status_label(false), "done"); + assert_eq!(chat_read_only_tool_status_label(true), "reported an error"); + assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); + assert_eq!( + chat_tool_display_label("gpu_snapshot"), + "Checked GPU status" + ); + assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); + assert!(mcp_tool_result_is_error(&serde_json::json!({ + "isError": true + }))); + assert!(!mcp_tool_result_is_error(&serde_json::json!({ + "isError": false + }))); + assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); + } + + #[test] + fn local_chat_without_service_explains_serve_before_chat_without_llm_setup() { + let (_root, paths) = test_paths("local-chat-no-service-guidance"); + let result = + render_chat_prompt_result(&paths, "local", None, "Check this ROCm setup", true) + .expect("missing local assistant should render guidance"); + assert!(result.approval.is_none()); + let rendered = result.rendered; + + assert!(rendered.contains("No local assistant is running yet.")); + assert!(rendered.contains("First-time ROCm setup does not need an LLM")); + assert!(rendered.contains("Recommended path:")); + assert!(rendered.contains("Advanced manual command")); + assert!(rendered.contains( + "rocm serve Qwen3-4B-Instruct-2507-GGUF --engine lemonade --device gpu_required --managed" + )); + assert!(!rendered.contains("sshleifer/tiny-gpt2")); + assert!(rendered.contains("rocm chat --tools --provider local --prompt")); + assert!(rendered.contains("Nothing was changed.")); + assert!(!rendered.contains("install sdk")); + assert!(!rendered.contains("setup TheRock with an LLM")); + } + + #[test] + fn local_chat_status_prompts_use_read_only_tools_without_assistant() -> Result<()> { + let (_root, paths) = test_paths("local-chat-status-fallback"); + + let running = + render_chat_prompt_result(&paths, "local", None, "Is vLLM running?", true)?.rendered; + assert!(!running.contains("No local assistant is running yet.")); + assert!(running.contains("Checked model servers: done"), "{running}"); + assert!(running.contains("ROCm CLI summary"), "{running}"); + assert!( + running.contains("Local model servers: none running under ROCm CLI."), + "{running}" + ); + assert!(running.contains("Nothing was changed.")); + + let installed = + render_chat_prompt_result(&paths, "local", None, "Is vLLM installed?", true)?.rendered; + assert!(!installed.contains("No local assistant is running yet.")); + assert!(installed.contains("Engine runtimes:"), "{installed}"); + assert!(installed.contains("vLLM:"), "{installed}"); + + let installed_and_running = render_chat_prompt_result( + &paths, + "local", + None, + "Is vLLM installed and is it running?", + true, + )? + .rendered; + assert!( + installed_and_running.contains("Checked local engines: done"), + "{installed_and_running}" + ); + assert!( + installed_and_running.contains("Checked model servers: done"), + "{installed_and_running}" + ); + assert!( + installed_and_running.contains("Engine runtimes:"), + "{installed_and_running}" + ); + assert!( + installed_and_running.contains("Local model servers: none running under ROCm CLI."), + "{installed_and_running}" + ); + + let port = render_chat_prompt_result( + &paths, + "local", + None, + "What is running on port 8188?", + true, + )? + .rendered; + assert!(!port.contains("No local assistant is running yet.")); + assert!(port.contains("Checked local port: done"), "{port}"); + assert!(port.contains("Port 8188:"), "{port}"); + Ok(()) + } + + #[test] + fn chat_tools_anthropic_reaches_provider_opt_in_boundary() { + let (_root, paths) = test_paths("anthropic-chat-tools-opt-in"); + + let error = render_chat_prompt_result( + &paths, + "anthropic", + Some("claude-test"), + "Check this ROCm setup", + true, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("cloud provider `anthropic` is disabled")); + assert!(error.contains("rocm config enable-provider anthropic")); + assert!(!error.contains("OpenAI-compatible provider")); + } +} diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 548fc7fb09..6104963818 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -12,13 +12,14 @@ //! the completion builder, and re-imports these types so every other module //! keeps reaching them as `crate::Command`, `crate::Provider` and so on. +pub(crate) use crate::cli_args::{ + DevicePolicyArg, FrameworkArg, InstallFormat, PermissionsModeArg, Provider, SandboxToolArg, + TelemetryModeArg, WatcherModeArg, +}; use crate::service_records::DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS; use crate::{bootstrap, chat, comfyui, remote, storage}; -use clap::{Parser, Subcommand, ValueEnum}; -use rocm_core::{ - DEFAULT_LOCAL_HOST, PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, TELEMETRY_MODE_LOCAL, - TELEMETRY_MODE_OFF, WatcherMode, -}; +use clap::{Parser, Subcommand}; +use rocm_core::DEFAULT_LOCAL_HOST; use std::path::PathBuf; /// User-selectable serving engines, in the order shown in `--help` and shell @@ -95,8 +96,33 @@ pub(crate) enum Command { /// takes the id, not the position. Diagnose { /// Raw error text from the user; sharpens keyword scoring. - #[arg(long)] + /// + /// A shell reads this text before `rocm` does, so quotes, `$(...)` + /// and a leading `-` in a pasted error can be changed, run, or taken + /// as a flag. Use `--symptom-file` to hand over a paste untouched. + /// + /// Each match then says whether this text contributed to it + /// (`symptom_matched` in --json). + #[arg(long, value_name = "TEXT")] symptom: Option, + /// Read the error text from a file instead, or from standard input + /// when PATH is `-`, so no shell quoting is needed. A file named `-` + /// is passed as `./-`. + /// + /// The text that arrives is matched as `--symptom` would match it, + /// with one trailing line break removed (shells and editors add one). + /// A UTF-8 byte-order mark is dropped, and a file with a UTF-16 + /// byte-order mark, as Windows PowerShell's `>` writes, is decoded; + /// anything else must be UTF-8. An unreadable file, input that is not + /// text, or input larger than 16 MiB is a usage error (exit code 2). + /// For example: `rocm diagnose --symptom-file error.txt`. In + /// PowerShell, save the clipboard first + /// (`Get-Clipboard -Raw > error.txt`): Windows PowerShell 5.1 pipes + /// text to a program as ASCII, turning any other character into `?`. + /// With nothing piped in, paste the text at the terminal and end it + /// with Ctrl-D at the start of a line (Ctrl-Z then Enter on Windows). + #[arg(long, value_name = "PATH", conflicts_with = "symptom")] + symptom_file: Option, /// Show at most this many matches (default 5). #[arg(long, default_value_t = 5)] top: usize, @@ -111,22 +137,70 @@ pub(crate) enum Command { /// Also answer whether a model would run on this machine, before /// downloading it. /// - /// Takes a curated model name or alias (see `rocm model`). Nothing is - /// fetched: the answer comes from the recipe and this host's GPU. A - /// model the catalog does not carry is reported as undetermined rather - /// than blocked - `rocm serve` still accepts it, this just cannot say - /// in advance whether it fits. + /// Takes a curated model name or alias (see `rocm model`), or a GGUF + /// model: a local `.gguf` path, or `owner/repo:QUANT` on Hugging Face. + /// No weights are downloaded. A curated model is answered from its + /// recipe and this host's GPU. A GGUF outside the catalog is estimated + /// from its file header (for a remote file: one repository listing, a + /// `HEAD` request for a file the listing does not size, then a few MB by + /// range request; no token is sent, so a gated or private repository + /// cannot be read): weights plus KV cache at `--context-size` plus a + /// compute buffer and a runtime reserve, against the dedicated VRAM of + /// the GPU with the most of it, labelled ESTIMATED rather than READY or + /// BLOCKED. A device-visibility mask on a host whose GPUs differ in + /// memory is flagged in the report, since which card it selects is not + /// verified there. Any other model the + /// catalog does not carry is reported as undetermined rather than + /// blocked - `rocm serve` still accepts it, this just cannot say in + /// advance whether it fits. + /// + /// For example: `rocm diagnose --model Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M + /// --context-size 32768 --kv-cache-type q8_0`. `HF_ENDPOINT` points the + /// header read at a Hugging Face mirror. /// /// A flag rather than a positional: `diagnose` takes no positional /// today, and one added now would be ambiguous against `--symptom`. #[arg(long, value_name = "MODEL")] model: Option, + /// Context size, in tokens, to estimate a GGUF model for (1 to + /// 16777216; default 4096). + /// + /// The largest context that fits (never more than the context the model + /// was trained for) is reported as well, but only when this machine's + /// GPU memory could be read; without a reading, the memory the model + /// needs is shown and nothing is compared, so no largest context is + /// given. The comparison is against the total dedicated VRAM of the GPU + /// with the most of it that ROCm can see, as for a curated model, not + /// what other processes have left free. Applies to a GGUF outside the + /// catalog (a curated model's memory minimum is fixed, and says so); + /// `rocm serve` has no context-size option yet, so this answers the + /// question without changing what serve does. + #[arg( + long, + value_name = "TOKENS", + requires = "model", + value_parser = clap::value_parser!(u64).range(1..=rocm_core::gguf_fit::MAX_CONTEXT_SIZE) + )] + context_size: Option, + /// KV cache element type to estimate for (default f16). + /// + /// A quantized cache is smaller. In llama.cpp a quantized V cache needs + /// flash attention (q8_0 and q4_0 here quantize both K and V), and the + /// estimate assumes flash attention is on. Applies to a GGUF outside the + /// catalog. + #[arg(long, value_enum, value_name = "TYPE", requires = "model")] + kv_cache_type: Option, /// Show the report this machine would contribute, and send nothing. /// /// Nothing leaves the machine: this prints the exact content so it can /// be read before any of it is shared. Hardware that is not on AMD's /// published compatibility matrix produces no report at all. /// + /// `--model` is not consulted with `--report`: a report carries no model + /// verdict, so nothing is looked up for it and nothing is fetched. A + /// note on stderr says that `--model` (and `--context-size` and + /// `--kv-cache-type`, if given) were ignored. + /// /// Not combinable with `--distro`: a report describes this machine, and /// a WSL distribution reached remotely is not fully examined (see /// `--distro`'s own help), so it cannot back the disclosure guard's @@ -180,11 +254,19 @@ pub(crate) enum Command { #[arg(long)] device_index: Option, /// Emit the catalog as JSON for tooling. Only valid without a fix id. + /// + /// It also names each exit code and says what it means. Exit 2 is + /// both an unknown fix id (a stderr line starting `Unknown fix-id:`) + /// and a malformed command line (a stderr line starting `error:`); + /// read the message to tell them apart. #[arg(long)] json: bool, }, /// Print the rocm-cli version, release tag or branch, and commit hash, /// plus the ROCm SDK and GPU driver this machine would use. + /// + /// A release tag `vX.Y.Z` is shown as `X.Y.Z (vX.Y.Z, )`. A build + /// from `main` is shown as `0.0.0-dev (main, )`. Version, /// Generate a shell completion script for the given shell. Completions { @@ -375,7 +457,9 @@ rocm model --verbose" }, /// Start a local OpenAI-compatible model server. /// - /// Picks an engine and ROCm runtime automatically unless overridden. By default the + /// Picks an engine and ROCm runtime automatically unless overridden. An + /// Anthropic `/v1/messages` endpoint is published beside the OpenAI + /// one (see `rocm services list`). By default the /// server runs as a managed background service and prints a deployment summary /// (status, endpoint, model, and smoke-test metrics); use --verbose to stream engine /// logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the @@ -617,6 +701,11 @@ rocm logs --search error timeout")] status: bool, }, /// Launch the unified telemetry dashboard (TUI) with an embedded daemon. + /// + /// On Linux/WSL, discovers loaded models from local Lemonade at 127.0.0.1:13305 + /// by default. Configure dashboard.daemon.enable_lemonade, lemonade_host, and + /// lemonade_port in config.json. External models have no Services stop/restart + /// controls; discovery is read-only and quitting leaves external servers running. Dash { /// Replay a recorded session NDJSON instead of connecting to a live daemon. #[arg(long, value_name = "FILE")] @@ -637,10 +726,16 @@ rocm logs --search error timeout")] /// Remove ROCm CLI-managed files from this computer. /// /// Removes the config, data, and cache directories and the installed - /// binaries, including the running rocm.exe on Windows. It also undoes the - /// PATH change the installer made (its marked shell-profile block, or the - /// user PATH entry it recorded adding on Windows), unless other programs - /// share the install folder. + /// binaries, including the running rocm.exe on Windows, and the fish + /// completion script the Linux installer wrote for them (a rocm.fish it + /// did not write is left in place, and one it cannot remove is only a + /// warning). It also undoes the PATH change the installer made (its marked + /// shell-profile block, or the user PATH entry it recorded adding on + /// Windows), unless other programs share the install folder. + #[command(after_help = "EXAMPLES:\n \ +rocm uninstall --dry-run\n \ +rocm uninstall\n \ +rocm uninstall --keep-data --yes")] Uninstall { /// Do not ask for interactive confirmation. Without it, the command /// asks `[y/N]` (default: no) in an interactive terminal; outside one, @@ -651,7 +746,8 @@ rocm logs --search error timeout")] /// Show what would be removed without deleting anything. #[arg(long)] dry_run: bool, - /// Keep installed rocm-cli binaries, and the installer's PATH change. + /// Keep installed rocm-cli binaries, the installer's PATH change, and the + /// fish completions it wrote. #[arg(long)] keep_binaries: bool, /// Keep saved settings. @@ -668,6 +764,11 @@ rocm logs --search error timeout")] /// Allow removing development binaries inside the current build tree. #[arg(long)] force_dev_binaries: bool, + /// Not an option; accepted only to say so. Without it clap answers a + /// mistyped `--force` (a likely guess for skipping the confirmation) + /// with `--force-dev-binaries`, which removes development binaries. + #[arg(long, hide = true, num_args = 0..=1, default_missing_value = "")] + force: Option, }, } @@ -718,11 +819,17 @@ pub(crate) enum InstallTarget { /// Install TheRock ROCm wheels into a Python folder managed by ROCm CLI. #[command(after_help = "EXAMPLES:\n \ rocm install sdk\n \ -rocm install sdk --channel nightly --build-date 2025-01-15\n \ -rocm install sdk --family gfx110X-all --dry-run\n \ -rocm install sdk --devel")] +rocm install sdk --channel nightly --build-date 2026-08-19\n \ +rocm install sdk --family gfx1100 --dry-run\n \ +rocm install sdk --version 7.14.1 --family gfx110X-all\n \ +rocm install sdk --devel\n \ +rocm install sdk --version 10.0.0 --python-version 3.12")] Sdk { - /// Package channel to install, such as release or nightly. + /// TheRock ROCm SDK channel to install from: `release` or `nightly`. + /// + /// This picks the ROCm SDK stream only. It is separate from the channel + /// the `rocm` binary itself was installed from (`install.sh`, + /// `install.ps1`); `rocm update` updates SDK runtimes, not the binary. #[arg(long, default_value = "release")] channel: String, /// Package format to install [possible values: wheel, tarball]. @@ -731,13 +838,20 @@ rocm install sdk --devel")] /// Full folder path where the ROCm Python environment should be created. #[arg(long)] prefix: Option, - /// Exact TheRock package version to install. + /// Exact TheRock package version to install; with none, the channel's + /// newest build. An earlier ROCm 7 release such as 7.14.1 is supported, + /// for a wheel install only. #[arg(long, conflicts_with = "build_date")] version: Option, - /// Pick the TheRock package built on this date. + /// Pick the TheRock package built on this date; nightlies from before + /// 2026-08-23 are still installable, for a wheel install only. #[arg(long, value_name = "YYYY-MM-DD", conflicts_with = "version")] build_date: Option, - /// TheRock GPU package family to install, such as gfx110X-all. + /// TheRock GPU package family to install, such as gfx1200. + /// + /// ROCm 10 and newer, which an install with no `--version` gets, need + /// an exact arch such as `gfx1200`; a label such as gfx110X-all is for + /// an earlier ROCm 7 (wheel installs only). /// /// A GPU target ROCm publishes no build for is refused rather than /// mapped to a near neighbour: `--family gfx90c` - the Vega iGPU in @@ -746,6 +860,17 @@ rocm install sdk --devel")] /// install sdk` reports the same refusal when it detects such a GPU. #[arg(long)] family: Option, + /// Python version for a ROCm 10 or newer install, one of 3.10 to 3.14. + /// + /// ROCm 10 installs use Python 3.14 unless you name another, because + /// vLLM, flash-attn and amd-aiter for ROCm 10 are published for 3.14 + /// only. With another version the SDK and PyTorch install and run, but + /// the vLLM engine does not: the engine install that follows an SDK + /// install on a vLLM GPU is skipped, and `rocm engines install vllm` + /// refuses that runtime. Re-run without this flag to switch it to 3.14. + /// Wheel installs only; a ROCm 7 install is built for Python 3.12. + #[arg(long, value_name = "X.Y")] + python_version: Option, /// Also install the ROCm compiler, headers, and static libraries for /// building GPU code. Roughly doubles the wheel download; already /// included by `--format tarball`. @@ -759,6 +884,11 @@ rocm install sdk --devel")] #[arg(long)] devel: bool, /// Resolve the install plan without changing files. + /// + /// Downloads no uv and prepares no Python. For ROCm 10 and newer on a + /// host with no uv installed, the plan marks the package versions it + /// would need uv to choose, and everything made from them, as not + /// resolved. #[arg(long)] dry_run: bool, /// Approve replacing the current active default ROCm runtime (and @@ -912,7 +1042,7 @@ left, so it cannot undo more than one activation." /// Python executable inside the existing ROCm environment. #[arg(long)] python: PathBuf, - /// Optional SDK root path. + /// Python environment folder. Taken from --python when that sits in a bin or Scripts folder; required otherwise. #[arg(long)] root: Option, /// Runtime id to assign. @@ -1075,9 +1205,10 @@ pub(crate) enum ServicesCommand { /// confirmed gone. Otherwise it reports on stderr that the stop was not /// confirmed and exits with an error, keeping the record, which still names /// the processes, and any endpoint key, which the server may still be - /// enforcing. It names each process that did not exit, and for one it could - /// not verify it signals nothing and names an operating-system command to - /// check on it. It does not suggest running the stop again. + /// enforcing. It names each process that did not exit (on Windows, also one + /// whose child processes may still be running), and for one it could not + /// verify it signals nothing and names an operating-system command to check + /// on it. It does not suggest running the stop again. Stop { /// Service id from `rocm services list`. service_id: String, @@ -1099,9 +1230,10 @@ pub(crate) enum ServicesCommand { }, /// Delete one local server record and its files. /// - /// Only works on a record that is not running: stop the server first. The - /// record's details file, log, engine state file, and endpoint key file are - /// all deleted, so its log can no longer be read and it can no longer be + /// Only works on a record that is not running: stop the server first (on + /// Windows a process `rocm` cannot query counts as running). The record's + /// details file, log, engine state file, and endpoint key file are all + /// deleted, so its log can no longer be read and it can no longer be /// restarted. Remove { /// Service id from `rocm services list --all`. @@ -1109,11 +1241,21 @@ pub(crate) enum ServicesCommand { /// Required; remove refuses without it. #[arg(long)] yes: bool, + /// Remove a record whose recorded process `rocm` cannot query. + /// + /// Only when a stop was requested, no recorded process is confirmed + /// running, and at least one could not be queried (on Windows, a live + /// process the account may not open). Deletes the record, log, engine + /// state and endpoint key, and says which process; if it is still + /// running it keeps running. Refused for a process confirmed running. + #[arg(long)] + force: bool, }, /// Delete local server records that are no longer running. /// - /// Running servers are always left alone. Leftover files whose record is - /// already gone are cleaned up too. + /// Running servers are always left alone; a record held only by a process + /// `rocm` cannot query (Windows) is listed with the `remove --force` command. + /// Leftover files whose record is already gone are cleaned up too. /// /// Waits for a managed launch already under way to publish its record /// before reading the directory, and waits as long as that launch takes. @@ -1251,168 +1393,10 @@ pub(crate) enum SetupCommand { }, } -/// Which framework `rocm examine` should probe. -/// -/// The CLI-facing mirror of [`rocm_core::FrameworkProbe`], which cannot derive -/// `ValueEnum` itself without pulling clap into `rocm-core`. Every variant of -/// that enum existed and was reachable in the library, but no caller ever passed -/// anything but the default, so the choice could not be made from the command -/// line at all. -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum FrameworkArg { - /// Try PyTorch first, then llama.cpp (the default). - Auto, - /// Probe PyTorch only. - Pytorch, - /// Probe llama.cpp only. - #[value(name = "llama-cpp", alias = "llamacpp")] - LlamaCpp, - /// Probe no framework at all - fastest, and enough for GPU and driver questions. - Skip, -} - -impl From for rocm_core::FrameworkProbe { - fn from(value: FrameworkArg) -> Self { - match value { - FrameworkArg::Auto => Self::Auto, - FrameworkArg::Pytorch => Self::PyTorch, - FrameworkArg::LlamaCpp => Self::LlamaCpp, - FrameworkArg::Skip => Self::Skip, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum InstallFormat { - /// Python wheel packages (recommended; smaller, pip-installable). - Wheel, - /// Self-contained tarball archive of the ROCm SDK. - Tarball, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum Provider { - /// A local OpenAI-compatible server started via `rocm serve` (no API key). - Local, - /// Anthropic Claude API (requires an API key). - Anthropic, - /// OpenAI API or OpenAI-compatible endpoint (requires an API key). - Openai, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum WatcherModeArg { - /// Only report findings; never take action. - Observe, - /// Report findings and propose changes for you to approve. - Propose, - /// Apply changes automatically within a contained, reversible scope. - Contained, -} - -impl From for WatcherMode { - fn from(value: WatcherModeArg) -> Self { - match value { - WatcherModeArg::Observe => Self::Observe, - WatcherModeArg::Propose => Self::Propose, - WatcherModeArg::Contained => Self::Contained, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum TelemetryModeArg { - /// Collect GPU telemetry on this machine only; nothing leaves the device. - Local, - /// Disable telemetry collection entirely. - Off, -} - -/// User-facing `rocm serve --device` policy. -/// -/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage -/// validation (exit code 2, with the valid choices listed) rather than a generic -/// application error (exit code 1) — matching how every other enum-style CLI -/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`, -/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden -/// from the advertised list. -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum DevicePolicyArg { - /// Require a ROCm GPU; fail if none is usable (the default). - #[value(alias = "auto", alias = "gpu")] - GpuRequired, - /// Accepted for compatibility; behaves identically to `gpu_required` - /// (rocm serve has no CPU fallback path). - GpuPreferred, - /// Accepted but always rejected: rocm serve has no CPU fallback path, so - /// this exits with an error. Hidden from `--help`, shell completions and the - /// invalid-value suggestion list so it is never advertised as usable, while - /// still parsing so the deliberate rejection message is preserved. - #[value(alias = "cpu", hide = true)] - CpuOnly, -} - -impl DevicePolicyArg { - /// Canonical policy string understood by [`crate::gpu_policy::parse_device_policy`]. Routing the - /// parsed variant back through the string keeps `parse_device_policy` the - /// single choke point that enforces the `cpu_only` rejection across every - /// caller, rather than adding a second conversion path that could bypass it. - pub(crate) const fn as_policy_str(self) -> &'static str { - match self { - Self::GpuRequired => "gpu_required", - Self::GpuPreferred => "gpu_preferred", - Self::CpuOnly => "cpu_only", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum SandboxToolArg { - ListServers, - RestartServer, - StopServer, -} - -impl SandboxToolArg { - pub(crate) const fn as_cli_value(self) -> &'static str { - match self { - Self::ListServers => "list_servers", - Self::RestartServer => "restart_server", - Self::StopServer => "stop_server", - } - } -} - -impl TelemetryModeArg { - pub(crate) const fn as_str(self) -> &'static str { - match self { - Self::Local => TELEMETRY_MODE_LOCAL, - Self::Off => TELEMETRY_MODE_OFF, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum PermissionsModeArg { - FullAccess, - Ask, -} - -impl PermissionsModeArg { - pub(crate) const fn as_str(self) -> &'static str { - match self { - Self::FullAccess => PERMISSIONS_MODE_FULL_ACCESS, - Self::Ask => PERMISSIONS_MODE_ASK, - } - } -} - #[cfg(test)] mod tests { use super::*; - use crate::{cli_command, device_policy_name, engine_inventory}; + use crate::{cli_command, device_policy_name, engine_inventory, uninstall::reject_force}; use clap::{CommandFactory, Parser}; use rocm_engine_protocol::DevicePolicy; @@ -1421,6 +1405,39 @@ mod tests { Cli::command().debug_assert(); } + /// `cli_version_string` is `ROCM_CLI_VERSION`, the one string + /// `format_cli_version` in `build.rs` embeds, clap's `-V`/`--version` is + /// that same string, and `cli_version_number` leads it. This does not run + /// the binary: `@id:examine-version` compares the printed `rocm version` + /// and `-V`/`--version` lines. Tag, `main`, and unknown shapes are the + /// `format_cli_version_*` tests in `build.rs`; this checks only the shape + /// of the ref this checkout resolved, which is never a release tag in CI. + #[test] + fn version_surfaces_share_one_build_script_string() { + let full = crate::cli_version_string(); + let number = crate::cli_version_number(); + let reference = env!("ROCM_CLI_VERSION_REF"); + let hash = env!("ROCM_CLI_GIT_HASH"); + assert_eq!(full, env!("ROCM_CLI_VERSION")); + assert_eq!( + cli_command().get_version(), + Some(full.as_str()), + "-V/--version must print the same string as `rocm version`" + ); + assert_eq!(full, format!("{number} ({reference}, {hash})")); + // A resolved ref whose text is not `v` plus the leading version is not + // a release tag. `main`, a feature branch, and a name such as `vllm` + // all land here, and none of them may report the workspace package + // version. + if reference != "unknown" && reference.strip_prefix('v') != Some(number.as_str()) { + assert_eq!(number, "0.0.0-dev", "{full}"); + assert_ne!(number, env!("CARGO_PKG_VERSION")); + } + if reference == "unknown" { + assert_eq!(number, env!("CARGO_PKG_VERSION")); + } + } + /// EAI-7362: the top-level `rocm --help` command list must be alphabetical, /// including clap's implicit `help` entry (it must sort into place, not be /// pinned last). @@ -1490,6 +1507,30 @@ mod tests { ); } + #[test] + fn dash_help_scopes_lemonade_discovery_to_linux_and_wsl() { + // The embedded daemon that does the discovery is Unix-only, so the help + // must not promise it on every platform. README.md says the same. + let help = Cli::command() + .find_subcommand_mut("dash") + .expect("dash subcommand") + .render_long_help() + .to_string(); + let flat = help.split_whitespace().collect::>().join(" "); + assert!( + flat.contains("On Linux/WSL, discovers loaded models from local Lemonade"), + "{help}" + ); + // This pins the wording only. Scenario + // `@id:dash-lemonade-external-protocol-discovery` proves the claim: the + // external server receives no request beyond reads after the dashboard + // quits. + assert!( + flat.contains("quitting leaves external servers running"), + "{help}" + ); + } + #[test] fn the_framework_choice_is_offered_on_the_command_line() { // Guards the actual regression: the variants existed in the library and @@ -1892,6 +1933,143 @@ mod tests { } } + /// `rocm diagnose --model` tells the user that `rocm serve` takes no context + /// size, KV cache type or GPU layer count and passes none to the engine + /// (`SERVE_OPTIONS_NOTE` and `SERVE_CONTEXT_NOTE` in `gguf_fit.rs`). This + /// fails when `rocm serve` gains one of them, so the notes are updated with it. + #[test] + fn serve_takes_no_estimate_options_yet() { + let serve = |extra: &[&str]| { + let mut argv = vec!["rocm", "serve", "o/r:Q4_K_M"]; + argv.extend_from_slice(extra); + Cli::try_parse_from(argv) + }; + assert!(serve(&[]).is_ok(), "the model alone parses"); + for option in [ + &["--context-size", "4096"][..], + &["--ctx-size", "4096"][..], + &["--kv-cache-type", "q8_0"][..], + &["--n-gpu-layers", "20"][..], + ] { + assert!(serve(option).is_err(), "rocm serve now takes {option:?}"); + } + } + + fn parse_diagnose(args: &[&str]) -> Result { + let mut argv = vec!["rocm", "diagnose"]; + argv.extend_from_slice(args); + Cli::try_parse_from(argv) + } + + // `--symptom-file` exists because a pasted error does not survive the shell + // as an argument: one starting with `-` is not even accepted as + // `--symptom`'s value. The file form takes `-` (stdin) and any path, and + // the two sources cannot both be given. + #[test] + fn diagnose_takes_the_symptom_from_a_file_or_stdin_but_not_with_symptom() { + assert!( + parse_diagnose(&["--symptom", "-x HIP error"]).is_err(), + "a leading minus is read as a flag, which is what --symptom-file avoids" + ); + for path in ["-", "error.txt"] { + match parse_diagnose(&["--symptom-file", path]) { + Ok(Cli { + command: + Some(Command::Diagnose { + symptom, + symptom_file, + .. + }), + .. + }) => { + assert_eq!(symptom, None); + assert_eq!(symptom_file, Some(PathBuf::from(path))); + } + other => panic!("expected diagnose for {path:?}, got {other:?}"), + } + } + let error = parse_diagnose(&["--symptom", "oom", "--symptom-file", "-"]) + .expect_err("--symptom and --symptom-file are exclusive"); + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); + } + + // The context options describe the model being asked about, so without + // `--model` they would be accepted and silently do nothing. + #[test] + fn diagnose_context_options_are_refused_without_a_model() { + for args in [ + &["--context-size", "8192"][..], + &["--kv-cache-type", "q8_0"][..], + ] { + let error = parse_diagnose(args).expect_err("an option with nothing to apply to"); + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument, + "{args:?}" + ); + assert!(error.to_string().contains("--model"), "{args:?}: {error}"); + } + } + + #[test] + fn diagnose_context_options_reach_the_request_and_bad_values_are_refused() { + let cli = parse_diagnose(&[ + "--model", + "o/r:Q4_K_M", + "--context-size", + "32768", + "--kv-cache-type", + "q4_0", + ]) + .expect("a model with both options parses"); + match cli.command { + Some(Command::Diagnose { + context_size, + kv_cache_type, + .. + }) => { + assert_eq!(context_size, Some(32768)); + assert_eq!( + kv_cache_type.map(rocm_core::gguf_fit::KvCacheType::from), + Some(rocm_core::gguf_fit::KvCacheType::Q4_0) + ); + } + other => panic!("expected diagnose, got {other:?}"), + } + // The smallest size accepted, beside the 0 refused below. + assert!(parse_diagnose(&["--model", "o/r:Q4_K_M", "--context-size", "1"]).is_ok()); + for bad in [ + &["--context-size", "0"][..], + &["--context-size", "-1"][..], + &["--context-size", "lots"][..], + &["--context-size", "18446744073709551615"][..], + &["--context-size", "16777217"][..], + &["--kv-cache-type", "q5_0"][..], + ] { + let mut args = vec!["--model", "o/r:Q4_K_M"]; + args.extend_from_slice(bad); + assert!(parse_diagnose(&args).is_err(), "{bad:?} must not parse"); + } + } + + // The largest context size the flag takes is the one the estimate is made for, + // not a number of its own: a figure past it would print an absurd memory + // need, so clap refuses it before anything is estimated. + #[test] + fn the_largest_context_size_accepted_is_the_largest_the_estimate_makes() { + let largest = rocm_core::gguf_fit::MAX_CONTEXT_SIZE.to_string(); + let cli = parse_diagnose(&["--model", "o/r:Q4_K_M", "--context-size", &largest]) + .expect("the cap itself parses"); + match cli.command { + Some(Command::Diagnose { context_size, .. }) => { + assert_eq!(context_size, Some(rocm_core::gguf_fit::MAX_CONTEXT_SIZE)); + } + other => panic!("expected diagnose, got {other:?}"), + } + let over = (rocm_core::gguf_fit::MAX_CONTEXT_SIZE + 1).to_string(); + assert!(parse_diagnose(&["--model", "o/r:Q4_K_M", "--context-size", &over]).is_err()); + } + // `--yes` approves a restart, so on its own it asks for something that was // never requested. Mutation: removing `requires = "restart_services"` from // either subcommand's `yes` makes that subcommand's argv parse and fails @@ -2259,6 +2437,124 @@ mod tests { .expect_err("an abbreviated subcommand name must not reach uninstall"); } + /// `--force` is a likely guess for "skip the confirmation", and without a + /// declared `--force` clap answers it, and every near miss of it, with + /// `--force-dev-binaries`, which removes the binaries of a development build. + /// The scenario `uninstall-mistyped-force-is-not-answered-with-a-destructive-flag` + /// proves the real binary's output; this pins the parse behind it. + #[test] + fn a_mistyped_force_never_reaches_the_dev_binaries_flag() { + // `--force` itself parses, so the handler can say what to use instead. + let cli = Cli::try_parse_from(["rocm", "uninstall", "--force"]).unwrap(); + let Some(Command::Uninstall { + force, + force_dev_binaries, + .. + }) = cli.command + else { + panic!("not an uninstall command"); + }; + assert!(force.is_some() && !force_dev_binaries); + let refusal = reject_force(force.is_some()).unwrap_err().to_string(); + assert!(refusal.contains("`--force` is not an option"), "{refusal}"); + assert!(refusal.contains("--yes"), "{refusal}"); + assert!(!refusal.contains("force-dev-binaries"), "{refusal}"); + assert!(reject_force(false).is_ok()); + // `--force=x` is the same mistake, and gets the same answer rather than + // a usage line that lists every flag. + let cli = Cli::try_parse_from(["rocm", "uninstall", "--force=x"]).unwrap(); + let Some(Command::Uninstall { force, .. }) = cli.command else { + panic!("not an uninstall command"); + }; + assert!(reject_force(force.is_some()).is_err()); + + // Near misses and the forms clap suggests from elsewhere in the command + // line never offer the destructive flag. + for args in [ + vec!["uninstall", "--forc"], + vec!["uninstall", "--forcee"], + vec!["uninstall", "--keep-binaries", "--forc"], + vec!["uninstall", "-f"], + vec!["--force", "uninstall"], + ] { + let error = cli_command() + .try_get_matches_from(std::iter::once("rocm").chain(args.iter().copied())) + .unwrap_err(); + let rendered = error.render().to_string(); + assert!( + !rendered.contains("force-dev-binaries"), + "{args:?} was answered with the destructive flag:\n{rendered}" + ); + assert_eq!(error.exit_code(), 2, "{args:?}"); + } + + // The guard depends on clap offering the hidden `--force` as the + // suggestion for a near miss: its candidates are every declared long + // name, hidden or not. If a clap upgrade stopped suggesting hidden + // arguments, `--forc` would be answered with `--force-dev-binaries` + // again, so the near misses must name `--force` and not the + // destructive flag. The two checks together are independent of + // clap's wording around the suggestion. + for args in [ + vec!["uninstall", "--forc"], + vec!["uninstall", "--keep-binaries", "--forc"], + ] { + let rendered = cli_command() + .try_get_matches_from(std::iter::once("rocm").chain(args.iter().copied())) + .unwrap_err() + .render() + .to_string(); + assert!( + rendered.contains("--force") && !rendered.contains("force-dev-binaries"), + "{args:?} must be steered to the declared `--force`, not the destructive flag:\n{rendered}" + ); + } + + // The whole flag misspelled is still met with the flag it was meant to be. + let error = cli_command() + .try_get_matches_from(["rocm", "uninstall", "--force-dev-binary"]) + .unwrap_err() + .render() + .to_string(); + assert!(error.contains("--force-dev-binaries"), "{error}"); + + // It stays out of the help. + let mut command = cli_command(); + let help = command + .find_subcommand_mut("uninstall") + .unwrap() + .render_long_help() + .to_string(); + assert!(!help.contains("Not an option"), "{help}"); + } + + /// The help leads with the safe preview, then the real run, as the tip a + /// person reads before running a command that deletes things. + #[test] + fn uninstall_help_shows_a_dry_run_before_the_real_run() { + let mut command = cli_command(); + let help = command + .find_subcommand_mut("uninstall") + .expect("uninstall subcommand") + .render_long_help() + .to_string(); + let examples = help.split_once("EXAMPLES:").map_or_else( + || panic!("`rocm uninstall --help` has no examples:\n{help}"), + |(_, rest)| rest, + ); + let rows: Vec<&str> = examples + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect(); + assert_eq!( + rows.first().copied(), + Some("rocm uninstall --dry-run"), + "{examples}" + ); + assert!(rows.contains(&"rocm uninstall"), "{examples}"); + } + #[test] fn update_rejects_zero_timeout_secs() { let error = Cli::try_parse_from(["rocm", "update", "--json", "--timeout-secs", "0"]) diff --git a/apps/rocm/src/cli_args.rs b/apps/rocm/src/cli_args.rs new file mode 100644 index 0000000000..a11fd1b520 --- /dev/null +++ b/apps/rocm/src/cli_args.rs @@ -0,0 +1,176 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The clap value enums behind the `--framework`, `--format`, `--provider`, +//! `--mode`, `--device`, `--telemetry` and `--permissions` style arguments, and +//! the conversions from them to the library types they stand for. +//! +//! Mechanically relocated from `cli.rs` with no behavior change: attributes, +//! help text, aliases and declaration order are unchanged. `cli.rs` re-exports +//! each type, so every module keeps reaching them as `crate::cli::Provider` and +//! so on. + +use clap::ValueEnum; +use rocm_core::{ + PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, TELEMETRY_MODE_LOCAL, TELEMETRY_MODE_OFF, + WatcherMode, +}; + +/// Which framework `rocm examine` should probe. +/// +/// The CLI-facing mirror of [`rocm_core::FrameworkProbe`], which cannot derive +/// `ValueEnum` itself without pulling clap into `rocm-core`. Every variant of +/// that enum existed and was reachable in the library, but no caller ever passed +/// anything but the default, so the choice could not be made from the command +/// line at all. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum FrameworkArg { + /// Try PyTorch first, then llama.cpp (the default). + Auto, + /// Probe PyTorch only. + Pytorch, + /// Probe llama.cpp only. + #[value(name = "llama-cpp", alias = "llamacpp")] + LlamaCpp, + /// Probe no framework at all - fastest, and enough for GPU and driver questions. + Skip, +} + +impl From for rocm_core::FrameworkProbe { + fn from(value: FrameworkArg) -> Self { + match value { + FrameworkArg::Auto => Self::Auto, + FrameworkArg::Pytorch => Self::PyTorch, + FrameworkArg::LlamaCpp => Self::LlamaCpp, + FrameworkArg::Skip => Self::Skip, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum InstallFormat { + /// Python wheel packages (recommended; smaller, pip-installable). + Wheel, + /// Self-contained tarball archive of the ROCm SDK. + Tarball, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum Provider { + /// A local OpenAI-compatible server started via `rocm serve` (no API key). + Local, + /// Anthropic Claude API (requires an API key). + Anthropic, + /// OpenAI API or OpenAI-compatible endpoint (requires an API key). + Openai, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum WatcherModeArg { + /// Only report findings; never take action. + Observe, + /// Report findings and propose changes for you to approve. + Propose, + /// Apply changes automatically within a contained, reversible scope. + Contained, +} + +impl From for WatcherMode { + fn from(value: WatcherModeArg) -> Self { + match value { + WatcherModeArg::Observe => Self::Observe, + WatcherModeArg::Propose => Self::Propose, + WatcherModeArg::Contained => Self::Contained, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum TelemetryModeArg { + /// Collect GPU telemetry on this machine only; nothing leaves the device. + Local, + /// Disable telemetry collection entirely. + Off, +} + +/// User-facing `rocm serve --device` policy. +/// +/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage +/// validation (exit code 2, with the valid choices listed) rather than a generic +/// application error (exit code 1) — matching how every other enum-style CLI +/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`, +/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden +/// from the advertised list. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum DevicePolicyArg { + /// Require a ROCm GPU; fail if none is usable (the default). + #[value(alias = "auto", alias = "gpu")] + GpuRequired, + /// Accepted for compatibility; behaves identically to `gpu_required` + /// (rocm serve has no CPU fallback path). + GpuPreferred, + /// Accepted but always rejected: rocm serve has no CPU fallback path, so + /// this exits with an error. Hidden from `--help`, shell completions and the + /// invalid-value suggestion list so it is never advertised as usable, while + /// still parsing so the deliberate rejection message is preserved. + #[value(alias = "cpu", hide = true)] + CpuOnly, +} + +impl DevicePolicyArg { + /// Canonical policy string understood by [`crate::gpu_policy::parse_device_policy`]. Routing the + /// parsed variant back through the string keeps `parse_device_policy` the + /// single choke point that enforces the `cpu_only` rejection across every + /// caller, rather than adding a second conversion path that could bypass it. + pub(crate) const fn as_policy_str(self) -> &'static str { + match self { + Self::GpuRequired => "gpu_required", + Self::GpuPreferred => "gpu_preferred", + Self::CpuOnly => "cpu_only", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum SandboxToolArg { + ListServers, + RestartServer, + StopServer, +} + +impl SandboxToolArg { + pub(crate) const fn as_cli_value(self) -> &'static str { + match self { + Self::ListServers => "list_servers", + Self::RestartServer => "restart_server", + Self::StopServer => "stop_server", + } + } +} + +impl TelemetryModeArg { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Local => TELEMETRY_MODE_LOCAL, + Self::Off => TELEMETRY_MODE_OFF, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum PermissionsModeArg { + FullAccess, + Ask, +} + +impl PermissionsModeArg { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::FullAccess => PERMISSIONS_MODE_FULL_ACCESS, + Self::Ask => PERMISSIONS_MODE_ASK, + } + } +} diff --git a/apps/rocm/src/comfyui.rs b/apps/rocm/src/comfyui.rs index 2d68ad6b81..99efe4f55c 100644 --- a/apps/rocm/src/comfyui.rs +++ b/apps/rocm/src/comfyui.rs @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: MIT +use crate::cli::ComfyuiCommand; +use crate::cli_audit::record_cli_audit_event; use crate::therock; use anyhow::{Context, Result, bail}; use process::{comfyui_run_state_cli_label, endpoint_is_reachable, evaluate_running_state}; @@ -60,6 +62,169 @@ pub(crate) struct ComfyUiStartOptions { pub no_open_browser: bool, } +pub(crate) fn run(command: Option) -> Result<()> { + let paths = AppPaths::discover()?; + let config = RocmCliConfig::load(&paths).unwrap_or_default(); + match command.unwrap_or(ComfyuiCommand::Status) { + ComfyuiCommand::Status => { + print!("{}", render_status(&paths, &config)?); + record_cli_audit_event( + &paths, + "app", + "comfyui_status", + "info", + "rendered ComfyUI status", + None, + ); + Ok(()) + } + ComfyuiCommand::ModelsPath => { + print!("{}", render_models_path(&paths)?); + record_cli_audit_event( + &paths, + "app", + "comfyui_models_path", + "info", + "rendered ComfyUI models path", + None, + ); + Ok(()) + } + ComfyuiCommand::Logs { lines } => { + print!("{}", render_logs(&paths, lines)?); + record_cli_audit_event( + &paths, + "app", + "comfyui_logs", + "info", + "rendered ComfyUI logs", + None, + ); + Ok(()) + } + ComfyuiCommand::Install { + runtime_id, + reinstall, + dry_run, + yes: _, + } => { + match install( + &paths, + &config, + ComfyUiInstallOptions { + runtime_id: runtime_id.clone(), + reinstall, + dry_run, + }, + ) { + Ok(text) => { + print!("{text}"); + record_cli_audit_event( + &paths, + "app", + if dry_run { + "comfyui_install_dry_run" + } else { + "comfyui_install" + }, + "info", + format!( + "ComfyUI install completed runtime={} reinstall={} dry_run={}", + runtime_id.as_deref().unwrap_or(""), + reinstall, + dry_run + ), + None, + ); + Ok(()) + } + Err(error) => { + record_cli_audit_event( + &paths, + "app", + if dry_run { + "comfyui_install_dry_run" + } else { + "comfyui_install" + }, + "error", + format!( + "ComfyUI install failed runtime={} reinstall={} dry_run={}: {error}", + runtime_id.as_deref().unwrap_or(""), + reinstall, + dry_run + ), + None, + ); + Err(error) + } + } + } + ComfyuiCommand::Start { + host, + port, + no_open_browser, + yes: _, + } => match start( + &paths, + ComfyUiStartOptions { + host, + port, + no_open_browser, + }, + ) { + Ok(text) => { + print!("{text}"); + record_cli_audit_event( + &paths, + "app", + "comfyui_start", + "info", + "ComfyUI start requested", + None, + ); + Ok(()) + } + Err(error) => { + record_cli_audit_event( + &paths, + "app", + "comfyui_start", + "error", + format!("ComfyUI start failed: {error}"), + None, + ); + Err(error) + } + }, + ComfyuiCommand::Stop { yes: _ } => match stop(&paths) { + Ok(text) => { + print!("{text}"); + record_cli_audit_event( + &paths, + "app", + "comfyui_stop", + "info", + "ComfyUI stop requested", + None, + ); + Ok(()) + } + Err(error) => { + record_cli_audit_event( + &paths, + "app", + "comfyui_stop", + "error", + format!("ComfyUI stop failed: {error}"), + None, + ); + Err(error) + } + }, + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] struct ComfyUiManifest { app_id: String, @@ -354,13 +519,20 @@ fn render_logs_with_options( if show_file_locations { writeln!(output, " saved file: {}", path.display())?; } - let lines = read_tail_lines(path, limit) + let tail = read_log_tail(path, limit) .with_context(|| format!("failed to read {}", path.display()))?; - if lines.is_empty() { + if tail.newest_line_exceeds_the_read_bound() { + // The file has content; saying it is empty would be false. + writeln!( + output, + " No output shown: {}.", + rocm_core::newest_line_exceeds_bound_notice() + )?; + } else if tail.lines.is_empty() { writeln!(output, " The saved log is empty.")?; } else { writeln!(output, " latest output:")?; - for line in lines { + for line in tail.lines { writeln!(output, " {line}")?; } } @@ -503,66 +675,17 @@ fn comfyui_log_title(path: &Path) -> &'static str { } } -fn read_tail_lines(path: &Path, limit: usize) -> Result> { - let bytes = fs::read(path)?; - let text = strip_ansi_sequences(&decode_log_bytes(&bytes)); - let mut lines = text - .lines() - .rev() - .take(limit) - .map(str::to_owned) - .collect::>(); - lines.reverse(); - Ok(lines) -} - -fn decode_log_bytes(bytes: &[u8]) -> String { - if bytes.starts_with(&[0xff, 0xfe]) || looks_like_utf16_le(bytes) { - let start = if bytes.starts_with(&[0xff, 0xfe]) { - 2 - } else { - 0 - }; - let units = bytes[start..] - .chunks_exact(2) - .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) - .collect::>(); - return String::from_utf16_lossy(&units); - } - String::from_utf8_lossy(bytes).into_owned() -} - -fn looks_like_utf16_le(bytes: &[u8]) -> bool { - if bytes.len() < 8 { - return false; - } - let sample_len = bytes.len().min(512); - let zero_odd_bytes = bytes[..sample_len] - .iter() - .enumerate() - .filter(|(index, byte)| index % 2 == 1 && **byte == 0) - .count(); - zero_odd_bytes * 4 >= sample_len -} - -fn strip_ansi_sequences(text: &str) -> String { - let mut cleaned = String::with_capacity(text.len()); - let mut chars = text.chars().peekable(); - while let Some(ch) = chars.next() { - if ch != '\x1b' { - cleaned.push(ch); - continue; - } - if chars.peek() == Some(&'[') { - let _ = chars.next(); - for next in chars.by_ref() { - if ('@'..='~').contains(&next) { - break; - } - } - } - } - cleaned +/// The last `limit` lines of a saved log, read through the shared bounded tail +/// routine. A ComfyUI log may have been redirected by a Windows shell as +/// UTF-16 and carries the colour codes of the Python process, so it is +/// decoded the console way: UTF-16 detected, ANSI sequences stripped. +fn read_log_tail(path: &Path, limit: usize) -> Result { + rocm_core::tail_log_decoded( + path, + limit, + rocm_core::LOG_VIEW_MAX_TAIL_READ, + rocm_core::LogDecode::CONSOLE, + ) } fn load_manifest(paths: &AppPaths) -> Result> { @@ -997,6 +1120,222 @@ mod tests { Ok(()) } + /// The lines of [`read_log_tail`], for the tests that compare lines. + fn read_tail_lines(path: &Path, limit: usize) -> Result> { + Ok(read_log_tail(path, limit)?.lines) + } + + fn tail_fixture(name: &str, bytes: &[u8]) -> Result { + let paths = test_paths(name); + let dir = app_root(&paths).join("logs"); + fs::create_dir_all(&dir)?; + let path = dir.join("start-fixture.log"); + fs::write(&path, bytes)?; + Ok(path) + } + + fn utf16_le(text: &str, bom: bool) -> Vec { + let mut bytes = if bom { vec![0xff, 0xfe] } else { Vec::new() }; + for unit in text.encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + bytes + } + + /// 5 MiB: a marked first line, 5120 lines of 1 KiB, a marked last line. + fn large_log_with_head_marker() -> Vec { + let mut bytes = b"HEAD-MARKER\n".to_vec(); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..5120 { + bytes.extend_from_slice(filler.as_bytes()); + } + bytes.extend_from_slice(b"TAIL-MARKER\n"); + bytes + } + + /// What `render_logs` prints for a lone saved log with these bytes, and + /// the log's size on disk, so a message is asserted with the state it + /// describes. + fn render_lone_log(name: &str, bytes: &[u8]) -> Result<(String, u64)> { + let paths = test_paths(name); + let logs = app_root(&paths).join("logs"); + fs::create_dir_all(&logs)?; + let log = logs.join("start-fixture.log"); + fs::write(&log, bytes)?; + let size = fs::metadata(&log)?.len(); + Ok((render_logs(&paths, 5)?, size)) + } + + #[test] + fn logs_say_a_log_is_empty_only_when_it_is_and_name_the_bound_when_it_is_not() -> Result<()> { + let bound = rocm_core::LOG_VIEW_MAX_TAIL_READ; + assert_eq!( + bound, + 4 * 1024 * 1024, + "the notice below names this many MiB" + ); + let notice = "No output shown: the newest line is longer than the 4 MiB a log view reads."; + let over_the_bound = usize::try_from(bound)? + 1024; + + // An empty file: the log really is empty, and says so. + let (rendered, size) = render_lone_log("comfyui-logs-empty", b"")?; + assert_eq!(size, 0); + assert!(rendered.contains("The saved log is empty."), "{rendered}"); + assert!(!rendered.contains(notice), "{rendered}"); + + // One line longer than the bound, with and without a final newline, + // and as sixteen-bit text: the file has content, so it is not empty, + // and the notice names the bound. + let one_line = "x".repeat(over_the_bound).into_bytes(); + let mut with_newline = one_line.clone(); + with_newline.push(b'\n'); + let wide = utf16_le(&"y".repeat(over_the_bound / 2 + 1), true); + for (name, bytes) in [ + ("comfyui-logs-long-line", one_line), + ("comfyui-logs-long-line-newline", with_newline), + ("comfyui-logs-long-line-wide", wide), + ] { + let (rendered, size) = render_lone_log(name, &bytes)?; + assert!(size > bound, "{name}: the log is bigger than the bound"); + assert!(rendered.contains(notice), "{name}: {rendered}"); + assert!(!rendered.contains("is empty"), "{name}: {rendered}"); + assert!(!rendered.contains("latest output:"), "{name}: {rendered}"); + } + + // A big log whose newest lines are short is shown, not described. + let (rendered, size) = render_lone_log("comfyui-logs-big", &large_log_with_head_marker())?; + assert!(size > bound); + assert!(rendered.contains("TAIL-MARKER"), "{rendered}"); + assert!(!rendered.contains(notice), "{rendered}"); + Ok(()) + } + + #[test] + fn read_tail_lines_a_large_file_is_read_through_a_bounded_window() -> Result<()> { + // A line far outside the tail window is not returned however large the + // limit: the log is not read whole. + let log = tail_fixture("comfyui-tail-large", &large_log_with_head_marker())?; + + let lines = read_tail_lines(&log, 10_000)?; + fs::remove_file(&log)?; + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + assert!(lines.len() < 5122, "got {} lines", lines.len()); + assert!(lines.len() > 3500, "the window holds about 4 MiB of lines"); + Ok(()) + } + + #[test] + fn read_tail_lines_a_large_sixteen_bit_log_is_read_through_a_bounded_window() -> Result<()> { + // The window of a UTF-16 log must start on a character boundary or + // every following character decodes as garbage. + let mut text = String::from("HEAD-MARKER\r\n"); + let filler = format!("\u{1b}[33m{}\u{1b}[0m\r\n", "y".repeat(1000)); + for _ in 0..3000 { + text.push_str(&filler); + } + text.push_str("TAIL-MARKER\r\n"); + let log = tail_fixture("comfyui-tail-large-utf16", &utf16_le(&text, true))?; + assert!(fs::metadata(&log)?.len() > 6_000_000); + + let lines = read_tail_lines(&log, 10_000)?; + fs::remove_file(&log)?; + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + let whole = "y".repeat(1000); + for line in &lines[..lines.len() - 1] { + assert_eq!(line, &whole, "a partial leading fragment was kept"); + } + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_a_bomless_utf16_log_and_strips_ansi() -> Result<()> { + let text = "\u{1b}[32m[INFO]\u{1b}[0m one\r\n\u{1b}[1;31mtwo\u{1b}[0m\r\n"; + let log = tail_fixture("comfyui-tail-utf16-nobom", &utf16_le(text, false))?; + + let lines = read_tail_lines(&log, 5)?; + + assert_eq!(lines, vec!["[INFO] one", "two"]); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_ansi_from_an_eight_bit_log() -> Result<()> { + let log = tail_fixture("comfyui-tail-ansi-utf8", b"\x1b[32mgreen\x1b[0m done\n")?; + + assert_eq!(read_tail_lines(&log, 5)?, vec!["green done"]); + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_invalid_eight_bit_text_lossily() -> Result<()> { + let log = tail_fixture("comfyui-tail-invalid", b"ok\nbad\xffbyte\nlast\n")?; + + assert_eq!( + read_tail_lines(&log, 5)?, + vec!["ok", "bad\u{FFFD}byte", "last"] + ); + Ok(()) + } + + #[test] + fn read_tail_lines_missing_file_is_a_not_found_error_naming_the_path() { + let paths = test_paths("comfyui-tail-missing"); + let missing = app_root(&paths).join("logs").join("absent.log"); + + let error = read_tail_lines(&missing, 5).expect_err("a missing log must not read"); + + assert_eq!( + error.to_string(), + format!("failed to open {}", missing.display()) + ); + let io_error = error + .downcast_ref::() + .expect("the io error is kept as the cause"); + assert_eq!(io_error.kind(), io::ErrorKind::NotFound); + // A limit of 0 returns before the file is looked at. + assert!( + read_tail_lines(&missing, 0) + .expect("limit 0 never opens the file") + .is_empty() + ); + } + + #[test] + fn read_tail_lines_limit_zero_returns_nothing_for_an_existing_log() -> Result<()> { + let log = tail_fixture("comfyui-tail-zero", b"a\nb\n")?; + + assert!(read_tail_lines(&log, 0)?.is_empty()); + Ok(()) + } + + #[test] + fn read_tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let log = tail_fixture("comfyui-tail-few", b"a\nb\n")?; + + assert_eq!(read_tail_lines(&log, 50)?, vec!["a", "b"]); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let log = tail_fixture("comfyui-tail-no-newline", b"a\nb\nc")?; + + assert_eq!(read_tail_lines(&log, 2)?, vec!["b", "c"]); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_crlf_line_endings() -> Result<()> { + let log = tail_fixture("comfyui-tail-crlf", b"a\r\nb\r\n")?; + + assert_eq!(read_tail_lines(&log, 5)?, vec!["a", "b"]); + Ok(()) + } + #[test] fn unmanaged_running_url_reports_reachable_loopback_endpoint() -> Result<()> { let listener = TcpListener::bind("127.0.0.1:0")?; diff --git a/apps/rocm/src/comfyui/install.rs b/apps/rocm/src/comfyui/install.rs index 0ac2a164de..c446ddc3b6 100644 --- a/apps/rocm/src/comfyui/install.rs +++ b/apps/rocm/src/comfyui/install.rs @@ -27,6 +27,7 @@ use super::{ }; use crate::cli_progress::AnimatedSpinner; use crate::freeform_planner::format_structured_tool_call; +use crate::runtime_adopt::SetupRecovery; use crate::therock; use anyhow::{Context, Result, bail}; use flate2::read::GzDecoder; @@ -48,7 +49,14 @@ pub(crate) fn install( options: ComfyUiInstallOptions, ) -> Result { paths.ensure()?; - let runtime = select_runtime(paths, config, options.runtime_id.as_deref())?; + // A preview names the runtime the real run will use but does not write the + // registry entry that run would restore. + let runtime = select_runtime( + paths, + config, + options.runtime_id.as_deref(), + SetupRecovery::for_dry_run(options.dry_run), + )?; let app_root = runtime_app_root(&runtime.manifest); let source_path = source_path_from_app_root(&app_root); let log_path = install_log_path_from_app_root(&app_root); @@ -1908,6 +1916,170 @@ mod tests { remove_test_root(&paths); Ok(()) } + + /// `--dry-run` promises not to change files. With nothing chosen, the install + /// finds the runtime `rocm setup` created even though its registry entry was + /// lost, but a preview must not write that entry back: it names the same + /// runtime the real run will use and leaves the registry as it found it. + #[test] + fn dry_run_previews_the_setup_runtime_without_restoring_its_registry_entry() -> Result<()> { + let paths = test_paths("comfyui-dry-run-setup-runtime"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new() + ); + + let rendered = install( + &paths, + &config, + ComfyUiInstallOptions { + runtime_id: None, + reinstall: false, + dry_run: true, + }, + )?; + + assert!(rendered.contains(" mode: dry-run\n"), "{rendered}"); + assert!( + rendered.contains(&format!( + " ROCm runtime: {}\n", + therock::runtime_version_display(&manifest.version) + )), + "{rendered}" + ); + assert!( + rendered + .lines() + .any(|line| line.contains("install command:") + && line.contains(&format!("--runtime-id {}", manifest.runtime_key))), + "{rendered}" + ); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new(), + "a preview must not restore the registry entry" + ); + remove_test_root(&paths); + Ok(()) + } + + /// A runtime that setup created and the registry forgot, with a cached + /// ComfyUI archive and a fake interpreter, so a real install into it needs + /// no network and no GPU. + #[cfg(unix)] + fn setup_runtime_the_registry_forgot( + name: &str, + ) -> Result<(AppPaths, RocmCliConfig, therock::InstalledRuntimeManifest)> { + let paths = test_paths(name); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + let app_root = runtime_app_root(&manifest); + plant_cached_source_archive( + &app_root, + &[("main.py", "new main"), ("requirements.txt", "torch\n")], + )?; + let python = PathBuf::from( + manifest + .python_executable + .as_deref() + .context("runtime has no Python")?, + ); + rocm_core::fake_executable::write_fake_executable( + &python, + "#!/bin/sh\nprintf '%s' '{\"torch_version\": \"2.4.0\", \"torch_cuda_available\": true, \"device_count\": 1, \"devices\": [\"Fake GPU\"]}' > \"$2\"\n", + )?; + Ok((paths, config, manifest)) + } + + /// The preview prints the install command it would run, and that command + /// has to work from the state the preview left: the registry still without + /// the entry. Its `--runtime-id` is an explicit choice, which used to look + /// only in the registry and fail with "Install ROCm first". Following the + /// printed command previews again without a write, and runs for real with + /// the one write the real run makes. + #[cfg(unix)] + #[test] + fn the_printed_install_command_works_from_the_state_the_preview_left() -> Result<()> { + let (paths, config, manifest) = + setup_runtime_the_registry_forgot("comfyui-printed-command-setup-runtime")?; + let options = |runtime_id: Option, dry_run| ComfyUiInstallOptions { + runtime_id, + reinstall: false, + dry_run, + }; + let printed_runtime_id = |rendered: &str| -> Result { + let line = rendered + .lines() + .find(|line| line.contains("install command:")) + .with_context(|| format!("the preview prints its command:\n{rendered}"))?; + let words: Vec<&str> = line.split_whitespace().collect(); + let at = words + .iter() + .position(|word| *word == "--runtime-id") + .context("the printed command names the runtime")?; + Ok(words[at + 1].to_owned()) + }; + + let preview = install(&paths, &config, options(None, true))?; + let printed = printed_runtime_id(&preview)?; + assert_eq!(printed, manifest.runtime_key, "{preview}"); + assert!( + crate::test_support::runtime_registry_entry_names(&paths).is_empty(), + "the premise: the preview leaves the registry without the entry" + ); + + let again = install(&paths, &config, options(Some(printed.clone()), true)) + .context("the printed command, previewed again")?; + assert_eq!(printed_runtime_id(&again)?, printed, "{again}"); + assert!( + crate::test_support::runtime_registry_entry_names(&paths).is_empty(), + "previewing the printed command must not write the entry either" + ); + + install(&paths, &config, options(Some(printed), false)) + .context("the printed command, run for real")?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)], + "the real run restores the entry the printed command's runtime needs" + ); + remove_test_root(&paths); + Ok(()) + } + + /// The other half: the real run does restore the entry, so the runtime the + /// preview named is one `rocm runtimes list` shows once the install has run. + #[cfg(unix)] + #[test] + fn install_restores_the_setup_runtime_registry_entry_the_preview_left_alone() -> Result<()> { + let (paths, config, manifest) = + setup_runtime_the_registry_forgot("comfyui-install-setup-runtime")?; + + install( + &paths, + &config, + ComfyUiInstallOptions { + runtime_id: None, + reinstall: false, + dry_run: false, + }, + )?; + + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + remove_test_root(&paths); + Ok(()) + } } /// Property tests for the ComfyUI source unpack, judged on the filesystem by diff --git a/apps/rocm/src/comfyui/process.rs b/apps/rocm/src/comfyui/process.rs index ec324098d5..5b17ef58c8 100644 --- a/apps/rocm/src/comfyui/process.rs +++ b/apps/rocm/src/comfyui/process.rs @@ -419,7 +419,7 @@ fn process_is_running(pid: u32) -> bool { fn windows_process_is_running(pid: u32) -> bool { let filter = format!("PID eq {pid}"); - let mut command = Command::new("tasklist"); + let mut command = Command::new(rocm_core::windows_system_tool("tasklist.exe")); command.args(["/FI", &filter, "/FO", "CSV", "/NH"]); let Ok(output) = capture_configured_command(command, "check ComfyUI process") else { return false; @@ -468,7 +468,7 @@ fn terminate_process_tree(pid: u32) -> Result<()> { } fn terminate_windows_process_tree(pid: u32) -> Result<()> { - let status = Command::new("taskkill") + let status = Command::new(rocm_core::windows_system_tool("taskkill.exe")) .args(["/PID", &pid.to_string(), "/T", "/F"]) .stdin(Stdio::null()) .stdout(Stdio::null()) diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 3e496900be..70956108bb 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -7,7 +7,10 @@ //! //! [`select_runtime`] turns an explicit `--runtime-id` or the configured //! default into the one ready managed Python runtime an install runs against, -//! and explains how to pick one when none or several qualify. The +//! and explains how to pick one when none or several qualify. A choice that +//! names the runtime `rocm setup` created finds it even when its registry entry +//! was lost (see `pick_from_registry`), so the `--runtime-id` an install preview +//! prints works from the state the preview left. The //! `runtime_environment_*` functions build the `VIRTUAL_ENV`, `ROCM_PATH`, //! `PATH` and library-path settings a ComfyUI child process runs with, and //! [`apply_runtime_environment`] puts them on a command. @@ -17,12 +20,16 @@ //! ComfyUI types. use super::{ComfyUiManifest, ComfyUiRuntimeEnvironment, SelectedRuntime}; +use crate::runtime_adopt::{ + SetupRecovery, SingleReadyRuntime, recover_setup_runtime_registration, + registered_runtimes_with_setup, select_single_ready_runtime, +}; use crate::runtimes_cmd::runtime_usability_status; use crate::therock; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, RocmCliConfig, runtime_is_linux, runtime_is_windows, runtime_path_list_join, - runtime_path_list_split, runtime_paths_equivalent, + runtime_path_list_split, runtime_paths_equivalent, runtime_python_env_root_for_executable, }; use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -32,16 +39,13 @@ pub(super) fn select_runtime( paths: &AppPaths, config: &RocmCliConfig, selector: Option<&str>, + recovery: SetupRecovery, ) -> Result { - let manifests = therock::load_runtime_manifests(paths)?; - if manifests.is_empty() { - bail!( - "Install ROCm first: run `rocm` and pick \"Set up this system\", then install ComfyUI." - ); - } let manifest = match selector.map(str::trim).filter(|value| !value.is_empty()) { - Some(selector) => select_runtime_by_selector(&manifests, selector)?.clone(), - None => select_default_runtime(config, &manifests)?.clone(), + Some(selector) => pick_from_registry(paths, config, recovery, |manifests| { + select_runtime_by_selector(manifests, selector) + })?, + None => select_default_runtime(paths, config, recovery)?, }; if runtime_usability_status(&manifest) != "ready" { bail!( @@ -119,16 +123,85 @@ fn format_runtime_statuses<'a>( .join(", ") } -fn select_default_runtime<'a>( +/// Apply `pick` (a choice that names a runtime: `--runtime-id`, the active +/// runtime or the configured default) to the registry. A runtime that `rocm +/// setup` created and whose registry entry was lost is not a refusal either: +/// when the registry alone has no answer, `pick` runs once more over the +/// registry plus that runtime, the way the engine paths resolve an explicit +/// `--runtime-id`. Which write that implies is `recovery`'s: a preview adds the +/// runtime to the list without writing its entry, so the `--runtime-id` an +/// install preview prints is a command that works from the state the preview +/// left. If setup's runtime cannot be used, the registry's own answer stands. +/// +/// The two failures are not alike. Not finding a usable setup runtime leaves the +/// registry's refusal, because the user named something else or setup left +/// nothing. Failing to write the entry of a setup runtime that was found and +/// picked is not a refusal about a missing runtime, and swallowing it would +/// answer "Install ROCm first" about a runtime that is there, so that error is +/// returned as it is. The runtime is therefore found and picked without writing, +/// and written back only for a pick that is going to be used. +fn pick_from_registry( + paths: &AppPaths, config: &RocmCliConfig, - manifests: &'a [therock::InstalledRuntimeManifest], -) -> Result<&'a therock::InstalledRuntimeManifest> { + recovery: SetupRecovery, + pick: impl for<'a> Fn( + &'a [therock::InstalledRuntimeManifest], + ) -> Result<&'a therock::InstalledRuntimeManifest>, +) -> Result { + let registered = therock::load_runtime_manifests(paths)?; + let refusal = if registered.is_empty() { + install_rocm_first() + } else { + match pick(®istered) { + Ok(manifest) => return Ok(manifest.clone()), + Err(error) => error, + } + }; + // Found without writing, whichever recovery was asked for: the write comes + // after the pick, so its failure is reported as what it is. + let with_setup = match registered_runtimes_with_setup(paths, config, SetupRecovery::PreviewOnly) + { + Ok(with_setup) if with_setup.len() > registered.len() => with_setup, + _ => return Err(refusal), + }; + let picked = pick(&with_setup)?.clone(); + if recovery == SetupRecovery::Persist { + recover_setup_runtime_registration(paths, config)?; + } + Ok(picked) +} + +fn install_rocm_first() -> anyhow::Error { + anyhow::anyhow!( + "Install ROCm first: run `rocm` and pick \"Set up this system\", then install ComfyUI." + ) +} + +fn select_default_runtime( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, +) -> Result { if let Some(active_key) = config.active_runtime_key.as_deref() { - return select_runtime_by_selector(manifests, active_key); + return pick_from_registry(paths, config, recovery, |manifests| { + select_runtime_by_selector(manifests, active_key) + }); } - let Some(default_runtime_id) = config.default_runtime_id.as_deref() else { - return select_single_ready_runtime(manifests); - }; + match config.default_runtime_id.as_deref() { + Some(default_runtime_id) => pick_from_registry(paths, config, recovery, |manifests| { + select_runtime_by_default_id(manifests, default_runtime_id) + }), + // Nothing says which runtime to use: the rule the engine paths share. + None => { + single_ready_runtime_or_refusal(select_single_ready_runtime(paths, config, recovery)?) + } + } +} + +fn select_runtime_by_default_id<'a>( + manifests: &'a [therock::InstalledRuntimeManifest], + default_runtime_id: &str, +) -> Result<&'a therock::InstalledRuntimeManifest> { let matches = manifests .iter() .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(default_runtime_id)) @@ -156,23 +229,24 @@ fn select_default_runtime<'a>( } } -fn select_single_ready_runtime( - manifests: &[therock::InstalledRuntimeManifest], -) -> Result<&therock::InstalledRuntimeManifest> { - let ready = manifests - .iter() - .filter(|manifest| runtime_usability_status(manifest) == "ready") - .collect::>(); - match ready.as_slice() { - [manifest] => Ok(*manifest), - [] => { - let statuses = format_runtime_statuses(manifests); +/// ComfyUI's words for what [`select_single_ready_runtime`] found. The rule +/// itself, including which runtimes count as ready, is not decided here. +fn single_ready_runtime_or_refusal( + found: SingleReadyRuntime, +) -> Result { + match found { + SingleReadyRuntime::One(manifest) => Ok(*manifest), + SingleReadyRuntime::Zero { registered } if registered.is_empty() => { + Err(install_rocm_first()) + } + SingleReadyRuntime::Zero { registered } => { + let statuses = format_runtime_statuses(®istered); bail!( "No ROCm runtime is ready. Fix one or install another: run `rocm` and pick \"Set up this system\" (or run `rocm install sdk`), then install ComfyUI. Runtimes found: {statuses}. See `rocm runtimes list`." ) } - _ => { - let available = format_available_runtime_keys(ready.iter().copied()); + SingleReadyRuntime::Many { ready } => { + let available = format_available_runtime_keys(&ready); bail!( "Multiple ROCm runtimes are ready. Pick one in `/runtimes`, set a default with `rocm runtimes activate `, or pass `--runtime-id `. Available: {available}. See `rocm runtimes list`." ) @@ -237,7 +311,7 @@ pub(super) fn runtime_environment_for_manifest( } let mut env = ComfyUiRuntimeEnvironment { - venv_root: infer_python_env_root(&manifest.python_executable), + venv_root: runtime_python_env_root_for_executable(&manifest.python_executable), rocm_root: Some(manifest.runtime_root.clone()), path_entries: Vec::new(), library_entries: Vec::new(), @@ -254,7 +328,7 @@ pub(super) fn runtime_environment_from_runtime( python: &Path, ) -> ComfyUiRuntimeEnvironment { let mut env = ComfyUiRuntimeEnvironment { - venv_root: infer_python_env_root(python), + venv_root: runtime_python_env_root_for_executable(python), rocm_root: manifest .rocm_sdk .as_ref() @@ -307,15 +381,6 @@ fn collect_rocm_runtime_paths(root: &Path, env: &mut ComfyUiRuntimeEnvironment) } } -fn infer_python_env_root(python: &Path) -> Option { - let bin_dir = python.parent()?; - let name = bin_dir.file_name()?.to_string_lossy().to_ascii_lowercase(); - if matches!(name.as_str(), "scripts" | "bin") { - return bin_dir.parent().map(Path::to_path_buf); - } - None -} - pub(super) fn apply_runtime_environment( command: &mut Command, env: &ComfyUiRuntimeEnvironment, @@ -397,9 +462,22 @@ mod tests { use super::*; use crate::comfyui::render_status; use crate::comfyui::tests::{ready_runtime_manifest, test_paths, write_runtime_manifest}; + use crate::runtime_adopt::classify_single_ready_runtime; + use crate::test_support::{runtime_registry_entry_names, write_test_pip_runtime}; use rocm_core::{runtime_python_executable_in_env, runtime_rocm_library_filename}; use std::fs; + /// `select_runtime` as a real install runs it: a runtime that only setup + /// recovery can make visible has its registry entry written back. The + /// preview mode has its own tests. + fn persisting_select_runtime( + paths: &AppPaths, + config: &RocmCliConfig, + selector: Option<&str>, + ) -> Result { + select_runtime(paths, config, selector, SetupRecovery::Persist) + } + #[test] fn default_runtime_selection_uses_single_ready_runtime() -> Result<()> { let paths = test_paths("comfyui-single-ready-runtime"); @@ -461,7 +539,8 @@ mod tests { }; let manifests = [manifest]; - let selected = select_default_runtime(&RocmCliConfig::default(), &manifests)?; + let selected = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec()))?; assert_eq!( selected.runtime_key, "release-pip-gfx120x-all-7-13-0a20260511" @@ -510,8 +589,9 @@ mod tests { let nightly = ready_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [release, nightly]; - let error = select_default_runtime(&RocmCliConfig::default(), &manifests) - .expect_err("two ready runtimes should be ambiguous"); + let error = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec())) + .expect_err("two ready runtimes should be ambiguous"); let message = error.to_string(); assert_actionable(&message); @@ -545,8 +625,9 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "release-wheel-gfx94x-dcgpu-7-13-0")?; let manifests = [unusable]; - let error = select_default_runtime(&RocmCliConfig::default(), &manifests) - .expect_err("an unusable runtime is not ready"); + let error = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec())) + .expect_err("an unusable runtime is not ready"); let message = error.to_string(); assert!( @@ -628,7 +709,7 @@ mod tests { ); // `select_runtime`'s bail when nothing is registered at all. - let error = select_runtime(&paths, &RocmCliConfig::default(), None) + let error = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) .expect_err("no registered runtime means no selection"); let message = error.to_string(); assert!( @@ -653,11 +734,7 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [ready, unusable]; - let config = RocmCliConfig { - default_runtime_id: Some("no-such-runtime-id".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "no-such-runtime-id") .expect_err("configured default id matches nothing"); let message = error.to_string(); @@ -689,11 +766,7 @@ mod tests { let nightly = ready_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [release, nightly]; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "therock-release:gfx120X-all") .expect_err("two manifests share the configured default id"); let message = error.to_string(); @@ -721,11 +794,7 @@ mod tests { ); let manifests = [ready, unusable]; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "therock-release:gfx120X-all") .expect_err("two manifests share the configured default id"); let message = error.to_string(); @@ -847,7 +916,7 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "release-wheel-gfx94x-dcgpu-7-13-0")?; write_runtime_manifest(&paths, &unusable)?; - let error = select_runtime( + let error = persisting_select_runtime( &paths, &RocmCliConfig::default(), Some("release-wheel-gfx94x-dcgpu-7-13-0"), @@ -958,10 +1027,961 @@ mod tests { Ok(()) } + /// Zero runtimes registered: the refusal points at the launcher's setup row, + /// and selecting wrote nothing. + #[test] + fn comfyui_selection_with_no_registered_runtime_says_to_install_rocm_and_writes_nothing() + -> Result<()> { + let paths = test_paths("comfyui-selection-none-registered"); + + let message = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) + .expect_err("no registered runtime means no selection") + .to_string(); + + assert!( + message.contains("Install ROCm first") + && message.contains("pick \"Set up this system\""), + "unexpected refusal: {message}" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + Ok(()) + } + + /// One ready runtime and nothing chosen: it is the selection. + #[test] + fn comfyui_selection_with_one_ready_runtime_selects_it() -> Result<()> { + let paths = test_paths("comfyui-selection-one-ready"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + + let selected = persisting_select_runtime(&paths, &RocmCliConfig::default(), None)?; + + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + assert_eq!( + Some(selected.python.display().to_string()), + manifest.python_executable + ); + Ok(()) + } + + /// Registered but none ready: the refusal lists the runtime with the reason + /// it is unusable, and fixing that reason (here: putting the interpreter + /// back) is what clears it. + #[test] + fn comfyui_selection_with_only_an_unusable_runtime_clears_once_it_is_fixed() -> Result<()> { + let paths = test_paths("comfyui-selection-none-ready"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let python = PathBuf::from( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + ); + fs::remove_file(&python)?; + + let message = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) + .expect_err("an unusable runtime is not ready") + .to_string(); + + assert!( + message.contains("No ROCm runtime is ready") + && message.contains("rocm runtimes list") + && message.contains(&format!("`{}` (unusable", manifest.runtime_key)), + "unexpected refusal: {message}" + ); + assert_eq!( + runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + + fs::write(&python, "python")?; + let selected = persisting_select_runtime(&paths, &RocmCliConfig::default(), None)?; + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + Ok(()) + } + + /// Several ready runtimes and none chosen: the refusal names the remedies, + /// and each one clears it. `rocm runtimes activate ` is run as + /// `activate_runtime`, its implementation, and `--runtime-id` as the selector. + #[test] + fn comfyui_selection_with_several_ready_runtimes_names_remedies_that_clear_it() -> Result<()> { + let paths = test_paths("comfyui-selection-several-ready"); + let first = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let second = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + + let message = persisting_select_runtime(&paths, &config, None) + .expect_err("several ready runtimes and no choice must not select") + .to_string(); + + assert!( + message.contains("Multiple ROCm runtimes are ready") + && message.contains("rocm runtimes activate ") + && message.contains("--runtime-id ") + && message.contains(&first.runtime_key) + && message.contains(&second.runtime_key), + "unexpected refusal: {message}" + ); + + let selected = persisting_select_runtime(&paths, &config, Some(&first.runtime_key))?; + assert_eq!(selected.manifest.runtime_key, first.runtime_key); + + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &second.runtime_key)?; + let selected = persisting_select_runtime(&paths, &config, None)?; + assert_eq!(selected.manifest.runtime_key, second.runtime_key); + Ok(()) + } + + /// The setup-recovery half of the rule: a ready runtime that only + /// `recover_setup_runtime_registration` can make visible is found by the + /// ComfyUI path, as it is by the engine paths, and finding it re-files its + /// registry entry, so the selection names a key `rocm runtimes list` shows. + #[test] + fn comfyui_selection_recovers_the_setup_runtime_the_registry_lost() -> Result<()> { + let paths = test_paths("comfyui-selection-setup-recovery"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + assert!( + config.setup.therock_venv.is_some(), + "setup must name the folder for this test to mean anything" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + + let selected = persisting_select_runtime(&paths, &config, None)?; + + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + assert_eq!( + runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + Ok(()) + } + + /// Recovering the setup runtime can fail, and the ComfyUI path now reports + /// that failure as the engine paths do, rather than carrying on as if setup + /// had named nothing: a setup folder whose runtime is no longer usable is + /// named, with the reason, and nothing is written to the registry. + #[test] + fn comfyui_selection_reports_a_setup_runtime_that_is_no_longer_usable() -> Result<()> { + let paths = test_paths("comfyui-selection-setup-unusable"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + let config = RocmCliConfig::load(&paths)?; + + let message = persisting_select_runtime(&paths, &config, None) + .expect_err("an unusable setup runtime is an error") + .to_string(); + + assert!( + message.contains("is not usable") && message.contains(&manifest.runtime_key), + "got: {message}" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + Ok(()) + } + + /// A choice that names the runtime setup created, by `--runtime-id` (key or + /// id), the active runtime or the configured default, finds it even though + /// its registry entry was lost. That is what makes the `--runtime-id` an + /// install preview prints a command that works from the state the preview + /// left: a preview adds the runtime to what it searches and writes nothing, + /// a real run writes the entry back. Each row asserts the pick and the + /// registry together. + #[test] + fn a_choice_naming_the_setup_runtime_finds_it_after_its_registry_entry_was_lost() -> Result<()> + { + type Choice = fn(&therock::InstalledRuntimeManifest, &mut RocmCliConfig) -> Option; + let choices: [(&str, Choice); 4] = [ + ("explicit key", |manifest, _| { + Some(manifest.runtime_key.clone()) + }), + ("explicit id", |manifest, _| { + Some(manifest.runtime_id.clone()) + }), + ("active runtime", |manifest, config| { + config.active_runtime_key = Some(manifest.runtime_key.clone()); + None + }), + ("configured default", |manifest, config| { + config.default_runtime_id = Some(manifest.runtime_id.clone()); + None + }), + ]; + for (label, choice) in choices { + for recovery in [SetupRecovery::PreviewOnly, SetupRecovery::Persist] { + let paths = test_paths(&format!( + "comfyui-choice-setup-{}-{recovery:?}", + label.replace(' ', "-") + )); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let mut config = RocmCliConfig::load(&paths)?; + let selector = choice(&manifest, &mut config); + + let selected = select_runtime(&paths, &config, selector.as_deref(), recovery) + .with_context(|| format!("{label}, {recovery:?}"))?; + + assert_eq!( + selected.manifest.runtime_key, manifest.runtime_key, + "{label}, {recovery:?}" + ); + let expected_entries = match recovery { + SetupRecovery::PreviewOnly => Vec::new(), + SetupRecovery::Persist => vec![format!("{}.json", manifest.runtime_key)], + }; + assert_eq!( + runtime_registry_entry_names(&paths), + expected_entries, + "{label}, {recovery:?}" + ); + } + } + Ok(()) + } + + /// Finding the setup runtime for a named choice must not hide that its + /// registry entry cannot be written back. The preview of such an install + /// succeeds, because it writes nothing; the printed command is the real run, + /// and it has to say why it cannot restore the entry rather than answer as if + /// there were no runtime ("Install ROCm first"). Each row asserts the error + /// and the registry together: the failing run leaves the registry as it + /// found it, and the runtime was still found (the preview, and the + /// unwritable registry's own refusal for a name that is not setup's). + #[test] + fn a_choice_naming_the_setup_runtime_reports_a_registry_that_cannot_take_it_back() -> Result<()> + { + type Choice = fn(&therock::InstalledRuntimeManifest, &mut RocmCliConfig) -> Option; + let choices: [(&str, Choice); 5] = [ + ("explicit key", |manifest, _| { + Some(manifest.runtime_key.clone()) + }), + ("explicit id", |manifest, _| { + Some(manifest.runtime_id.clone()) + }), + ("active runtime", |manifest, config| { + config.active_runtime_key = Some(manifest.runtime_key.clone()); + None + }), + ("configured default", |manifest, config| { + config.default_runtime_id = Some(manifest.runtime_id.clone()); + None + }), + // Not a named choice, but the same persistence, reported the same way. + ("no choice", |_, _| None), + ]; + for (label, choice) in choices { + let paths = test_paths(&format!( + "comfyui-choice-unwritable-{}", + label.replace(' ', "-") + )); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let mut config = RocmCliConfig::load(&paths)?; + let selector = choice(&manifest, &mut config); + let registry = crate::test_support::make_runtime_registry_unwritable(&paths)?; + + let previewed = select_runtime( + &paths, + &config, + selector.as_deref(), + SetupRecovery::PreviewOnly, + ) + .with_context(|| format!("{label}: the preview writes nothing"))?; + assert_eq!( + previewed.manifest.runtime_key, manifest.runtime_key, + "{label}" + ); + + let error = + select_runtime(&paths, &config, selector.as_deref(), SetupRecovery::Persist) + .expect_err("a registry that cannot be written cannot take the entry back"); + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "failed to restore setup runtime `{}`", + manifest.runtime_key + )), + "{label}: got: {message}" + ); + assert!( + !message.contains("Install ROCm first") && !message.contains("not found"), + "{label}: the persistence error must not be replaced by a refusal about a missing runtime: {message}" + ); + assert_eq!( + fs::read_to_string(®istry)?, + "not a directory", + "{label}: a failed restore leaves the registry as it was" + ); + } + Ok(()) + } + + /// The same, with the registry readable and holding another runtime, which is + /// the state a read-only registry directory leaves. The refusal that stands + /// without the fix is then the registry's "not found", so the error is + /// compared with that. Where the directory stays writable regardless (an + /// administrator), the write succeeds and the test checks the restore + /// instead, so it cannot pass by writing nothing. + #[cfg(unix)] + #[test] + fn a_read_only_registry_directory_is_reported_not_hidden_behind_a_not_found() -> Result<()> { + use crate::runtimes_cmd::runtime_registry_dir; + use std::os::unix::fs::PermissionsExt; + + let paths = test_paths("comfyui-choice-read-only-registry"); + let setup = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig::load(&paths)?; + let registry = runtime_registry_dir(&paths); + let original = fs::metadata(®istry)?.permissions(); + fs::set_permissions(®istry, fs::Permissions::from_mode(0o555))?; + + let selected = select_runtime( + &paths, + &config, + Some(&setup.runtime_key), + SetupRecovery::Persist, + ); + + fs::set_permissions(®istry, original)?; + match selected { + Err(error) => { + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "failed to restore setup runtime `{}`", + setup.runtime_key + )) && !message.contains("not found"), + "got: {message}" + ); + assert!( + !runtime_registry_entry_names(&paths) + .contains(&format!("{}.json", setup.runtime_key)), + "a failed restore writes no entry" + ); + } + Ok(selected) => { + assert_eq!(selected.manifest.runtime_key, setup.runtime_key); + assert!( + runtime_registry_entry_names(&paths) + .contains(&format!("{}.json", setup.runtime_key)), + "the directory stayed writable, so the entry is restored" + ); + } + } + Ok(()) + } + + /// What the recovery must not turn into: a choice that names some other + /// runtime is answered by the registry whether or not setup's runtime is + /// usable, a name that matches nothing is still refused (with the setup + /// runtime among those found when it is usable), and an unusable setup + /// runtime leaves the registry's own refusal standing instead of replacing + /// it with an error about a runtime the user did not ask for. + #[test] + fn recovering_the_setup_runtime_does_not_change_the_answer_for_other_choices() -> Result<()> { + let paths = test_paths("comfyui-choice-other-than-setup"); + let setup = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let other = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig::load(&paths)?; + let registry = runtime_registry_entry_names(&paths); + assert_eq!(registry, [format!("{}.json", other.runtime_key)]); + + let picked = select_runtime( + &paths, + &config, + Some(&other.runtime_key), + SetupRecovery::PreviewOnly, + )?; + assert_eq!(picked.manifest.runtime_key, other.runtime_key); + + let miss = select_runtime( + &paths, + &config, + Some("no-such-runtime"), + SetupRecovery::PreviewOnly, + ) + .expect_err("nothing is called that") + .to_string(); + assert!( + miss.contains("ROCm runtime not found: `no-such-runtime`") + && miss.contains(&setup.runtime_key), + "{miss}" + ); + assert_eq!(runtime_registry_entry_names(&paths), registry); + + fs::remove_file( + setup + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + for recovery in [SetupRecovery::PreviewOnly, SetupRecovery::Persist] { + let picked = select_runtime(&paths, &config, Some(&other.runtime_key), recovery) + .with_context(|| format!("an unusable setup runtime, {recovery:?}"))?; + assert_eq!(picked.manifest.runtime_key, other.runtime_key); + let miss = select_runtime(&paths, &config, Some("no-such-runtime"), recovery) + .expect_err("nothing is called that") + .to_string(); + assert!( + miss.contains("ROCm runtime not found: `no-such-runtime`") + && !miss.contains("is not usable"), + "{recovery:?}: {miss}" + ); + assert_eq!( + runtime_registry_entry_names(&paths), + registry, + "{recovery:?}" + ); + } + Ok(()) + } + + /// A registry directory that stops writing, as far as this process can be + /// stopped: a read-only directory, which does not stop `root`. `None` where it + /// would not bite (the process can write anyway, or the host has no such + /// modes), so a caller skips the row instead of passing for the wrong reason. + /// Restores the mode on drop. + #[cfg(unix)] + struct ReadOnlyRegistry { + path: PathBuf, + original: fs::Permissions, + } + + #[cfg(unix)] + impl Drop for ReadOnlyRegistry { + fn drop(&mut self) { + let _ = fs::set_permissions(&self.path, self.original.clone()); + } + } + + #[cfg(unix)] + fn make_registry_read_only(paths: &AppPaths) -> Option { + use std::os::unix::fs::PermissionsExt; + + let path = crate::runtimes_cmd::runtime_registry_dir(paths); + let original = fs::metadata(&path).ok()?.permissions(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o555)).ok()?; + let guard = ReadOnlyRegistry { path, original }; + let probe = guard.path.join(".write-probe"); + if fs::write(&probe, "").is_ok() { + let _ = fs::remove_file(probe); + return None; + } + Some(guard) + } + + #[cfg(not(unix))] + struct ReadOnlyRegistry; + + #[cfg(not(unix))] + fn make_registry_read_only(_paths: &AppPaths) -> Option { + None + } + + /// Every state `rocm comfyui install` can pick a runtime from, one row each: + /// what the user chose (nothing, a registered runtime by name, the setup + /// runtime by name) against how many ready runtimes the registry holds (0, 1, + /// 2), what the setup folder holds (nothing, a usable runtime whose registry + /// entry was lost, a runtime that is no longer usable), whether the registry + /// takes a new entry, and whether the install is a preview or a real run. + /// + /// The rules the rows pin: a choice that names a registered runtime is + /// answered by the registry whatever setup holds. With nothing chosen the + /// ready runtimes are counted, and a setup runtime that cannot be used is not + /// one of them: it does not stop a registered ready runtime from being picked + /// (one) or refused as ambiguous (two), and is reported only when it was the + /// last hope (none registered). A setup runtime that can be used is counted, + /// and a failure to write its entry back is reported as that, whatever else + /// the count would have said. Each row asserts the message and the registry + /// together. + #[test] + fn comfyui_runtime_selection_state_matrix() -> Result<()> { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Chosen { + Nothing, + Registered, + SetupRuntime, + } + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum SetupFolder { + Absent, + Usable, + Unusable, + } + struct Expected { + selected: Option<&'static str>, + error_has: Vec, + error_lacks: &'static [&'static str], + setup_entry_written: bool, + } + + const SETUP_KEY: &str = "runtime-s"; + const MULTIPLE: &str = "Multiple ROCm runtimes are ready"; + let registered_keys = ["runtime-a", "runtime-b"]; + let restore_failed = format!("failed to restore setup runtime `{SETUP_KEY}`"); + let mut rows_run = 0; + let mut rows_skipped = 0; + + for chosen in [Chosen::Nothing, Chosen::Registered, Chosen::SetupRuntime] { + for ready in 0..=2usize { + for setup in [ + SetupFolder::Absent, + SetupFolder::Usable, + SetupFolder::Unusable, + ] { + for registry_writable in [true, false] { + for recovery in [SetupRecovery::Persist, SetupRecovery::PreviewOnly] { + if chosen == Chosen::Registered && ready == 0 { + continue; // nothing registered to name + } + let row = format!( + "{chosen:?}, {ready} ready, setup {setup:?}, writable {registry_writable}, {recovery:?}" + ); + let paths = test_paths(&format!( + "comfyui-selection-matrix-{}", + rows_run + rows_skipped + )); + for (index, key) in registered_keys.iter().take(ready).enumerate() { + write_test_pip_runtime( + &paths, + key, + "therock-release:gfx120X-all", + "7.14.0", + 10 + index as u128, + )?; + } + if setup != SetupFolder::Absent { + let manifest = + crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, SETUP_KEY, + )?; + if setup == SetupFolder::Unusable { + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + } + } + let config = RocmCliConfig::load(&paths)?; + // A registry that cannot take an entry: a plain file for + // an empty one (stops `root` too), a read-only directory + // where it holds runtimes. + let registry_path = crate::runtimes_cmd::runtime_registry_dir(&paths); + let _read_only; + if !registry_writable { + if ready == 0 { + crate::test_support::make_runtime_registry_unwritable(&paths)?; + } else { + let Some(guard) = make_registry_read_only(&paths) else { + rows_skipped += 1; + continue; + }; + _read_only = guard; + } + } + rows_run += 1; + + let persists = recovery == SetupRecovery::Persist; + let setup_usable = setup == SetupFolder::Usable; + let write_fails = persists && !registry_writable && setup_usable; + let writes = persists && registry_writable && setup_usable; + let ready_keys = registered_keys[..ready].to_vec(); + let expected = match chosen { + Chosen::Nothing => match (setup, ready) { + (SetupFolder::Usable, _) if write_fails => Expected { + selected: None, + error_has: vec![restore_failed.clone()], + error_lacks: &["Install ROCm first", MULTIPLE], + setup_entry_written: false, + }, + (SetupFolder::Usable, 0) => Expected { + selected: Some(SETUP_KEY), + error_has: vec![], + error_lacks: &[], + setup_entry_written: writes, + }, + (SetupFolder::Usable, _) => Expected { + selected: None, + error_has: std::iter::once(MULTIPLE.to_owned()) + .chain(ready_keys.iter().map(|key| (*key).to_owned())) + .chain([SETUP_KEY.to_owned()]) + .collect(), + error_lacks: &["Install ROCm first"], + setup_entry_written: writes, + }, + (SetupFolder::Unusable, 0) => Expected { + selected: None, + error_has: vec![ + format!("setup runtime `{SETUP_KEY}`"), + "is not usable".to_owned(), + ], + error_lacks: &["Install ROCm first"], + setup_entry_written: false, + }, + (SetupFolder::Absent, 0) => Expected { + selected: None, + error_has: vec!["Install ROCm first".to_owned()], + error_lacks: &["is not usable"], + setup_entry_written: false, + }, + (_, 1) => Expected { + selected: Some("runtime-a"), + error_has: vec![], + error_lacks: &[], + setup_entry_written: false, + }, + (_, _) => Expected { + selected: None, + error_has: std::iter::once(MULTIPLE.to_owned()) + .chain(ready_keys.iter().map(|key| (*key).to_owned())) + .collect(), + error_lacks: &["is not usable", SETUP_KEY], + setup_entry_written: false, + }, + }, + Chosen::Registered => Expected { + selected: Some("runtime-a"), + error_has: vec![], + error_lacks: &[], + setup_entry_written: false, + }, + Chosen::SetupRuntime => match setup { + SetupFolder::Usable if write_fails => Expected { + selected: None, + error_has: vec![restore_failed.clone()], + error_lacks: &["Install ROCm first", "not found"], + setup_entry_written: false, + }, + SetupFolder::Usable => Expected { + selected: Some(SETUP_KEY), + error_has: vec![], + error_lacks: &[], + setup_entry_written: writes, + }, + SetupFolder::Absent | SetupFolder::Unusable => Expected { + selected: None, + error_has: vec![if ready == 0 { + "Install ROCm first".to_owned() + } else { + format!("ROCm runtime not found: `{SETUP_KEY}`") + }], + error_lacks: &["is not usable"], + setup_entry_written: false, + }, + }, + }; + + let selector = match chosen { + Chosen::Nothing => None, + Chosen::Registered => Some("runtime-a"), + Chosen::SetupRuntime => Some(SETUP_KEY), + }; + let outcome = select_runtime(&paths, &config, selector, recovery); + + match (outcome, expected.selected) { + (Ok(selected), Some(key)) => { + assert_eq!(selected.manifest.runtime_key, key, "{row}"); + } + (Ok(selected), None) => panic!( + "{row}: expected a refusal, picked {}", + selected.manifest.runtime_key + ), + (Err(error), Some(key)) => { + panic!("{row}: expected {key}, got: {error:#}") + } + (Err(error), None) => { + let message = format!("{error:#}"); + for needle in &expected.error_has { + assert!( + message.contains(needle.as_str()), + "{row}: `{needle}` missing from: {message}" + ); + } + for needle in expected.error_lacks { + assert!( + !message.contains(needle), + "{row}: `{needle}` must not appear in: {message}" + ); + } + } + } + + let mut entries: Vec = + ready_keys.iter().map(|key| format!("{key}.json")).collect(); + if expected.setup_entry_written { + entries.push(format!("{SETUP_KEY}.json")); + } + entries.sort(); + assert_eq!(runtime_registry_entry_names(&paths), entries, "{row}"); + if !registry_writable && ready == 0 { + assert_eq!( + fs::read_to_string(®istry_path)?, + "not a directory", + "{row}: a failed restore leaves the registry as it was" + ); + } + } + } + } + } + } + // The rows that need a registry holding runtimes that refuses a write skip + // themselves where the process can write anyway; the rest must have run. + assert_eq!(rows_run + rows_skipped, 96, "the matrix has 96 rows"); + assert!(rows_run >= 60, "ran {rows_run}, skipped {rows_skipped}"); + Ok(()) + } + + /// The two paths that pick "the one ready runtime" must not drift apart: + /// for each registry state below, ComfyUI selects a runtime exactly when + /// the engine paths do, and the same one. Both reach the rule through + /// `select_single_ready_runtime`, so a copy of it re-introduced in either + /// caller (a different readiness test, a missing setup recovery) shows up as + /// a disagreement on one of these states. + #[test] + fn comfyui_and_engine_paths_agree_on_the_single_ready_runtime() -> Result<()> { + use crate::engine_runtime_select::{ + resolve_engine_selection, validate_engine_selection_runtime, + }; + + #[derive(Clone, Copy)] + enum State { + Empty, + OneUnusable, + OneReady, + TwoReady, + ReadyAndUnusable, + SetupRuntimeLost, + SetupRuntimeLostBesideAReadyOne, + UnusableSetupRuntimeBesideAReadyOne, + } + for (label, state, expected) in [ + ("empty", State::Empty, None), + ("one-unusable", State::OneUnusable, None), + ("one-ready", State::OneReady, Some("runtime-a")), + ("two-ready", State::TwoReady, None), + ( + "ready-and-unusable", + State::ReadyAndUnusable, + Some("runtime-a"), + ), + ( + "setup-runtime-lost", + State::SetupRuntimeLost, + Some("runtime-s"), + ), + ( + "setup-runtime-lost-beside-a-ready-one", + State::SetupRuntimeLostBesideAReadyOne, + None, + ), + ( + "unusable-setup-runtime-beside-a-ready-one", + State::UnusableSetupRuntimeBesideAReadyOne, + Some("runtime-a"), + ), + ] { + // Each path gets its own copy of the state: a recovery run by one + // writes the registry, which would otherwise hand the other path a + // state it could not have found for itself. + let plant_state = |paths: &AppPaths| -> Result<()> { + let plant = |key: &str| { + write_test_pip_runtime(paths, key, "therock-release:gfx120X-all", "7.14.0", 20) + }; + let break_runtime = |manifest: &therock::InstalledRuntimeManifest| -> Result<()> { + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + Ok(()) + }; + match state { + State::Empty => {} + State::OneUnusable => break_runtime(&plant("runtime-a")?)?, + State::OneReady => { + plant("runtime-a")?; + } + State::TwoReady => { + plant("runtime-a")?; + plant("runtime-b")?; + } + State::ReadyAndUnusable => { + plant("runtime-a")?; + break_runtime(&plant("runtime-b")?)?; + } + State::SetupRuntimeLost => { + crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?; + } + State::SetupRuntimeLostBesideAReadyOne => { + plant("runtime-a")?; + crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?; + } + State::UnusableSetupRuntimeBesideAReadyOne => { + plant("runtime-a")?; + break_runtime( + &crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?, + )?; + } + } + Ok(()) + }; + let comfyui_paths = test_paths(&format!("comfyui-engine-agree-{label}-comfyui")); + plant_state(&comfyui_paths)?; + let comfyui = persisting_select_runtime( + &comfyui_paths, + &RocmCliConfig::load(&comfyui_paths)?, + None, + ) + .ok() + .map(|selected| selected.manifest.runtime_key); + let engine_paths = test_paths(&format!("comfyui-engine-agree-{label}-engine")); + plant_state(&engine_paths)?; + let engine = validate_engine_selection_runtime( + &engine_paths, + resolve_engine_selection(&RocmCliConfig::load(&engine_paths)?, "vllm", None, None), + )? + .runtime_id; + + assert_eq!(comfyui, engine, "{label}: the two paths disagree"); + assert_eq!(comfyui.as_deref(), expected, "{label}"); + } + Ok(()) + } + fn command_env_value(command: &Command, key: &str) -> Option { command .get_envs() .find(|(name, _)| name.to_string_lossy() == key) .and_then(|(_, value)| value.map(OsString::from)) } + + /// The environment folder ComfyUI exports as `VIRTUAL_ENV`, by the folder + /// the interpreter sits in. Today only a `bin` or `Scripts` folder + /// (any case) identifies a venv; anything else leaves `VIRTUAL_ENV` unset + /// rather than pointing it at a guessed folder. + #[test] + fn runtime_environment_exports_virtual_env_only_for_a_bin_or_scripts_folder() -> Result<()> { + let cases: [(&str, Option<&str>); 7] = [ + ("/opt/venv/bin/python", Some("/opt/venv")), + ("/opt/venv/Scripts/python.exe", Some("/opt/venv")), + ("/opt/venv/SCRIPTS/python.exe", Some("/opt/venv")), + ("/opt/venv/BIN/python", Some("/opt/venv")), + ("/opt/venv/tools/python", None), + ("python", None), + ("/python", None), + ]; + for (python, expected) in cases { + let python = PathBuf::from(python); + let manifest = minimal_runtime_manifest(&python); + + let env = runtime_environment_from_runtime(&manifest, &python); + let mut command = Command::new(&python); + apply_runtime_environment(&mut command, &env)?; + + assert_eq!( + command_env_value(&command, "VIRTUAL_ENV").as_deref(), + expected.map(std::ffi::OsStr::new), + "{}", + python.display() + ); + } + Ok(()) + } + + fn minimal_runtime_manifest(python: &Path) -> therock::InstalledRuntimeManifest { + therock::InstalledRuntimeManifest { + runtime_key: "therock-release-gfx120x-all".to_owned(), + runtime_id: "therock-release:gfx120X-all".to_owned(), + channel: "release".to_owned(), + format: "wheel".to_owned(), + family: "gfx120X-all".to_owned(), + family_source: "test".to_owned(), + version: "7.13.0a20260511".to_owned(), + install_root: PathBuf::from("/nonexistent/runtime"), + selected_artifact_url: "https://example.invalid/simple".to_owned(), + source_layout_generation: None, + index_url: None, + tarball_file_name: None, + python_launcher: None, + python_executable: Some(python.display().to_string()), + pip_cache_dir: None, + rocm_sdk: None, + sdk_torch: None, + wheel_composition: None, + read_only: false, + imported_from: None, + devel: true, + installed_at_unix_ms: 100, + } + } } diff --git a/apps/rocm/src/config_cmd.rs b/apps/rocm/src/config_cmd.rs index 576a0813df..ffc81f3cd6 100644 --- a/apps/rocm/src/config_cmd.rs +++ b/apps/rocm/src/config_cmd.rs @@ -13,7 +13,7 @@ //! on, lives in `cli.rs` with the rest of the `Command`/`Cli` definition, and //! `dispatch()` calls `config` through the re-import `use //! crate::config_cmd::{...};`. `render_config_text` is also `pub(crate)`: -//! `chat.rs`'s read-only handling of `rocm config show` reaches it as +//! `chat/mcp_bridge.rs`'s read-only handling of `rocm config show` reaches it as //! `crate::render_config_text` through that re-import. `read_provider_key_from_user` //! and `telemetry_policy_summary` have no caller outside this module and stay //! private. diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 28346a4c05..c09f55f3c9 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -58,10 +58,11 @@ pub fn runner_options( instance_tick: Duration::from_secs_f64(d.instance_tick_secs), disable_vllm_metrics: false, vllm_metrics_host: "127.0.0.1".into(), - // Lemonade discovery stays opt-in (mirrors a no-flag embedded daemon). - enable_lemonade: false, - lemonade_host: "127.0.0.1".into(), - lemonade_port: 13305, + // Read-only discovery of local external Lemonade; managed registry + // discovery remains independent of this switch. + enable_lemonade: d.enable_lemonade, + lemonade_host: d.lemonade_host.clone(), + lemonade_port: d.lemonade_port, persist_dir: Some(paths.telemetry_state_dir()), // D7 seam consumer: managed services from `rocm serve --managed`. services_dir: Some(paths.services_dir()), @@ -258,6 +259,29 @@ fn automation_summaries(config: &RocmCliConfig) -> Vec { .collect() } +/// Read-only managed-engine inventory, independent of GPU telemetry and +/// endpoint liveness. An unavailable detect response means unknown, not absent. +fn lemonade_managed_installed(paths: &AppPaths) -> Option { + let detect = crate::engine_ipc::engine_request::<_, rocm_engine_protocol::DetectResponse>( + Some(paths), + "lemonade", + rocm_engine_protocol::EngineMethod::Detect, + &rocm_engine_protocol::DetectRequest::default(), + ) + .ok(); + managed_installed_from_detect(detect.as_ref()) +} + +/// Pure verdict over a Detect result: `None` when Detect failed (unknown, not +/// absent); otherwise installed only counts when the runtime is managed. A +/// legacy response without `managed_env` is not proof of a managed install. +fn managed_installed_from_detect( + detect: Option<&rocm_engine_protocol::DetectResponse>, +) -> Option { + let detect = detect?; + Some(detect.installed && detect.managed_env == Some(true)) +} + /// Resolve the TUI args from the unified config + environment. /// /// MUST be called on a synchronous thread *before* any tokio runtime is entered: @@ -328,6 +352,7 @@ pub fn resolved_args( // `resolved_args` itself has no opinion, so this always starts `false` // here. launcher_hosted: false, + lemonade_managed_installed: lemonade_managed_installed(paths), } } @@ -345,8 +370,9 @@ pub fn resolved_args( /// `rocm_dash_tui::client`'s `#[cfg(windows)]` arm does connect and stream /// live GPU/CPU/memory every tick but — by design, there being no /// `rocmdashd` unix socket to source one from (`maybe_spawn_embedded_daemon` -/// is unix-only) — never sends an instance snapshot, so -/// `AppState::instances` never gets a first one there. +/// is unix-only) — never sends an instance snapshot, so this probe is also +/// where `AppState::instances` comes from there: the dashboard polls it +/// (`ManagedRegistrySnapshot::instances`) once a second off the UI thread. /// /// **Known divergence, same shape as [`services_past_attempts`]'s:** this /// reads the record's raw on-disk `status`, not process liveness. A service @@ -366,11 +392,31 @@ impl ManagedRegistryProbe for LiveManagedRegistryProbe { let discovery = discover_managed_services(&records); ManagedRegistrySnapshot { has_live_service: discovery.svcs.iter().any(|svc| svc.status.is_serving()), + instances: managed_instances(&discovery), managed_ids: discovery.seen, } } } +/// The live managed services as dashboard instances with no telemetry yet — +/// the same mapping the daemon applies to a newly discovered service. +fn managed_instances( + discovery: &rocm_dash_daemon::registry::ManagedDiscovery, +) -> Vec { + discovery + .svcs + .iter() + .map(|svc| { + rocm_dash_core::traits::merge_instance( + svc, + &rocm_dash_core::traits::InstanceSample::default(), + 0, + 0, + ) + }) + .collect() +} + /// Managed-service records that are no longer running, read from the same /// registry `rocm services` reads. A status-only read over already-loaded /// records: no readiness probes, no daemon, no disk I/O of its own. @@ -706,18 +752,7 @@ pub fn run_launcher(chat_mock: bool) -> Result<()> { fn launcher_serving_instances(paths: &AppPaths) -> Vec { use rocm_dash_daemon::registry::{discover_managed_services, load_service_records}; let records = load_service_records(&paths.services_dir()); - discover_managed_services(&records) - .svcs - .into_iter() - .map(|svc| rocm_dash_core::metrics::Instance { - container_id: svc.container_id, - container_name: svc.container_name, - model_name: svc.model_name, - status: svc.status, - port: svc.port, - ..Default::default() - }) - .collect() + managed_instances(&discover_managed_services(&records)) } /// Entry point for a focused launcher flow (Set up / Serve / Diagnose). @@ -1080,6 +1115,63 @@ async fn maybe_spawn_embedded_daemon( mod tests { use super::*; + fn detect(installed: bool, managed_env: Option) -> rocm_engine_protocol::DetectResponse { + let mut v = serde_json::json!({ + "installed": installed, "env_id": null, "python_version": null, + "torch_version": null, "transformers_version": null, + "available_devices": [], "notes": [], + "capabilities": {"cpu": false, "rocm_gpu": false, "openai_compatible": true, + "tool_calling": false, "quantized_models": "", "reasoning_parser": false} + }); + if let Some(m) = managed_env { + v["managed_env"] = m.into(); + } + serde_json::from_value(v).unwrap() + } + + #[test] + fn managed_installed_verdict_follows_detect_result() { + assert_eq!( + managed_installed_from_detect(Some(&detect(true, Some(true)))), + Some(true) + ); + assert_eq!( + managed_installed_from_detect(Some(&detect(true, Some(false)))), + Some(false) + ); + // Legacy response without the ownership field proves nothing managed. + assert_eq!( + managed_installed_from_detect(Some(&detect(true, None))), + Some(false) + ); + assert_eq!( + managed_installed_from_detect(Some(&detect(false, Some(true)))), + Some(false) + ); + // A failed Detect is unknown, never "not installed". + assert_eq!(managed_installed_from_detect(None), None); + } + + #[test] + fn lemonade_discovery_defaults_and_overrides_reach_daemon() { + let default = runner_options(&RocmCliConfig::default(), &paths(), false); + assert!(default.enable_lemonade); + assert_eq!(default.lemonade_host, "127.0.0.1"); + assert_eq!(default.lemonade_port, 13305); + let config: RocmCliConfig = serde_json::from_str( + r#"{ + "dashboard": {"daemon": {"enable_lemonade": false, + "lemonade_host": "localhost", "lemonade_port": 14567}} + }"#, + ) + .unwrap(); + let overridden = runner_options(&config, &paths(), false); + assert!(!overridden.enable_lemonade); + assert_eq!(overridden.lemonade_host, "localhost"); + assert_eq!(overridden.lemonade_port, 14567); + assert_eq!(overridden.services_dir, Some(paths().services_dir())); + } + fn cfg() -> RocmCliConfig { RocmCliConfig::default() } @@ -1157,8 +1249,10 @@ mod tests { /// `running`, and there is no process or listener behind it. That status /// string is the whole of what decides the count — this path never consults /// a pid and never probes the port, and the on-disk view it deserializes - /// into (`rocm_dash_daemon::registry::ServiceRecord`) carries no pid fields - /// at all. Asserting 0 is asserting that no refresh happens on this path — + /// into (`rocm_dash_daemon::registry::ServiceRecord`) does carry pid fields + /// now, for the daemon's process attribution. Discovery on this path derives + /// ancestry roots from them too, but the count asserted here comes from the + /// record's status string alone and never reads a pid. Asserting 0 is asserting that no refresh happens on this path — /// if someone later adds one, this test fails and they are pointed at the /// doc comment explaining the trade-off rather than silently flipping a /// launch-time read into network I/O. @@ -1299,6 +1393,61 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// Windows has no daemon, so the dashboard's instance list comes from the + /// probe's `instances` (see `AppState::push_host_gpu_snapshot`). A ready + /// Lemonade record must come through with its model and port; a stopped + /// record and one with no bound port must not. + #[test] + fn live_registry_probe_carries_the_live_managed_instances() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join(".rocm-work") + .join("tests") + .join("dash") + .join(format!( + "probe-instances-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let p = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + std::fs::create_dir_all(p.services_dir()).unwrap(); + std::fs::write( + p.services_dir().join("ready.json"), + br#"{"service_id":"lemonade-qwen","engine":"lemonade","model_ref":"Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M","port":11436,"status":"ready","created_at_unix_ms":3}"#, + ) + .unwrap(); + std::fs::write( + p.services_dir().join("stopped.json"), + br#"{"service_id":"lemonade-old","engine":"lemonade","model_ref":"Qwen/Qwen2.5-1.5B-Instruct","port":11435,"status":"stopped","created_at_unix_ms":2}"#, + ) + .unwrap(); + std::fs::write( + p.services_dir().join("unbound.json"), + br#"{"service_id":"lemonade-unbound","engine":"lemonade","port":0,"status":"ready","created_at_unix_ms":1}"#, + ) + .unwrap(); + + let args = resolved_args(&cfg(), &p, ActiveTab::Home); + let instances = probe_snapshot(&args).instances; + assert_eq!( + instances.len(), + 1, + "only the live, bound record: {instances:?}" + ); + let inst = &instances[0]; + assert_eq!(inst.container_id, "lemonade-qwen"); + assert_eq!(inst.model_name, "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M"); + assert_eq!(inst.port, Some(11436)); + assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready); + + let _ = std::fs::remove_dir_all(&root); + } + /// Paired with `resolved_args_reports_no_live_managed_service_when_none_are_ready_or_running`: /// `managed_ids` and `has_live_service` (`ManagedRegistrySnapshot`) are two /// different questions over the same live read (liveness vs. reachability), so a record @@ -1913,6 +2062,9 @@ mod tests { assert_eq!(inst.model_name, "meta-llama/Llama-3.1-8B"); assert_eq!(inst.port, Some(11435)); assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready); + // Registry records are owned by this CLI; the flag must survive the map + // so lifecycle surfaces treat the launcher row as managed. + assert!(inst.managed, "registry-owned record must stay managed"); let _ = std::fs::remove_dir_all(&root); } diff --git a/apps/rocm/src/deletion_properties.rs b/apps/rocm/src/deletion_properties.rs index 243e2ab06c..a664dbef0b 100644 --- a/apps/rocm/src/deletion_properties.rs +++ b/apps/rocm/src/deletion_properties.rs @@ -673,6 +673,7 @@ fn uninstall_case( dry_run: true, ..options }, + crate::uninstall_plan::PlanAudience::Captured, ); let planned: BTreeSet = plan.actions.iter().map(|e| e.path.clone()).collect(); prop_assert_eq!(&listed_in_render(&rendered), &planned, "{}", rendered); diff --git a/apps/rocm/src/diagnose_cmd.rs b/apps/rocm/src/diagnose_cmd.rs index 0d53e615a6..7558cadeda 100644 --- a/apps/rocm/src/diagnose_cmd.rs +++ b/apps/rocm/src/diagnose_cmd.rs @@ -4,15 +4,17 @@ //! `rocm diagnose` and `rocm fix`: the command handlers (`diagnose`, `fix`) and //! the report step behind `rocm diagnose --report` — `show_prepared_report` -//! prints what a report would carry and offers it through `perform_delivery`, +//! prints what a report would carry (built by `prepare_cli_report`, which names +//! this build's leading version) and offers it through `perform_delivery`, //! which acts on a delivery decision with an injected `Opener`, and //! `established_entry` picks the catalog entry a report names. //! -//! Mechanically relocated from `main.rs` with no behavior change; see -//! `docs/architecture.md`. `diagnose` and `fix` are `pub(crate)`: `dispatch()` -//! calls them through the re-import `use crate::diagnose_cmd::{diagnose, -//! fix};`. The other three functions have no caller outside this module and -//! stay private. `diagnose` gets its `--model` verdict from +//! Relocated from `main.rs` with no behavior change; see +//! `docs/architecture.md`. Since then `--model` has gained the GGUF estimate's +//! flags, and `--report` ignores a model question and says so on stderr. +//! `diagnose` and `fix` are `pub(crate)`: `dispatch()` calls them through the +//! re-import `use crate::diagnose_cmd::{diagnose, fix};`. The other functions +//! have no caller outside this module and stay private. `diagnose` gets its `--model` verdict from //! `model_fit.rs`'s `assess_model_on_this_host`, and `fix` reports a failing //! exit code to `main()` through `FixExitCode`, which stays in `main.rs` with //! the rest of the exit-code plumbing and is reached as `crate::FixExitCode`. @@ -31,6 +33,7 @@ use serde::Serialize; use crate::FixExitCode; use crate::cli_report::{JsonOrText, print_json_or_text}; +use crate::diagnose_model; use crate::model_fit::assess_model_on_this_host; pub(crate) fn diagnose( @@ -38,7 +41,7 @@ pub(crate) fn diagnose( top: usize, json: bool, distro: Option, - model: Option, + model: Option, report_requested: bool, send: bool, ) -> Result<()> { @@ -95,8 +98,18 @@ pub(crate) fn diagnose( .as_ref() .is_some_and(|wsl| !wsl.locally_probed); let mut report = rocm_core::run_diagnose(&examination, &symptom.unwrap_or_default()); - if let Some(model_ref) = &model { - report.model = Some(assess_model_on_this_host(model_ref, &examination)); + // A prepared report carries no model verdict, and a GGUF estimate may read + // the model's header from the network -- which `--report` promises it never + // does -- so under `--report` the model is not assessed at all. + if report_requested && let Some(query) = &model { + eprintln!("{}", diagnose_model::report_ignores_model_note(query)); + } + if let Some(query) = model.as_ref().filter(|_| !report_requested) { + report.model = Some(assess_model_on_this_host( + &query.model_ref, + &examination, + &query.context, + )); } if report_requested { return show_prepared_report(&examination, &report, json, send); @@ -212,6 +225,21 @@ fn perform_delivery( format!("{SENDER_NOTICE}\n{outcome}") } +/// The report this machine would contribute, naming this build's leading +/// version: the one `rocm version` starts with. +fn prepare_cli_report( + examination: &rocm_core::Examination, + entry: Option<&str>, + fix_offered: bool, +) -> std::result::Result { + rocm_core::prepare_report( + examination, + entry, + fix_offered, + &crate::cli_version_number(), + ) +} + /// Print the report this machine would contribute, and send nothing. fn show_prepared_report( examination: &rocm_core::Examination, @@ -225,7 +253,7 @@ fn show_prepared_report( // looking for a fault. Anything scripting this reads the outcome from // `--json` rather than from the exit code, exactly as `rocm diagnose` itself // already asks callers to do. - match rocm_core::prepare_report(examination, entry, fix_offered) { + match prepare_cli_report(examination, entry, fix_offered) { Ok(prepared) => { if json { println!("{}", serde_json::to_string_pretty(&prepared)?); @@ -331,6 +359,35 @@ mod tests { use super::*; + /// `rocm diagnose --report` publishes the version `rocm version` leads + /// with, not `rocm-core`'s frozen package version. The e2e side is + /// `@id:diagnose-report-names-the-cli-version`. + #[test] + fn a_problem_report_names_the_version_rocm_version_leads_with() { + let examination = rocm_core::Examination { + os_family: "linux".to_owned(), + distro_id: "ubuntu".to_owned(), + distro_version: "22.04".to_owned(), + has_amd_gpu: true, + gpus: vec![rocm_core::examine::Gpu { + name: "test".to_owned(), + gfx_target: "gfx1100".to_owned(), + pci_id: String::new(), + is_apu: Some(false), + is_amd: true, + }], + ..rocm_core::Examination::default() + }; + let report = prepare_cli_report(&examination, None, false) + .expect("a released GPU must produce a report"); + let number = crate::cli_version_number(); + assert_eq!(report.cli_version, number); + assert!( + crate::cli_version_string().starts_with(&format!("{number} (")), + "{number:?} does not lead `rocm version`'s string" + ); + } + /// An opener that records rather than opens, and can be told to fail. /// /// The whole reason the opener is a trait: the real one spawns a browser @@ -474,6 +531,7 @@ mod tests { rocm_core::DiagnoseReport { has_match, matched: vec![rocm_core::Diagnosis { + symptom_matched: false, id: id.to_owned(), title: "under test".to_owned(), score, @@ -570,6 +628,7 @@ mod tests { route: None, undetermined_reason: None, warnings: Vec::new(), + estimate: None, }; let readiness_text = rocm_core::model_readiness::render_model_readiness_text(&readiness); report.model = Some(readiness); diff --git a/apps/rocm/src/diagnose_model.rs b/apps/rocm/src/diagnose_model.rs new file mode 100644 index 0000000000..789b61136f --- /dev/null +++ b/apps/rocm/src/diagnose_model.rs @@ -0,0 +1,501 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The binary's side of `rocm diagnose --model`'s GGUF context-size estimate. +//! +//! The estimate itself lives in `rocm-core` ([`rocm_core::model_readiness`] and +//! the `gguf*` modules). What is here is what only the binary knows: the flags +//! as clap parsed them ([`KvCacheTypeArg`], [`ModelQuery`]), what `--report` +//! says when it is handed a model question it will not answer, and the mask +//! caveat: whether a device-visibility variable is set (read here), over the +//! `amd-smi` rows the host's memory was classified from (passed in). +//! +//! The `Diagnose` command variant and its dispatch arm stay in `main.rs` with the +//! rest of the command tree; `diagnose()` is in `diagnose_cmd.rs`, and the host +//! assessment that calls [`attach_gguf_estimate`] in `model_fit.rs`. + +use crate::gpu_policy::GpuVramUsage; +use clap::ValueEnum; +use rocm_core::gguf_fit::KvCacheType; +use rocm_core::gguf_source::GgufSource; +use rocm_core::model_readiness::{ + ContextRequest, HostFacts, ModelReadiness, UndeterminedReason, attach_context_estimate, + note_unverified_mask, +}; + +/// KV cache element type for `rocm diagnose --model`'s GGUF estimate. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum KvCacheTypeArg { + /// 16-bit floats (llama.cpp's default). + #[value(name = "f16")] + F16, + /// 8-bit quantized: about half the size of f16. + #[value(name = "q8_0")] + Q8_0, + /// 4-bit quantized: about a quarter the size of f16. + #[value(name = "q4_0")] + Q4_0, +} + +impl From for KvCacheType { + fn from(arg: KvCacheTypeArg) -> Self { + match arg { + KvCacheTypeArg::F16 => Self::F16, + KvCacheTypeArg::Q8_0 => Self::Q8_0, + KvCacheTypeArg::Q4_0 => Self::Q4_0, + } + } +} + +/// What `rocm diagnose --model` was asked about. +pub(crate) struct ModelQuery { + pub(crate) model_ref: String, + pub(crate) context: ContextRequest, +} + +impl ModelQuery { + /// The question `--model` asks, with `--context-size` and `--kv-cache-type` + /// as given (clap refuses either without `--model`). + pub(crate) fn new( + model_ref: String, + context_size: Option, + kv_cache_type: Option, + ) -> Self { + Self { + model_ref, + context: ContextRequest { + context_size, + kv_cache_type: kv_cache_type.map(Into::into), + }, + } + } +} + +/// What `--report` says to stderr when it was given a model question: the report +/// is about this machine, so the model flags did nothing, and nothing about the +/// model was fetched. A flag that is accepted and silently has no effect reads as +/// an answer that was never asked for. +pub(crate) fn report_ignores_model_note(query: &ModelQuery) -> String { + let mut ignored = vec!["--model"]; + if query.context.context_size.is_some() { + ignored.push("--context-size"); + } + if query.context.kv_cache_type.is_some() { + ignored.push("--kv-cache-type"); + } + let verb = if ignored.len() == 1 { "was" } else { "were" }; + format!( + "note: --report describes this machine, not a model, so {} {verb} ignored and nothing \ + about the model was fetched", + ignored.join(", ") + ) +} + +/// Add the GGUF estimate to a readiness answer, and the mask caveat that goes +/// with it. +/// +/// `vram` is the `amd-smi` reading the host's memory was classified from, passed +/// in rather than read again: each read is a subprocess. +/// +/// A header read from the hub (a listing, a `HEAD`, then range requests, within +/// a two-minute budget) can take a while with nothing printed, so a spinner +/// covers it. It is drawn on stderr and only when stderr is a terminal, so +/// `--json` and piped output are unchanged. +pub(crate) fn attach_gguf_estimate( + readiness: &mut ModelReadiness, + context: &ContextRequest, + host: &HostFacts, + gguf_source: &dyn GgufSource, + vram: Option<&[GpuVramUsage]>, +) { + let _spinner = reads_from_the_hub(readiness).then(|| { + crate::cli_progress::AnimatedSpinner::start(format!( + "Reading the GGUF header of {} from Hugging Face...", + readiness.model_ref + )) + }); + attach_gguf_estimate_with( + readiness, + context, + host, + gguf_source, + vram, + device_visibility_mask_is_set(), + ); +} + +/// Whether estimating this answer reads a file header from the hub: a catalog +/// miss named as `owner/repo:QUANT` (or `owner/repo:file.gguf`). A curated model, +/// a local path and a bare `owner/repo` make no request. +fn reads_from_the_hub(readiness: &ModelReadiness) -> bool { + readiness.undetermined_reason == Some(UndeterminedReason::ModelNotCurated) + && readiness.canonical_model_id.is_none() + && matches!( + rocm_core::gguf_source::parse_gguf_ref(&readiness.model_ref), + Some(rocm_core::gguf_source::GgufRef::Hub { + variant: Some(_), + .. + }) + ) +} + +/// [`attach_gguf_estimate`] with whether a mask is set passed in, so a test can +/// drive it without touching the process environment. +fn attach_gguf_estimate_with( + readiness: &mut ModelReadiness, + context: &ContextRequest, + host: &HostFacts, + gguf_source: &dyn GgufSource, + vram: Option<&[GpuVramUsage]>, + mask_set: bool, +) { + // The one step that can touch a file or the network, and only for a GGUF + // reference the catalog does not carry; it reads the header, not the weights. + attach_context_estimate(readiness, context, host, gguf_source); + // Which card a device-visibility mask selects, in `amd-smi`'s numbering, has + // not been verified on a multi-GPU machine, so an estimate that compared + // memory says so when it could have compared the wrong card. After the + // estimate, never before: the caveat is attached to the estimate it doubts. + note_mask_for_rows(readiness, vram, mask_set); +} + +/// [`note_unverified_mask`] over the rows `amd-smi` reported. +fn note_mask_for_rows( + readiness: &mut ModelReadiness, + vram: Option<&[GpuVramUsage]>, + mask_set: bool, +) { + let totals: Vec = vram + .iter() + .flat_map(|rows| rows.iter()) + .map(|usage| usage.total_mb) + .collect(); + note_unverified_mask(readiness, mask_set, &totals); +} + +/// Whether any of the variables that hide GPUs from the runtime is set, even to +/// nothing (which hides every one). +/// +/// Wider than the mask [`rocm_core::usable_amd_gpu_indices`] applies, which +/// reads `HIP_VISIBLE_DEVICES` and `ROCR_VISIBLE_DEVICES` only: the HIP runtime +/// also honours `CUDA_VISIBLE_DEVICES`, which the memory classification does +/// not, so a figure compared under it is just as open to doubt. +fn device_visibility_mask_is_set() -> bool { + mask_set_where(|name| std::env::var_os(name).is_some()) +} + +/// The variables that hide GPUs from the runtime. The caveat text +/// ([`rocm_core::model_readiness::MASK_UNVERIFIED_NOTE`]) names each of them; +/// a test holds the two together. +const DEVICE_VISIBILITY_VARIABLES: [&str; 3] = [ + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", +]; + +/// Whether `is_set` holds for any of [`DEVICE_VISIBILITY_VARIABLES`]. +fn mask_set_where(is_set: impl Fn(&str) -> bool) -> bool { + DEVICE_VISIBILITY_VARIABLES.iter().any(|name| is_set(name)) +} + +#[cfg(test)] +mod tests { + use super::*; + use rocm_core::gguf_fit::{ContextEstimate, FitRequest}; + use rocm_core::model_readiness::{AcceleratorMemory, ModelCatalogSource}; + + /// The spinner is for a hub read only: a catalog miss named `owner/repo:QUANT`. + /// A local path, a bare repository, a name that is not a GGUF reference and a + /// curated model make no request, so they get none. + #[test] + fn only_a_hub_gguf_outside_the_catalog_is_read_from_the_hub() { + let registry = rocm_core::builtin_model_recipe_registry(); + let host = HostFacts { + accelerator_memory: AcceleratorMemory::Dedicated(24.0), + system_ram_gib: None, + }; + let answer = |model: &str| { + crate::model_fit::assess_model_for_host( + model, + &ModelCatalogSource::Available(®istry), + &host, + None, + None, + ) + }; + assert!(reads_from_the_hub(&answer( + "some-lab/never-curated-7b:Q4_K_M" + ))); + assert!(reads_from_the_hub(&answer( + "some-lab/never-curated-7b:dir/m-Q4_K_M.gguf" + ))); + for model in [ + "models/m.gguf", + "some-lab/never-curated-7b", + "not-a-gguf-at-all", + ] { + assert!(!reads_from_the_hub(&answer(model)), "{model}"); + } + let curated = ®istry.recipes[0].canonical_model_id; + assert!(!reads_from_the_hub(&answer(curated)), "{curated}"); + } + + /// A readiness whose estimate compared memory, built without a network or a file. + fn readiness_with_a_compared_estimate() -> ModelReadiness { + let mut readiness = crate::model_fit::assess_model_for_host( + "some-lab/never-curated-7b:Q4_K_M", + &ModelCatalogSource::Unreachable { + detail: "not needed here".to_owned(), + }, + &HostFacts { + accelerator_memory: AcceleratorMemory::Dedicated(24.0), + system_ram_gib: None, + }, + None, + None, + ); + let mut estimate = ContextEstimate::unreadable( + "x.gguf", + &FitRequest { + context_size: 4096, + context_size_defaulted: true, + kv_cache_type: KvCacheType::F16, + }, + &anyhow::anyhow!("not read"), + ); + estimate.available_gib = Some(24.0); + readiness.estimate = Some(estimate); + readiness + } + + fn gpu_row(index: u32, total_gib: u64) -> GpuVramUsage { + GpuVramUsage { + index, + used_mb: 0, + total_mb: total_gib * 1024, + } + } + + /// `--report` ignores the model flags it was given, and says which, so a flag + /// that did nothing is not taken for one that was answered. + #[test] + fn the_report_note_names_exactly_the_model_flags_that_were_ignored() { + let note = |context_size: Option, kv: Option| { + report_ignores_model_note(&ModelQuery::new("o/r:Q4_K_M".to_owned(), context_size, kv)) + }; + let alone = note(None, None); + assert!(alone.contains("--model was ignored"), "{alone}"); + assert!(!alone.contains("--context-size"), "{alone}"); + assert!(!alone.contains("--kv-cache-type"), "{alone}"); + let both = note(Some(4096), Some(KvCacheTypeArg::Q8_0)); + assert!( + both.contains("--model, --context-size, --kv-cache-type were ignored"), + "{both}" + ); + let size_only = note(Some(4096), None); + assert!( + size_only.contains("--model, --context-size were ignored") + && !size_only.contains("--kv-cache-type"), + "{size_only}" + ); + assert!( + both.contains("nothing about the model was fetched"), + "{both}" + ); + } + + /// The flags reach the request as given. + #[test] + fn the_query_carries_the_flags_as_given() { + let query = ModelQuery::new( + "o/r:Q4_K_M".to_owned(), + Some(8192), + Some(KvCacheTypeArg::Q4_0), + ); + assert_eq!(query.model_ref, "o/r:Q4_K_M"); + assert_eq!( + query.context, + ContextRequest { + context_size: Some(8192), + kv_cache_type: Some(KvCacheType::Q4_0), + } + ); + let unasked = ModelQuery::new("o/r:Q4_K_M".to_owned(), None, None); + assert_eq!(unasked.context, ContextRequest::default()); + } + + /// The glue between `amd-smi`'s rows and the caveat: cards of two sizes and a + /// mask say it, and every other host does not. + #[test] + fn the_unverified_mask_note_reaches_an_estimate_when_the_gpus_differ() { + let noted = |rows: Option<&[GpuVramUsage]>, mask_set: bool| { + let mut readiness = readiness_with_a_compared_estimate(); + note_mask_for_rows(&mut readiness, rows, mask_set); + readiness + .estimate + .unwrap() + .notes + .iter() + .any(|note| note.contains("has not been verified")) + }; + let mixed = [gpu_row(0, 24), gpu_row(1, 8)]; + let same = [gpu_row(0, 24), gpu_row(1, 24)]; + assert!(noted(Some(&mixed), true)); + assert!(!noted(Some(&mixed), false), "no mask, nothing to doubt"); + assert!(!noted(Some(&same), true), "the same card either way"); + assert!(!noted(None, true), "no rows, no figure compared"); + } + + /// Each variable on its own counts as a mask, and the caveat names each. + #[test] + fn each_visibility_variable_alone_is_a_mask_and_the_caveat_names_it() { + // The three the runtime honours, named here so dropping one from the + // constant fails; and every entry of the constant, so adding one that + // the caveat does not name fails too. + for variable in [ + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "CUDA_VISIBLE_DEVICES", + ] { + assert!( + DEVICE_VISIBILITY_VARIABLES.contains(&variable), + "{variable}" + ); + } + for variable in DEVICE_VISIBILITY_VARIABLES { + assert!(mask_set_where(|name| name == variable), "{variable}"); + assert!( + rocm_core::model_readiness::MASK_UNVERIFIED_NOTE.contains(variable), + "{variable}" + ); + } + assert!(!mask_set_where(|_| false)); + } + + /// A GGUF of a few kilobytes: 32 layers, 8 KV heads of width 128, so its + /// KV cache is real and an estimate compares it against the card. + fn tiny_gguf() -> Vec { + fn string(out: &mut Vec, value: &str) { + out.extend_from_slice(&(value.len() as u64).to_le_bytes()); + out.extend_from_slice(value.as_bytes()); + } + fn entry_u32(out: &mut Vec, key: &str, value: u32) { + string(out, key); + out.extend_from_slice(&4_u32.to_le_bytes()); + out.extend_from_slice(&value.to_le_bytes()); + } + let mut metadata = Vec::new(); + string(&mut metadata, "general.architecture"); + metadata.extend_from_slice(&8_u32.to_le_bytes()); + string(&mut metadata, "t"); + entry_u32(&mut metadata, "t.block_count", 32); + entry_u32(&mut metadata, "t.embedding_length", 4096); + entry_u32(&mut metadata, "t.feed_forward_length", 14336); + entry_u32(&mut metadata, "t.attention.head_count", 32); + entry_u32(&mut metadata, "t.attention.head_count_kv", 8); + entry_u32(&mut metadata, "t.context_length", 8192); + let mut file = Vec::new(); + file.extend_from_slice(b"GGUF"); + file.extend_from_slice(&3_u32.to_le_bytes()); + file.extend_from_slice(&1_u64.to_le_bytes()); + file.extend_from_slice(&7_u64.to_le_bytes()); + file.extend_from_slice(&metadata); + string(&mut file, "w"); + file.extend_from_slice(&1_u32.to_le_bytes()); + file.extend_from_slice(&1024_u64.to_le_bytes()); + file.extend_from_slice(&0_u32.to_le_bytes()); + file.extend_from_slice(&0_u64.to_le_bytes()); + file.resize(file.len().div_ceil(32) * 32, 0); + file + } + + struct OneFile(Vec); + + impl GgufSource for OneFile { + fn open( + &self, + _reference: &rocm_core::gguf_source::GgufRef, + ) -> anyhow::Result { + Ok(rocm_core::gguf_source::OpenedGguf { + reader: Box::new(std::io::Cursor::new(self.0.clone())), + size: self.0.len() as u64 + 1024 * 1024 * 1024, + description: "t.gguf".to_owned(), + }) + } + } + + /// The glue end to end: an estimate is read through the source and compared + /// with the card, and the mask caveat lands on it -- which it can only do + /// once the estimate exists, so the order of the two steps matters. + #[test] + fn attaching_an_estimate_under_a_mask_on_mixed_gpus_carries_the_caveat() { + let registry = rocm_core::builtin_model_recipe_registry(); + let host = HostFacts { + accelerator_memory: AcceleratorMemory::Dedicated(24.0), + system_ram_gib: Some(64.0), + }; + let attach = |mask_set: bool, rows: &[GpuVramUsage]| { + let mut readiness = crate::model_fit::assess_model_for_host( + "some-lab/never-curated-7b:Q4_K_M", + &ModelCatalogSource::Available(®istry), + &host, + None, + None, + ); + attach_gguf_estimate_with( + &mut readiness, + &ContextRequest::default(), + &host, + &OneFile(tiny_gguf()), + Some(rows), + mask_set, + ); + readiness + }; + let mixed = [gpu_row(0, 24), gpu_row(1, 8)]; + let noted = attach(true, &mixed); + let estimate = noted.estimate.as_ref().expect("an estimate was attached"); + // Sized and compared, not just tagged with the card: `available_gib` is + // set before sizing, so on its own it would hold for an estimate that + // failed. + assert_eq!( + estimate.outcome, + rocm_core::gguf_fit::EstimateOutcome::Fits, + "{estimate:?}" + ); + assert!(estimate.total_gib.is_some(), "{estimate:?}"); + assert!( + estimate + .notes + .iter() + .any(|note| note == rocm_core::model_readiness::MASK_UNVERIFIED_NOTE), + "{:?}", + estimate.notes + ); + let unmasked = attach(false, &mixed); + assert!( + !unmasked + .estimate + .expect("an estimate") + .notes + .iter() + .any(|note| note == rocm_core::model_readiness::MASK_UNVERIFIED_NOTE) + ); + } + + // Every spelling the flag takes maps to the type the estimate uses, so a + // mismatch between the two enums cannot silently estimate another cache. + #[test] + fn every_kv_cache_flag_value_names_the_matching_estimate_type() { + for arg in KvCacheTypeArg::value_variants() { + let name = arg + .to_possible_value() + .expect("named") + .get_name() + .to_owned(); + let kind = KvCacheType::from(*arg); + assert_eq!(kind.name(), name); + } + } +} diff --git a/apps/rocm/src/driver_install.rs b/apps/rocm/src/driver_install.rs index 3c2a73f942..59d6cb7bef 100644 --- a/apps/rocm/src/driver_install.rs +++ b/apps/rocm/src/driver_install.rs @@ -37,7 +37,6 @@ use crate::driver_plan::{ DriverCommandPhase, DriverInstallPlan, PrivilegeEscalation, build_driver_install_plan_read, render_driver_install_plan, }; -use crate::system_packages::real_os_release; /// Whether `rocm install driver` goes on to run the plan's commands. Pure so /// the gate that keeps native Windows from installing anything (its plan is @@ -62,7 +61,7 @@ pub(crate) fn install_driver( ) -> std::result::Result { let examine = ExamineSummary::gather().map_err(|source| DriverInstallError::new(source, false))?; - let os_release = real_os_release(); + let os_release = rocm_core::hardware_root::real_os_release(); // The only place the real privilege level is read; every builder below takes // it as a parameter so both branches stay testable on any host. let plan = diff --git a/apps/rocm/src/driver_plan.rs b/apps/rocm/src/driver_plan.rs index 0c71110f07..44cd8810e5 100644 --- a/apps/rocm/src/driver_plan.rs +++ b/apps/rocm/src/driver_plan.rs @@ -1445,9 +1445,9 @@ pub(crate) mod tests { } /// `build_driver_install_plan_read` is the one seam `install_driver` hands - /// its `real_os_release()` result to. Both are now one-line delegations - /// to `hardware_root::read_os_release`, so reverting either back to a raw - /// `/etc/os-release` read is a type error here, not a silent regression. + /// `hardware_root::real_os_release()`'s result to, so reverting that back to + /// a raw `/etc/os-release` read is a type error here, not a silent + /// regression. /// This still pins the behaviour at the seam itself: an `OsRelease` that /// says the fallback path was read plans exactly what a primary-path read /// with the same content would. diff --git a/apps/rocm/src/endpoint_keys.rs b/apps/rocm/src/endpoint_keys.rs index a40ee68ab6..e85fc3d216 100644 --- a/apps/rocm/src/endpoint_keys.rs +++ b/apps/rocm/src/endpoint_keys.rs @@ -5,7 +5,7 @@ //! Secure storage for per-service inference-endpoint API keys. //! //! When `rocm serve` binds a public (non-loopback) interface it protects the -//! endpoint with an API key (see `resolve_endpoint_auth` in `service_launch.rs`). +//! endpoint with an API key (see `resolve_endpoint_auth` in `service_launch/endpoint_auth.rs`). //! The key is stored in a 0600 file under the services directory, keyed by //! service id. //! diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 3e9565d00e..3c061bc26f 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -4,35 +4,34 @@ //! Engine plugin binary resolution and the engine IPC request/response //! envelope: locating an engine's executable (packaged plugin vs. the -//! platform-packaged fallback), encoding/decoding the JSON-over-stdio -//! envelope used to talk to it, and the built-in (in-process) fast path for -//! `lemonade`/`vllm` that skips spawning a child process entirely. +//! platform-packaged fallback), talking to it over JSON-over-stdio, and the +//! built-in (in-process) fast path for `lemonade`/`vllm`/`aim` that skips +//! spawning a child process entirely. +//! +//! The stdio conversation itself is not implemented here. The request framing, +//! the child's environment, stderr handling, the SIGPIPE guard around the write +//! and the reading of the answer are the client in +//! [`rocm_engine_protocol`]'s `stdio_client` module, which `rocmd` uses too. +//! This module chooses what to run (the built-in handler, or the resolved +//! engine binary with `stdio`) and what `rocm` asks of the client for it +//! ([`engine_stdio_child_request`]). //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-213) of the `main.rs` modularization (see `docs/architecture.md`). //! `dispatch()` calls directly into this module at two points — //! `Command::EngineStdio { engine } => run_builtin_engine_stdio(&engine)` and //! the `__engine-serve-http` arm's `run_builtin_engine_serve_http(...)` call — //! both re-imported via `use crate::engine_ipc::{run_builtin_engine_serve_http, -//! run_builtin_engine_stdio};`; `dispatch()`'s own body is otherwise -//! unaffected. +//! run_builtin_engine_stdio};`. //! //! This module owns: resolving an engine's binary path //! (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, -//! `find_engine_plugin_binary`), the stdio request/response envelope used to -//! talk to an external engine binary or the in-process built-in handler +//! `find_engine_plugin_binary`), the request entry points //! (`engine_request`, `engine_request_with_env_root`, -//! `decode_engine_response`, the `ScopedEnvVar`/ +//! `engine_stdio_child_request`), the `ScopedEnvVar`/ //! `with_scoped_builtin_engine_env` env-scoping helpers and the //! `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, -//! `run_builtin_engine_stdio`), and the built-in HTTP-serving path +//! `run_builtin_engine_stdio`, and the built-in HTTP-serving path //! (`builtin_engine_available`, `run_builtin_engine_serve_http`, -//! `builtin_engine_serve_http_args`). One item sits physically inside this -//! span in `main.rs` but stays there rather than moving: `impl -//! From for WatcherMode` is unrelated clap-arg-to-domain-type -//! conversion that only happens to sit between `find_engine_plugin_binary` and -//! `engine_request` in the original file (it has since moved to `cli.rs`, -//! next to the enum it converts). +//! `builtin_engine_serve_http_args`). //! //! `main.rs` calls into it for `resolve_engine_binary_path`, //! `resolve_engine_binary_path_with_paths`, `engine_request`, @@ -42,39 +41,31 @@ //! `runtime_install_finalize.rs` call into it for //! `engine_request_with_env_root`; `service_launch.rs` calls into it for //! `builtin_engine_serve_http_args`. `find_engine_plugin_binary`, -//! `decode_engine_response`, `with_scoped_builtin_engine_env`, -//! `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this -//! module and stay private. Shared helpers this module calls back into at the -//! crate root — `with_sigpipe_ignored` and `device_policy_name` — stay in -//! `main.rs` and are reached unqualified via `use crate::{...}`, same as for -//! other mechanically relocated modules. `parse_device_policy` moved out of -//! `main.rs` into `gpu_policy.rs` in ROCMAI-214 (Phase 6f); it is reached via -//! `use crate::gpu_policy::parse_device_policy;` instead. `apply_app_path_env`, -//! `app_path_env_vars`, `optional_arg`, and `engine_recipe_json_arg` moved out -//! of `main.rs` into `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f); this -//! file's `use` list below points there now instead of at the crate root. +//! `with_scoped_builtin_engine_env`, `builtin_engine_request`, and +//! `ScopedEnvVar` have no callers outside this module and stay private. The +//! one helper this module calls back into at the crate root, `device_policy_name`, +//! stays in `main.rs` and is reached via `use crate::device_policy_name;`. +//! `parse_device_policy` is in `gpu_policy.rs`, `engine_recipe_json_arg` is in +//! `engine_runtime_select.rs`, and `optional_arg` is in `rocm_engine_protocol`. use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; -use std::process::{Command as ProcessCommand, Stdio}; use std::sync::{Mutex, OnceLock}; -use std::thread; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, engine_binary_path, engine_plugin_dirs}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, engine_binary_path, engine_plugin_dirs}; use rocm_engine_protocol::{ DevicePolicy, EngineMethod, EnginePluginDescriptor, EngineRecipeHint, EngineRequestEnvelope, - EngineResponseEnvelope, + EngineResponseEnvelope, EngineStdioOptions, decode_engine_response, engine_request_envelope, + optional_arg, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use crate::engine_runtime_select::{ - app_path_env_vars, apply_app_path_env, engine_recipe_json_arg, optional_arg, -}; +use crate::device_policy_name; +use crate::engine_runtime_select::engine_recipe_json_arg; use crate::gpu_policy::parse_device_policy; -use crate::{device_policy_name, with_sigpipe_ignored}; static BUILTIN_ENGINE_ENV_LOCK: OnceLock> = OnceLock::new(); @@ -130,11 +121,7 @@ where R: DeserializeOwned, { let stream_progress = matches!(&method, EngineMethod::Install); - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request) - .context("failed to encode engine request payload")?, - }; + let envelope = engine_request_envelope(method, request)?; if let Some(envelope) = with_scoped_builtin_engine_env(paths, env_root, || { builtin_engine_request(engine, &envelope) }) { @@ -146,118 +133,57 @@ where "unable to locate engine binary for {engine}; build the workspace or install the engine package" ) })?; - let mut command = ProcessCommand::new(engine_binary); - command.arg("stdio"); - if let Some(paths) = paths { - apply_app_path_env(&mut command, paths); - } - if let Some(env_root) = env_root { - command.env("ROCM_CLI_ENGINE_ENVS_ROOT", env_root); - } - if stream_progress { - command.env("ROCM_ENGINE_PROGRESS_STDERR", "1"); - } - let mut child = command - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn engine stdio process")?; - - { - let mut stdin = child - .stdin - .take() - .context("engine stdio child did not expose stdin")?; - // Ignore SIGPIPE for the duration of the write: with SIG_DFL in effect - // process-wide, an engine that exits before reading its stdin would - // otherwise kill `rocm` with SIGPIPE before the exit-status diagnostics - // below can run. Under SIG_IGN the write returns EPIPE instead, which we - // surface as the normal "failed to write engine request" error and then - // let `wait()` report why the engine exited. - with_sigpipe_ignored(|| { - serde_json::to_writer(&mut stdin, &envelope) - .context("failed to write engine request")?; - stdin.write_all(b"\n")?; - Ok::<(), anyhow::Error>(()) - })?; - } - - let stderr_handle = child.stderr.take().map(|stderr| { - thread::spawn(move || { - let mut reader = io::BufReader::new(stderr); - let mut collected = Vec::new(); - let mut line = Vec::new(); - loop { - line.clear(); - match reader.read_until(b'\n', &mut line) { - Ok(0) => break, - Ok(_) => { - if let Ok(text) = std::str::from_utf8(&line) { - eprint!("{text}"); - } else { - let _ = io::stderr().write_all(&line); - } - collected.extend_from_slice(&line); - } - Err(error) => { - let message = format!("failed to read engine progress: {error}\n"); - eprint!("{message}"); - collected.extend_from_slice(message.as_bytes()); - break; - } - } - } - collected - }) - }); - let mut stdout = Vec::new(); - child - .stdout - .take() - .context("engine stdio child did not expose stdout")? - .read_to_end(&mut stdout) - .context("failed to read engine stdio response")?; - let status = child - .wait() - .context("failed waiting for engine stdio response")?; - let stderr = stderr_handle - .map(|handle| handle.join().unwrap_or_else(|_| Vec::new())) - .unwrap_or_default(); - - if !status.success() && stdout.is_empty() { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {status}"); - } - bail!("engine stdio process exited with status {status}: {stderr}"); - } - let envelope: EngineResponseEnvelope = serde_json::from_slice(&stdout).with_context(|| { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - "failed to parse engine response envelope".to_owned() - } else { - format!("failed to parse engine response envelope; stderr: {stderr}") - } - })?; - decode_engine_response(envelope) + engine_stdio_child_request(&engine_binary, paths, &envelope, env_root, stream_progress) } -fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result +/// The out-of-process half of [`engine_request_with_env_root`]: run +/// `engine_binary stdio` through the client `rocm` shares with `rocmd`, send +/// `envelope` on its stdin and decode its answer. +/// +/// What `rocm` asks of that client: the child runs on this process's app paths +/// when it has them and on the engine environments root when the request names +/// one, an install asks for progress on stderr, and the child's stderr is +/// written to this process's as it arrives, which is how a terminal user +/// watches an install. No endpoint key is handed over, because none of the +/// requests `rocm` makes is about a running service's endpoint. Split out so a +/// test can point it at a stand-in binary. +fn engine_stdio_child_request( + engine_binary: &Path, + paths: Option<&AppPaths>, + envelope: &EngineRequestEnvelope, + env_root: Option<&Path>, + stream_progress: bool, +) -> Result where R: DeserializeOwned, { - if !envelope.ok { - let error = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |value| format!("{}: {}", value.code, value.message), - ); - bail!("{error}"); + let options = stdio_options(paths, env_root, stream_progress); + decode_engine_response(run_engine_stdio( + engine_binary, + ["stdio"], + &options, + envelope, + )?) +} + +/// What `rocm` asks of the shared stdio client; see +/// [`engine_stdio_child_request`]. +const fn stdio_options<'a>( + paths: Option<&'a AppPaths>, + env_root: Option<&'a Path>, + stream_progress: bool, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: paths, + env_root, + endpoint_key_file: None, + // An external plugin is a third-party binary: never hand it a path to + // a live endpoint credential, including one we inherited ourselves. + withhold_inherited_key_file: true, + request_progress: stream_progress, + forward_stderr: true, + note_failing_exit_after_answer: false, } - let data = envelope - .data - .context("engine response envelope did not contain data")?; - serde_json::from_value(data).context("failed to decode engine response payload") } struct ScopedEnvVar { @@ -300,15 +226,12 @@ fn with_scoped_builtin_engine_env( let _guard = lock.lock().expect("builtin engine env lock poisoned"); let mut vars = Vec::new(); if let Some(paths) = paths { - for (key, value) in app_path_env_vars(paths) { + for (key, value) in paths.env_vars() { vars.push(ScopedEnvVar::set_path(key, value)); } } if let Some(env_root) = env_root { - vars.push(ScopedEnvVar::set_path( - "ROCM_CLI_ENGINE_ENVS_ROOT", - env_root, - )); + vars.push(ScopedEnvVar::set_path(ENGINE_ENVS_ROOT_ENV, env_root)); } let result = f(); drop(vars); @@ -362,6 +285,16 @@ pub(crate) fn run_builtin_engine_serve_http( engine_recipe: Option, ) -> Result<()> { let parsed_policy = parse_device_policy(Some(device_policy))?; + // Every engine is fronted the same way, so the Anthropic endpoint is started + // here once rather than inside each adapter. It runs on a thread of this + // process, which is the service's supervisor: it ends when the service does. + rocmd::anthropic_proxy::spawn_alongside_engine(rocmd::anthropic_proxy::AnthropicProxyConfig { + host: host.clone(), + engine_port: port, + served_model: model_ref.clone(), + state_path: state_path.clone(), + probe_api_key: rocm_engine_protocol::resolve_endpoint_api_key(), + }); match engine { "lemonade" => rocm_engine_lemonade::builtin_serve_http( service_id, @@ -455,6 +388,30 @@ mod tests { use super::*; use crate::tests::test_paths; + /// `rocm` must actually ASK for the withholding, not merely be able to. + /// + /// The protocol crate's own test proves the flag causes the clearing; this + /// is the other half of the chain, and without it flipping the value here + /// to `false` would re-open the leak with every test still green. Asserted + /// on `stdio_options` rather than on a built `Command` because + /// `engine_stdio_command` is private to the protocol crate. + #[test] + fn every_external_plugin_spawn_asks_to_withhold_the_endpoint_key_file() { + for stream_progress in [false, true] { + let options = stdio_options(None, None, stream_progress); + assert!( + options.withhold_inherited_key_file, + "an engine plugin is a third-party binary: `rocm` must withhold \ + an inherited key-file path from it, whatever else this request \ + asks for" + ); + assert!( + options.endpoint_key_file.is_none(), + "and it must never hand one over either" + ); + } + } + #[test] fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { let (root, paths) = test_paths("engine-plugin"); @@ -490,4 +447,289 @@ mod tests { assert_eq!(discovered, Some(primary_path)); Ok(()) } + + // ---- What `rocm` asks of the shared stdio client. ---- + // + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocm`'s entry point against a stand-in engine child and pin what + // `rocm` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocmd/src/engine_ipc.rs`. + + #[test] + fn rocm_asks_for_forwarded_stderr_and_no_endpoint_key() { + let (root, paths) = crate::tests::test_paths("engine-ipc-options"); + let env_root = root.join("envs"); + + let options = stdio_options(Some(&paths), Some(&env_root), true); + assert_eq!( + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) + ); + assert_eq!(options.env_root, Some(env_root.as_path())); + assert!(options.request_progress); + // The terminal user watches an install through the child's stderr. + assert!(options.forward_stderr); + assert!(!options.note_failing_exit_after_answer); + // No request `rocm` makes is about a running service's endpoint. + assert_eq!(options.endpoint_key_file, None); + + let bare = stdio_options(None, None, false); + assert!(bare.app_paths.is_none() && bare.env_root.is_none()); + assert!(!bare.request_progress); + assert!(bare.forward_stderr); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn envelope(method: EngineMethod, payload: serde_json::Value) -> EngineRequestEnvelope { + EngineRequestEnvelope { method, payload } + } + + fn stop_request() -> EngineRequestEnvelope { + envelope(EngineMethod::Stop, json!({"service_id": "svc"})) + } + + fn request( + engine: &FakeEngine, + paths: Option<&AppPaths>, + env_root: Option<&Path>, + stream_progress: bool, + ) -> Result { + engine_stdio_child_request::( + engine.path(), + paths, + &stop_request(), + env_root, + stream_progress, + ) + } + + /// Run `f` holding the lock that serialises every test that sets + /// `ROCM_CLI_*` process-wide (`with_scoped_builtin_engine_env`), so that + /// what a child inherits when `f` starts it and what [`ambient`] reads + /// inside `f` come from the same instant. + fn under_env_lock(f: impl FnOnce() -> R) -> R { + let lock = BUILTIN_ENGINE_ENV_LOCK.get_or_init(|| Mutex::new(())); + let _guard = lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f() + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_gets_the_stdio_verb_and_one_line_of_request() { + let engine = FakeEngine::answering("rocm-framing", STOPPED, 0); + let answer = request(&engine, None, None, false).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn the_child_runs_on_the_callers_app_paths_when_given_them() { + let (root, paths) = crate::tests::test_paths("engine-ipc-app-paths"); + let engine = FakeEngine::answering("rocm-app-paths", STOPPED, 0); + request(&engine, Some(&paths), None, false).unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!( + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() + ); + } + + #[test] + fn the_child_inherits_the_callers_environment_when_no_app_paths_are_given() { + const KEYS: [&str; 3] = [ + "ROCM_CLI_CONFIG_DIR", + "ROCM_CLI_DATA_DIR", + "ROCM_CLI_CACHE_DIR", + ]; + let engine = FakeEngine::answering("rocm-no-app-paths", STOPPED, 0); + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + KEYS.map(ambient) + }); + + for (key, expected) in KEYS.into_iter().zip(expected) { + assert_eq!(engine.env(key), expected, "{key}"); + } + } + + /// The comparison of a child's inherited environment with the ambient one + /// holds while another thread scopes `ROCM_CLI_*` process-wide, which + /// other tests of this module do: the spawn and the read happen under the + /// same lock. Without the lock the child can start while the variables + /// are scoped and the read see them restored (or the other way round). + #[test] + fn the_inherited_environment_is_compared_at_the_instant_the_child_starts() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let (root, paths) = crate::tests::test_paths("engine-ipc-env-race"); + let done = Arc::new(AtomicBool::new(false)); + let flipper = { + let done = Arc::clone(&done); + std::thread::spawn(move || { + while !done.load(Ordering::Relaxed) { + with_scoped_builtin_engine_env(Some(&paths), None, || { + std::thread::sleep(std::time::Duration::from_millis(2)); + }); + // Outside the lock, so the test's own turn is not starved. + std::thread::sleep(std::time::Duration::from_millis(2)); + } + }) + }; + + for round in 0..25 { + let engine = FakeEngine::answering("rocm-env-race", STOPPED, 0); + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + ambient("ROCM_CLI_DATA_DIR") + }); + assert_eq!(engine.env("ROCM_CLI_DATA_DIR"), expected, "round {round}"); + } + + done.store(true, Ordering::Relaxed); + flipper.join().unwrap(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn an_env_root_reaches_the_child_as_the_engine_envs_root() { + let engine = FakeEngine::answering("rocm-env-root", STOPPED, 0); + let env_root = Path::new("/an/engine/envs/root"); + request(&engine, None, Some(env_root), false).unwrap(); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/an/engine/envs/root") + ); + + let without = FakeEngine::answering("rocm-no-env-root", STOPPED, 0); + let expected = under_env_lock(|| { + request(&without, None, None, false).unwrap(); + ambient("ROCM_CLI_ENGINE_ENVS_ROOT") + }); + assert_eq!(without.env("ROCM_CLI_ENGINE_ENVS_ROOT"), expected); + } + + #[test] + fn progress_streaming_is_asked_for_only_when_the_caller_wants_it() { + let streaming = FakeEngine::answering("rocm-progress-on", STOPPED, 0); + request(&streaming, None, None, true).unwrap(); + assert_eq!( + streaming.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + + let quiet = FakeEngine::answering("rocm-progress-off", STOPPED, 0); + let expected = under_env_lock(|| { + request(&quiet, None, None, false).unwrap(); + ambient("ROCM_ENGINE_PROGRESS_STDERR") + }); + assert_eq!(quiet.env("ROCM_ENGINE_PROGRESS_STDERR"), expected); + } + + #[test] + fn the_endpoint_key_is_not_passed_to_the_child() { + // `rocm`'s requests are not about a running service's endpoint (detect, + // install, resolve-model, stop), so this client has no service to find + // a key for. `rocmd`'s healthcheck is the call that threads one. + let engine = FakeEngine::answering("rocm-no-key", STOPPED, 0); + let key = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + ambient(key) + }); + assert_eq!(engine.env(key), expected); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + // The engine's own error envelope is the useful diagnostic, so an + // answer on stdout wins over the exit status. + let engine = FakeEngine::answering( + "rocm-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = request(&engine, None, None, false).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("rocm-answers-then-exits-2", STOPPED, 2); + assert_eq!( + request(&engine, None, None, false).unwrap(), + json!({"stopped": true}) + ); + } + + /// The write of a request to an engine that has already exited must come + /// back as an error even where SIGPIPE is at its default action, which is + /// how `main` runs `rocm`. Re-executes this test in a fresh process that + /// installs `SIG_DFL` first (the test harness leaves SIGPIPE ignored, and + /// changing it process-wide would race sibling tests). + #[test] + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_ENGINE_SIGPIPE_ARM").is_some() { + crate::reset_sigpipe(); + let engine = FakeEngine::new("rocm-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let result = engine_stdio_child_request::( + engine.path(), + None, + &envelope(EngineMethod::Stop, json!({ "blob": big })), + None, + false, + ); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_ENGINE_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } } diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index e0c744891f..88d1019187 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -7,29 +7,29 @@ //! to (`EngineSelection`, `resolve_engine_selection`, //! `validate_engine_selection_runtime`, `single_ready_runtime_key`, //! `resolve_runtime_selector_to_exact_key`), small argv-building helpers used -//! when relaunching a managed engine (`optional_arg`, -//! `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, +//! when relaunching a managed engine (`engine_recipe_json_arg`, +//! `parse_engine_recipe_json_arg`, //! `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` //! environment variables every managed child inherits -//! (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/ -//! `apply_app_path_env`), and polling a freshly launched service until it can +//! (`app_path_env_var_values`/`app_path_env_var_refs`, over +//! [`AppPaths::env_vars`]; a command gets them from `AppPaths::apply_env`), and polling a freshly launched service until it can //! actually answer inference requests (`wait_for_service_http_ready`, //! `wait_for_service_http_ready_with_progress`, //! `service_http_readiness_paths`, `http_get_local_service`, //! `http_post_local_service_json`, `service_http_readiness_response_ready`, //! `lemonade_health_ready_for_model`, `model_list_ready_for_model`, -//! `service_model_gpu_ready`, `service_model_reports_rocm_backend`, +//! `service_model_gpu_ready`, `service_model_reports_gpu_backend`, //! `service_model_names_match`). //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-215) of the `main.rs` modularization (see -//! `docs/architecture.md`). This is the last sub-phase of the `main.rs` -//! Phase 6f split that lands before the file's single shared -//! `#[cfg(test)] mod tests` block starts: the function immediately after this -//! cluster in the original file, `treat_as_natural_language`, is -//! dispatch-adjacent natural-language-invocation routing left over from an -//! earlier phase (ROCMAI-87) and stays in `main.rs` on purpose — it is not -//! part of this module and is not touched by this change. +//! Mechanically relocated from `main.rs` with no behavior change (the +//! `relocation` class in `docs/architecture.md`). The cluster ended where the +//! file's single shared `#[cfg(test)] mod tests` block starts: the function +//! immediately after it in the original file, `treat_as_natural_language`, is +//! dispatch-adjacent natural-language-invocation routing that stays in +//! `main.rs` on purpose — it is not part of this module and is not touched by +//! this change. Behavior here has changed since the move: a Lemonade router +//! entry served on Vulkan now counts as ready (`service_model_gpu_ready`), not +//! only one served on ROCm. //! //! One item physically interleaved in this span in `main.rs` stayed there //! instead of moving: `const fn status_for_readiness` sits between @@ -57,16 +57,14 @@ //! `resolve_runtime_selector_to_exact_key` is also `pub(crate)`: `engines_cmd.rs` //! calls it in production, and `runtimes_cmd.rs`'s own test module calls it //! directly too. `single_ready_runtime_key` is called only from -//! `validate_engine_selection_runtime` in this same module and stays private. -//! `optional_arg` and -//! `engine_recipe_json_arg` are `pub(crate)`, read from `engine_ipc.rs`'s +//! `validate_engine_selection_runtime` in this same module and stays private; +//! it does not decide which runtime counts as "the one ready runtime" itself, +//! it asks `runtime_adopt::select_single_ready_runtime`, the rule +//! `rocm comfyui install` shares. +//! `engine_recipe_json_arg` is `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read //! from `main.rs`'s `dispatch()` and `service_control.rs`'s `restart_child_argv`. -//! `app_path_env_vars`/`apply_app_path_env` are `pub(crate)`, read from -//! `engine_ipc.rs`, `app_path_env_vars` is also read from `chat.rs`, and -//! `apply_app_path_env` is also read from `engines_cmd.rs`, -//! `service_launch.rs`, and `service_control.rs`'s -//! `restart_internal_managed_service_with`; `app_path_env_var_values`/ +//! `app_path_env_var_values`/ //! `app_path_env_var_refs` are `pub(crate)`, read from `service_launch.rs` //! and `service_control.rs`'s `restart_internal_managed_service_with`. //! `daemon_run_argv` is `pub(crate)`, read from `main.rs`'s `dispatch()`; @@ -76,11 +74,11 @@ //! `restart_internal_managed_service_with`; `wait_for_service_http_ready_with_progress` //! is `pub(crate)`, read from `service_launch.rs`. `http_post_local_service_json` //! is `pub(crate)`, read from `service_control.rs`'s `unload_lemonade_service_model`. -//! `service_model_names_match` is `pub(crate)`, read from `chat.rs`. +//! `service_model_names_match` is `pub(crate)`, read from `chat/turn.rs`. //! `service_http_readiness_paths`, `http_get_local_service`, //! `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, //! `model_list_ready_for_model`, `service_model_gpu_ready`, and -//! `service_model_reports_rocm_backend` have no callers outside this module +//! `service_model_reports_gpu_backend` have no callers outside this module //! and stay private. //! //! The cluster's regression tests live in this module's own `mod tests`, @@ -92,7 +90,6 @@ use std::ffi::OsString; use std::path::{Path, PathBuf}; -use std::process::Command as ProcessCommand; use std::thread; use std::time::Duration; @@ -103,10 +100,8 @@ use rocm_core::{ }; use rocm_engine_protocol::EngineRecipeHint; -use crate::runtimes_cmd::{ - recover_setup_runtime_registration, select_runtime_manifest, - validate_runtime_manifest_for_activation, -}; +use crate::runtime_adopt::{SetupRecovery, SingleReadyRuntime, select_single_ready_runtime}; +use crate::runtimes_cmd::{recover_setup_runtime_registration, select_runtime_manifest}; use crate::therock; #[derive(Debug, Clone)] @@ -211,16 +206,14 @@ pub(crate) fn validate_engine_selection_runtime( fn single_ready_runtime_key(paths: &AppPaths) -> Result> { let config = RocmCliConfig::load(paths).unwrap_or_default(); - recover_setup_runtime_registration(paths, &config)?; - let manifests = therock::load_runtime_manifests(paths)?; - let ready = manifests - .iter() - .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) - .collect::>(); - Ok(match ready.as_slice() { - [manifest] => Some(manifest.runtime_key.clone()), - _ => None, - }) + // Zero or several ready runtimes leave the selection open: the caller that + // needs a runtime says so in its own words. + Ok( + match select_single_ready_runtime(paths, &config, SetupRecovery::Persist)? { + SingleReadyRuntime::One(manifest) => Some(manifest.runtime_key), + SingleReadyRuntime::Zero { .. } | SingleReadyRuntime::Many { .. } => None, + }, + ) } pub(crate) fn resolve_runtime_selector_to_exact_key( @@ -246,13 +239,6 @@ pub(crate) fn resolve_runtime_selector_to_exact_key( } } -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } -} - pub(crate) fn engine_recipe_json_arg( engine_recipe: Option<&EngineRecipeHint>, ) -> Result> { @@ -275,20 +261,13 @@ pub(crate) fn parse_engine_recipe_json_arg( .transpose() } -pub(crate) fn app_path_env_vars(paths: &AppPaths) -> [(&'static str, &Path); 3] { - [ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ] -} - #[cfg_attr(not(windows), allow(dead_code))] pub(crate) fn app_path_env_var_values( paths: &AppPaths, env_root: Option<&Path>, ) -> Vec<(&'static str, PathBuf)> { - let mut vars = app_path_env_vars(paths) + let mut vars = paths + .env_vars() .into_iter() .map(|(key, value)| (key, value.to_path_buf())) .collect::>(); @@ -325,12 +304,6 @@ pub(crate) fn managed_service_launcher_path() -> Result { Ok(current_exe) } -pub(crate) fn apply_app_path_env(command: &mut ProcessCommand, paths: &AppPaths) { - for (key, value) in app_path_env_vars(paths) { - command.env(key, value); - } -} - pub(crate) fn wait_for_service_http_ready( engine: &str, host: &str, @@ -524,7 +497,7 @@ fn lemonade_health_ready_for_model(body: &str, canonical_model_id: &str) -> bool .into_iter() .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) .any(|loaded| service_model_names_match(loaded, canonical_model_id)); - name_matches && service_model_reports_rocm_backend(model) + name_matches && service_model_reports_gpu_backend(model) }) }) } @@ -532,7 +505,7 @@ fn lemonade_health_ready_for_model(body: &str, canonical_model_id: &str) -> bool fn model_list_ready_for_model( body: &str, canonical_model_id: &str, - require_rocm_backend: bool, + require_gpu_backend: bool, ) -> bool { let Ok(value) = serde_json::from_str::(body.trim()) else { return false; @@ -546,7 +519,7 @@ fn model_list_ready_for_model( .into_iter() .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) .any(|loaded| service_model_names_match(loaded, canonical_model_id)); - name_matches && (!require_rocm_backend || service_model_gpu_ready(model)) + name_matches && (!require_gpu_backend || service_model_gpu_ready(model)) }) }) } @@ -554,22 +527,26 @@ fn model_list_ready_for_model( /// Whether a lemonade `/v1/models` entry is served on GPU. A stock `llama-server` /// (direct-serve) entry has no `recipe_options` and is accepted (that path only runs /// GPU backends); a Lemonade-router entry carries `recipe_options`, so it must report a -/// ROCm backend — which keeps a registered-but-unloaded model (empty `recipe_options`) +/// GPU backend (ROCm or Vulkan) — which keeps a registered-but-unloaded model (empty `recipe_options`) /// from reading as ready. fn service_model_gpu_ready(model: &serde_json::Value) -> bool { match model.get("recipe_options") { None => true, - Some(_) => service_model_reports_rocm_backend(model), + Some(_) => service_model_reports_gpu_backend(model), } } -fn service_model_reports_rocm_backend(model: &serde_json::Value) -> bool { +fn service_model_reports_gpu_backend(model: &serde_json::Value) -> bool { model .get("recipe_options") .and_then(|options| options.get("llamacpp_backend")) .or_else(|| model.get("llamacpp_backend")) .and_then(serde_json::Value::as_str) - .is_some_and(|backend| backend.trim().to_ascii_lowercase().starts_with("rocm")) + .is_some_and(|backend| { + // The GPU backends the engine can select (ROCm, else Vulkan); never `cpu`. + let backend = backend.trim().to_ascii_lowercase(); + backend.starts_with("rocm") || backend.starts_with("vulkan") + }) } /// Whether two model names refer to the same model, leniently. @@ -611,6 +588,7 @@ mod tests { use serde_json::json; use super::*; + use crate::runtime_registry::validate_runtime_manifest_for_activation; use crate::status_for_readiness; use crate::test_support::write_test_pip_runtime; use crate::tests::{test_app_paths, test_paths}; @@ -753,6 +731,49 @@ mod tests { )); } + #[test] + fn lemonade_router_readiness_accepts_any_gpu_backend_the_engine_selects() { + // The engine selects ROCm first and Vulkan otherwise, and records either as + // `backend_requested`; a router-served model on a Vulkan host must read as + // ready on both probe paths. CPU (or no backend) must never read as ready. + for (backend, ready) in [ + ("rocm", true), + ("rocm-stable", true), + ("vulkan", true), + ("Vulkan", true), + ("cpu", false), + ("", false), + ] { + let health = json!({ + "all_models_loaded": [{ + "model_name": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": backend } + }] + }) + .to_string(); + let models = json!({ + "data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": backend } + }] + }) + .to_string(); + for (path, body) in [("/v1/health", &health), ("/v1/models", &models)] { + assert_eq!( + service_http_readiness_response_ready( + "lemonade", + path, + 200, + body, + "Qwen3-0.6B-GGUF" + ), + ready, + "{path} with backend `{backend}`" + ); + } + } + } + #[test] fn serve_readiness_wait_withholds_ready_while_the_model_only_lists() -> Result<()> { use std::io::{Read, Write}; @@ -922,16 +943,6 @@ mod tests { Ok(()) } - #[test] - fn app_path_env_vars_include_config_data_and_cache() { - let paths = test_app_paths(); - let vars = app_path_env_vars(&paths); - - assert_eq!(vars[0], ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path())); - assert_eq!(vars[1], ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path())); - assert_eq!(vars[2], ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path())); - } - #[test] fn app_path_env_var_values_include_engine_env_root_when_needed() { let paths = test_app_paths(); @@ -1048,4 +1059,137 @@ mod tests { let _ = std::fs::remove_dir_all(root); Ok(()) } + + /// Zero ready runtimes, the registry empty: the engine path answers with no + /// selection rather than an error, and writes nothing. The caller reports + /// the missing runtime in its own words (`engines_cmd.rs` and `serve_cmd.rs`). + #[test] + fn engine_selection_with_no_registered_runtime_stays_unselected_and_writes_nothing() + -> Result<()> { + let (root, paths) = test_paths("no-registered-runtime-selection"); + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert!(selection.runtime_id.is_none()); + assert!(selection.env_id.is_none()); + assert!(selection.source.is_none()); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new() + ); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// Zero ready runtimes, one registered but unusable: still no selection. + /// Readiness is `validate_runtime_manifest_for_activation`, so a runtime + /// whose interpreter is gone does not count, and it is not picked as "the + /// only one" either. + #[test] + fn engine_selection_does_not_count_an_unusable_runtime_as_ready() -> Result<()> { + let (root, paths) = test_paths("unusable-runtime-selection"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let python = manifest + .python_executable + .as_deref() + .expect("the fixture records an interpreter"); + std::fs::remove_file(python)?; + assert!(validate_runtime_manifest_for_activation(&manifest).is_err()); + + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert!(selection.runtime_id.is_none()); + assert!(selection.source.is_none()); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// The setup-recovery half of the rule: a ready runtime that only + /// `recover_setup_runtime_registration` can make visible is found by the + /// engine path, and finding it re-files its registry entry, so the message + /// and the state are one fact (the selection names a key that + /// `rocm runtimes list` now shows). + #[test] + fn engine_selection_recovers_the_setup_runtime_the_registry_lost() -> Result<()> { + let (root, paths) = test_paths("setup-recovery-selection"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new(), + "the registry must start without the runtime for this test to mean anything" + ); + + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert_eq!( + selection.runtime_id.as_deref(), + Some(manifest.runtime_key.as_str()) + ); + assert_eq!(selection.source.as_deref(), Some("single_ready_runtime")); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// Recovering the setup runtime writes its registry entry, and a registry + /// that cannot take it is reported as that on every path that recovers: + /// an explicit `--runtime-id` and the one-ready-runtime default (which + /// `serve` and `engines install` both reach). Neither may answer as if there + /// were no runtime, or carry on without one: the runtime is there, and the + /// failure is what stops the command. The error and the registry are + /// asserted together. + #[test] + fn engine_selection_reports_a_registry_that_cannot_take_the_setup_runtime_back() -> Result<()> { + for explicit in [true, false] { + let (root, paths) = test_paths("setup-recovery-unwritable-registry"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let registry = crate::test_support::make_runtime_registry_unwritable(&paths)?; + let runtime_id = explicit.then_some(manifest.runtime_key.as_str()); + + let error = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::load(&paths)?, "vllm", runtime_id, None), + ) + .expect_err("a registry that cannot be written cannot take the entry back"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "failed to restore setup runtime `{}`", + manifest.runtime_key + )), + "explicit: {explicit}, got: {message}" + ); + assert_eq!( + std::fs::read_to_string(®istry)?, + "not a directory", + "explicit: {explicit}: a failed restore leaves the registry as it was" + ); + let _ = std::fs::remove_dir_all(root); + } + Ok(()) + } } diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index 84301ee13f..dab1629cba 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -41,7 +41,7 @@ use crate::engine_install_env::{ }; use crate::engine_ipc::{engine_request, engine_request_with_env_root}; use crate::engine_runtime_select::{ - apply_app_path_env, resolve_engine_selection, resolve_runtime_selector_to_exact_key, + resolve_engine_selection, resolve_runtime_selector_to_exact_key, validate_engine_selection_runtime, }; use crate::runtime_install_finalize::{ @@ -469,7 +469,7 @@ fn engine_shell( .env("ROCM_CLI_ENV_ID", &resolved.env_id) .env("ROCM_CLI_RUNTIME_ID", &resolved.runtime_id) .env("ROCM_CLI_PYTHON", &resolved.python_executable); - apply_app_path_env(&mut command, &paths); + paths.apply_env(&mut command); let prompt = format!("(rocm:{engine}) "); let mut prompt_marked = false; @@ -835,4 +835,72 @@ mod tests { ); } } + + /// Several ready runtimes and none chosen: the engine path does not guess. + /// `resolve_engine_env` reports the gap in its own words, and each remedy + /// those words name must clear it: `rocm runtimes activate ` + /// (run here as `activate_runtime`, its implementation) and `--runtime-id`. + #[test] + fn engine_install_with_several_ready_runtimes_names_remedies_that_clear_it() -> Result<()> { + let (root, paths) = test_paths("engine-install-several-ready-runtimes"); + let first = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + + let message = resolve_engine_env(&paths, &config, "vllm", None, None) + .expect_err("several ready runtimes and no choice must not resolve") + .to_string(); + assert!( + message.contains("no active ROCm runtime is configured") + && message.contains("rocm runtimes list") + && message.contains("rocm runtimes activate ") + && message.contains("--runtime-id"), + "unexpected refusal: {message}" + ); + + // `rocm runtimes activate `: the choice is now made. + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &first.runtime_key)?; + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&config, "vllm", None, None), + )?; + assert_eq!( + selection.runtime_id.as_deref(), + Some(first.runtime_key.as_str()) + ); + assert_eq!( + selection.source.as_deref(), + Some("config_active_runtime_key") + ); + + // `--runtime-id `: the same choice, for one invocation. + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection( + &RocmCliConfig::default(), + "vllm", + Some(&first.runtime_key), + None, + ), + )?; + assert_eq!( + selection.runtime_id.as_deref(), + Some(first.runtime_key.as_str()) + ); + assert_eq!(selection.source.as_deref(), Some("cli_runtime_id")); + let _ = fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/examine_cmd.rs b/apps/rocm/src/examine_cmd.rs index 6ae1a22fae..06b1a2cab6 100644 --- a/apps/rocm/src/examine_cmd.rs +++ b/apps/rocm/src/examine_cmd.rs @@ -82,13 +82,13 @@ pub(crate) fn examine(json: bool, framework: rocm_core::FrameworkProbe) -> Resul // torch lives only in its site-packages, so probing PATH would report // `unknown` for a host that has one. let interpreter = rocm_core::active_managed_framework_interpreter(&paths, &config); - let mut examination = - rocm_core::Examination::probe_with_interpreter(framework, interpreter.as_ref()); - // `gather` rather than `examine_human_report`: the latter first runs - // `recover_setup_runtime_registration`, which writes. Asking a machine a - // question should not change it, and `--json` is the form tooling calls - // in a loop. - let host = ExamineSummary::gather()?; + // `probe_with_summary` rather than `examine_human_report`, whose recovery + // step writes: asking a machine a question should not change it, and + // `--json` is the form tooling calls in a loop. It gathers the summary + // from the examination it probes, so the kernel release is read once for + // both documents and the two fields cannot disagree. + let (mut examination, host) = + rocm_core::Examination::probe_with_summary(framework, interpreter.as_ref())?; // The human report's "AMD GPU detected" line reads this same // `detected_gfx_target`; WSL skips the probes that would set // `has_amd_gpu`, so take it from here rather than leave the two forms diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 2674b2a7d5..ec67be84b1 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -9,7 +9,7 @@ //! Relocated from `main.rs` with no behavior change (see //! `docs/architecture.md`). The `rocm model` catalog renderer used to live //! here too; it moved to `model_catalog_text.rs`, which this module does not -//! call into outside its own tests. +//! call into. //! //! This module owns: the launch-summary/examine text surfaces //! (`render_launch_summary`, `render_examine_text`, @@ -29,8 +29,8 @@ //! `launch_default` calls `render_launch_summary`, re-imported there. //! `render_examine_text`, `render_examine_text_with_paths` and //! `render_engine_inventory_text_with_paths` are `pub(crate)`, read from -//! [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output -//! summaries; `render_engine_inventory_text_with_paths` is also read from +//! [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only +//! command runner; `render_engine_inventory_text_with_paths` is also read from //! [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s //! `render_engine_inventory_text`. `append_examine_runtime_state` is also //! `pub(crate)`, read only from @@ -49,11 +49,9 @@ //! The regression tests for these surfaces live in this module's own //! `mod tests`, reaching the shared `test_paths` fixture through //! `crate::tests::` and `write_test_pip_runtime` through -//! `crate::test_support::`. One of them, -//! `linux_only_engine_runtime_status_is_explicit_on_native_windows`, also -//! calls `model_catalog_text.rs`'s `model_registry_adapter_availability_note`, -//! which is `pub(crate)` for that reason; its primary subject is -//! `engine_runtime_status_label`, so it stays here. +//! `crate::test_support::`. The note the `rocm model --verbose` listing prints +//! for a rule-out engine is tested in `model_catalog_text.rs`, next to the +//! renderer that prints it. use std::fmt::Write as _; use std::path::PathBuf; @@ -578,7 +576,6 @@ mod tests { use std::fs; use super::*; - use crate::model_catalog_text::model_registry_adapter_availability_note; use crate::runtimes_cmd::runtime_manifest_path; use crate::test_support::write_test_pip_runtime; use crate::tests::test_paths; @@ -891,13 +888,8 @@ mod tests { engine_runtime_status_label("vllm", &detect), "unsupported_native_windows" ); - assert!( - model_registry_adapter_availability_note("vllm", &|| true) - .is_some_and(|note| note.contains("unsupported_native_windows")) - ); } else { assert_eq!(engine_runtime_status_label("vllm", &detect), "not found"); - assert!(model_registry_adapter_availability_note("vllm", &|| true).is_none()); } } diff --git a/apps/rocm/src/freeform_execution.rs b/apps/rocm/src/freeform_execution.rs index 9838e40668..33de867cb2 100644 --- a/apps/rocm/src/freeform_execution.rs +++ b/apps/rocm/src/freeform_execution.rs @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: MIT -//! The freeform natural-language planner's plan-*execution*/consent cluster -//! (ROCMAI-726, a Phase 6e, ROCMAI-87 follow-up). +//! The freeform natural-language planner's plan-*execution*/consent cluster, +//! the half that `freeform_planner.rs` left behind. //! //! Mechanically relocated from `main.rs` with no behavior change — `main.rs`'s //! `execute_freeform_next_action()` call sites stay byte-identical @@ -18,7 +18,7 @@ //! to execute. //! //! This is the other half of the freeform-planner cluster that -//! `freeform_planner.rs` (Phase 6e, ROCMAI-87) deliberately left behind: that +//! `freeform_planner.rs` deliberately left behind: that //! module builds a plan; this one reads/acts on its `FreeformPlanAction` //! internals, grants it the consent the outer `--yes` already carries, and //! computes planner-specific output text — a dispatch-adjacent concern diff --git a/apps/rocm/src/freeform_planner.rs b/apps/rocm/src/freeform_planner.rs index 42f4593052..877c91e904 100644 --- a/apps/rocm/src/freeform_planner.rs +++ b/apps/rocm/src/freeform_planner.rs @@ -47,8 +47,6 @@ use std::fmt::Write as _; // as `freeform_planner::format_structured_tool_call` (read by `comfyui.rs`, // `freeform_execution.rs` and `providers.rs`'s tests). pub(crate) use crate::rocm_argv::format_structured_tool_call; -use crate::rocm_argv::serve_args_request_cpu_device; -use crate::service_launch::is_loopback_host; use crate::shell_quote::has_terminal_control_character; use crate::{Cli, prompt_text, providers, rocm_argv, tool_policy}; @@ -88,7 +86,7 @@ pub(crate) fn freeform_plan_next_action( /// Builds the freeform plan exactly once, returning both the rendered plan /// text and the next action derived from that same plan. /// -/// Used by `chat.rs`'s `natural_language_plan` MCP tool, which used to call +/// Used by `chat/mcp_bridge.rs`'s `natural_language_plan` MCP tool, which used to call /// `render_freeform_plan` and (the now-removed) /// `freeform_plan_next_action_with_context` independently — each building its /// own `StructuredRequestPlan`. When the request is ambiguous and a cloud @@ -1016,24 +1014,7 @@ pub(crate) fn validate_provider_planner_tool_call( [command, subcommand, ..] if command == "install" && subcommand == "driver" => {} [command, ..] if command == "update" => {} [command, ..] if command == "serve" => { - if rocm_argv::chat_cli_has_flag(&call.args, "--allow-public-bind") { - bail!("provider planner cannot request public network binding"); - } - if serve_args_request_cpu_device(&call.args) { - bail!( - "provider planner cannot request CPU execution; rocm-cli requires ROCm GPU execution" - ); - } - if let Some(host) = rocm_argv::chat_cli_arg_value(&call.args, "--host") - && !is_loopback_host(host) - { - bail!("provider planner cannot request non-local host `{host}`"); - } - if rocm_argv::chat_cli_has_flag(&call.args, "--foreground") - || !rocm_argv::chat_cli_has_flag(&call.args, "--managed") - { - bail!("provider planner must request managed serving with --managed"); - } + tool_policy::validate_serve_command_safety(&call.args, "provider planner")?; } [command, ..] if command == "uninstall" => {} _ => bail!( @@ -2020,4 +2001,209 @@ mod tests { ); Ok(()) } + + /// The planner's response for a `serve` call, as the provider would send it. + fn serve_planner_response(args: &[&str]) -> String { + serde_json::json!({ + "intent": "serve", + "tool_call": {"tool": "rocm", "args": args}, + }) + .to_string() + } + + /// The serve safety rules as the provider-assisted planner meets them: every + /// refusal in the shared table, at the validator and at the response parser + /// that builds the plan. A refused response yields no `StructuredRequestPlan`, + /// so there is no planned action for anything to run, and the operator sees + /// the refusal as the planner's failure note. + /// + /// The chat suite (`tool_policy`) runs the same table. + #[test] + fn planner_refuses_each_unsafe_serve_and_plans_nothing() { + use crate::test_support::{SERVE_SAFETY_REFUSALS, owned_argv}; + + for case in SERVE_SAFETY_REFUSALS { + let context = format!("{}: {:?}", case.rule, case.args); + let call = ProviderPlannerToolCall { + tool: "rocm".to_owned(), + args: owned_argv(case.args), + }; + let error = validate_provider_planner_tool_call(&call).expect_err(&context); + assert_eq!(error.to_string(), case.planner_error, "{context}"); + + let error = provider_planner_response_to_plan( + "serve a model", + "local", + &serve_planner_response(case.args), + ) + .expect_err(&context); + assert_eq!(error.to_string(), case.planner_error, "{context}"); + assert_eq!( + planner_failure_note("local", &error), + format!( + "Provider-assisted planner `local` could not resolve this request: {}", + case.planner_error + ), + "{context}" + ); + } + } + + #[test] + fn planner_plans_a_safe_serve_for_approval_with_its_argv_unchanged() -> Result<()> { + use crate::test_support::{SERVE_SAFETY_ACCEPTED, owned_argv}; + + for args in SERVE_SAFETY_ACCEPTED { + let plan = provider_planner_response_to_plan( + "serve a model", + "local", + &serve_planner_response(args), + )?; + assert_eq!(plan.intent, PlannerIntent::Serve, "{args:?}"); + // A serve starts a server, so the plan always asks for approval. + assert_eq!(plan.approval, "required before execution", "{args:?}"); + assert_eq!(plan.actions.len(), 1, "{args:?}"); + assert_eq!(plan.actions[0].args, owned_argv(args), "{args:?}"); + } + Ok(()) + } + + /// The planner has no port rule: a loopback host other than `127.0.0.1` + /// without `--port` is planned, where the chat policy refuses it + /// (`chat_requires_a_port_on_a_loopback_host_other_than_the_default`). + #[test] + fn planner_accepts_a_loopback_host_without_a_port() -> Result<()> { + use crate::test_support::{SERVE_PORT_RULE_CHAT_ONLY, owned_argv}; + + for (args, _) in SERVE_PORT_RULE_CHAT_ONLY { + let plan = provider_planner_response_to_plan( + "serve a model", + "local", + &serve_planner_response(args), + )?; + assert_eq!(plan.actions[0].args, owned_argv(args), "{args:?}"); + } + Ok(()) + } + + /// The rules fire in a fixed order, so an argv that breaks several reports + /// the first one: public bind, then CPU, then host, then managed. + #[test] + fn planner_reports_the_first_broken_serve_rule() { + use crate::test_support::owned_argv; + + for (args, expected) in [ + ( + &[ + "serve", + "qwen", + "--allow-public-bind", + "--device", + "cpu", + "--host", + "0.0.0.0", + "--foreground", + ][..], + "provider planner cannot request public network binding", + ), + ( + &[ + "serve", + "qwen", + "--device", + "cpu", + "--host", + "0.0.0.0", + "--foreground", + ][..], + "provider planner cannot request CPU execution; ROCm GPU execution is required", + ), + ( + &["serve", "qwen", "--host", "0.0.0.0", "--foreground"][..], + "provider planner cannot request non-local host `0.0.0.0`", + ), + ( + &["serve", "qwen", "--foreground"][..], + "provider planner must request managed serving with --managed", + ), + ] { + let call = ProviderPlannerToolCall { + tool: "rocm".to_owned(), + args: owned_argv(args), + }; + let error = validate_provider_planner_tool_call(&call) + .expect_err("unsafe") + .to_string(); + assert_eq!(error, expected, "{args:?}"); + } + } + + /// The planner and the chat tool policy enforce one set of serve rules, so + /// they refuse an argv with the same words and differ only in who is named + /// at the front of the message. + #[test] + fn planner_and_chat_serve_refusals_differ_in_the_actor_only() { + use crate::test_support::SERVE_SAFETY_REFUSALS; + + for case in SERVE_SAFETY_REFUSALS { + let chat = case + .chat_error + .strip_prefix("local assistant ") + .unwrap_or_else(|| panic!("chat refusal names the assistant: {}", case.chat_error)); + let planner = case + .planner_error + .strip_prefix("provider planner ") + .unwrap_or_else(|| { + panic!("planner refusal names the planner: {}", case.planner_error) + }); + assert_eq!(chat, planner, "{}: {:?}", case.rule, case.args); + } + } + + /// `--foreground` next to `--managed` is refused by clap before the planner's + /// safety check runs, where the chat check refuses it for the managed rule. + #[test] + fn planner_refuses_foreground_with_managed_as_an_invalid_command() { + use crate::test_support::{SERVE_FOREGROUND_AND_MANAGED, owned_argv}; + + let call = ProviderPlannerToolCall { + tool: "rocm".to_owned(), + args: owned_argv(SERVE_FOREGROUND_AND_MANAGED), + }; + let error = validate_provider_planner_tool_call(&call) + .expect_err("clap refuses the conflicting flags") + .to_string(); + assert_eq!( + error, + "provider planner returned a rocm command that is not valid" + ); + } + + /// The planner matches the `serve` word exactly, where the chat check + /// ignores case, so `Serve` and `SERVE` never reach the planner's safety + /// check: the clap parse that precedes it refuses them. Either way an + /// uppercase `serve` is refused; only the reason differs. + #[test] + fn planner_refuses_an_uppercase_serve_word_before_the_safety_check() { + use crate::test_support::owned_argv; + + for word in ["Serve", "SERVE"] { + for args in [ + &[word, "qwen", "--device", "cpu", "--managed"][..], + &[word, "qwen", "--managed"][..], + ] { + let call = ProviderPlannerToolCall { + tool: "rocm".to_owned(), + args: owned_argv(args), + }; + let error = validate_provider_planner_tool_call(&call) + .expect_err("an uppercase serve word is not a rocm command") + .to_string(); + assert_eq!( + error, "provider planner returned a rocm command that is not valid", + "{args:?}" + ); + } + } + } } diff --git a/apps/rocm/src/gpu_policy.rs b/apps/rocm/src/gpu_policy.rs index 81bef34323..108a642d35 100644 --- a/apps/rocm/src/gpu_policy.rs +++ b/apps/rocm/src/gpu_policy.rs @@ -10,10 +10,9 @@ //! --json`, with an amdgpu DRM sysfs fallback), and building the serve-plan //! low-VRAM warning from it. //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-214) of the `main.rs` modularization (see -//! `docs/architecture.md`). `dispatch()` calls directly into this module at -//! one point — the `Command::EngineServeHttp` arm's +//! Mechanically relocated from `main.rs` with no behavior change (the +//! `relocation` class in `docs/architecture.md`). `dispatch()` calls directly +//! into this module at one point — the `Command::EngineServeHttp` arm's //! `parse_gpu_indices_arg(gpu.as_deref())?` — re-imported via `use //! crate::gpu_policy::parse_gpu_indices_arg;`; `dispatch()`'s own body is //! otherwise unaffected. @@ -32,8 +31,9 @@ //! //! One item physically interleaved in this span in `main.rs` stayed there //! instead of moving: `const fn device_policy_name` sits immediately after -//! `detect_gpu_count` and before the next cluster's (ROCMAI-215) -//! `EngineSelection`/`resolve_engine_selection`, but it already had callers +//! `detect_gpu_count` and before the engine-selection cluster's +//! `EngineSelection`/`resolve_engine_selection` (now in +//! `engine_runtime_select.rs`), but it already had callers //! outside this cluster — `engine_ipc.rs` and `service_launch.rs` reach it //! unqualified via `use crate::{...}`, same as `serve_cmd.rs` does — plus its //! own test, so it is not part of this cluster's relocation. diff --git a/apps/rocm/src/install_cmd.rs b/apps/rocm/src/install_cmd.rs index 76d0c97194..1230258395 100644 --- a/apps/rocm/src/install_cmd.rs +++ b/apps/rocm/src/install_cmd.rs @@ -15,8 +15,8 @@ //! live in `cli.rs` with the rest of the `Command`/`Cli` definition. //! `dispatch()` calls `install` through the re-import `use //! crate::install_cmd::{...};`. `therock_install_version_selector` is also -//! `pub(crate)`: `chat.rs`'s `render_install_sdk_dry_run_for_args` calls it as -//! `crate::therock_install_version_selector` through that re-import, which is +//! `pub(crate)`: `chat/mcp_bridge.rs`'s `render_install_sdk_dry_run_for_args` +//! calls it as `crate::therock_install_version_selector` through that re-import, which is //! why `sdk_install_request` moved here rather than staying in `chat.rs` — //! this module does not depend on `chat.rs`. `sdk_install_request` keeps the //! `pub(crate)` it had in `chat.rs`, although only `install` and this module's @@ -73,6 +73,7 @@ pub(crate) const fn sdk_install_request<'a>( prefix: Option, version_selector: Option, family_override: Option<&'a str>, + python_version: Option<&'a str>, dry_run: bool, devel: bool, consent: therock::SdkInstallConsent, @@ -83,6 +84,7 @@ pub(crate) const fn sdk_install_request<'a>( prefix, version_selector, family_override, + python_version, dry_run, include_devel: devel, consent, @@ -100,6 +102,7 @@ pub(crate) fn install(target: InstallTarget) -> Result<()> { version, build_date, family, + python_version, dry_run, yes, approve_replacing_active_default, @@ -125,6 +128,7 @@ pub(crate) fn install(target: InstallTarget) -> Result<()> { prefix, version_selector, family.as_deref(), + python_version.as_deref(), dry_run, devel, consents.replace_active_default, @@ -301,6 +305,7 @@ mod tests { None, None, None, + None, false, devel, therock::SdkInstallConsent::Ask, @@ -318,6 +323,7 @@ mod tests { None, None, None, + None, true, false, therock::SdkInstallConsent::Ask, @@ -348,6 +354,7 @@ mod tests { None, None, None, + None, false, devel, therock::SdkInstallConsent::Ask, diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 573044fb9a..f3fc7f3faf 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -10,12 +10,14 @@ mod advised_commands; #[path = "../../../crates/rocm-core/src/archive_props.rs"] mod archive_props; mod automations; +mod background_helper; mod bootstrap; mod bridge_snapshot; mod chat; mod chat_cmd; mod chat_host_facts; mod cli; +mod cli_args; mod cli_audit; mod cli_progress; mod cli_report; @@ -23,6 +25,7 @@ mod comfyui; mod config_cmd; mod dash; mod dash_seam; +mod diagnose_model; // Unix-only: its generators plant symlinks with `std::os::unix::fs::symlink`. #[cfg(all(test, unix))] mod deletion_properties; @@ -69,6 +72,7 @@ mod services_text; mod setup_cmd; mod shell_quote; mod storage; +mod symptom_input; mod system_packages; #[cfg(test)] mod test_support; @@ -93,7 +97,7 @@ mod update_cmd; // for other root-level commands outside `dispatch()` that call it directly. // `select_runtime_manifest` is no longer needed here: // `single_ready_runtime_key`/`resolve_runtime_selector_to_exact_key` moved out -// to `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f), reaching it via that +// to `engine_runtime_select.rs`, reaching it via that // module's own grouped `use crate::runtimes_cmd::{...}` import instead. // `update_cmd.rs`'s `apply_runtime_update`/`append_update_activate_summary` // call into `runtimes_cmd` directly too, via their own imports rather than @@ -102,17 +106,18 @@ mod update_cmd; // `managed_service_process_command`, and // `ensure_public_service_has_endpoint_key` are no longer needed here either: // their only caller outside `dispatch()`, `restart_internal_managed_service` -// (and the endpoint-key check inside it), moved out to `services_text.rs` in -// ROCMAI-209 (Phase 6f) and has since moved on to `service_control.rs`, which -// reaches them via its own imports instead. +// (and the endpoint-key check inside it), moved out to `services_text.rs` and +// has since moved on to `service_control.rs`, which reaches them via its own +// imports instead. // `record_cli_audit_event`, `cli_lifecycle_log_path` and `sanitize_log_value` // (from `cli_audit.rs`) are re-imported so other modules keep reaching them as -// `crate::`; `main.rs` itself calls `record_cli_audit_event`. +// `crate::`. // `therock_install_version_selector` (from `install_cmd.rs`) is re-imported for -// the same reason: `chat.rs` reaches it as `crate::therock_install_version_selector`. +// the same reason: `chat/mcp_bridge.rs` reaches it as +// `crate::therock_install_version_selector`. // `build_codex_bridge_gpu_snapshot` and `builtin_codex_bridge_engine_inventory` -// (from `bridge_snapshot.rs`) are re-imported only so `chat.rs` keeps reaching -// them as `crate::`; `build_codex_bridge_snapshot` is called from +// (from `bridge_snapshot.rs`) are re-imported only so `chat/mcp_bridge.rs` keeps +// reaching them as `crate::`; `build_codex_bridge_snapshot` is called from // `dispatch()` as well. // The clap command tree and its value enums (from `cli.rs`) are re-imported so // every module keeps reaching them as `crate::Command`, `crate::Provider` and so @@ -124,9 +129,9 @@ use crate::bridge_snapshot::{ }; use crate::chat_cmd::run_chat_command; use crate::cli::{ - AutomationsCommand, BenchCommand, Cli, ComfyuiCommand, Command, ConfigCommand, DevicePolicyArg, - EnginesCommand, InstallFormat, InstallTarget, Provider, RuntimesCommand, SandboxToolArg, - ServicesCommand, SetupCommand, StorageCommand, + AutomationsCommand, BenchCommand, Cli, Command, ConfigCommand, DevicePolicyArg, EnginesCommand, + InstallFormat, InstallTarget, Provider, RuntimesCommand, SandboxToolArg, ServicesCommand, + SetupCommand, StorageCommand, }; // Reached only as `super::SUPPORTED_ENGINES` from `advised_commands.rs`'s test, // so a non-test build does not carry it. @@ -188,10 +193,6 @@ use std::fs; use std::io::{self, Write}; use std::path::PathBuf; use std::process::ExitCode; -#[cfg(all(test, not(windows)))] -use std::process::Stdio; -#[cfg(all(test, not(windows)))] -use std::process::{Command as ProcessCommand, ExitStatus}; #[cfg(test)] use std::sync::Mutex; @@ -220,33 +221,6 @@ fn reset_sigpipe() { #[allow(clippy::missing_const_for_fn)] fn reset_sigpipe() {} -/// Run `f` with SIGPIPE temporarily ignored, restoring the previous -/// disposition afterwards. -/// -/// [`reset_sigpipe`] sets `SIG_DFL` process-wide so stdout/stderr pipe writes -/// terminate the process conventionally. But a write to a spawned child's stdin -/// whose reader has already exited would then kill `rocm` with SIGPIPE before -/// the caller can turn the resulting `EPIPE` into a diagnostic. Wrap such writes -/// in this helper so the failure surfaces as an `io::Error` (`BrokenPipe`) -/// instead of a signal. -#[cfg(unix)] -#[allow(unsafe_code)] // libc FFI -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - // SAFETY: reading/installing a signal disposition is always safe. - let previous = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; - let result = f(); - // SAFETY: restoring the disposition captured above is always safe. - unsafe { - libc::signal(libc::SIGPIPE, previous); - } - result -} - -#[cfg(not(unix))] -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - f() -} - /// Marker error carrying `rocm fix`'s exit code back through `main()`'s /// ordinary return path, instead of calling `std::process::exit` mid-stack /// and skipping the `_log_guard` destructor held in `run()`. @@ -412,17 +386,24 @@ fn parse_cli() -> Result { } /// What every version-reporting surface (`-V`/`--version`, `rocm version`, and -/// the MCP in-process handler) prints: the semantic version Cargo built, plus -/// the git ref and commit hash [`build.rs`] embedded, additively — never a -/// replacement, so a build with an "unknown" ref (no tag, no branch, no git) -/// still reports a real version number rather than none at all. +/// the MCP in-process handler) prints. `format_cli_version` in `build.rs` +/// computes this once from the ref and commit hash it already resolves and +/// exports it as `ROCM_CLI_VERSION`, so these surfaces cannot disagree. +/// +/// A release tag `vX.Y.Z` is shown as `X.Y.Z (vX.Y.Z, )`. A branch, +/// including `main`, is shown as `0.0.0-dev (, )`. Only an +/// unresolved ref falls back to the workspace package version, which is not +/// bumped per release (see the module docs in `build.rs`). fn cli_version_string() -> String { - format!( - "{} ({}, {})", - env!("CARGO_PKG_VERSION"), - env!("ROCM_CLI_VERSION_REF"), - env!("ROCM_CLI_GIT_HASH") - ) + env!("ROCM_CLI_VERSION").to_owned() +} + +/// The leading version of [`cli_version_string`], without the parenthetical +/// ref and hash. For one-token fields such as `rocm update`'s `cli: installed=` +/// and a problem report's `cli_version`. `build.rs` exports it beside the full +/// string, from the same function, so nothing here parses that string. +fn cli_version_number() -> String { + env!("ROCM_CLI_VERSION_NUMBER").to_owned() } /// `rocm version`: the traceable build string, plus the ROCm SDK and GPU @@ -679,13 +660,23 @@ fn dispatch(cli: Cli) -> Result<()> { Some(Command::Examine { json, framework }) => examine(json, framework.into()), Some(Command::Diagnose { symptom, + symptom_file, top, json, distro, model, + context_size, + kv_cache_type, report, send, - }) => diagnose(symptom, top, json, distro, model, report, send), + }) => { + let symptom = symptom_input::resolve(symptom, symptom_file.as_deref()) + .map_err(|error| clap_exit_code(symptom_input::usage_error(&error)))?; + let model = model.map(|model_ref| { + diagnose_model::ModelQuery::new(model_ref, context_size, kv_cache_type) + }); + diagnose(symptom, top, json, distro, model, report, send) + } // Keep this error chained rather than discarding it into a fresh // `anyhow!(...)` (e.g. via a `.map_err` that restringifies it) -- see // `FixExitCode`'s doc comment for why that would silently break its @@ -860,7 +851,7 @@ fn dispatch(cli: Cli) -> Result<()> { api_key, engine_env, }), - Some(Command::Comfyui { command }) => comfyui(command), + Some(Command::Comfyui { command }) => comfyui::run(command), Some(Command::Services { command }) => services(command), Some(Command::Remote { command }) => remote::run(command), Some(Command::Automations { command }) => automations(command), @@ -945,15 +936,19 @@ fn dispatch(cli: Cli) -> Result<()> { keep_data, keep_cache, force_dev_binaries, - }) => uninstall(UninstallOptions { - yes, - dry_run, - keep_binaries, - keep_config, - keep_data, - keep_cache, - force_dev_binaries, - }), + force, + }) => { + uninstall::reject_force(force.is_some())?; + uninstall(UninstallOptions { + yes, + dry_run, + keep_binaries, + keep_config, + keep_data, + keep_cache, + force_dev_binaries, + }) + } Some(Command::Completions { shell }) => { let mut cmd = completion_command(); clap_complete::generate(shell, &mut cmd, "rocm", &mut std::io::stdout()); @@ -1098,169 +1093,6 @@ fn platform_refusal(ruled_out: PlatformRuleOut, action: GatedEngineAction) -> Op } } -fn comfyui(command: Option) -> Result<()> { - let paths = AppPaths::discover()?; - let config = RocmCliConfig::load(&paths).unwrap_or_default(); - match command.unwrap_or(ComfyuiCommand::Status) { - ComfyuiCommand::Status => { - print!("{}", comfyui::render_status(&paths, &config)?); - record_cli_audit_event( - &paths, - "app", - "comfyui_status", - "info", - "rendered ComfyUI status", - None, - ); - Ok(()) - } - ComfyuiCommand::ModelsPath => { - print!("{}", comfyui::render_models_path(&paths)?); - record_cli_audit_event( - &paths, - "app", - "comfyui_models_path", - "info", - "rendered ComfyUI models path", - None, - ); - Ok(()) - } - ComfyuiCommand::Logs { lines } => { - print!("{}", comfyui::render_logs(&paths, lines)?); - record_cli_audit_event( - &paths, - "app", - "comfyui_logs", - "info", - "rendered ComfyUI logs", - None, - ); - Ok(()) - } - ComfyuiCommand::Install { - runtime_id, - reinstall, - dry_run, - yes: _, - } => { - match comfyui::install( - &paths, - &config, - comfyui::ComfyUiInstallOptions { - runtime_id: runtime_id.clone(), - reinstall, - dry_run, - }, - ) { - Ok(text) => { - print!("{text}"); - record_cli_audit_event( - &paths, - "app", - if dry_run { - "comfyui_install_dry_run" - } else { - "comfyui_install" - }, - "info", - format!( - "ComfyUI install completed runtime={} reinstall={} dry_run={}", - runtime_id.as_deref().unwrap_or(""), - reinstall, - dry_run - ), - None, - ); - Ok(()) - } - Err(error) => { - record_cli_audit_event( - &paths, - "app", - if dry_run { - "comfyui_install_dry_run" - } else { - "comfyui_install" - }, - "error", - format!( - "ComfyUI install failed runtime={} reinstall={} dry_run={}: {error}", - runtime_id.as_deref().unwrap_or(""), - reinstall, - dry_run - ), - None, - ); - Err(error) - } - } - } - ComfyuiCommand::Start { - host, - port, - no_open_browser, - yes: _, - } => match comfyui::start( - &paths, - comfyui::ComfyUiStartOptions { - host, - port, - no_open_browser, - }, - ) { - Ok(text) => { - print!("{text}"); - record_cli_audit_event( - &paths, - "app", - "comfyui_start", - "info", - "ComfyUI start requested", - None, - ); - Ok(()) - } - Err(error) => { - record_cli_audit_event( - &paths, - "app", - "comfyui_start", - "error", - format!("ComfyUI start failed: {error}"), - None, - ); - Err(error) - } - }, - ComfyuiCommand::Stop { yes: _ } => match comfyui::stop(&paths) { - Ok(text) => { - print!("{text}"); - record_cli_audit_event( - &paths, - "app", - "comfyui_stop", - "info", - "ComfyUI stop requested", - None, - ); - Ok(()) - } - Err(error) => { - record_cli_audit_event( - &paths, - "app", - "comfyui_stop", - "error", - format!("ComfyUI stop failed: {error}"), - None, - ); - Err(error) - } - }, - } -} - /// Shown after any activation (direct `runtimes activate`, `update --apply /// --activate`, or an `install sdk` that switches the active runtime) that /// recorded a previous runtime, so every path that reaches this state gives @@ -1442,6 +1274,134 @@ pub(crate) mod tests { ); } + /// The catalog manifest tells a caller that exit 2 is shared between an + /// unknown fix-id and a malformed command line. That claim is about clap, + /// which only this crate links, so it is proved here: a bad `rocm fix` + /// command line really exits with the code published for `unknown_id`, + /// through the same `clap_exit_code -> exit_code_for` chain `main` uses, + /// and the refusal has a line starting with the `error:` the description + /// tells a caller to look for (not `Usage:`, which a missing value does not + /// print). The + /// description's own wording is pinned in + /// `rocm-core` (`the_unknown_id_description_names_the_usage_error_it_shares_a_code_with`). + #[test] + fn a_usage_error_exits_with_the_code_published_for_an_unknown_fix_id() { + let manifest = rocm_core::catalog_manifest(); + let published = manifest.exit_codes["unknown_id"]; + for argv in [ + &["rocm", "fix", "--no-such-option"][..], + &["rocm", "fix", "fix-9-igpu-dgpu", "--device-index"][..], + &[ + "rocm", + "fix", + "fix-9-igpu-dgpu", + "--device-index", + "not-a-number", + ][..], + ] { + let err = super::cli_command() + .try_get_matches_from(argv) + .expect_err("a malformed command line must be a parse error"); + let rendered = err.render().to_string(); + assert!( + rendered.lines().any(|line| line.starts_with("error:")) + && !rendered.contains(rocm_core::fix_exit::UNKNOWN_ID_PREFIX), + "{argv:?}: the description promises `error:` and no unknown-id message:\n{rendered}" + ); + let result: Result<()> = Err(super::clap_exit_code(err)); + assert_eq!( + super::exit_code_for(result), + ExitCode::from(published as u8), + "{argv:?}: a usage error must exit with the code the manifest names for it" + ); + } + assert!( + manifest.exit_code_descriptions["unknown_id"].contains("malformed command line"), + "the published description must warn that a usage error exits {published} too" + ); + } + + /// A negative device index exits with one of two published codes, and the + /// catalog says which by how it is written: `--device-index=-1` reaches the + /// fix, which refuses it as `not_applicable`; `--device-index -1` never gets + /// past the parser, which reads `-1` as an unknown option, a usage error that + /// exits with the `unknown_id` code. Both go through the real parser and + /// the same exit path `main` uses, the first through `dispatch` as well. + /// Under `--dry-run`, so a lost guard in the runner would only preview. + /// + /// "Reaches the fix" holds on Windows and Linux, where fix-9 applies. On + /// WSL it does not apply, so `apply` returns 3 from the platform gate before + /// the runner, and the `=-1` half cannot notice a lost runner guard there; + /// `a_negative_device_index_is_not_applicable_and_nothing_runs` in + /// `rocm-core`'s `fix/runners.rs` holds the guard on every host. + #[test] + fn a_negative_device_index_reaches_the_fix_only_when_written_with_equals() { + let manifest = rocm_core::catalog_manifest(); + let not_applicable = manifest.exit_codes["not_applicable"]; + let unknown_id = manifest.exit_codes["unknown_id"]; + + let cli = super::Cli::try_parse_from([ + "rocm", + "fix", + "fix-9-igpu-dgpu", + "--device-index=-1", + "--dry-run", + ]) + .expect("`--device-index=-1` must parse: the fix, not the parser, refuses it"); + assert!( + matches!( + cli.command, + Some(super::Command::Fix { + device_index: Some(-1), + .. + }) + ), + "the parsed value must be -1" + ); + let mut env = ScopedTestEnv::new(); + env.set("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", "1"); + let result = super::dispatch(cli); + drop(env); + assert_eq!( + super::exit_code_for(result), + ExitCode::from(not_applicable as u8), + "`--device-index=-1` must exit with the code published as not_applicable" + ); + + let err = super::cli_command() + .try_get_matches_from([ + "rocm", + "fix", + "fix-9-igpu-dgpu", + "--device-index", + "-1", + "--dry-run", + ]) + .expect_err("`--device-index -1` must be a usage error"); + let rendered = err.render().to_string(); + assert!( + rendered.lines().any(|line| line.starts_with("error:")) + && !rendered.contains(rocm_core::fix_exit::UNKNOWN_ID_PREFIX), + "`--device-index -1` must be refused as a usage error:\n{rendered}" + ); + let result: Result<()> = Err(super::clap_exit_code(err)); + assert_eq!( + super::exit_code_for(result), + ExitCode::from(unknown_id as u8), + "`--device-index -1` must exit with the code published as unknown_id" + ); + + for (name, spelling) in [ + ("not_applicable", "`--device-index=-1`"), + ("unknown_id", "`--device-index -1`"), + ] { + assert!( + manifest.exit_code_descriptions[name].contains(spelling), + "the {name} description must name {spelling}" + ); + } + } + /// Any other error must still fail with exit 1, matching what /// `Result<(), anyhow::Error>`'s `Termination` impl already does today for /// every subcommand other than `fix`. @@ -1609,89 +1569,6 @@ pub(crate) mod tests { } } - /// Regression test for the engine child-stdin write under the process-wide - /// `SIG_DFL` that `main` installs via [`reset_sigpipe`]. If an engine child - /// exits before reading its stdin, the parent's write to that pipe must - /// surface as a `BrokenPipe` error — handled by the caller's diagnostics — - /// rather than killing `rocm` with SIGPIPE before those diagnostics run. - /// - /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE - /// at Rust's default `SIG_IGN`, and flipping it process-wide would race - /// sibling tests. So this re-execs itself in fresh processes that install - /// `SIG_DFL` first, and checks BOTH directions: - /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — - /// this is the regression the plain `SIG_DFL` reset introduced; - /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns - /// an error instead — this is the fix. - /// - /// Each child reaps its short-lived grandchild before writing, so the read - /// end is closed deterministically and the outcome does not race. - #[cfg(unix)] - #[test] - fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { - use std::os::unix::process::ExitStatusExt as _; - - // Grandchild helper: reproduce main()'s SIG_DFL, spawn a process that - // exits immediately, reap it, then write to its now-closed stdin. The - // "guarded" arm wraps the write; the "unguarded" arm does not. - if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { - reset_sigpipe(); - let mut grandchild = ProcessCommand::new("true") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn a process that exits immediately"); - let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); - grandchild.wait().expect("reap grandchild"); - // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; - // it must exceed the pipe buffer so the write actually reaches the - // closed read end rather than being swallowed by kernel buffering. - let payload = vec![b'x'; 64 * 1024]; - if mode == "guarded" { - let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); - assert!( - result.is_err(), - "guarded write to a dead pipe should return BrokenPipe, not succeed" - ); - } else { - // Unguarded: under SIG_DFL this write delivers SIGPIPE and never - // returns. If the process somehow survives, exit 0 so the parent's - // "should have been signalled" assertion fails loudly. - let _ = stdin.write_all(&payload); - } - return; - } - - let exe = std::env::current_exe().expect("current test executable"); - let run_arm = |mode: &str| -> ExitStatus { - ProcessCommand::new(&exe) - .args([ - "tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", - "--exact", - "--nocapture", - ]) - .env("ROCM_TEST_SIGPIPE_ARM", mode) - .status() - .expect("re-exec the test arm in a child process") - }; - - let unguarded = run_arm("unguarded"); - assert_eq!( - unguarded.signal(), - Some(libc::SIGPIPE), - "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ - SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" - ); - - let guarded = run_arm("guarded"); - assert!( - guarded.success(), - "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ - SIGPIPE reached the process instead of surfacing as an error" - ); - } - #[test] fn completions_generate_for_every_shell() { use clap_complete::Shell; diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index f0fd2b5bff..ccd8826ef1 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -5,10 +5,10 @@ //! Managed-service liveness classification and the dashboard-sidebar //! status/counts helpers derived from it. //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-211) of the `main.rs` modularization (see -//! `docs/architecture.md`). `dispatch()`'s call sites are unaffected: none of -//! this module's functions are called directly from `dispatch()`. +//! Mechanically relocated from `main.rs` with no behavior change (the +//! `relocation` class in `docs/architecture.md`). `dispatch()`'s call sites are +//! unaffected: none of this module's functions are called directly from +//! `dispatch()`. //! //! This module owns: reading and liveness-refreshing the on-disk managed-service //! registry (`load_managed_services`, `refresh_managed_service_runtime_liveness`, @@ -22,13 +22,14 @@ //! attempts counts and one-line status text //! (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/ //! `local_server_sidebar_status`) plus the engine display-name helper -//! (`friendly_engine_label`), which only `chat.rs` reads. `service_records.rs` +//! (`friendly_engine_label`), which only `chat/summaries.rs` reads. `service_records.rs` //! calls into it for the in-use check behind service-record removal and pruning //! (`managed_service_is_live`, `recorded_service_pids`) and for //! `load_managed_services` in prune planning, and `bridge_snapshot.rs` calls //! `load_managed_services` for the codex-bridge snapshot; //! `services_text.rs` reads the sidebar helpers for the local-servers view; -//! `chat.rs` calls into it for the deterministic services-tool summary; +//! `chat/turn.rs` and `chat/mcp_bridge.rs` call into it for the services tool +//! and the local-assistant readiness check; //! `runtime_services.rs` calls into it when reconciling live servers against a //! runtime activation; `service_launch.rs` is the only caller of the //! `existing_live_managed_service_in` duplicate-launch guard it runs before @@ -41,8 +42,8 @@ use std::time::Duration; use anyhow::{Context, Result}; use rocm_core::{ - AppPaths, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, - managed_service_endpoint_readiness, process_is_running, + AppPaths, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, ProcessProbe, + managed_service_endpoint_readiness, }; use crate::endpoint_keys; @@ -189,9 +190,40 @@ pub(crate) fn recorded_service_pids(record: &ManagedServiceRecord) -> Vec { .collect() } +/// Whether any process the record names may still be running: everything but a +/// confirmed exit counts (see [`ProcessProbe::may_be_running`]), and a `0` +/// placeholder PID is never asked about. +/// +/// The one reading behind every destructive decision made from a recorded +/// process's liveness: the deferred key cleanup, the reconciliation to +/// `stopped`, and the in-use check behind `rocm services remove` and `prune`. +/// Kept in one place so those cannot drift apart on what counts as gone. +pub(crate) fn recorded_pids_may_be_running( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { + recorded_service_pids(record) + .iter() + .any(|pid| probe(*pid).may_be_running()) +} + +/// Whether the record is a container-backed engine's stop that was requested and +/// has not been confirmed. +/// +/// Such a stop is decided by the engine `Stop` (`engine_stop_decides_confirmation`), +/// not by the recorded supervisor, which can be gone while the container still +/// runs. Until a `services stop` confirms it, nothing here may read the record as +/// stopped, complete its cleanup, or let `remove`/`prune` delete it, on the +/// evidence of the supervisor's PID alone. +pub(crate) fn container_stop_unresolved(record: &ManagedServiceRecord) -> bool { + record.stop_requested_unix_ms.is_some() + && rocm_core::engine_stop_decides_confirmation(&record.engine) +} + /// Complete a stop that was requested but could not confirm termination, once -/// the processes are actually observed gone: drop the endpoint key and clear the -/// marker. Returns whether the record changed. +/// the processes are actually observed gone: drop the endpoint key, clear the +/// marker, forget the recorded processes and read the record as stopped. Returns +/// whether the record changed. /// /// This is deliberately keyed on `stop_requested_unix_ms` rather than on "the /// tracked pids are dead". A *crashed* service also has dead pids, but its @@ -205,18 +237,66 @@ pub(crate) fn recorded_service_pids(record: &ManagedServiceRecord) -> Vec { /// [`refresh_managed_service_runtime_liveness`], so a record that reached /// "stopped" by another route still gets its deferred cleanup rather than /// stranding the key. -fn settle_pending_stop_key_cleanup(paths: &AppPaths, record: &mut ManagedServiceRecord) -> bool { +/// +/// A container-backed engine never completes this way: its supervisor can exit +/// while the container runs, so only a stop whose engine `Stop` confirms may end +/// it. +/// +/// "Observed gone" means every recorded process is *confirmed* gone: a process +/// that could not be queried (on Windows, a live one this process may not open) +/// is not gone, and the stop that reported it unverified must keep its key and +/// its marker until it is. +fn settle_pending_stop_key_cleanup( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { if record.stop_requested_unix_ms.is_none() { return false; } - if recorded_service_pids(record) - .iter() - .any(|pid| process_is_running(*pid)) - { + // A container-backed engine (`engine_stop_decides_confirmation`) is stopped by + // its engine `Stop`, not by the recorded supervisor: that process can be gone + // while the container still runs, so PID evidence alone cannot say the stop + // completed. The stop that could not confirm keeps its key, marker and + // recorded processes, and the next `services stop` asks the engine again. + if container_stop_unresolved(record) { + return false; + } + if recorded_pids_may_be_running(record, probe) { return false; } endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); - record.stop_requested_unix_ms = None; + let consumed_marker = record.stop_requested_unix_ms.take(); + // A record that named no process has no stop to complete: the key and the + // stale marker go, as they always did, but it is not read as stopped. It may + // be a launch that has not recorded its supervisor yet. The marker's value + // stays on the record (`settled_stop_unix_ms`), because a stop that began on + // this empty record and finds it with no marker would otherwise read this + // cleanup as a relaunch and report itself unconfirmed. + if recorded_service_pids(record).is_empty() { + record.settled_stop_unix_ms = consumed_marker; + return true; + } + // Every recorded process is confirmed gone, so none of them is the + // service's any more, and their PIDs are stale from here on. Left in the + // record, a later `services stop` would signal whatever the OS has handed + // those numbers to, and a PID recorded without a start-time token cannot be + // refuted, so on Windows the tree kill would take that process and its + // subtree. The same forgetting as a stop that confirmed the exit itself + // (`clear_recorded_processes`), and it is done only here, behind the + // confirmed-gone check above: a process that may still be running stays named. + record.clear_recorded_processes(); + // And the status a confirmed stop leaves, whatever the record read before: + // `stopped`, as `rocm services stop` and `rocmd`'s stop set on a confirmed + // stop for a `failed` record too. Set here rather than left to the + // reconciliation below, which has no PID to read once they are cleared, so a + // `ready` or `running` record would stay live; and not only for a live status, + // because a `failed` record with its processes cleared is not the `stopped` + // record `is_covered_by_stop_of` accepts as this same stop completing, so a + // stop still finalizing would report itself unconfirmed. Setting it also makes + // the caller's liveness gate return at once, so the one write the caller makes + // covers the key cleanup, the cleared processes and the status together. + "stopped".clone_into(&mut record.status); true } @@ -224,7 +304,20 @@ pub(crate) fn refresh_managed_service_runtime_liveness( paths: &AppPaths, record: &mut ManagedServiceRecord, ) -> bool { - let settled_pending_stop = settle_pending_stop_key_cleanup(paths, record); + refresh_managed_service_runtime_liveness_with(paths, record, rocm_core::probe_process) +} + +/// [`refresh_managed_service_runtime_liveness`] with the read of each recorded +/// PID supplied by the caller, so a test drives the answer a host can give for a +/// process it cannot query. Every decision here that a process is gone (the +/// deferred key cleanup, the status reconciled to `stopped`) takes a confirmed +/// exit; a process that could not be queried counts as possibly running. +pub(crate) fn refresh_managed_service_runtime_liveness_with( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { + let settled_pending_stop = settle_pending_stop_key_cleanup(paths, record, &probe); if !managed_service_is_live(record) { return settled_pending_stop; } @@ -268,9 +361,14 @@ pub(crate) fn refresh_managed_service_runtime_liveness( return settled_pending_stop || outcome.record_changed; } + // An unresolved container stop is not read as stopped from the supervisor's + // PID, whether or not the endpoint answers: only the engine `Stop` decides it. + if container_stop_unresolved(record) { + return settled_pending_stop || outcome.record_changed; + } let tracked_pids = recorded_service_pids(record); let has_tracked_pid = !tracked_pids.is_empty(); - let has_live_pid = tracked_pids.iter().any(|pid| process_is_running(*pid)); + let has_live_pid = recorded_pids_may_be_running(record, &probe); if has_tracked_pid && !has_live_pid { // Reconcile the status only. The endpoint key is *not* dropped here: // dead pids alone do not distinguish a stopped service from a crashed @@ -922,4 +1020,260 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// A record whose stop was requested but not confirmed, with its endpoint + /// key on disk: the state a stop leaves when a recorded process cannot be + /// verified. + fn pending_stop_record(paths: &AppPaths, service_id: &str) -> Result { + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + "model-ref", + "canonical/model", + "127.0.0.1", + port, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.stop_requested_unix_ms = Some(1); + record.write()?; + endpoint_keys::store_endpoint_api_key(paths, service_id, "key-123")?; + Ok(record) + } + + /// A process that cannot be queried (on Windows, a live one this process may + /// not open) is not one that is gone. Whatever is reading liveness must keep + /// the key and the stop marker the unverified stop left, and must not + /// reconcile the record to `stopped`; only a confirmed exit may. + #[test] + fn a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried() -> Result<()> { + let (root, paths) = test_paths("liveness-unqueryable-pending-stop"); + paths.ensure()?; + let service_id = "svc-unqueryable"; + let mut record = pending_stop_record(&paths, service_id)?; + // The dangerous shape: a supervisor recorded without a start-time token, + // and an engine with one. + record.supervisor_pid = 4242; + record.supervisor_start_ticks = None; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Unqueryable + }); + assert_eq!( + ( + record.supervisor_pid, + record.engine_pid, + record.engine_start_ticks + ), + (4242, Some(4243), Some(77)), + "a process that may still be running stays named: the record is the only thing that names it" + ); + + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), + "the key of a service that may still be running is kept" + ); + assert_eq!( + record.stop_requested_unix_ms, + Some(1), + "so is the stop marker that hands the cleanup to a later confirmed exit" + ); + assert_ne!( + record.status, "stopped", + "a process that could not be looked at does not make the service stopped" + ); + + // The same record once the process is confirmed gone: now the deferred + // cleanup and the reconciliation both happen. + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none(), + "a confirmed exit drops the key" + ); + assert_eq!(record.stop_requested_unix_ms, None); + assert_eq!(record.status, "stopped"); + assert_eq!( + ( + record.supervisor_pid, + record.supervisor_start_ticks, + record.engine_pid, + record.engine_start_ticks + ), + (0, None, None, None), + "the processes confirmed gone are forgotten, so a later stop cannot reach a recycled PID" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The deferred cleanup forgets the PIDs, so the reconciliation after it has + /// none to read: every live status must still end as `stopped`, in the one + /// refresh, rather than a record such as `starting` staying live with no + /// process behind it, and a `failed` one must too, as a confirmed stop makes it. + /// A container-backed engine's supervisor can be gone while the container + /// still runs, and its stop is decided by the engine `Stop`, which the + /// unconfirmed stop did not get. A listing that sees the supervisor gone must + /// not complete that stop: key, marker and recorded processes stay for the + /// next `services stop`. A process engine with the same PID evidence does + /// complete (the sibling tests above). + #[test] + fn a_pending_stop_of_a_container_engine_is_not_completed_by_a_gone_supervisor() -> Result<()> { + assert!(rocm_core::engine_stop_decides_confirmation("aim")); + let (root, paths) = test_paths("liveness-container-pending-stop"); + paths.ensure()?; + let service_id = "svc-container"; + let mut record = pending_stop_record(&paths, service_id)?; + "aim".clone_into(&mut record.engine); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), + "the key stays: the container may still be serving with it" + ); + assert_eq!(record.stop_requested_unix_ms, Some(1)); + assert_eq!( + (record.supervisor_pid, record.supervisor_start_ticks), + (4242, Some(5)), + "the record keeps naming what the stop could not confirm" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A record that names no process has no stop to complete: a `starting` + /// record with a stale stop marker gets the marker and key cleaned, as before, + /// but is not read as stopped, since it may be a launch that has not recorded + /// its supervisor yet. + #[test] + fn a_stale_stop_marker_on_a_record_naming_no_process_is_cleaned_without_reading_it_stopped() + -> Result<()> { + let (root, paths) = test_paths("liveness-pending-stop-no-process"); + paths.ensure()?; + let service_id = "svc-no-process"; + let mut record = pending_stop_record(&paths, service_id)?; + "starting".clone_into(&mut record.status); + record.clear_recorded_processes(); + + let changed = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!(changed, "the stale marker is consumed and persisted"); + assert_eq!(record.stop_requested_unix_ms, None); + assert!(endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none()); + assert_eq!(record.status, "starting", "it is not read as stopped"); + assert_eq!( + record.settled_stop_unix_ms, + Some(1), + "the consumed marker's stamp stays, so a stop that began on this record can tell its own marker was consumed from a relaunch having cleared it" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn a_completed_pending_stop_reads_stopped_whatever_live_status_it_had() -> Result<()> { + for status in ["starting", "running", "ready", "recovering", "failed"] { + let (root, paths) = test_paths(&format!("liveness-pending-stop-{status}")); + paths.ensure()?; + let mut record = pending_stop_record(&paths, "svc-status")?; + record.status = status.to_owned(); + + let changed = + refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!(changed, "{status}"); + assert_eq!(record.status, "stopped", "{status}"); + assert_eq!(record.supervisor_pid, 0, "{status}"); + assert_eq!( + record.settled_stop_unix_ms, None, + "{status}: a record that names processes reads stopped and needs no stamp" + ); + let _ = fs::remove_dir_all(root); + } + Ok(()) + } + + #[test] + fn a_pending_stop_keeps_its_key_and_marker_while_a_process_runs() -> Result<()> { + let (root, paths) = test_paths("liveness-running-pending-stop"); + paths.ensure()?; + let service_id = "svc-running"; + let mut record = pending_stop_record(&paths, service_id)?; + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Running + }); + + assert!(endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some()); + assert_eq!(record.stop_requested_unix_ms, Some(1)); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The one reading behind the three destructive decisions: any recorded + /// process that is not confirmed gone counts, both roles are asked about, and + /// a placeholder `0` PID never is. + #[test] + fn recorded_pids_may_be_running_unless_every_recorded_process_is_confirmed_gone() { + use std::cell::RefCell; + + let (root, paths) = test_paths("recorded-pids-may-be-running"); + let mut record = pending_stop_record(&paths, "svc-pids").expect("record"); + let _ = fs::remove_dir_all(root); + record.supervisor_pid = 111; + record.engine_pid = Some(222); + + let asked_cell = RefCell::new(Vec::new()); + let asked = &asked_cell; + let by_pid = |running: &'static [(u32, ProcessProbe)]| { + move |pid: u32| { + asked.borrow_mut().push(pid); + running + .iter() + .find(|(candidate, _)| *candidate == pid) + .map_or(ProcessProbe::Gone, |(_, probe)| *probe) + } + }; + + assert!(recorded_pids_may_be_running( + &record, + by_pid(&[(222, ProcessProbe::Unqueryable)]) + )); + assert!(recorded_pids_may_be_running( + &record, + by_pid(&[(111, ProcessProbe::Running)]) + )); + assert!(!recorded_pids_may_be_running(&record, by_pid(&[]))); + let engine_only = asked.borrow().iter().filter(|pid| **pid == 222).count(); + assert!(engine_only >= 1, "the engine PID is asked about too"); + + record.supervisor_pid = 0; + record.engine_pid = None; + asked.borrow_mut().clear(); + assert!(!recorded_pids_may_be_running( + &record, + by_pid(&[(0, ProcessProbe::Running)]) + )); + assert!(asked.borrow().is_empty(), "a 0 placeholder is never probed"); + } } diff --git a/apps/rocm/src/model_catalog_text.rs b/apps/rocm/src/model_catalog_text.rs index c9cafd54e1..c10b270e5b 100644 --- a/apps/rocm/src/model_catalog_text.rs +++ b/apps/rocm/src/model_catalog_text.rs @@ -20,10 +20,8 @@ //! `select_serve_engine`. //! //! The two renderers and `format_bytes` are `pub(crate)`; `main.rs` imports them -//! and `chat.rs`, `storage.rs` and `automations.rs` reach them as -//! `crate::`. `model_registry_adapter_availability_note` is `pub(crate)` -//! only so the engine-status test that stays in `examine_report_text.rs` can -//! call it. Every other item is private. +//! and `chat/mcp_bridge.rs`, `storage.rs` and `automations.rs` reach them as +//! `crate::`. Every other item is private. use std::collections::BTreeMap; use std::fmt::Write as _; @@ -819,7 +817,7 @@ fn append_model_engine_support_lines( } #[allow(dead_code)] -pub(crate) fn model_registry_adapter_availability_note( +fn model_registry_adapter_availability_note( engine: &str, docker_reachable: &dyn Fn() -> bool, ) -> Option<&'static str> { @@ -1407,4 +1405,21 @@ mod tests { assert!(!rendered.contains("CPU fallback")); assert!(rendered.contains("engine_action: use /engine install ")); } + + /// The adapter-availability note the verbose listing appends for vLLM: a + /// native Windows host gets the explicit `unsupported_native_windows` + /// status, every other host gets no note. Moved here from the + /// `engine_runtime_status_label` test in `examine_report_text.rs`, which + /// keeps the `runtime:` label half of the same check. + #[test] + fn vllm_adapter_availability_note_is_explicit_on_native_windows() { + if cfg!(windows) { + assert!( + model_registry_adapter_availability_note("vllm", &|| true) + .is_some_and(|note| note.contains("unsupported_native_windows")) + ); + } else { + assert!(model_registry_adapter_availability_note("vllm", &|| true).is_none()); + } + } } diff --git a/apps/rocm/src/model_fit.rs b/apps/rocm/src/model_fit.rs index 4ce02b58b6..2d4fadeb27 100644 --- a/apps/rocm/src/model_fit.rs +++ b/apps/rocm/src/model_fit.rs @@ -15,8 +15,11 @@ //! out for the platform (native Windows for vLLM and AIM, an unreachable Docker //! daemon for AIM) and word the reason. //! -//! Mechanically relocated from `main.rs` with no behavior change; see -//! `docs/architecture.md`. It is a module of its own, rather than part of the +//! Relocated from `main.rs` with no behavior change; see +//! `docs/architecture.md`. Since then the memory classification answers "no GPU +//! is visible" when a device mask hides every card (it reported a card of 0 GiB, +//! or an APU's unreadable pool), and the assessment attaches the GGUF estimate. +//! It is a module of its own, rather than part of the //! `rocm diagnose` handler, so `examine_report_text.rs` (which reads //! `PlatformRuleOut` and `engine_ruled_out_by_platform`) and //! `model_catalog_text.rs` (which reads `PlatformRuleOut` and @@ -36,13 +39,16 @@ //! `model_fit::tests::docker_unreachable_subprocess_helper`; that path moved //! with the helper. +use rocm_core::gguf_source::{GgufSource, LiveGgufSource}; use rocm_core::model_readiness::{ - AcceleratorMemory, HostEngineChoice, HostFacts, ModelCatalogSource, ModelReadiness, + AcceleratorMemory, ContextRequest, HostEngineChoice, HostFacts, ModelCatalogSource, + ModelReadiness, }; use rocm_core::{ AppPaths, ModelRecipeRecord, RocmCliConfig, detect_host_gpu_summary, load_model_recipe_registry, }; +use crate::diagnose_model; use crate::gpu_policy::{GpuVramUsage, gpu_vram_usage, vram_capacity_is_meaningful}; use crate::serve_cmd::select_serve_engine; @@ -56,9 +62,16 @@ use crate::serve_cmd::select_serve_engine; pub(crate) fn assess_model_on_this_host( model_ref: &str, examination: &rocm_core::Examination, + context: &ContextRequest, ) -> ModelReadiness { let paths = AppPaths::discover().ok(); - assess_model_with_host_paths(model_ref, examination, paths.as_ref()) + assess_model_with_host_paths( + model_ref, + examination, + paths.as_ref(), + context, + &LiveGgufSource::new(), + ) } /// Testable core of [`assess_model_on_this_host`]. @@ -77,16 +90,25 @@ fn assess_model_with_host_paths( model_ref: &str, examination: &rocm_core::Examination, paths: Option<&AppPaths>, + context: &ContextRequest, + gguf_source: &dyn GgufSource, ) -> ModelReadiness { let host_gpu_summary = detect_host_gpu_summary(paths); + // Read once: the memory classification and the mask caveat below both need + // the rows, and each read is an `amd-smi` subprocess. + let vram = gpu_vram_usage(); let host = HostFacts { - accelerator_memory: host_accelerator_memory(&host_gpu_summary, examination), + accelerator_memory: host_accelerator_memory( + &host_gpu_summary, + examination, + vram.as_deref(), + ), system_ram_gib: rocm_core::detect_system_ram_gib(), }; let configured_engine = paths .and_then(|paths| RocmCliConfig::load(paths).ok()) .and_then(|config| config.default_engine); - match load_model_recipe_registry() { + let mut readiness = match load_model_recipe_registry() { Ok(registry) => assess_model_for_host( model_ref, &ModelCatalogSource::Available(®istry), @@ -103,7 +125,15 @@ fn assess_model_with_host_paths( configured_engine.as_deref(), Some(&host_gpu_summary), ), - } + }; + diagnose_model::attach_gguf_estimate( + &mut readiness, + context, + &host, + gguf_source, + vram.as_deref(), + ); + readiness } /// The memory pool an engine on this host would allocate a model from. @@ -147,9 +177,10 @@ fn examination_reports_amd_gpu(examination: &rocm_core::Examination) -> bool { fn host_accelerator_memory( host_gpu_summary: &rocm_core::HostGpuSummary, examination: &rocm_core::Examination, + vram: Option<&[GpuVramUsage]>, ) -> AcceleratorMemory { classify_accelerator_memory( - gpu_vram_usage().as_deref(), + vram, host_gpu_summary.gfx_target.as_deref(), examination_reports_amd_gpu(examination), rocm_core::usable_amd_gpu_indices().as_deref(), @@ -190,8 +221,11 @@ fn classify_accelerator_memory( .iter() .filter(|usage| usable_indices.is_none_or(|indices| indices.contains(&usage.index))) .map(|usage| usage.total_mb as f64 / 1024.0) - .fold(0.0_f64, f64::max); - return AcceleratorMemory::Dedicated(max_gib); + .reduce(f64::max); + // A mask that leaves no card is "no GPU is visible", the same answer + // `has_usable_amd_gpu` gives -- not a card with 0 GiB, which a verdict + // would then print as a measured figure ("this host offers 0 GiB"). + return max_gib.map_or(AcceleratorMemory::None, AcceleratorMemory::Dedicated); } // This is an APU: the carve-out `amd-smi` reports above is the wrong pool, // per `vram_capacity_is_meaningful`'s own doc. The right pool is the GTT @@ -210,6 +244,14 @@ fn classify_accelerator_memory( // read, this reports `UnifiedMemoryUnreadable`, so the verdict reads // `Undetermined` with no dead-end command attached, rather than a // confident wrong one. + // + // A mask that leaves the APU's one device out is "no GPU is visible", as + // for a discrete card above: that is the answer a command can change. + let masked_out = usable_indices + .is_some_and(|indices| !vram.iter().any(|usage| indices.contains(&usage.index))); + if masked_out { + return AcceleratorMemory::None; + } AcceleratorMemory::UnifiedMemoryUnreadable } @@ -222,7 +264,7 @@ fn classify_accelerator_memory( /// reported as such rather than silently swapped for another: `serve` does not /// fall back either, and a verdict that pretended otherwise would be wrong in /// the user's favour. -fn assess_model_for_host( +pub(crate) fn assess_model_for_host( model_ref: &str, catalog: &ModelCatalogSource<'_>, host: &HostFacts, @@ -571,8 +613,13 @@ mod tests { let summary = detect_host_gpu_summary(Some(&paths)); let expected = select_serve_engine(None, Some("vllm"), Some(recipe), Some(&summary)); - let readiness = - assess_model_with_host_paths(&recipe.canonical_model_id, &examination, Some(&paths)); + let readiness = assess_model_with_host_paths( + &recipe.canonical_model_id, + &examination, + Some(&paths), + &ContextRequest::default(), + &LiveGgufSource::new(), + ); let _ = std::fs::remove_dir_all(&dir); @@ -623,6 +670,16 @@ mod tests { "an APU's memory pool must read UnifiedMemoryUnreadable, not a fabricated figure, \ and not the generic Unknown either: {result:?}" ); + // A mask that leaves the APU visible changes nothing; one that hides it + // is "no GPU is visible", as for a discrete card. + assert_eq!( + classify_accelerator_memory(Some(&apu_carveout), Some("gfx1151"), true, Some(&[0])), + AcceleratorMemory::UnifiedMemoryUnreadable + ); + assert_eq!( + classify_accelerator_memory(Some(&apu_carveout), Some("gfx1151"), true, Some(&[])), + AcceleratorMemory::None + ); } /// No VRAM telemetry at all, but the examination positively saw an AMD @@ -719,6 +776,44 @@ mod tests { ); } + /// A mask that hides every card is "no GPU is visible", not a card with no + /// memory: the fold over the visible rows used to start from 0.0 and report + /// `Dedicated(0.0)`, which a verdict printed as a measured "0 MiB". + #[test] + fn classify_accelerator_memory_reports_none_when_the_mask_leaves_no_gpu() { + let vram = [ + GpuVramUsage { + index: 0, + used_mb: 0, + total_mb: 24 * 1024, + }, + GpuVramUsage { + index: 1, + used_mb: 0, + total_mb: 8 * 1024, + }, + ]; + // `HIP_VISIBLE_DEVICES=""`: no usable device at all. + assert_eq!( + classify_accelerator_memory(Some(&vram), Some("gfx942"), true, Some(&[])), + AcceleratorMemory::None + ); + // A mask naming only a device `amd-smi` has no row for. + assert_eq!( + classify_accelerator_memory(Some(&vram), Some("gfx942"), true, Some(&[5])), + AcceleratorMemory::None + ); + // Unchanged: no mask, and a mask that leaves a card. + assert_eq!( + classify_accelerator_memory(Some(&vram), Some("gfx942"), true, None), + AcceleratorMemory::Dedicated(24.0) + ); + assert_eq!( + classify_accelerator_memory(Some(&vram), Some("gfx942"), true, Some(&[1])), + AcceleratorMemory::Dedicated(8.0) + ); + } + /// The end-to-end proof, one layer above `classify_accelerator_memory`'s own /// unit tests: not just that the function picks the right number, but that /// picking the wrong one would have changed the verdict a user actually diff --git a/apps/rocm/src/prompt_text.rs b/apps/rocm/src/prompt_text.rs index 7647c532f7..8324c1fd89 100644 --- a/apps/rocm/src/prompt_text.rs +++ b/apps/rocm/src/prompt_text.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Text extraction from a natural-language request, shared by `chat.rs`'s +//! Text extraction from a natural-language request, shared by `chat/intent.rs`'s //! prompt-intent heuristics (`fallback_rocm_tool_call_for_prompt`, //! `fallback_config_tool_call_for_prompt`, `install_sdk_chat_approval_for_prompt` //! and the `prompt_*` predicates) and the freeform planner: which of a list of diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index 87d6c9ff44..d35d738698 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -1068,7 +1068,7 @@ mod tests { #[test] fn freeform_plan_dedup_prevents_provider_assisted_divergence() -> Result<()> { - // Regression for the `chat.rs` `natural_language_plan` call site, + // Regression for the `chat/mcp_bridge.rs` `natural_language_plan` call site, // which used to call `render_freeform_plan` and the now-removed // `freeform_plan_next_action_with_context` independently -- each // calling `build_freeform_plan_with_context` on its own. @@ -1918,6 +1918,7 @@ mod tests { requested_model: "Qwen/Qwen3.5".to_owned(), api_model: "Qwen/Qwen3.5".to_owned(), chat_endpoint: "http://127.0.0.1:8000/v1/chat/completions".to_owned(), + anthropic_endpoint: None, service_id: "svc-ready".to_owned(), status: "ready".to_owned(), already_running: false, diff --git a/apps/rocm/src/providers/wire.rs b/apps/rocm/src/providers/wire.rs index e30a6ae433..4892bf2866 100644 --- a/apps/rocm/src/providers/wire.rs +++ b/apps/rocm/src/providers/wire.rs @@ -7,8 +7,9 @@ //! non-streaming responses, and the SSE emitters that turn a streamed response //! into `ProviderStreamEvent`s. //! -//! Depends on the shared request/response types in `providers` and on the ROCm -//! tool schema in `tool_schema`; the local HTTP transport in `local_http` +//! Depends on the shared request/response types in `providers`, on the ROCm +//! tool schema in `tool_schema` and on the chat-template rule in +//! [`rocm_dash_core::chat_template`]; the local HTTP transport in `local_http` //! reads [`OpenAiSseEmitter`] from here, so the dependency is one-way. use super::tool_schema::{rocm_anthropic_tool_definitions, rocm_openai_tool_definitions}; @@ -46,19 +47,18 @@ pub(super) fn openai_local_stream_chat_request_body( body } +/// The chat-template rule itself lives in +/// [`rocm_dash_core::chat_template`], not here: the dashboard's chat client +/// sends the same request to the same locally served model and has to decide +/// this identically, and a rule kept in two places drifts. fn apply_local_model_request_defaults(model: &str, body: &mut serde_json::Value) { - if local_model_should_disable_thinking(model) { + if rocm_dash_core::chat_template::local_model_should_disable_thinking(model) { body["chat_template_kwargs"] = serde_json::json!({ "enable_thinking": false }); } } -fn local_model_should_disable_thinking(model: &str) -> bool { - let normalized = model.to_ascii_lowercase(); - normalized.contains("qwen3") || normalized.contains("qwen/qwen3") -} - fn local_openai_compatible_request(request: &ChatRequest) -> ChatRequest { let system = request .messages diff --git a/apps/rocm/src/remote/doctor.rs b/apps/rocm/src/remote/doctor.rs index 66e8e279e0..33379fa766 100644 --- a/apps/rocm/src/remote/doctor.rs +++ b/apps/rocm/src/remote/doctor.rs @@ -382,6 +382,7 @@ mod tests { fn real_report_with_a_fix() -> DiagnoseReport { use rocm_core::diagnose::{Diagnosis, Fix}; report(vec![Diagnosis { + symptom_matched: false, id: "dkms-mismatch".to_owned(), title: "DKMS built against another kernel".to_owned(), score: 90, @@ -590,6 +591,7 @@ mod tests { let rendered = render_report( "gpu-box", &report(vec![Diagnosis { + symptom_matched: false, id: "annotated".to_owned(), title: "a fix that explains itself".to_owned(), score: 90, @@ -630,6 +632,7 @@ mod tests { let rendered = render_report( "gpu-box", &report(vec![Diagnosis { + symptom_matched: false, id: "container".to_owned(), title: "container missing devices".to_owned(), score: 90, @@ -704,6 +707,7 @@ mod tests { let rendered = render_report( "gpu-box", &report(vec![Diagnosis { + symptom_matched: false, id: "wheels".to_owned(), title: "wrong wheels".to_owned(), score: 90, @@ -740,4 +744,46 @@ mod tests { let rendered = render_report("gpu-box", &report(vec![]), 5); assert!(rendered.starts_with("Health of gpu-box"), "{rendered}"); } + + /// The note under a cause the symptom did not contribute to describes the + /// examined host, which here is the remote one, and it marks only that + /// cause. + #[test] + fn the_symptom_note_is_true_of_a_remote_host_and_marks_only_its_cause() { + use rocm_core::diagnose::{Diagnosis, SYMPTOM_DID_NOT_CONTRIBUTE_NOTE}; + let row = |id: &str, score: i32, symptom_matched: bool| Diagnosis { + id: id.to_owned(), + title: id.to_owned(), + score, + symptom_matched, + ..Diagnosis::default() + }; + let rendered = render_report( + "gpu-box", + &report(vec![ + row("fix-host-only", 60, false), + row("fix-from-the-paste", 55, true), + ]), + 5, + ); + let block = |id: &str| -> Vec<&str> { + let marker = format!("({id})"); + rendered + .lines() + .skip_while(|l| !l.contains(&marker)) + .take_while(|l| !l.trim().is_empty()) + .collect() + }; + let marked = |id: &str| { + block(id) + .iter() + .any(|l| l.trim() == SYMPTOM_DID_NOT_CONTRIBUTE_NOTE) + }; + assert!(marked("fix-host-only"), "{rendered}"); + assert!(!marked("fix-from-the-paste"), "{rendered}"); + assert!( + !SYMPTOM_DID_NOT_CONTRIBUTE_NOTE.contains("this machine"), + "the note is printed about a remote host too, where `this machine` is the wrong one" + ); + } } diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 549753818a..18e594f44b 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -5,9 +5,13 @@ //! Registering a runtime that was not installed by `rocm install sdk`: //! importing a manifest written elsewhere, adopting an existing Python //! environment by probing it, and recovering the registration of the runtime -//! `rocm setup` created. +//! `rocm setup` created. It also holds the one rule for picking "the one ready +//! runtime" when nothing else says which to use +//! ([`select_single_ready_runtime`]), which sits here because it recovers the +//! setup registration first; `rocm comfyui install` and the engine paths both +//! ask it and each words the zero and several cases itself. //! -//! Relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` +//! The rest was relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` //! calls into this module for `rocm runtimes import` and `rocm runtimes adopt`, //! and re-exports `newest_installed_runtime_manifest`, //! `recover_setup_runtime_registration` and `setup_runtime_paths` for the @@ -19,11 +23,15 @@ use std::fs; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, RocmCliConfig, managed_pip_cache_dir, runtime_python_executable_in_env}; +use rocm_core::{ + AppPaths, RocmCliConfig, managed_pip_cache_dir, runtime_python_env_root_for_executable, + runtime_python_executable_in_env, +}; use crate::runtime_registry::{ - paths_equivalent, runtime_manifest_path, runtime_registry_dir, - validate_runtime_manifest_for_activation, + ManifestOverwrite, paths_equivalent, runtime_manifest_path, runtime_registry_dir, + sort_manifests_newest_install_first, validate_runtime_manifest_for_activation, + write_runtime_manifest, }; use crate::therock; @@ -58,7 +66,11 @@ pub(crate) fn import_runtime_manifest( validate_runtime_manifest_for_activation(&manifest) .with_context(|| format!("imported runtime `{}` is not usable", manifest.runtime_key))?; - write_runtime_registry_manifest(paths, &manifest, replace)?; + write_runtime_manifest( + paths, + &manifest, + ManifestOverwrite::for_replace_flag(replace), + )?; Ok(manifest) } @@ -230,7 +242,9 @@ fn resolve_adopt_python_input(input: &Path) -> Result<(PathBuf, Option) return Ok((python, Some(env_root))); } if absolute.is_file() { - let inferred_root = infer_python_env_root(&absolute); + // `None` for an interpreter outside a `bin`/`Scripts` folder: the + // request then asks for `--root` rather than adopting a guessed folder. + let inferred_root = runtime_python_env_root_for_executable(&absolute); return Ok((absolute, inferred_root)); } bail!( @@ -239,15 +253,6 @@ fn resolve_adopt_python_input(input: &Path) -> Result<(PathBuf, Option) ); } -fn infer_python_env_root(python_executable: &Path) -> Option { - let bin_dir = python_executable.parent()?; - let bin_name = bin_dir.file_name()?.to_string_lossy(); - if bin_name.eq_ignore_ascii_case("Scripts") || bin_name == "bin" { - return bin_dir.parent().map(Path::to_path_buf); - } - bin_dir.parent().map(Path::to_path_buf) -} - pub(crate) fn adopt_runtime_from_probe( paths: &AppPaths, request: AdoptRuntimeRequest, @@ -314,7 +319,11 @@ pub(crate) fn adopt_runtime_from_probe( }; validate_runtime_manifest_for_activation(&manifest) .with_context(|| format!("adopted runtime `{}` is not usable", manifest.runtime_key))?; - write_runtime_registry_manifest(paths, &manifest, request.replace)?; + write_runtime_manifest( + paths, + &manifest, + ManifestOverwrite::for_replace_flag(request.replace), + )?; Ok(manifest) } @@ -350,37 +359,17 @@ fn parse_therock_runtime_id(runtime_id: &str) -> Result<(String, String)> { Ok((channel.to_owned(), family.to_owned())) } -fn write_runtime_registry_manifest( - paths: &AppPaths, - manifest: &therock::InstalledRuntimeManifest, - replace: bool, -) -> Result<()> { - let manifest = manifest.clone().normalize_storage_paths(); - let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); - if registry_path.exists() && !replace { - bail!( - "runtime registry entry already exists: {}; pass --replace to overwrite it", - registry_path.display() - ); - } - fs::create_dir_all( - registry_path - .parent() - .context("runtime registry path has no parent directory")?, - )?; - fs::write( - ®istry_path, - serde_json::to_vec_pretty(&manifest) - .context("failed to serialize runtime registry manifest")?, - ) - .with_context(|| format!("failed to write {}", registry_path.display()))?; - Ok(()) +/// The runtime `rocm setup` created, as the manifest in its install tree +/// describes it, and whether the runtime registry already holds its entry. +struct SetupRuntime { + manifest: therock::InstalledRuntimeManifest, + registered: bool, } -pub(crate) fn recover_setup_runtime_registration( - paths: &AppPaths, - config: &RocmCliConfig, -) -> Result> { +/// Find the runtime `rocm setup` created and check that it is usable, without +/// changing anything. `None` when setup named no folder or the folder holds no +/// manifest; an error when it does and the runtime cannot be used. +fn find_setup_runtime(paths: &AppPaths, config: &RocmCliConfig) -> Result> { let Some(setup_root) = config .setup .therock_venv @@ -424,16 +413,190 @@ pub(crate) fn recover_setup_runtime_registration( ) })?; - if !runtime_manifest_path(paths, &manifest.runtime_key).is_file() { - write_runtime_registry_manifest(paths, &manifest, false).with_context(|| { - format!( - "failed to restore setup runtime `{}` into {}", - manifest.runtime_key, - runtime_registry_dir(paths).display() - ) - })?; + let registered = runtime_manifest_path(paths, &manifest.runtime_key).is_file(); + Ok(Some(SetupRuntime { + manifest, + registered, + })) +} + +/// Write the registry entry of a setup runtime that the registry has lost, and +/// nothing when it is there. The one place that write and its error message live. +fn restore_setup_registration(paths: &AppPaths, setup: &SetupRuntime) -> Result<()> { + if !setup.registered { + write_runtime_manifest(paths, &setup.manifest, ManifestOverwrite::Refuse).with_context( + || { + format!( + "failed to restore setup runtime `{}` into {}", + setup.manifest.runtime_key, + runtime_registry_dir(paths).display() + ) + }, + )?; + } + Ok(()) +} + +pub(crate) fn recover_setup_runtime_registration( + paths: &AppPaths, + config: &RocmCliConfig, +) -> Result> { + let Some(setup) = find_setup_runtime(paths, config)? else { + return Ok(None); + }; + restore_setup_registration(paths, &setup)?; + Ok(Some(setup.manifest.runtime_key)) +} + +/// Whether finding the runtime `rocm setup` created may write its lost registry +/// entry back while a runtime is being picked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SetupRecovery { + /// Write the entry back, as `rocm serve` and `rocm engines install` do. + Persist, + /// Count the setup runtime as if its entry were there, and write nothing: + /// a `--dry-run` preview must name the runtime the real run will use + /// without changing any file. + PreviewOnly, +} + +impl SetupRecovery { + /// The recovery a command with a `--dry-run` flag uses. + pub(crate) const fn for_dry_run(dry_run: bool) -> Self { + if dry_run { + Self::PreviewOnly + } else { + Self::Persist + } + } +} + +/// What picking "the one ready runtime" found. The caller decides how to report +/// the two cases that are not a pick, because the advice differs by command +/// (`rocm comfyui install` and `rocm serve` name different remedies). +#[derive(Debug)] +pub(crate) enum SingleReadyRuntime { + /// Exactly one registered runtime is ready. + One(Box), + /// No registered runtime is ready. `registered` is everything the registry + /// holds, unusable entries included and possibly none, so a caller can say + /// why each one does not count. + Zero { + registered: Vec, + }, + /// More than one is ready, so there is nothing to pick without a choice. + Many { + ready: Vec, + }, +} + +/// Pick the one ready runtime when nothing else says which to use: no +/// `--runtime-id`, no active runtime, no configured default. This is the only +/// place that rule lives; `rocm comfyui install` and the engine paths +/// (`rocm serve`, `rocm engines install`) both call it. +/// +/// Ready means `validate_runtime_manifest_for_activation` accepts the runtime, +/// the same check `rocm runtimes activate` applies. Setup is recovered first +/// (`recover_setup_runtime_registration`), so the runtime `rocm setup` created +/// counts even if its registry entry was lost. With [`SetupRecovery::Persist`] +/// that writes the entry back; with [`SetupRecovery::PreviewOnly`] nothing is +/// written, and the setup runtime is counted from its own manifest as the +/// registry would hold it once restored, so the answer is the one `Persist` +/// gives. +/// +/// A setup runtime that cannot be used (its manifest is unreadable or points +/// elsewhere, or it fails the readiness check) is not a ready runtime and +/// cannot change the count, so it does not stop a registered ready runtime from +/// being picked, or from being refused as ambiguous with another. That is the +/// answer a choice that names a runtime gets from the registry too. When the +/// registry holds no ready runtime the setup runtime was the only candidate and +/// its error is the answer. A setup runtime that can be used but whose registry +/// entry cannot be written is different: it is one of the runtimes being +/// counted, so the failed write is reported as it is. +pub(crate) fn select_single_ready_runtime( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, +) -> Result { + let setup = match find_setup_runtime(paths, config) { + Ok(setup) => setup, + Err(unusable) => { + return match classify_single_ready_runtime(therock::load_runtime_manifests(paths)?) { + SingleReadyRuntime::Zero { .. } => Err(unusable), + answer => Ok(answer), + }; + } + }; + Ok(classify_single_ready_runtime(registered_runtimes_with( + paths, setup, recovery, + )?)) +} + +/// Every runtime the registry holds, with the one `rocm setup` created counted +/// even if its registry entry was lost. With [`SetupRecovery::Persist`] the +/// entry is written back first (`recover_setup_runtime_registration`); with +/// [`SetupRecovery::PreviewOnly`] nothing is written, and the setup runtime is +/// added from its own manifest as the registry would hold it once restored, so +/// both return the same list. An error when the setup runtime exists but +/// cannot be used. +pub(crate) fn registered_runtimes_with_setup( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, +) -> Result> { + registered_runtimes_with(paths, find_setup_runtime(paths, config)?, recovery) +} + +/// [`registered_runtimes_with_setup`] for a setup runtime already found. +fn registered_runtimes_with( + paths: &AppPaths, + setup: Option, + recovery: SetupRecovery, +) -> Result> { + match recovery { + SetupRecovery::Persist => { + if let Some(setup) = &setup { + restore_setup_registration(paths, setup)?; + } + therock::load_runtime_manifests(paths) + } + SetupRecovery::PreviewOnly => { + let mut registered = therock::load_runtime_manifests(paths)?; + if let Some(setup) = setup + && !setup.registered + { + // What `load_runtime_manifests` would return for the entry that + // `Persist` would have written. + registered.push( + setup + .manifest + .normalize_storage_paths() + .normalize_host_paths(), + ); + sort_manifests_newest_install_first(&mut registered); + } + Ok(registered) + } + } +} + +/// The pure half of [`select_single_ready_runtime`]: which of `registered` are +/// ready, and whether that is none, one or several. +pub(crate) fn classify_single_ready_runtime( + registered: Vec, +) -> SingleReadyRuntime { + let ready: Vec<_> = registered + .iter() + .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) + .cloned() + .collect(); + if ready.is_empty() { + return SingleReadyRuntime::Zero { registered }; + } + match <[therock::InstalledRuntimeManifest; 1]>::try_from(ready) { + Ok([manifest]) => SingleReadyRuntime::One(Box::new(manifest)), + Err(ready) => SingleReadyRuntime::Many { ready }, } - Ok(Some(manifest.runtime_key)) } /// The folder setup was configured for and its pip cache — the pair both @@ -819,4 +982,387 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// `rocm runtimes adopt --python ` without `--root` takes the + /// environment folder from the shared rule (see + /// `rocm_core::runtime_python_env_root_for_executable`): only an + /// interpreter in a `bin` or `Scripts` folder, in any case, names one. An + /// interpreter in an unrelated folder yields no root, and the request is + /// then refused with `--root` guidance rather than adopting a guessed + /// folder. + #[test] + fn adopt_python_file_infers_a_root_only_from_a_bin_or_scripts_folder() -> Result<()> { + let (root, _) = test_paths("runtime-adopt-infer-root"); + for (folder, expected_root) in [ + ("bin", true), + ("Scripts", true), + ("SCRIPTS", true), + ("tools", false), + ] { + let env_root = root.join(format!("env-{folder}")); + let python = env_root.join(folder).join("python"); + fs::create_dir_all(python.parent().unwrap())?; + fs::write(&python, "python")?; + + let (resolved, inferred) = resolve_adopt_python_input(&python)?; + + assert_eq!(resolved, python, "{folder}"); + assert_eq!( + inferred.as_deref(), + expected_root.then_some(env_root.as_path()), + "{folder}" + ); + } + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + fn pip_runtime(paths: &AppPaths, key: &str) -> Result { + write_test_pip_runtime(paths, key, "therock-release:gfx120X-all", "7.14.0", 20) + } + + /// Readiness is the activation check and nothing looser: a runtime whose + /// interpreter is gone is registered but does not count, and neither is it + /// "the only one". + #[test] + fn single_ready_runtime_counts_only_what_activation_accepts() -> Result<()> { + let (root, paths) = test_paths("single-ready-classify"); + let ready = pip_runtime(&paths, "runtime-a")?; + let broken = pip_runtime(&paths, "runtime-b")?; + fs::remove_file( + broken + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + assert!(validate_runtime_manifest_for_activation(&ready).is_ok()); + assert!(validate_runtime_manifest_for_activation(&broken).is_err()); + + match classify_single_ready_runtime(vec![ready.clone(), broken.clone()]) { + SingleReadyRuntime::One(manifest) => assert_eq!(manifest.runtime_key, "runtime-a"), + other => panic!("expected the one ready runtime, got {other:?}"), + } + match classify_single_ready_runtime(vec![broken.clone()]) { + SingleReadyRuntime::Zero { registered } => { + assert_eq!(registered.len(), 1, "unusable entries stay listed"); + } + other => panic!("expected no ready runtime, got {other:?}"), + } + match classify_single_ready_runtime(Vec::new()) { + SingleReadyRuntime::Zero { registered } => assert!(registered.is_empty()), + other => panic!("expected no ready runtime, got {other:?}"), + } + let second = pip_runtime(&paths, "runtime-c")?; + match classify_single_ready_runtime(vec![ready, broken, second]) { + SingleReadyRuntime::Many { ready } => { + let keys: Vec<_> = ready.iter().map(|m| m.runtime_key.as_str()).collect(); + assert_eq!(keys, ["runtime-a", "runtime-c"], "only ready ones listed"); + } + other => panic!("expected several ready runtimes, got {other:?}"), + } + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// What no inferred root costs the user: the request is refused and asks + /// for `--root`, and an explicit `--root` still adopts the same + /// interpreter. + #[test] + fn adopt_request_without_an_inferred_root_asks_for_root() { + let python = PathBuf::from("python"); + let request = |install_root: Option| { + infer_adopt_runtime_request( + python.clone(), + install_root, + None, + None, + None, + false, + &therock::RocmSdkPythonProbe { + rocm_sdk_version: Some("7.13.0".to_owned()), + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }, + ) + }; + + let error = request(None).unwrap_err().to_string(); + assert!(error.contains("pass --root"), "{error}"); + + let explicit = request(Some(PathBuf::from("/opt/rocm-env"))).unwrap(); + assert_eq!(explicit.install_root, PathBuf::from("/opt/rocm-env")); + } + + /// The selection recovers the setup runtime first, so a runtime the registry + /// lost is the one found, and finding it is what writes its entry back. + #[test] + fn select_single_ready_runtime_recovers_setup_before_counting() -> Result<()> { + let (root, paths) = test_paths("single-ready-recovers-setup"); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + let config = RocmCliConfig::load(&paths)?; + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + + match select_single_ready_runtime(&paths, &config, SetupRecovery::Persist)? { + SingleReadyRuntime::One(manifest) => assert_eq!(manifest.runtime_key, "runtime-s"), + other => panic!("expected the recovered setup runtime, got {other:?}"), + } + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", lost.runtime_key)] + ); + + // Without setup naming a folder there is nothing to recover and the + // registry is not touched. + let (other_root, other_paths) = test_paths("single-ready-no-setup"); + match select_single_ready_runtime( + &other_paths, + &RocmCliConfig::default(), + SetupRecovery::Persist, + )? { + SingleReadyRuntime::Zero { registered } => assert!(registered.is_empty()), + other => panic!("expected no ready runtime, got {other:?}"), + } + assert!(crate::test_support::runtime_registry_entry_names(&other_paths).is_empty()); + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(other_root); + Ok(()) + } + + /// The keys `select_single_ready_runtime` found ready, in the order it lists + /// them, or the message of its refusal to recover. + fn single_ready_outcome( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, + ) -> Result, String> { + match select_single_ready_runtime(paths, config, recovery) { + Ok(SingleReadyRuntime::One(manifest)) => Ok(vec![manifest.runtime_key]), + Ok(SingleReadyRuntime::Many { ready }) => { + Ok(ready.into_iter().map(|m| m.runtime_key).collect()) + } + Ok(SingleReadyRuntime::Zero { registered }) => { + Err(format!("none ready of {}", registered.len())) + } + Err(error) => Err(format!("{error:#}")), + } + } + + /// A preview counts the runtime setup created from its own manifest and + /// writes nothing; the real run then finds the same runtime and writes the + /// entry. Both orders of install time are covered: the ready runtimes are + /// listed newest first, and a preview must list them the way the registry + /// will once the entry is back. + #[test] + fn preview_counts_the_setup_runtime_without_writing_and_agrees_with_persist() -> Result<()> { + for (label, other_installed_at) in [("other-newer", 30), ("other-older", 10)] { + let (root, paths) = test_paths(&format!("single-ready-preview-{label}")); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-setup", + )?; + write_test_pip_runtime( + &paths, + "runtime-other", + "therock-release:gfx120X-all", + "7.14.0", + other_installed_at, + )?; + let config = RocmCliConfig::load(&paths)?; + let registry_before = crate::test_support::runtime_registry_entry_names(&paths); + assert_eq!(registry_before, ["runtime-other.json"], "{label}"); + + let preview = single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly) + .expect("the setup runtime and the other one are both ready"); + + assert_eq!(preview.len(), 2, "{label}: {preview:?}"); + assert!(preview.contains(&lost.runtime_key), "{label}: {preview:?}"); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + registry_before, + "{label}: a preview must not write the lost entry" + ); + let persisted = single_ready_outcome(&paths, &config, SetupRecovery::Persist) + .expect("recovery finds the same two"); + assert_eq!(preview, persisted, "{label}: preview and real run disagree"); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + [ + "runtime-other.json".to_owned(), + "runtime-setup.json".to_owned() + ], + "{label}: the real run writes the entry back" + ); + let _ = fs::remove_dir_all(root); + } + Ok(()) + } + + /// The setup runtime alone is "the one ready runtime" in a preview, and a + /// registry that already holds it is left as it is. + #[test] + fn preview_picks_a_lost_setup_runtime_and_ignores_a_registered_one() -> Result<()> { + let (root, paths) = test_paths("single-ready-preview-one"); + crate::test_support::write_test_setup_runtime_missing_from_registry(&paths, "runtime-s")?; + let config = RocmCliConfig::load(&paths)?; + + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly), + Ok(vec!["runtime-s".to_owned()]) + ); + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + + // Once the entry exists a preview and a real run see one runtime, and + // neither adds a second copy of it. + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::Persist), + Ok(vec!["runtime-s".to_owned()]) + ); + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly), + Ok(vec!["runtime-s".to_owned()]) + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A setup runtime that is no longer usable is reported by a preview as the + /// real run reports it, and neither writes anything. + #[test] + fn preview_reports_an_unusable_setup_runtime_as_persist_does() -> Result<()> { + let (root, paths) = test_paths("single-ready-preview-unusable"); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + fs::remove_file( + lost.python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + let config = RocmCliConfig::load(&paths)?; + + let preview = single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly) + .expect_err("an unusable setup runtime is an error"); + let persist = single_ready_outcome(&paths, &config, SetupRecovery::Persist) + .expect_err("an unusable setup runtime is an error"); + + assert!(preview.contains("is not usable"), "{preview}"); + assert_eq!(preview, persist); + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Recovering the setup runtime restores the registry entry and writes no + /// install-root file, whatever shape that file has: the manifest it restores + /// is the one read from that very file, so it is always what would be + /// written. A hand-edited file (compact, an extra key, no final newline) is + /// therefore left byte for byte as it was, and recovery cannot fail on a + /// setup folder that is read-only. Asserted on the bytes, so it holds when + /// the process can write anyway. The restored entry is what the file says. + #[test] + fn setup_recovery_restores_the_entry_and_leaves_a_hand_edited_install_root_file_alone() + -> Result<()> { + let (root, paths) = test_paths("setup-recovery-hand-edited-file"); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + let file = lost.install_root.join(".rocm-cli-runtime.json"); + let mut document = serde_json::to_value(&lost)?; + document["a_key_a_future_release_adds"] = serde_json::json!(true); + let hand_edited = serde_json::to_vec(&document)?; + fs::write(&file, &hand_edited)?; + let config = RocmCliConfig::load(&paths)?; + + let recovered = recover_setup_runtime_registration(&paths, &config)?; + + assert_eq!(recovered.as_deref(), Some("runtime-s")); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec!["runtime-s.json".to_owned()] + ); + assert_eq!( + fs::read(&file)?, + hand_edited, + "recovery must not rewrite the install-root file it restored from" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A setup runtime that is no longer usable is not a ready runtime, so it + /// does not change what the registry alone answers: one registered ready + /// runtime is picked, two are refused as ambiguous with exactly those two + /// named, and the registry is not written. Only when the registry holds no + /// ready runtime is the setup runtime's own error the answer. Both recoveries + /// agree, and the answer and the registry are asserted together. + #[test] + fn an_unusable_setup_runtime_does_not_hide_the_registrys_ready_runtimes() -> Result<()> { + for recovery in [SetupRecovery::Persist, SetupRecovery::PreviewOnly] { + for (registered, expected) in [ + (vec![], Err("is not usable")), + (vec!["runtime-a"], Ok(vec!["runtime-a"])), + ( + vec!["runtime-a", "runtime-b"], + Ok(vec!["runtime-b", "runtime-a"]), + ), + ] { + let (root, paths) = test_paths("single-ready-unusable-setup"); + for (index, key) in registered.iter().enumerate() { + write_test_pip_runtime( + &paths, + key, + "therock-release:gfx120X-all", + "7.14.0", + 10 + index as u128, + )?; + } + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + fs::remove_file( + lost.python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + let config = RocmCliConfig::load(&paths)?; + let entries = crate::test_support::runtime_registry_entry_names(&paths); + assert_eq!(entries.len(), registered.len()); + + let outcome = single_ready_outcome(&paths, &config, recovery); + + let label = format!("{recovery:?}, {} registered", registered.len()); + match expected { + Ok(keys) => assert_eq!( + outcome, + Ok(keys.into_iter().map(str::to_owned).collect()), + "{label}" + ), + Err(text) => { + let message = outcome.expect_err("nothing else is ready"); + assert!(message.contains(text), "{label}: {message}"); + } + } + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + entries, + "{label}: the unusable setup runtime is not registered" + ); + let _ = fs::remove_dir_all(root); + } + } + Ok(()) + } + + #[test] + fn a_dry_run_previews_and_a_real_run_persists() { + assert_eq!(SetupRecovery::for_dry_run(true), SetupRecovery::PreviewOnly); + assert_eq!(SetupRecovery::for_dry_run(false), SetupRecovery::Persist); + } } diff --git a/apps/rocm/src/runtime_install_finalize.rs b/apps/rocm/src/runtime_install_finalize.rs index 8e69c8124d..04c99b568f 100644 --- a/apps/rocm/src/runtime_install_finalize.rs +++ b/apps/rocm/src/runtime_install_finalize.rs @@ -47,7 +47,7 @@ mod torch_alignment; use torch_alignment::{ TorchAlignment, TorchRetention, classify_retained_torch, deliberately_diverged_package, - report_torch_alignment, report_torch_retention, retained_diverged_package, + diverged_packages, report_torch_alignment, report_torch_retention, retained_diverged_package, runtime_library_paths_for_key, runtime_python_for_key, sdk_torch_build_for_key, }; @@ -58,6 +58,11 @@ pub(crate) struct SdkInstallFinalization { family: String, pub(crate) previous_runtime_key: Option, services: runtime_services::RuntimeServiceReconciliation, + /// The Python the installed runtime was built on, for a ROCm 10+ runtime and + /// `None` for any other (see [`therock::recorded_python_version`]). Read from + /// the installed manifest rather than the `--python-version` text, so a ROCm 7 + /// install that was handed a redundant `3.12` is not mistaken for a moved one. + sdk_python_version: Option<&'static str>, } pub(crate) fn print_sdk_install_success(finalized: &SdkInstallFinalization) { @@ -117,6 +122,27 @@ impl SdkInstallConsents { } } +/// Why `engine` cannot be installed onto a ROCm 10+ SDK runtime built on +/// `sdk_python_version` (`rocm install sdk --python-version`), or `None` when it can +/// (including `None` for a runtime of any other layout). +/// +/// Only vLLM has a Python of its own: for ROCm 10 it is published for +/// [`therock::NEXT_MANAGED_PYTHON_VERSION`] alone, so installing it onto a runtime +/// the user moved off that Python would fail. The line names the flag that clears +/// the condition: repeating the install with that Python moves the runtime back. +fn engine_skipped_for_sdk_python(engine: &str, sdk_python_version: Option<&str>) -> Option { + let version = sdk_python_version?; + if engine != "vllm" || version == therock::NEXT_MANAGED_PYTHON_VERSION { + return None; + } + Some(format!( + "note: vLLM for ROCm 10 needs Python {}, and this runtime uses Python {version}; \ + re-run this `rocm install sdk` with --python-version {} to move it there and install vLLM", + therock::NEXT_MANAGED_PYTHON_VERSION, + therock::NEXT_MANAGED_PYTHON_VERSION + )) +} + pub(crate) fn maybe_auto_install_sdk_preferred_engine( paths: &AppPaths, finalized: &SdkInstallFinalization, @@ -133,6 +159,11 @@ pub(crate) fn maybe_auto_install_sdk_preferred_engine( ); println!(" engine: {engine}"); println!(" runtime_id: {}", finalized.runtime_key); + if let Some(note) = engine_skipped_for_sdk_python(engine, finalized.sdk_python_version) { + println!(" status: skipped"); + println!(" {note}"); + return Ok(()); + } ensure_openmpi_for_vllm(approved)?; @@ -164,7 +195,7 @@ pub(crate) fn maybe_auto_install_sdk_preferred_engine( &finalized.runtime_key, // The engine install failed, so no alignment ran and any // divergence found here is a real one. - None, + &[], ); return Err(error); } @@ -245,9 +276,9 @@ fn report_engine_dependency_check( engine: &str, python: Option<&Path>, runtime_key: &str, - realigned_package: Option<&str>, + diverged: &[&str], ) { - let outcome = engine_dependency_check(paths, engine, python, realigned_package); + let outcome = engine_dependency_check(paths, engine, python, diverged); print!("{}", render_engine_dependency_check(engine, &outcome)); let (level, message) = match &outcome { EngineDependencyCheck::Satisfied => ( @@ -764,7 +795,13 @@ pub(crate) fn settle_engine_install( (retained_diverged_package(&retention), probed) } }; - report_engine_dependency_check(paths, engine, Some(python), runtime_key, diverged); + report_engine_dependency_check( + paths, + engine, + Some(python), + runtime_key, + &diverged_packages(diverged, rocm_core::torch_alignment_disabled()), + ); let devices = report_runtime_device_check(paths, engine, runtime_key, devices); // Only a runtime that cannot serve raises the question the host answers, so @@ -792,7 +829,7 @@ fn engine_dependency_check( paths: &AppPaths, engine: &str, python: Option<&Path>, - realigned_package: Option<&str>, + diverged: &[&str], ) -> EngineDependencyCheck { let Some(python) = python else { return EngineDependencyCheck::NotVerified( @@ -806,7 +843,7 @@ fn engine_dependency_check( .iter() .map(|violation| violation.detail.clone()) .collect(); - classify_dependency_details(details, realigned_package) + classify_dependency_details(details, diverged) } Err(error) => EngineDependencyCheck::NotVerified(error.to_string()), } @@ -819,25 +856,25 @@ fn engine_dependency_check( /// and must not drag it along. Lumping the two together loses the distinction /// exactly where it matters most — the remedy for the genuine violation would /// reinstall the engine and undo the alignment. -fn classify_dependency_details( - details: Vec, - realigned_package: Option<&str>, -) -> EngineDependencyCheck { +fn classify_dependency_details(details: Vec, diverged: &[&str]) -> EngineDependencyCheck { if details.is_empty() { return EngineDependencyCheck::Satisfied; } - let Some(package) = realigned_package else { + if diverged.is_empty() { return EngineDependencyCheck::Violated { violations: details, expected: Vec::new(), }; - }; + } // A line whose subject cannot be parsed is not evidence of a divergence, so it // stays a violation: the conservative side keeps saying something is wrong. let (expected, violations): (Vec, Vec) = details.into_iter().partition(|detail| { - rocm_core::violation_subject(detail) - .is_some_and(|subject| subject.package.eq_ignore_ascii_case(package)) + rocm_core::violation_subject(detail).is_some_and(|subject| { + diverged + .iter() + .any(|package| subject.package.eq_ignore_ascii_case(package)) + }) }); if violations.is_empty() { EngineDependencyCheck::ExpectedDivergence(expected) @@ -901,7 +938,7 @@ fn render_engine_dependency_check(engine: &str, outcome: &EngineDependencyCheck) // the two this is, the block above has already said. let _ = writeln!( output, - " action: none; this torch is kept on purpose (see torch_alignment above)" + " action: none; this torch stack is kept on purpose (see torch_alignment above)" ); } } @@ -964,6 +1001,7 @@ pub(crate) fn finalize_successful_sdk_install( Ok(Some(SdkInstallFinalization { runtime_key: activation.runtime_key, + sdk_python_version: therock::recorded_python_version(&manifest), install_root: manifest.install_root, family: manifest.family, previous_runtime_key: activation.previous_runtime_key, @@ -1119,7 +1157,7 @@ pub(crate) mod tests { vec![ "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed".to_owned(), ], - Some("torch"), + &["torch"], ); assert!(matches!( @@ -1145,7 +1183,7 @@ pub(crate) mod tests { "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed".to_owned(), "The package `vllm` requires `numpy==1.26.4`, but `2.0.0` is installed".to_owned(), ], - Some("torch"), + &["torch"], ); let EngineDependencyCheck::Violated { @@ -1176,7 +1214,7 @@ pub(crate) mod tests { // the divergence was intended, so it must not be quietly excused. let outcome = classify_dependency_details( vec!["something uv said that this parser does not recognise".to_owned()], - Some("torch"), + &["torch"], ); assert!(matches!(outcome, EngineDependencyCheck::Violated { .. })); @@ -1317,6 +1355,7 @@ pub(crate) mod tests { family: "gfx94X-dcgpu".to_owned(), previous_runtime_key: None, services: runtime_services::RuntimeServiceReconciliation::default(), + sdk_python_version: None, } } @@ -1336,6 +1375,51 @@ pub(crate) mod tests { (result, finalized) } + /// vLLM is the one engine with a Python of its own: for ROCm 10 it exists for 3.14 + /// only, so an SDK the user moved off 3.14 cannot take it. The note names the + /// command that clears the condition, and every other row has no note. + #[test] + fn engine_is_skipped_only_for_vllm_on_an_sdk_moved_off_the_vllm_python() { + let note = engine_skipped_for_sdk_python("vllm", Some("3.12")) + .expect("vLLM cannot be installed onto a Python 3.12 ROCm 10 runtime"); + assert!( + note.contains("--python-version 3.14") && note.contains("Python 3.14"), + "{note}" + ); + assert_eq!(engine_skipped_for_sdk_python("vllm", Some("3.14")), None); + assert_eq!(engine_skipped_for_sdk_python("vllm", None), None); + assert_eq!( + engine_skipped_for_sdk_python("lemonade", Some("3.12")), + None + ); + } + + /// The note is a promise that the engine install does not run, so the entry + /// point is held to it: on a vLLM family and a runtime on 3.12 it returns + /// without an engine request, which would otherwise fail here (no engine + /// binary exists in this test). + /// + /// Native Windows never prefers vLLM (`preferred_serve_engine_for_host_gpu_summary`), + /// so the skip path is unreachable there and the precondition below cannot hold. + #[cfg_attr( + windows, + ignore = "vLLM is never the preferred engine on native Windows" + )] + #[test] + fn auto_install_skips_vllm_on_an_sdk_moved_off_its_python() { + let (_root, paths) = test_paths("auto-engine-skipped-for-python"); + let finalized = SdkInstallFinalization { + sdk_python_version: Some("3.12"), + ..sdk_install_finalization() + }; + assert_eq!( + preferred_engine_for_sdk_family(&finalized.family), + Some("vllm") + ); + maybe_auto_install_sdk_preferred_engine(&paths, &finalized, false) + .expect("the engine install is skipped, not attempted"); + } + #[test] fn render_sdk_install_success_includes_service_reconciliation_section() { // Verify that the reconciliation section is printed even when no services @@ -1547,7 +1631,7 @@ pub(crate) mod tests { // than fall through to `satisfied`. let (root, paths) = test_paths("engine-dependency-no-python"); - let outcome = engine_dependency_check(&paths, "vllm", None, None); + let outcome = engine_dependency_check(&paths, "vllm", None, &[]); let _ = fs::remove_dir_all(root); assert_eq!( diff --git a/apps/rocm/src/runtime_install_finalize/torch_alignment.rs b/apps/rocm/src/runtime_install_finalize/torch_alignment.rs index 16b7188d0d..d604a44e66 100644 --- a/apps/rocm/src/runtime_install_finalize/torch_alignment.rs +++ b/apps/rocm/src/runtime_install_finalize/torch_alignment.rs @@ -451,6 +451,26 @@ pub(super) const fn deliberately_diverged_package( } } +/// Every package the dependency check should expect to diverge, given the one the +/// install settled on (`diverged`, from [`deliberately_diverged_package`] or +/// [`retained_diverged_package`]). +/// +/// Opted out, the torch the runtime holds is the user's, and so is the rest of the +/// stack the SDK wrote with it: vLLM pins an exact build of each, so reading +/// torchvision, torchaudio or triton as violations would answer the opt-out with a +/// remedy the engine's repair no longer offers. With nothing settled, nothing is +/// expected. +pub(super) fn diverged_packages( + diverged: Option<&'static str>, + alignment_disabled: bool, +) -> Vec<&'static str> { + match diverged { + Some(_) if alignment_disabled => rocm_core::SDK_TORCH_STACK_PACKAGES.to_vec(), + Some(package) => vec![package], + None => Vec::new(), + } +} + /// The build identifier of the torch that belongs to this runtime's SDK. /// /// Taken from the manifest, never from whatever torch happens to be installed. @@ -1248,7 +1268,7 @@ mod tests { vec![ "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.9.0+cpu` is installed".to_owned(), ], - deliberately_diverged_package(&outcome), + &diverged_packages(deliberately_diverged_package(&outcome), false), ); assert!(matches!( dependencies, @@ -1260,4 +1280,63 @@ mod tests { "the remedy would undo the opt-out, got {rendered:?}" ); } + + /// What the CI GPU lanes print after the SDK is installed over an engine runtime + /// with the opt-out set: the SDK wrote torchvision, torchaudio and triton as well + /// as torch, and each one misses vLLM's pin. Judged one package at a time, the + /// three siblings were violations carrying a "repair" remedy the engine no longer + /// performs, and the install reported `dependency_check: violated`. + #[test] + fn an_opted_out_runtime_expects_the_whole_sdk_torch_stack_to_diverge() { + let details = vec![ + "The package `vllm` requires `triton==3.6.0`, but `3.8.0+git4cff872c.rocm7.14.1` is installed".to_owned(), + "The package `vllm` requires `torchvision==0.24.1+d801a34`, but `0.26.0+rocm7.14.1` is installed".to_owned(), + "The package `vllm` requires `torchaudio==2.9.0+eaa9e4e`, but `2.11.0+rocm7.14.1` is installed".to_owned(), + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.14.1` is installed".to_owned(), + ]; + let kept = TorchRetention::SdkBuild { + version: "2.11.0+rocm7.14.1".to_owned(), + }; + + let opted_out = classify_dependency_details( + details.clone(), + &diverged_packages(retained_diverged_package(&kept), true), + ); + assert!( + matches!(&opted_out, EngineDependencyCheck::ExpectedDivergence(expected) if expected.len() == 4), + "opted out, all four are the user's: {opted_out:?}" + ); + + // Not opted out, only torch is ever an intended divergence: the siblings + // are real violations, because rocm-cli realigns torch and nothing else. + let aligned = classify_dependency_details( + details, + &diverged_packages(retained_diverged_package(&kept), false), + ); + assert!( + matches!(&aligned, EngineDependencyCheck::Violated { violations, expected } + if violations.len() == 3 && expected.len() == 1), + "aligned, the siblings stay violations: {aligned:?}" + ); + } + + /// The opt-out spares the torch stack and nothing else, and a runtime with no + /// settled divergence expects none. + #[test] + fn the_opt_out_widens_the_divergence_to_the_torch_stack_only() { + assert!(diverged_packages(None, true).is_empty()); + assert_eq!(diverged_packages(Some("torch"), false), vec!["torch"]); + let widened = diverged_packages(Some("torch"), true); + assert_eq!(widened, rocm_core::SDK_TORCH_STACK_PACKAGES.to_vec()); + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `numpy==1.26.4`, but `2.0.0` is installed".to_owned(), + ], + &widened, + ); + assert!( + matches!(outcome, EngineDependencyCheck::Violated { .. }), + "an unrelated pin is still a violation under the opt-out" + ); + } } diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 9f9fa17ce7..41eabb776e 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -121,7 +121,16 @@ impl InstalledRuntimeManifest { wheel_composition_includes_devel(self.wheel_composition.as_ref()).unwrap_or(self.devel) } - fn normalize_host_paths(mut self) -> Self { + /// Whether ROCm CLI owns the folder this runtime lives in: it installed it, + /// rather than importing or adopting somebody else's. Ownership is what + /// decides whether the install root carries a `.rocm-cli-runtime.json` (see + /// [`write_runtime_manifest`]) and whether uninstall and prune may delete + /// the folder, so those cannot disagree about which runtimes are whose. + pub(crate) const fn owns_install_root(&self) -> bool { + !self.read_only && self.imported_from.is_none() + } + + pub(crate) fn normalize_host_paths(mut self) -> Self { self.install_root = normalize_manifest_path(self.install_root); self.python_launcher = self .python_launcher @@ -336,33 +345,112 @@ pub(crate) fn runtime_manifest_path(paths: &AppPaths, runtime_key: &str) -> Path runtime_registry_dir(paths).join(format!("{runtime_key}.json")) } -pub(crate) fn save_runtime_manifest( +/// What [`write_runtime_manifest`] does when the registry already holds an entry +/// under the manifest's key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ManifestOverwrite { + /// Write over it. An install of a runtime ROCm CLI already installed is the + /// same runtime again. + Replace, + /// Refuse, and name `--replace`. Importing or adopting a runtime must not + /// displace a registration the user made earlier unless they ask to. + Refuse, +} + +impl ManifestOverwrite { + /// The policy for a command with a `--replace` flag. + pub(crate) const fn for_replace_flag(replace: bool) -> Self { + if replace { Self::Replace } else { Self::Refuse } + } +} + +/// Write the manifest files of a runtime: the registry entry, and, for a +/// runtime ROCm CLI owns ([`InstalledRuntimeManifest::owns_install_root`]), the +/// `.rocm-cli-runtime.json` in its install root. +/// +/// Every way a runtime gets registered goes through this one routine: an SDK +/// install, `rocm runtimes import`, `rocm runtimes adopt`, and the recovery of +/// the runtime `rocm setup` created. `overwrite` is the only thing a caller +/// chooses. +/// +/// The file in the install root is ROCm CLI's proof that it owns the folder: +/// uninstall and prune delete a folder only when it carries one that matches +/// the registry, and activation requires one of a runtime that is not +/// read-only. An imported or adopted runtime is somebody else's folder, which +/// uninstall and prune never delete and activation accepts without one, so it +/// gets none: a root the user may not be able to write to must not make +/// registering it fail, and an import's source manifest may be that very file +/// in another install. +/// +/// The registry entry is written last, after the install-root file. A failed +/// install-root write therefore leaves the registry exactly as it was, and a +/// failed registry write leaves no new entry, so an error that says a runtime +/// could not be registered or restored never describes one whose entry exists. +/// The install-root file may be left behind by a failed registry write; it holds +/// what a retry writes. +/// +/// The file is not rewritten when it already holds what would be written, so +/// recovering a runtime from its own manifest leaves it alone. "Holds" is judged +/// on the decoded manifest, not the bytes ([`install_root_manifest_is_current`]): +/// a file an older binary wrote omits keys that now read back as defaults and +/// are written out explicitly, and a root the user cannot write to must not turn +/// that into a failure. +/// +/// Neither file is written atomically; that is a separate piece of work shared +/// with the other writers in the workspace. +pub(crate) fn write_runtime_manifest( paths: &AppPaths, manifest: &InstalledRuntimeManifest, + overwrite: ManifestOverwrite, ) -> Result<()> { let manifest = manifest.clone().normalize_storage_paths(); let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); + if overwrite == ManifestOverwrite::Refuse && registry_path.exists() { + bail!( + "runtime registry entry already exists: {}; pass --replace to overwrite it", + registry_path.display() + ); + } fs::create_dir_all( registry_path .parent() .context("runtime manifest registry path has no parent directory")?, )?; - fs::write( - ®istry_path, - serde_json::to_vec_pretty(&manifest).context("failed to serialize runtime manifest")?, - ) - .with_context(|| format!("failed to write {}", registry_path.display()))?; - - let local_manifest_path = manifest.install_root.join(".rocm-cli-runtime.json"); - fs::write( - &local_manifest_path, - serde_json::to_vec_pretty(&manifest) - .context("failed to serialize local runtime manifest")?, - ) - .with_context(|| format!("failed to write {}", local_manifest_path.display()))?; + let encoded = + serde_json::to_vec_pretty(&manifest).context("failed to serialize runtime manifest")?; + + if manifest.owns_install_root() { + let local_manifest_path = manifest.install_root.join(".rocm-cli-runtime.json"); + let already_current = fs::read(&local_manifest_path) + .is_ok_and(|existing| install_root_manifest_is_current(&existing, &encoded)); + if !already_current { + fs::write(&local_manifest_path, &encoded) + .with_context(|| format!("failed to write {}", local_manifest_path.display()))?; + } + } + // Last, because the registry entry is what registering a runtime means; see + // the doc comment. + fs::write(®istry_path, &encoded) + .with_context(|| format!("failed to write {}", registry_path.display()))?; Ok(()) } +/// Whether an install root manifest already holds the manifest that `encoded` +/// is the storage form of. Equal bytes qualify, and so does a file that decodes +/// to the same manifest once normalised for storage: a manifest written before +/// a field existed has no key for it, reads back as the field's default, and +/// serializes with an explicit value, so a byte comparison reads it as changed. +/// A file that does not decode is not current. +fn install_root_manifest_is_current(existing: &[u8], encoded: &[u8]) -> bool { + if existing == encoded { + return true; + } + serde_json::from_slice::(existing) + .ok() + .and_then(|decoded| serde_json::to_vec_pretty(&decoded.normalize_storage_paths()).ok()) + .is_some_and(|normalised| normalised == encoded) +} + pub(crate) fn load_runtime_manifests(paths: &AppPaths) -> Result> { Ok(load_runtime_manifests_reporting_unparsed(paths)?.0) } @@ -628,7 +716,7 @@ fn validate_wheel_runtime_manifest(manifest: &InstalledRuntimeManifest) -> Resul } fn validate_tarball_runtime_manifest(manifest: &InstalledRuntimeManifest) -> Result<()> { - if runtime_install_root_has_payload(&manifest.install_root)? { + if rocm_core::install_root_has_payload(&manifest.install_root)? { Ok(()) } else { bail!( @@ -638,16 +726,6 @@ fn validate_tarball_runtime_manifest(manifest: &InstalledRuntimeManifest) -> Res } } -fn runtime_install_root_has_payload(path: &Path) -> Result { - for entry in fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))? { - let entry = entry?; - if !entry.file_name().to_string_lossy().starts_with('.') { - return Ok(true); - } - } - Ok(false) -} - pub(crate) fn paths_equivalent(left: &Path, right: &Path) -> bool { let left = normalize_path_for_compare(left); let right = normalize_path_for_compare(right); @@ -791,13 +869,13 @@ mod tests { /// that neither `rocm runtimes list` nor `rocm examine` surfaces `devel`, /// so the state is invisible. /// - /// LIMIT: this covers `save_runtime_manifest` only. It does NOT reach + /// LIMIT: this covers `write_runtime_manifest` only. It does NOT reach /// `install_sdk`, which is what actually threads `include_devel` into the /// manifest — that call needs `uv`, a live index, and a real `rocm_sdk` /// probe, so it has no unit coverage here. Hardcoding `devel: true` at that /// write site passes this test and the rest of the suite. #[test] - fn save_runtime_manifest_replaces_devel_rather_than_merging() -> Result<()> { + fn write_runtime_manifest_replaces_devel_rather_than_merging() -> Result<()> { let (root, paths) = test_paths("manifest-replaces-devel"); let install_root = root.join("install-root"); fs::create_dir_all(&install_root)?; @@ -813,14 +891,14 @@ mod tests { with_devel.devel, "test fixture should start with devel: true" ); - save_runtime_manifest(&paths, &with_devel)?; + write_runtime_manifest(&paths, &with_devel, ManifestOverwrite::Replace)?; let without_devel = InstalledRuntimeManifest { devel: false, installed_at_unix_ms: 2, ..with_devel.clone() }; - save_runtime_manifest(&paths, &without_devel)?; + write_runtime_manifest(&paths, &without_devel, ManifestOverwrite::Replace)?; let manifests = load_runtime_manifests(&paths)?; let reloaded = manifests @@ -836,6 +914,703 @@ mod tests { Ok(()) } + /// A manifest for `runtime_key` whose install root is a folder under `root`. + fn manifest_in( + root: &Path, + runtime_key: &str, + read_only: bool, + imported_from: Option, + ) -> Result { + let install_root = root.join(runtime_key); + fs::create_dir_all(&install_root)?; + Ok(InstalledRuntimeManifest { + install_root, + read_only, + imported_from, + ..test_runtime_manifest(runtime_key, "therock-release:gfx120X-all", 1) + }) + } + + /// The policy is the only thing a caller chooses, and it decides one thing: + /// whether an existing registry entry is replaced or left alone. The refusal + /// is asserted as message and state together, and with the entry still the + /// old bytes, because a refusal that wrote first would print the same text. + #[test] + fn write_runtime_manifest_refuses_an_existing_entry_only_under_the_refuse_policy() -> Result<()> + { + let (root, paths) = test_paths("manifest-overwrite-policy"); + let manifest = manifest_in(&root, "release-wheel-gfx120X-all-7.13.0", false, None)?; + let entry = runtime_manifest_path(&paths, &manifest.runtime_key); + + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Refuse)?; + assert!( + entry.is_file(), + "nothing registered yet, so nothing to refuse" + ); + + fs::write(&entry, "stale entry")?; + let error = write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Refuse) + .unwrap_err() + .to_string(); + assert!( + error.contains("already exists") && error.contains("--replace"), + "{error}" + ); + assert_eq!(fs::read_to_string(&entry)?, "stale entry"); + + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; + let replaced: InstalledRuntimeManifest = serde_json::from_slice(&fs::read(&entry)?)?; + assert_eq!(replaced.runtime_key, manifest.runtime_key); + + assert_eq!( + ManifestOverwrite::for_replace_flag(true), + ManifestOverwrite::Replace + ); + assert_eq!( + ManifestOverwrite::for_replace_flag(false), + ManifestOverwrite::Refuse + ); + let _ = fs::remove_dir_all(&root); + Ok(()) + } + + /// Which manifests get a `.rocm-cli-runtime.json` in the install root: + /// exactly the ones ROCm CLI owns. A read-only runtime and an imported one + /// (the two ways a runtime is not ROCm CLI's) get a registry entry and leave + /// their folder alone, whichever policy registered them. + #[test] + fn write_runtime_manifest_writes_the_install_root_manifest_only_for_an_owned_runtime() + -> Result<()> { + let (root, paths) = test_paths("manifest-install-root-file"); + let cases = [ + ("owned", false, None, true), + ("read-only", true, None, false), + ( + "imported", + false, + Some(PathBuf::from("elsewhere/manifest.json")), + false, + ), + ]; + for overwrite in [ManifestOverwrite::Replace, ManifestOverwrite::Refuse] { + for (label, read_only, imported_from, gets_the_file) in &cases { + let key = format!("{label}-{overwrite:?}"); + let manifest = manifest_in(&root, &key, *read_only, imported_from.clone())?; + + write_runtime_manifest(&paths, &manifest, overwrite)?; + + assert_eq!( + manifest.owns_install_root(), + *gets_the_file, + "{label} / {overwrite:?}" + ); + assert!( + runtime_manifest_path(&paths, &key).is_file(), + "{label} / {overwrite:?}: registry entry" + ); + assert_eq!( + manifest + .install_root + .join(".rocm-cli-runtime.json") + .is_file(), + *gets_the_file, + "{label} / {overwrite:?}: install root manifest" + ); + } + } + let _ = fs::remove_dir_all(&root); + Ok(()) + } + + /// The registry entry is what registers a runtime, so it is written last and + /// each failing write is asserted as the error and the state together. When + /// the install-root file cannot be written (a directory stands where the file + /// should be, which stops an administrator as well) the error names that file + /// and the registry is exactly as it was: no entry, so a caller's "could not + /// register" or "could not restore" is true, under either policy. When the + /// registry entry cannot be written (a directory stands where it should be) + /// the error names the entry and there is still no entry that reads as a + /// runtime. + #[test] + fn a_failed_manifest_write_never_leaves_a_registered_runtime_behind() -> Result<()> { + for overwrite in [ManifestOverwrite::Replace, ManifestOverwrite::Refuse] { + let (root, paths) = test_paths("manifest-write-fails"); + let manifest = manifest_in(&root, "release-wheel-gfx120X-all-7.13.0", false, None)?; + let entry = runtime_manifest_path(&paths, &manifest.runtime_key); + let local = manifest.install_root.join(".rocm-cli-runtime.json"); + + fs::create_dir_all(&local)?; + let error = format!( + "{:#}", + write_runtime_manifest(&paths, &manifest, overwrite) + .expect_err("the install-root manifest cannot be written") + ); + assert!( + error.contains("failed to write") && error.contains(".rocm-cli-runtime.json"), + "{overwrite:?}: {error}" + ); + assert!( + !entry.exists() && load_runtime_manifests(&paths)?.is_empty(), + "{overwrite:?}: a failed install-root write leaves no registry entry" + ); + + fs::remove_dir(&local)?; + fs::create_dir_all(&entry)?; + let error = format!( + "{:#}", + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace) + .expect_err("the registry entry cannot be written") + ); + assert!( + error.contains("failed to write") && error.contains(&entry.display().to_string()), + "{overwrite:?}: {error}" + ); + assert!( + !entry.is_file(), + "{overwrite:?}: a failed registry write leaves no entry" + ); + let _ = fs::remove_dir_all(&root); + } + Ok(()) + } + + /// Recovering a runtime from its own install-root manifest must not touch + /// that file when it already holds what would be written, so a setup folder + /// the user cannot write to does not turn a recovery into an error. The file + /// is made read-only: a write would fail where a skip succeeds. (Not + /// asserted when the file stays writable regardless, as for an + /// administrator.) + #[test] + #[allow(clippy::permissions_set_readonly_false)] + fn write_runtime_manifest_leaves_a_current_install_root_manifest_alone() -> Result<()> { + let (root, paths) = test_paths("manifest-current-install-root-file"); + let manifest = manifest_in(&root, "release-wheel-gfx120X-all-7.13.0", false, None)?; + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; + let local = manifest.install_root.join(".rocm-cli-runtime.json"); + let mut permissions = fs::metadata(&local)?.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&local, permissions.clone())?; + let still_writable = fs::OpenOptions::new().append(true).open(&local).is_ok(); + + let unchanged = write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace); + let changed = InstalledRuntimeManifest { + version: "7.14.0".to_owned(), + ..manifest + }; + let rewritten = write_runtime_manifest(&paths, &changed, ManifestOverwrite::Replace); + + permissions.set_readonly(false); + fs::set_permissions(&local, permissions)?; + unchanged.context("an install root manifest that is already current is not rewritten")?; + if !still_writable { + assert!( + rewritten.is_err(), + "a different manifest is written, which the read-only file refuses" + ); + } + let _ = fs::remove_dir_all(&root); + Ok(()) + } + + /// The state matrix for "is the install root manifest already current": + /// the same bytes, the same manifest in an older shape (keys that hold no + /// value omitted, so a byte comparison reads it as changed), a different + /// manifest in that shape, and a file that is not a manifest at all. Pure, so + /// it runs on every platform, unlike the read-only-file tests that show the + /// effect on the filesystem. + #[test] + fn install_root_manifest_is_current_compares_the_decoded_manifest() -> Result<()> { + let (root, _paths) = test_paths("manifest-current-decoded"); + let manifest = manifest_in(&root, "release-wheel-gfx120X-all-7.13.0", false, None)?; + let stored = manifest.clone().normalize_storage_paths(); + let encoded = serde_json::to_vec_pretty(&stored)?; + let legacy_shaped = |manifest: &InstalledRuntimeManifest| -> Result> { + let mut document = serde_json::to_value(manifest.clone().normalize_storage_paths())?; + let object = document.as_object_mut().context("a JSON object")?; + let empty: Vec = object + .iter() + .filter(|(_, value)| value.is_null()) + .map(|(key, _)| key.clone()) + .collect(); + assert!(!empty.is_empty(), "the fixture needs a null-valued key"); + for key in empty { + object.remove(&key); + } + Ok(serde_json::to_vec_pretty(&document)?) + }; + + assert!(install_root_manifest_is_current(&encoded, &encoded)); + let legacy = legacy_shaped(&manifest)?; + assert_ne!(legacy, encoded, "the legacy shape must differ in bytes"); + assert!( + install_root_manifest_is_current(&legacy, &encoded), + "a legacy-shaped file that decodes to the same manifest is current" + ); + let newer = InstalledRuntimeManifest { + version: "7.14.0".to_owned(), + ..manifest + }; + assert!( + !install_root_manifest_is_current(&legacy_shaped(&newer)?, &encoded), + "a legacy-shaped file that decodes to a different manifest is not current" + ); + assert!(!install_root_manifest_is_current(b"not json", &encoded)); + assert!(!install_root_manifest_is_current(b"", &encoded)); + let _ = fs::remove_dir_all(&root); + Ok(()) + } + + /// Calls that put bytes in a file, which is what a second writer of a registry + /// entry would use. Matched as text, so `tokio::fs::write(` counts too. The + /// file builders (`OpenOptions`, `File::options`) are not here: they write + /// only when a write option is chained on them, see [`open_options_write`]. + const WRITE_PRIMITIVES: [&str; 7] = [ + "fs::write(", + "File::create(", + "File::create_new(", + "fs::copy(", + "fs::rename(", + "fs::hard_link(", + "write_all(", + ]; + + /// The file builders, and the options that make one write. `OpenOptions::new()` + /// followed by `.read(true).open(..)` is a read-only probe and not a writer. + const FILE_BUILDERS: [&str; 2] = ["OpenOptions::new(", "File::options("]; + const WRITE_OPTIONS: [&str; 5] = ["write", "append", "create", "create_new", "truncate"]; + + /// The index just past the `)` that closes the `(` at `open`, skipping string + /// literals, or the end of `text` when it never closes. + fn skip_parens(text: &[u8], open: usize) -> usize { + let mut depth = 0usize; + let mut at = open; + let mut in_string = false; + while at < text.len() { + let byte = text[at]; + if in_string { + match byte { + b'\\' => at += 1, + b'"' => in_string = false, + _ => {} + } + } else { + match byte { + b'"' => in_string = true, + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return at + 1; + } + } + _ => {} + } + } + at += 1; + } + text.len() + } + + /// Whether `production` builds a file with a builder that can write. The + /// method chain after the builder is read across lines and whitespace: it is + /// a read-only probe only when the chain reaches `.open(` and none of + /// [`WRITE_OPTIONS`] is called on it. A builder that is stored (`let mut o = + /// OpenOptions::new();`) or whose chain does not reach `.open(` cannot be + /// judged and counts as a writer. + fn open_options_write(production: &str) -> bool { + let text = production.as_bytes(); + for builder in FILE_BUILDERS { + for (at, _) in production.match_indices(builder) { + let mut next = skip_parens(text, at + builder.len() - 1); + let mut methods = Vec::new(); + loop { + while text.get(next).is_some_and(u8::is_ascii_whitespace) { + next += 1; + } + if text.get(next) != Some(&b'.') { + break; + } + next += 1; + while text.get(next).is_some_and(u8::is_ascii_whitespace) { + next += 1; + } + let start = next; + while text + .get(next) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + { + next += 1; + } + methods.push(&production[start..next]); + while text.get(next).is_some_and(u8::is_ascii_whitespace) { + next += 1; + } + if text.get(next) == Some(&b'(') { + next = skip_parens(text, next); + } + } + let writes = methods.iter().any(|method| WRITE_OPTIONS.contains(method)); + if writes || !methods.contains(&"open") { + return true; + } + } + } + false + } + + /// Every way `production` writes a file, by name. + fn write_calls(production: &str) -> Vec<&'static str> { + let mut calls: Vec<&'static str> = WRITE_PRIMITIVES + .iter() + .copied() + .filter(|primitive| production.contains(primitive)) + .collect(); + if open_options_write(production) { + calls.push("OpenOptions with a write option"); + } + calls + } + + /// `source` as the compiler sees production code, for a text scan: everything + /// from the file's `#[cfg(test)]` test module on is cut, and comments are + /// removed, so neither a fixture nor a sentence in a comment can be mistaken + /// for a writer. String literals stay, because the path segments of the + /// registry directory are string literals. The scan is not a lexer: a raw + /// string that holds a double quote would confuse it. + fn production_code(source: &str) -> String { + let end = [ + "#[cfg(test)]\nmod tests", + "#[cfg(test)]\npub(crate) mod tests", + ] + .iter() + .filter_map(|marker| source.find(marker)) + .min() + .unwrap_or(source.len()); + let chars: Vec = source[..end].chars().collect(); + let mut code = String::with_capacity(end); + let mut at = 0; + let mut in_string = false; + while at < chars.len() { + let c = chars[at]; + if in_string { + code.push(c); + if c == '\\' && at + 1 < chars.len() { + code.push(chars[at + 1]); + at += 2; + continue; + } + in_string = c != '"'; + at += 1; + continue; + } + match (c, chars.get(at + 1)) { + ('/', Some('/')) => { + while at < chars.len() && chars[at] != '\n' { + at += 1; + } + } + ('/', Some('*')) => { + at += 2; + while at + 1 < chars.len() && !(chars[at] == '*' && chars[at + 1] == '/') { + at += 1; + } + at += 2; + } + // The character literal `'"'` must not open a string. + ('\'', Some('"')) if chars.get(at + 2) == Some(&'\'') => { + code.push_str("'\"'"); + at += 3; + } + ('"', _) => { + in_string = true; + code.push(c); + at += 1; + } + _ => { + code.push(c); + at += 1; + } + } + } + code + } + + /// The ways production code can name the registry directory: through the + /// two helpers in this file, or by spelling the path itself (the `runtimes` + /// and `registry` segments, or one literal holding both). + fn registry_location_spellings(production: &str) -> Vec<&'static str> { + let mut spellings = Vec::new(); + for helper in ["runtime_manifest_path(", "runtime_registry_dir("] { + if production.contains(helper) { + spellings.push(helper); + } + } + if production.contains("\"runtimes\"") && production.contains("\"registry\"") { + spellings.push("the \"runtimes\" and \"registry\" path segments"); + } + if production.contains("runtimes/registry") || production.contains("runtimes\\\\registry") { + spellings.push("a literal path through runtimes/registry"); + } + spellings + } + + /// Why `production` writes a registry entry on its own, one line per way it + /// can reach the registry, or nothing. A file that both writes (any of + /// [`WRITE_PRIMITIVES`], or a file builder with a write option, see + /// [`open_options_write`]) and names the registry directory (by any of + /// [`registry_location_spellings`]) is a writer, however far apart the two + /// are and whatever the path variable is called. A file that only reads or + /// removes entries has no write call and is not one. + fn registry_writer_findings(production: &str) -> Vec { + let writes = write_calls(production); + if writes.is_empty() { + return Vec::new(); + } + registry_location_spellings(production) + .into_iter() + .map(|spelling| format!("{spelling} in a file that calls {}", writes.join(", "))) + .collect() + } + + /// The detector sees a writer however it builds its path, and nothing that + /// only reads. The cases are the ways the previous scan (a 200-character + /// window after `fs::write(` looking for three names) went blind: a path built + /// from literals, a path held in a variable away from the write, one literal, + /// a `//` inside a string literal before the write. A file builder + /// (`OpenOptions`, `File::options`) counts as a write only when a write option + /// is chained on it or the chain cannot be read; a read-only probe in a file + /// that builds the registry path, as `amd_smi.rs` does, is not a writer. + #[test] + fn registry_writer_detector_sees_every_way_of_building_the_path() { + let filler = "// padding\n".repeat(40); + let far_apart = format!( + "fn dir(p: &AppPaths) -> PathBuf {{ p.data_dir.join(\"runtimes\").join(\"registry\") }}\n{filler}fn save(d: &Path) {{ fs::write(d.join(\"x.json\"), b\"\").ok(); }}" + ); + let registry = "fn dir(p: &AppPaths) -> PathBuf { p.data_dir.join(\"runtimes\").join(\"registry\") }\n"; + let cases: [(&str, String, bool); 17] = [ + ( + "amd_smi shape: the registry path is built, and only read-only probes open files", + format!( + "{registry}fn probe(path: &Path) -> bool {{ fs::OpenOptions::new().read(true).open(path).is_ok() }}\nfn device(path: &Path) -> bool {{ std::fs::OpenOptions::new().read(true).open(path).is_ok() }}" + ), + false, + ), + ( + "a read-only probe whose chain spans lines", + format!( + "{registry}fn probe(path: &Path) -> bool {{\n OpenOptions::new()\n .read(true)\n .open(path)\n .is_ok()\n}}" + ), + false, + ), + ( + "a read-only probe through File::options", + format!("{registry}fn probe(p: &Path) -> bool {{ File::options().read(true).open(p).is_ok() }}"), + false, + ), + ( + "OpenOptions with .write(true)", + format!( + "{registry}fn save(d: &Path) {{ OpenOptions::new().write(true).create(true).open(d.join(\"k.json\")).ok(); }}" + ), + true, + ), + ( + "OpenOptions with a write option on a later line", + format!( + "{registry}fn save(d: &Path) {{\n fs::OpenOptions::new()\n .read(true)\n .truncate(true)\n .open(d.join(\"k.json\"))\n .ok();\n}}" + ), + true, + ), + ( + "File::options with .append(true)", + format!("{registry}fn save(d: &Path) {{ File::options().append(true).open(d).ok(); }}"), + true, + ), + ( + "a builder kept in a variable cannot be judged", + format!( + "{registry}fn save(d: &Path) {{ let mut options = OpenOptions::new(); options.write(true); options.open(d).ok(); }}" + ), + true, + ), + ( + "a chain that never reaches .open( cannot be judged", + format!("{registry}fn save() {{ let options = OpenOptions::new().read(true).clone(); drop(options); }}"), + true, + ), + ( + "through the helper", + "fn f(p: &AppPaths) { fs::write(runtime_manifest_path(p, \"k\"), b\"\").ok(); }" + .to_owned(), + true, + ), + ("segments, far from the write", far_apart, true), + ( + "one literal", + "fn f(base: &Path) { let d = base.join(\"runtimes/registry\"); File::create(d.join(\"k.json\")).ok(); }" + .to_owned(), + true, + ), + ( + "a // inside a string before the write", + "fn f(p: &AppPaths) { let note = \"see http://x\"; fs::write(runtime_registry_dir(p).join(note), b\"\").ok(); }" + .to_owned(), + true, + ), + ( + "reads only", + "fn f(p: &AppPaths) { let _ = fs::read_dir(p.data_dir.join(\"runtimes\").join(\"registry\")); }" + .to_owned(), + false, + ), + ( + "writes something else", + "fn f(p: &Path) { fs::write(p.join(\"config.json\"), b\"\").ok(); }".to_owned(), + false, + ), + ( + "only a comment names the writer", + "// fs::write(runtime_manifest_path(p, k), b\"\")\nfn f() {}".to_owned(), + false, + ), + ( + "the writer is in the test module", + "fn f() {}\n#[cfg(test)]\nmod tests {\n fn t(p: &AppPaths) { fs::write(runtime_manifest_path(p, \"k\"), b\"\").ok(); }\n}" + .to_owned(), + false, + ), + ( + "a block comment holds the path", + "fn f(p: &Path) { /* runtime_registry_dir( */ fs::write(p, b\"\").ok(); }" + .to_owned(), + false, + ), + ]; + for (label, source, is_a_writer) in cases { + assert_eq!( + !registry_writer_findings(&production_code(&source)).is_empty(), + is_a_writer, + "{label}" + ); + } + } + + /// Every way of registering a runtime goes through + /// [`write_runtime_manifest`]. A second writer would be free to drift again + /// (a different overwrite rule, a missing install-root manifest), so no + /// production code outside this file may both name the registry directory and + /// write a file. This reads the sources of the `rocm` app, `rocmd`, + /// `rocm-core` and the engines: the writers differ only in how they are + /// called, which no behavioural test of one caller can see in another. + /// + /// Two further rules keep that from being walked around. The files that call + /// the two path helpers are an explicit list, each with the reason it needs + /// the path and none of them writing, so a new caller is reviewed rather than + /// passed, and a stale entry fails too. The two fixture modules are exempt + /// because the modules are `#[cfg(test)]` where they are declared, which is + /// checked, not because of what they are called. + /// + /// What this does not catch, by design: a writer that is handed the registry + /// path by a function in another file and never spells it, a write primitive + /// that is not in [`WRITE_PRIMITIVES`] (a new helper with its own name), a + /// file builder whose write option is set through something other than the + /// methods in [`WRITE_OPTIONS`] (the chain is read as text; a chain that + /// reaches `.open(` with only read options is a read), and code outside the + /// source roots it reads. Review catches those; this test + /// catches the ordinary ways of growing a second writer. + #[test] + fn only_the_shared_routine_writes_a_registry_entry() { + fn rust_files(dir: &Path, found: &mut Vec) { + for entry in fs::read_dir(dir).expect("source folder readable").flatten() { + let path = entry.path(); + if path.is_dir() { + rust_files(&path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") { + found.push(path); + } + } + } + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("the app sits two levels below the workspace root"); + let mut roots = vec![ + workspace.join("apps/rocm/src"), + workspace.join("apps/rocmd/src"), + workspace.join("crates/rocm-core/src"), + ]; + for engine in fs::read_dir(workspace.join("engines")) + .expect("engines folder readable") + .flatten() + { + roots.push(engine.path().join("src")); + } + let mut sources = Vec::new(); + for root in &roots { + rust_files(root, &mut sources); + } + assert!(sources.len() > 100, "the scan found the source tree"); + + // This file holds the routine. + let routine = "apps/rocm/src/runtime_registry.rs"; + // Fixture modules, each with the file that declares it. + let fixtures = [ + ("apps/rocm/src/test_support.rs", "apps/rocm/src/main.rs"), + ( + "apps/rocm/src/therock/test_support.rs", + "apps/rocm/src/therock.rs", + ), + ]; + for (fixture, declared_in) in fixtures { + let declaration = + fs::read_to_string(workspace.join(declared_in)).expect("declaring file readable"); + assert!( + declaration.contains("#[cfg(test)]\nmod test_support;"), + "{fixture} is exempt only while {declared_in} declares it under #[cfg(test)]" + ); + } + // Production files that name an entry's path without writing it, and why + // (in sorted order, which is how they are compared). + let path_helper_callers = [ + // Whether setup's runtime is registered, and the error that names the registry. + "apps/rocm/src/runtime_adopt.rs", + // The entry uninstall removes. + "apps/rocm/src/runtime_uninstall.rs", + // The path an activation, import or adopt report prints. + "apps/rocm/src/runtimes_cmd.rs", + // The path an install plan prints. + "apps/rocm/src/therock.rs", + ]; + + let mut helper_callers = Vec::new(); + for path in sources { + let relative = path + .strip_prefix(workspace) + .expect("source under the workspace") + .to_string_lossy() + .replace('\\', "/"); + if relative == routine || fixtures.iter().any(|(fixture, _)| *fixture == relative) { + continue; + } + let production = production_code(&fs::read_to_string(&path).expect("source readable")); + let findings = registry_writer_findings(&production); + assert!( + findings.is_empty(), + "{relative} writes a registry entry itself instead of calling write_runtime_manifest: {findings:?}" + ); + if production.contains("runtime_manifest_path(") + || production.contains("runtime_registry_dir(") + { + helper_callers.push(relative); + } + } + helper_callers.sort(); + assert_eq!( + helper_callers, path_helper_callers, + "the files that call runtime_manifest_path or runtime_registry_dir changed: a new caller must be reviewed (and listed with its reason) and a file that stopped calling them removed" + ); + } + /// A manifest written before `devel` became opt-in has no such field, and /// those installs all had the toolchain. Reading one back must not claim /// otherwise, or the next update would silently strip it. @@ -946,4 +1721,57 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// Activation validation of a tarball runtime follows + /// `rocm_core::install_root_has_payload`: the root's own `.rocm-cli-runtime.json` + /// marker is a dot-entry and does not make a payload, so a root holding only + /// that is reported unusable (by the message the runtime listing prints), and + /// one more entry makes it ready. A root that is gone is reported by the + /// earlier "install root is missing" check, never reaching the payload rule. + #[test] + fn tarball_validation_follows_the_shared_payload_rule() { + let (root, _) = test_paths("tarball-shared-payload-rule"); + let install_root = root.join("tarball-runtime"); + fs::create_dir_all(&install_root).unwrap(); + fs::write(install_root.join(".rocm-cli-runtime.json"), b"{}").unwrap(); + let manifest = InstalledRuntimeManifest { + format: "tarball".to_owned(), + install_root: install_root.clone(), + ..test_runtime_manifest( + "release-tarball-gfx120X-all-7-13-0", + "therock-release:gfx120X-all", + 1, + ) + }; + + let error = validate_runtime_manifest_for_activation(&manifest) + .expect_err("a root with only the dot-marker has no payload") + .to_string(); + assert_eq!( + error, + format!( + "tarball runtime install root has no payload files: {}", + install_root.display() + ) + ); + assert_eq!( + runtime_usability_status(&manifest), + format!("unusable ({error})") + ); + + fs::write(install_root.join("rocm-install-file"), b"x").unwrap(); + validate_runtime_manifest_for_activation(&manifest) + .expect("one entry beside the marker is a payload"); + assert_eq!(runtime_usability_status(&manifest), "ready"); + + fs::remove_dir_all(&install_root).unwrap(); + let error = validate_runtime_manifest_for_activation(&manifest) + .expect_err("a missing root is unusable") + .to_string(); + assert_eq!( + error, + format!("install root is missing: {}", install_root.display()) + ); + fs::remove_dir_all(root).ok(); + } } diff --git a/apps/rocm/src/runtime_services.rs b/apps/rocm/src/runtime_services.rs index 636a3e4549..84f020ef89 100644 --- a/apps/rocm/src/runtime_services.rs +++ b/apps/rocm/src/runtime_services.rs @@ -16,8 +16,8 @@ //! [`ServiceRuntimeState`], [`RuntimeServiceEntry`], [`FailedServiceRestart`], //! [`RuntimeServiceReconciliation`] and [`ServiceRuntimePin`]. `RuntimesCommand` //! lives in `cli.rs` (part of the `Command`/`Cli` clap definition); `fn -//! runtimes()` itself moved to `runtimes_cmd.rs` in Phase 6c (ROCMAI-89), which -//! calls back into this file via `crate::runtime_services`. +//! runtimes()` itself lives in `runtimes_cmd.rs`, which calls back into this +//! file via `crate::runtime_services`. use std::fmt::Write as _; diff --git a/apps/rocm/src/runtime_uninstall.rs b/apps/rocm/src/runtime_uninstall.rs index 8b102383e6..e140cf1550 100644 --- a/apps/rocm/src/runtime_uninstall.rs +++ b/apps/rocm/src/runtime_uninstall.rs @@ -5,12 +5,16 @@ //! `rocm runtimes uninstall`: planning the removal of a registered runtime, //! re-validating that plan after the confirmation prompt, and applying it //! (unregistering the runtime, clearing the active and default state that named -//! it, and removing its install root when ROCm CLI owns it). +//! it, and removing its install root when ROCm CLI owns it). When that root +//! holds the Lemonade install its record points at (a self-managed engine +//! installs inside the active runtime's folder), the plan says so and applying +//! it clears the record, so Lemonade does not keep naming deleted files. //! -//! Relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` -//! calls into this module from `runtimes()` and re-exports `uninstall_runtime`, -//! `should_remove_runtime_install_root` and `InstallRootDecision` for -//! `storage.rs`. The active-runtime marker check this module makes +//! Relocated from `runtimes_cmd.rs`; the Lemonade record handling is new. +//! `runtimes_cmd.rs` calls into this module from `runtimes()` and re-exports +//! `uninstall_runtime`, `should_remove_runtime_install_root`, +//! `InstallRootDecision` and `lemonade_record_report_line` for `storage.rs`. +//! The active-runtime marker check this module makes //! (`active_runtime_marker_matches`, over the `ActiveRuntimeMarker` the //! activation writes) and the path comparison it needs live in //! `runtime_registry.rs`, so this module does not depend on `runtimes_cmd.rs`. @@ -20,6 +24,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, RocmCliConfig, runtime_install_root_is_protected}; +use rocm_engine_lemonade::RecordCleanup; use crate::cli_report; use crate::runtime_registry::{ @@ -34,6 +39,9 @@ pub(crate) struct RuntimeUninstallResult { pub(crate) runtime_key: String, pub(crate) registry_path: PathBuf, pub(crate) removed_install_root: Option, + /// What happened to Lemonade's install record once the folder was removed: + /// cleared, left dangling because deleting it failed, or not checkable. + pub(crate) lemonade_record: RecordCleanup, pub(crate) read_only: bool, pub(crate) manifest_mismatch: bool, pub(crate) was_active: bool, @@ -70,6 +78,14 @@ pub(crate) struct RuntimeUninstallPlan { /// does not by itself orphan the default while others remain). clears_default_runtime: bool, install_root_decision: InstallRootDecision, + /// Lemonade's runtime folder when its install record points inside this + /// runtime's folder: a self-managed engine installs there while the runtime + /// is active, so removing the folder removes that engine too. + lemonade_install_inside: Option, + /// Whether this runtime's folder holds a Lemonade tree at all, installed or + /// partly installed. Only then can an unreadable install record matter to + /// this uninstall. + holds_lemonade_tree: bool, } impl RuntimeUninstallPlan { @@ -78,6 +94,40 @@ impl RuntimeUninstallPlan { } } +/// The line `runtimes uninstall` (and `storage remove-old-installs`, which +/// delegates to it) report about Lemonade's install record, or `None` when the +/// removed runtime had nothing to do with it. +/// +/// Built from the outcome the record handling returned, so each claim is the one +/// that happened: "no longer recorded" only follows a record that was deleted, +/// and a record that could not be deleted or read says so and how to fix it. +pub(crate) fn lemonade_record_report_line(outcome: &RecordCleanup) -> Option { + match outcome { + RecordCleanup::NotAffected => None, + RecordCleanup::Cleared(lemonade) => Some(format!( + "its install at {} was inside this runtime and was removed with it; \ + Lemonade is no longer recorded as installed", + lemonade.display() + )), + RecordCleanup::Failed { + runtime_dir, + record, + error, + } => Some(format!( + "its install at {} was inside this runtime and was removed with it, but \ + Lemonade's install record could not be deleted ({error}); delete {} \ + so Lemonade is not left pointing at files that are gone", + runtime_dir.display(), + record.display() + )), + RecordCleanup::Unreadable(error) => Some(format!( + "Lemonade's install record could not be read ({error}), so it was not \ + checked against this runtime; `rocm engines install lemonade` rewrites it \ + and keeps a current Lemonade install" + )), + } +} + pub(crate) fn print_runtime_uninstall_plan(plan: &RuntimeUninstallPlan) { let install_folder = if plan.will_remove_install_root() { format!( @@ -104,6 +154,17 @@ pub(crate) fn print_runtime_uninstall_plan(plan: &RuntimeUninstallPlan) { if plan.clears_default_runtime { report = report.detail("default_runtime", "would be cleared"); } + if plan.will_remove_install_root() + && let Some(lemonade) = plan.lemonade_install_inside.as_ref() + { + report = report.detail( + "lemonade", + format!( + "its install at {} is inside this runtime folder and would be removed with it", + lemonade.display() + ), + ); + } print!("{}", report.render()); } @@ -127,12 +188,23 @@ pub(crate) fn plan_runtime_uninstall( && other.runtime_id.eq_ignore_ascii_case(&manifest.runtime_id) })); let install_root_decision = should_remove_runtime_install_root(&manifest)?; + let holds_lemonade_tree = + rocm_engine_lemonade::install_tree_dirs(&rocm_engine_lemonade::install_tree_root( + paths, + Some(&manifest.install_root.join("engines")), + )) + .iter() + .any(|part| part.exists()); + let lemonade_install_inside = rocm_engine_lemonade::recorded_runtime_dir(paths) + .filter(|dir| rocm_core::runtime_path_is_same_or_inside(dir, &manifest.install_root)); Ok(RuntimeUninstallPlan { manifest, registry_path, was_active, clears_default_runtime, install_root_decision, + lemonade_install_inside, + holds_lemonade_tree, }) } @@ -215,6 +287,8 @@ pub(crate) fn apply_runtime_uninstall( was_active, clears_default_runtime, install_root_decision, + lemonade_install_inside: _, + holds_lemonade_tree, } = plan; let mut removed_install_root = None; @@ -288,11 +362,27 @@ pub(crate) fn apply_runtime_uninstall( } } + // Decided from the folder actually removed, not from the plan: the record + // is only stale once its files are gone. + let lemonade_record = removed_install_root + .as_deref() + .map_or(RecordCleanup::NotAffected, |removed| { + rocm_engine_lemonade::forget_install_record_inside(paths, removed) + }); + // An unreadable record says nothing about this runtime unless the runtime + // could have held Lemonade; otherwise the line would repeat on every runtime + // uninstall for as long as the record stays unreadable. + let lemonade_record = match lemonade_record { + RecordCleanup::Unreadable(_) if !holds_lemonade_tree => RecordCleanup::NotAffected, + other => other, + }; + Ok(RuntimeUninstallResult { runtime_id: manifest.runtime_id, runtime_key: manifest.runtime_key, registry_path, removed_install_root, + lemonade_record, read_only: manifest.read_only, manifest_mismatch: matches!(install_root_decision, InstallRootDecision::ManifestMismatch), was_active, @@ -357,6 +447,180 @@ mod tests { use crate::test_support::write_test_pip_runtime; use crate::tests::test_paths; + /// While a runtime is active, `rocm serve` installs Lemonade inside that + /// runtime's folder and records it there. Uninstalling the runtime deletes + /// the folder; the record must go with it, or Lemonade keeps pointing at + /// files that no longer exist. A record pointing elsewhere is left alone. + #[test] + fn runtime_uninstall_clears_a_lemonade_record_pointing_into_the_removed_folder() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-lemonade"); + let held = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-14-1", + "therock-release:gfx1151", + "7.14.1", + 20, + )?; + let other = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let tree = rocm_engine_lemonade::install_tree_root( + &paths, + Some(&held.install_root.join("engines")), + ); + crate::test_support::plant_lemonade_tree(&tree, 16)?; + crate::test_support::record_lemonade_install(&paths, &tree)?; + let mut config = RocmCliConfig::default(); + + // Removing a runtime that does not hold the tree keeps the record. + let removed = uninstall_runtime(&paths, &mut config, &other.runtime_key)?; + assert_eq!(removed.lemonade_record, RecordCleanup::NotAffected); + assert_eq!( + rocm_engine_lemonade::recorded_runtime_dir(&paths), + Some(tree.join("runtime")) + ); + + let plan = plan_runtime_uninstall(&paths, &config, &held.runtime_key)?; + assert_eq!(plan.lemonade_install_inside, Some(tree.join("runtime"))); + let removed = apply_runtime_uninstall(&paths, &mut config, plan)?; + + assert_eq!( + removed.lemonade_record, + RecordCleanup::Cleared(tree.join("runtime")) + ); + assert!(!tree.exists()); + assert_eq!(rocm_engine_lemonade::recorded_runtime_dir(&paths), None); + let line = lemonade_record_report_line(&removed.lemonade_record).unwrap(); + assert!( + line.contains(&tree.join("runtime").display().to_string()) + && line.ends_with("Lemonade is no longer recorded as installed"), + "{line}" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A record that cannot be read may well name the runtime being removed, so + /// the uninstall cannot tell whether it is left dangling. When the runtime + /// held a Lemonade tree it must say so rather than treat the record as + /// absent, name the cheap command that rewrites it (a plain + /// `rocm engines install lemonade`, shown to do so by + /// `a_plain_install_replaces_a_corrupt_record_and_keeps_the_current_tree` in + /// the engine), not the wiping `--reinstall`, and leave the file alone. + #[test] + fn runtime_uninstall_reports_a_lemonade_record_it_could_not_read() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-lemonade-unreadable"); + let held = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-14-1", + "therock-release:gfx1151", + "7.14.1", + 20, + )?; + crate::test_support::plant_lemonade_tree( + &rocm_engine_lemonade::install_tree_root( + &paths, + Some(&held.install_root.join("engines")), + ), + 16, + )?; + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + fs::create_dir_all(record.parent().expect("record has a parent"))?; + fs::write(&record, b"{ not json")?; + let mut config = RocmCliConfig::default(); + + let removed = uninstall_runtime(&paths, &mut config, &held.runtime_key)?; + + assert!( + matches!(removed.lemonade_record, RecordCleanup::Unreadable(_)), + "{:?}", + removed.lemonade_record + ); + let line = lemonade_record_report_line(&removed.lemonade_record).unwrap(); + assert!(line.contains("could not be read"), "{line}"); + assert!( + line.contains("`rocm engines install lemonade` rewrites it"), + "{line}" + ); + assert!(!line.contains("--reinstall"), "{line}"); + assert!(record.exists()); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The same unreadable record must not be reported on a runtime that never + /// held Lemonade: nothing about that uninstall depends on it, and the line + /// would otherwise repeat on every runtime uninstall. + #[test] + fn an_unreadable_lemonade_record_is_not_reported_for_a_runtime_without_lemonade() -> Result<()> + { + let (root, paths) = test_paths("runtime-uninstall-lemonade-unrelated"); + let runtime = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-14-1", + "therock-release:gfx1151", + "7.14.1", + 20, + )?; + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + fs::create_dir_all(record.parent().expect("record has a parent"))?; + fs::write(&record, b"{ not json")?; + let mut config = RocmCliConfig::default(); + + let plan = plan_runtime_uninstall(&paths, &config, &runtime.runtime_key)?; + assert!(!plan.holds_lemonade_tree); + let removed = apply_runtime_uninstall(&paths, &mut config, plan)?; + + assert_eq!(removed.lemonade_record, RecordCleanup::NotAffected); + assert_eq!(lemonade_record_report_line(&removed.lemonade_record), None); + assert!( + removed.removed_install_root.is_some(), + "the runtime was removed" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Each outcome's line says only what happened. A failed deletion must + /// name the record to delete and must not claim Lemonade is no longer + /// recorded; no outcome for an unrelated runtime prints anything. (That + /// deleting the named record really clears the condition is shown by + /// `a_record_that_cannot_be_deleted_is_reported_not_swallowed` in the + /// engine.) + #[test] + fn the_lemonade_record_line_matches_what_happened() { + assert_eq!( + lemonade_record_report_line(&RecordCleanup::NotAffected), + None + ); + let cleared = + lemonade_record_report_line(&RecordCleanup::Cleared(PathBuf::from("/rt/lemonade"))) + .unwrap(); + assert!( + cleared.contains("no longer recorded as installed"), + "{cleared}" + ); + let failed = lemonade_record_report_line(&RecordCleanup::Failed { + runtime_dir: PathBuf::from("/rt/lemonade"), + record: PathBuf::from("/data/engines/lemonade/manifests/runtime.json"), + error: "sharing violation".to_owned(), + }) + .unwrap(); + assert!(failed.contains("sharing violation"), "{failed}"); + assert!( + failed.contains("delete /data/engines/lemonade/manifests/runtime.json"), + "{failed}" + ); + assert!(!failed.contains("no longer recorded"), "{failed}"); + } + #[test] fn runtime_uninstall_unregisters_read_only_runtime_without_deleting_external_root() -> Result<()> { diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index bd7033b402..950960a6b2 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -55,7 +55,8 @@ pub(crate) use crate::runtime_registry::{ validate_runtime_manifest_for_activation, }; pub(crate) use crate::runtime_uninstall::{ - InstallRootDecision, should_remove_runtime_install_root, uninstall_runtime, + InstallRootDecision, lemonade_record_report_line, should_remove_runtime_install_root, + uninstall_runtime, }; use crate::runtime_uninstall::{ RuntimeUninstallConfirmation, apply_runtime_uninstall, @@ -257,6 +258,9 @@ pub(crate) fn runtimes(command: Option) -> Result<()> { report = report.detail("folder_removed", "no"); } } + if let Some(line) = lemonade_record_report_line(&result.lemonade_record) { + report = report.detail("lemonade", line); + } if result.default_runtime_cleared { report = report .detail("default_runtime", "cleared") @@ -473,13 +477,9 @@ impl ActivationSnapshot { // failed, so it is the write least able to afford leaving a // truncated marker behind — which runtime resolution and the // storage retention holds would both read as "nothing is active". + // The helper creates the `runtimes` folder when it is missing. MarkerSnapshot::Contents(bytes) => { - let parent = path - .parent() - .context("active runtime marker path has no parent directory")?; - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - therock::write_file_atomically(&path, &bytes) + rocm_core::atomic_write::write_file_atomically(&path, &bytes) .with_context(|| format!("failed to restore {}", path.display()))?; } // Only a regular file is removed: whatever else may sit at that @@ -527,11 +527,11 @@ fn restore_after_failed_activation( error: anyhow::Error, ) -> anyhow::Error { match snapshot.restore(paths, config) { - Ok(()) => error.context( + Ok(()) => discard_failed_marker_replacement(error).context( "failed to record the active ROCm runtime; the previously active runtime is still \ the active one", ), - Err(restore_error) => error.context(format!( + Err(restore_error) => discard_failed_marker_replacement(error).context(format!( "failed to record the active ROCm runtime, and restoring the previous state failed \ too ({restore_error:#}); the ROCm CLI config and the active runtime marker may now \ disagree. Run `rocm runtimes list` to see both, then re-run `rocm runtimes activate \ @@ -540,6 +540,38 @@ fn restore_after_failed_activation( } } +/// Drop the marker a failed activation left behind, and the advice to use it. +/// +/// On Windows a marker publish can fail after taking the old marker away, and +/// the shared writer then keeps the new marker in its temp file and advises +/// renaming it into place (`rocm_core::atomic_write::KeptReplacement`). The +/// activation is being rolled back, so that file holds the marker of an +/// activation that failed, and renaming it would install that marker over +/// whatever the rollback restored. So the file is removed, and the error is +/// rebuilt without the advice. If the file cannot be removed, the error names +/// it and says to delete it instead. +fn discard_failed_marker_replacement(error: anyhow::Error) -> anyhow::Error { + let Some(kept) = error.downcast_ref::() else { + return error; + }; + let failure = format!( + "failed to write active runtime marker {}: {}", + kept.destination().display(), + kept.cause() + ); + match fs::remove_file(kept.kept()) { + Ok(()) => anyhow::anyhow!(failure), + Err(remove_error) if remove_error.kind() == std::io::ErrorKind::NotFound => { + anyhow::anyhow!(failure) + } + Err(remove_error) => anyhow::anyhow!( + "{failure}; {} holds the marker of the activation that failed and could not be \ + removed ({remove_error}): delete it, do not rename it into place", + kept.kept().display() + ), + } +} + /// Marker shown beside the active runtime in `rocm runtimes list`. Every /// place that renders this glyph MUST use this constant so the rendered /// character and the legend text stay in sync. @@ -843,33 +875,46 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { } } +/// Replace the active-runtime marker in one step. +/// +/// Two `rocm` processes can activate a runtime against the same data +/// directory at once (two terminals, a script), so the write goes through the +/// shared atomic writer: each writer stages into its own exclusively created +/// temp file, and the marker is replaced rather than removed first, so a +/// reader always finds a complete marker: on Windows the publish renames first, +/// which replaces in one step. Only when that rename fails, because another +/// process holds the marker open without delete sharing, does it fall back to +/// `ReplaceFileW`, and a `ReplaceFileW` that fails part-way can leave the marker +/// missing, which the writer repairs with a plain rename when that succeeds +/// (see `rocm_core::atomic_write`); two replacements in flight at once through +/// the fallback are not tested. It does not order the two activations: the last +/// one to publish wins, as before. fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { let path = active_runtime_marker_path(paths); - fs::create_dir_all( - path.parent() - .context("active runtime marker path has no parent directory")?, - )?; // The shared helper rather than a local temp-file-and-rename: it is the same // write the failed-activation restore uses, and it is strictly stronger than // a bare `fs::rename`, which on Windows is `MoveFileExW` and fails with a // sharing violation when another process holds the marker open. The helper - // uses `ReplaceFileW` there whenever the destination exists. The + // renames first there and falls back to `ReplaceFileW` when that fails. The // destination is never removed first, on any platform — that would leave a // window with no marker at all, which runtime resolution and the storage // retention holds both read as "nothing is active" — and a failed publish // takes its temp file with it, so a retried activation cannot strand one in - // the folder `rocm runtimes` reads. - therock::write_file_atomically( - &path, - &serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?, - ) - .with_context(|| format!("failed to write {}", path.display()))?; - Ok(()) + // the folder `rocm runtimes` reads. The one exception is a Windows publish + // that took the old marker away and could not put the new one in place: + // the temp file is then the only copy, so it is kept and the error names + // it; a failed activation removes it again once the restore has put the + // previous marker back (`discard_failed_marker_replacement`). It also + // creates the `runtimes` folder when it is missing. + let bytes = + serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; + rocm_core::atomic_write::write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write active runtime marker {}", path.display())) } #[cfg(test)] pub(crate) mod tests { - use std::path::PathBuf; + use std::path::{Path, PathBuf}; use super::*; use crate::append_examine_runtime_state; @@ -878,6 +923,7 @@ pub(crate) mod tests { use crate::runtime_install_finalize::{ finalize_successful_sdk_install, render_sdk_install_success, }; + use crate::runtime_registry::{ManifestOverwrite, write_runtime_manifest}; use crate::test_support::write_test_pip_runtime; use crate::tests::{ NEW_RUNTIME_KEY, OLD_RUNTIME_KEY, plant_service_record_on_runtime, test_paths, @@ -1193,6 +1239,145 @@ pub(crate) mod tests { Ok(()) } + #[cfg(unix)] + fn test_active_runtime_marker(paths: &AppPaths, runtime_key: &str) -> ActiveRuntimeMarker { + ActiveRuntimeMarker { + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: runtime_key.to_owned(), + manifest_path: runtime_manifest_path(paths, runtime_key), + install_root: paths.data_dir.join("runtimes").join(runtime_key), + previous_runtime_id: None, + previous_runtime_key: None, + activated_at_unix_ms: rocm_core::unix_time_millis(), + } + } + + #[cfg(unix)] + fn leftover_marker_temp_files(paths: &AppPaths) -> Result> { + let dir = active_runtime_marker_path(paths) + .parent() + .context("marker has a parent")? + .to_path_buf(); + let mut leftovers = Vec::new(); + for entry in fs::read_dir(dir)? { + let name = entry?.file_name().to_string_lossy().into_owned(); + if name.starts_with("active.json.tmp-") { + leftovers.push(name); + } + } + Ok(leftovers) + } + + /// Two activations writing the marker at once must both succeed, and a + /// reader must find a complete marker throughout. #605 reported a writer + /// that removed `active.json` before renaming the replacement in, and two + /// writers in the same millisecond sharing one temp file, so one failed + /// with "failed to move active runtime marker" and the other could publish + /// the shorter payload followed by the longer one's tail. #424 fixed that; + /// this change routes the marker through `rocm_core::atomic_write`, the + /// writer `rocm` and `rocmd` now share, and this test pins that the marker + /// keeps the property through it. + /// + /// The bug is a race, so against the old code this fails with high + /// probability per round rather than deterministically; against the + /// current writer nothing in it is timing-dependent. Unix only: the + /// property under test is that `rename` replaces in one step. The Windows + /// publish tries `rename` first too, but this test has not been run + /// against it there, nor against the `ReplaceFileW` fallback. + #[cfg(unix)] + #[test] + fn concurrent_active_runtime_marker_writes_keep_a_complete_marker_published() -> Result<()> { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Barrier}; + + const ROUNDS: usize = 200; + let (root, paths) = test_paths("marker-concurrent"); + write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "initial"))?; + let marker_path = active_runtime_marker_path(&paths); + + let done = Arc::new(AtomicBool::new(false)); + let reader = { + let done = Arc::clone(&done); + let marker_path = marker_path.clone(); + std::thread::spawn(move || { + let mut problems = Vec::new(); + while !done.load(Ordering::Acquire) { + match fs::read(&marker_path) { + Ok(bytes) => { + if let Err(error) = + serde_json::from_slice::(&bytes) + { + problems.push(format!("unparsable marker: {error}")); + } + } + Err(error) => problems.push(format!("marker unreadable: {error}")), + } + } + problems + }) + }; + + let barrier = Arc::new(Barrier::new(2)); + let writers: Vec<_> = [ + "short", + "a-runtime-key-long-enough-to-leave-a-tail-behind-a-shorter-payload", + ] + .into_iter() + .map(|runtime_key| { + let barrier = Arc::clone(&barrier); + let paths = paths.clone(); + std::thread::spawn(move || { + let mut errors = Vec::new(); + for _ in 0..ROUNDS { + barrier.wait(); + if let Err(error) = write_active_runtime_marker( + &paths, + test_active_runtime_marker(&paths, runtime_key), + ) { + errors.push(format!("{error:#}")); + } + } + errors + }) + }) + .collect(); + + let mut writer_errors = Vec::new(); + for writer in writers { + writer_errors.extend(writer.join().expect("writer thread panicked")); + } + done.store(true, Ordering::Release); + let reader_problems = reader.join().expect("reader thread panicked"); + + let published: ActiveRuntimeMarker = serde_json::from_slice(&fs::read(&marker_path)?)?; + let leftovers = leftover_marker_temp_files(&paths)?; + let _ = fs::remove_dir_all(root); + + assert!( + writer_errors.is_empty(), + "{} of {} marker writes failed; first: {:?}", + writer_errors.len(), + 2 * ROUNDS, + writer_errors.first() + ); + assert!( + reader_problems.is_empty(), + "a reader found no complete marker {} times; first: {:?}", + reader_problems.len(), + reader_problems.first() + ); + assert!( + [ + "short", + "a-runtime-key-long-enough-to-leave-a-tail-behind-a-shorter-payload" + ] + .contains(&published.runtime_key.as_str()), + "unexpected marker: {published:?}" + ); + assert_eq!(leftovers, Vec::::new()); + Ok(()) + } + #[test] fn runtime_activation_records_exact_key_and_rollback() -> Result<()> { let (root, paths) = test_paths("runtime-activation"); @@ -1298,10 +1483,10 @@ pub(crate) mod tests { } /// Replace the active-runtime marker with a NON-EMPTY directory, so the - /// next `write_active_runtime_marker` fails at a predictable point: its - /// `create_dir_all` of the parent succeeds, and the final `fs::rename` of - /// the temp file onto a non-empty directory fails on both supported - /// platforms without needing privileges or a real full disk. + /// next `write_active_runtime_marker` fails at a predictable point: the + /// temp file is staged beside it, and publishing that file onto a + /// non-empty directory fails on both supported platforms without needing + /// privileges or a real full disk. fn break_active_runtime_marker(paths: &AppPaths) -> Result { let marker = active_runtime_marker_path(paths); let _ = fs::remove_file(&marker); @@ -1993,6 +2178,198 @@ pub(crate) mod tests { Ok(()) } + /// The Windows partial-replace failure during activation: the marker publish + /// took the old marker away and kept the new one in its temp file. Once the + /// restore has put the previous marker back, the kept file holds the failed + /// activation's marker, so it is removed, and the error no longer tells the + /// user to rename it into place. Driven through the shared writer's publish + /// seam, so it runs on every platform. + #[test] + fn a_restored_activation_drops_the_marker_a_partial_replace_kept() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + let previous_marker = fs::read(active_runtime_marker_path(&paths))?; + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + config.previous_runtime_key = Some(OLD_RUNTIME_KEY.to_owned()); + config.save(&paths)?; + + let marker_path = active_runtime_marker_path(&paths); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + assert!( + !marker_temp_file_names(&paths)?.is_empty(), + "the shared writer keeps the new marker when the old one is gone" + ); + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let restored_marker = fs::read(&marker_path)?; + let leftovers = marker_temp_file_names(&paths)?; + let persisted = RocmCliConfig::load(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert_eq!(restored_marker, previous_marker); + assert_eq!( + persisted.active_runtime_key.as_deref(), + Some(OLD_RUNTIME_KEY) + ); + assert!( + leftovers.is_empty(), + "the failed activation's marker was left beside the restored one: {leftovers:?}" + ); + assert!( + rendered.contains("the previously active runtime is still the active one"), + "{rendered}" + ); + assert!( + !rendered.contains("rename that file"), + "renaming would install the failed activation's marker:\n{rendered}" + ); + Ok(()) + } + + /// Produce the error the shared writer returns when a marker publish took + /// the old marker away and kept the new one, through its publish seam. + /// `prepare_kept` runs on the kept temp file before the error is returned. + fn partial_replace_marker_error( + paths: &AppPaths, + prepare_kept: impl FnOnce(&std::path::Path) -> std::io::Result<()>, + ) -> anyhow::Error { + let marker_path = active_runtime_marker_path(paths); + rocm_core::atomic_write::write_file_atomically_with_publish( + &marker_path, + b"{\"runtime_key\":\"new\"}", + |attempt| std::ffi::OsString::from(format!("partial-{attempt}")), + || {}, + |tmp, path| { + fs::remove_file(path)?; + prepare_kept(tmp)?; + Err(rocm_core::atomic_write::destination_removed_error( + std::io::Error::from_raw_os_error(1176), + std::io::Error::from(std::io::ErrorKind::PermissionDenied), + )) + }, + ) + .expect_err("the simulated partial replace fails the write") + .context(format!( + "failed to write active runtime marker {}", + marker_path.display() + )) + } + + /// When the kept marker cannot be removed, the error names it and says to + /// delete it, and nothing in it still says to rename it into place. The + /// kept path is made a non-empty directory, which `remove_file` refuses on + /// every platform. + #[test] + fn a_kept_marker_that_cannot_be_removed_is_named_with_delete_advice_only() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-stuck"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + let previous_marker = fs::read(active_runtime_marker_path(&paths))?; + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + config.save(&paths)?; + let write_error = partial_replace_marker_error(&paths, |tmp| { + fs::remove_file(tmp)?; + fs::create_dir(tmp)?; + fs::write(tmp.join("occupied.txt"), "stuck") + }); + let kept = marker_temp_file_names(&paths)?; + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let restored_marker = fs::read(active_runtime_marker_path(&paths))?; + let still_there = marker_temp_file_names(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert_eq!(kept.len(), 1, "{kept:?}"); + assert_eq!(still_there, kept, "the stuck file is left where it was"); + assert_eq!(restored_marker, previous_marker); + assert!( + rendered.contains(&kept[0]), + "the stuck file is named:\n{rendered}" + ); + assert!( + rendered.contains("delete it, do not rename it"), + "{rendered}" + ); + assert!( + !rendered.contains("rename that file"), + "renaming would install the failed activation's marker:\n{rendered}" + ); + Ok(()) + } + + /// A kept marker that is already gone by the time the rollback looks for it + /// is as good as removed: the error neither names it nor tells the user to + /// delete or rename it. + #[test] + fn a_kept_marker_already_gone_is_treated_as_removed() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-gone"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + let kept = marker_temp_file_names(&paths)?; + let runtimes_dir = active_runtime_marker_path(&paths) + .parent() + .context("active runtime marker path has no parent directory")? + .to_path_buf(); + for name in &kept { + fs::remove_file(runtimes_dir.join(name))?; + } + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let _ = fs::remove_dir_all(&root); + + assert_eq!(kept.len(), 1, "{kept:?}"); + assert!( + rendered.contains("the previously active runtime is still the active one"), + "{rendered}" + ); + assert!(!rendered.contains(&kept[0]), "{rendered}"); + assert!(!rendered.contains("rename that file"), "{rendered}"); + assert!(!rendered.contains("delete it"), "{rendered}"); + Ok(()) + } + + /// When the rollback itself fails as well, the kept marker is still the + /// failed activation's, so it goes too, and the error keeps the rollback's + /// own recovery advice without the advice to rename it. The config path is + /// made a non-empty directory so the restore's config save fails. + #[test] + fn a_failed_rollback_still_drops_the_marker_a_partial_replace_kept() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-double-fault"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + assert!(!marker_temp_file_names(&paths)?.is_empty()); + let config_path = paths.config_path(); + let _ = fs::remove_file(&config_path); + fs::create_dir_all(&config_path)?; + fs::write(config_path.join("occupied.txt"), "not a config")?; + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let leftovers = marker_temp_file_names(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert!( + rendered.contains("restoring the previous state failed too"), + "{rendered}" + ); + assert!(rendered.contains("rocm runtimes list"), "{rendered}"); + assert!(!rendered.contains("rename that file"), "{rendered}"); + assert!(leftovers.is_empty(), "{leftovers:?}"); + Ok(()) + } + // A restore that fails too must say so and name the way out, rather than // claim the previous runtime is still the active one. The marker's parent is // made a regular file, so writing the captured bytes back fails at @@ -2311,9 +2688,14 @@ pub(crate) mod tests { Ok(()) } - #[test] - fn runtime_adopt_records_read_only_manifest_from_probe() -> Result<()> { - let (root, paths) = test_paths("runtime-adopt"); + /// A Python environment outside ROCm CLI's folders that `rocm runtimes adopt` + /// accepts: an interpreter, the SDK libraries the probe names, and the probe + /// that reports them. Returns the environment folder, the request that adopts + /// it, and the probe. + fn adoptable_environment( + root: &Path, + replace: bool, + ) -> Result<(PathBuf, AdoptRuntimeRequest, therock::RocmSdkPythonProbe)> { let external_root = root.join("external-therock-venv"); let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); let python_executable = scripts_dir.join(if cfg!(windows) { @@ -2342,37 +2724,43 @@ pub(crate) mod tests { fs::write(&amdhip, "amdhip")?; fs::write(&hipblas, "hipblas")?; - let adopted = adopt_runtime_from_probe( - &paths, - AdoptRuntimeRequest { - python_executable, - install_root: external_root.clone(), - runtime_id: "therock-release:gfx120X-all".to_owned(), - runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), - replace: false, - }, - therock::RocmSdkPythonProbe { - import_ok: true, - rocm_sdk_version: Some("7.13.0".to_owned()), - root_path: Some(sdk_root.clone()), - bin_path: Some(sdk_bin.clone()), - runtime_roots: vec![sdk_root], - bin_paths: vec![sdk_bin.clone()], - library_paths: vec![sdk_bin], - resolved_libraries: vec![ - therock::RocmSdkLibraryProbe { - shortname: "amdhip64".to_owned(), - paths: vec![amdhip], - }, - therock::RocmSdkLibraryProbe { - shortname: "hipblas".to_owned(), - paths: vec![hipblas], - }, - ], - resolved_target_family: Some("gfx120X-all".to_owned()), - ..therock::RocmSdkPythonProbe::default() - }, - )?; + let request = AdoptRuntimeRequest { + python_executable, + install_root: external_root.clone(), + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), + replace, + }; + let probe = therock::RocmSdkPythonProbe { + import_ok: true, + rocm_sdk_version: Some("7.13.0".to_owned()), + root_path: Some(sdk_root.clone()), + bin_path: Some(sdk_bin.clone()), + runtime_roots: vec![sdk_root], + bin_paths: vec![sdk_bin.clone()], + library_paths: vec![sdk_bin], + resolved_libraries: vec![ + therock::RocmSdkLibraryProbe { + shortname: "amdhip64".to_owned(), + paths: vec![amdhip], + }, + therock::RocmSdkLibraryProbe { + shortname: "hipblas".to_owned(), + paths: vec![hipblas], + }, + ], + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }; + Ok((external_root, request, probe)) + } + + #[test] + fn runtime_adopt_records_read_only_manifest_from_probe() -> Result<()> { + let (root, paths) = test_paths("runtime-adopt"); + let (external_root, request, probe) = adoptable_environment(&root, false)?; + + let adopted = adopt_runtime_from_probe(&paths, request, probe)?; assert!(adopted.read_only); assert_eq!(adopted.channel, "release"); @@ -2407,6 +2795,310 @@ pub(crate) mod tests { Ok(()) } + /// Which of the two files a registered runtime can have exist: its registry + /// entry, and the `.rocm-cli-runtime.json` ROCm CLI keeps in the install + /// root as proof that it owns the folder. + fn registered_files(paths: &AppPaths, install_root: &Path, runtime_key: &str) -> (bool, bool) { + ( + runtime_manifest_path(paths, runtime_key).is_file(), + install_root.join(".rocm-cli-runtime.json").is_file(), + ) + } + + /// A runtime ROCm CLI installed itself, with both of its manifest files + /// deleted so a test sees only what the write under test creates. + fn installed_runtime_without_manifest_files( + paths: &AppPaths, + ) -> Result { + let manifest = write_test_pip_runtime( + paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + fs::remove_file(runtime_manifest_path(paths, &manifest.runtime_key))?; + fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; + assert_eq!( + registered_files(paths, &manifest.install_root, &manifest.runtime_key), + (false, false) + ); + Ok(manifest) + } + + /// Registering a runtime whose registry entry exists is refused until the + /// caller asks to replace it. `register(replace)` makes the attempt; the + /// message and the entry on disk are asserted together, and the same call + /// with `replace` set is what clears the refusal. + fn assert_duplicate_registration_needs_replace( + paths: &AppPaths, + runtime_key: &str, + register: impl Fn(bool) -> Result<()>, + ) -> Result<()> { + let entry = runtime_manifest_path(paths, runtime_key); + fs::write(&entry, "stale entry")?; + + let error = register(false).unwrap_err().to_string(); + + assert!( + error.contains("already exists") && error.contains("--replace"), + "{error}" + ); + assert_eq!( + fs::read_to_string(&entry)?, + "stale entry", + "a refused registration must leave the existing entry alone" + ); + register(true)?; + let replaced: therock::InstalledRuntimeManifest = + serde_json::from_slice(&fs::read(&entry)?)?; + assert_eq!(replaced.runtime_key, runtime_key); + Ok(()) + } + + /// Installing writes both files, and a second install of the same key + /// overwrites both without being asked: it is the same runtime again. + #[test] + fn runtime_install_writes_the_registry_entry_and_the_install_root_manifest() -> Result<()> { + let (root, paths) = test_paths("runtime-files-install"); + let manifest = installed_runtime_without_manifest_files(&paths)?; + + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; + + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, true) + ); + let reinstalled = therock::InstalledRuntimeManifest { + version: "7.14.0".to_owned(), + ..manifest.clone() + }; + write_runtime_manifest(&paths, &reinstalled, ManifestOverwrite::Replace)?; + for path in [ + runtime_manifest_path(&paths, &manifest.runtime_key), + manifest.install_root.join(".rocm-cli-runtime.json"), + ] { + let written: therock::InstalledRuntimeManifest = + serde_json::from_slice(&fs::read(&path)?)?; + assert_eq!(written.version, "7.14.0", "{}", path.display()); + } + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Importing registers a folder somebody else owns, so it writes the + /// registry entry and nothing into that folder, and refuses to replace an + /// existing entry unless asked. + #[test] + fn runtime_import_writes_the_registry_entry_only_and_refuses_a_duplicate() -> Result<()> { + let (root, paths) = test_paths("runtime-files-import"); + let manifest = installed_runtime_without_manifest_files(&paths)?; + let exported_manifest = root.join("external-runtime.json"); + fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; + + import_runtime_manifest(&paths, &exported_manifest, false)?; + + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, false) + ); + assert_duplicate_registration_needs_replace(&paths, &manifest.runtime_key, |replace| { + import_runtime_manifest(&paths, &exported_manifest, replace).map(drop) + })?; + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, false), + "replacing an import must not start writing into the folder either" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Adopting has the same contract as importing: registry entry only, and a + /// duplicate needs `--replace`. + #[test] + fn runtime_adopt_writes_the_registry_entry_only_and_refuses_a_duplicate() -> Result<()> { + let (root, paths) = test_paths("runtime-files-adopt"); + let (external_root, request, probe) = adoptable_environment(&root, false)?; + let runtime_key = request.runtime_key.clone(); + + adopt_runtime_from_probe(&paths, request.clone(), probe.clone())?; + + assert_eq!( + registered_files(&paths, &external_root, &runtime_key), + (true, false) + ); + assert_duplicate_registration_needs_replace(&paths, &runtime_key, |replace| { + let request = AdoptRuntimeRequest { + replace, + ..request.clone() + }; + adopt_runtime_from_probe(&paths, request, probe.clone()).map(drop) + })?; + assert_eq!( + registered_files(&paths, &external_root, &runtime_key), + (true, false), + "replacing an adoption must not start writing into the environment either" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Recovering the runtime `rocm setup` created restores its registry entry + /// from the manifest in its folder, leaves that manifest in place, and does + /// not replace an entry that is already there. + #[test] + fn setup_recovery_restores_the_registry_entry_and_keeps_the_install_root_manifest() -> Result<()> + { + let (root, paths) = test_paths("runtime-files-recovery"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + config.setup.therock_venv = Some(manifest.install_root.clone()); + let entry = runtime_manifest_path(&paths, &manifest.runtime_key); + fs::remove_file(&entry)?; + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (false, true) + ); + + let recovered = recover_setup_runtime_registration(&paths, &config)?; + + assert_eq!(recovered.as_deref(), Some(manifest.runtime_key.as_str())); + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, true) + ); + fs::write(&entry, "entry written since")?; + recover_setup_runtime_registration(&paths, &config)?; + assert_eq!( + fs::read_to_string(&entry)?, + "entry written since", + "recovery only fills a missing entry" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Rewrite a manifest file as an older rocm-cli wrote it: without the keys + /// that hold no value, which a newer binary reads back as their defaults and + /// writes out as explicit `null`s. Returns the bytes now on disk. + fn rewrite_as_legacy_shaped(path: &Path) -> Result> { + let mut document: serde_json::Value = serde_json::from_slice(&fs::read(path)?)?; + let object = document + .as_object_mut() + .context("a manifest is a JSON object")?; + let empty: Vec = object + .iter() + .filter(|(_, value)| value.is_null()) + .map(|(key, _)| key.clone()) + .collect(); + assert!( + !empty.is_empty(), + "the fixture must hold a null-valued key for this to mean anything" + ); + for key in empty { + object.remove(&key); + } + let legacy = serde_json::to_vec_pretty(&document)?; + fs::write(path, &legacy)?; + Ok(legacy) + } + + /// Recovering the setup runtime from a manifest an older binary wrote, in a + /// folder the user cannot write to: the file decodes to the very manifest the + /// recovery would write, so it is already current and is left alone. Judging + /// it by bytes would read the missing keys as a change, write the registry + /// entry, and then fail on the read-only file. The registry entry and the + /// file's bytes are asserted together. (Where the file stays writable + /// regardless, as for an administrator, the unchanged bytes still prove the + /// file was not rewritten.) + #[test] + #[allow(clippy::permissions_set_readonly_false)] + fn setup_recovery_leaves_a_read_only_legacy_shaped_install_root_manifest_alone() -> Result<()> { + let (root, paths) = test_paths("runtime-files-recovery-legacy"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + config.setup.therock_venv = Some(manifest.install_root.clone()); + let entry = runtime_manifest_path(&paths, &manifest.runtime_key); + let local = manifest.install_root.join(".rocm-cli-runtime.json"); + fs::remove_file(&entry)?; + let legacy = rewrite_as_legacy_shaped(&local)?; + let mut permissions = fs::metadata(&local)?.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&local, permissions.clone())?; + + let recovered = recover_setup_runtime_registration(&paths, &config); + + permissions.set_readonly(false); + fs::set_permissions(&local, permissions)?; + let recovered = recovered + .context("a usable manifest in a folder that cannot be written is recovered")?; + assert_eq!(recovered.as_deref(), Some(manifest.runtime_key.as_str())); + assert!(entry.is_file(), "the registry entry is restored"); + assert_eq!( + fs::read(&local)?, + legacy, + "the install root manifest is already current and must not be rewritten" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The writer gives a runtime a manifest in its folder exactly when + /// uninstall, which prune asks the same question of, may delete that folder. + /// The two are written in different modules; this is what ties them. + #[test] + fn a_runtime_gets_an_install_root_manifest_exactly_when_uninstall_may_remove_its_folder() + -> Result<()> { + let (root, paths) = test_paths("runtime-files-ownership"); + for (label, read_only, imported) in [ + ("owned", false, false), + ("read-only", true, false), + ("imported", false, true), + ] { + let mut manifest = write_test_pip_runtime( + &paths, + &format!("release-pip-{label}"), + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + manifest.read_only = read_only; + manifest.imported_from = imported.then(|| PathBuf::from("elsewhere/manifest.json")); + fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; + + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; + + let has_manifest = manifest + .install_root + .join(".rocm-cli-runtime.json") + .is_file(); + let may_remove = + should_remove_runtime_install_root(&manifest)? == InstallRootDecision::Remove; + assert_eq!(has_manifest, may_remove, "{label}"); + assert_eq!(has_manifest, manifest.owns_install_root(), "{label}"); + } + + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn runtime_uninstall_removes_managed_root_and_clears_active_state() -> Result<()> { let (root, paths) = test_paths("runtime-uninstall-managed"); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 9f5895d3e3..2336899be7 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -28,16 +28,16 @@ //! `parse_device_policy`, `parse_gpu_selection`, //! `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, //! and `validate_pinned_gpu_index` moved out of `main.rs` into -//! `gpu_policy.rs` in ROCMAI-214 (Phase 6f); `resolve_engine_selection` and +//! `gpu_policy.rs`; `resolve_engine_selection` and //! `validate_engine_selection_runtime` moved out of `main.rs` into -//! `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f); this file's `use` +//! `engine_runtime_select.rs`; this file's `use` //! list below points to both now instead of at the crate root. The //! managed-service-spawning tail this file calls into — `start_managed_service`, //! `run_attached_service`, `resolve_endpoint_auth`, //! `ensure_public_bind_engine_supported`, //! `drop_orphaned_endpoint_key_on_already_running`, `print_managed_launch_plain`, //! `validate_bind_host`, `ensure_background_helper_running_quiet` — was relocated -//! to `service_launch.rs` in ROCMAI-91 (Phase 6b), along with its test module; +//! to `service_launch.rs`, along with its test module; //! this file's `use` list below points there now instead of at the crate root. use anyhow::{Context as _, Result, bail}; @@ -74,6 +74,7 @@ use crate::serve_recipe::{ parse_gpu_memory_utilization, protocol_engine_recipe_hint, render_serve_engine_recipe_lines, }; use crate::serve_summary; +use crate::service_control::run_internal_sandbox_tool; use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, e2e_simulate_oom_launch, ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, @@ -82,8 +83,8 @@ use crate::service_launch::{ wait_begin_client_config, }; use crate::{ - GatedEngineAction, cli_progress, device_policy_name, ensure_engine_allowed_on_os, - ensure_engine_allowed_on_platform, + GatedEngineAction, SandboxToolArg, cli_progress, device_policy_name, + ensure_engine_allowed_on_os, ensure_engine_allowed_on_platform, }; #[derive(Debug, Clone, Eq, PartialEq)] @@ -865,8 +866,17 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // test reaches Lemonade's backend boundary without real GPU hardware, and the // E2E-only OOM-launch fault injection likewise stands in for the GPU it does // not have. + // + // The scripted free-space override opens the same waiver: it exists to refuse a + // Lemonade install on a drive that cannot hold it, which `rocm serve` only + // reaches past this bail. let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") - && std::env::var_os("ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE").is_some(); + && [ + "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", + "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES", + ] + .iter() + .any(|variable| std::env::var_os(variable).is_some()); if !cpu_only && !scripted_backend_failure && !e2e_simulate_oom_launch() @@ -1249,6 +1259,24 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { engine_serves_vllm, ); notes.extend(engine_env_notes.iter().cloned()); + // Published by the serving process shortly after the engine lists its + // model. A service that was already running published it long ago, so + // only a fresh launch is waited for. + let anthropic_endpoint = rocm_core::wait_for_published_anthropic_endpoint( + &paths.service_engine_state_path(&selected_engine, &report.service_id), + if report.already_running || report.status != "ready" { + std::time::Duration::ZERO + } else { + std::time::Duration::from_secs(5) + }, + ); + if anthropic_endpoint.is_none() && !report.already_running && report.status == "ready" { + notes.push( + "the Anthropic-compatible endpoint was not published yet; it appears as \ + `anthropic endpoint:` in `rocm services list` once the engine lists its model" + .to_owned(), + ); + } // A managed serve that failed on VRAM lands here not-ready with the OOM // traceback only in its own log. Read that log tail so the summary can // name the memory knobs, since the pre-launch low-VRAM warning cannot @@ -1281,6 +1309,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { requested_model: model, api_model: resolve.served_model_name().to_owned(), chat_endpoint: format!("{}/chat/completions", report.endpoint_url), + anthropic_endpoint, service_id: report.service_id.clone(), status: report.status.clone(), already_running: report.already_running, @@ -1289,7 +1318,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { notes, }; // Same intentional one-time key display as `print_managed_launch_plain` - // in service_launch.rs; see its rationale for why the tag below is currently inert. + // in service_launch/start.rs; see its rationale for why the tag below is + // currently inert. // codeql[rust/cleartext-logging] print!("{}", serve_summary::render_summary(&summary)); } else { @@ -1314,6 +1344,14 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { launch_lock, require_api_key, gpu_refusal_waived_for_reuse, + |paths, service_id| { + run_internal_sandbox_tool( + paths, + SandboxToolArg::StopServer, + Some(service_id.to_owned()), + true, + ) + }, ) } @@ -1625,6 +1663,94 @@ mod tests { ); } + #[test] + fn serve_small_assistant_resolves_the_gguf_build_on_lemonade() { + // The small assistant recipe is published as safetensors under its + // canonical id and as GGUF under a companion repo. Lemonade can only + // serve the latter, and only with a quantization variant pinned, so + // every way of naming the recipe has to arrive at that ref. + for model in [ + "qwen2.5", + "qwen2.5-1.5b", + "qwen2.5-1.5b-instruct", + "qwen-small", + "Qwen/Qwen2.5-1.5B-Instruct", + ] { + let recipe = resolve_builtin_model_recipe(model) + .unwrap_or_else(|| panic!("{model} resolves to a recipe")); + // With no engine named and no GPU family preferring one, automatic + // selection follows the recipe. A host whose family prefers vLLM + // takes the earlier branch in `select_serve_engine` and is not + // covered here; the ref each engine resolves to is, below. + assert_eq!( + select_serve_engine(None, None, Some(&recipe), None), + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + }, + "{model}" + ); + assert_eq!( + serve_model_ref_for_engine(model, Some(&recipe), "lemonade"), + "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "{model}" + ); + // vLLM keeps the safetensors repo the canonical id names. + assert_eq!( + serve_model_ref_for_engine(model, Some(&recipe), "vllm"), + "Qwen/Qwen2.5-1.5B-Instruct", + "{model}" + ); + } + } + + #[test] + fn every_lemonade_catalog_recipe_resolves_to_a_servable_ref() { + // Lemonade passes a Hugging Face `owner/repo` through untouched, and the + // engine then refuses it without a `:variant` -- on Windows from inside + // the detached engine process, where the user never sees the reason. The + // catalog is what has to supply the variant, so no recipe that lists + // Lemonade may resolve to a bare `owner/repo`. A bare registry name (no + // `/`) is a Lemonade model id, not a Hugging Face repo, and is fine. + // + // Scope: recipes that list Lemonade, and only those. Lemonade can still + // be reached for a recipe that does not -- an explicit `--engine + // lemonade`, a configured `default_engine`, or the platform default when + // nothing else fits -- and that recipe is handed its canonical id, which + // is what a recipe with no Lemonade build should get. Refusing a bare + // ref before the engine starts is the other half of that, and is not + // this test's job. + let offenders = rocm_core::builtin_model_recipes() + .into_iter() + .filter(|recipe| model_recipe_supports_engine(recipe, "lemonade")) + .map(|recipe| { + let resolved = serve_model_ref_for_engine( + &recipe.canonical_model_id, + Some(&recipe), + "lemonade", + ); + (recipe.canonical_model_id, resolved) + }) + .filter(|(_, resolved)| { + // A trailing `:` is as unservable as no colon at all, so the + // variant has to be there, not merely the separator. + resolved.contains('/') + && resolved + .rsplit('/') + .next() + .and_then(|name| name.split_once(':')) + .is_none_or(|(_, variant)| variant.trim().is_empty()) + }) + .collect::>(); + assert!( + offenders.is_empty(), + "these recipes hand Lemonade a Hugging Face repo with no `:variant`, which the \ + engine refuses after it has already been launched; give each one a lemonade \ + engine_recipes entry whose model_id_override names the GGUF repo and variant: \ + {offenders:?}" + ); + } + #[test] fn serve_model_ref_for_engine_pins_the_shipped_aim_image_to_its_digest() { let recipe = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("aim catalog entry"); diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 2f501506fc..0b4a509d70 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -233,9 +233,9 @@ pub(crate) fn resolve_serve_port( if let Some(owner) = reserved.iter().find(|entry| entry.port == port) { // The "stop it" advice below is proven, not just worded: see // `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` - // in `service_launch.rs`, which recreates the exact on-disk status - // transition `rocm services stop` performs and confirms the same - // explicit port then resolves. A full CLI-level E2E for this needs + // in `service_launch/launch_transaction.rs`, which recreates the exact + // on-disk status transition `rocm services stop` performs and confirms + // the same explicit port then resolves. A full CLI-level E2E for this needs // a real GPU, since reaching this branch requires a completed // model/engine resolution first. bail!("{}", reserved_message(DEFAULT_LOCAL_HOST, port, owner)); diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 73db2a4b0d..cd0ab34bc6 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -53,6 +53,9 @@ pub(crate) struct DeploymentSummary { pub api_model: String, /// Full chat-completions endpoint, e.g. `http://127.0.0.1:1337/v1/chat/completions`. pub chat_endpoint: String, + /// Base URL of the Anthropic-compatible endpoint (clients append + /// `/v1/messages`), when the service has published one. + pub anthropic_endpoint: Option, pub service_id: String, /// `"ready"`, `"starting"`, or an existing service's status. pub status: String, @@ -122,14 +125,20 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ] }; + let anthropic_rows: Vec<(&str, String)> = summary + .anthropic_endpoint + .iter() + .map(|url| ("anthropic endpoint", url.clone())) + .collect(); let rows: Vec<(&str, String)> = [ vec![ ("status", summary.status.clone()), ("engine", summary.engine.clone()), ], model_rows, + vec![("endpoint", summary.chat_endpoint.clone())], + anthropic_rows, vec![ - ("endpoint", summary.chat_endpoint.clone()), ("time to first token", format_ttft(summary.metrics.ttft)), ("throughput (approx)", format_tps(summary.metrics.gen_tps)), ("service", summary.service_id.clone()), @@ -449,6 +458,7 @@ mod tests { requested_model: "Qwen/Qwen2.5-7B-Instruct".to_owned(), api_model: "Qwen/Qwen2.5-7B-Instruct".to_owned(), chat_endpoint: "http://127.0.0.1:1337/v1/chat/completions".to_owned(), + anthropic_endpoint: None, service_id: "vllm-qwen-1720000000".to_owned(), status: "ready".to_owned(), already_running: false, @@ -476,6 +486,32 @@ mod tests { assert_eq!(format_tps(Some(f64::INFINITY)), "n/a"); } + #[test] + fn summary_lists_the_anthropic_endpoint_beside_the_openai_one_when_published() { + let mut summary = base_summary(); + summary.anthropic_endpoint = Some("http://127.0.0.1:41000".to_owned()); + let rendered = render_summary(&summary); + let endpoint = rendered + .find("http://127.0.0.1:1337/v1/chat/completions") + .unwrap(); + let anthropic = rendered.lines().any(|line| { + line.trim_start().starts_with("anthropic endpoint") + && line.ends_with("http://127.0.0.1:41000") + }); + assert!( + anthropic, + "no `anthropic endpoint` row ending in the published URL" + ); + let anthropic = rendered.find("anthropic endpoint").unwrap(); + assert!( + endpoint < anthropic, + "the anthropic row must follow the endpoint row" + ); + // Nothing published (the service is not ready, or the wait ran out): + // no row rather than a made-up one. + assert!(!render_summary(&base_summary()).contains("anthropic")); + } + #[test] fn summary_shows_endpoint_model_and_metrics() { let rendered = render_summary(&base_summary()); diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 7bc72177c3..d2e8410696 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -8,9 +8,9 @@ //! Mechanically relocated from `services_text.rs` with no behavior change. //! //! This module owns: the sandbox-tool runner (`run_internal_sandbox_tool`), the -//! stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` -//! constant, the private `RecordedPidOutcome`/`RecordedStopTermination` types -//! and the `scripted_stop_outcome` end-to-end hook, `stop_internal_managed_service`, +//! stop path (`stop_recorded_processes`, which hands the recorded PIDs to the +//! routine `rocm-core` shares with `rocmd`, the `scripted_stop_outcome` +//! end-to-end hook, `stop_internal_managed_service`, //! `stop_internal_managed_service_with`, //! `stop_internal_managed_service_confirmed_with`, //! `unload_lemonade_service_model`), and the restart path @@ -22,7 +22,9 @@ //! Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are //! `pub(crate)`. `run_internal_sandbox_tool` is read from `main.rs`'s //! `dispatch()`, `services_cmd.rs`'s `run_approved_service_action` and -//! `service_launch.rs`; `restart_internal_managed_service` from +//! `serve_cmd.rs`, which hands it to `service_launch.rs`'s +//! `run_attached_service` as the stop for a Ctrl-C; +//! `restart_internal_managed_service` from //! `runtime_services.rs`. Every other item has no caller outside this module and //! stays private. //! @@ -30,9 +32,9 @@ //! (`load_managed_service`, `read_managed_service_manifest`) and shapes the //! `stop_server` answer with its `internal_stop_server_value`, which stays there //! next to `service_stop_unconfirmed` and the renderer that reads the same -//! `stop_unconfirmed` status. `service_launch.rs` and this module import each -//! other: this module takes `ensure_public_service_has_endpoint_key` and the -//! spawn helpers from it, and it takes `run_internal_sandbox_tool` from here. +//! `stop_unconfirmed` status. This module takes +//! `ensure_public_service_has_endpoint_key` and the spawn helpers from +//! `service_launch.rs`, which does not import this module back. //! //! The regression tests for the stop and restart paths live in this module's own //! `mod tests`. They reach shared fixtures (`test_paths`, `ScopedTestEnv`, @@ -54,8 +56,6 @@ use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use crate::endpoint_keys; use crate::engine_ipc::{builtin_engine_serve_http_args, engine_request}; -#[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; use crate::engine_runtime_select::{ @@ -124,119 +124,14 @@ pub(crate) fn run_internal_sandbox_tool( })) } -/// How long a managed stop waits for each recorded process to actually exit -/// after `SIGTERM` before escalating to `SIGKILL` and then reporting a timeout. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// One recorded PID's fate in a managed stop: the role the record gave it, and -/// what [`rocm_core::terminate_verified`] did about it. Kept per PID because the -/// outcomes that leave a stop unconfirmed are reported differently: a -/// `TimedOut` PID was signalled and it or a process under it outlived the -/// deadline, while an `Unverified` one is not signalled by this stop, and the -/// report names a command to check on it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct RecordedPidOutcome { - pid: u32, - /// `supervisor`, `engine`, or `supervisor_and_engine` when the record names - /// one PID for both — the same labels `rocmd`'s `pid_outcomes` uses. - role: &'static str, - outcome: rocm_core::TerminationOutcome, -} - -/// What a managed stop did to the processes its record names. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(crate) struct RecordedStopTermination { - /// Every PID that received a signal, including one that then outlived the - /// deadline. - signaled_pids: Vec, - /// Whether every recorded process is confirmed no longer running. - all_stopped: bool, - pid_outcomes: Vec, -} - -impl RecordedStopTermination { - /// A stop stand-in that confirms every recorded process gone without - /// signalling any, for tests in other modules that drive the real stop path. - #[cfg(test)] - pub(crate) fn confirmed_stopped() -> Self { - Self { - all_stopped: true, - ..Self::default() - } - } -} - -/// Terminate the processes recorded for a managed service, verifying each PID's -/// identity first so a recycled PID never causes an unrelated process tree to be -/// killed. -/// -/// The supervisor (launcher) and the engine server are distinct processes, so -/// each PID is paired with **its own** start-time token (`supervisor_start_ticks` -/// vs `engine_start_ticks`). When they resolve to the same PID (e.g. before the -/// engine state has been refreshed) the entry is de-duplicated, preferring a -/// known token. Returns the PIDs actually signalled, whether every recorded -/// process is confirmed no longer running, and each PID's own outcome. -fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> RecordedStopTermination { - // Build the (pid, own-token, role) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. - let mut entries: Vec<(u32, Option, &'static str)> = Vec::new(); - for (pid, ticks, role) in [ - ( - Some(record.supervisor_pid), - record.supervisor_start_ticks, - "supervisor", - ), - (record.engine_pid, record.engine_start_ticks, "engine"), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - if existing.1.is_none() { - existing.1 = ticks; - } - existing.2 = "supervisor_and_engine"; - } else { - entries.push((pid, ticks, role)); - } - } - - let mut termination = RecordedStopTermination { - all_stopped: true, - ..RecordedStopTermination::default() - }; - for (pid, ticks, role) in entries { - let identity = rocm_core::ProcessIdentity::new(pid, ticks); - // The managed stop contract is definitive: after the adapter gets its - // graceful opportunity, this cleanup pass forces any verified survivor - // so the command does not report success while a GPU worker remains. - let outcome = scripted_stop_outcome().unwrap_or_else(|| { - rocm_core::terminate_verified( - &identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) - }); - if !outcome.stopped() { - termination.all_stopped = false; - } - // Report every PID that received a signal. TimedOut belongs here because - // signalling was attempted even though the process was not confirmed - // stopped; `all_stopped` separately carries the truthful completion state. - if matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ) { - termination.signaled_pids.push(pid); - } - termination - .pid_outcomes - .push(RecordedPidOutcome { pid, role, outcome }); - } - termination +/// Terminate the processes recorded for a managed service through the routine +/// `rocmd` shares, [`rocm_core::terminate_recorded_service_pids`], with +/// [`rocm_core::terminate_recorded_pid`] as the termination of each PID. The +/// one thing this adds is the e2e seam below. +fn stop_recorded_processes(record: &ManagedServiceRecord) -> Vec { + rocm_core::terminate_recorded_service_pids(record, |identity| { + scripted_stop_outcome().unwrap_or_else(|| rocm_core::terminate_recorded_pid(identity)) + }) } /// E2E hook: when `ROCM_E2E_STOP_OUTCOME` is `timed_out` or `unverified`, a @@ -245,7 +140,8 @@ fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> RecordedSto /// /// Neither outcome can be staged from outside the binary: `TimedOut` needs a /// process that survives `SIGKILL`, and `Unverified` one whose recorded -/// start-time cannot be read back. Without this seam the "stop not confirmed" +/// start-time cannot be read back, or (on Windows) that cannot be queried at +/// all. Without this seam the "stop not confirmed" /// report, its exit status and what it keeps on disk have no live coverage. /// The recorded process is left untouched. Compiled only into the /// `e2e-test-hooks` binary; production builds never see it. @@ -264,12 +160,12 @@ const fn scripted_stop_outcome() -> Option { } fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_internal_managed_service_with(paths, service_id, terminate_recorded_service_pids) + stop_internal_managed_service_with(paths, service_id, stop_recorded_processes) } /// [`stop_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller. Production passes -/// [`terminate_recorded_service_pids`]; a test passes a stand-in that observes +/// [`stop_recorded_processes`]; a test passes a stand-in that observes /// what is on disk at the moment the processes would be signalled. Everything /// else — the engine stop, the manifest writes, the key cleanup — runs for real. /// @@ -278,7 +174,7 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { stop_internal_managed_service_confirmed_with(paths, service_id, terminate) .map(|(report, _)| report) @@ -292,7 +188,7 @@ pub(crate) fn stop_internal_managed_service_with( fn stop_internal_managed_service_confirmed_with( paths: &AppPaths, service_id: &str, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result<(serde_json::Value, bool)> { let mut stopping = load_managed_service(paths, service_id)?; // Record the request on disk before anything is stopped, not only after. @@ -320,11 +216,17 @@ fn stop_internal_managed_service_confirmed_with( }, ) }; - let RecordedStopTermination { - signaled_pids, - all_stopped: terminated_all, - pid_outcomes, - } = terminate(&stopping); + let stops = terminate(&stopping); + // Every PID that received a signal, including one that then outlived the + // deadline: signalling was attempted even though the process was not + // confirmed stopped, and `terminated_all` separately carries the truthful + // completion state. + let signaled_pids: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); + let terminated_all = stops.iter().all(|stop| stop.stopped()); // Re-read the record rather than write back the copy loaded before the // stop, which can be some forty seconds old by now: `write` replaces the // whole file, and a supervisor, the daemon's restart or another stop may @@ -348,11 +250,20 @@ fn stop_internal_managed_service_confirmed_with( // supervisor that follows it, so a failed `Stop` leaves the stop unconfirmed. let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.is_ok(); - let record_changed = !stopping.names_same_processes(&record); + let record_changed = !record.is_covered_by_stop_of(&stopping); let all_stopped = terminated_all && !record_changed && engine_stop_confirms; if all_stopped { record.status = "stopped".to_owned(); record.stop_requested_unix_ms = None; + // The processes this stop confirmed gone are not the service's any more, + // and their PIDs are stale the instant they exited. Left in the record, + // a later stop would signal them again: a PID recorded without a + // start-time token (every one a Windows record had before it recorded + // a creation time) cannot be refuted, so if the + // OS had given it to another process the Windows tree kill would take + // that process and its subtree. The same call, on the same confirmed + // path, as `rocmd`'s stop. + record.clear_recorded_processes(); } else { // The marker records that a stop was *asked for* even though it could // not be confirmed. `refresh_managed_service_runtime_liveness` needs it @@ -399,15 +310,9 @@ fn stop_internal_managed_service_confirmed_with( "status": record.status, "engine_stop": engine_stop, "signaled_pids": signaled_pids, - "pid_outcomes": pid_outcomes + "pid_outcomes": stops .iter() - .map(|entry| { - serde_json::json!({ - "pid": entry.pid, - "role": entry.role, - "outcome": entry.outcome.as_str(), - }) - }) + .map(|stop| stop.report_entry()) .collect::>(), "record_changed": record_changed, // A container engine is confirmed stopped by its own `Stop`; when that @@ -515,12 +420,7 @@ pub(crate) fn restart_internal_managed_service( service_id: &str, runtime_pin: Option<&str>, ) -> Result { - restart_internal_managed_service_with( - paths, - service_id, - runtime_pin, - terminate_recorded_service_pids, - ) + restart_internal_managed_service_with(paths, service_id, runtime_pin, stop_recorded_processes) } /// [`restart_internal_managed_service`], with the termination of the recorded @@ -536,7 +436,7 @@ fn restart_internal_managed_service_with( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { restart_internal_managed_service_inner(paths, service_id, runtime_pin, terminate, None) } @@ -571,7 +471,7 @@ fn restart_internal_managed_service_inner( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, spawn_override: Option<&RestartSpawnOverride<'_>>, ) -> Result { #[cfg(windows)] @@ -606,6 +506,26 @@ fn restart_internal_managed_service_inner( stop_report["signaled_pids"] ); } + // Publish that a relaunch has begun before anything is restored or spawned. + // From here until `record_restarted_child` the record is an empty `stopped` + // manifest, which an older stop still finalizing would take for its own + // completion and delete the restored key under; the generation stamp is what + // keeps `is_covered_by_stop_of` from covering it. Applied to the record as it + // is on disk now, guarded on it still being what the confirmed stop left. + let (confirmed_record, _) = read_managed_service_manifest(paths, service_id)?; + if update_restarting_record( + paths, + &confirmed_record, + ManagedServiceRecord::begin_relaunch, + )? + .is_none() + { + bail!( + "`{service_id}` was stopped but NOT restarted: it changed on disk while the \ + restart was beginning; run `{}` to try again", + service_hints::service_action_hint("restart", service_id) + ); + } if let Some(key) = preserved_endpoint_key.as_deref() { endpoint_keys::store_endpoint_api_key(paths, service_id, key)?; } @@ -713,7 +633,7 @@ fn restart_internal_managed_service_inner( command.stdin(Stdio::null()); attach_background_stdio(&mut command, Some(&record.log_path))?; detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); + paths.apply_env(&mut command); if let Some(engine_envs_root) = engine_envs_root.as_deref() { command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); } @@ -770,8 +690,8 @@ fn restart_internal_managed_service_inner( /// against for the supervisor's writes. /// /// Nothing is written, and `None` returned, when the record no longer reads -/// as `expected` left it — it is gone, names other processes, or has another -/// status: whoever wrote since holds a truer view than this restart. `update` +/// as `expected` left it — it is gone, names other processes, has another +/// status, or is in another relaunch generation: whoever wrote since holds a truer view than this restart. `update` /// never touches the stop marker, so a stop asked for meanwhile survives. fn update_restarting_record( paths: &AppPaths, @@ -782,7 +702,10 @@ fn update_restarting_record( return Ok(None); } let (mut current, _) = read_managed_service_manifest(paths, &expected.service_id)?; - if !current.names_same_processes(expected) || current.status != expected.status { + if !current.names_same_processes(expected) + || current.status != expected.status + || current.relaunch_started_unix_ms != expected.relaunch_started_unix_ms + { return Ok(None); } update(&mut current); @@ -1073,7 +996,7 @@ mod tests { // Proves the *ordering*, not just the refusal: had the guard run after // `stop_internal_managed_service`, the stop would have written // "stopped". It reaches that state here because the record's only pid is - // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` + // the test's own (`engine_pid` is None and the shared stop routine // skips the caller's pid), so the stop confirms termination trivially. let after = load_managed_service(&paths, service_id).unwrap(); assert_ne!( @@ -1216,6 +1139,275 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); } + /// A liveness refresh that lands after the stop's processes have exited but + /// before the stop reads the record again completes the same stop (key, + /// marker, recorded processes cleared, `stopped`) and writes it. That record + /// names no process, which must not read as a restart: the stop is confirmed, + /// the marker is not put back, and a restart is not refused for it. A record + /// that read `failed` is completed to `stopped` too, as a confirmed stop makes + /// it, so it is covered as well and not left reading `failed` with no process. + #[test] + fn a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it() { + use rocm_core::TerminationOutcome::Graceful; + for status in ["ready", "failed"] { + let (root, paths) = test_paths(&format!("stop-listing-before-finalization-{status}")); + paths.ensure().unwrap(); + let service_id = "svc-stop-listing-before-finalization"; + let mut planted = stop_test_record(&paths, service_id); + planted.status = status.to_owned(); + planted.supervisor_pid = 4242; + planted.supervisor_start_ticks = None; + planted.engine_pid = Some(4243); + planted.engine_start_ticks = Some(77); + planted.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { + assert!(stopping.stop_requested_unix_ms.is_some()); + // The processes have exited; a concurrent `services` listing sees + // it, completes the pending stop and persists that, before this + // stop reads the record again. + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + // The fake PIDs read as gone to the stop's own initial refresh, + // which may already have demoted the status: put back the status + // the listing under test starts from. + status.clone_into(&mut listed.status); + let changed = + crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!( + changed, + "premise: the listing completed the stop ({status})" + ); + listed.write().unwrap(); + assert_eq!(raw_manifest(&paths, service_id).supervisor_pid, 0); + vec![stop_of(4242, Graceful), stop_of(4243, Graceful)] + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], true, "{status}: {report}"); + assert_eq!(on_disk.status, "stopped", "{status}"); + assert_eq!( + on_disk.stop_requested_unix_ms, None, + "{status}: a confirmed stop does not put the marker back" + ); + assert_eq!(on_disk.supervisor_pid, 0, "{status}"); + assert_eq!(key, None, "{status}: the key goes with the confirmed stop"); + } + } + + /// The same race through `restart`: the restart's own stop is completed by a + /// listing before it finalizes, and the restart must carry on past the stop + /// (here to the launch lock, which a holder keeps, so it stops there) + /// instead of refusing with "could not confirm". + #[test] + fn a_restart_is_not_refused_because_a_listing_completed_its_stop() -> Result<()> { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("restart-listing-before-finalization"); + paths.ensure()?; + let service_id = "svc-restart-listing-before-finalization"; + stop_test_record(&paths, service_id).write()?; + let holder = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + LAUNCH_LOCK_TIMEOUT_OVERRIDE.with(|cell| cell.set(Some(Duration::from_millis(100)))); + let result = restart_internal_managed_service_with(&paths, service_id, None, |_| { + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + listed.write().unwrap(); + vec![stop_of(999_999_999, Graceful)] + }); + LAUNCH_LOCK_TIMEOUT_OVERRIDE.with(|cell| cell.set(None)); + drop(holder); + let _ = fs::remove_dir_all(root); + + let message = format!( + "{:#}", + result.expect_err("the held launch lock fails the restart") + ); + assert!( + !message.contains("could not confirm"), + "the stop was completed, the restart must not be refused for it: {message}" + ); + assert!(message.contains("stopped but NOT restarted"), "{message}"); + Ok(()) + } + + /// The other half of the same check: a restart that has begun launching but + /// has recorded no process yet (status `starting`, nothing named) is not the + /// stop completing, and the stop must not claim it or drop the key it will + /// serve with. + #[test] + fn stop_does_not_confirm_over_a_restart_that_has_recorded_no_process_yet() { + let (root, paths) = test_paths("stop-restart-in-flight"); + paths.ensure().unwrap(); + let service_id = "svc-stop-restart-in-flight"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let mut restarting = raw_manifest(&paths, service_id); + restarting.status = "starting".to_owned(); + restarting.clear_recorded_processes(); + restarting.stop_requested_unix_ms = None; + restarting.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["status"], "starting", "{report}"); + assert_eq!(on_disk.status, "starting", "the restart's status stays"); + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + } + + /// The same as above when the record being stopped names no process either: + /// a relaunch's empty `starting` record then equals the stopping snapshot's + /// processes, which must not make it the stop's own. Its marker is cleared and + /// it reads `starting`, so the stop does not claim it, overwrite its status or + /// delete the key it will serve with. A stop of an empty record that nobody + /// relaunched is still confirmed. + #[test] + fn stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record() { + let (root, paths) = test_paths("stop-empty-relaunch"); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-relaunch"; + let mut empty = stop_test_record(&paths, service_id); + empty.clear_recorded_processes(); + empty.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let mut relaunch = raw_manifest(&paths, service_id); + relaunch.status = "starting".to_owned(); + relaunch.stop_requested_unix_ms = None; + relaunch.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + + // And with nobody relaunching, the stop of the same empty record is a stop. + let untouched = stop_internal_managed_service_with(&paths, service_id, |_| { + bare_termination(Vec::new(), true) + }) + .unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.status, "starting", "the relaunch's status stays"); + assert_eq!(key.as_deref(), Some("key-123"), "its key is kept"); + assert_eq!(untouched["stopped"], true, "{untouched}"); + } + + /// The interleaving the empty-relaunch guard above must not mistake for a + /// relaunch: the record being stopped names no process and reads `starting` or + /// `failed` (a liveness refresh leaves both as they are), and a concurrent + /// listing finds the stop marker with nothing recorded to wait for and clears + /// it before the stop reads the record again. Nobody relaunched, so the stop is + /// confirmed as it is when no listing intervenes (the last block), and the + /// record reads `stopped` afterwards with its key gone. + #[test] + fn a_listing_clearing_the_marker_of_an_empty_stop_does_not_make_it_unconfirmed() { + for status in ["starting", "failed"] { + let (root, paths) = test_paths(&format!("stop-empty-listing-{status}")); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-listing"; + let mut empty = stop_test_record(&paths, service_id); + status.clone_into(&mut empty.status); + empty.clear_recorded_processes(); + empty.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { + assert!(stopping.stop_requested_unix_ms.is_some()); + // The listing under test: it reads the record with the stop's + // marker on it, finds no process to wait for, and persists the + // cleanup before this stop reads the record again. + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let changed = + crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!( + changed, + "premise: the listing cleaned the marker ({status})" + ); + listed.write().unwrap(); + let after_listing = raw_manifest(&paths, service_id); + assert_eq!(after_listing.stop_requested_unix_ms, None, "{status}"); + assert_eq!(after_listing.status, status, "premise: status unchanged"); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], true, "{status}: {report}"); + assert_eq!(on_disk.status, "stopped", "{status}"); + assert_eq!(on_disk.stop_requested_unix_ms, None, "{status}"); + assert_eq!(key, None, "{status}: the key goes with the confirmed stop"); + } + } + + /// The same interleaving followed by a real relaunch: the listing consumed the + /// marker, then the daemon restarted the empty record (a restart counts itself, + /// clears the marker and reads `recovering` or `starting`). The stamp the + /// listing left is the stop's own, but the relaunch wipes it, so the stop still + /// does not claim the relaunch's record. + #[test] + fn a_relaunch_after_a_listing_consumed_the_marker_is_still_not_the_stop() { + let (root, paths) = test_paths("stop-empty-listing-then-relaunch"); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-listing-then-relaunch"; + let mut empty = stop_test_record(&paths, service_id); + "starting".clone_into(&mut empty.status); + empty.clear_recorded_processes(); + empty.write().unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + listed.write().unwrap(); + let mut relaunch = raw_manifest(&paths, service_id); + assert!( + relaunch.settled_stop_unix_ms.is_some(), + "premise: the listing left the stop's stamp" + ); + "recovering".clone_into(&mut relaunch.status); + relaunch.reset_for_restart(); + relaunch.stop_requested_unix_ms = None; + relaunch.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.status, "recovering", "the relaunch's status stays"); + assert_eq!(on_disk.restart_count, 1); + } + #[test] fn stop_of_a_container_engine_is_unconfirmed_when_the_engine_stop_fails() { // For `aim` the recorded PID is only the supervisor; killing it says @@ -1370,27 +1562,41 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the endpoint key is kept"); } - /// A stop stand-in's bare verdict, without per-PID detail. - fn bare_termination(signaled_pids: Vec, all_stopped: bool) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids, - all_stopped, - pid_outcomes: Vec::new(), + /// One recorded process's stop, for a stand-in to return. + fn stop_of(pid: u32, outcome: rocm_core::TerminationOutcome) -> rocm_core::RecordedPidStop { + rocm_core::RecordedPidStop { + pid, + role: "supervisor_and_engine", + outcome: rocm_core::RecordedPidOutcome::Terminated(outcome), + } + } + + /// A stop stand-in's bare verdict, without per-PID detail: `signaled_pids` + /// were signalled and `all_stopped` says whether they exited. A stop that + /// signalled nothing and is not confirmed has one unverifiable process. + fn bare_termination( + signaled_pids: Vec, + all_stopped: bool, + ) -> Vec { + use rocm_core::TerminationOutcome::{Graceful, TimedOut, Unverified}; + if signaled_pids.is_empty() { + return if all_stopped { + Vec::new() + } else { + vec![stop_of(999_999_999, Unverified)] + }; } + let outcome = if all_stopped { Graceful } else { TimedOut }; + signaled_pids + .into_iter() + .map(|pid| stop_of(pid, outcome)) + .collect() } /// A stop stand-in whose one recorded process (the record's shared /// supervisor/engine PID) was signalled and outlived the deadline. - fn timed_out_termination(pid: u32) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids: vec![pid], - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::TimedOut, - }], - } + fn timed_out_termination(pid: u32) -> Vec { + vec![stop_of(pid, rocm_core::TerminationOutcome::TimedOut)] } /// The most recent audit event written under `paths`. @@ -1410,17 +1616,13 @@ mod tests { stop_test_record(&paths, service_id).write().unwrap(); endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: Vec::new(), - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::Unverified, + )] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); @@ -1483,31 +1685,26 @@ mod tests { record.engine_pid = Some(888_888_888); record.write().unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: vec![999_999_999], - all_stopped: false, - pid_outcomes: vec![ - RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor", - outcome: rocm_core::TerminationOutcome::TimedOut, - }, - RecordedPidOutcome { - pid: 888_888_888, - role: "engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }, - ], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + role: "supervisor", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(888_888_888, rocm_core::TerminationOutcome::Unverified) + }, + ] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); let _ = fs::remove_dir_all(root); let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); assert_no_retry_line(&rendered); assert!( - rendered.contains("check: `ps -p 888888888 -o pid,args`"), + rendered.contains(&format!("check: `{}`", process_check_command(888_888_888))), "{rendered}" ); assert!(!rendered.contains("the record changed"), "{rendered}"); @@ -1608,6 +1805,128 @@ mod tests { assert_eq!(key, None, "a confirmed stop drops the key"); } + /// A confirmed stop must not leave the processes it stopped named in the + /// record. Their PIDs are stale the moment they exit, and a recorded PID + /// with no start-time token is signalled on trust alone once the OS has + /// handed it to another process; on Windows the tree kill then takes that + /// process and everything under it. `rocmd`'s stop clears them for the same + /// reason (`stop_managed_service_clears_recorded_pids_once_the_processes_are_gone`). + #[test] + fn a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing() { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("services-stop-clears-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-clears-pids"; + let mut record = stop_test_record(&paths, service_id); + // The dangerous shape: a supervisor recorded without a start-time token, + // and an engine with one. + record.supervisor_pid = 4242; + record.supervisor_start_ticks = None; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + // The termination of each recorded PID, observed: the real routine runs + // (which PIDs it visits, in which order), only the signalling is replaced. + let signalled = std::cell::RefCell::new(Vec::new()); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + signalled.borrow_mut().push(*identity); + Graceful + }) + }; + + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let signalled_by_first = signalled.borrow().clone(); + + let second = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_second = raw_manifest(&paths, service_id); + let signalled_in_all = signalled.borrow().clone(); + let _ = fs::remove_dir_all(root); + + assert_eq!(first["stopped"], true, "{first}"); + assert_eq!( + signalled_by_first, + vec![ + rocm_core::ProcessIdentity::new(4242, None), + rocm_core::ProcessIdentity::new(4243, Some(77)), + ] + ); + // The persisted state after the confirmed stop names no process. + assert_eq!(after_first.status, "stopped"); + assert_eq!(after_first.stop_requested_unix_ms, None); + assert_eq!(after_first.supervisor_pid, 0); + assert_eq!(after_first.supervisor_start_ticks, None); + assert_eq!(after_first.engine_pid, None); + assert_eq!(after_first.engine_start_ticks, None); + // So a stop of the same record signals nothing, and still reads stopped. + assert_eq!( + signalled_in_all, signalled_by_first, + "the second stop reached a PID the first one had already stopped" + ); + assert_eq!(second["stopped"], true, "{second}"); + assert_eq!(second["signaled_pids"], serde_json::json!([]), "{second}"); + assert_eq!(second["pid_outcomes"], serde_json::json!([]), "{second}"); + assert_eq!(after_second.status, "stopped"); + assert_eq!(after_second.supervisor_pid, 0); + assert_eq!(after_second.engine_pid, None); + } + + // Unix only: the processes must really be running. The second stop reads the + // record through the liveness refresh, which forgets the PIDs of processes it + // observes gone (`a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop`), + // so made-up PIDs would be forgotten and not kept. + #[cfg(unix)] + #[test] + fn an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop() { + use rocm_core::TerminationOutcome::TimedOut; + let mut supervisor = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn supervisor"); + let mut engine = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn engine"); + let (supervisor_pid, engine_pid) = (supervisor.id(), engine.id()); + let (root, paths) = test_paths("services-stop-keeps-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-keeps-pids"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = supervisor_pid; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(engine_pid); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + let signalled = std::cell::Cell::new(0_usize); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |_| { + signalled.set(signalled.get() + 1); + TimedOut + }) + }; + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let _ = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let signalled_in_all = signalled.get(); + let _ = supervisor.kill(); + let _ = supervisor.wait(); + let _ = engine.kill(); + let _ = engine.wait(); + let _ = fs::remove_dir_all(root); + + // Nothing was confirmed gone, so the record still names what survived: + // that is the only thing a later stop can find them by. + assert_eq!(first["stopped"], false, "{first}"); + assert_eq!(after_first.supervisor_pid, supervisor_pid); + assert_eq!(after_first.supervisor_start_ticks, Some(5)); + assert_eq!(after_first.engine_pid, Some(engine_pid)); + assert_eq!(after_first.engine_start_ticks, Some(77)); + assert_eq!(signalled_in_all, 4, "each stop tried both processes"); + } + #[test] fn restart_does_not_start_an_engine_over_an_unconfirmed_stop() { // A restart used to discard its stop's result and start the new engine @@ -1751,6 +2070,49 @@ mod tests { assert_eq!(on_disk.supervisor_pid, std::process::id()); } + /// The relaunch generation a restart stamps lives only until the restart + /// publishes: recording the child clears it, and a record whose generation + /// moved since the restart read it (another restart began, or this one's + /// stamp is gone) is not written over, as one with other processes is not. + #[test] + fn a_restart_publishing_its_child_ends_the_relaunch_generation_and_respects_a_moved_one() { + let (root, paths) = test_paths("restart-relaunch-generation"); + paths.ensure().unwrap(); + let service_id = "svc-restart-relaunch-generation"; + let mut stopped = stop_test_record(&paths, service_id); + stopped.status = "stopped".to_owned(); + stopped.clear_recorded_processes(); + stopped.begin_relaunch(); + stopped.write().unwrap(); + let stopped = raw_manifest(&paths, service_id); + assert!(stopped.relaunch_started_unix_ms.is_some(), "premise"); + + // Another restart began meanwhile: the generation on disk is not the one + // this restart read. + let mut moved = stopped.clone(); + moved.begin_relaunch(); + moved.write().unwrap(); + assert!( + moved.relaunch_started_unix_ms > stopped.relaunch_started_unix_ms, + "a new generation is always distinct from the one before it" + ); + let refused = record_restarted_child(&paths, &stopped, None, std::process::id()); + let untouched = raw_manifest(&paths, service_id); + + // With the generation as the restart read it, publication goes through. + stopped.write().unwrap(); + let started = record_restarted_child(&paths, &stopped, None, std::process::id()).unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert!(refused.is_err(), "a moved generation is not written over"); + assert_eq!(untouched.status, "stopped"); + assert_eq!(untouched.supervisor_pid, 0, "no child was recorded"); + assert_eq!(started.relaunch_started_unix_ms, None); + assert_eq!(on_disk.relaunch_started_unix_ms, None); + assert_eq!(on_disk.supervisor_pid, std::process::id()); + } + #[test] fn restart_reports_a_child_it_could_not_record_and_writes_nothing() { // Between the restart's stop and its spawn, another restart recorded @@ -1907,8 +2269,11 @@ mod tests { // The claim the message makes: the service really is stopped, and no // replacement child was started behind the failed wait. assert_eq!(record.status, "stopped"); + // The confirmed stop cleared the recorded processes (the record names + // none), so a supervisor recorded behind the failed wait would show as a + // nonzero PID. assert_eq!( - record.supervisor_pid, 999_999_999, + record.supervisor_pid, 0, "no new supervisor may be recorded behind the failed wait" ); Ok(()) @@ -2002,6 +2367,60 @@ mod tests { Ok(()) } + /// An older stop finalizing while a restart is between its own confirmed stop + /// and the publication of its child. The record is then an empty `stopped` + /// manifest, which is also what an older stop's own completion looks like, but + /// the restart has restored the endpoint key and is about to serve with it. + /// The older stop (here the outer one, whose termination step runs the whole + /// restart, so its snapshot predates it) must neither claim that record nor + /// delete the key. The restart's spawn is failed on purpose, which leaves the + /// record exactly as it is at the spawn: published nothing, key restored. + #[cfg(not(windows))] + #[test] + fn an_older_stop_finalizing_before_a_restart_publishes_does_not_delete_its_key() -> Result<()> { + let (root, paths) = test_paths("older-stop-during-restart"); + paths.ensure()?; + let service_id = "svc-older-stop-during-restart"; + // A live PID (this test's own), so the restart's own load does not take + // the older stop's marker for a completed stop and clear the key before + // the restart has captured it. Nothing signals it: every termination + // step here is a stand-in. + let mut planted = stop_test_record(&paths, service_id); + planted.supervisor_pid = std::process::id(); + planted.write()?; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123")?; + let restart_reached_spawn = std::cell::Cell::new(false); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let spawn: &RestartSpawnOverride<'_> = &|_command| { + restart_reached_spawn.set(true); + Err(io::Error::other("spawn intercepted by test")) + }; + let restart = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| bare_termination(Vec::new(), true), + Some(spawn), + ); + assert!(restart.is_err(), "the intercepted spawn fails the restart"); + bare_termination(Vec::new(), true) + })?; + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert!(restart_reached_spawn.get(), "premise: the restart ran"); + assert_eq!( + key.as_deref(), + Some("key-123"), + "the key the restart restored for its child is not the older stop's to delete" + ); + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.restart_count, 0, "nothing was published"); + Ok(()) + } + #[cfg(not(windows))] #[test] fn restart_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { @@ -2383,7 +2802,7 @@ mod tests { "svc-cpu-policy", "vllm", "starting", - // Skipped by `terminate_recorded_service_pids`, which never + // Skipped by the shared stop routine, which never // signals this process — so the stop runs for real without the // test killing itself. std::process::id(), @@ -2469,15 +2888,14 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real.wrapping_add(1))); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); - assert!(signaled.is_empty(), "must not signal a mismatched identity"); + assert!( + stops.iter().all(|stop| !stop.signaled()), + "must not signal a mismatched identity" + ); // A recycled PID means the recorded service process itself is gone. - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); assert!( rocm_core::process_is_running(pid), "the unrelated live process must survive" @@ -2507,11 +2925,13 @@ mod tests { record.engine_pid = Some(eng_pid); record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); - let RecordedStopTermination { - signaled_pids: mut signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let all_stopped = stops.iter().all(|stop| stop.stopped()); + let mut signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); signaled.sort_unstable(); let mut expected = vec![sup_pid, eng_pid]; expected.sort_unstable(); @@ -2539,14 +2959,15 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real)); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); assert_eq!(signaled, vec![pid]); - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); // Reap our own child (a detached managed process would be reaped by init) // so the liveness check does not observe a not-yet-reaped zombie. let mut child = child; @@ -2557,4 +2978,468 @@ mod tests { ); let _ = fs::remove_dir_all(root); } + + // ---- This module's stop, through the routine it shares with `rocmd`. ---- + // + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. Every PID below is one that cannot be + // running, so nothing is signalled and the tests behave the same on every + // host. + + /// A PID no host has in use, so the stop reports it as already gone. + const ABSENT_SUPERVISOR_PID: u32 = 999_999_998; + const ABSENT_ENGINE_PID: u32 = 999_999_997; + + /// Run the production stop of a record naming `supervisor` and `engine`, + /// and return its report. + fn stop_report_for(name: &str, supervisor: u32, engine: Option) -> serde_json::Value { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let service_id = "svc-stop-through-the-shared-routine"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = supervisor; + record.engine_pid = engine; + record.write().unwrap(); + + let report = + stop_internal_managed_service_with(&paths, service_id, stop_recorded_processes) + .unwrap(); + let _ = fs::remove_dir_all(root); + report + } + + #[test] + fn a_stop_reports_the_supervisor_then_the_engine_in_that_order() { + let report = stop_report_for( + "managed-stop-order", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_ENGINE_PID), + ); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor", "outcome": "already_gone"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!(report["signaled_pids"], serde_json::json!([])); + assert_eq!(report["stopped"], true); + } + + #[test] + fn a_stop_reports_a_pid_recorded_twice_once_under_both_roles() { + let report = stop_report_for( + "managed-stop-aliased", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_SUPERVISOR_PID), + ); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor_and_engine", "outcome": "already_gone"}, + ]) + ); + } + + /// Both binaries report the stopping process's own PID as `self`, never + /// signal it, and do not let it stand in the way of a confirmed stop. `rocm` + /// used to drop it from the report, so `pid_outcomes` showed one entry fewer + /// than `rocmd`'s for the same record. + #[test] + fn a_stop_reports_its_own_pid_as_self() { + let own = std::process::id(); + + let report = stop_report_for("managed-stop-own-pid", own, Some(ABSENT_ENGINE_PID)); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": own, "role": "supervisor", "outcome": "self"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!( + report["signaled_pids"], + serde_json::json!([]), + "the stopping process is never signalled" + ); + assert_eq!(report["stopped"], true); + } + + #[test] + fn a_stop_of_a_record_naming_only_its_own_pid_confirms() { + let own = std::process::id(); + + let report = stop_report_for("managed-stop-only-own-pid", own, None); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([{"pid": own, "role": "supervisor", "outcome": "self"}]) + ); + assert_eq!(report["stopped"], true); + } + + /// The `self` row is not one of the outcomes that leave a stop unconfirmed, + /// so it never reaches the report's `still running` or `not signaled` lines. + #[test] + fn a_self_row_adds_no_line_to_an_unconfirmed_report() { + let (root, paths) = test_paths("managed-stop-self-row-unconfirmed"); + paths.ensure().unwrap(); + let service_id = "svc-self-row-unconfirmed"; + stop_test_record(&paths, service_id).write().unwrap(); + let own = std::process::id(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + pid: own, + role: "supervisor", + outcome: rocm_core::RecordedPidOutcome::OwnProcess, + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + ] + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let _ = fs::remove_dir_all(root); + let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); + + assert!( + rendered.contains("still running: process 999999999 (engine)"), + "{rendered}" + ); + assert!( + !rendered.contains(&format!("process {own}")), + "the stopping process is not a survivor: {rendered}" + ); + } + + #[cfg(unix)] + #[test] + fn managed_stop_signals_the_supervisor_before_the_engine() { + let sup = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn supervisor"); + let eng = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn engine"); + let (sup_pid, eng_pid) = (sup.id(), eng.id()); + let (root, paths) = test_paths("managed-stop-signal-order"); + let mut record = + managed_record_for_pid(&paths, sup_pid, rocm_core::process_start_ticks(sup_pid)); + record.engine_pid = Some(eng_pid); + record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); + + let stops = stop_recorded_processes(&record); + let (mut sup, mut eng) = (sup, eng); + let _ = sup.wait(); + let _ = eng.wait(); + let _ = fs::remove_dir_all(root); + + // Not sorted: the order is the routine's, and a copy that visited the + // engine first would fail here. + assert_eq!( + stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect::>(), + vec![sup_pid, eng_pid] + ); + assert!(stops.iter().all(|stop| stop.stopped())); + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service_control.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } + } + + /// The stop and the liveness read after it, end to end: a stop that cannot + /// verify a recorded process leaves the key and the stop request, and a later + /// refresh that still cannot query the process must not take them away or + /// reconcile the record to `stopped`. The refresh reads the record the stop + /// wrote, as a service listing does. + #[test] + fn an_unverified_stop_keeps_its_key_and_marker_through_a_later_liveness_refresh() { + let (root, paths) = test_paths("stop-unverified-then-refresh"); + paths.ensure().unwrap(); + let service_id = "svc-unverified-then-refresh"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::Unverified, + )] + }) + .unwrap(); + assert_eq!(report["stopped"], false, "{report}"); + let (mut record, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + assert!( + record.stop_requested_unix_ms.is_some(), + "the stop is pending" + ); + + let changed = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut record, + |_| rocm_core::ProcessProbe::Unqueryable, + ); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + assert!( + record.stop_requested_unix_ms.is_some(), + "the stop request is kept" + ); + assert!( + !changed, + "nothing about the record may change while the process is unqueryable" + ); + } + + /// An unconfirmed stop that a later listing completes (every recorded process + /// is observed gone) must leave a record that names no process, as a confirmed + /// stop does: otherwise the next `services stop` signals whatever the OS has + /// since given those PIDs to, and a PID with no start-time token cannot be + /// refuted. The same record, written back and stopped again, visits no PID. + #[test] + fn a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop() { + let (root, paths) = test_paths("stop-completed-by-listing-clears-pids"); + paths.ensure().unwrap(); + let service_id = "svc-completed-by-listing"; + let mut planted = stop_test_record(&paths, service_id); + planted.supervisor_pid = 4242; + planted.supervisor_start_ticks = None; + planted.engine_pid = Some(4243); + planted.engine_start_ticks = Some(77); + planted.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of(4242, rocm_core::TerminationOutcome::Unverified)] + }) + .unwrap(); + assert_eq!(report["stopped"], false, "{report}"); + let (mut record, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + assert_eq!( + record.supervisor_pid, 4242, + "an unconfirmed stop keeps the PIDs" + ); + + // The listing that finds both processes gone completes the stop. + let changed = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut record, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!(changed); + record.write().unwrap(); + let after_listing = raw_manifest(&paths, service_id); + + let visited = std::cell::RefCell::new(Vec::new()); + let second = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + visited.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Graceful + }) + }) + .unwrap(); + let visited = visited.into_inner(); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(after_listing.status, "stopped"); + assert_eq!(after_listing.stop_requested_unix_ms, None); + assert_eq!(key, None, "the deferred cleanup drops the key"); + assert_eq!( + ( + after_listing.supervisor_pid, + after_listing.supervisor_start_ticks, + after_listing.engine_pid, + after_listing.engine_start_ticks + ), + (0, None, None, None) + ); + assert!( + visited.is_empty(), + "a later stop reached a PID the listing had observed gone: {visited:?}" + ); + assert_eq!(second["stopped"], true, "{second}"); + assert_eq!(second["pid_outcomes"], serde_json::json!([]), "{second}"); + } + + /// A stop whose recorded process was seen to exit but whose tree kill did not + /// complete says so in those words, not as a timeout, and keeps what an + /// unconfirmed stop keeps: the claim and the state are checked together. + #[test] + fn a_stop_with_an_unconfirmed_tree_kill_says_what_happened_and_keeps_the_service() { + let (root, paths) = test_paths("services-stop-descendants-unconfirmed"); + paths.ensure().unwrap(); + let service_id = "svc-descendants-unconfirmed"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::DescendantsUnconfirmed, + )] + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + // What the report says about the warning, checked against what happens + // next: a later stop finds the root gone and, with no new evidence about + // what was under it, reports the stop confirmed. + let later = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::AlreadyGone, + )] + }) + .unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!(answer["status"], "stop_unconfirmed", "{answer}"); + let rendered = format!("{:#}", outcome.expect_err("not a confirmed stop")); + assert!( + rendered.contains( + "not confirmed: process 999999999 (supervisor_and_engine) exited, but a process under it may still be running" + ), + "{rendered}" + ); + assert!( + rendered.contains("rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed"), + "the report must not imply the warning outlives this run: {rendered}" + ); + assert_eq!( + later["stopped"], true, + "the claim the report makes is true: {later}" + ); + assert!( + !rendered.contains("still running: process") + && !rendered.contains("did not exit after it was signaled"), + "the root was seen to exit, so nothing outlived a deadline: {rendered}" + ); + assert!(!rendered.contains("Local server stopped"), "{rendered}"); + assert_no_retry_line(&rendered); + // The claim of what was kept, against what was kept. + assert!(rendered.contains("kept: the server record"), "{rendered}"); + assert_eq!(on_disk.supervisor_pid, 999_999_999); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123")); + } + + /// A root that exited by itself before its tree could be taken down confirms + /// the stop only through the record's other roles, through the real routine + /// and this caller's own folding: the root's row alone is not enough, and the + /// record, its PIDs, its stop request and its key are kept unless the engine + /// role is confirmed gone too. + #[test] + fn a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does() { + use rocm_core::TerminationOutcome::{ExitedBeforeTreeKill, Forced, TimedOut}; + + for (engine, confirmed) in [(Forced, true), (TimedOut, false)] { + let (root, paths) = test_paths("services-stop-root-exited-alone"); + paths.ensure().unwrap(); + let service_id = "svc-root-exited-alone"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = 999_999_998; + record.engine_pid = Some(999_999_997); + record.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + if identity.pid == 999_999_998 { + ExitedBeforeTreeKill + } else { + engine + } + }) + }) + .unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(report["stopped"], confirmed, "{engine:?}: {report}"); + if confirmed { + assert_eq!(on_disk.status, "stopped"); + assert_eq!( + on_disk.supervisor_pid, 0, + "the confirmed stop clears the PIDs" + ); + assert_eq!(key, None, "and the key"); + } else { + assert_eq!(on_disk.supervisor_pid, 999_999_998, "the PIDs are kept"); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + } + } + } + + /// A lone root that exited by itself is not vouched for by anything else the + /// record names, so this caller does not clear the record on its say-so. + #[test] + fn a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed() { + let (root, paths) = test_paths("services-stop-lone-root-exited-alone"); + paths.ensure().unwrap(); + let service_id = "svc-lone-root-exited-alone"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |_| { + rocm_core::TerminationOutcome::ExitedBeforeTreeKill + }) + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + let rendered = format!("{:#}", outcome.expect_err("not a confirmed stop")); + assert!( + rendered.contains("was already gone when the processes under it were to be taken down"), + "{rendered}" + ); + assert!(!rendered.contains("Local server stopped"), "{rendered}"); + assert_no_retry_line(&rendered); + assert_eq!(on_disk.supervisor_pid, 999_999_999); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123")); + } } diff --git a/apps/rocm/src/service_hints.rs b/apps/rocm/src/service_hints.rs index 49d659dc10..6556c2e70a 100644 --- a/apps/rocm/src/service_hints.rs +++ b/apps/rocm/src/service_hints.rs @@ -38,6 +38,16 @@ pub(crate) fn service_action_hint(action: &str, service_id: &str) -> String { format!("rocm services {action} {} --yes", shell_quote(service_id)) } +/// The directly-runnable `rocm services remove --force --yes` hint: the +/// removal that goes ahead for a record whose recorded processes cannot be +/// queried (see `remove_managed_service_record`). +pub(crate) fn service_forced_remove_hint(service_id: &str) -> String { + format!( + "rocm services remove {} --force --yes", + shell_quote(service_id) + ) +} + /// The directly-runnable `rocm services stop --yes` hint. pub(crate) fn service_stop_hint(service_id: &str) -> String { service_action_hint("stop", service_id) diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 720d9312c5..ae5c2fdb5e 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -8,13 +8,13 @@ //! Relocated from `main.rs`, since extended with the port-allocation and //! launch-lock transaction. This module //! is the managed-service-spawning tail that was deliberately deferred out of -//! the Phase 6a (ROCMAI-82) split: `start_managed_service`, +//! the earlier relocation of the service-record code: `start_managed_service`, //! `run_attached_service`, `spawn_managed_engine_child`, `stream_attached_logs`, -//! and the rest of this file. It was entangled with crate-root launch helpers +//! and the rest of what was then a single file. It was entangled with crate-root launch helpers //! — `record_cli_audit_event`, `existing_live_managed_service_in`, and the stop //! action of the service-record-management cluster (this module reaches only //! stop, via `AttachOutcome::Stop`; restart/remove/prune are not called here) -//! — and this later pass (ROCMAI-91, Phase 6b) separates the spawning tail out +//! — and this later pass separates the spawning tail out //! from them; see below for where they live now. //! //! `dispatch()`'s call sites are unaffected: `Command::Services { command } => @@ -28,5100 +28,57 @@ //! here uses) to `managed_service_status.rs`. Several functions here are `pub(crate)` //! because other modules call into this cluster across the module boundary; //! `main.rs` only declares the module. +//! +//! # Layout +//! +//! This file only declares the submodules and re-exports the items that other +//! modules call, so callers keep reaching them as `crate::service_launch::name`. +//! Items a submodule shares with its siblings are `pub(super)` and reached through +//! `super::`; nothing is wider than that unless a module outside this +//! one calls it. +//! +//! - `endpoint_auth`: bind-host validation and the endpoint API key checks. +//! - `launch_transaction`: the locked port allocation decision, the cleanup of a +//! failed launch, and the reuse refusals. +//! - `spawn`: the detached engine child, its process and stdio plumbing, and +//! `ManagedLaunchReport`. +//! - `readiness`: waiting on a service, its liveness, and the OOM fault-injection +//! seam. +//! - `start`: the background launch path and its plain output. +//! - `helper_autostart`: starting the background automation helper. +//! - `attach`: the attached run and its log streaming. + +mod attach; +mod endpoint_auth; +mod helper_autostart; +mod launch_transaction; +mod readiness; +mod spawn; +mod start; +#[cfg(test)] +mod test_support; -use std::fmt::Write as _; -use std::fs; -use std::io::{self, BufRead, Write}; -use std::path::{Path, PathBuf}; -#[cfg(not(windows))] -use std::process::{Command as ProcessCommand, Stdio}; -use std::thread; -use std::time::Duration; - -use anyhow::{Context, Result, anyhow, bail}; -use rocm_core::{AppPaths, AutomationRuntimeState, EndpointReadiness, ManagedServiceRecord}; -use rocm_core::{format_http_base_url, process_is_running}; -use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; - -use crate::endpoint_keys; -use crate::engine_ipc::builtin_engine_serve_http_args; -#[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; -#[cfg(windows)] -use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; -use crate::engine_runtime_select::{ - managed_service_launcher_path, wait_for_service_http_ready_with_progress, +pub(crate) use attach::run_attached_service; +pub(crate) use endpoint_auth::{ + drop_orphaned_endpoint_key_on_already_running, ensure_public_bind_engine_supported, + ensure_public_service_has_endpoint_key, is_loopback_host, resolve_endpoint_auth, + validate_bind_host, }; -use crate::engines_cmd::env_root_for_service; -use crate::managed_service_status::{ - existing_live_managed_service_in, load_managed_services, managed_service_is_live, +pub(crate) use helper_autostart::{ + ensure_background_helper_running, ensure_background_helper_running_quiet, }; -use crate::serve_port; -use crate::service_control::run_internal_sandbox_tool; -use crate::service_hints; -use crate::services_text::read_optional_tail_lines; -use crate::services_text::{ - render_service_action_result, render_service_action_result_with_helper_note, - service_stop_unconfirmed, -}; -use crate::{SandboxToolArg, device_policy_name, record_cli_audit_event, status_for_readiness}; - -pub(crate) fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { - if !is_loopback_host(host) && !allow_public_bind { - bail!( - "`rocm serve --host {host}` is not loopback; pass `--allow-public-bind` before binding a non-local interface" - ); - } - Ok(()) -} - -/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the -/// policy so `rocmd` classifies a recorded `host` identically when it respawns -/// the service. -pub(crate) fn is_loopback_host(host: &str) -> bool { - !rocm_engine_protocol::is_public_bind_host(host) -} - -/// Resolve the API key that will guard this endpoint, applying the -/// loopback-vs-public policy for `rocm serve`. -/// -/// - **Loopback host** → `None`: local serving stays credential-free (the -/// unchanged default). Any key supplied for a loopback bind is ignored and -/// nothing is persisted — loopback needs no auth. -/// - **Public host** → `Some(key)`: use the user-supplied key when present, -/// otherwise generate a strong random one so a public endpoint can never come -/// up anonymous. An empty/whitespace supplied key is rejected rather than -/// silently treated as "no auth". -/// - **`required`** → treat a loopback bind as public for this purpose. -/// -/// That last case exists because "loopback" is a statement about the bind -/// address, not about who can reach the port. Publishing the port onto a -/// tailnet, proxying it, or mapping it out of a container all leave the bind -/// loopback while widening the audience — and the policy above would then hand -/// out an unauthenticated endpoint. Whoever widens the reach is responsible for -/// asking for the credential, so this is an explicit flag rather than a guess. -pub(crate) fn resolve_endpoint_auth( - host: &str, - supplied: Option<&str>, - required: bool, -) -> Result> { - if is_loopback_host(host) && !required { - return Ok(None); - } - match supplied { - Some(raw) => { - let trimmed = raw.trim(); - if trimmed.is_empty() { - bail!( - "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) was empty; a public \ - endpoint must be protected by a non-empty API key" - ); - } - // The key is later interpolated verbatim into raw `Authorization: - // Bearer {key}\r\n` header lines; reject a control character (e.g. an - // embedded CR/LF) here so a crafted key cannot inject extra headers. - if rocm_core::endpoint_api_key_has_forbidden_chars(trimmed) { - bail!( - "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) contained a control \ - character such as a carriage return or newline; an endpoint API key \ - must be a single line of printable characters" - ); - } - Ok(Some(trimmed.to_owned())) - } - None => Ok(Some(rocm_core::generate_endpoint_api_key())), - } -} - -/// When an equivalent managed service is already running, or the launch fails -/// before any record exists, the endpoint key serve() freshly stored for this -/// attempt is unused — drop it so it is not orphaned in storage. The already-running service keeps its own key. -/// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) -/// is a no-op. -pub(crate) fn drop_orphaned_endpoint_key_on_already_running( - paths: &AppPaths, - service_id: &str, - freshly_stored: Option<&str>, -) { - if freshly_stored.is_some() { - endpoint_keys::clear_endpoint_api_key(paths, service_id); - } -} - -/// Reject engine/platform combinations that cannot enforce a public endpoint's -/// API key, so a public bind fails closed instead of coming up unauthenticated. -/// -/// The one such case today: Windows managed Lemonade. Its server reads the -/// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn -/// primitive only carries path-valued env overrides, so the key never reaches it. -/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds -/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter -/// so both branches are unit-testable off-Windows. -pub(crate) fn ensure_public_bind_engine_supported( - engine: &str, - public_bind: bool, - is_windows: bool, -) -> Result<()> { - if public_bind && is_windows && engine == "lemonade" { - bail!( - "public binding with the lemonade engine is not supported on Windows: the endpoint \ - API key cannot be enforced there. Use `--engine vllm`, or bind a loopback host \ - (the default 127.0.0.1)." - ); - } - Ok(()) -} - -/// Refuse to (re)spawn a managed service recorded on a public host when its -/// endpoint API key is gone, so a respawn cannot reopen the endpoint anonymously. -/// -/// `serve()` applies the loopback-vs-public policy once, via -/// [`resolve_endpoint_auth`], and persists the resulting key. Every later spawn -/// — `rocm services restart`, and `rocmd`'s recovery supervisor — reads that key -/// file back and would otherwise treat "no key file" as "no auth wanted", -/// silently downgrading a protected public endpoint to an open one. The real -/// invariant is a property of the *host*, not of the file: a non-loopback bind -/// must always be authenticated. -/// -/// The gap is reachable through ordinary commands, because a stop deletes the -/// key file and `rocm services restart` accepts a stopped service id (its help -/// points at `rocm services list --all`). -/// -/// `key_present` is a plain `bool` rather than a path so both branches are -/// unit-testable without touching the filesystem, mirroring `is_windows` in -/// [`ensure_public_bind_engine_supported`]. -pub(crate) fn ensure_public_service_has_endpoint_key( - host: &str, - key_present: bool, - requires_api_key: bool, -) -> Result<()> { - // Two ways a service can need a key. A public bind is the obvious one. The - // other is a service that asked for auth on a loopback bind, because - // something outside this process republishes the port — a tailnet publish - // survives a reboot, let alone a restart, so "loopback" stops meaning - // "only this machine" and the bind address can no longer be trusted to - // answer the question on its own. - if requires_api_key && !key_present { - bail!( - "managed service was launched with `--require-api-key` but has no endpoint API key, \ - so restarting it would reopen it without authentication. Something outside this \ - machine may still be publishing its port. The key is dropped when a service stops \ - and cannot be recovered. Launch it again with \ - `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ - to choose the key instead of generating one)." - ); - } - if rocm_engine_protocol::is_public_bind_host(host) && !key_present { - bail!( - "managed service is bound to the public host `{host}` but has no endpoint API key, \ - so restarting it would reopen it without authentication. The key is dropped when a \ - service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --port --allow-public-bind` (add `--api-key `, or set \ - ROCM_SERVE_API_KEY, to choose the key instead of generating one)." - ); - } - Ok(()) -} - -#[cfg(not(windows))] -pub(crate) fn detach_background_command(command: &mut ProcessCommand) { - rocm_core::detach_command_session(command); -} - +pub(crate) use launch_transaction::{managed_service_reuse_refusal, no_usable_gpu_refusal}; +pub(crate) use readiness::e2e_simulate_oom_launch; #[cfg(not(windows))] -pub(crate) fn attach_background_stdio( - command: &mut ProcessCommand, - log_path: Option<&Path>, -) -> Result<()> { - if let Some(log_path) = log_path { - let log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .with_context(|| format!("failed to open {}", log_path.display()))?; - command - .stdout(Stdio::from(log.try_clone()?)) - .stderr(Stdio::from(log)); - } else { - command.stdout(Stdio::null()).stderr(Stdio::null()); - } - Ok(()) -} - -/// Windows spawn-liveness check: wait out the startup window, then fail with the -/// startup log when the child is already gone. The wait and the liveness probe -/// are injected so a unit test runs the exit-detected path on every platform, -/// including the Windows lane that actually calls it. -/// -/// # Errors -/// Fails with the startup-failure detail when `is_running` reports the child dead. -#[cfg_attr(not(windows), allow(dead_code))] -fn ensure_spawned_child_survives_startup( - child_pid: u32, - log_path: &Path, - settle: impl FnOnce(), - is_running: impl FnOnce(u32) -> bool, -) -> Result<()> { - settle(); - if is_running(child_pid) { - Ok(()) - } else { - bail!("{}", managed_engine_startup_failure_detail("", log_path)) - } -} - -/// `status_clause` is a pre-formatted fragment inserted right after "exited -/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real -/// `ExitStatus` is available; empty on Windows, where the liveness check -/// below only has a yes/no answer, not an exit code). -pub(crate) fn managed_engine_startup_failure_detail( - status_clause: &str, - log_path: &Path, -) -> String { - let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); - if recent_lines.is_empty() { - for _ in 0..5 { - thread::sleep(Duration::from_millis(120)); - recent_lines = read_optional_tail_lines(log_path, 80, "service log"); - if !recent_lines.is_empty() { - break; - } - } - } - if recent_lines.is_empty() { - return format!( - "managed engine exited immediately{status_clause}; inspect {}", - log_path.display() - ); - } - format!( - "managed engine exited immediately{status_clause}; inspect {}\n\nrecent startup log output:\n{}", - log_path.display(), - recent_lines.join("\n") - ) -} -#[cfg(not(windows))] -pub(crate) fn managed_service_process_command(program: &Path, args: &[String]) -> ProcessCommand { - let mut command = ProcessCommand::new(program); - command.args(args); - command -} - -/// Engine-neutral result of a managed launch. Returned rather than printed so the -/// caller can render it either as the rich deployment summary (interactive TTY) or -/// as the plain line-by-line form (piped output, chat assistant), from one code path. -pub(crate) struct ManagedLaunchReport { - pub(crate) service_id: String, - /// `http://host:port/v1`. - pub(crate) endpoint_url: String, - /// `"ready"` (inference confirmed), `"running"` (model listed but not serving - /// yet), `"starting"` (endpoint not answering), or the existing service's - /// status when nothing was spawned. - pub(crate) status: String, - /// True when an equivalent service was already live and nothing was spawned. - pub(crate) already_running: bool, - pub(crate) child_pid: Option, - pub(crate) log_path: Option, - pub(crate) manifest_path: Option, -} - -/// Either an already-live service (nothing spawned) or a freshly spawned engine -/// child that is `running` but not yet HTTP-ready. -/// -/// Split out of [`start_managed_service`] so the attached (`--verbose` / -/// `--foreground`) serve path can spawn the very same detached child and stream -/// its log live from the first line — including startup — instead of blocking on -/// the readiness wait before any output appears. -enum ManagedSpawn { - AlreadyRunning(ManagedLaunchReport), - // `ManagedServiceRecord` is large; box it so the two variants stay a similar - // size (clippy::large_enum_variant). - Spawned { - record: Box, - child_pid: u32, - }, -} - -/// Outcome of the locked allocation step. -/// -/// `Debug` so tests can assert on `Result::expect_err`. -#[derive(Debug)] -enum ManagedPortDecision { - /// An equivalent managed service is already live; nothing may be spawned. - /// Boxed to keep the variants a similar size (clippy::large_enum_variant). - AlreadyLive(Box), - /// The concrete port this launch owns until it hands it to the child. - Lease(rocm_core::LoopbackPortLease), -} - -/// Live managed services, with liveness refreshed by [`load_managed_services`] -/// (dead PIDs demote to "stopped" and therefore neither satisfy the duplicate -/// check nor reserve a port). -/// -/// A read failure degrades to "nothing is live": for an already-`ready` -/// service the OS preflight still guards its port (the engine holds it), and -/// refusing to serve because the services directory could not be listed would -/// be a worse failure than launching. That guard does not cover a `starting` -/// record: its port is released back to the OS in -/// `launch_managed_engine_child_locked` (see `lease.release()`), right before the -/// spawn, and stays unbound until the new engine child has actually bound it, so a concurrent launch that hits this -/// read failure in that narrow window can probe the same port successfully. -/// Accepted as a known gap — narrower than the general services-dir read -/// failure case, and only reachable by a second, near-simultaneous I/O -/// failure on top of one already in flight. -fn live_managed_services(paths: &AppPaths) -> Vec { - load_managed_services(paths) - .unwrap_or_default() - .into_iter() - .filter(managed_service_is_live) - .collect() -} - -/// Ports that live managed services still own. -/// -/// Conservative by design: on the default loopback host a record reserves its -/// port regardless of the host it recorded, because a `0.0.0.0` (or `localhost`) -/// listener also blocks the loopback address a new local server would bind. -/// Over-reserving costs one candidate in a 101-port scan; under-reserving costs -/// a collision. On a custom host the recorded host is compared (see -/// `serve_port::hosts_overlap`). -fn reserved_service_ports(live: &[ManagedServiceRecord]) -> Vec { - live.iter() - .map(|record| serve_port::ReservedPort { - port: record.port, - host: record.host.clone(), - service_id: record.service_id.clone(), - status: record.status.clone(), - }) - .collect() -} - -/// The decision half of the launch transaction: refresh liveness, apply the -/// `(engine, canonical_model_id)` idempotency guard, then lease a concrete port -/// against both live records and the real OS. -/// -/// Callers MUST hold [`rocm_core::AppPaths::managed_launch_lock_path`] across -/// this call *and* the record publication that follows, or two launches can -/// pick the same candidate. -/// `probe` is injected so tests can drive deterministic bind outcomes without a -/// GPU or a spawn. -fn resolve_managed_port_in_transaction( - paths: &AppPaths, - engine: &str, - canonical_model_id: &str, - host: &str, - request: serve_port::PortRequest, - probe: &mut dyn FnMut(u16) -> std::io::Result, -) -> Result { - let live = live_managed_services(paths); - // Idempotency first: an already-live equivalent service means no port is - // needed at all, so it must be answered before any candidate is leased. - if let Some(existing) = existing_live_managed_service_in(&live, engine, canonical_model_id) { - return Ok(ManagedPortDecision::AlreadyLive(Box::new(existing.clone()))); - } - let reserved = reserved_service_ports(&live); - serve_port::resolve_serve_port(host, request, &reserved, probe).map(ManagedPortDecision::Lease) -} - -/// Grace given to a child that must be terminated because its record could not -/// be published. Short: the child is milliseconds old and has nothing to flush. -const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); - -/// Undo a failed managed launch attempt before the launch lock is released, -/// so no failed transaction leaves a live reservation or a stray endpoint key. -/// -/// Order matters: a spawned child is terminated *first*, so it is not still -/// holding the port or writing to the log by the time cleanup runs. The -/// termination outcome is audited but not acted on: the record is marked -/// abandoned (`stopped`, releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), -/// every artifact this attempt created is removed: there is nothing for -/// `rocm services`/`rocm logs` to show. If a child did run, its record and log -/// are kept instead (marked `status = "stopped"`, see -/// [`ManagedServiceRecord::mark_launch_abandoned`]), since the error that led here -/// already told the user to go inspect that log; only the endpoint key, which a -/// failed attempt has no use for, is cleared. -/// Every step is best-effort and idempotent; cleanup must never mask the -/// original failure with a second one. -fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { - if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { - // The token captured at spawn, not a fresh `capture`: when the child - // already exited, a fresh one would describe a recycled pid's new owner - // and the forced tree kill would hit it. Where start tokens exist - // (Linux), a missing one means the child was gone before it could be - // captured, so the pid is unverifiable and nothing is signalled - // (`terminate_verified` alone would "best-effort proceed"). Windows - // records no token at all, so there the kill proceeds as before. - let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { - rocm_core::TerminationOutcome::Unverified - } else { - rocm_core::terminate_verified( - &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), - rocm_core::KillScope::Tree, - MANAGED_LAUNCH_CLEANUP_GRACE, - true, - ) - }; - record_cli_audit_event( - paths, - "service", - "managed_service_launch_cleanup", - "warn", - format!( - "terminated engine child pid={pid} after a failed launch publication outcome={}", - outcome.as_str() - ), - Some(&record.service_id), - ); - let mut abandoned = record.clone(); - abandoned.mark_launch_abandoned(); - let _ = abandoned.write(); - endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); - return; - } - for path in [ - record.manifest_path.as_path(), - record.log_path.as_path(), - record.engine_state_path.as_path(), - ] { - let _ = fs::remove_file(path); - } - endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); -} - -/// The ways a live managed service that already matches on liveness, engine and -/// canonical model id can still be unusable by this invocation. -/// -/// Those three terms are necessary but not sufficient: three further terms can -/// still refuse, one variant each. [`managed_service_reuse_refusal`] is the -/// single decision procedure both callers ask, so the two cannot answer -/// differently — which they previously did; see that function for what the -/// disagreement cost. -pub(crate) enum ManagedReuseRefusal { - /// The live service was started with different serve options. - RecipeMismatch, - /// `--require-api-key` was asked for and the live service has no key. - /// - /// The engine reads its key once, at launch, from the environment the spawn - /// builds — so a server started without one keeps serving anonymously no - /// matter what is written afterwards. Upgrading the record would be worse - /// than doing nothing: it would claim auth the live process does not - /// enforce, and `ensure_public_service_has_endpoint_key` would pass on the - /// strength of a key file nothing reads. `rocm remote serve` relies on this: - /// it publishes a loopback port onto the tailnet and prints "the API key - /// above is what stops anyone else calling it", which reusing an - /// unauthenticated service silently would make false about an endpoint the - /// whole tailnet can reach. Refusing is the only answer that fails closed. - MissingApiKey, - /// The live service runs under a device policy that does not satisfy this - /// invocation's. Reusing a `cpu_only` service to answer a `gpu_required` - /// serve is the silent CPU fallback AGENTS.md section 6 forbids. - DevicePolicyTooWeak, -} - -/// Why this invocation cannot reuse `existing` — or `None` when it genuinely -/// can. See [`ManagedReuseRefusal`] for the terms this answers on. -/// -/// This exists because `serve()`'s reuse pre-gate lets a matching live service -/// skip the no-usable-AMD-GPU refusal. Deciding that bypass on a *weaker* test -/// than the spawn then uses to accept reuse produced a dead end: on a GPU-less -/// host (`HIP_VISIBLE_DEVICES=` with a server live from another shell) a serve -/// with different options bypassed the refusal, reached the spawn, and was told -/// to stop the server and run the command again — and the rerun, with nothing -/// left to reuse, hit the no-GPU refusal. The advice named a command that could -/// not clear the condition, which AGENTS.md section 3 forbids. Sharing the -/// predicate means a service that cannot actually be reused never buys the -/// bypass, so the user gets the accurate refusal first and keeps their server. -/// -/// `requested_device_policy` is taken by value, not as an `Option`, on purpose. -/// It was optional so the spawn could opt out of the device-policy term, and -/// that left two holes at once: the spawn's `DevicePolicyTooWeak` arm was -/// unreachable (an untestable bail), and `serve()` — the caller that *does* -/// gate the GPU refusal on this answer — could silently drop the term by -/// passing `None`, with the whole suite still green. Both callers have a -/// `DevicePolicy` in hand, so requiring it costs nothing and makes that -/// mutation a compile error. It widens nothing observable today either: the -/// only policy a record can carry is the one `parse_device_policy` admits -/// (see below), so `existing.device_policy == Some("cpu_only")` is not -/// reachable from any `rocm serve` this CLI can run. -/// The `rocm serve` no-usable-GPU refusal, worded once. -/// -/// Two places raise it and they must say the same thing: `serve()`'s fail-fast -/// pre-flight, and the spawn below when the reuse that waived that pre-flight -/// turns out not to be available after all. -pub(crate) fn no_usable_gpu_refusal(device_policy: &DevicePolicy) -> String { - // Keyed on the same `is_wsl_host()` the usable-GPU probe branches on, so the - // advice follows the platform whose answer produced the refusal. - no_usable_gpu_refusal_text(device_policy_name(device_policy), rocm_core::is_wsl_host()) -} - -/// The refusal text for `policy`, split on whether the probe that refused was -/// the WSL one. -/// -/// WSL reaches the GPU through `/dev/dxg` and ROCDXG, and has no `/dev/kfd`, so -/// telling a WSL user to confirm that device is present names something that can -/// never clear the condition. There the advice points at the status `rocm -/// examine` reports for the path the probe actually asked about -/// ([`rocm_core::WSL_ROCDXG_READY`]), and at `rocm diagnose`, which names what is -/// missing. -fn no_usable_gpu_refusal_text(policy: &str, on_wsl: bool) -> String { - let remedy = if on_wsl { - format!( - "WSL reaches the GPU through /dev/dxg and ROCDXG rather than /dev/kfd: run \ - `rocm examine` and check that `driver_status` is `{ready}` (`rocm diagnose` \ - names what is missing; a ROCDXG library the loader reaches only through \ - LD_LIBRARY_PATH needs that variable exported in this shell, see docs/wsl.md), and ensure HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES \ - are not masking every device.", - ready = rocm_core::WSL_ROCDXG_READY - ) - } else { - "Check the driver with `rocm examine`, confirm /dev/kfd is present, and ensure \ - HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES are not masking every device." - .to_owned() - }; - format!( - "no usable AMD GPU detected; `rocm serve` requires a GPU under the {policy} policy \ - and does not fall back to CPU. {remedy}" - ) -} - -pub(crate) fn managed_service_reuse_refusal( - existing: &ManagedServiceRecord, - requested_recipe_json: Option<&str>, - require_api_key: bool, - requested_device_policy: &DevicePolicy, -) -> Option { - if existing.engine_recipe_json.as_deref() != requested_recipe_json { - return Some(ManagedReuseRefusal::RecipeMismatch); - } - if require_api_key && !existing.requires_api_key { - return Some(ManagedReuseRefusal::MissingApiKey); - } - // Only `gpu_required` can be let down by the record's policy, and the - // enumeration is exhaustive rather than narrow: `parse_device_policy` - // (`main.rs`) maps `auto`, `gpu`, `gpu_required` *and* `gpu_preferred` onto - // `GpuRequired`, and rejects `cpu_only` with an error, so `GpuRequired` is - // the only policy `serve()` can hand this predicate — pinned by - // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`. There - // is deliberately no wider match on "anything but `CpuOnly`": it would be - // identical at runtime and would read as a guard against a `gpu_preferred` - // bypass that no CLI invocation can reach. A `cpu_only` invocation would be - // satisfied by anything anyway and never reaches the bail. README.md's - // "that is the only policy pair refused" is the contract, and this matches - // it exactly. - if matches!(requested_device_policy, DevicePolicy::GpuRequired) - && existing.device_policy.as_deref() == Some("cpu_only") - { - return Some(ManagedReuseRefusal::DevicePolicyTooWeak); - } - None -} - -/// Spawn the detached engine child shared by the managed (background) and -/// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP -/// readiness wait; callers decide whether to block on readiness -/// ([`start_managed_service`]) or start tailing the log immediately -/// ([`run_attached_service`]). -/// -/// The whole body is one serialized transaction under the shared allocation -/// lock: duplicate detection, live-reservation refresh, candidate leasing, -/// manifest/log creation, argv preparation, the bounded immediate-exit check, -/// and record publication. Readiness deliberately runs after the lock is -/// released. -#[allow(clippy::too_many_arguments)] -fn spawn_managed_engine_child( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port_request: serve_port::PortRequest, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - require_api_key: bool, - gpu_refusal_waived_for_reuse: bool, -) -> Result { - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // No lock is acquired here: every production caller - // ([`start_managed_service`], [`run_attached_service`]) already holds - // [`rocm_core::AppPaths::managed_launch_lock_path`] for the whole of this - // call, so no two transactions can choose the same port or publish - // competing records for one engine+model. A second acquisition in this - // frame would self-deadlock, since the lock is exclusive per open file - // description, not per process. - // - // Idempotency guard: if a managed service for this engine+model is already - // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the - // chat assistant re-issuing the same request) is treated as satisfied, not - // an error. Keyed on engine+canonical model — the freshly generated - // `service_id` is timestamp-unique and would never match an existing one. - // Stale/dead services fall through and relaunch normally. - let requested_recipe_json = engine_recipe - .map(serde_json::to_string) - .transpose() - .context("failed to encode engine recipe hint")?; - let lease = match resolve_managed_port_in_transaction( - paths, - engine, - &resolve.canonical_model_id, - host, - port_request, - &mut serve_port::loopback_bind_probe, - )? { - ManagedPortDecision::AlreadyLive(existing) => { - let existing = *existing; - // Asked through the shared predicate rather than re-tested here, so this - // refusal and `serve()`'s reuse pre-gate cannot disagree about whether a - // service is reusable. When they disagreed, the pre-gate let a - // non-reusable service bypass the no-GPU refusal and the user was sent - // here to be told to stop a working server for nothing. - // - // The same device policy the serve path passes. This side used to pass - // `None` to keep the spawn's acceptance untouched, but that opted the - // spawn out of a term it has the input for, left its own - // `DevicePolicyTooWeak` bail unreachable, and made the serve path's - // argument droppable. Nothing observable widens: `parse_device_policy` - // is the only producer of a `DevicePolicy` on every path that reaches - // here and it rejects `cpu_only`, so no record this CLI writes can - // carry the policy this term refuses. - match managed_service_reuse_refusal( - &existing, - requested_recipe_json.as_deref(), - require_api_key, - device_policy, - ) { - Some(ManagedReuseRefusal::RecipeMismatch) => bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", - existing.service_id, - resolve.canonical_model_id - ), - Some(ManagedReuseRefusal::MissingApiKey) => bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` \ - without authentication, and a running server cannot be given a key it did not \ - start with; stop it with `{}` and run the command again to \ - serve it with `--require-api-key`", - existing.service_id, - resolve.canonical_model_id, - service_hints::service_stop_hint(&existing.service_id) - ), - // Reachable only from a record whose `device_policy` says - // `cpu_only`, which no `rocm serve` can write today - // (`parse_device_policy` refuses that policy outright) — so this is - // a guard against a record written by some future path, not a - // refusal users see now. The advice still names a command that - // clears the condition: stopping the service removes the only thing - // this term can object to, and the rerun then launches normally - // (AGENTS.md section 3). - Some(ManagedReuseRefusal::DevicePolicyTooWeak) => bail!( - "managed service `{}` is already running under a device policy that cannot \ - satisfy this request; stop it with `{}` and run the command again", - existing.service_id, - service_hints::service_stop_hint(&existing.service_id) - ), - None => {} - } - // An explicit port is a demand, not a hint: reporting success on a - // different port than the one asked for would silently drop it. - if let Some(port) = port_request.explicit() - && port != existing.port - { - bail!( - "managed service `{}` is already serving engine `{engine}` and model `{}` on port {}, so port {port} would not be used; stop it with `{}` and run the command again to serve on that port", - existing.service_id, - resolve.canonical_model_id, - existing.port, - service_hints::service_stop_hint(&existing.service_id) - ); - } - record_cli_audit_event( - paths, - "service", - "managed_service_launch_skipped", - "info", - format!( - "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", - resolve.canonical_model_id, existing.service_id, existing.status - ), - Some(&existing.service_id), - ); - return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { - service_id: existing.service_id, - endpoint_url: existing.endpoint_url, - status: existing.status, - already_running: true, - child_pid: None, - log_path: None, - manifest_path: None, - })); - } - ManagedPortDecision::Lease(lease) => lease, - }; - - // Nothing live to reuse — and if the only reason this invocation got past - // `serve()`'s no-usable-GPU pre-flight was a reusable service, that premise - // has just been falsified. `serve()` decided it from a snapshot taken before - // GPU selection and the launch lock; the check above is the authoritative - // one, so a service that died in between (an OOM, say) would otherwise fall - // through to a real spawn on a host with no usable GPU, and the user would - // get whatever the engine says about having no device instead of the - // actionable refusal the pre-flight exists to give. Raise it here, where the - // premise is known to be false. - if gpu_refusal_waived_for_reuse { - bail!("{}", no_usable_gpu_refusal(device_policy)); - } - - let mut record = ManagedServiceRecord::new( - paths, - service_id, - engine, - requested_model, - resolve.canonical_model_id.clone(), - host, - lease.port(), - "managed", - 0, - runtime_id.map(str::to_owned), - env_id.map(str::to_owned), - Some(device_policy_name(device_policy).to_owned()), - ); - record.gpu_indices = gpu_indices.to_vec(); - record.served_model_id = resolve.served_model_id.clone(); - record.engine_recipe_json = requested_recipe_json; - // The flag the user actually passed, carried through rather than re-derived. - // - // Deriving it from key-file presence looked equivalent and was not: - // `resolve_endpoint_auth` mints a key for *every* non-loopback bind whether or - // not auth was demanded, so a plain `--host 0.0.0.0 --allow-public-bind` - // recorded `true` here. The guard below tests this field before the bind - // address, so that service was then refused with a message naming a flag it - // never used and a relaunch command that drops `--allow-public-bind` — the - // public-bind branch, which carries the right command, became unreachable. - // - // This field means "the user demanded auth on a bind that would not otherwise - // require it". A public bind needs no such record; its address still says so. - record.requires_api_key = require_api_key; - // Publishing the record inside the lock is what reserves the leased port for - // every other transaction. From here on, any failure must remove it again. - record.write()?; - - match launch_managed_engine_child_locked( - paths, - &mut record, - lease, - engine, - service_id, - resolve, - host, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - #[cfg(not(windows))] - None, - ) { - Ok(child_pid) => Ok(ManagedSpawn::Spawned { - record: Box::new(record), - child_pid, - }), - Err(error) => { - // The single cleanup choke point for every failure path below the - // record's first write: `record.engine_pid` tells it whether a child - // ever ran, so a pre-spawn failure deletes everything and a post-spawn - // one keeps the record and log (marked `status = "stopped"`) instead. - abandon_failed_managed_launch(paths, &record); - Err(error) - } - } -} - -// Test-only, thread-scoped spawn interceptor for callers that cannot take a -// `spawn_override` argument (`start_managed_service`, `run_attached_service`): -// lets a test observe state at the instant production would spawn the child. -#[cfg(all(test, not(windows)))] -thread_local! { - static TEST_SPAWN_HOOK: std::cell::RefCell>>> = - const { std::cell::RefCell::new(None) }; -} - -#[cfg(all(test, not(windows)))] -fn test_spawn_hook() -> Option>> { - TEST_SPAWN_HOOK.with(|hook| hook.borrow().clone()) -} - -#[cfg(not(windows))] -type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; - -/// The side-effecting half of the launch transaction: create the log, prepare -/// concrete argv, hand the leased port to the child, spawn it, reject a child -/// that has already exited, and publish the running record. -/// -/// Split out of [`spawn_managed_engine_child`] so that *every* failure below the -/// record's first write is funnelled through one cleanup path before the -/// launch lock is released. Returns the child pid. -#[allow(clippy::too_many_arguments)] -fn launch_managed_engine_child_locked( - paths: &AppPaths, - record: &mut ManagedServiceRecord, - lease: rocm_core::LoopbackPortLease, - engine: &str, - service_id: &str, - resolve: &ResolveModelResponse, - host: &str, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - // Test-only seam: lets a test assert the leased port is actually bindable - // (i.e. released) at the exact point production would spawn the child, - // without needing to fake or intercept the real spawn mechanics. - #[cfg(not(windows))] spawn_override: Option<&SpawnOverride<'_>>, -) -> Result { - let port = record.port; - if let Some(parent) = record.engine_state_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::File::create(&record.log_path) - .with_context(|| format!("failed to create {}", record.log_path.display()))?; - let current_exe = managed_service_launcher_path() - .context("failed to resolve current rocm executable path")?; - let serve_args = builtin_engine_serve_http_args( - engine, - service_id, - &resolve.canonical_model_id, - host, - port, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - &record.engine_state_path, - Some(&record.log_path), - )?; - let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; - // Hand the child the *path* to the endpoint key file (public bind only) via the - // environment. A path — not the secret value — is what the detached-spawn - // primitives accept as an env override, and it keeps the key off both the argv - // and the environment block. `serve()` wrote the file before spawning. - // Validity, not mere existence: the engine adapters resolve the key with - // `endpoint_api_key_from_file` and enforce nothing when it yields `None`, so - // an empty or malformed key file would otherwise satisfy the guard below and - // still produce an unauthenticated public listener. - let endpoint_key_file = endpoint_keys::endpoint_key_file_if_present(paths, service_id) - .filter(|path| rocm_engine_protocol::endpoint_api_key_from_file(path).is_some()); - // `serve()` already resolved and stored the key for a public bind, so the - // public-bind branch cannot fire on the fresh-launch path today. It is the - // shared choke point for managed spawns, so enforce the invariant here too - // rather than relying on every future caller having done so. - // - // `record.requires_api_key` is passed, not a literal, and the two arguments - // are deliberately different things: that field is the `--require-api-key` - // flag the caller passed, `endpoint_key_file` is whether a *usable* key is on - // disk. A present but empty or malformed key file is where they disagree, and - // is exactly what the `requires_api_key` branch exists to refuse. - // - // The field is threaded, never derived from the key file. Deriving it marked - // every public bind as having demanded auth — see the assignment above. - ensure_public_service_has_endpoint_key( - host, - endpoint_key_file.is_some(), - record.requires_api_key, - )?; - // Hand the port over: close the probe socket immediately before the spawn, - // still under the caller's launch lock, so no other ROCm transaction can - // take it in the meantime. On a custom host the lease never held a socket - // and this is a no-op — the engine owns that bind. - debug_assert_eq!(lease.port(), port); - lease.release(); - #[cfg(windows)] - let child_pid = { - let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); - let mut env_refs = app_path_env_var_refs(&env_values); - if let Some(key_file) = endpoint_key_file.as_deref() { - env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); - } - let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) - .context("failed to launch managed engine process")?; - // Set before the immediate-exit check below, so a bail! here still leaves - // `record.engine_pid` telling cleanup that a child was spawned. The - // start-time token is captured here too, while the child is certainly - // ours: cleanup binds its kill to it rather than to whatever owns the - // pid by then. - record.record_engine_identity(child_pid); - ensure_spawned_child_survives_startup( - child_pid, - &record.log_path, - || thread::sleep(Duration::from_millis(200)), - process_is_running, - )?; - child_pid - }; - #[cfg(not(windows))] - let child_pid = { - let mut command = managed_service_process_command(¤t_exe, &serve_args); - command.stdin(Stdio::null()); - attach_background_stdio(&mut command, Some(&record.log_path))?; - detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); - if let Some(engine_envs_root) = engine_envs_root.as_deref() { - command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); - } - if let Some(key_file) = endpoint_key_file.as_deref() { - command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); - } - #[cfg(all(test, not(windows)))] - let hook = test_spawn_hook(); - #[cfg(all(test, not(windows)))] - let spawn_override = spawn_override.or(hook.as_deref()); - let mut child = match spawn_override { - Some(spawn) => spawn(&mut command), - None => command.spawn(), - } - .context("failed to launch managed engine process")?; - let child_pid = child.id(); - // Set before the immediate-exit check below, so a bail! here still leaves - // `record.engine_pid` telling cleanup that a child was spawned. The - // start-time token is captured here too, while the child is certainly - // ours: cleanup binds its kill to it rather than to whatever owns the - // pid by then. - record.record_engine_identity(child_pid); - thread::sleep(Duration::from_millis(200)); - if let Some(status) = child - .try_wait() - .context("failed to check managed engine startup state")? - { - bail!( - "{}", - managed_engine_startup_failure_detail( - &format!(" with status {status}"), - &record.log_path - ) - ); - } - child_pid - }; - // The child is both roles until the engine state file names a separate - // server process. Each role gets its own token, captured while the child is - // alive: a stop pairs every recorded PID with its own token, and the two - // PIDs diverge as soon as `rocmd` restarts the service — at which point an - // engine PID recorded without one could only be signalled blind. - // The engine's token was already captured right after the spawn (so cleanup - // can bind to it); re-capturing here could replace it with a later owner's. - record.record_supervisor_identity(child_pid); - record.status = "running".to_owned(); - if let Err(error) = record.write() { - // Cleanup is the caller's job now: `record.engine_pid` (set above) is - // already enough for `abandon_failed_managed_launch` to terminate this - // child, so there is nothing left for this inner path to do beyond - // surfacing the error. - return Err(error.context("failed to publish the managed service record")); - } - - Ok(child_pid) -} - -/// How long one readiness wait polls before `rocm serve` reports `starting`. -const READINESS_WAIT: Duration = Duration::from_secs(45); - -/// `rocm serve --wait`: keep waiting past [`READINESS_WAIT`] (a first start -/// downloads the model, which can take far longer) until the server answers or -/// stops being live. A server that exited ends the wait with the last reading -/// instead of blocking forever. -fn wait_past_readiness_budget( - mut probe: impl FnMut() -> EndpointReadiness, - mut still_live: impl FnMut() -> bool, -) -> EndpointReadiness { - loop { - let readiness = probe(); - if readiness == EndpointReadiness::Serving || !still_live() { - return readiness; - } - } -} - -/// Whether `pid` is a running process that is not a zombie. The serve -/// supervisor is an unreaped child of this process, so after it dies -/// `process_is_running` alone keeps reporting it alive. -fn service_pid_is_alive(pid: u32) -> bool { - rocm_core::process_is_running(pid) && !process_is_zombie(pid) -} - -#[cfg(target_os = "linux")] -fn process_is_zombie(pid: u32) -> bool { - let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { - return false; - }; - // `comm` may contain spaces and parentheses; the state follows the last `)`. - stat.rfind(')') - .and_then(|end| stat.get(end + 1..)) - .and_then(|rest| rest.split_whitespace().next()) - == Some("Z") -} - -#[cfg(not(target_os = "linux"))] -const fn process_is_zombie(_pid: u32) -> bool { - false -} - -/// Whether the record still describes a server that can finish starting: its -/// status is a live one *and* a tracked process is alive. The status alone is -/// not enough, since a killed engine never writes a terminal state. -fn record_still_live(record: &ManagedServiceRecord, pid_alive: impl Fn(u32) -> bool) -> bool { - if !crate::managed_service_status::managed_service_is_live(record) { - return false; - } - let pids = crate::managed_service_status::recorded_service_pids(record); - pids.is_empty() || pids.into_iter().any(pid_alive) -} - -/// [`record_still_live`] over the manifest on disk, so a `rocm services stop` -/// from another terminal (which writes only the manifest) ends the wait too. -fn service_still_live(paths: &AppPaths, service_id: &str, pid_alive: impl Fn(u32) -> bool) -> bool { - let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { - return false; - }; - let _ = latest.refresh_from_engine_state(); - record_still_live(&latest, pid_alive) -} - -/// Record a server that died mid-start as stopped, leaving a manifest that a -/// `services stop` already settled untouched. A `failed` state the engine -/// reported is persisted rather than overwritten, so the reason survives. -fn settle_exited_service(paths: &AppPaths, service_id: &str) { - let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { - return; - }; - let on_disk = latest.status.clone(); - let _ = latest.refresh_from_engine_state(); - if crate::managed_service_status::managed_service_is_live(&latest) { - latest.status = "stopped".to_owned(); - } else if latest.status == on_disk { - return; - } - let _ = latest.write(); -} - -/// The failure a `--wait` ends with when the server died before it became -/// ready, after recording the death in the manifest. `None` while the wait did -/// not apply, the endpoint is serving, or the server is still alive. -fn died_during_wait( - paths: &AppPaths, - service_id: &str, - wait_until_ready: bool, - readiness: EndpointReadiness, - pid_alive: impl Fn(u32) -> bool, -) -> Option { - if !wait_until_ready - || readiness == EndpointReadiness::Serving - || service_still_live(paths, service_id, pid_alive) - { - return None; - } - settle_exited_service(paths, service_id); - let mut message = format!( - "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" - ); - // A plain `serve` adds the memory guidance for a vLLM launch that ran out of - // VRAM; the wait's failure is the same launch, so it gets the same note. - // "starting" is the status the note keys on for "never became ready", which is - // what a `--wait` death is, whatever the settled record now says. - if let Ok(record) = ManagedServiceRecord::read_manifest(paths, service_id) { - for note in crate::serve_summary::append_oom_serve_note( - Vec::new(), - &record.engine, - false, - "starting", - Some(&record.log_path), - ) { - let _ = write!(message, "\n note: {note}"); - } - } - Some(anyhow!(message)) -} - -/// Record the outcome of a finished wait on the manifest as it is now, changing -/// only what the wait learned. The copy read at spawn time can be long stale: -/// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` -/// refreshes the record, so writing it back whole would revert those. -fn record_wait_outcome( - paths: &AppPaths, - held: &mut ManagedServiceRecord, - status: &str, - verified: bool, -) -> Result<()> { - let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) - .unwrap_or_else(|_| held.clone()); - status.clone_into(&mut latest.status); - if verified { - // Latch the verification the wait just performed, so the readiness checks - // behind `services list` and chat read it instead of re-probing. - latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); - } - latest.write()?; - *held = latest; - Ok(()) -} - -/// Test-only fault-injection switch, off (`const false`) in shipped binaries: -/// the `e2e-oom-fault-injection` feature is enabled only by the `xtask e2e` -/// mock-lane build, never by the release build. When enabled and -/// `ROCM_E2E_SIMULATE_OOM_LAUNCH=1` is set on the process, `rocm serve` fabricates -/// the on-disk state of a managed vLLM launch that spawned, wrote an -/// out-of-memory traceback to the log it owns, and never became ready — so the -/// positive OOM-guidance summary path can be verified on a GPU-less CI host, -/// where a real launch cannot run. See [`simulate_oom_managed_launch`]. -#[cfg(feature = "e2e-oom-fault-injection")] -pub(crate) fn e2e_simulate_oom_launch() -> bool { - std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "1") -} - -/// Release builds carry no fault-injection switch: the seam compiles out -/// entirely, so `rocm serve` behaves identically to a build without the feature. -#[cfg(not(feature = "e2e-oom-fault-injection"))] -pub(crate) const fn e2e_simulate_oom_launch() -> bool { - false -} - -/// Test-only (see [`e2e_simulate_oom_launch`]): fabricate the on-disk state of a -/// managed vLLM launch that spawned, OOM'd, and never became ready, then return -/// the launch report the summary path would see for it — `already_running: false` -/// (this invocation launched it), `status: "starting"` (it failed to become -/// ready), and a `log_path` whose contents carry a real allocator OOM signature. -/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob guidance. No process is -/// spawned. Compiled out of shipped binaries. -#[cfg(feature = "e2e-oom-fault-injection")] -#[allow(clippy::too_many_arguments)] -fn simulate_oom_managed_launch( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port: u16, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, -) -> Result { - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - let mut record = ManagedServiceRecord::new( - paths, - service_id, - engine, - requested_model, - resolve.canonical_model_id.clone(), - host, - port, - "managed", - std::process::id(), - runtime_id.map(str::to_owned), - env_id.map(str::to_owned), - Some(device_policy_name(device_policy).to_owned()), - ); - record.gpu_indices = gpu_indices.to_vec(); - record.status = "starting".to_owned(); - record.write()?; - // A real allocator OOM signature in the log this invocation owns, so the - // summary renders memory guidance for a launch that actually OOM'd. - // - // The engine's own error block is reproduced below it, because that is what - // a real managed launch leaves on disk, and leaving it out hid a bug this - // scenario should have caught: the supervisor and the engine share one log, - // so when vLLM OOMs the engine bails with its startup-log context — which - // embeds rocm-cli's own OOM hint — and `exit_code_for` writes that whole - // error back into the same file, *after* the real line. A fixture carrying - // only the raw OOM line can never see the newest-first scan land on - // rocm-cli's own sentence instead of the user's error. - let real_line = "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB."; - fs::write( - &record.log_path, - format!( - "{real_line}\n\ - Error: vLLM server process exited before becoming ready (status: exit status: 1)\n\ - \n\ - Startup log, last {} lines as the terminal drew them:\n\ - {real_line}\n\ - \n\ - {}{real_line}). {}\n\ - For conditional remediation, run `{} 'vllm: {real_line}'`.\n", - rocm_engine_protocol::DEFAULT_LOG_TAIL_LINES, - rocm_core::VLLM_OOM_DETECTED_PREFIX, - rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT, - rocm_core::VLLM_OOM_DIAGNOSE_COMMAND, - ), - ) - .with_context(|| format!("failed to write {}", record.log_path.display()))?; - Ok(ManagedLaunchReport { - service_id: service_id.to_owned(), - endpoint_url: format!("{}/v1", format_http_base_url(host, port)), - status: "starting".to_owned(), - already_running: false, - child_pid: None, - log_path: Some(record.log_path), - manifest_path: Some(record.manifest_path), - }) -} - -/// Probe `record`'s endpoint until it answers: once within the readiness -/// budget, or (with `wait_until_ready`) for as long as the server stays alive. -fn wait_for_record_ready( - paths: &AppPaths, - record: &ManagedServiceRecord, - wait_until_ready: bool, - endpoint_api_key: Option<&str>, - on_wait_tick: &mut dyn FnMut(Duration), -) -> EndpointReadiness { - let probe = |on_tick: &mut dyn FnMut(Duration)| { - wait_for_service_http_ready_with_progress( - &record.engine, - &record.host, - record.port, - &|| { - let mut latest = record.clone(); - let _ = latest.refresh_from_engine_state(); - latest.served_model_name().to_owned() - }, - endpoint_api_key, - READINESS_WAIT, - on_tick, - ) - }; - if wait_until_ready { - wait_past_readiness_budget( - || probe(&mut *on_wait_tick), - || service_still_live(paths, &record.service_id, service_pid_is_alive), - ) - } else { - probe(on_wait_tick) - } -} - -/// An equivalent service is already live. Without `--wait`, or once it is -/// serving, report it as is; otherwise wait for it like a fresh launch would. -fn wait_on_existing_service( - paths: &AppPaths, - mut report: ManagedLaunchReport, - wait_until_ready: bool, - on_wait_begin: &mut dyn FnMut(&str, u16), - on_wait_tick: &mut dyn FnMut(Duration), -) -> Result { - if !wait_until_ready || report.status == "ready" { - return Ok(report); - } - let Ok(mut record) = ManagedServiceRecord::read_manifest(paths, &report.service_id) else { - return Ok(report); - }; - // The id of the server being waited on, which is not the caller's freshly - // generated one when it attached to an existing server. - on_wait_begin(&report.service_id, record.port); - // The key the caller generated belongs to a launch that did not happen; the - // existing service answers to its own. - let stored_key = endpoint_keys::endpoint_api_key(paths, &report.service_id); - let readiness = - wait_for_record_ready(paths, &record, true, stored_key.as_deref(), on_wait_tick); - if let Some(error) = died_during_wait( - paths, - &report.service_id, - true, - readiness, - service_pid_is_alive, - ) { - return Err(error); - } - report.status = status_for_readiness(readiness).to_owned(); - record_wait_outcome( - paths, - &mut record, - &report.status, - readiness == EndpointReadiness::Serving, - )?; - Ok(report) -} - -/// Launch a managed service in the background: run the locked allocation and -/// spawn transaction, release the launch lock once the record is published, -/// then wait for HTTP readiness. -/// -/// Drops the endpoint key `serve()` freshly staged for this attempt if the -/// launch fails before any `ManagedServiceRecord` exists to own it (port -/// reserved/in use, range exhausted, other bind error) — such a failure -/// returns before `abandon_failed_managed_launch` ever runs, so nothing else -/// would clear it. Scoped to the spawn call only, mirroring -/// [`run_attached_service`]'s identical guard: a later failure (the readiness -/// wait, `record.write()`) leaves a record a live child actually owns, so it -/// must not also wipe that key. -#[allow(clippy::too_many_arguments)] -pub(crate) fn start_managed_service( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port_request: serve_port::PortRequest, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - endpoint_api_key: Option<&str>, - launch_lock: rocm_core::FileLock, - require_api_key: bool, - gpu_refusal_waived_for_reuse: bool, - wait_until_ready: bool, - on_wait_begin: &mut dyn FnMut(&str, u16), - on_wait_tick: &mut dyn FnMut(Duration), -) -> Result { - #[cfg(feature = "e2e-oom-fault-injection")] - if e2e_simulate_oom_launch() { - return simulate_oom_managed_launch( - paths, - engine, - service_id, - requested_model, - resolve, - host, - port_request - .explicit() - .unwrap_or(rocm_core::DEFAULT_LOCAL_PORT), - device_policy, - gpu_indices, - runtime_id, - env_id, - ); - } - let (mut record, child_pid) = match spawn_managed_engine_child( - paths, - engine, - service_id, - requested_model, - resolve, - host, - port_request, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - require_api_key, - gpu_refusal_waived_for_reuse, - ) - .inspect_err(|_| { - drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); - })? { - ManagedSpawn::AlreadyRunning(report) => { - // `--wait` applies to a service that is still starting too: the - // natural retry after `rocm serve m` printed `starting`. - drop(launch_lock); - return wait_on_existing_service( - paths, - report, - wait_until_ready, - on_wait_begin, - on_wait_tick, - ); - } - ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), - }; - // The transaction resolved the request to one concrete port; everything - // below — readiness, the endpoint URL, the audit line — uses that. - let port = record.port; - // The claiming service record is now persisted, so the selected GPU is - // visible to any concurrent auto-selection. Release the launch lock before - // the readiness wait below, which can block for many seconds — holding it - // that long would needlessly serialize unrelated serves. - drop(launch_lock); - - #[cfg(windows)] - thread::sleep(Duration::from_millis(200)); - - if wait_until_ready { - on_wait_begin(service_id, port); - } - let readiness = wait_for_record_ready( - paths, - &record, - wait_until_ready, - endpoint_api_key, - on_wait_tick, - ); - if let Some(error) = died_during_wait( - paths, - service_id, - wait_until_ready, - readiness, - service_pid_is_alive, - ) { - return Err(error); - } - let launch_status = status_for_readiness(readiness); - record_wait_outcome( - paths, - &mut record, - launch_status, - readiness == EndpointReadiness::Serving, - )?; - let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); - record_cli_audit_event( - paths, - "service", - "managed_service_launch", - "info", - format!( - "launched managed service engine={} model={} endpoint={} readiness={}", - engine, resolve.canonical_model_id, endpoint_url, launch_status - ), - Some(service_id), - ); - Ok(ManagedLaunchReport { - service_id: service_id.to_owned(), - endpoint_url, - status: launch_status.to_owned(), - already_running: false, - child_pid: Some(child_pid), - log_path: Some(record.log_path), - manifest_path: Some(record.manifest_path), - }) -} - -/// Reproduce the original plain, line-by-line managed-launch output. Used for -/// non-interactive output (piped, CI, the chat assistant's `serve --managed`), -/// where the animated summary is inappropriate. The interactive path renders the -/// summary table via [`crate::serve_summary`] instead. -pub(crate) fn print_managed_launch_plain( - report: &ManagedLaunchReport, - endpoint_api_key: Option<&str>, -) { - if report.already_running { - println!("managed service already running"); - println!(" service_id: {}", report.service_id); - println!(" endpoint: {}", report.endpoint_url); - println!(" status: {}", report.status); - println!(" note: existing service detected; no second process spawned"); - return; - } - println!("managed service launched"); - println!(" service_id: {}", report.service_id); - if let Some(child_pid) = report.child_pid { - println!(" process_pid: {child_pid}"); - } - println!(" endpoint: {}", report.endpoint_url); - if let Some(key) = endpoint_api_key { - // Intentional one-time display of a freshly generated API key to the - // terminal so the user can copy it — the designed delivery channel - // documented on `render_endpoint_client_config`, not a log. The tag below - // is currently inert (Rust's CodeQL pack has no AlertSuppression.ql yet — - // github/codeql#21637) but will start working once that lands, since the - // tag must be the single line immediately before the flagged code. - print!( - // codeql[rust/cleartext-logging] - "{}", - render_endpoint_client_config(&report.endpoint_url, key) - ); - } - if let Some(log_path) = report.log_path.as_deref() { - println!(" log_path: {}", log_path.display()); - } - if let Some(manifest_path) = report.manifest_path.as_deref() { - println!(" manifest_path: {}", manifest_path.display()); - } - println!(" readiness: {}", report.status); -} - -/// The client configuration to show when a `--wait` begins, so a key generated for -/// a public bind reaches the user even if the wait is interrupted or fails. -/// `None` unless the wait is on the launch that key belongs to: an attach to an -/// already-starting server (`waiting_on` differs from `service_id`) answers to -/// its own key, which is not shown here. -pub(crate) fn wait_begin_client_config( - waiting_on: &str, - service_id: &str, - host: &str, - port: u16, - api_key: Option<&str>, -) -> Option { - let key = api_key.filter(|_| waiting_on == service_id)?; - let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); - Some(render_endpoint_client_config(&endpoint_url, key)) -} - -/// Render the one-time secure client configuration for a public, authenticated -/// endpoint. This is the *intended* channel for delivering the key to the user -/// (unlike logs/status, which must never contain it) — it prints the key once at -/// launch alongside a ready-to-use example. Callers only invoke this for a -/// non-loopback bind that generated/received a key. -fn render_endpoint_client_config(endpoint_url: &str, api_key: &str) -> String { - let mut out = String::new(); - let _ = writeln!(out, " api key: {api_key}"); - let _ = writeln!( - out, - " note: this key is shown only now - clients must send `Authorization: Bearer `" - ); - let _ = writeln!( - out, - " example: curl -H \"Authorization: Bearer {api_key}\" {endpoint_url}/models" - ); - out -} - -/// The "should we spawn?" decision for [`ensure_background_helper_running`], -/// factored out so it is testable hermetically (no spawn side effect). Returns -/// `true` when the file-based runtime state says the daemon is `running` AND its -/// recorded `daemon_pid` is a live process — i.e. a second spawn must be guarded. -/// A missing state file, `running=false`, or a dead/zero pid returns `false`. -fn background_helper_already_running(paths: &AppPaths) -> Result { - Ok(AutomationRuntimeState::load(paths)? - .is_some_and(|state| state.running && rocm_core::process_is_running(state.daemon_pid))) -} - -/// What the user should be told about the background automation helper -/// (`rocm daemon`) after stopping a service or when planning an uninstall. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct HelperNote { - /// The sentence to print. - pub(crate) text: String, - /// The helper is running right now. - pub(crate) running: bool, - /// An automation watcher is enabled, so the helper stays up on its own. - pub(crate) outlives: bool, -} - -/// The note for one combination of the two facts the helper itself acts on: -/// whether a watcher is enabled -/// ([`rocm_core::RocmCliConfig::automation_daemon_enabled`] gates both the spawn -/// and the helper's exit) and whether it is running. `None` for neither. -fn helper_note_for(enabled: &[&str], running: bool) -> Option { - let text = match (enabled.is_empty(), running) { - (true, false) => return None, - (true, true) => "the background automation helper (`rocm daemon`) is still running; no automation is enabled, so it exits on its own within a few seconds".to_owned(), - (false, true) => { - let disable = enabled - .iter() - .map(|id| format!("`rocm automations disable {id}`")) - .collect::>() - .join(", "); - format!( - "the background automation helper (`rocm daemon`) keeps running while an automation is enabled ({}); to stop it, run {disable}, after which it exits on its own within a few seconds", - enabled.join(", ") - ) - } - (false, false) => format!( - "an automation is enabled ({}) but the background automation helper (`rocm daemon`) is not running; `rocm serve` or `rocm automations enable` starts it", - enabled.join(", ") - ), - }; - Some(HelperNote { - text, - running, - outlives: !enabled.is_empty() && running, - }) -} - -/// [`helper_note_for`] over the facts on disk. An unreadable config counts as -/// "no watcher enabled". -pub(crate) fn background_helper_note(paths: &AppPaths) -> Option { - let config = rocm_core::RocmCliConfig::load(paths).unwrap_or_default(); - let enabled: Vec<&str> = config - .automations - .watchers - .iter() - .filter(|(_, watcher)| watcher.enabled) - .map(|(id, _)| id.as_str()) - .collect(); - let running = background_helper_already_running(paths).unwrap_or(false); - helper_note_for(&enabled, running) -} - -/// Shared daemon-lifecycle entrypoint: ensures the background automation helper -/// (`rocm daemon`) is running when an automation watcher needs it, spawning it -/// detached if not. Liveness is read from the file-based automation runtime -/// state. Intentionally `pub(crate)` — reused by both the `serve --managed` path -/// and `automations enable`. Only the spawn result itself (`command.spawn()` / -/// `spawn_detached_no_inherit`) is logged rather than propagated; setup errors -/// (path discovery, config load, stdio attach) still return `Err`. -/// -/// The daemon's only work is running enabled automation watchers (server -/// recovery included — `server-recover` is one of them and is off by default), -/// and it exits on its own once none is enabled. So with no watcher enabled -/// nothing is spawned: a `serve` would otherwise leave a no-op daemon running -/// long after its service was stopped (ROCMAI-827). -pub(crate) fn ensure_background_helper_running() -> Result<()> { - ensure_background_helper_running_quiet(false) -} - -/// As [`ensure_background_helper_running`], but suppresses the stdout status line -/// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so -/// the daemon-spawn note does not appear above the deployment summary table. -pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { - let paths = AppPaths::discover()?; - ensure_background_helper_running_with(&paths, quiet, spawn_background_helper, &|| {}) -} - -/// The body of [`ensure_background_helper_running_quiet`], with the actual -/// process spawn injected so the decision around it is testable without -/// launching a daemon. `spawn` returns `Err` for a setup failure (propagated) -/// and `Ok(Err(..))` for a failed spawn (reported, not fatal). `on_lock_busy` is -/// called each time the autostart lock is found held by another process, which -/// lets a test wait for this call to be genuinely queued behind it. -fn ensure_background_helper_running_with( - paths: &AppPaths, - quiet: bool, - spawn: impl FnOnce(&AppPaths) -> Result>, - on_lock_busy: &dyn Fn(), -) -> Result<()> { - // An unreadable config is "no watcher": this also runs after `serve` has - // already launched its child, where failing would report a launch that - // happened as an error. - if !rocm_core::RocmCliConfig::load(paths).is_ok_and(|config| config.automation_daemon_enabled()) - { - return Ok(()); - } - - // Two concurrent callers (e.g. two `rocm serve`) can both read "not running" - // and each spawn a daemon. Serialize the decision on a lock file — the first - // holder spawns, later holders observe the now-running daemon and return - // without spawning. - // - // The "already running" read is deliberately made only under the lock, with - // no lock-free fast path. The daemon exits by itself once the last watcher - // is disabled, and it decides that under this same lock and publishes - // `running = false` before releasing it. Caller order is: save the enabling - // config, then come here. A lock-free read could still see the daemon that is - // about to exit as running, spawn nothing, and leave a watcher enabled with - // no daemon; under the lock the daemon has either seen the new config and - // stays, or has already published that it stopped. - let _autostart_lock = loop { - if let Some(lock) = - rocm_core::FileLock::try_acquire(paths.automation_autostart_lock_path())? - { - break lock; - } - on_lock_busy(); - std::thread::sleep(std::time::Duration::from_millis(10)); - }; - if background_helper_already_running(paths)? { - return Ok(()); - } - - // The lock alone does not close the window: the spawned daemon does not - // publish its `running` runtime state until well after `spawn()` (clap parse, - // runtime build, config load, banner flush). A second caller that acquires - // this lock during that gap still sees "not running" and would spawn a - // duplicate. Bridge the gap with a short-lived claim recording the child PID - // and spawn time: a holder that finds a live, recent claim defers instead. - let claim_path = paths.automation_autostart_claim_path(); - if autostart_spawn_in_flight( - read_autostart_claim(&claim_path), - now_unix_millis(), - AUTOSTART_CLAIM_TTL_MS, - rocm_core::process_is_running, - ) { - return Ok(()); - } - - match spawn(paths)? { - Ok(daemon_pid) => { - // Record the claim before returning (and thus releasing the lock) so a - // concurrent holder in the spawn→publish window defers. Best-effort: a - // failed write only reopens the original, already-tolerated race. - let _ = write_autostart_claim( - &claim_path, - AutostartClaim { - daemon_pid, - spawned_at_ms: now_unix_millis(), - }, - ); - if !quiet { - println!(" helper: started background automation daemon"); - } - } - Err(error) if !quiet => { - println!(" helper: could not start background automation daemon: {error}"); - } - Err(_) => {} - } - Ok(()) -} - -/// Spawn `rocm daemon` detached from this process. The outer `Result` is a setup -/// failure (resolving the executable, attaching stdio); the inner one is the -/// spawn itself. -fn spawn_background_helper(paths: &AppPaths) -> Result> { - let exe = managed_service_launcher_path() - .context("failed to resolve current rocm executable path")?; - let args = vec!["daemon".to_owned()]; - #[cfg(windows)] - let spawn_result = { - let env_values = app_path_env_var_values(paths, None); - let env_refs = app_path_env_var_refs(&env_values); - rocm_core::spawn_detached_no_inherit(&exe, &args, &env_refs) - }; - #[cfg(not(windows))] - let spawn_result = { - let mut command = managed_service_process_command(&exe, &args); - command.stdin(Stdio::null()); - attach_background_stdio(&mut command, None)?; - detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); - command - .spawn() - .map(|child| child.id()) - .map_err(anyhow::Error::from) - }; - Ok(spawn_result) -} - -/// How long an autostart claim is honoured before it is treated as stale even if -/// its recorded PID is still alive. Comfortably longer than a cold daemon boot -/// (clap parse → runtime build → config load → state publish) yet short enough -/// that a crashed spawn cannot suppress autostart for long. -const AUTOSTART_CLAIM_TTL_MS: u128 = 30_000; - -/// A just-spawned daemon's autostart claim: the child PID and the wall-clock time -/// (milliseconds since the Unix epoch) the spawn was recorded. It lets a -/// concurrent autostart holder distinguish a live, in-flight spawn from a stale -/// leftover. Serialized as a single `" "` line — no dependency and -/// trivially forward-compatible. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct AutostartClaim { - daemon_pid: u32, - spawned_at_ms: u128, -} - -/// Milliseconds since the Unix epoch, or `0` if the clock is before the epoch -/// (which only makes a fresh claim look old — safe, it just permits a respawn). -fn now_unix_millis() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_millis()) -} - -/// Read an autostart claim, returning `None` when the file is absent or -/// unparseable (either is treated as "no claim", so a respawn is permitted). -fn read_autostart_claim(path: &Path) -> Option { - let text = std::fs::read_to_string(path).ok()?; - let mut parts = text.split_whitespace(); - let daemon_pid = parts.next()?.parse().ok()?; - let spawned_at_ms = parts.next()?.parse().ok()?; - Some(AutostartClaim { - daemon_pid, - spawned_at_ms, - }) -} - -/// Write an autostart claim as `" "`. Best-effort at the call site. -fn write_autostart_claim(path: &Path, claim: AutostartClaim) -> std::io::Result<()> { - std::fs::write( - path, - format!("{} {}", claim.daemon_pid, claim.spawned_at_ms), - ) -} - -/// Whether an existing autostart `claim` means a daemon spawn is still in flight, -/// so the current lock holder should defer rather than spawn a duplicate. A claim -/// counts as in-flight only while its child PID is alive *and* it is younger than -/// `ttl_ms` — the TTL bounds how long a crashed spawn (or a PID later reused by an -/// unrelated process) can suppress autostart. `pid_alive` is injected so the -/// decision is unit-testable without a live process. -fn autostart_spawn_in_flight( - claim: Option, - now_ms: u128, - ttl_ms: u128, - pid_alive: impl Fn(u32) -> bool, -) -> bool { - claim.is_some_and(|claim| { - now_ms.saturating_sub(claim.spawned_at_ms) < ttl_ms && pid_alive(claim.daemon_pid) - }) -} - -/// What ended an attached (`--verbose`/`--foreground`) streaming session. Kept -/// as a plain enum, separate from any terminal I/O, so the follow-up action -/// (detach note vs. stop the server) is unit-testable without a TTY. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum AttachOutcome { - /// Ctrl-D: leave the server running and hand the terminal back. - Detach, - /// Ctrl-C: stop the server, then hand the terminal back. - Stop, - /// The engine process exited on its own while we were streaming its log. - ServerExited, -} - -/// Attached serve path for `--verbose`/`--foreground`: spawn the engine as a -/// detached managed child (the same child the background path spawns) and stream -/// its log in this terminal. Unlike the old in-process foreground, the server -/// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. -#[allow(clippy::too_many_arguments)] -pub(crate) fn run_attached_service( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port_request: serve_port::PortRequest, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - endpoint_api_key: Option<&str>, - launch_lock: rocm_core::FileLock, - require_api_key: bool, - gpu_refusal_waived_for_reuse: bool, -) -> Result<()> { - let spawn = spawn_managed_engine_child( - paths, - engine, - service_id, - requested_model, - resolve, - host, - port_request, - &resolve.device_policy, - gpu_indices, - runtime_id, - env_id, - resolve.engine_recipe.as_ref(), - require_api_key, - gpu_refusal_waived_for_reuse, - ) - // Mirrors `start_managed_service`'s cleanup: a pre-spawn - // failure (port reserved/in use, range exhausted, other bind error) returns - // before any service record exists, so nothing else clears the key just - // stored for this attempt. - .inspect_err(|_| { - drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); - })?; - // The claiming record is persisted (or an existing service was found), so the - // selected GPU is now visible to concurrent auto-selection. Release the launch - // lock before streaming logs, which blocks for the whole attached session. - drop(launch_lock); - - // Attached mode is a managed record too, so it shares the transaction's - // resolved concrete port. - let (service_id, log_path, port, child_pid) = match spawn { - // A server for this engine+model is already live. Don't fight it for the - // port — point the user at the existing one instead of tailing a log we - // did not start. - ManagedSpawn::AlreadyRunning(report) => { - println!("model already being served"); - println!(" service_id: {}", report.service_id); - println!(" endpoint: {}", report.endpoint_url); - println!(" status: {}", report.status); - println!(" logs: rocm logs {}", report.service_id); - println!( - " stop: {}", - service_hints::service_stop_hint(&report.service_id) - ); - drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); - return Ok(()); - } - ManagedSpawn::Spawned { record, child_pid } => ( - service_id.to_owned(), - record.log_path.clone(), - record.port, - child_pid, - ), - }; - - // The child is a managed service that outlives this session once detached. - // The background helper adds one thing for it: with the `server-recover` - // watcher enabled, it health-checks managed servers and restarts failed ones. - // Every duty of the helper is watcher-gated, so with no watcher enabled no - // helper is started. Nothing else depends on it: a self-exited server's - // record is settled by its own supervisor (`supervise_service` writes the - // exit via `record_supervised_exit` in `apps/rocmd/src/supervise.rs`) and on - // every read (`refresh_managed_service_runtime_liveness`), and the dashboard - // has its own embedded daemon. Match the background ordering (spawn, then - // ensure the helper) and keep it quiet so no status line breaks into the - // log stream. - ensure_background_helper_running_quiet(true)?; - - // The resolution detail (model, engine, runtime, GPU, warnings) was already - // printed as the "serve plan" block in `serve()`; extend it with the launch - // coordinates and the streaming hint rather than repeating it. - let endpoint = format!("{}/v1", format_http_base_url(host, port)); - println!(" service_id: {service_id}"); - println!(" endpoint: {endpoint}"); - if let Some(key) = endpoint_api_key { - // Same intentional one-time key display as `print_managed_launch_plain` - // above; see its rationale for why the tag below is currently inert. - // codeql[rust/cleartext-logging] - print!("{}", render_endpoint_client_config(&endpoint, key)); - } - println!(" streaming engine logs - Ctrl-D detaches (leaves it running), Ctrl-C stops it"); - println!(); - - let outcome = stream_attached_logs(&log_path, child_pid)?; - println!(); - - match outcome { - AttachOutcome::Detach => { - // Audit before printing: a closed stdout makes `println!` panic, - // which must not lose the event. - record_cli_audit_event( - paths, - "service", - "serve_detach", - "info", - format!("detached from attached serve service_id={service_id} endpoint={endpoint}"), - Some(&service_id), - ); - println!("detached - server still running"); - println!(" service_id: {service_id}"); - println!(" endpoint: {endpoint}"); - println!(" list: rocm services"); - println!(" logs: rocm logs {service_id}"); - println!(" stop: {}", service_hints::service_stop_hint(&service_id)); - Ok(()) - } - AttachOutcome::Stop => { - println!("stopping server..."); - let stop = run_internal_sandbox_tool( - paths, - SandboxToolArg::StopServer, - Some(service_id.clone()), - true, - ); - finish_attached_stop( - paths, - &service_id, - child_pid, - stop, - rocm_core::terminate_process_tree, - &mut io::stdout().lock(), - &mut io::stderr().lock(), - ) - } - AttachOutcome::ServerExited => { - println!("server process exited"); - println!(" service_id: {service_id}"); - println!(" recent logs: rocm logs {service_id}"); - Ok(()) - } - } -} - -/// Restores cooked terminal mode when dropped, so [`stream_attached_logs`] leaves -/// the terminal usable on every exit path (normal return, `?` error, or panic). -struct RawModeGuard; - -impl Drop for RawModeGuard { - fn drop(&mut self) { - let _ = crossterm::terminal::disable_raw_mode(); - } -} - -/// Map a key press (control modifier + lowercased character) to the attach -/// action it triggers, if any. Factored out of the raw-mode reader loop so the -/// Ctrl-D/Ctrl-C mapping is unit-testable without a terminal. -const fn detach_key_outcome(ctrl: bool, ch: char) -> Option { - if !ctrl { - return None; - } - match ch { - 'c' => Some(AttachOutcome::Stop), - 'd' => Some(AttachOutcome::Detach), - _ => None, - } -} - -/// Follow `log_path` in the terminal until the user presses Ctrl-D (detach) or -/// Ctrl-C (stop), or the engine process exits. Uses crossterm raw mode to -/// capture the keys directly (in raw mode Ctrl-C does not raise SIGINT, so we see -/// it as a key event). When stdin is not a TTY (piped/CI), keystroke capture is -/// impossible, so we follow the log until the process exits instead. -fn stream_attached_logs(log_path: &Path, child_pid: u32) -> Result { - use std::io::IsTerminal as _; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::mpsc; - - if !std::io::stdin().is_terminal() { - return stream_attached_logs_no_tty(log_path, child_pid); - } - - // Enter raw mode *before* spawning the key reader. In raw mode Ctrl-C arrives - // as a key event instead of SIGINT; if the reader started first, a Ctrl-C in - // that window would kill the CLI outright (leaving the detached child alive - // but printing no detach/stop message). The guard restores cooked mode on - // every exit path (normal return, `?` error, panic). - crossterm::terminal::enable_raw_mode().context("failed to enter raw terminal mode")?; - let _raw_guard = RawModeGuard; - - let stop = Arc::new(AtomicBool::new(false)); - let (tx, rx) = mpsc::channel::(); - - let reader_stop = Arc::clone(&stop); - let reader = thread::spawn(move || { - use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; - while !reader_stop.load(Ordering::Relaxed) { - match event::poll(Duration::from_millis(100)) { - Ok(true) => match event::read() { - Ok(Event::Key(key)) if key.kind != KeyEventKind::Release => { - let outcome = match key.code { - KeyCode::Char(ch) => detach_key_outcome( - key.modifiers.contains(KeyModifiers::CONTROL), - ch.to_ascii_lowercase(), - ), - _ => None, - }; - if let Some(outcome) = outcome { - let _ = tx.send(outcome); - break; - } - } - Ok(_) => {} - Err(_) => break, - }, - Ok(false) => {} - Err(_) => break, - } - } - }); - - let mut stdout = io::stdout(); - let mut log_reader: Option> = None; - let mut line = String::new(); - let outcome = loop { - if log_reader.is_none() { - log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); - } - if let Some(reader) = log_reader.as_mut() { - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - // Raw mode disables the terminal's own \n -> \r\n - // translation, so emit an explicit carriage return to - // keep the log left-aligned instead of stair-stepping. - let _ = write!(stdout, "{}\r\n", line.trim_end_matches('\n')); - let _ = stdout.flush(); - } - Err(_) => break, - } - } - } - - if let Ok(signal) = rx.try_recv() { - break signal; - } - if !process_is_running(child_pid) { - break AttachOutcome::ServerExited; - } - thread::sleep(Duration::from_millis(150)); - }; - - stop.store(true, Ordering::Relaxed); - let _ = reader.join(); - Ok(outcome) -} - -/// Non-interactive fallback for [`stream_attached_logs`]: no keystroke capture, -/// so just follow the log until the (detached) engine process exits. A Ctrl-C -/// here delivers SIGINT to this process and leaves the managed server running. -fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result { - let mut stdout = io::stdout(); - let mut log_reader: Option> = None; - let mut line = String::new(); - loop { - if log_reader.is_none() { - log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); - } - if let Some(reader) = log_reader.as_mut() { - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - let _ = write!(stdout, "{line}"); - let _ = stdout.flush(); - } - Err(_) => break, - } - } - } - if !process_is_running(child_pid) { - return Ok(AttachOutcome::ServerExited); - } - thread::sleep(Duration::from_millis(200)); - } -} - -/// Finish the attached `rocm serve` Ctrl-C stop once the managed stop has -/// answered: what is printed where, what is signalled, what is audited, and -/// how the command exits. Everything but the stop itself and the real -/// signal is here, so a test can drive it. -/// -/// - Confirmed: the report, with the background-helper note `rocm services -/// stop` also prints, goes to `out` and the command succeeds. -/// - Unconfirmed: the same report `rocm services stop` gives, check lines -/// included, becomes the error, so it reaches stderr and the command exits -/// non-zero. Nothing is signalled on the stop's behalf: the stop has already -/// done what it safely could. -/// - The stop itself failed: `signal_child` gets the child this session -/// launched, so Ctrl-C never leaves the model orphaned when the stop path -/// breaks. The stop's error, and the signal's if that fails too, are noted on -/// `err`, and the audit event records whether the signal worked. The command -/// still exits 0, as it always has here. -fn finish_attached_stop( - paths: &AppPaths, - service_id: &str, - child_pid: u32, - stop: Result, - signal_child: impl FnOnce(u32) -> Result<()>, - out: &mut dyn Write, - err: &mut dyn Write, -) -> Result<()> { - match stop { - Ok(result) if service_stop_unconfirmed(&result) => { - record_cli_audit_event( - paths, - "service", - "serve_stop", - "error", - format!("stop of attached serve not confirmed service_id={service_id}"), - Some(service_id), - ); - bail!( - "{}", - render_service_action_result("stop_server", &result).trim_end() - ) - } - Ok(result) => { - record_cli_audit_event( - paths, - "service", - "serve_stop", - "info", - format!("stopped attached serve service_id={service_id}"), - Some(service_id), - ); - write!( - out, - "{}", - render_service_action_result_with_helper_note(paths, "stop_server", &result) - )?; - Ok(()) - } - Err(error) => { - let signal = signal_child(child_pid); - // Record the audit event before writing to stderr, so a failed - // write cannot skip it. - let message = match &signal { - Ok(()) => format!( - "stop failed; signaled attached serve child {child_pid} service_id={service_id}" - ), - Err(signal_error) => format!( - "stop failed; signaling attached serve child {child_pid} also failed: {signal_error:#} service_id={service_id}" - ), - }; - record_cli_audit_event( - paths, - "service", - "serve_stop", - if signal.is_ok() { "warn" } else { "error" }, - message, - Some(service_id), - ); - writeln!(err, " note: {error:#}")?; - if let Err(signal_error) = &signal { - writeln!( - err, - " note: signaling this session's server process {child_pid} directly also failed: {signal_error:#}" - )?; - } - Ok(()) - } - } -} - -#[cfg(test)] -mod tests { - #[test] - fn the_no_gpu_refusal_sends_a_bare_metal_user_to_dev_kfd() { - let text = super::no_usable_gpu_refusal_text("gpu_required", false); - assert!(text.contains("confirm /dev/kfd is present"), "{text}"); - assert!(!text.contains("ROCDXG"), "{text}"); - } - - #[test] - fn the_no_gpu_refusal_does_not_send_a_wsl_user_to_dev_kfd() { - // WSL has no /dev/kfd, so advice to confirm it can never clear the - // condition. The text names the status `examine` reports for the path - // the probe asked about, taken from the constant the producer uses, so - // the two cannot drift. That the status really is the one a refused - // host shows is asserted end to end by serve-42. - let text = super::no_usable_gpu_refusal_text("gpu_required", true); - assert!(!text.contains("/dev/kfd is present"), "{text}"); - assert!(text.contains("rocm examine"), "{text}"); - assert!(text.contains("rocm diagnose"), "{text}"); - // The advice is true only because `rocdxg_ready` counts a library on - // LD_LIBRARY_PATH; `a_rocdxg_only_on_the_loader_path_is_ready` in - // `rocm-core`'s host_gpu/wsl.rs proves that behavior. - assert!(text.contains("LD_LIBRARY_PATH"), "{text}"); - assert!( - text.contains(&format!("`{}`", rocm_core::WSL_ROCDXG_READY)), - "{text}" - ); - } - - #[test] - fn the_no_gpu_refusal_names_the_policy_on_both_platforms() { - for on_wsl in [false, true] { - let text = super::no_usable_gpu_refusal_text("gpu_required", on_wsl); - assert!( - text.starts_with("no usable AMD GPU detected; `rocm serve` requires a GPU under the gpu_required policy"), - "{text}" - ); - } - } - use super::*; - - #[test] - fn a_child_that_exited_during_the_startup_window_is_reported_with_its_log() -> Result<()> { - let dir = tempfile::tempdir()?; - let log = dir.path().join("engine.log"); - fs::write(&log, "boom: engine could not start\n")?; - let settled = std::cell::Cell::new(false); - let error = ensure_spawned_child_survives_startup( - 4242, - &log, - || settled.set(true), - |pid| { - assert_eq!(pid, 4242); - assert!(settled.get(), "liveness must be probed after the wait"); - false - }, - ) - .expect_err("a dead child must fail the launch"); - assert!(settled.get(), "the startup window must elapse first"); - assert!(format!("{error:#}").contains("boom"), "{error:#}"); - ensure_spawned_child_survives_startup(4242, &log, || {}, |_| true)?; - Ok(()) - } - use crate::tests::test_paths; - use rocm_core::DEFAULT_LOCAL_HOST; - - #[test] - fn wait_past_readiness_budget_keeps_probing_a_live_starting_server() { - let mut readings = vec![ - EndpointReadiness::Unreachable, - EndpointReadiness::Unreachable, - EndpointReadiness::Serving, - ] - .into_iter(); - let mut probes = 0; - let result = wait_past_readiness_budget( - || { - probes += 1; - readings.next().expect("no probe after Serving") - }, - || true, - ); - assert_eq!(result, EndpointReadiness::Serving); - assert_eq!(probes, 3, "must outlast the first (45s) budget"); - } - - #[test] - fn wait_past_readiness_budget_stops_when_the_server_exits() { - let mut probes = 0; - let result = wait_past_readiness_budget( - || { - probes += 1; - EndpointReadiness::Unreachable - }, - || false, - ); - assert_eq!(result, EndpointReadiness::Unreachable); - assert_eq!(probes, 1); - } - - #[test] - fn wait_past_readiness_budget_returns_at_once_when_already_serving() { - let result = wait_past_readiness_budget( - || EndpointReadiness::Serving, - || panic!("liveness is irrelevant once serving"), - ); - assert_eq!(result, EndpointReadiness::Serving); - } - - fn record_tracking_pid(paths: &AppPaths, pid: u32, status: &str) -> ManagedServiceRecord { - let mut record = reusable_record(paths); - record.supervisor_pid = pid; - record.status = status.to_owned(); - record - } - - #[cfg(unix)] - fn exited_child_pid() -> u32 { - let mut child = std::process::Command::new("true").spawn().expect("spawn"); - let pid = child.id(); - child.wait().expect("reap"); - pid - } - - #[test] - #[cfg(unix)] - fn record_still_live_ends_the_wait_for_a_dead_pid_with_a_live_status() { - // The engine was killed before writing any terminal state: the status - // still says `starting`, but nothing tracked is running. - let (root, paths) = test_paths("wait-dead-pid"); - paths.ensure().expect("paths"); - let dead = record_tracking_pid(&paths, exited_child_pid(), "starting"); - assert!(!record_still_live(&dead, service_pid_is_alive)); - - let alive = record_tracking_pid(&paths, std::process::id(), "starting"); - assert!(record_still_live(&alive, service_pid_is_alive)); - - // A settled status ends the wait even while the pid is still around. - let stopped = record_tracking_pid(&paths, std::process::id(), "stopped"); - assert!(!record_still_live(&stopped, service_pid_is_alive)); - let _ = fs::remove_dir_all(root); - } - - #[test] - #[cfg(target_os = "linux")] - fn service_pid_is_alive_treats_an_unreaped_child_as_dead() { - // The serve supervisor is our own unreaped child; once it dies it is a - // zombie that `kill(pid, 0)` still reports as running. - let mut child = std::process::Command::new("true").spawn().expect("spawn"); - let pid = child.id(); - let deadline = std::time::Instant::now() + Duration::from_secs(10); - while !process_is_zombie(pid) { - assert!(std::time::Instant::now() < deadline, "child never exited"); - thread::sleep(Duration::from_millis(10)); - } - assert!( - process_is_running(pid), - "a zombie still answers kill(pid, 0)" - ); - assert!(!service_pid_is_alive(pid)); - child.wait().expect("reap"); - } - - #[test] - fn service_still_live_follows_the_manifest_on_disk() { - // `rocm services stop` from another terminal writes only the manifest. - let (root, paths) = test_paths("wait-manifest"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); - record.write().expect("write"); - assert!(service_still_live(&paths, &record.service_id, |_| true)); - - record.status = "stopped".to_owned(); - record.write().expect("write"); - assert!(!service_still_live(&paths, &record.service_id, |_| true)); - assert!(!service_still_live(&paths, "no-such-service", |_| true)); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn settle_exited_service_demotes_a_live_manifest_and_spares_a_settled_one() { - let (root, paths) = test_paths("wait-settle"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, 0, "starting"); - record.write().expect("write"); - settle_exited_service(&paths, &record.service_id); - let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); - assert_eq!(after.status, "stopped"); - - // A record that already failed keeps its reason: the guard, not the - // overwrite, is what spares it (a `stopped` -> `stopped` rewrite would - // be invisible). - record.status = "failed".to_owned(); - record.write().expect("write"); - settle_exited_service(&paths, &record.service_id); - let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); - assert_eq!(after.status, "failed"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn wait_on_existing_service_only_waits_for_one_that_is_not_serving_yet() { - // The probing path itself blocks for the readiness budget, so only the - // cases that must return at once are driven here; a manifest planted - // as `starting` would hang a wrong guard for 45 seconds and fail it. - let (root, paths) = test_paths("wait-existing"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, 0, "starting"); - record.write().expect("write"); - let report = |status: &str| ManagedLaunchReport { - service_id: record.service_id.clone(), - endpoint_url: record.endpoint_url.clone(), - status: status.to_owned(), - already_running: true, - child_pid: None, - log_path: None, - manifest_path: None, - }; - let mut no_tick = |_: Duration| panic!("must not probe"); - - let untouched = wait_on_existing_service( - &paths, - report("starting"), - false, - &mut |_, _| panic!("no --wait, no banner"), - &mut no_tick, - ) - .expect("no --wait"); - assert_eq!(untouched.status, "starting"); - let ready = wait_on_existing_service( - &paths, - report("ready"), - true, - &mut |_, _| panic!("already ready, no banner"), - &mut no_tick, - ) - .expect("already ready"); - assert_eq!(ready.status, "ready"); - record.status = "starting".to_owned(); - assert_eq!( - ManagedServiceRecord::read_manifest(&paths, &record.service_id) - .unwrap() - .status, - "starting" - ); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn died_during_wait_fails_with_a_logs_pointer_and_records_the_death() { - let (root, paths) = test_paths("wait-died"); - paths.ensure().expect("paths"); - let record = record_tracking_pid(&paths, std::process::id(), "starting"); - record.write().expect("write"); - let id = record.service_id; - let status = || { - ManagedServiceRecord::read_manifest(&paths, &id) - .unwrap() - .status - }; - - // Not waiting, already serving, or still alive: no failure, no change. - let pending = EndpointReadiness::Unreachable; - assert!(died_during_wait(&paths, &id, false, pending, |_| false).is_none()); - assert!( - died_during_wait(&paths, &id, true, EndpointReadiness::Serving, |_| false).is_none() - ); - assert!(died_during_wait(&paths, &id, true, pending, |_| true).is_none()); - assert_eq!(status(), "starting"); - - // Dead server: the error names the logs command and the record is settled. - let error = died_during_wait(&paths, &id, true, pending, |_| false).expect("died"); - let message = error.to_string(); - assert!( - message.contains("exited before it became ready"), - "{message}" - ); - assert!( - message.contains(&format!("rocm logs --service {id}")), - "{message}" - ); - assert_eq!(status(), "stopped"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn died_during_wait_adds_the_oom_guidance_for_a_vllm_launch_only() { - let (root, paths) = test_paths("wait-died-oom"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); - record.engine = "vllm".to_owned(); - record.write().expect("write"); - fs::write( - &record.log_path, - "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.\n", - ) - .expect("log"); - let id = record.service_id; - let pending = EndpointReadiness::Unreachable; - let message = died_during_wait(&paths, &id, true, pending, |_| false) - .expect("died") - .to_string(); - assert!(message.contains("--gpu-memory-utilization"), "{message}"); - - // Same log, another engine: that engine has no such flag to suggest. - let mut record = ManagedServiceRecord::read_manifest(&paths, &id).unwrap(); - record.engine = "lemonade".to_owned(); - "starting".clone_into(&mut record.status); - record.write().expect("write"); - let message = died_during_wait(&paths, &id, true, pending, |_| false) - .expect("died") - .to_string(); - assert!(!message.contains("--gpu-memory-utilization"), "{message}"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn record_wait_outcome_keeps_what_changed_on_disk_during_the_wait() { - let (root, paths) = test_paths("wait-outcome"); - paths.ensure().expect("paths"); - let mut held = record_tracking_pid(&paths, 111, "starting"); - held.write().expect("write"); - // While the wait ran, `rocmd` recorded a restart and a new supervisor. - let mut elsewhere = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); - elsewhere.restart_count = 3; - elsewhere.supervisor_pid = 222; - elsewhere.write().expect("write"); - - record_wait_outcome(&paths, &mut held, "ready", true).expect("outcome"); - let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); - assert_eq!(after.status, "ready"); - assert!(after.inference_verified_at_unix_ms.is_some()); - assert_eq!(after.restart_count, 3, "the restart count was reverted"); - assert_eq!(after.supervisor_pid, 222, "the supervisor pid was reverted"); - - record_wait_outcome(&paths, &mut held, "starting", false).expect("outcome"); - let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); - assert_eq!(after.status, "starting"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn wait_begin_shows_the_key_only_for_the_launch_it_belongs_to() { - let own = wait_begin_client_config("svc-new", "svc-new", "0.0.0.0", 8000, Some("k-secret")) - .expect("own launch shows its key"); - // No `{own}` in the failure messages: the text carries the dummy key, - // and formatting it into an assert is what CodeQL flags as logging it. - assert!(own.contains("api key: k-secret"), "key line missing"); - assert!( - own.contains("http://0.0.0.0:8000/v1/models"), - "models url missing" - ); - // Attached to an already-starting server: its key is not this one. - assert!( - wait_begin_client_config("svc-old", "svc-new", "0.0.0.0", 8000, Some("k-secret")) - .is_none() - ); - // Loopback binds have no key to show. - assert!(wait_begin_client_config("svc-new", "svc-new", "127.0.0.1", 8000, None).is_none()); - } - - #[test] - fn serve_bind_validation_requires_public_ack() { - validate_bind_host("127.0.0.1", false).unwrap(); - validate_bind_host("localhost", false).unwrap(); - validate_bind_host("::1", false).unwrap(); - let error = validate_bind_host("0.0.0.0", false).unwrap_err(); - assert!( - error.to_string().contains("--allow-public-bind"), - "{error:#}" - ); - validate_bind_host("0.0.0.0", true).unwrap(); - } - - /// A `stop_server` answer whose one recorded process ended with `outcome`. - fn stop_answer(confirmed: bool, outcome: &str) -> serde_json::Value { - crate::services_text::internal_stop_server_value(serde_json::json!({ - "service_id": "svc-attached", - "status": "ready", - "signaled_pids": if outcome == "unverified" { vec![] } else { vec![4242] }, - "pid_outcomes": [{ "pid": 4242, "role": "supervisor_and_engine", "outcome": outcome }], - // An attached serve on a public bind stores a key; an unconfirmed - // stop keeps it. - "endpoint_key_kept": !confirmed, - "stopped": confirmed, - })) - } - - /// Run [`finish_attached_stop`] for `stop`, returning its result, what it - /// wrote to stdout, the PID it signalled (if any) and the audit event. - fn finish( - name: &str, - stop: Result, - ) -> (Result<()>, String, Option, serde_json::Value) { - finish_signalling(name, stop, Ok(())) - } - - /// [`finish`], with the direct signal answering `signal_result`. - fn finish_signalling( - name: &str, - stop: Result, - signal_result: Result<()>, - ) -> (Result<()>, String, Option, serde_json::Value) { - let (result, out, _err, signalled, audit) = finish_capturing(name, stop, signal_result); - (result, out, signalled, audit) - } - - /// [`finish_signalling`], also returning what was written to stderr. - fn finish_capturing( - name: &str, - stop: Result, - signal_result: Result<()>, - ) -> (Result<()>, String, String, Option, serde_json::Value) { - let (root, paths) = test_paths(name); - paths.ensure().unwrap(); - let mut out = Vec::new(); - let mut err = Vec::new(); - let mut signalled = None; - let result = finish_attached_stop( - &paths, - "svc-attached", - 777, - stop, - |pid| { - signalled = Some(pid); - signal_result - }, - &mut out, - &mut err, - ); - let audit_text = fs::read_to_string(paths.audit_events_path()).expect("audit log"); - let audit = serde_json::from_str(audit_text.lines().last().expect("an audit event")) - .expect("audit json"); - let _ = fs::remove_dir_all(root); - ( - result, - String::from_utf8(out).unwrap(), - String::from_utf8(err).unwrap(), - signalled, - audit, - ) - } - - #[test] - fn attached_ctrl_c_reports_a_confirmed_stop_on_stdout() { - let (result, out, signalled, audit) = - finish("attached-confirmed", Ok(stop_answer(true, "graceful"))); - result.expect("a confirmed stop succeeds"); - assert!(out.contains("Local server stopped"), "{out}"); - assert_eq!(signalled, None); - assert_eq!(audit["level"], "info", "{audit}"); - assert_eq!( - audit["message"], - "stopped attached serve service_id=svc-attached" - ); - } - - #[test] - fn attached_ctrl_c_fails_an_unconfirmed_stop_like_services_stop_and_signals_nothing() { - // The same report `rocm services stop` gives, as the error (stderr, - // non-zero exit), naming no retry; nothing is signalled on the stop's - // behalf and nothing reaches stdout. - let answer = stop_answer(false, "timed_out"); - let (result, out, signalled, audit) = finish("attached-timed-out", Ok(answer.clone())); - let error = format!("{:#}", result.expect_err("an unconfirmed stop fails")); - assert_eq!( - error, - render_service_action_result("stop_server", &answer).trim_end(), - "the report is `rocm services stop`'s own" - ); - assert!( - error.contains( - "process 4242 (supervisor_and_engine) or a process under it did not exit" - ), - "{error}" - ); - assert!( - error.lines().any(|line| line - == " kept: the server record, which still names its processes, and its endpoint key, which the server may still be enforcing"), - "{error}" - ); - crate::services_text::tests::assert_no_retry_line(&error); - assert!(out.is_empty(), "nothing on stdout: {out}"); - assert_eq!(signalled, None, "an unconfirmed stop signals nothing more"); - assert_eq!(audit["level"], "error", "{audit}"); - assert_eq!( - audit["message"], - "stop of attached serve not confirmed service_id=svc-attached" - ); - } - - #[test] - fn attached_ctrl_c_names_a_check_for_an_unverified_process() { - let (result, out, signalled, audit) = - finish("attached-unverified", Ok(stop_answer(false, "unverified"))); - let error = format!("{:#}", result.expect_err("an unverified stop fails")); - assert!(error.contains("not signaled: process 4242"), "{error}"); - assert!(error.contains("check: `ps -p 4242 -o pid,args`"), "{error}"); - crate::services_text::tests::assert_no_retry_line(&error); - assert!(out.is_empty(), "{out}"); - assert_eq!(signalled, None); - assert_eq!(audit["level"], "error", "{audit}"); - } - - #[test] - fn attached_ctrl_c_signals_its_own_child_when_the_stop_itself_fails() { - let (result, out, err, signalled, audit) = finish_capturing( - "attached-stop-error", - Err(anyhow::anyhow!("managed service `svc-attached` not found") - .context("failed to stop svc-attached")), - Ok(()), - ); - result.expect("a failed stop falls back and still exits cleanly, as before"); - assert_eq!(signalled, Some(777)); - assert_eq!( - err, " note: failed to stop svc-attached: managed service `svc-attached` not found\n", - "the stop's error goes to stderr, and a signal that worked adds nothing" - ); - assert!(out.is_empty(), "nothing on stdout: {out}"); - assert_eq!(audit["level"], "warn", "{audit}"); - assert_eq!( - audit["message"], - "stop failed; signaled attached serve child 777 service_id=svc-attached" - ); - - // The direct signal failing too is printed on stderr and recorded, with - // its error. - let (result, _, err, signalled, audit) = finish_capturing( - "attached-stop-error-signal-fails", - Err(anyhow::anyhow!("managed service `svc-attached` not found")), - Err(anyhow::anyhow!("no such process")), - ); - result.expect("still exits 0, as on main"); - assert_eq!( - err, - " note: managed service `svc-attached` not found\n \ - note: signaling this session's server process 777 directly also failed: no such process\n", - "both notes go to stderr" - ); - assert_eq!(signalled, Some(777)); - assert_eq!(audit["level"], "error", "{audit}"); - assert_eq!( - audit["message"], - "stop failed; signaling attached serve child 777 also failed: no such process service_id=svc-attached" - ); - } - - /// A writer whose every write fails, standing in for a closed stdout/stderr. - struct BrokenWriter; - - impl Write for BrokenWriter { - fn write(&mut self, _: &[u8]) -> std::io::Result { - Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe)) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - #[test] - fn attached_ctrl_c_records_the_audit_event_even_when_writing_the_outcome_fails() { - // One case per branch that writes after deciding: a confirmed stop - // (stdout) and a stop that itself failed (stderr notes). The write - // error is returned, but the audit event is already on disk. - let cases: [(&str, Result, &str); 2] = [ - ( - "attached-broken-out", - Ok(stop_answer(true, "graceful")), - "stopped attached serve service_id=svc-attached", - ), - ( - "attached-broken-err", - Err(anyhow::anyhow!("managed service `svc-attached` not found")), - "stop failed; signaled attached serve child 777 service_id=svc-attached", - ), - ]; - for (name, stop, expected_message) in cases { - let (root, paths) = test_paths(name); - paths.ensure().unwrap(); - let result = finish_attached_stop( - &paths, - "svc-attached", - 777, - stop, - |_| Ok(()), - &mut BrokenWriter, - &mut BrokenWriter, - ); - let audit_text = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); - let _ = fs::remove_dir_all(root); - assert!(result.is_err(), "{name}: the write error is surfaced"); - let audit: serde_json::Value = serde_json::from_str( - audit_text - .lines() - .last() - .unwrap_or_else(|| panic!("{name}: no audit event recorded")), - ) - .expect("audit json"); - assert_eq!(audit["message"], expected_message, "{name}: {audit}"); - } - } - - #[test] - fn detach_key_ctrl_d_detaches_ctrl_c_stops() { - assert_eq!(detach_key_outcome(true, 'd'), Some(AttachOutcome::Detach)); - assert_eq!(detach_key_outcome(true, 'c'), Some(AttachOutcome::Stop)); - } - - #[test] - fn detach_key_ignores_plain_and_unrelated_keys() { - // Without the control modifier, `d`/`c` are ordinary log-scroll input. - assert_eq!(detach_key_outcome(false, 'd'), None); - assert_eq!(detach_key_outcome(false, 'c'), None); - // Other control combos are not detach/stop triggers. - assert_eq!(detach_key_outcome(true, 'q'), None); - assert_eq!(detach_key_outcome(true, 'z'), None); - } - - /// A record with every reuse-relevant field set to the "reusable" value, so - /// each test below breaks exactly one term and nothing else. - fn reusable_record(paths: &AppPaths) -> ManagedServiceRecord { - let mut record = ManagedServiceRecord::new( - paths, - "vllm-qwen-1000", - "vllm", - "qwen", - "qwen-canonical".to_owned(), - "127.0.0.1", - 8000, - "managed", - 0, - None, - None, - Some("gpu_required".to_owned()), - ); - record.engine_recipe_json = None; - record.requires_api_key = true; - record - } - - #[test] - fn reuse_is_refused_on_every_term_the_spawn_and_the_gpu_bail_depend_on() { - let (root, paths) = test_paths("managed-reuse-predicate"); - paths.ensure().expect("paths"); - let base = reusable_record(&paths); - - // The baseline must be reusable, or the negative cases below could pass - // for the wrong reason. - assert!( - managed_service_reuse_refusal(&base, None, true, &DevicePolicy::GpuRequired).is_none(), - "a matching service with no recipe, a key, and a gpu_required policy is reusable" - ); - - // Recipe mismatch — the spawn's first refusal. - let mut mismatched = base.clone(); - mismatched.engine_recipe_json = Some("{\"engine\":\"vllm\"}".to_owned()); - assert!(matches!( - managed_service_reuse_refusal(&mismatched, None, true, &DevicePolicy::GpuRequired), - Some(ManagedReuseRefusal::RecipeMismatch) - )); - - // Unmet `--require-api-key` — the spawn's second refusal. Also pins that - // it is *only* refused when the key is actually demanded. - let mut anonymous = base.clone(); - anonymous.requires_api_key = false; - assert!(matches!( - managed_service_reuse_refusal(&anonymous, None, true, &DevicePolicy::GpuRequired), - Some(ManagedReuseRefusal::MissingApiKey) - )); - assert!( - managed_service_reuse_refusal(&anonymous, None, false, &DevicePolicy::GpuRequired) - .is_none(), - "an anonymous service is fine when no key was demanded" - ); - - // Device policy — the term `serve()` depends on, because it is the one - // gating the no-usable-GPU bail on the answer. Reusing a `cpu_only` - // service to answer a `gpu_required` serve is the silent CPU fallback. - let mut cpu_only = base; - cpu_only.device_policy = Some("cpu_only".to_owned()); - assert!(matches!( - managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::GpuRequired), - Some(ManagedReuseRefusal::DevicePolicyTooWeak) - )); - // `GpuRequired` is the only policy the CLI can produce — `--device - // gpu_preferred`/`auto`/`gpu` all parse to it and `cpu_only` is rejected - // (`parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`) — - // so there is no second policy arm to discriminate here, and asserting - // one would pin an input `serve()` never produces. Both callers pass - // the same policy; the parameter is not optional, so neither can drop - // this term without a compile error. - // - // A cpu_only *invocation* never reaches the bail, so nothing to refuse. - assert!( - managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::CpuOnly).is_none() - ); - - let _ = std::fs::remove_dir_all(&root); - } - - #[test] - fn spawn_managed_engine_child_refuses_an_explicit_port_a_live_equivalent_does_not_serve() - -> Result<()> { - let (root, paths) = test_paths("dup-managed-explicit-port-differs"); - paths.ensure()?; - let mut existing = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11510, - "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let launch = |port_request| { - spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-2000", - "qwen", - &resolve, - "127.0.0.1", - port_request, - &resolve.device_policy, - &[], - None, - None, - None, - false, - false, - ) - }; - - let different = launch(serve_port::PortRequest::Explicit(11511)); - // The same port, and no port at all, still reuse the live service. - let same = launch(serve_port::PortRequest::Explicit(11510)); - let auto = launch(serve_port::PortRequest::Auto); - let _ = fs::remove_dir_all(root); - - let Err(error) = different else { - panic!("a port the live service does not serve must be refused") - }; - let message = error.to_string(); - assert!( - message.contains("on port 11510") && message.contains("port 11511 would not be used"), - "{message}" - ); - assert!(matches!(same?, ManagedSpawn::AlreadyRunning(_))); - assert!(matches!(auto?, ManagedSpawn::AlreadyRunning(_))); - Ok(()) - } - - #[test] - fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { - // A live service recorded with one recipe (e.g. a tool-call parser flag) - // must reject a relaunch requesting a different recipe rather than - // silently reusing the old server, and the error must not claim the - // mismatch is specifically about generation defaults when it could stem - // from any recipe field. - let (root, paths) = test_paths("dup-managed-recipe-mismatch"); - paths.ensure()?; - let mut existing = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11510, - "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { - contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], - ..EngineRecipeHint::default() - })?); - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let requested_recipe = EngineRecipeHint { - contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], - ..EngineRecipeHint::default() - }; - - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-2000", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11511), - &resolve.device_policy, - &[], - None, - None, - Some(&requested_recipe), - false, - false, - ); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!("mismatched recipe on a live service must be rejected") - }; - let message = error.to_string(); - assert!( - message.contains("different serve options"), - "message should describe the mismatch generically: {message}" - ); - assert!( - message.contains("recipe hint, tool-call parser, or generation defaults"), - "message should not single out generation defaults as the sole cause: {message}" - ); - Ok(()) - } - - #[test] - fn spawn_managed_engine_child_refuses_to_reuse_an_unauthenticated_service() -> Result<()> { - // `--require-api-key` used to be accepted and dropped on this path. The - // reuse branch returns before the flag is recorded and before the - // endpoint key is checked, so a caller demanding auth got a server that - // never had it, with no error. `rocm remote serve` then published that - // endpoint onto the tailnet and printed a freshly minted key under "the - // API key above is what stops anyone else calling it" — a false - // assurance about an endpoint the whole tailnet can reach. - // - // Asserted through `spawn_managed_engine_child` rather than against the - // guard's own arguments: the defect was the early return, so only the - // real call site can fail for it. - let (root, paths) = test_paths("dup-managed-unauthenticated-reuse"); - paths.ensure()?; - let mut existing = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11520, - "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - // The state that matters: live, matching, and serving without auth. - existing.requires_api_key = false; - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-3001", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11520), - &resolve.device_policy, - &[], - None, - None, - None, - // The demand that used to be silently discarded. - true, - false, - ); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!( - "reusing an unauthenticated service must not satisfy `--require-api-key`; a \ - satisfied reuse leaves the endpoint open while the caller is told it is not" - ) - }; - let message = error.to_string(); - assert!( - message.contains("without authentication"), - "the refusal must say why it refused: {message}" - ); - // The printed hint must be the exact, directly-runnable command: - // `main.rs`'s `service_actions_require_yes_and_render_sandbox_result` - // proves this exact form (`rocm services stop --yes`) is the one - // the approval gate itself recommends. That running this form actually - // clears *this* refusal — a service reused without the key it is now - // being asked to require — is proven by - // `@id:serve-require-api-key-refuses-unauthenticated-reuse` - // (`tests/e2e-cucumber/features/model_serving.feature`, serve-24), - // which reproduces this exact condition, runs the printed hint, and - // asserts the record is left `stopped` — not by `stop_managed_services` - // in `tests/e2e-cucumber/tests/e2e.rs`, which runs this same - // invocation as best-effort teardown and never asserts success. - assert!( - message.contains("rocm services stop lemonade-qwen-3000 --yes"), - "the refusal must name the way out, with the service to stop, directly runnable with --yes: {message}" - ); - Ok(()) - } - - #[test] - fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { - // Drives the real call site, not the guard's own arguments. The service - // was launched with `--require-api-key`, and its key file is present but - // invalid — so `requires_api_key` is true while `key_present` is false, - // which is the only way to reach the first branch. A test calling - // `ensure_public_service_has_endpoint_key` directly cannot catch a - // mis-wired call site, which is the defect that has occurred here twice. - // - // The flag is passed explicitly rather than inferred from the key file. - // Inferring it marked every public bind as having demanded auth, because - // a public bind always has a key file whether or not it asked for one. - let (root, paths) = test_paths("managed-spawn-invalid-key"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // Non-empty but invalid (embedded CR/LF), so the file exists and is not - // merely empty (requires_api_key = true) but yields no usable key - // (key_present = false) — `endpoint_api_key_from_file` rejects embedded - // control characters. An empty seed here would let the post-condition - // below pass even if the real cleanup call never ran, since - // `endpoint_api_key` already reads back `None` for an empty file whether - // or not it was ever deleted. - endpoint_keys::store_endpoint_api_key( - &paths, - "lemonade-qwen-3000", - "bad-key\r\nX-Injected: value", - )?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-3000", - "qwen", - &resolve, - // Loopback on purpose: the public-bind branch must not be what - // refuses this, or the test would pass with the guard disabled. - "127.0.0.1", - serve_port::PortRequest::Explicit(11512), - &resolve.device_policy, - &[], - None, - None, - None, - true, - false, - ); - - // This guard fires before any child is spawned, so `record.engine_pid` - // is still `None` when `spawn_managed_engine_child`'s cleanup arm calls - // `abandon_failed_managed_launch` — proving its pre-spawn branch - // (delete everything) runs from the real call site, not just from the - // `abandoned_launch_leaves_no_record_log_or_endpoint_key` test that - // calls the helper directly. - let manifest_path = paths.service_manifest_path("lemonade-qwen-3000"); - let log_path = paths.service_log_path("lemonade-qwen-3000"); - let engine_state_path = paths.service_engine_state_path("lemonade", "lemonade-qwen-3000"); - let key_file_path = endpoint_keys::endpoint_key_file_path(&paths, "lemonade-qwen-3000"); - let live = live_managed_services(&paths); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!("a service requiring a key must not spawn with an unusable key file") - }; - let message = error.to_string(); - assert!( - message.contains("--require-api-key"), - "the refusal must name the flag the service was launched with: {message}" - ); - assert!( - message.contains("without authentication"), - "the refusal must say what the risk is: {message}" - ); - assert!( - !manifest_path.exists(), - "a pre-spawn failure must not leave a record" - ); - assert!( - !log_path.exists(), - "a pre-spawn failure must not leave a log" - ); - assert!( - !engine_state_path.exists(), - "a pre-spawn failure must not leave engine state" - ); - assert!( - !key_file_path.exists(), - "a pre-spawn failure must delete the unusable key file from disk, not merely \ - leave it unreadable as a key" - ); - assert!( - live.is_empty(), - "a pre-spawn failure must leave no reservation" - ); - Ok(()) - } - - #[test] - fn a_failed_spawn_past_the_first_write_terminates_the_child_and_keeps_an_abandoned_record() - -> Result<()> { - // Drives the real call site's *post*-spawn cleanup branch, which - // neither `abandoned_launch_*` test reaches (they call - // `abandon_failed_managed_launch` directly). An engine name this - // binary does not build in makes the real spawned child (a second - // `rocm` process) exit almost immediately via - // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after - // `launch_managed_engine_child_locked` has already set - // `record.engine_pid` — so cleanup must terminate (already-dead) the - // child, keep the record and log for inspection, mark the record - // `stopped` (a `failed` one would be respawned by rocmd recovery), and - // clear only the endpoint key. - // - // The child always dies near-instantly, but `spawn_managed_engine_child` - // only reports that within a fixed startup-check window. On a loaded or - // antivirus-scanning Windows CI box, a fresh debug `rocm.exe` can - // occasionally be slow enough to start that the window is missed even - // though the child is already doomed — so poll by retrying a fresh - // attempt (its own service id/port, so no stale manifest collides) up - // to a deadline instead of asserting on one race-prone attempt. - let deadline = std::time::Instant::now() + Duration::from_secs(10); - let mut attempt: u16 = 0; - let (root, paths, service_id, result) = loop { - attempt += 1; - let (root, paths) = - test_paths(&format!("managed-spawn-unknown-engine-cleanup-{attempt}")); - paths.ensure()?; - let service_id = format!("no-such-engine-qwen-{}", 4000 + attempt); - endpoint_keys::store_endpoint_api_key(&paths, &service_id, "test-key")?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let result = spawn_managed_engine_child( - &paths, - "no-such-engine", - &service_id, - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11_514 + attempt), - &resolve.device_policy, - &[], - None, - None, - None, - false, - false, - ); - - if result.is_ok() && std::time::Instant::now() < deadline { - let _ = fs::remove_dir_all(&root); - continue; - } - break (root, paths, service_id, result); - }; - - let manifest_path = paths.service_manifest_path(&service_id); - let log_path = paths.service_log_path(&service_id); - let persisted = load_managed_services(&paths) - .expect("re-read manifests") - .into_iter() - .find(|candidate| candidate.service_id == service_id); - let key = endpoint_keys::endpoint_api_key(&paths, &service_id); - let live = live_managed_services(&paths); - - let Err(error) = result else { - panic!("an engine this binary does not build in must not report success") - }; - let message = error.to_string(); - assert!( - message.contains("managed engine exited immediately"), - "the failure must come from the spawned child dying, not an earlier guard: {message}" - ); - assert!( - manifest_path.exists(), - "a post-spawn failure must keep the record for inspection" - ); - assert!( - log_path.exists(), - "a post-spawn failure must keep the log for inspection" - ); - let persisted = - persisted.expect("a post-spawn failure must leave the record on disk, not delete it"); - // `stopped`, not `failed`: rocmd's recovery restarts a `failed` record - // without a stop marker, which would respawn a launch the CLI just - // reported as failed (pinned on the rocmd side by - // `abandoned_launch_record_is_not_a_recovery_candidate`). - assert_eq!( - persisted.status, "stopped", - "a post-spawn failure must mark the record abandoned, not leave it stuck" - ); - assert_eq!( - key, None, - "a post-spawn failure must still clear the endpoint key" - ); - assert!( - live.is_empty(), - "an abandoned record must not still reserve its port" - ); - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { - // serve() stores the new attempt's endpoint key before calling - // start_managed_service. If spawn_managed_engine_child fails inside - // resolve_managed_port_in_transaction (explicit port already reserved - // by another live service), it returns before any - // ManagedServiceRecord exists for this attempt, so - // abandon_failed_managed_launch never runs. Driving the real function - // end to end (not a simulated call to - // drop_orphaned_endpoint_key_on_already_running) proves - // `start_managed_service`'s own `.inspect_err` actually wires the cleanup, mirroring how - // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key - // drives run_attached_service for the foreground path. - let (root, paths) = test_paths("orphaned-key-on-reserved-port"); - paths.ensure()?; - let mut other = ManagedServiceRecord::new( - &paths, - "lemonade-llama-1000", - "lemonade", - "llama", - "llama-canonical", - "127.0.0.1", - 11513, - "managed", - std::process::id(), - None, - None, - None, - ); - other.status = "ready".to_owned(); - other.engine_pid = Some(std::process::id()); - other.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let service_id = "lemonade-qwen-2000"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; - let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - - let result = start_managed_service( - &paths, - "lemonade", - service_id, - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11513), - &resolve.device_policy, - &[], - None, - None, - None, - Some("test-key"), - launch_lock, - false, - false, - false, - &mut |_, _| {}, - &mut |_elapsed| {}, - ); - let Err(error) = result else { - let _ = fs::remove_dir_all(root); - panic!("explicit port already reserved by a live service must be rejected") - }; - assert!( - error - .to_string() - .contains("already reserved by managed service"), - "unexpected error: {error}" - ); - - assert!( - endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), - "cleanup must drop the key this failed attempt staged before launching" - ); - - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - fn attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key() -> Result<()> { - // The background path's cleanup is the `.inspect_err` in - // start_managed_service. run_attached_service is the other caller of - // spawn_managed_engine_child and needs the identical .inspect_err() - // cleanup, not a simulated one: drive the real function end to end. - let (root, paths) = test_paths("orphaned-key-attached-reserved-port"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - let mut other = ManagedServiceRecord::new( - &paths, - "lemonade-llama-1000", - "lemonade", - "llama", - "llama-canonical", - "127.0.0.1", - 11514, - "managed", - std::process::id(), - None, - None, - None, - ); - other.status = "ready".to_owned(); - other.engine_pid = Some(std::process::id()); - other.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let service_id = "lemonade-qwen-attached-2000"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; - let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - - let result = run_attached_service( - &paths, - "lemonade", - service_id, - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11514), - &[], - None, - None, - Some("test-key"), - launch_lock, - false, - false, - ); - let Err(error) = result else { - let _ = fs::remove_dir_all(root); - panic!("explicit port already reserved by a live service must be rejected") - }; - assert!( - error - .to_string() - .contains("already reserved by managed service"), - "unexpected error: {error}" - ); - assert!( - endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), - "a pre-spawn failure must drop the key this attempt staged before launching" - ); - - let _ = fs::remove_dir_all(root); - Ok(()) - } - - /// The launch lock must still be held when the child is spawned: it is what - /// serializes allocation against publication. Observed through the spawn - /// hook, since releasing it early changes nothing else a test can see. - #[cfg(not(windows))] - #[test] - fn start_and_attached_launches_hold_the_launch_lock_until_the_spawn() -> Result<()> { - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - for attached in [false, true] { - let (root, paths) = test_paths("launch-lock-held-until-spawn"); - paths.ensure()?; - let lock_path = paths.managed_launch_lock_path(); - let lock_was_held = std::rc::Rc::new(std::cell::Cell::new(false)); - let observed = lock_was_held.clone(); - TEST_SPAWN_HOOK.with(|hook| { - *hook.borrow_mut() = Some(std::rc::Rc::new(move |_command| { - observed.set( - rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err(), - ); - Err(io::Error::other("spawn intercepted by test")) - })); - }); - let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - let result = if attached { - run_attached_service( - &paths, - "lemonade", - "lemonade-lock-attached", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Auto, - &[], - None, - None, - None, - launch_lock, - false, - false, - ) - } else { - start_managed_service( - &paths, - "lemonade", - "lemonade-lock-start", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Auto, - &resolve.device_policy, - &[], - None, - None, - None, - None, - launch_lock, - false, - false, - false, - &mut |_, _| {}, - &mut |_elapsed| {}, - ) - .map(|_| ()) - }; - TEST_SPAWN_HOOK.with(|hook| *hook.borrow_mut() = None); - let _ = fs::remove_dir_all(root); - assert!( - format!("{:#}", result.expect_err("the intercepted spawn fails")) - .contains("spawn intercepted by test") - ); - assert!( - lock_was_held.get(), - "the launch lock must be held at spawn (attached: {attached})" - ); - } - Ok(()) - } - - #[test] - fn resolve_endpoint_auth_loopback_stays_credential_free() { - // Loopback binds never require auth, even if a key is supplied. - for host in ["127.0.0.1", "localhost", "::1"] { - assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); - assert_eq!( - resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), - None - ); - } - } - - #[test] - fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { - // "Loopback" describes the bind address, not who can reach the port. A - // tailnet publish, a proxy, or a container port map all leave the bind - // loopback while widening the audience, and the default policy would - // hand out an unauthenticated endpoint. Whoever widens the reach asks - // for the credential explicitly. - for host in ["127.0.0.1", "localhost", "::1"] { - let generated = resolve_endpoint_auth(host, None, true) - .unwrap() - .expect("a required key must be generated, not skipped"); - assert!(!generated.trim().is_empty()); - - assert_eq!( - resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), - Some("supplied-key".to_owned()), - "a supplied key must be honoured rather than ignored as it is by default" - ); - } - - // The same validation a public bind gets: an empty key is a refusal, not - // a silent downgrade to no auth. - assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); - } - - #[test] - fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { - let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) - .unwrap() - .expect("public bind must have a key"); - assert_eq!(key, "my-key"); - } - - #[test] - fn resolve_endpoint_auth_public_generates_key_when_absent() { - let key = resolve_endpoint_auth("0.0.0.0", None, false) - .unwrap() - .expect("public bind must generate a key"); - assert_eq!(key.len(), 48); - assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); - } - - #[test] - fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { - let error = resolve_endpoint_auth("0.0.0.0", Some(" "), false).unwrap_err(); - assert!(error.to_string().contains("non-empty"), "{error:#}"); - } - - #[test] - fn resolve_endpoint_auth_public_rejects_embedded_crlf() { - // A supplied key survives `trim()` with embedded CR/LF intact and would - // otherwise be interpolated into a raw `Authorization: Bearer` header, - // injecting an extra header line. It must be rejected at input validation. - for supplied in [ - "good-key\r\nX-Injected: value", - "good-key\nmore", - "line\rreturn", - ] { - let error = resolve_endpoint_auth("0.0.0.0", Some(supplied), false).unwrap_err(); - assert!(error.to_string().contains("control character"), "{error:#}"); - } - } - - #[test] - fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { - let (root, paths) = test_paths("drop-orphaned-key-stored"); - let service_id = "svc-orphaned"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); - - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); - - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { - // A loopback attempt never stores a key (`freshly_stored == None`), so the - // helper must not panic or error, and no file must appear. - let (root, paths) = test_paths("drop-orphaned-key-loopback"); - let service_id = "svc-loopback"; - - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); - - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn public_bind_fails_closed_for_windows_lemonade_only() { - // Windows + Lemonade + public bind: refuse (cannot enforce the key). - let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); - assert!(error.to_string().contains("lemonade"), "{error:#}"); - // Every other combination is allowed: - ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows - ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows - ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key - } - - #[test] - fn a_public_bind_is_refused_with_the_command_that_restores_it() { - // `resolve_endpoint_auth` mints a key for every non-loopback bind whether - // or not auth was demanded, so deriving `requires_api_key` from key-file - // presence marked ordinary public binds as having asked for it. The guard - // tests that field first, so those services were refused with a message - // naming a flag they never passed and a relaunch command that drops - // `--allow-public-bind` — coming back on loopback instead. - // - // A plain `--host 0.0.0.0 --allow-public-bind` launch: key file present, - // `--require-api-key` never passed. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false) - .expect_err("a public bind with no key must be refused"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("--allow-public-bind"), - "the refusal must name the command that restores the public bind: {rendered}" - ); - assert!( - !rendered.contains("--require-api-key"), - "a service that never passed the flag must not be told it did: {rendered}" - ); - - // And the loopback-with-forced-auth case still reports its own reason. - let error = ensure_public_service_has_endpoint_key("127.0.0.1", false, true) - .expect_err("a service that demanded auth must not come back without it"); - assert!( - format!("{error:#}").contains("--require-api-key"), - "{error:#}" - ); - } - - #[test] - fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { - // A stop deletes the key file, so a later restart of a public service - // would otherwise respawn it with no auth at all. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); - let message = error.to_string(); - assert!(message.contains("0.0.0.0"), "{error:#}"); - assert!(message.contains("without authentication"), "{error:#}"); - // Actionable: name the command that mints a fresh key. - assert!(message.contains("--allow-public-bind"), "{error:#}"); - - // A public service that still has its key restarts normally. - ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); - } - - #[test] - fn respawn_allows_loopback_services_without_an_endpoint_key() { - // Loopback stays credential-free, so every accepted spelling must pass - // the guard with no key present — when nothing asked for auth. - for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false, false) - .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); - } - } - - #[test] - fn respawn_refuses_a_loopback_service_that_was_launched_with_a_key() { - // The hole this closes: a loopback bind that something else republishes - // — a tailnet publish, a proxy, a container port map. The publish - // outlives the process, so a restart after the key was dropped would - // reopen a reachable endpoint with no authentication, and the bind - // address gives the guard no way to notice. - for host in ["127.0.0.1", "localhost", "::1"] { - let error = ensure_public_service_has_endpoint_key(host, false, true) - .expect_err("a service launched with a key must not restart without one"); - let message = format!("{error:#}"); - assert!(message.contains("without authentication"), "{message}"); - assert!(message.contains("--require-api-key"), "{message}"); - } - - // With its key still present it restarts normally. - ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); - } - - #[test] - fn endpoint_client_config_shows_key_once_with_bearer_guidance() { - let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); - // Dummy literal exercising the intentional one-time key display documented - // on `render_endpoint_client_config`; not a real credential or a log write. - // codeql[rust/cleartext-logging] - assert!(rendered.contains("secret-123"), "{rendered}"); - // codeql[rust/cleartext-logging]: see rationale above. - assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); - // codeql[rust/cleartext-logging]: see rationale above. - assert!(rendered.contains("shown only now"), "{rendered}"); - } - - /// Build an `AutomationRuntimeState` for the no-double-spawn guard tests. - fn runtime_state(running: bool, daemon_pid: u32) -> AutomationRuntimeState { - AutomationRuntimeState { - running, - automations_enabled: true, - daemon_pid, - started_at_unix_ms: 1, - last_tick_unix_ms: 1, - local_webhook_endpoint: None, - active_watchers: Vec::new(), - } - } - - #[test] - fn background_helper_already_running_true_for_live_pid() { - // Phase-10 daemon no-double-spawn: a runtime-state.json with running=true - // and a LIVE daemon_pid (this very test process) means the helper is - // already up — the "should spawn?" decision must say NO (true ⇒ skip). - // Hermetic + offline: no spawn, just the file-based liveness check. - let (root, paths) = test_paths("helper-live-pid"); - runtime_state(true, std::process::id()) - .write(&paths) - .expect("write runtime state"); - assert!( - background_helper_already_running(&paths).expect("liveness check ok"), - "live recorded pid + running=true ⇒ do not spawn a second daemon" - ); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn background_helper_already_running_false_for_dead_or_missing() { - // The inverse guard cases — each must report NOT running (false ⇒ spawn): - // (1) no state file at all, (2) running=true but a dead/zero pid, - // (3) a live pid but running=false. None must spawn from this decision. - let (root, paths) = test_paths("helper-dead-or-missing"); - // (1) No state file yet. - assert!( - !background_helper_already_running(&paths).expect("missing state ⇒ ok"), - "no runtime state ⇒ not running" - ); - // (2) running=true but pid 0 is never a live process. - runtime_state(true, 0).write(&paths).expect("write state"); - assert!( - !background_helper_already_running(&paths).expect("dead pid ⇒ ok"), - "running=true + dead pid ⇒ not running (spawn)" - ); - // (3) live pid but running flag is false. - runtime_state(false, std::process::id()) - .write(&paths) - .expect("write state"); - assert!( - !background_helper_already_running(&paths).expect("not-running flag ⇒ ok"), - "running=false ⇒ not running even with a live pid" - ); - let _ = fs::remove_dir_all(&root); - } - - /// ROCMAI-827: `rocm serve` started the background daemon on every launch, - /// and with no automation watcher enabled that daemon had nothing to do yet - /// kept running after every service was stopped. With the default config - /// (no watcher enabled) the helper must not be spawned at all. - #[test] - fn background_helper_not_spawned_when_no_automation_is_enabled() { - let (root, paths) = test_paths("helper-no-automation"); - paths.ensure().expect("create app dirs"); - // The stored flag alone is not "a watcher is enabled": nothing would run. - let mut config = rocm_core::RocmCliConfig::default(); - config.automations.daemon_enabled = true; - config.save(&paths).expect("save default config"); - - let mut spawned = false; - ensure_background_helper_running_with( - &paths, - true, - |_| { - spawned = true; - Ok(Ok(4242)) - }, - &|| {}, - ) - .expect("ensure helper"); - - let claim_written = paths.automation_autostart_claim_path().exists(); - let _ = fs::remove_dir_all(&root); - assert!( - !spawned, - "no watcher enabled ⇒ the daemon has no work and must not be started" - ); - assert!(!claim_written, "no spawn ⇒ no autostart claim"); - } - - /// The other half of the gate: once a watcher is enabled the helper is still - /// started (exactly once) and its autostart claim recorded. - #[test] - fn background_helper_spawned_when_an_automation_is_enabled() { - let (root, paths) = test_paths("helper-automation-enabled"); - paths.ensure().expect("create app dirs"); - let mut config = rocm_core::RocmCliConfig::default(); - config.watcher_config_mut("server-recover").enabled = true; - config.save(&paths).expect("save config"); - - let mut spawns = 0; - ensure_background_helper_running_with( - &paths, - true, - |_| { - spawns += 1; - Ok(Ok(4242)) - }, - &|| {}, - ) - .expect("ensure helper"); - - let claim = read_autostart_claim(&paths.automation_autostart_claim_path()); - let _ = fs::remove_dir_all(&root); - assert_eq!(spawns, 1, "an enabled watcher needs the daemon"); - assert_eq!(claim.map(|claim| claim.daemon_pid), Some(4242)); - } - - /// ROCMAI-827 race, enable side: the daemon that is about to exit still has - /// `running = true` and a live pid in its state when `automations enable` - /// (config already saved) asks for a helper. It decides under the autostart - /// lock and publishes "stopped" before releasing it, so the enable must wait - /// on that lock and then start a new helper. A lock-free "already running" - /// read would return here without spawning, leaving a watcher with no daemon. - #[test] - fn background_helper_is_started_when_the_exiting_daemon_publishes_stopped() { - let (root, paths) = test_paths("helper-exit-window"); - paths.ensure().expect("create app dirs"); - let mut config = rocm_core::RocmCliConfig::default(); - config.watcher_config_mut("server-recover").enabled = true; - config.save(&paths).expect("save config"); - // The old daemon: alive (this process stands in) and still `running`. - runtime_state(true, std::process::id()) - .write(&paths) - .expect("write runtime state"); - - // The exiting daemon holds the autostart lock while it decides. - let exiting_daemon_lock = - rocm_core::FileLock::acquire(paths.automation_autostart_lock_path()) - .expect("hold the lock"); - let enable_paths = paths.clone(); - let queued_flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let queued = queued_flag.clone(); - let enable = std::thread::spawn(move || { - let mut spawns = 0; - ensure_background_helper_running_with( - &enable_paths, - true, - |_| { - spawns += 1; - Ok(Ok(4242)) - }, - &|| queued.store(true, std::sync::atomic::Ordering::SeqCst), - ) - .expect("ensure helper"); - spawns - }); - // Wait until the enable path has actually found the lock held and queued - // behind it, rather than assuming how long that takes. - let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); - // A caller that skipped the lock finishes without ever queuing. - while !queued_flag.load(std::sync::atomic::Ordering::SeqCst) && !enable.is_finished() { - assert!(std::time::Instant::now() < deadline, "enable never queued"); - std::thread::yield_now(); - } - runtime_state(false, std::process::id()) - .write(&paths) - .expect("daemon publishes stopped"); - drop(exiting_daemon_lock); - - let spawns = enable.join().expect("enable thread"); - let _ = fs::remove_dir_all(&root); - assert_eq!( - spawns, 1, - "the enable must start a helper once the old one has published it stopped" - ); - } - - fn stop_result() -> serde_json::Value { - serde_json::json!({"output": {"result": { - "service": {"service_id": "svc-a", "status": "stopped"}, - "signaled_pids": [1] - }}}) - } - - /// Put the on-disk facts in one of the four combinations: whether a watcher - /// is enabled, and whether the helper is running (this process stands in for - /// a live daemon). - fn set_helper_facts(paths: &AppPaths, watcher_enabled: bool, helper_running: bool) { - let mut config = rocm_core::RocmCliConfig::default(); - config.watcher_config_mut("server-recover").enabled = watcher_enabled; - config.save(paths).expect("save config"); - runtime_state(helper_running, std::process::id()) - .write(paths) - .expect("write runtime state"); - } - - /// ROCMAI-827: the helper note is derived from both facts, so it cannot - /// disagree with the state it describes. Each combination: the exact wording, - /// and the running/outlives flags the uninstall plan acts on. - #[test] - fn helper_note_follows_watcher_enabled_and_helper_running() { - let (root, paths) = test_paths("helper-note-combos"); - paths.ensure().expect("create app dirs"); - - set_helper_facts(&paths, false, false); - let neither = background_helper_note(&paths); - set_helper_facts(&paths, false, true); - let idle_running = background_helper_note(&paths).expect("note"); - set_helper_facts(&paths, true, false); - let enabled_dead = background_helper_note(&paths).expect("note"); - set_helper_facts(&paths, true, true); - let enabled_running = background_helper_note(&paths).expect("note"); - let _ = fs::remove_dir_all(&root); - - assert_eq!(neither, None); - assert_eq!( - idle_running.text, - "the background automation helper (`rocm daemon`) is still running; no automation is enabled, so it exits on its own within a few seconds" - ); - assert!(idle_running.running && !idle_running.outlives); - assert_eq!( - enabled_dead.text, - "an automation is enabled (server-recover) but the background automation helper (`rocm daemon`) is not running; `rocm serve` or `rocm automations enable` starts it" - ); - assert!(!enabled_dead.running && !enabled_dead.outlives); - assert_eq!( - enabled_running.text, - "the background automation helper (`rocm daemon`) keeps running while an automation is enabled (server-recover); to stop it, run `rocm automations disable server-recover`, after which it exits on its own within a few seconds" - ); - assert!(enabled_running.running && enabled_running.outlives); - } - - /// `rocm services stop` prints the note for a stop only, and only the variant - /// that matches the state. - #[test] - fn stop_output_carries_the_helper_note_that_matches_the_state() { - use crate::services_text::render_service_action_result_with_helper_note as render; - - let (root, paths) = test_paths("stop-helper-note"); - paths.ensure().expect("create app dirs"); - set_helper_facts(&paths, false, false); - let quiet = render(&paths, "stop_server", &stop_result()); - set_helper_facts(&paths, true, true); - let outlives = render(&paths, "stop_server", &stop_result()); - let restart = render(&paths, "restart_server", &stop_result()); - set_helper_facts(&paths, true, false); - let not_running = render(&paths, "stop_server", &stop_result()); - let _ = fs::remove_dir_all(&root); - - assert!(!quiet.contains("background automation helper"), "{quiet}"); - assert!( - outlives.contains("stopped processes: 1") - && outlives.contains("keeps running while an automation is enabled") - && outlives.contains("`rocm automations disable server-recover`"), - "{outlives}" - ); - assert!( - not_running.contains("is not running") && !not_running.contains("keeps running"), - "{not_running}" - ); - assert!( - !restart.contains("background automation helper"), - "{restart}" - ); - } - - /// The command the note names must exist and accept exactly those arguments, - /// and its effect (the helper exiting once the last watcher is off) is - /// `automations-11` in `features/automations.feature` plus the rocmd - /// `automation_loop_` tests. - #[test] - fn the_command_named_by_the_helper_note_parses() { - use clap::Parser as _; - - let (root, paths) = test_paths("helper-note-command"); - paths.ensure().expect("create app dirs"); - set_helper_facts(&paths, true, true); - let note = background_helper_note(&paths).expect("a note"); - let _ = fs::remove_dir_all(&root); - - let command = note - .text - .split('`') - .find(|part| part.starts_with("rocm automations disable")) - .expect("the note names the disable command"); - let argv: Vec<&str> = command.split_whitespace().collect(); - assert!(crate::Cli::try_parse_from(argv).is_ok(), "{command}"); - } - - /// `serve` calls the helper autostart after its child is already running. An - /// unreadable config must not turn that launch into an error: it reads as "no - /// watcher enabled" and nothing is spawned. - #[test] - fn background_helper_autostart_treats_an_unreadable_config_as_no_watcher() { - let (root, paths) = test_paths("helper-bad-config"); - paths.ensure().expect("create app dirs"); - fs::write(paths.config_path(), b"{ this is not json").expect("write bad config"); - assert!(rocm_core::RocmCliConfig::load(&paths).is_err()); - - let mut spawned = false; - let result = ensure_background_helper_running_with( - &paths, - true, - |_| { - spawned = true; - Ok(Ok(4242)) - }, - &|| {}, - ); - let _ = fs::remove_dir_all(&root); - - assert!(result.is_ok(), "{result:?}"); - assert!(!spawned); - } - - #[test] - fn autostart_spawn_in_flight_defers_only_for_a_live_recent_claim() { - let claim = AutostartClaim { - daemon_pid: 4242, - spawned_at_ms: 1_000, - }; - // A recent claim whose PID is still alive ⇒ a spawn is in flight, defer. - assert!(autostart_spawn_in_flight( - Some(claim), - 1_000 + 5_000, - AUTOSTART_CLAIM_TTL_MS, - |_| true, - )); - // Same claim but the child PID is gone (crashed spawn) ⇒ respawn. - assert!(!autostart_spawn_in_flight( - Some(claim), - 1_000 + 5_000, - AUTOSTART_CLAIM_TTL_MS, - |_| false, - )); - // Older than the TTL, even with a live PID (possible PID reuse) ⇒ respawn. - assert!(!autostart_spawn_in_flight( - Some(claim), - 1_000 + AUTOSTART_CLAIM_TTL_MS, - AUTOSTART_CLAIM_TTL_MS, - |_| true, - )); - // No claim at all ⇒ nothing in flight, spawn. - assert!(!autostart_spawn_in_flight( - None, - 1_000, - AUTOSTART_CLAIM_TTL_MS, - |_| true, - )); - } - - #[test] - fn autostart_claim_round_trips_and_absent_or_garbage_reads_as_none() { - let (root, paths) = test_paths("autostart-claim-roundtrip"); - let path = paths.automation_autostart_claim_path(); - fs::create_dir_all(path.parent().expect("claim parent")).expect("mkdir claim dir"); - // Absent file ⇒ no claim. - assert_eq!(read_autostart_claim(&path), None); - // Round-trip a written claim. - let claim = AutostartClaim { - daemon_pid: 9987, - spawned_at_ms: 1_726_000_000_123, - }; - write_autostart_claim(&path, claim).expect("write claim"); - assert_eq!(read_autostart_claim(&path), Some(claim)); - // Garbage / partial content ⇒ no claim (permits a respawn, never panics). - fs::write(&path, "not-a-pid").expect("write garbage"); - assert_eq!(read_autostart_claim(&path), None); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { - // `generate_service_id` embeds a timestamp, so a second launch for the - // same engine+model has a DIFFERENT service_id. The guard must still - // detect the live service by (engine, canonical_model_id), and return - // the newest live match. `starting` skips the endpoint probe; the - // current process id is a guaranteed-live PID. - let (root, paths) = test_paths("dup-managed-distinct-ids"); - paths.ensure()?; - - // Older, dead manifest for the same engine+model (distinct service_id). - let mut dead = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11500, - "managed", - 999_999_999, - None, - None, - None, - ); - dead.status = "ready".to_owned(); - dead.engine_pid = Some(999_999_999); - dead.created_at_unix_ms = 1000; - dead.write()?; - - // Newer, live manifest for the same engine+model (distinct service_id). - let mut live = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-2000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11501, - "managed", - std::process::id(), - None, - None, - None, - ); - live.status = "starting".to_owned(); - live.engine_pid = Some(std::process::id()); - live.created_at_unix_ms = 2000; - live.write()?; - - let live_services = live_managed_services(&paths); - let found = - existing_live_managed_service_in(&live_services, "lemonade", "qwen-canonical").cloned(); - let _ = fs::remove_dir_all(root); - - let found = found.expect("a live managed service should be detected by engine+model"); - assert_eq!( - found.service_id, "lemonade-qwen-2000", - "should return the newest live match" - ); - assert!(managed_service_is_live(&found)); - Ok(()) - } - - #[test] - fn dead_managed_service_allows_relaunch() -> Result<()> { - // A stale manifest with dead PIDs must NOT block a relaunch: liveness - // refresh demotes it to "stopped", so the guard returns None. - let (root, paths) = test_paths("dup-managed-dead"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11502, - "managed", - 999_999_999, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; - - let live = live_managed_services(&paths); - let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a dead managed service must not block relaunch" - ); - Ok(()) - } - - #[test] - fn live_service_for_other_model_does_not_block() -> Result<()> { - // A live service for a DIFFERENT model must not match — the guard keys - // on the model, not just the engine. - let (root, paths) = test_paths("dup-managed-other-model"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-other-1", - "lemonade", - "other", - "other-canonical", - "127.0.0.1", - 11503, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = "starting".to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; - - let live = live_managed_services(&paths); - let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a live service for a different model must not match" - ); - Ok(()) - } - - #[test] - fn missing_manifest_allows_launch() { - // No services dir / manifests → nothing to detect, launch proceeds. - let (root, paths) = test_paths("dup-managed-missing"); - let live = live_managed_services(&paths); - let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - assert!(found.is_none()); - } - - fn write_live_service_record( - paths: &AppPaths, - service_id: &str, - canonical_model_id: &str, - port: u16, - status: &str, - ) -> Result { - let mut record = ManagedServiceRecord::new( - paths, - service_id, - "lemonade", - canonical_model_id, - canonical_model_id.to_owned(), - rocm_core::DEFAULT_LOCAL_HOST, - port, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = status.to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; - Ok(record) - } - - fn never_probe(_port: u16) -> std::io::Result { - panic!("no port may be probed on this path") - } - - fn always_free(port: u16) -> std::io::Result { - Ok(rocm_core::LoopbackPortLease::engine_owned(port)) - } - - #[test] - fn allocation_transaction_answers_an_equivalent_live_service_before_leasing() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-already-live"); - paths.ensure()?; - let existing = write_live_service_record( - &paths, - "lemonade-qwen-9000", - "qwen-canonical", - 11501, - "starting", - )?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "qwen-canonical", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut never_probe, - )?; - let _ = fs::remove_dir_all(root); - - match decision { - ManagedPortDecision::AlreadyLive(record) => { - assert_eq!(record.service_id, existing.service_id); - assert_eq!(record.port, existing.port); - } - ManagedPortDecision::Lease(_) => { - panic!("an equivalent live service must be answered, not leased a new port") - } - } - Ok(()) - } - - #[test] - fn allocation_transaction_skips_ports_reserved_by_other_live_services() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-skips-reserved"); - paths.ensure()?; - write_live_service_record(&paths, "lemonade-a", "model-a", 11_435, "running")?; - write_live_service_record(&paths, "lemonade-b", "model-b", 11_436, "recovering")?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-c", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut always_free, - )?; - let _ = fs::remove_dir_all(root); - - match decision { - ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_437), - ManagedPortDecision::AlreadyLive(record) => { - panic!("model-c has no live service, expected a lease, got {record:?}") - } - } - Ok(()) - } - - #[test] - fn allocation_transaction_ignores_stopped_records() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-ignores-stopped"); - paths.ensure()?; - let mut stale = - write_live_service_record(&paths, "lemonade-stale", "model-stale", 11_435, "running")?; - // A dead pid demotes the record to stopped when liveness is refreshed, so - // it must not reserve its port. - stale.engine_pid = Some(999_999_999); - stale.supervisor_pid = 999_999_999; - stale.write()?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-fresh", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut always_free, - )?; - let _ = fs::remove_dir_all(root); - - match decision { - ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), - ManagedPortDecision::AlreadyLive(record) => { - panic!("a stopped record must not block a fresh launch, got {record:?}") - } - } - Ok(()) - } - - #[test] - fn stopping_the_reserving_record_frees_its_explicit_port_for_reuse() -> Result<()> { - // Proves the mechanism `rocm services stop` relies on: it never touches - // the port directly, it only flips the stopped record's `status` (see - // the `all_stopped` branch in `main.rs`'s stop handler). This recreates - // that exact disk state and checks the same explicit port a live - // competitor refused now resolves, closing the loop the E2E - // `assert_port_reserved_message` step only checks by wording. - let (root, paths) = test_paths("alloc-explicit-port-freed-by-stop"); - paths.ensure()?; - let mut holder = ManagedServiceRecord::new( - &paths, - "lemonade-a-1", - "lemonade", - "model-a", - "model-a".to_owned(), - DEFAULT_LOCAL_HOST, - 11_435, - "managed", - std::process::id(), - None, - None, - None, - ); - holder.status = "running".to_owned(); - holder.engine_pid = Some(std::process::id()); - holder.write()?; - - let still_reserved = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-b", - DEFAULT_LOCAL_HOST, - serve_port::PortRequest::Explicit(11_435), - &mut always_free, - ); - // The refusal names the service AND the command that frees the port; - // the stop below is that command's own implementation, and the grant - // after it is the outcome, so advice and result are pinned together. - let stop_hint = service_hints::service_stop_hint("lemonade-a-1"); - assert!( - matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1") - && error.to_string().contains(&stop_hint)), - "unexpected result while still running: {still_reserved:?}" - ); - - // Go through the real stop path (`rocm services stop --yes`'s own - // implementation) rather than writing `status = "stopped"` to disk - // directly, so this test actually proves the advice serve-45 pins: that - // running the stop command frees the port. The termination closure is - // the same seam production injects a real signal through; here it just - // confirms every recorded process gone without touching the test - // process itself. - crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { - crate::service_control::RecordedStopTermination::confirmed_stopped() - })?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-b", - DEFAULT_LOCAL_HOST, - serve_port::PortRequest::Explicit(11_435), - &mut always_free, - ); - let _ = fs::remove_dir_all(root); - - match decision? { - ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), - ManagedPortDecision::AlreadyLive(existing) => { - panic!( - "a stopped record must not block reuse: {}", - existing.service_id - ) - } - } - Ok(()) - } - - #[test] - fn allocation_transaction_rejects_an_explicit_port_a_recovering_service_holds() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-rejects-explicit-collision"); - paths.ensure()?; - let existing = write_live_service_record( - &paths, - "lemonade-recovering", - "model-recovering", - 11_435, - "recovering", - )?; - - let result = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-other", - "127.0.0.1", - serve_port::PortRequest::Explicit(11_435), - &mut never_probe, - ); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!("an explicit port a recovering service holds must be refused") - }; - let message = error.to_string(); - assert!( - message.contains(&existing.service_id), - "the refusal must name the service holding the port: {message}" - ); - assert!( - message.contains("recovering"), - "the refusal must say why the port is unavailable: {message}" - ); - Ok(()) - } - - #[test] - fn concurrent_auto_transactions_publish_distinct_ports() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-concurrent-distinct"); - paths.ensure()?; - let paths = std::sync::Arc::new(paths); - - let spawn_one = |model: &'static str| { - let paths = std::sync::Arc::clone(&paths); - thread::spawn(move || -> Result { - let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - let lease = match resolve_managed_port_in_transaction( - &paths, - "lemonade", - model, - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut serve_port::loopback_bind_probe, - )? { - ManagedPortDecision::Lease(lease) => lease, - ManagedPortDecision::AlreadyLive(record) => { - panic!("fresh model must not already be live: {record:?}") - } - }; - let port = lease.port(); - write_live_service_record(&paths, model, model, port, "running")?; - lease.release(); - Ok(port) - }) - }; - - let first = spawn_one("model-one"); - let second = spawn_one("model-two"); - let first_port = first.join().expect("thread one panicked")?; - let second_port = second.join().expect("thread two panicked")?; - let _ = fs::remove_dir_all(root); - - assert_ne!(first_port, second_port); - for port in [first_port, second_port] { - assert!( - (serve_port::AUTO_PORT_FIRST..=serve_port::AUTO_PORT_LAST).contains(&port), - "leased port {port} must fall within the auto-select range" - ); - } - Ok(()) - } - - #[test] - fn concurrent_equivalent_transactions_converge_on_one_record() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-concurrent-converge"); - paths.ensure()?; - let paths = std::sync::Arc::new(paths); - - let race_once = || { - let paths = std::sync::Arc::clone(&paths); - thread::spawn(move || -> Result { - let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - match resolve_managed_port_in_transaction( - &paths, - "lemonade", - "shared-model", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut serve_port::loopback_bind_probe, - )? { - ManagedPortDecision::Lease(lease) => { - let port = lease.port(); - write_live_service_record( - &paths, - "lemonade-shared", - "shared-model", - port, - "running", - )?; - lease.release(); - Ok(true) - } - ManagedPortDecision::AlreadyLive(_) => Ok(false), - } - }) - }; - - let first = race_once(); - let second = race_once(); - let first_launched = first.join().expect("thread one panicked")?; - let second_launched = second.join().expect("thread two panicked")?; - - let live_count = live_managed_services(&paths).len(); - let _ = fs::remove_dir_all(root); - - assert_ne!( - first_launched, second_launched, - "exactly one racing transaction must launch" - ); - assert_eq!(live_count, 1, "only one record may end up live"); - Ok(()) - } - - #[test] - fn select_gpu_indices_under_launch_lock_blocks_until_the_holder_releases() -> Result<()> { - // Exercises the real lock-acquisition function production callers use - // (`select_gpu_indices_under_launch_lock`) against a lock another thread - // actually holds, so a refactor that dropped the lock early would fail - // here. `spawn_managed_engine_child` is then run for real, but only via - // its `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no - // child process is ever launched. The holder writes nothing: this does - // not show a write made under the lock being observed. - use std::sync::{Arc, Barrier}; - use std::time::Instant; - - let (root, paths) = test_paths("launch-lock-holds-across-spawn"); - paths.ensure()?; - write_live_service_record(&paths, "lemonade-held-1", "held-canonical", 11_520, "ready")?; - - let barrier = Arc::new(Barrier::new(2)); - let hold_for = Duration::from_millis(250); - - let holder_paths = paths.clone(); - let holder_barrier = Arc::clone(&barrier); - let holder = thread::spawn(move || -> Result<()> { - let _lock = rocm_core::FileLock::acquire(holder_paths.managed_launch_lock_path())?; - holder_barrier.wait(); - thread::sleep(hold_for); - Ok(()) - }); - - barrier.wait(); - let started = Instant::now(); - let (gpu_indices, launch_lock) = crate::gpu_policy::select_gpu_indices_under_launch_lock( - &paths, - true, - None, - || None, - None, - None, - )?; - let waited = started.elapsed(); - - let resolve = ResolveModelResponse { - canonical_model_id: "held-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let spawn = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-held-2", - "held-model", - &resolve, - DEFAULT_LOCAL_HOST, - serve_port::PortRequest::Auto, - &resolve.device_policy, - &gpu_indices, - None, - None, - None, - false, - false, - ); - drop(launch_lock); - holder.join().expect("holder thread")?; - let _ = fs::remove_dir_all(root); - - // `thread::sleep` only guarantees *at least* `hold_for`, but under a - // loaded host (e.g. a full parallel `cargo test` run on WSL2) timer - // coalescing can make the measured wait a hair short of the nominal - // hold. Allow a small margin so the test still fails loudly if the - // call returns early, without being flaky on actual block duration. - let margin = Duration::from_millis(20); - assert!( - waited + margin >= hold_for, - "select_gpu_indices_under_launch_lock returned before the holder released the lock: waited {waited:?}, held {hold_for:?}" - ); - match spawn? { - ManagedSpawn::AlreadyRunning(report) => { - assert_eq!(report.service_id, "lemonade-held-1"); - } - ManagedSpawn::Spawned { .. } => { - panic!( - "expected the pre-seeded AlreadyLive record to short-circuit, not a real spawn" - ) - } - } - Ok(()) - } - - #[test] - fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { - let (root, paths) = test_paths("abandon-launch-cleanup"); - paths.ensure()?; - let mut record = write_live_service_record( - &paths, - "lemonade-abandon", - "abandon-model", - 11_490, - "running", - )?; - fs::create_dir_all( - record - .engine_state_path - .parent() - .expect("engine state parent"), - )?; - fs::write(&record.engine_state_path, "{}")?; - endpoint_keys::store_endpoint_api_key(&paths, &record.service_id, "secret-key")?; - // Simulate a pre-spawn failure: nothing ever ran. - record.engine_pid = None; - - abandon_failed_managed_launch(&paths, &record); - - assert!(!record.manifest_path.exists()); - assert!(!record.log_path.exists()); - assert!(!record.engine_state_path.exists()); - assert_eq!( - endpoint_keys::endpoint_api_key(&paths, &record.service_id), - None - ); - assert!(live_managed_services(&paths).is_empty()); - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - #[cfg(not(windows))] - #[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] - fn managed_launch_cleanup_sleeper_child() { - if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { - thread::sleep(Duration::from_mins(2)); - } - } - - #[test] - #[cfg(target_os = "linux")] - fn abandoned_launch_never_signals_a_pid_that_no_longer_matches_the_spawn_token() -> Result<()> { - // The child exited and its pid now belongs to an unrelated live - // process: the start token recorded at spawn no longer matches, so the - // forced tree kill must not touch it. - let (root, paths) = test_paths("abandon-launch-recycled-pid"); - paths.ensure()?; - let mut record = write_live_service_record( - &paths, - "lemonade-abandon-recycled", - "abandon-recycled-model", - 11_493, - "running", - )?; - let current_exe = std::env::current_exe().expect("current test exe"); - let mut bystander = ProcessCommand::new(current_exe) - .arg("--exact") - .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") - .arg("--ignored") - .arg("--test-threads=1") - .env("ROCM_TEST_SLEEPER_CHILD", "1") - .spawn() - .expect("spawn bystander"); - record.record_engine_identity(bystander.id()); - record.engine_start_ticks = record.engine_start_ticks.map(|ticks| ticks + 1); - - abandon_failed_managed_launch(&paths, &record); - - let mismatched_left_alone = bystander.try_wait().expect("poll bystander").is_none(); - // A child that was gone before its token could be captured leaves none: - // that must not fall back to "best-effort" signalling either. - record.engine_start_ticks = None; - abandon_failed_managed_launch(&paths, &record); - let still_running = - mismatched_left_alone && bystander.try_wait().expect("poll bystander").is_none(); - let _ = bystander.kill(); - let _ = bystander.wait(); - let _ = fs::remove_dir_all(root); - assert!( - still_running, - "a pid with a mismatched token must be left alone" - ); - Ok(()) - } - - #[test] - #[cfg(not(windows))] - fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { - let (root, paths) = test_paths("abandon-launch-terminates-child"); - paths.ensure()?; - let mut record = write_live_service_record( - &paths, - "lemonade-abandon-child", - "abandon-child-model", - 11_491, - "running", - )?; - - let current_exe = std::env::current_exe().expect("current test exe"); - let mut child = ProcessCommand::new(current_exe) - .arg("--exact") - .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") - .arg("--ignored") - .arg("--test-threads=1") - .env("ROCM_TEST_SLEEPER_CHILD", "1") - .spawn() - .expect("spawn sleeper child"); - let child_pid = child.id(); - record.record_engine_identity(child_pid); - - abandon_failed_managed_launch(&paths, &record); - - let wait_result = child.wait(); - - match wait_result { - Ok(status) => assert!( - !status.success(), - "the sleeper child must be terminated, not exit cleanly" - ), - Err(error) => panic!("failed to wait for sleeper child: {error}"), - } - let persisted = load_managed_services(&paths) - .expect("re-read manifests") - .into_iter() - .find(|candidate| candidate.service_id == record.service_id); - let _ = fs::remove_dir_all(root); - let persisted = persisted.expect( - "a launch that spawned a child must keep its record so `rocm services`/`rocm logs` can show the attempt", - ); - // `stopped`, not `failed`: `rocmd`'s recovery restarts a `failed` record - // that carries no stop marker, which would respawn a launch the CLI just - // reported as failed. The rocmd side of the pair is - // `abandoned_launch_record_is_not_a_recovery_candidate` in - // `apps/rocmd/src/watchers.rs`. - assert_eq!( - persisted.status, "stopped", - "a spawned child's record must be kept, but in a status recovery ignores" - ); - Ok(()) - } - - #[test] - #[cfg(not(windows))] - fn launch_releases_the_leased_port_before_spawning_the_child() -> Result<()> { - // `lease.release()` must run before the child is spawned, not after, - // or the engine is handed a port the CLI is still holding open. - // Proven via the injected `spawn_override` seam: it asserts a fresh - // bind on the leased port succeeds at the exact instant production - // would spawn, which only holds if the lease was already released. - let (root, paths) = test_paths("launch-releases-port-before-spawn"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // Ephemeral, not a fixed number in the automatic range, which a real - // `rocm serve` on the developer's machine could be holding. - let port = std::net::TcpListener::bind(("127.0.0.1", 0))? - .local_addr()? - .port(); - let lease = rocm_core::lease_loopback_port(port)?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-release-before-spawn", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - port, - "managed", - 0, - None, - None, - None, - ); - record.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let bind_succeeded = std::cell::Cell::new(false); - let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = - &|_command| { - // Retry briefly: a sibling test's concurrent fork can hold a copy - // of the just-released socket until its exec closes it. - let deadline = std::time::Instant::now() + Duration::from_secs(2); - while !bind_succeeded.get() && std::time::Instant::now() < deadline { - bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); - } - // A long-lived no-op child stands in for the engine: `_command` - // carries real serve args this test binary cannot execute as - // itself (see `managed_launch_cleanup_sleeper_child`). - ProcessCommand::new(std::env::current_exe().expect("test binary")) - .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") - .arg("--exact") - .arg("--ignored") - .env("ROCM_TEST_SLEEPER_CHILD", "1") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - }; - - let result = launch_managed_engine_child_locked( - &paths, - &mut record, - lease, - "lemonade", - "lemonade-release-before-spawn", - &resolve, - "127.0.0.1", - &resolve.device_policy, - &[], - None, - None, - None, - Some(spawn_override), - ); - if let Ok(child_pid) = result.as_ref() { - let _ = rocm_core::terminate_process(*child_pid); - } - let _ = fs::remove_dir_all(root); - - assert!( - bind_succeeded.get(), - "the port must already be bindable (released) by the time the child is spawned" - ); - result?; - Ok(()) - } -} +pub(crate) use spawn::{ + attach_background_stdio, detach_background_command, managed_engine_startup_failure_detail, + managed_service_process_command, +}; +// The helper is platform-independent, but outside tests nothing on Windows +// calls it through this path, so a plain re-export there would be unused. The +// `services_text` log-tail tests call it and are compiled on Windows too. +#[cfg(all(windows, test))] +pub(crate) use spawn::managed_engine_startup_failure_detail; +pub(crate) use start::{ + print_managed_launch_plain, start_managed_service, wait_begin_client_config, +}; diff --git a/apps/rocm/src/service_launch/attach.rs b/apps/rocm/src/service_launch/attach.rs new file mode 100644 index 0000000000..d76a90e8d4 --- /dev/null +++ b/apps/rocm/src/service_launch/attach.rs @@ -0,0 +1,774 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The attached serve path (`--verbose`/`--foreground`): spawn the managed child +//! and stream its log in this terminal until the user detaches or stops it. + +use super::endpoint_auth::drop_orphaned_endpoint_key_on_already_running; +use super::helper_autostart::ensure_background_helper_running_quiet; +use super::spawn::{ManagedSpawn, spawn_managed_engine_child}; +use super::start::render_endpoint_client_config; +use crate::record_cli_audit_event; +use crate::serve_port; +use crate::service_hints; +use crate::services_text::{ + render_service_action_result, render_service_action_result_with_helper_note, + service_stop_unconfirmed, +}; +use anyhow::{Context, Result, bail}; +use rocm_core::AppPaths; +use rocm_core::{format_http_base_url, process_is_running}; +use rocm_engine_protocol::ResolveModelResponse; +use std::fs; +use std::io::{self, BufRead, Write}; +use std::path::Path; +use std::thread; +use std::time::Duration; + +/// What ended an attached (`--verbose`/`--foreground`) streaming session. Kept +/// as a plain enum, separate from any terminal I/O, so the follow-up action +/// (detach note vs. stop the server) is unit-testable without a TTY. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AttachOutcome { + /// Ctrl-D: leave the server running and hand the terminal back. + Detach, + /// Ctrl-C: stop the server, then hand the terminal back. + Stop, + /// The engine process exited on its own while we were streaming its log. + ServerExited, +} + +/// Attached serve path for `--verbose`/`--foreground`: spawn the engine as a +/// detached managed child (the same child the background path spawns) and stream +/// its log in this terminal. Unlike the old in-process foreground, the server +/// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. +/// +/// `stop_service` is how Ctrl-C stops the server: the caller passes the managed +/// stop, so the `service_launch` module does not import the stop path, which +/// itself imports its spawn helpers. It returns the same `stop_server` answer +/// [`finish_attached_stop`] reads. +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_attached_service( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port_request: serve_port::PortRequest, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + endpoint_api_key: Option<&str>, + launch_lock: rocm_core::FileLock, + require_api_key: bool, + gpu_refusal_waived_for_reuse: bool, + stop_service: impl FnOnce(&AppPaths, &str) -> Result, +) -> Result<()> { + let spawn = spawn_managed_engine_child( + paths, + engine, + service_id, + requested_model, + resolve, + host, + port_request, + &resolve.device_policy, + gpu_indices, + runtime_id, + env_id, + resolve.engine_recipe.as_ref(), + require_api_key, + gpu_refusal_waived_for_reuse, + ) + // Mirrors `start_managed_service`'s cleanup: a pre-spawn + // failure (port reserved/in use, range exhausted, other bind error) returns + // before any service record exists, so nothing else clears the key just + // stored for this attempt. + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })?; + // The claiming record is persisted (or an existing service was found), so the + // selected GPU is now visible to concurrent auto-selection. Release the launch + // lock before streaming logs, which blocks for the whole attached session. + drop(launch_lock); + + // Attached mode is a managed record too, so it shares the transaction's + // resolved concrete port. + let (service_id, log_path, port, child_pid) = match spawn { + // A server for this engine+model is already live. Don't fight it for the + // port — point the user at the existing one instead of tailing a log we + // did not start. + ManagedSpawn::AlreadyRunning(report) => { + println!("model already being served"); + println!(" service_id: {}", report.service_id); + println!(" endpoint: {}", report.endpoint_url); + println!(" status: {}", report.status); + println!(" logs: rocm logs {}", report.service_id); + println!( + " stop: {}", + service_hints::service_stop_hint(&report.service_id) + ); + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + return Ok(()); + } + ManagedSpawn::Spawned { record, child_pid } => ( + service_id.to_owned(), + record.log_path.clone(), + record.port, + child_pid, + ), + }; + + // The child is a managed service that outlives this session once detached. + // The background helper adds one thing for it: with the `server-recover` + // watcher enabled, it health-checks managed servers and restarts failed ones. + // Every duty of the helper is watcher-gated, so with no watcher enabled no + // helper is started. Nothing else depends on it: a self-exited server's + // record is settled by its own supervisor (`supervise_service` writes the + // exit via `record_supervised_exit` in `apps/rocmd/src/supervise.rs`) and on + // every read (`refresh_managed_service_runtime_liveness`), and the dashboard + // has its own embedded daemon. Match the background ordering (spawn, then + // ensure the helper) and keep it quiet so no status line breaks into the + // log stream. + ensure_background_helper_running_quiet(true)?; + + // The resolution detail (model, engine, runtime, GPU, warnings) was already + // printed as the "serve plan" block in `serve()`; extend it with the launch + // coordinates and the streaming hint rather than repeating it. + let endpoint = format!("{}/v1", format_http_base_url(host, port)); + println!(" service_id: {service_id}"); + println!(" endpoint: {endpoint}"); + if let Some(key) = endpoint_api_key { + // Same intentional one-time key display as `print_managed_launch_plain` + // in `service_launch/start.rs`; see its rationale for why the tag below is + // currently inert. + // codeql[rust/cleartext-logging] + print!("{}", render_endpoint_client_config(&endpoint, key)); + } + println!(" streaming engine logs - Ctrl-D detaches (leaves it running), Ctrl-C stops it"); + println!(); + + let outcome = stream_attached_logs(&log_path, child_pid)?; + println!(); + + match outcome { + AttachOutcome::Detach => { + // Audit before printing: a closed stdout makes `println!` panic, + // which must not lose the event. + record_cli_audit_event( + paths, + "service", + "serve_detach", + "info", + format!("detached from attached serve service_id={service_id} endpoint={endpoint}"), + Some(&service_id), + ); + println!("detached - server still running"); + println!(" service_id: {service_id}"); + println!(" endpoint: {endpoint}"); + println!(" list: rocm services"); + println!(" logs: rocm logs {service_id}"); + println!(" stop: {}", service_hints::service_stop_hint(&service_id)); + Ok(()) + } + AttachOutcome::Stop => { + println!("stopping server..."); + let stop = stop_service(paths, &service_id); + finish_attached_stop( + paths, + &service_id, + child_pid, + stop, + rocm_core::terminate_process_tree, + &mut io::stdout().lock(), + &mut io::stderr().lock(), + ) + } + AttachOutcome::ServerExited => { + println!("server process exited"); + println!(" service_id: {service_id}"); + println!(" recent logs: rocm logs {service_id}"); + Ok(()) + } + } +} + +/// Restores cooked terminal mode when dropped, so [`stream_attached_logs`] leaves +/// the terminal usable on every exit path (normal return, `?` error, or panic). +struct RawModeGuard; + +impl Drop for RawModeGuard { + fn drop(&mut self) { + let _ = crossterm::terminal::disable_raw_mode(); + } +} + +/// Map a key press (control modifier + lowercased character) to the attach +/// action it triggers, if any. Factored out of the raw-mode reader loop so the +/// Ctrl-D/Ctrl-C mapping is unit-testable without a terminal. +const fn detach_key_outcome(ctrl: bool, ch: char) -> Option { + if !ctrl { + return None; + } + match ch { + 'c' => Some(AttachOutcome::Stop), + 'd' => Some(AttachOutcome::Detach), + _ => None, + } +} + +/// Follow `log_path` in the terminal until the user presses Ctrl-D (detach) or +/// Ctrl-C (stop), or the engine process exits. Uses crossterm raw mode to +/// capture the keys directly (in raw mode Ctrl-C does not raise SIGINT, so we see +/// it as a key event). When stdin is not a TTY (piped/CI), keystroke capture is +/// impossible, so we follow the log until the process exits instead. +fn stream_attached_logs(log_path: &Path, child_pid: u32) -> Result { + use std::io::IsTerminal as _; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc; + + if !std::io::stdin().is_terminal() { + return stream_attached_logs_no_tty(log_path, child_pid); + } + + // Enter raw mode *before* spawning the key reader. In raw mode Ctrl-C arrives + // as a key event instead of SIGINT; if the reader started first, a Ctrl-C in + // that window would kill the CLI outright (leaving the detached child alive + // but printing no detach/stop message). The guard restores cooked mode on + // every exit path (normal return, `?` error, panic). + crossterm::terminal::enable_raw_mode().context("failed to enter raw terminal mode")?; + let _raw_guard = RawModeGuard; + + let stop = Arc::new(AtomicBool::new(false)); + let (tx, rx) = mpsc::channel::(); + + let reader_stop = Arc::clone(&stop); + let reader = thread::spawn(move || { + use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; + while !reader_stop.load(Ordering::Relaxed) { + match event::poll(Duration::from_millis(100)) { + Ok(true) => match event::read() { + Ok(Event::Key(key)) if key.kind != KeyEventKind::Release => { + let outcome = match key.code { + KeyCode::Char(ch) => detach_key_outcome( + key.modifiers.contains(KeyModifiers::CONTROL), + ch.to_ascii_lowercase(), + ), + _ => None, + }; + if let Some(outcome) = outcome { + let _ = tx.send(outcome); + break; + } + } + Ok(_) => {} + Err(_) => break, + }, + Ok(false) => {} + Err(_) => break, + } + } + }); + + let mut stdout = io::stdout(); + let mut log_reader: Option> = None; + let mut line = String::new(); + let outcome = loop { + if log_reader.is_none() { + log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); + } + if let Some(reader) = log_reader.as_mut() { + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + // Raw mode disables the terminal's own \n -> \r\n + // translation, so emit an explicit carriage return to + // keep the log left-aligned instead of stair-stepping. + let _ = write!(stdout, "{}\r\n", line.trim_end_matches('\n')); + let _ = stdout.flush(); + } + Err(_) => break, + } + } + } + + if let Ok(signal) = rx.try_recv() { + break signal; + } + if !process_is_running(child_pid) { + break AttachOutcome::ServerExited; + } + thread::sleep(Duration::from_millis(150)); + }; + + stop.store(true, Ordering::Relaxed); + let _ = reader.join(); + Ok(outcome) +} + +/// Non-interactive fallback for [`stream_attached_logs`]: no keystroke capture, +/// so just follow the log until the (detached) engine process exits. A Ctrl-C +/// here delivers SIGINT to this process and leaves the managed server running. +fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result { + let mut stdout = io::stdout(); + let mut log_reader: Option> = None; + let mut line = String::new(); + loop { + if log_reader.is_none() { + log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); + } + if let Some(reader) = log_reader.as_mut() { + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let _ = write!(stdout, "{line}"); + let _ = stdout.flush(); + } + Err(_) => break, + } + } + } + if !process_is_running(child_pid) { + return Ok(AttachOutcome::ServerExited); + } + thread::sleep(Duration::from_millis(200)); + } +} + +/// Finish the attached `rocm serve` Ctrl-C stop once the managed stop has +/// answered: what is printed where, what is signalled, what is audited, and +/// how the command exits. Everything but the stop itself and the real +/// signal is here, so a test can drive it. +/// +/// - Confirmed: the report, with the background-helper note `rocm services +/// stop` also prints, goes to `out` and the command succeeds. +/// - Unconfirmed: the same report `rocm services stop` gives, check lines +/// included, becomes the error, so it reaches stderr and the command exits +/// non-zero. Nothing is signalled on the stop's behalf: the stop has already +/// done what it safely could. +/// - The stop itself failed: `signal_child` gets the child this session +/// launched, so Ctrl-C never leaves the model orphaned when the stop path +/// breaks. The stop's error, and the signal's if that fails too, are noted on +/// `err`, and the audit event records whether the signal worked. The command +/// still exits 0, as it always has here. +fn finish_attached_stop( + paths: &AppPaths, + service_id: &str, + child_pid: u32, + stop: Result, + signal_child: impl FnOnce(u32) -> Result<()>, + out: &mut dyn Write, + err: &mut dyn Write, +) -> Result<()> { + match stop { + Ok(result) if service_stop_unconfirmed(&result) => { + record_cli_audit_event( + paths, + "service", + "serve_stop", + "error", + format!("stop of attached serve not confirmed service_id={service_id}"), + Some(service_id), + ); + bail!( + "{}", + render_service_action_result("stop_server", &result).trim_end() + ) + } + Ok(result) => { + record_cli_audit_event( + paths, + "service", + "serve_stop", + "info", + format!("stopped attached serve service_id={service_id}"), + Some(service_id), + ); + write!( + out, + "{}", + render_service_action_result_with_helper_note(paths, "stop_server", &result) + )?; + Ok(()) + } + Err(error) => { + let signal = signal_child(child_pid); + // Record the audit event before writing to stderr, so a failed + // write cannot skip it. + let message = match &signal { + Ok(()) => format!( + "stop failed; signaled attached serve child {child_pid} service_id={service_id}" + ), + Err(signal_error) => format!( + "stop failed; signaling attached serve child {child_pid} also failed: {signal_error:#} service_id={service_id}" + ), + }; + record_cli_audit_event( + paths, + "service", + "serve_stop", + if signal.is_ok() { "warn" } else { "error" }, + message, + Some(service_id), + ); + writeln!(err, " note: {error:#}")?; + if let Err(signal_error) = &signal { + writeln!( + err, + " note: signaling this session's server process {child_pid} directly also failed: {signal_error:#}" + )?; + } + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endpoint_keys; + use crate::tests::test_paths; + use rocm_core::ManagedServiceRecord; + use rocm_engine_protocol::DevicePolicy; + + /// A `stop_server` answer whose one recorded process ended with `outcome`. + fn stop_answer(confirmed: bool, outcome: &str) -> serde_json::Value { + crate::services_text::internal_stop_server_value(serde_json::json!({ + "service_id": "svc-attached", + "status": "ready", + "signaled_pids": if outcome == "unverified" { vec![] } else { vec![4242] }, + "pid_outcomes": [{ "pid": 4242, "role": "supervisor_and_engine", "outcome": outcome }], + // An attached serve on a public bind stores a key; an unconfirmed + // stop keeps it. + "endpoint_key_kept": !confirmed, + "stopped": confirmed, + })) + } + + /// Run [`finish_attached_stop`] for `stop`, returning its result, what it + /// wrote to stdout, the PID it signalled (if any) and the audit event. + fn finish( + name: &str, + stop: Result, + ) -> (Result<()>, String, Option, serde_json::Value) { + finish_signalling(name, stop, Ok(())) + } + + /// [`finish`], with the direct signal answering `signal_result`. + fn finish_signalling( + name: &str, + stop: Result, + signal_result: Result<()>, + ) -> (Result<()>, String, Option, serde_json::Value) { + let (result, out, _err, signalled, audit) = finish_capturing(name, stop, signal_result); + (result, out, signalled, audit) + } + + /// [`finish_signalling`], also returning what was written to stderr. + fn finish_capturing( + name: &str, + stop: Result, + signal_result: Result<()>, + ) -> (Result<()>, String, String, Option, serde_json::Value) { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let mut out = Vec::new(); + let mut err = Vec::new(); + let mut signalled = None; + let result = finish_attached_stop( + &paths, + "svc-attached", + 777, + stop, + |pid| { + signalled = Some(pid); + signal_result + }, + &mut out, + &mut err, + ); + let audit_text = fs::read_to_string(paths.audit_events_path()).expect("audit log"); + let audit = serde_json::from_str(audit_text.lines().last().expect("an audit event")) + .expect("audit json"); + let _ = fs::remove_dir_all(root); + ( + result, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + signalled, + audit, + ) + } + + #[test] + fn attached_ctrl_c_reports_a_confirmed_stop_on_stdout() { + let (result, out, signalled, audit) = + finish("attached-confirmed", Ok(stop_answer(true, "graceful"))); + result.expect("a confirmed stop succeeds"); + assert!(out.contains("Local server stopped"), "{out}"); + assert_eq!(signalled, None); + assert_eq!(audit["level"], "info", "{audit}"); + assert_eq!( + audit["message"], + "stopped attached serve service_id=svc-attached" + ); + } + + #[test] + fn attached_ctrl_c_fails_an_unconfirmed_stop_like_services_stop_and_signals_nothing() { + // The same report `rocm services stop` gives, as the error (stderr, + // non-zero exit), naming no retry; nothing is signalled on the stop's + // behalf and nothing reaches stdout. + let answer = stop_answer(false, "timed_out"); + let (result, out, signalled, audit) = finish("attached-timed-out", Ok(answer.clone())); + let error = format!("{:#}", result.expect_err("an unconfirmed stop fails")); + assert_eq!( + error, + render_service_action_result("stop_server", &answer).trim_end(), + "the report is `rocm services stop`'s own" + ); + assert!( + error.contains( + "process 4242 (supervisor_and_engine) or a process under it did not exit" + ), + "{error}" + ); + assert!( + error.lines().any(|line| line + == " kept: the server record, which still names its processes, and its endpoint key, which the server may still be enforcing"), + "{error}" + ); + crate::services_text::tests::assert_no_retry_line(&error); + assert!(out.is_empty(), "nothing on stdout: {out}"); + assert_eq!(signalled, None, "an unconfirmed stop signals nothing more"); + assert_eq!(audit["level"], "error", "{audit}"); + assert_eq!( + audit["message"], + "stop of attached serve not confirmed service_id=svc-attached" + ); + } + + #[test] + fn attached_ctrl_c_names_a_check_for_an_unverified_process() { + let (result, out, signalled, audit) = + finish("attached-unverified", Ok(stop_answer(false, "unverified"))); + let error = format!("{:#}", result.expect_err("an unverified stop fails")); + assert!(error.contains("not signaled: process 4242"), "{error}"); + assert!( + error.contains(&format!( + "check: `{}`", + crate::services_text::process_check_command(4242) + )), + "{error}" + ); + crate::services_text::tests::assert_no_retry_line(&error); + assert!(out.is_empty(), "{out}"); + assert_eq!(signalled, None); + assert_eq!(audit["level"], "error", "{audit}"); + } + + #[test] + fn attached_ctrl_c_signals_its_own_child_when_the_stop_itself_fails() { + let (result, out, err, signalled, audit) = finish_capturing( + "attached-stop-error", + Err(anyhow::anyhow!("managed service `svc-attached` not found") + .context("failed to stop svc-attached")), + Ok(()), + ); + result.expect("a failed stop falls back and still exits cleanly, as before"); + assert_eq!(signalled, Some(777)); + assert_eq!( + err, " note: failed to stop svc-attached: managed service `svc-attached` not found\n", + "the stop's error goes to stderr, and a signal that worked adds nothing" + ); + assert!(out.is_empty(), "nothing on stdout: {out}"); + assert_eq!(audit["level"], "warn", "{audit}"); + assert_eq!( + audit["message"], + "stop failed; signaled attached serve child 777 service_id=svc-attached" + ); + + // The direct signal failing too is printed on stderr and recorded, with + // its error. + let (result, _, err, signalled, audit) = finish_capturing( + "attached-stop-error-signal-fails", + Err(anyhow::anyhow!("managed service `svc-attached` not found")), + Err(anyhow::anyhow!("no such process")), + ); + result.expect("still exits 0, as on main"); + assert_eq!( + err, + " note: managed service `svc-attached` not found\n \ + note: signaling this session's server process 777 directly also failed: no such process\n", + "both notes go to stderr" + ); + assert_eq!(signalled, Some(777)); + assert_eq!(audit["level"], "error", "{audit}"); + assert_eq!( + audit["message"], + "stop failed; signaling attached serve child 777 also failed: no such process service_id=svc-attached" + ); + } + + /// A writer whose every write fails, standing in for a closed stdout/stderr. + struct BrokenWriter; + + impl Write for BrokenWriter { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe)) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn attached_ctrl_c_records_the_audit_event_even_when_writing_the_outcome_fails() { + // One case per branch that writes after deciding: a confirmed stop + // (stdout) and a stop that itself failed (stderr notes). The write + // error is returned, but the audit event is already on disk. + let cases: [(&str, Result, &str); 2] = [ + ( + "attached-broken-out", + Ok(stop_answer(true, "graceful")), + "stopped attached serve service_id=svc-attached", + ), + ( + "attached-broken-err", + Err(anyhow::anyhow!("managed service `svc-attached` not found")), + "stop failed; signaled attached serve child 777 service_id=svc-attached", + ), + ]; + for (name, stop, expected_message) in cases { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let result = finish_attached_stop( + &paths, + "svc-attached", + 777, + stop, + |_| Ok(()), + &mut BrokenWriter, + &mut BrokenWriter, + ); + let audit_text = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); + let _ = fs::remove_dir_all(root); + assert!(result.is_err(), "{name}: the write error is surfaced"); + let audit: serde_json::Value = serde_json::from_str( + audit_text + .lines() + .last() + .unwrap_or_else(|| panic!("{name}: no audit event recorded")), + ) + .expect("audit json"); + assert_eq!(audit["message"], expected_message, "{name}: {audit}"); + } + } + + #[test] + fn detach_key_ctrl_d_detaches_ctrl_c_stops() { + assert_eq!(detach_key_outcome(true, 'd'), Some(AttachOutcome::Detach)); + assert_eq!(detach_key_outcome(true, 'c'), Some(AttachOutcome::Stop)); + } + + #[test] + fn detach_key_ignores_plain_and_unrelated_keys() { + // Without the control modifier, `d`/`c` are ordinary log-scroll input. + assert_eq!(detach_key_outcome(false, 'd'), None); + assert_eq!(detach_key_outcome(false, 'c'), None); + // Other control combos are not detach/stop triggers. + assert_eq!(detach_key_outcome(true, 'q'), None); + assert_eq!(detach_key_outcome(true, 'z'), None); + } + + #[test] + fn attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key() -> Result<()> { + // The background path's cleanup is the `.inspect_err` in + // start_managed_service. run_attached_service is the other caller of + // spawn_managed_engine_child and needs the identical .inspect_err() + // cleanup, not a simulated one: drive the real function end to end. + let (root, paths) = test_paths("orphaned-key-attached-reserved-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11514, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-attached-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let result = run_attached_service( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11514), + &[], + None, + None, + Some("test-key"), + launch_lock, + false, + false, + |_, _| unreachable!("the launch fails before a Ctrl-C could stop it"), + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "a pre-spawn failure must drop the key this attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } +} diff --git a/apps/rocm/src/service_launch/endpoint_auth.rs b/apps/rocm/src/service_launch/endpoint_auth.rs new file mode 100644 index 0000000000..6a647314bc --- /dev/null +++ b/apps/rocm/src/service_launch/endpoint_auth.rs @@ -0,0 +1,377 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Bind-host validation and the endpoint API key checks that gate a public +//! bind: [`validate_bind_host`], [`resolve_endpoint_auth`] and the guards that +//! keep a respawn from reopening a service without its key. + +use crate::endpoint_keys; +use anyhow::{Result, bail}; +use rocm_core::AppPaths; + +pub(crate) fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { + if !is_loopback_host(host) && !allow_public_bind { + bail!( + "`rocm serve --host {host}` is not loopback; pass `--allow-public-bind` before binding a non-local interface" + ); + } + Ok(()) +} + +/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the +/// policy so `rocmd` classifies a recorded `host` identically when it respawns +/// the service. +pub(crate) fn is_loopback_host(host: &str) -> bool { + !rocm_engine_protocol::is_public_bind_host(host) +} + +/// Resolve the API key that will guard this endpoint, applying the +/// loopback-vs-public policy for `rocm serve`. +/// +/// - **Loopback host** → `None`: local serving stays credential-free (the +/// unchanged default). Any key supplied for a loopback bind is ignored and +/// nothing is persisted — loopback needs no auth. +/// - **Public host** → `Some(key)`: use the user-supplied key when present, +/// otherwise generate a strong random one so a public endpoint can never come +/// up anonymous. An empty/whitespace supplied key is rejected rather than +/// silently treated as "no auth". +/// - **`required`** → treat a loopback bind as public for this purpose. +/// +/// That last case exists because "loopback" is a statement about the bind +/// address, not about who can reach the port. Publishing the port onto a +/// tailnet, proxying it, or mapping it out of a container all leave the bind +/// loopback while widening the audience — and the policy above would then hand +/// out an unauthenticated endpoint. Whoever widens the reach is responsible for +/// asking for the credential, so this is an explicit flag rather than a guess. +pub(crate) fn resolve_endpoint_auth( + host: &str, + supplied: Option<&str>, + required: bool, +) -> Result> { + if is_loopback_host(host) && !required { + return Ok(None); + } + match supplied { + Some(raw) => { + let trimmed = raw.trim(); + if trimmed.is_empty() { + bail!( + "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) was empty; a public \ + endpoint must be protected by a non-empty API key" + ); + } + // The key is later interpolated verbatim into raw `Authorization: + // Bearer {key}\r\n` header lines; reject a control character (e.g. an + // embedded CR/LF) here so a crafted key cannot inject extra headers. + if rocm_core::endpoint_api_key_has_forbidden_chars(trimmed) { + bail!( + "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) contained a control \ + character such as a carriage return or newline; an endpoint API key \ + must be a single line of printable characters" + ); + } + Ok(Some(trimmed.to_owned())) + } + None => Ok(Some(rocm_core::generate_endpoint_api_key())), + } +} + +/// When an equivalent managed service is already running, or the launch fails +/// before any record exists, the endpoint key serve() freshly stored for this +/// attempt is unused — drop it so it is not orphaned in storage. The already-running service keeps its own key. +/// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) +/// is a no-op. +pub(crate) fn drop_orphaned_endpoint_key_on_already_running( + paths: &AppPaths, + service_id: &str, + freshly_stored: Option<&str>, +) { + if freshly_stored.is_some() { + endpoint_keys::clear_endpoint_api_key(paths, service_id); + } +} + +/// Reject engine/platform combinations that cannot enforce a public endpoint's +/// API key, so a public bind fails closed instead of coming up unauthenticated. +/// +/// The one such case today: Windows managed Lemonade. Its server reads the +/// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn +/// primitive only carries path-valued env overrides, so the key never reaches it. +/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds +/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter +/// so both branches are unit-testable off-Windows. +pub(crate) fn ensure_public_bind_engine_supported( + engine: &str, + public_bind: bool, + is_windows: bool, +) -> Result<()> { + if public_bind && is_windows && engine == "lemonade" { + bail!( + "public binding with the lemonade engine is not supported on Windows: the endpoint \ + API key cannot be enforced there. Use `--engine vllm`, or bind a loopback host \ + (the default 127.0.0.1)." + ); + } + Ok(()) +} + +/// Refuse to (re)spawn a managed service recorded on a public host when its +/// endpoint API key is gone, so a respawn cannot reopen the endpoint anonymously. +/// +/// `serve()` applies the loopback-vs-public policy once, via +/// [`resolve_endpoint_auth`], and persists the resulting key. Every later spawn +/// — `rocm services restart`, and `rocmd`'s recovery supervisor — reads that key +/// file back and would otherwise treat "no key file" as "no auth wanted", +/// silently downgrading a protected public endpoint to an open one. The real +/// invariant is a property of the *host*, not of the file: a non-loopback bind +/// must always be authenticated. +/// +/// The gap is reachable through ordinary commands, because a stop deletes the +/// key file and `rocm services restart` accepts a stopped service id (its help +/// points at `rocm services list --all`). +/// +/// `key_present` is a plain `bool` rather than a path so both branches are +/// unit-testable without touching the filesystem, mirroring `is_windows` in +/// [`ensure_public_bind_engine_supported`]. +pub(crate) fn ensure_public_service_has_endpoint_key( + host: &str, + key_present: bool, + requires_api_key: bool, +) -> Result<()> { + // Two ways a service can need a key. A public bind is the obvious one. The + // other is a service that asked for auth on a loopback bind, because + // something outside this process republishes the port — a tailnet publish + // survives a reboot, let alone a restart, so "loopback" stops meaning + // "only this machine" and the bind address can no longer be trusted to + // answer the question on its own. + if requires_api_key && !key_present { + bail!( + "managed service was launched with `--require-api-key` but has no endpoint API key, \ + so restarting it would reopen it without authentication. Something outside this \ + machine may still be publishing its port. The key is dropped when a service stops \ + and cannot be recovered. Launch it again with \ + `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ + to choose the key instead of generating one)." + ); + } + if rocm_engine_protocol::is_public_bind_host(host) && !key_present { + bail!( + "managed service is bound to the public host `{host}` but has no endpoint API key, \ + so restarting it would reopen it without authentication. The key is dropped when a \ + service stops and cannot be recovered. Launch it again with \ + `rocm serve --host {host} --port --allow-public-bind` (add `--api-key `, or set \ + ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::test_paths; + use std::fs; + + #[test] + fn serve_bind_validation_requires_public_ack() { + validate_bind_host("127.0.0.1", false).unwrap(); + validate_bind_host("localhost", false).unwrap(); + validate_bind_host("::1", false).unwrap(); + let error = validate_bind_host("0.0.0.0", false).unwrap_err(); + assert!( + error.to_string().contains("--allow-public-bind"), + "{error:#}" + ); + validate_bind_host("0.0.0.0", true).unwrap(); + } + + #[test] + fn resolve_endpoint_auth_loopback_stays_credential_free() { + // Loopback binds never require auth, even if a key is supplied. + for host in ["127.0.0.1", "localhost", "::1"] { + assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); + assert_eq!( + resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), + None + ); + } + } + + #[test] + fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { + // "Loopback" describes the bind address, not who can reach the port. A + // tailnet publish, a proxy, or a container port map all leave the bind + // loopback while widening the audience, and the default policy would + // hand out an unauthenticated endpoint. Whoever widens the reach asks + // for the credential explicitly. + for host in ["127.0.0.1", "localhost", "::1"] { + let generated = resolve_endpoint_auth(host, None, true) + .unwrap() + .expect("a required key must be generated, not skipped"); + assert!(!generated.trim().is_empty()); + + assert_eq!( + resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), + Some("supplied-key".to_owned()), + "a supplied key must be honoured rather than ignored as it is by default" + ); + } + + // The same validation a public bind gets: an empty key is a refusal, not + // a silent downgrade to no auth. + assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); + } + + #[test] + fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { + let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) + .unwrap() + .expect("public bind must have a key"); + assert_eq!(key, "my-key"); + } + + #[test] + fn resolve_endpoint_auth_public_generates_key_when_absent() { + let key = resolve_endpoint_auth("0.0.0.0", None, false) + .unwrap() + .expect("public bind must generate a key"); + assert_eq!(key.len(), 48); + assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); + } + + #[test] + fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { + let error = resolve_endpoint_auth("0.0.0.0", Some(" "), false).unwrap_err(); + assert!(error.to_string().contains("non-empty"), "{error:#}"); + } + + #[test] + fn resolve_endpoint_auth_public_rejects_embedded_crlf() { + // A supplied key survives `trim()` with embedded CR/LF intact and would + // otherwise be interpolated into a raw `Authorization: Bearer` header, + // injecting an extra header line. It must be rejected at input validation. + for supplied in [ + "good-key\r\nX-Injected: value", + "good-key\nmore", + "line\rreturn", + ] { + let error = resolve_endpoint_auth("0.0.0.0", Some(supplied), false).unwrap_err(); + assert!(error.to_string().contains("control character"), "{error:#}"); + } + } + + #[test] + fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { + let (root, paths) = test_paths("drop-orphaned-key-stored"); + let service_id = "svc-orphaned"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); + + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); + + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { + // A loopback attempt never stores a key (`freshly_stored == None`), so the + // helper must not panic or error, and no file must appear. + let (root, paths) = test_paths("drop-orphaned-key-loopback"); + let service_id = "svc-loopback"; + + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); + + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn public_bind_fails_closed_for_windows_lemonade_only() { + // Windows + Lemonade + public bind: refuse (cannot enforce the key). + let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); + assert!(error.to_string().contains("lemonade"), "{error:#}"); + // Every other combination is allowed: + ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows + ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows + ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key + } + + #[test] + fn a_public_bind_is_refused_with_the_command_that_restores_it() { + // `resolve_endpoint_auth` mints a key for every non-loopback bind whether + // or not auth was demanded, so deriving `requires_api_key` from key-file + // presence marked ordinary public binds as having asked for it. The guard + // tests that field first, so those services were refused with a message + // naming a flag they never passed and a relaunch command that drops + // `--allow-public-bind` — coming back on loopback instead. + // + // A plain `--host 0.0.0.0 --allow-public-bind` launch: key file present, + // `--require-api-key` never passed. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false) + .expect_err("a public bind with no key must be refused"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("--allow-public-bind"), + "the refusal must name the command that restores the public bind: {rendered}" + ); + assert!( + !rendered.contains("--require-api-key"), + "a service that never passed the flag must not be told it did: {rendered}" + ); + + // And the loopback-with-forced-auth case still reports its own reason. + let error = ensure_public_service_has_endpoint_key("127.0.0.1", false, true) + .expect_err("a service that demanded auth must not come back without it"); + assert!( + format!("{error:#}").contains("--require-api-key"), + "{error:#}" + ); + } + + #[test] + fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { + // A stop deletes the key file, so a later restart of a public service + // would otherwise respawn it with no auth at all. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("0.0.0.0"), "{error:#}"); + assert!(message.contains("without authentication"), "{error:#}"); + // Actionable: name the command that mints a fresh key. + assert!(message.contains("--allow-public-bind"), "{error:#}"); + + // A public service that still has its key restarts normally. + ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); + } + + #[test] + fn respawn_allows_loopback_services_without_an_endpoint_key() { + // Loopback stays credential-free, so every accepted spelling must pass + // the guard with no key present — when nothing asked for auth. + for host in ["127.0.0.1", "localhost", "::1"] { + ensure_public_service_has_endpoint_key(host, false, false) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + } + } + + #[test] + fn respawn_refuses_a_loopback_service_that_was_launched_with_a_key() { + // The hole this closes: a loopback bind that something else republishes + // — a tailnet publish, a proxy, a container port map. The publish + // outlives the process, so a restart after the key was dropped would + // reopen a reachable endpoint with no authentication, and the bind + // address gives the guard no way to notice. + for host in ["127.0.0.1", "localhost", "::1"] { + let error = ensure_public_service_has_endpoint_key(host, false, true) + .expect_err("a service launched with a key must not restart without one"); + let message = format!("{error:#}"); + assert!(message.contains("without authentication"), "{message}"); + assert!(message.contains("--require-api-key"), "{message}"); + } + + // With its key still present it restarts normally. + ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); + } +} diff --git a/apps/rocm/src/service_launch/helper_autostart.rs b/apps/rocm/src/service_launch/helper_autostart.rs new file mode 100644 index 0000000000..430d152097 --- /dev/null +++ b/apps/rocm/src/service_launch/helper_autostart.rs @@ -0,0 +1,475 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Starting the background automation helper (`rocm daemon`) when a watcher needs +//! it, with the autostart claim that keeps concurrent callers from spawning two. + +#[cfg(not(windows))] +use super::spawn::{ + attach_background_stdio, detach_background_command, managed_service_process_command, +}; +use crate::background_helper::background_helper_already_running; +use crate::engine_runtime_select::managed_service_launcher_path; +#[cfg(windows)] +use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; +use anyhow::{Context, Result}; +use rocm_core::AppPaths; +use std::path::Path; +#[cfg(not(windows))] +use std::process::Stdio; + +/// Shared daemon-lifecycle entrypoint: ensures the background automation helper +/// (`rocm daemon`) is running when an automation watcher needs it, spawning it +/// detached if not. Liveness is read from the file-based automation runtime +/// state. Intentionally `pub(crate)` — reused by both the `serve --managed` path +/// and `automations enable`. Only the spawn result itself (`command.spawn()` / +/// `spawn_detached_no_inherit`) is logged rather than propagated; setup errors +/// (path discovery, config load, stdio attach) still return `Err`. +/// +/// The daemon's only work is running enabled automation watchers (server +/// recovery included — `server-recover` is one of them and is off by default), +/// and it exits on its own once none is enabled. So with no watcher enabled +/// nothing is spawned: a `serve` would otherwise leave a no-op daemon running +/// long after its service was stopped (ROCMAI-827). +pub(crate) fn ensure_background_helper_running() -> Result<()> { + ensure_background_helper_running_quiet(false) +} + +/// As [`ensure_background_helper_running`], but suppresses the stdout status line +/// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so +/// the daemon-spawn note does not appear above the deployment summary table. +pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { + let paths = AppPaths::discover()?; + ensure_background_helper_running_with(&paths, quiet, spawn_background_helper, &|| {}) +} + +/// The body of [`ensure_background_helper_running_quiet`], with the actual +/// process spawn injected so the decision around it is testable without +/// launching a daemon. `spawn` returns `Err` for a setup failure (propagated) +/// and `Ok(Err(..))` for a failed spawn (reported, not fatal). `on_lock_busy` is +/// called each time the autostart lock is found held by another process, which +/// lets a test wait for this call to be genuinely queued behind it. +fn ensure_background_helper_running_with( + paths: &AppPaths, + quiet: bool, + spawn: impl FnOnce(&AppPaths) -> Result>, + on_lock_busy: &dyn Fn(), +) -> Result<()> { + // An unreadable config is "no watcher": this also runs after `serve` has + // already launched its child, where failing would report a launch that + // happened as an error. + if !rocm_core::RocmCliConfig::load(paths).is_ok_and(|config| config.automation_daemon_enabled()) + { + return Ok(()); + } + + // Two concurrent callers (e.g. two `rocm serve`) can both read "not running" + // and each spawn a daemon. Serialize the decision on a lock file — the first + // holder spawns, later holders observe the now-running daemon and return + // without spawning. + // + // The "already running" read is deliberately made only under the lock, with + // no lock-free fast path. The daemon exits by itself once the last watcher + // is disabled, and it decides that under this same lock and publishes + // `running = false` before releasing it. Caller order is: save the enabling + // config, then come here. A lock-free read could still see the daemon that is + // about to exit as running, spawn nothing, and leave a watcher enabled with + // no daemon; under the lock the daemon has either seen the new config and + // stays, or has already published that it stopped. + let _autostart_lock = loop { + if let Some(lock) = + rocm_core::FileLock::try_acquire(paths.automation_autostart_lock_path())? + { + break lock; + } + on_lock_busy(); + std::thread::sleep(std::time::Duration::from_millis(10)); + }; + if background_helper_already_running(paths)? { + return Ok(()); + } + + // The lock alone does not close the window: the spawned daemon does not + // publish its `running` runtime state until well after `spawn()` (clap parse, + // runtime build, config load, banner flush). A second caller that acquires + // this lock during that gap still sees "not running" and would spawn a + // duplicate. Bridge the gap with a short-lived claim recording the child PID + // and spawn time: a holder that finds a live, recent claim defers instead. + let claim_path = paths.automation_autostart_claim_path(); + if autostart_spawn_in_flight( + read_autostart_claim(&claim_path), + now_unix_millis(), + AUTOSTART_CLAIM_TTL_MS, + rocm_core::process_is_running, + ) { + return Ok(()); + } + + match spawn(paths)? { + Ok(daemon_pid) => { + // Record the claim before returning (and thus releasing the lock) so a + // concurrent holder in the spawn→publish window defers. Best-effort: a + // failed write only reopens the original, already-tolerated race. + let _ = write_autostart_claim( + &claim_path, + AutostartClaim { + daemon_pid, + spawned_at_ms: now_unix_millis(), + }, + ); + if !quiet { + println!(" helper: started background automation daemon"); + } + } + Err(error) if !quiet => { + println!(" helper: could not start background automation daemon: {error}"); + } + Err(_) => {} + } + Ok(()) +} + +/// Spawn `rocm daemon` detached from this process. The outer `Result` is a setup +/// failure (resolving the executable, attaching stdio); the inner one is the +/// spawn itself. +fn spawn_background_helper(paths: &AppPaths) -> Result> { + let exe = managed_service_launcher_path() + .context("failed to resolve current rocm executable path")?; + let args = vec!["daemon".to_owned()]; + #[cfg(windows)] + let spawn_result = { + let env_values = app_path_env_var_values(paths, None); + let env_refs = app_path_env_var_refs(&env_values); + rocm_core::spawn_detached_no_inherit(&exe, &args, &env_refs) + }; + #[cfg(not(windows))] + let spawn_result = { + let mut command = managed_service_process_command(&exe, &args); + command.stdin(Stdio::null()); + attach_background_stdio(&mut command, None)?; + detach_background_command(&mut command); + paths.apply_env(&mut command); + command + .spawn() + .map(|child| child.id()) + .map_err(anyhow::Error::from) + }; + Ok(spawn_result) +} + +/// How long an autostart claim is honoured before it is treated as stale even if +/// its recorded PID is still alive. Comfortably longer than a cold daemon boot +/// (clap parse → runtime build → config load → state publish) yet short enough +/// that a crashed spawn cannot suppress autostart for long. +const AUTOSTART_CLAIM_TTL_MS: u128 = 30_000; + +/// A just-spawned daemon's autostart claim: the child PID and the wall-clock time +/// (milliseconds since the Unix epoch) the spawn was recorded. It lets a +/// concurrent autostart holder distinguish a live, in-flight spawn from a stale +/// leftover. Serialized as a single `" "` line — no dependency and +/// trivially forward-compatible. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AutostartClaim { + daemon_pid: u32, + spawned_at_ms: u128, +} + +/// Milliseconds since the Unix epoch, or `0` if the clock is before the epoch +/// (which only makes a fresh claim look old — safe, it just permits a respawn). +fn now_unix_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis()) +} + +/// Read an autostart claim, returning `None` when the file is absent or +/// unparseable (either is treated as "no claim", so a respawn is permitted). +fn read_autostart_claim(path: &Path) -> Option { + let text = std::fs::read_to_string(path).ok()?; + let mut parts = text.split_whitespace(); + let daemon_pid = parts.next()?.parse().ok()?; + let spawned_at_ms = parts.next()?.parse().ok()?; + Some(AutostartClaim { + daemon_pid, + spawned_at_ms, + }) +} + +/// Write an autostart claim as `" "`. Best-effort at the call site. +fn write_autostart_claim(path: &Path, claim: AutostartClaim) -> std::io::Result<()> { + std::fs::write( + path, + format!("{} {}", claim.daemon_pid, claim.spawned_at_ms), + ) +} + +/// Whether an existing autostart `claim` means a daemon spawn is still in flight, +/// so the current lock holder should defer rather than spawn a duplicate. A claim +/// counts as in-flight only while its child PID is alive *and* it is younger than +/// `ttl_ms` — the TTL bounds how long a crashed spawn (or a PID later reused by an +/// unrelated process) can suppress autostart. `pid_alive` is injected so the +/// decision is unit-testable without a live process. +fn autostart_spawn_in_flight( + claim: Option, + now_ms: u128, + ttl_ms: u128, + pid_alive: impl Fn(u32) -> bool, +) -> bool { + claim.is_some_and(|claim| { + now_ms.saturating_sub(claim.spawned_at_ms) < ttl_ms && pid_alive(claim.daemon_pid) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{runtime_state, set_helper_facts}; + use crate::tests::test_paths; + use std::fs; + + /// ROCMAI-827: `rocm serve` started the background daemon on every launch, + /// and with no automation watcher enabled that daemon had nothing to do yet + /// kept running after every service was stopped. With the default config + /// (no watcher enabled) the helper must not be spawned at all. + #[test] + fn background_helper_not_spawned_when_no_automation_is_enabled() { + let (root, paths) = test_paths("helper-no-automation"); + paths.ensure().expect("create app dirs"); + // The stored flag alone is not "a watcher is enabled": nothing would run. + let mut config = rocm_core::RocmCliConfig::default(); + config.automations.daemon_enabled = true; + config.save(&paths).expect("save default config"); + + let mut spawned = false; + ensure_background_helper_running_with( + &paths, + true, + |_| { + spawned = true; + Ok(Ok(4242)) + }, + &|| {}, + ) + .expect("ensure helper"); + + let claim_written = paths.automation_autostart_claim_path().exists(); + let _ = fs::remove_dir_all(&root); + assert!( + !spawned, + "no watcher enabled ⇒ the daemon has no work and must not be started" + ); + assert!(!claim_written, "no spawn ⇒ no autostart claim"); + } + + /// The other half of the gate: once a watcher is enabled the helper is still + /// started (exactly once) and its autostart claim recorded. + #[test] + fn background_helper_spawned_when_an_automation_is_enabled() { + let (root, paths) = test_paths("helper-automation-enabled"); + paths.ensure().expect("create app dirs"); + let mut config = rocm_core::RocmCliConfig::default(); + config.watcher_config_mut("server-recover").enabled = true; + config.save(&paths).expect("save config"); + + let mut spawns = 0; + ensure_background_helper_running_with( + &paths, + true, + |_| { + spawns += 1; + Ok(Ok(4242)) + }, + &|| {}, + ) + .expect("ensure helper"); + + let claim = read_autostart_claim(&paths.automation_autostart_claim_path()); + let _ = fs::remove_dir_all(&root); + assert_eq!(spawns, 1, "an enabled watcher needs the daemon"); + assert_eq!(claim.map(|claim| claim.daemon_pid), Some(4242)); + } + + /// ROCMAI-827 race, enable side: the daemon that is about to exit still has + /// `running = true` and a live pid in its state when `automations enable` + /// (config already saved) asks for a helper. It decides under the autostart + /// lock and publishes "stopped" before releasing it, so the enable must wait + /// on that lock and then start a new helper. A lock-free "already running" + /// read would return here without spawning, leaving a watcher with no daemon. + #[test] + fn background_helper_is_started_when_the_exiting_daemon_publishes_stopped() { + let (root, paths) = test_paths("helper-exit-window"); + paths.ensure().expect("create app dirs"); + let mut config = rocm_core::RocmCliConfig::default(); + config.watcher_config_mut("server-recover").enabled = true; + config.save(&paths).expect("save config"); + // The old daemon: alive (this process stands in) and still `running`. + runtime_state(true, std::process::id()) + .write(&paths) + .expect("write runtime state"); + + // The exiting daemon holds the autostart lock while it decides. + let exiting_daemon_lock = + rocm_core::FileLock::acquire(paths.automation_autostart_lock_path()) + .expect("hold the lock"); + let enable_paths = paths.clone(); + let queued_flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let queued = queued_flag.clone(); + let enable = std::thread::spawn(move || { + let mut spawns = 0; + ensure_background_helper_running_with( + &enable_paths, + true, + |_| { + spawns += 1; + Ok(Ok(4242)) + }, + &|| queued.store(true, std::sync::atomic::Ordering::SeqCst), + ) + .expect("ensure helper"); + spawns + }); + // Wait until the enable path has actually found the lock held and queued + // behind it, rather than assuming how long that takes. + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); + // A caller that skipped the lock finishes without ever queuing. + while !queued_flag.load(std::sync::atomic::Ordering::SeqCst) && !enable.is_finished() { + assert!(std::time::Instant::now() < deadline, "enable never queued"); + std::thread::yield_now(); + } + runtime_state(false, std::process::id()) + .write(&paths) + .expect("daemon publishes stopped"); + drop(exiting_daemon_lock); + + let spawns = enable.join().expect("enable thread"); + let _ = fs::remove_dir_all(&root); + assert_eq!( + spawns, 1, + "the enable must start a helper once the old one has published it stopped" + ); + } + + fn stop_result() -> serde_json::Value { + serde_json::json!({"output": {"result": { + "service": {"service_id": "svc-a", "status": "stopped"}, + "signaled_pids": [1] + }}}) + } + + /// `rocm services stop` prints the note for a stop only, and only the variant + /// that matches the state. + #[test] + fn stop_output_carries_the_helper_note_that_matches_the_state() { + use crate::services_text::render_service_action_result_with_helper_note as render; + + let (root, paths) = test_paths("stop-helper-note"); + paths.ensure().expect("create app dirs"); + set_helper_facts(&paths, false, false); + let quiet = render(&paths, "stop_server", &stop_result()); + set_helper_facts(&paths, true, true); + let outlives = render(&paths, "stop_server", &stop_result()); + let restart = render(&paths, "restart_server", &stop_result()); + set_helper_facts(&paths, true, false); + let not_running = render(&paths, "stop_server", &stop_result()); + let _ = fs::remove_dir_all(&root); + + assert!(!quiet.contains("background automation helper"), "{quiet}"); + assert!( + outlives.contains("stopped processes: 1") + && outlives.contains("keeps running while an automation is enabled") + && outlives.contains("`rocm automations disable server-recover`"), + "{outlives}" + ); + assert!( + not_running.contains("is not running") && !not_running.contains("keeps running"), + "{not_running}" + ); + assert!( + !restart.contains("background automation helper"), + "{restart}" + ); + } + + /// `serve` calls the helper autostart after its child is already running. An + /// unreadable config must not turn that launch into an error: it reads as "no + /// watcher enabled" and nothing is spawned. + #[test] + fn background_helper_autostart_treats_an_unreadable_config_as_no_watcher() { + let (root, paths) = test_paths("helper-bad-config"); + paths.ensure().expect("create app dirs"); + fs::write(paths.config_path(), b"{ this is not json").expect("write bad config"); + assert!(rocm_core::RocmCliConfig::load(&paths).is_err()); + + let mut spawned = false; + let result = ensure_background_helper_running_with( + &paths, + true, + |_| { + spawned = true; + Ok(Ok(4242)) + }, + &|| {}, + ); + let _ = fs::remove_dir_all(&root); + + assert!(result.is_ok(), "{result:?}"); + assert!(!spawned); + } + + #[test] + fn autostart_spawn_in_flight_defers_only_for_a_live_recent_claim() { + let claim = AutostartClaim { + daemon_pid: 4242, + spawned_at_ms: 1_000, + }; + // A recent claim whose PID is still alive ⇒ a spawn is in flight, defer. + assert!(autostart_spawn_in_flight( + Some(claim), + 1_000 + 5_000, + AUTOSTART_CLAIM_TTL_MS, + |_| true, + )); + // Same claim but the child PID is gone (crashed spawn) ⇒ respawn. + assert!(!autostart_spawn_in_flight( + Some(claim), + 1_000 + 5_000, + AUTOSTART_CLAIM_TTL_MS, + |_| false, + )); + // Older than the TTL, even with a live PID (possible PID reuse) ⇒ respawn. + assert!(!autostart_spawn_in_flight( + Some(claim), + 1_000 + AUTOSTART_CLAIM_TTL_MS, + AUTOSTART_CLAIM_TTL_MS, + |_| true, + )); + // No claim at all ⇒ nothing in flight, spawn. + assert!(!autostart_spawn_in_flight( + None, + 1_000, + AUTOSTART_CLAIM_TTL_MS, + |_| true, + )); + } + + #[test] + fn autostart_claim_round_trips_and_absent_or_garbage_reads_as_none() { + let (root, paths) = test_paths("autostart-claim-roundtrip"); + let path = paths.automation_autostart_claim_path(); + fs::create_dir_all(path.parent().expect("claim parent")).expect("mkdir claim dir"); + // Absent file ⇒ no claim. + assert_eq!(read_autostart_claim(&path), None); + // Round-trip a written claim. + let claim = AutostartClaim { + daemon_pid: 9987, + spawned_at_ms: 1_726_000_000_123, + }; + write_autostart_claim(&path, claim).expect("write claim"); + assert_eq!(read_autostart_claim(&path), Some(claim)); + // Garbage / partial content ⇒ no claim (permits a respawn, never panics). + fs::write(&path, "not-a-pid").expect("write garbage"); + assert_eq!(read_autostart_claim(&path), None); + let _ = fs::remove_dir_all(&root); + } +} diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs new file mode 100644 index 0000000000..4fb530a5b8 --- /dev/null +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -0,0 +1,1137 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The locked launch transaction around a managed service: the port allocation +//! decision against live records and the OS, the cleanup of a failed launch +//! attempt, and the reuse refusals (including the no-usable-GPU wording) that +//! decide whether a live service may answer a `serve` request. + +use crate::endpoint_keys; +use crate::managed_service_status::{ + existing_live_managed_service_in, load_managed_services, managed_service_is_live, +}; +use crate::serve_port; +use crate::{device_policy_name, record_cli_audit_event}; +use anyhow::Result; +use rocm_core::{AppPaths, ManagedServiceRecord}; +use rocm_engine_protocol::DevicePolicy; +use std::fs; +use std::time::Duration; + +/// Outcome of the locked allocation step. +/// +/// `Debug` so tests can assert on `Result::expect_err`. +#[derive(Debug)] +pub(super) enum ManagedPortDecision { + /// An equivalent managed service is already live; nothing may be spawned. + /// Boxed to keep the variants a similar size (clippy::large_enum_variant). + AlreadyLive(Box), + /// The concrete port this launch owns until it hands it to the child. + Lease(rocm_core::LoopbackPortLease), +} + +/// Live managed services, with liveness refreshed by [`load_managed_services`] +/// (dead PIDs demote to "stopped" and therefore neither satisfy the duplicate +/// check nor reserve a port). +/// +/// A read failure degrades to "nothing is live": for an already-`ready` +/// service the OS preflight still guards its port (the engine holds it), and +/// refusing to serve because the services directory could not be listed would +/// be a worse failure than launching. That guard does not cover a `starting` +/// record: its port is released back to the OS in +/// `launch_managed_engine_child_locked` (see `lease.release()`), right before the +/// spawn, and stays unbound until the new engine child has actually bound it, so a concurrent launch that hits this +/// read failure in that narrow window can probe the same port successfully. +/// Accepted as a known gap — narrower than the general services-dir read +/// failure case, and only reachable by a second, near-simultaneous I/O +/// failure on top of one already in flight. +pub(super) fn live_managed_services(paths: &AppPaths) -> Vec { + load_managed_services(paths) + .unwrap_or_default() + .into_iter() + .filter(managed_service_is_live) + .collect() +} + +/// Ports that live managed services still own. +/// +/// Conservative by design: on the default loopback host a record reserves its +/// port regardless of the host it recorded, because a `0.0.0.0` (or `localhost`) +/// listener also blocks the loopback address a new local server would bind. +/// Over-reserving costs one candidate in a 101-port scan; under-reserving costs +/// a collision. On a custom host the recorded host is compared (see +/// `serve_port::hosts_overlap`). +fn reserved_service_ports(live: &[ManagedServiceRecord]) -> Vec { + live.iter() + .map(|record| serve_port::ReservedPort { + port: record.port, + host: record.host.clone(), + service_id: record.service_id.clone(), + status: record.status.clone(), + }) + .collect() +} + +/// The decision half of the launch transaction: refresh liveness, apply the +/// `(engine, canonical_model_id)` idempotency guard, then lease a concrete port +/// against both live records and the real OS. +/// +/// Callers MUST hold [`rocm_core::AppPaths::managed_launch_lock_path`] across +/// this call *and* the record publication that follows, or two launches can +/// pick the same candidate. +/// `probe` is injected so tests can drive deterministic bind outcomes without a +/// GPU or a spawn. +pub(super) fn resolve_managed_port_in_transaction( + paths: &AppPaths, + engine: &str, + canonical_model_id: &str, + host: &str, + request: serve_port::PortRequest, + probe: &mut dyn FnMut(u16) -> std::io::Result, +) -> Result { + let live = live_managed_services(paths); + // Idempotency first: an already-live equivalent service means no port is + // needed at all, so it must be answered before any candidate is leased. + if let Some(existing) = existing_live_managed_service_in(&live, engine, canonical_model_id) { + return Ok(ManagedPortDecision::AlreadyLive(Box::new(existing.clone()))); + } + let reserved = reserved_service_ports(&live); + serve_port::resolve_serve_port(host, request, &reserved, probe).map(ManagedPortDecision::Lease) +} + +/// Grace given to a child that must be terminated because its record could not +/// be published. Short: the child is milliseconds old and has nothing to flush. +const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); + +/// Whether this host records a start token for a process: Linux (the kernel +/// start-time) and Windows (the creation time). Anywhere else +/// `rocm_core::process_start_ticks` has nothing to read. +const fn host_records_start_tokens() -> bool { + cfg!(any(target_os = "linux", windows)) +} + +/// The verdict that ends a failed launch's cleanup before anything is signalled: +/// `Some(Unverified)` for a child whose token was not captured on a host that +/// records tokens, because that child was gone before it could be captured and +/// its pid may now belong to another process. `None` otherwise: a captured token +/// is verified by the termination, and on a host with no tokens there is nothing +/// to verify, so the kill proceeds as it always did there. +const fn launch_cleanup_refusal( + engine_start_ticks: Option, + host_records_tokens: bool, +) -> Option { + if engine_start_ticks.is_none() && host_records_tokens { + Some(rocm_core::TerminationOutcome::Unverified) + } else { + None + } +} + +/// Undo a failed managed launch attempt before the launch lock is released, +/// so no failed transaction leaves a live reservation or a stray endpoint key. +/// +/// Order matters: a spawned child is terminated *first*, so it is not still +/// holding the port or writing to the log by the time cleanup runs. The +/// termination outcome is audited but not acted on: the record is marked +/// abandoned (`stopped`, releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), +/// every artifact this attempt created is removed: there is nothing for +/// `rocm services`/`rocm logs` to show. If a child did run, its record and log +/// are kept instead (marked `status = "stopped"`, see +/// [`ManagedServiceRecord::mark_launch_abandoned`]), since the error that led here +/// already told the user to go inspect that log; only the endpoint key, which a +/// failed attempt has no use for, is cleared. +/// Every step is best-effort and idempotent; cleanup must never mask the +/// original failure with a second one. +pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { + // The shared routine `rocm services stop` and `rocmd` use, not + // `terminate_verified` alone: on Windows only the routine takes the + // processes under the child down, so a failed launch cannot leave the engine + // one level below the supervisor running. + abandon_failed_managed_launch_with(paths, record, |identity| { + rocm_core::terminate_recorded_pid_within(identity, MANAGED_LAUNCH_CLEANUP_GRACE) + }); +} + +/// [`abandon_failed_managed_launch`] with the termination of the child supplied +/// by the caller, so a test sees what the cleanup asks it to terminate, and +/// whether it asks at all. +fn abandon_failed_managed_launch_with( + paths: &AppPaths, + record: &ManagedServiceRecord, + terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, +) { + if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { + // The token captured at spawn, not a fresh `capture`: when the child + // already exited, a fresh one would describe a recycled pid's new owner + // and the forced tree kill would hit it. Where start tokens exist + // (Linux, and Windows, which records the process creation time), a + // missing one means the child was gone before it could be captured, so + // the pid is unverifiable and nothing is signalled (`terminate_verified` + // alone would "best-effort proceed"). A token that was captured is + // verified by the termination on both. + // + // The tradeoff on Windows, chosen deliberately: a child whose creation + // time could not be read at capture is not killed, so if `OpenProcess` + // failed for a child that is in fact alive (it is this account's own + // fresh child, so that is not expected), a failed launch leaves its + // engine running and the audit event says `unverified`. The alternative + // is a forced tree kill of a pid that may by now belong to another + // process, which cannot be undone. The spawn primitive returns only the + // pid and closes its handle, so a handle-based identity would mean + // changing it; that is not done here. The refusal decision + // (`launch_cleanup_refusal`) and what the cleanup asks the termination to + // do are under tests that run on Windows; the cleanup of a real Windows + // child is run by no test. + let outcome = if let Some(refusal) = + launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) + { + refusal + } else { + terminate(&rocm_core::ProcessIdentity::new( + pid, + record.engine_start_ticks, + )) + }; + record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "terminated engine child pid={pid} after a failed launch publication outcome={}", + outcome.as_str() + ), + Some(&record.service_id), + ); + let mut abandoned = record.clone(); + abandoned.mark_launch_abandoned(); + let _ = abandoned.write(); + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + return; + } + for path in [ + record.manifest_path.as_path(), + record.log_path.as_path(), + record.engine_state_path.as_path(), + ] { + let _ = fs::remove_file(path); + } + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); +} + +/// The ways a live managed service that already matches on liveness, engine and +/// canonical model id can still be unusable by this invocation. +/// +/// Those three terms are necessary but not sufficient: three further terms can +/// still refuse, one variant each. [`managed_service_reuse_refusal`] is the +/// single decision procedure both callers ask, so the two cannot answer +/// differently — which they previously did; see that function for what the +/// disagreement cost. +pub(crate) enum ManagedReuseRefusal { + /// The live service was started with different serve options. + RecipeMismatch, + /// `--require-api-key` was asked for and the live service has no key. + /// + /// The engine reads its key once, at launch, from the environment the spawn + /// builds — so a server started without one keeps serving anonymously no + /// matter what is written afterwards. Upgrading the record would be worse + /// than doing nothing: it would claim auth the live process does not + /// enforce, and `ensure_public_service_has_endpoint_key` would pass on the + /// strength of a key file nothing reads. `rocm remote serve` relies on this: + /// it publishes a loopback port onto the tailnet and prints "the API key + /// above is what stops anyone else calling it", which reusing an + /// unauthenticated service silently would make false about an endpoint the + /// whole tailnet can reach. Refusing is the only answer that fails closed. + MissingApiKey, + /// The live service runs under a device policy that does not satisfy this + /// invocation's. Reusing a `cpu_only` service to answer a `gpu_required` + /// serve is the silent CPU fallback AGENTS.md section 6 forbids. + DevicePolicyTooWeak, +} + +/// Why this invocation cannot reuse `existing` — or `None` when it genuinely +/// can. See [`ManagedReuseRefusal`] for the terms this answers on. +/// +/// This exists because `serve()`'s reuse pre-gate lets a matching live service +/// skip the no-usable-AMD-GPU refusal. Deciding that bypass on a *weaker* test +/// than the spawn then uses to accept reuse produced a dead end: on a GPU-less +/// host (`HIP_VISIBLE_DEVICES=` with a server live from another shell) a serve +/// with different options bypassed the refusal, reached the spawn, and was told +/// to stop the server and run the command again — and the rerun, with nothing +/// left to reuse, hit the no-GPU refusal. The advice named a command that could +/// not clear the condition, which AGENTS.md section 3 forbids. Sharing the +/// predicate means a service that cannot actually be reused never buys the +/// bypass, so the user gets the accurate refusal first and keeps their server. +/// +/// `requested_device_policy` is taken by value, not as an `Option`, on purpose. +/// It was optional so the spawn could opt out of the device-policy term, and +/// that left two holes at once: the spawn's `DevicePolicyTooWeak` arm was +/// unreachable (an untestable bail), and `serve()` — the caller that *does* +/// gate the GPU refusal on this answer — could silently drop the term by +/// passing `None`, with the whole suite still green. Both callers have a +/// `DevicePolicy` in hand, so requiring it costs nothing and makes that +/// mutation a compile error. It widens nothing observable today either: the +/// only policy a record can carry is the one `parse_device_policy` admits +/// (see below), so `existing.device_policy == Some("cpu_only")` is not +/// reachable from any `rocm serve` this CLI can run. +/// The `rocm serve` no-usable-GPU refusal, worded once. +/// +/// Two places raise it and they must say the same thing: `serve()`'s fail-fast +/// pre-flight, and [`spawn_managed_engine_child`](super::spawn::spawn_managed_engine_child) +/// when the reuse that waived that pre-flight turns out not to be available after +/// all. +pub(crate) fn no_usable_gpu_refusal(device_policy: &DevicePolicy) -> String { + // Keyed on the same `is_wsl_host()` the usable-GPU probe branches on, so the + // advice follows the platform whose answer produced the refusal. + no_usable_gpu_refusal_text(device_policy_name(device_policy), rocm_core::is_wsl_host()) +} + +/// The refusal text for `policy`, split on whether the probe that refused was +/// the WSL one. +/// +/// WSL reaches the GPU through `/dev/dxg` and ROCDXG, and has no `/dev/kfd`, so +/// telling a WSL user to confirm that device is present names something that can +/// never clear the condition. There the advice points at the status `rocm +/// examine` reports for the path the probe actually asked about +/// ([`rocm_core::WSL_ROCDXG_READY`]), and at `rocm diagnose`, which names what is +/// missing. +fn no_usable_gpu_refusal_text(policy: &str, on_wsl: bool) -> String { + let remedy = if on_wsl { + format!( + "WSL reaches the GPU through /dev/dxg and ROCDXG rather than /dev/kfd: run \ + `rocm examine` and check that `driver_status` is `{ready}` (`rocm diagnose` \ + names what is missing; a ROCDXG library the loader reaches only through \ + LD_LIBRARY_PATH needs that variable exported in this shell, see docs/wsl.md), and ensure HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES \ + are not masking every device.", + ready = rocm_core::WSL_ROCDXG_READY + ) + } else { + "Check the driver with `rocm examine`, confirm /dev/kfd is present, and ensure \ + HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES are not masking every device." + .to_owned() + }; + format!( + "no usable AMD GPU detected; `rocm serve` requires a GPU under the {policy} policy \ + and does not fall back to CPU. {remedy}" + ) +} + +pub(crate) fn managed_service_reuse_refusal( + existing: &ManagedServiceRecord, + requested_recipe_json: Option<&str>, + require_api_key: bool, + requested_device_policy: &DevicePolicy, +) -> Option { + if existing.engine_recipe_json.as_deref() != requested_recipe_json { + return Some(ManagedReuseRefusal::RecipeMismatch); + } + if require_api_key && !existing.requires_api_key { + return Some(ManagedReuseRefusal::MissingApiKey); + } + // Only `gpu_required` can be let down by the record's policy, and the + // enumeration is exhaustive rather than narrow: `parse_device_policy` + // (`main.rs`) maps `auto`, `gpu`, `gpu_required` *and* `gpu_preferred` onto + // `GpuRequired`, and rejects `cpu_only` with an error, so `GpuRequired` is + // the only policy `serve()` can hand this predicate — pinned by + // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`. There + // is deliberately no wider match on "anything but `CpuOnly`": it would be + // identical at runtime and would read as a guard against a `gpu_preferred` + // bypass that no CLI invocation can reach. A `cpu_only` invocation would be + // satisfied by anything anyway and never reaches the bail. README.md's + // "that is the only policy pair refused" is the contract, and this matches + // it exactly. + if matches!(requested_device_policy, DevicePolicy::GpuRequired) + && existing.device_policy.as_deref() == Some("cpu_only") + { + return Some(ManagedReuseRefusal::DevicePolicyTooWeak); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::service_hints; + #[cfg(not(windows))] + use crate::service_launch::test_support::sleeper_test_name; + use crate::service_launch::test_support::{reusable_record, write_live_service_record}; + use crate::tests::test_paths; + use rocm_core::DEFAULT_LOCAL_HOST; + #[cfg(not(windows))] + use std::process::Command as ProcessCommand; + use std::thread; + + #[test] + fn the_no_gpu_refusal_sends_a_bare_metal_user_to_dev_kfd() { + let text = super::no_usable_gpu_refusal_text("gpu_required", false); + assert!(text.contains("confirm /dev/kfd is present"), "{text}"); + assert!(!text.contains("ROCDXG"), "{text}"); + } + + #[test] + fn the_no_gpu_refusal_does_not_send_a_wsl_user_to_dev_kfd() { + // WSL has no /dev/kfd, so advice to confirm it can never clear the + // condition. The text names the status `examine` reports for the path + // the probe asked about, taken from the constant the producer uses, so + // the two cannot drift. That the status really is the one a refused + // host shows is asserted end to end by serve-42. + let text = super::no_usable_gpu_refusal_text("gpu_required", true); + assert!(!text.contains("/dev/kfd is present"), "{text}"); + assert!(text.contains("rocm examine"), "{text}"); + assert!(text.contains("rocm diagnose"), "{text}"); + // The advice is true only because `rocdxg_ready` counts a library on + // LD_LIBRARY_PATH; `a_rocdxg_only_on_the_loader_path_is_ready` in + // `rocm-core`'s host_gpu/wsl.rs proves that behavior. + assert!(text.contains("LD_LIBRARY_PATH"), "{text}"); + assert!( + text.contains(&format!("`{}`", rocm_core::WSL_ROCDXG_READY)), + "{text}" + ); + } + + #[test] + fn the_no_gpu_refusal_names_the_policy_on_both_platforms() { + for on_wsl in [false, true] { + let text = super::no_usable_gpu_refusal_text("gpu_required", on_wsl); + assert!( + text.starts_with("no usable AMD GPU detected; `rocm serve` requires a GPU under the gpu_required policy"), + "{text}" + ); + } + } + + #[test] + fn reuse_is_refused_on_every_term_the_spawn_and_the_gpu_bail_depend_on() { + let (root, paths) = test_paths("managed-reuse-predicate"); + paths.ensure().expect("paths"); + let base = reusable_record(&paths); + + // The baseline must be reusable, or the negative cases below could pass + // for the wrong reason. + assert!( + managed_service_reuse_refusal(&base, None, true, &DevicePolicy::GpuRequired).is_none(), + "a matching service with no recipe, a key, and a gpu_required policy is reusable" + ); + + // Recipe mismatch — the spawn's first refusal. + let mut mismatched = base.clone(); + mismatched.engine_recipe_json = Some("{\"engine\":\"vllm\"}".to_owned()); + assert!(matches!( + managed_service_reuse_refusal(&mismatched, None, true, &DevicePolicy::GpuRequired), + Some(ManagedReuseRefusal::RecipeMismatch) + )); + + // Unmet `--require-api-key` — the spawn's second refusal. Also pins that + // it is *only* refused when the key is actually demanded. + let mut anonymous = base.clone(); + anonymous.requires_api_key = false; + assert!(matches!( + managed_service_reuse_refusal(&anonymous, None, true, &DevicePolicy::GpuRequired), + Some(ManagedReuseRefusal::MissingApiKey) + )); + assert!( + managed_service_reuse_refusal(&anonymous, None, false, &DevicePolicy::GpuRequired) + .is_none(), + "an anonymous service is fine when no key was demanded" + ); + + // Device policy — the term `serve()` depends on, because it is the one + // gating the no-usable-GPU bail on the answer. Reusing a `cpu_only` + // service to answer a `gpu_required` serve is the silent CPU fallback. + let mut cpu_only = base; + cpu_only.device_policy = Some("cpu_only".to_owned()); + assert!(matches!( + managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::GpuRequired), + Some(ManagedReuseRefusal::DevicePolicyTooWeak) + )); + // `GpuRequired` is the only policy the CLI can produce — `--device + // gpu_preferred`/`auto`/`gpu` all parse to it and `cpu_only` is rejected + // (`parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`) — + // so there is no second policy arm to discriminate here, and asserting + // one would pin an input `serve()` never produces. Both callers pass + // the same policy; the parameter is not optional, so neither can drop + // this term without a compile error. + // + // A cpu_only *invocation* never reaches the bail, so nothing to refuse. + assert!( + managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::CpuOnly).is_none() + ); + + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { + // `generate_service_id` embeds a timestamp, so a second launch for the + // same engine+model has a DIFFERENT service_id. The guard must still + // detect the live service by (engine, canonical_model_id), and return + // the newest live match. `starting` skips the endpoint probe; the + // current process id is a guaranteed-live PID. + let (root, paths) = test_paths("dup-managed-distinct-ids"); + paths.ensure()?; + + // Older, dead manifest for the same engine+model (distinct service_id). + let mut dead = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11500, + "managed", + 999_999_999, + None, + None, + None, + ); + dead.status = "ready".to_owned(); + dead.engine_pid = Some(999_999_999); + dead.created_at_unix_ms = 1000; + dead.write()?; + + // Newer, live manifest for the same engine+model (distinct service_id). + let mut live = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-2000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11501, + "managed", + std::process::id(), + None, + None, + None, + ); + live.status = "starting".to_owned(); + live.engine_pid = Some(std::process::id()); + live.created_at_unix_ms = 2000; + live.write()?; + + let live_services = live_managed_services(&paths); + let found = + existing_live_managed_service_in(&live_services, "lemonade", "qwen-canonical").cloned(); + let _ = fs::remove_dir_all(root); + + let found = found.expect("a live managed service should be detected by engine+model"); + assert_eq!( + found.service_id, "lemonade-qwen-2000", + "should return the newest live match" + ); + assert!(managed_service_is_live(&found)); + Ok(()) + } + + #[test] + fn dead_managed_service_allows_relaunch() -> Result<()> { + // A stale manifest with dead PIDs must NOT block a relaunch: liveness + // refresh demotes it to "stopped", so the guard returns None. + let (root, paths) = test_paths("dup-managed-dead"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11502, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a dead managed service must not block relaunch" + ); + Ok(()) + } + + #[test] + fn live_service_for_other_model_does_not_block() -> Result<()> { + // A live service for a DIFFERENT model must not match — the guard keys + // on the model, not just the engine. + let (root, paths) = test_paths("dup-managed-other-model"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-other-1", + "lemonade", + "other", + "other-canonical", + "127.0.0.1", + 11503, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a live service for a different model must not match" + ); + Ok(()) + } + + #[test] + fn missing_manifest_allows_launch() { + // No services dir / manifests → nothing to detect, launch proceeds. + let (root, paths) = test_paths("dup-managed-missing"); + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + assert!(found.is_none()); + } + + fn never_probe(_port: u16) -> std::io::Result { + panic!("no port may be probed on this path") + } + + fn always_free(port: u16) -> std::io::Result { + Ok(rocm_core::LoopbackPortLease::engine_owned(port)) + } + + #[test] + fn allocation_transaction_answers_an_equivalent_live_service_before_leasing() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-already-live"); + paths.ensure()?; + let existing = write_live_service_record( + &paths, + "lemonade-qwen-9000", + "qwen-canonical", + 11501, + "starting", + )?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "qwen-canonical", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut never_probe, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::AlreadyLive(record) => { + assert_eq!(record.service_id, existing.service_id); + assert_eq!(record.port, existing.port); + } + ManagedPortDecision::Lease(_) => { + panic!("an equivalent live service must be answered, not leased a new port") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_skips_ports_reserved_by_other_live_services() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-skips-reserved"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-a", "model-a", 11_435, "running")?; + write_live_service_record(&paths, "lemonade-b", "model-b", 11_436, "recovering")?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-c", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_437), + ManagedPortDecision::AlreadyLive(record) => { + panic!("model-c has no live service, expected a lease, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_ignores_stopped_records() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-ignores-stopped"); + paths.ensure()?; + let mut stale = + write_live_service_record(&paths, "lemonade-stale", "model-stale", 11_435, "running")?; + // A dead pid demotes the record to stopped when liveness is refreshed, so + // it must not reserve its port. + stale.engine_pid = Some(999_999_999); + stale.supervisor_pid = 999_999_999; + stale.write()?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-fresh", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(record) => { + panic!("a stopped record must not block a fresh launch, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn stopping_the_reserving_record_frees_its_explicit_port_for_reuse() -> Result<()> { + // Proves the mechanism `rocm services stop` relies on: it never touches + // the port directly, it only flips the stopped record's `status` (see + // the `all_stopped` branch in `main.rs`'s stop handler). This recreates + // that exact disk state and checks the same explicit port a live + // competitor refused now resolves, closing the loop the E2E + // `assert_port_reserved_message` step only checks by wording. + let (root, paths) = test_paths("alloc-explicit-port-freed-by-stop"); + paths.ensure()?; + let mut holder = ManagedServiceRecord::new( + &paths, + "lemonade-a-1", + "lemonade", + "model-a", + "model-a".to_owned(), + DEFAULT_LOCAL_HOST, + 11_435, + "managed", + std::process::id(), + None, + None, + None, + ); + holder.status = "running".to_owned(); + holder.engine_pid = Some(std::process::id()); + holder.write()?; + + let still_reserved = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + // The refusal names the service AND the command that frees the port; + // the stop below is that command's own implementation, and the grant + // after it is the outcome, so advice and result are pinned together. + let stop_hint = service_hints::service_stop_hint("lemonade-a-1"); + assert!( + matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1") + && error.to_string().contains(&stop_hint)), + "unexpected result while still running: {still_reserved:?}" + ); + + // Go through the real stop path (`rocm services stop --yes`'s own + // implementation) rather than writing `status = "stopped"` to disk + // directly, so this test actually proves the advice serve-45 pins: that + // running the stop command frees the port. The termination closure is + // the same seam production injects a real signal through; here it just + // confirms every recorded process gone without touching the test + // process itself. + crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { + Vec::new() + })?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + let _ = fs::remove_dir_all(root); + + match decision? { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(existing) => { + panic!( + "a stopped record must not block reuse: {}", + existing.service_id + ) + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_rejects_an_explicit_port_a_recovering_service_holds() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-rejects-explicit-collision"); + paths.ensure()?; + let existing = write_live_service_record( + &paths, + "lemonade-recovering", + "model-recovering", + 11_435, + "recovering", + )?; + + let result = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-other", + "127.0.0.1", + serve_port::PortRequest::Explicit(11_435), + &mut never_probe, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("an explicit port a recovering service holds must be refused") + }; + let message = error.to_string(); + assert!( + message.contains(&existing.service_id), + "the refusal must name the service holding the port: {message}" + ); + assert!( + message.contains("recovering"), + "the refusal must say why the port is unavailable: {message}" + ); + Ok(()) + } + + #[test] + fn concurrent_auto_transactions_publish_distinct_ports() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-distinct"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let spawn_one = |model: &'static str| { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let lease = match resolve_managed_port_in_transaction( + &paths, + "lemonade", + model, + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => lease, + ManagedPortDecision::AlreadyLive(record) => { + panic!("fresh model must not already be live: {record:?}") + } + }; + let port = lease.port(); + write_live_service_record(&paths, model, model, port, "running")?; + lease.release(); + Ok(port) + }) + }; + + let first = spawn_one("model-one"); + let second = spawn_one("model-two"); + let first_port = first.join().expect("thread one panicked")?; + let second_port = second.join().expect("thread two panicked")?; + let _ = fs::remove_dir_all(root); + + assert_ne!(first_port, second_port); + for port in [first_port, second_port] { + assert!( + (serve_port::AUTO_PORT_FIRST..=serve_port::AUTO_PORT_LAST).contains(&port), + "leased port {port} must fall within the auto-select range" + ); + } + Ok(()) + } + + #[test] + fn concurrent_equivalent_transactions_converge_on_one_record() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-converge"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let race_once = || { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + match resolve_managed_port_in_transaction( + &paths, + "lemonade", + "shared-model", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => { + let port = lease.port(); + write_live_service_record( + &paths, + "lemonade-shared", + "shared-model", + port, + "running", + )?; + lease.release(); + Ok(true) + } + ManagedPortDecision::AlreadyLive(_) => Ok(false), + } + }) + }; + + let first = race_once(); + let second = race_once(); + let first_launched = first.join().expect("thread one panicked")?; + let second_launched = second.join().expect("thread two panicked")?; + + let live_count = live_managed_services(&paths).len(); + let _ = fs::remove_dir_all(root); + + assert_ne!( + first_launched, second_launched, + "exactly one racing transaction must launch" + ); + assert_eq!(live_count, 1, "only one record may end up live"); + Ok(()) + } + + #[test] + fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-cleanup"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon", + "abandon-model", + 11_490, + "running", + )?; + fs::create_dir_all( + record + .engine_state_path + .parent() + .expect("engine state parent"), + )?; + fs::write(&record.engine_state_path, "{}")?; + endpoint_keys::store_endpoint_api_key(&paths, &record.service_id, "secret-key")?; + // Simulate a pre-spawn failure: nothing ever ran. + record.engine_pid = None; + + abandon_failed_managed_launch(&paths, &record); + + assert!(!record.manifest_path.exists()); + assert!(!record.log_path.exists()); + assert!(!record.engine_state_path.exists()); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, &record.service_id), + None + ); + assert!(live_managed_services(&paths).is_empty()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// What the cleanup asks of the termination, through the seam and so on every + /// host: the child's PID with the token captured at spawn, and nothing at all + /// for a child with no token where tokens are recorded. The record is marked + /// stopped either way and the audit event carries the verdict the termination + /// gave. + #[test] + fn abandoned_launch_asks_the_termination_for_the_spawned_child_with_its_token() -> Result<()> { + use std::cell::RefCell; + let cases = [ + // (token captured, whether the cleanup is expected to ask) + (Some(77_u64), true), + (None, !host_records_start_tokens()), + ]; + for (token, expect_call) in cases { + let (root, paths) = test_paths("abandon-launch-asks-termination"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-seam", + "abandon-seam-model", + 11_494, + "running", + )?; + record.engine_pid = Some(4242); + record.engine_start_ticks = token; + let asked = RefCell::new(Vec::new()); + + abandon_failed_managed_launch_with(&paths, &record, |identity| { + asked.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Forced + }); + + let asked = asked.into_inner(); + let persisted = load_managed_services(&paths)? + .into_iter() + .find(|candidate| candidate.service_id == record.service_id) + .expect("a launch that spawned a child keeps its record"); + let _ = fs::remove_dir_all(root); + let expected = if expect_call { + vec![rocm_core::ProcessIdentity::new(4242, token)] + } else { + Vec::new() + }; + assert_eq!(asked, expected, "token {token:?}"); + assert_eq!(persisted.status, "stopped", "token {token:?}"); + } + Ok(()) + } + + #[test] + #[cfg(target_os = "linux")] + fn abandoned_launch_never_signals_a_pid_that_no_longer_matches_the_spawn_token() -> Result<()> { + // The child exited and its pid now belongs to an unrelated live + // process: the start token recorded at spawn no longer matches, so the + // forced tree kill must not touch it. + let (root, paths) = test_paths("abandon-launch-recycled-pid"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-recycled", + "abandon-recycled-model", + 11_493, + "running", + )?; + let current_exe = std::env::current_exe().expect("current test exe"); + let mut bystander = ProcessCommand::new(current_exe) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .spawn() + .expect("spawn bystander"); + record.record_engine_identity(bystander.id()); + record.engine_start_ticks = record.engine_start_ticks.map(|ticks| ticks + 1); + + abandon_failed_managed_launch(&paths, &record); + + let mismatched_left_alone = bystander.try_wait().expect("poll bystander").is_none(); + // A child that was gone before its token could be captured leaves none: + // that must not fall back to "best-effort" signalling either. + record.engine_start_ticks = None; + abandon_failed_managed_launch(&paths, &record); + let still_running = + mismatched_left_alone && bystander.try_wait().expect("poll bystander").is_none(); + let _ = bystander.kill(); + let _ = bystander.wait(); + let _ = fs::remove_dir_all(root); + assert!( + still_running, + "a pid with a mismatched token must be left alone" + ); + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-terminates-child"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-child", + "abandon-child-model", + 11_491, + "running", + )?; + + let current_exe = std::env::current_exe().expect("current test exe"); + let mut child = ProcessCommand::new(current_exe) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .spawn() + .expect("spawn sleeper child"); + let child_pid = child.id(); + record.record_engine_identity(child_pid); + + abandon_failed_managed_launch(&paths, &record); + + let wait_result = child.wait(); + + match wait_result { + Ok(status) => assert!( + !status.success(), + "the sleeper child must be terminated, not exit cleanly" + ), + Err(error) => panic!("failed to wait for sleeper child: {error}"), + } + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == record.service_id); + let _ = fs::remove_dir_all(root); + let persisted = persisted.expect( + "a launch that spawned a child must keep its record so `rocm services`/`rocm logs` can show the attempt", + ); + // `stopped`, not `failed`: `rocmd`'s recovery restarts a `failed` record + // that carries no stop marker, which would respawn a launch the CLI just + // reported as failed. The rocmd side of the pair is + // `abandoned_launch_record_is_not_a_recovery_candidate` in + // `apps/rocmd/src/watchers.rs`. + assert_eq!( + persisted.status, "stopped", + "a spawned child's record must be kept, but in a status recovery ignores" + ); + Ok(()) + } + + /// A failed launch's cleanup refuses to signal a child whose token was not + /// captured on any host that records tokens. Windows is one of them, since + /// it records the creation time: a missing token there means the child was + /// gone before it could be captured, and its pid may now be another + /// process's, so the kill must not proceed as it did when Windows had none. + #[test] + fn launch_cleanup_refuses_a_child_without_a_token_only_where_tokens_exist() { + use rocm_core::TerminationOutcome::Unverified; + + // A host that records tokens: a missing one is unverifiable, a captured + // one goes on to be verified by the termination. + assert_eq!(launch_cleanup_refusal(None, true), Some(Unverified)); + assert_eq!(launch_cleanup_refusal(Some(7), true), None); + // A host with no tokens has nothing to verify and proceeds as before. + assert_eq!(launch_cleanup_refusal(None, false), None); + assert_eq!(launch_cleanup_refusal(Some(7), false), None); + } + + /// Ties the cleanup's idea of "this host records tokens" to what + /// `process_start_ticks` really does, on the host the test runs on: Linux and + /// Windows read a token for a live process (this one), anywhere else there + /// is none. If a platform gains or loses tokens without the cleanup being + /// told, this fails there. + #[test] + fn the_cleanup_guard_agrees_with_whether_this_host_reads_tokens() { + assert_eq!( + host_records_start_tokens(), + rocm_core::process_start_ticks(std::process::id()).is_some() + ); + } +} diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs new file mode 100644 index 0000000000..1e14c85599 --- /dev/null +++ b/apps/rocm/src/service_launch/readiness.rs @@ -0,0 +1,634 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Waiting on a launched or already-live managed service: the readiness budget, +//! liveness of the tracked process, how a wait ends, and the end-to-end OOM +//! fault-injection seam that fabricates a launch that never became ready. + +// There is deliberately no `use std::fs`, `use std::thread` or +// `use rocm_core::process_is_running` here: this file names them by full path. +// Their use sites sit under different `cfg` combinations (the Linux-only zombie +// probe, the OOM fault-injection seam, one Linux-only test), so any single import +// would be unused in some build, and a gate on it would have to be kept in step +// with every use site. Keep the full paths. The tests module imports `fs` for +// itself because its tests use it on every platform. +use super::spawn::ManagedLaunchReport; +#[cfg(feature = "e2e-oom-fault-injection")] +use crate::device_policy_name; +use crate::endpoint_keys; +use crate::engine_runtime_select::wait_for_service_http_ready_with_progress; +use crate::status_for_readiness; +#[cfg(feature = "e2e-oom-fault-injection")] +use anyhow::Context; +use anyhow::{Result, anyhow}; +#[cfg(feature = "e2e-oom-fault-injection")] +use rocm_core::format_http_base_url; +use rocm_core::{AppPaths, EndpointReadiness, ManagedServiceRecord}; +#[cfg(feature = "e2e-oom-fault-injection")] +use rocm_engine_protocol::{DevicePolicy, ResolveModelResponse}; +use std::fmt::Write as _; +use std::time::Duration; + +/// How long one readiness wait polls before `rocm serve` reports `starting`. +const READINESS_WAIT: Duration = Duration::from_secs(45); + +/// `rocm serve --wait`: keep waiting past [`READINESS_WAIT`] (a first start +/// downloads the model, which can take far longer) until the server answers or +/// stops being live. A server that exited ends the wait with the last reading +/// instead of blocking forever. +fn wait_past_readiness_budget( + mut probe: impl FnMut() -> EndpointReadiness, + mut still_live: impl FnMut() -> bool, +) -> EndpointReadiness { + loop { + let readiness = probe(); + if readiness == EndpointReadiness::Serving || !still_live() { + return readiness; + } + } +} + +/// Whether `pid` is a running process that is not a zombie. The serve +/// supervisor is an unreaped child of this process, so after it dies +/// `process_is_running` alone keeps reporting it alive. +pub(super) fn service_pid_is_alive(pid: u32) -> bool { + rocm_core::process_is_running(pid) && !process_is_zombie(pid) +} + +#[cfg(target_os = "linux")] +fn process_is_zombie(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + // `comm` may contain spaces and parentheses; the state follows the last `)`. + stat.rfind(')') + .and_then(|end| stat.get(end + 1..)) + .and_then(|rest| rest.split_whitespace().next()) + == Some("Z") +} + +#[cfg(not(target_os = "linux"))] +const fn process_is_zombie(_pid: u32) -> bool { + false +} + +/// Whether the record still describes a server that can finish starting: its +/// status is a live one *and* a tracked process is alive. The status alone is +/// not enough, since a killed engine never writes a terminal state. +fn record_still_live(record: &ManagedServiceRecord, pid_alive: impl Fn(u32) -> bool) -> bool { + if !crate::managed_service_status::managed_service_is_live(record) { + return false; + } + let pids = crate::managed_service_status::recorded_service_pids(record); + pids.is_empty() || pids.into_iter().any(pid_alive) +} + +/// [`record_still_live`] over the manifest on disk, so a `rocm services stop` +/// from another terminal (which writes only the manifest) ends the wait too. +fn service_still_live(paths: &AppPaths, service_id: &str, pid_alive: impl Fn(u32) -> bool) -> bool { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return false; + }; + let _ = latest.refresh_from_engine_state(); + record_still_live(&latest, pid_alive) +} + +/// Record a server that died mid-start as stopped, leaving a manifest that a +/// `services stop` already settled untouched. A `failed` state the engine +/// reported is persisted rather than overwritten, so the reason survives. +fn settle_exited_service(paths: &AppPaths, service_id: &str) { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return; + }; + let on_disk = latest.status.clone(); + let _ = latest.refresh_from_engine_state(); + if crate::managed_service_status::managed_service_is_live(&latest) { + latest.status = "stopped".to_owned(); + } else if latest.status == on_disk { + return; + } + let _ = latest.write(); +} + +/// The failure a `--wait` ends with when the server died before it became +/// ready, after recording the death in the manifest. `None` while the wait did +/// not apply, the endpoint is serving, or the server is still alive. +pub(super) fn died_during_wait( + paths: &AppPaths, + service_id: &str, + wait_until_ready: bool, + readiness: EndpointReadiness, + pid_alive: impl Fn(u32) -> bool, +) -> Option { + if !wait_until_ready + || readiness == EndpointReadiness::Serving + || service_still_live(paths, service_id, pid_alive) + { + return None; + } + settle_exited_service(paths, service_id); + let mut message = format!( + "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" + ); + // A plain `serve` adds the memory guidance for a vLLM launch that ran out of + // VRAM; the wait's failure is the same launch, so it gets the same note. + // "starting" is the status the note keys on for "never became ready", which is + // what a `--wait` death is, whatever the settled record now says. + if let Ok(record) = ManagedServiceRecord::read_manifest(paths, service_id) { + for note in crate::serve_summary::append_oom_serve_note( + Vec::new(), + &record.engine, + false, + "starting", + Some(&record.log_path), + ) { + let _ = write!(message, "\n note: {note}"); + } + } + Some(anyhow!(message)) +} + +/// Record the outcome of a finished wait on the manifest as it is now, changing +/// only what the wait learned. The copy read at spawn time can be long stale: +/// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` +/// refreshes the record, so writing it back whole would revert those. +pub(super) fn record_wait_outcome( + paths: &AppPaths, + held: &mut ManagedServiceRecord, + status: &str, + verified: bool, +) -> Result<()> { + let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) + .unwrap_or_else(|_| held.clone()); + status.clone_into(&mut latest.status); + if verified { + // Latch the verification the wait just performed, so the readiness checks + // behind `services list` and chat read it instead of re-probing. + latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + } + latest.write()?; + *held = latest; + Ok(()) +} + +/// Test-only fault-injection switch, off (`const false`) in shipped binaries: +/// the `e2e-oom-fault-injection` feature is enabled only by the `xtask e2e` +/// mock-lane build, never by the release build. When enabled and +/// `ROCM_E2E_SIMULATE_OOM_LAUNCH=1` is set on the process, `rocm serve` fabricates +/// the on-disk state of a managed vLLM launch that spawned, wrote an +/// out-of-memory traceback to the log it owns, and never became ready — so the +/// positive OOM-guidance summary path can be verified on a GPU-less CI host, +/// where a real launch cannot run. See [`simulate_oom_managed_launch`]. +#[cfg(feature = "e2e-oom-fault-injection")] +pub(crate) fn e2e_simulate_oom_launch() -> bool { + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "1") +} + +/// Release builds carry no fault-injection switch: the seam compiles out +/// entirely, so `rocm serve` behaves identically to a build without the feature. +#[cfg(not(feature = "e2e-oom-fault-injection"))] +pub(crate) const fn e2e_simulate_oom_launch() -> bool { + false +} + +/// Test-only (see [`e2e_simulate_oom_launch`]): fabricate the on-disk state of a +/// managed vLLM launch that spawned, OOM'd, and never became ready, then return +/// the launch report the summary path would see for it — `already_running: false` +/// (this invocation launched it), `status: "starting"` (it failed to become +/// ready), and a `log_path` whose contents carry a real allocator OOM signature. +/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob guidance. No process is +/// spawned. Compiled out of shipped binaries. +#[cfg(feature = "e2e-oom-fault-injection")] +#[allow(clippy::too_many_arguments)] +pub(super) fn simulate_oom_managed_launch( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port: u16, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, +) -> Result { + paths.ensure()?; + std::fs::create_dir_all(paths.services_dir())?; + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + requested_model, + resolve.canonical_model_id.clone(), + host, + port, + "managed", + std::process::id(), + runtime_id.map(str::to_owned), + env_id.map(str::to_owned), + Some(device_policy_name(device_policy).to_owned()), + ); + record.gpu_indices = gpu_indices.to_vec(); + record.status = "starting".to_owned(); + record.write()?; + // A real allocator OOM signature in the log this invocation owns, so the + // summary renders memory guidance for a launch that actually OOM'd. + // + // The engine's own error block is reproduced below it, because that is what + // a real managed launch leaves on disk, and leaving it out hid a bug this + // scenario should have caught: the supervisor and the engine share one log, + // so when vLLM OOMs the engine bails with its startup-log context — which + // embeds rocm-cli's own OOM hint — and `exit_code_for` writes that whole + // error back into the same file, *after* the real line. A fixture carrying + // only the raw OOM line can never see the newest-first scan land on + // rocm-cli's own sentence instead of the user's error. + let real_line = "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB."; + std::fs::write( + &record.log_path, + format!( + "{real_line}\n\ + Error: vLLM server process exited before becoming ready (status: exit status: 1)\n\ + \n\ + Startup log, last {} lines as the terminal drew them:\n\ + {real_line}\n\ + \n\ + {}{real_line}). {}\n\ + For conditional remediation, run `{} 'vllm: {real_line}'`.\n", + rocm_engine_protocol::DEFAULT_LOG_TAIL_LINES, + rocm_core::VLLM_OOM_DETECTED_PREFIX, + rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT, + rocm_core::VLLM_OOM_DIAGNOSE_COMMAND, + ), + ) + .with_context(|| format!("failed to write {}", record.log_path.display()))?; + Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url: format!("{}/v1", format_http_base_url(host, port)), + status: "starting".to_owned(), + already_running: false, + child_pid: None, + log_path: Some(record.log_path), + manifest_path: Some(record.manifest_path), + }) +} + +/// Probe `record`'s endpoint until it answers: once within the readiness +/// budget, or (with `wait_until_ready`) for as long as the server stays alive. +pub(super) fn wait_for_record_ready( + paths: &AppPaths, + record: &ManagedServiceRecord, + wait_until_ready: bool, + endpoint_api_key: Option<&str>, + on_wait_tick: &mut dyn FnMut(Duration), +) -> EndpointReadiness { + let probe = |on_tick: &mut dyn FnMut(Duration)| { + wait_for_service_http_ready_with_progress( + &record.engine, + &record.host, + record.port, + &|| { + let mut latest = record.clone(); + let _ = latest.refresh_from_engine_state(); + latest.served_model_name().to_owned() + }, + endpoint_api_key, + READINESS_WAIT, + on_tick, + ) + }; + if wait_until_ready { + wait_past_readiness_budget( + || probe(&mut *on_wait_tick), + || service_still_live(paths, &record.service_id, service_pid_is_alive), + ) + } else { + probe(on_wait_tick) + } +} + +/// An equivalent service is already live. Without `--wait`, or once it is +/// serving, report it as is; otherwise wait for it like a fresh launch would. +pub(super) fn wait_on_existing_service( + paths: &AppPaths, + mut report: ManagedLaunchReport, + wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str, u16), + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + if !wait_until_ready || report.status == "ready" { + return Ok(report); + } + let Ok(mut record) = ManagedServiceRecord::read_manifest(paths, &report.service_id) else { + return Ok(report); + }; + // The id of the server being waited on, which is not the caller's freshly + // generated one when it attached to an existing server. + on_wait_begin(&report.service_id, record.port); + // The key the caller generated belongs to a launch that did not happen; the + // existing service answers to its own. + let stored_key = endpoint_keys::endpoint_api_key(paths, &report.service_id); + let readiness = + wait_for_record_ready(paths, &record, true, stored_key.as_deref(), on_wait_tick); + if let Some(error) = died_during_wait( + paths, + &report.service_id, + true, + readiness, + service_pid_is_alive, + ) { + return Err(error); + } + report.status = status_for_readiness(readiness).to_owned(); + record_wait_outcome( + paths, + &mut record, + &report.status, + readiness == EndpointReadiness::Serving, + )?; + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::service_launch::test_support::reusable_record; + use crate::tests::test_paths; + use std::fs; + + #[test] + fn wait_past_readiness_budget_keeps_probing_a_live_starting_server() { + let mut readings = vec![ + EndpointReadiness::Unreachable, + EndpointReadiness::Unreachable, + EndpointReadiness::Serving, + ] + .into_iter(); + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + readings.next().expect("no probe after Serving") + }, + || true, + ); + assert_eq!(result, EndpointReadiness::Serving); + assert_eq!(probes, 3, "must outlast the first (45s) budget"); + } + + #[test] + fn wait_past_readiness_budget_stops_when_the_server_exits() { + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + EndpointReadiness::Unreachable + }, + || false, + ); + assert_eq!(result, EndpointReadiness::Unreachable); + assert_eq!(probes, 1); + } + + #[test] + fn wait_past_readiness_budget_returns_at_once_when_already_serving() { + let result = wait_past_readiness_budget( + || EndpointReadiness::Serving, + || panic!("liveness is irrelevant once serving"), + ); + assert_eq!(result, EndpointReadiness::Serving); + } + + fn record_tracking_pid(paths: &AppPaths, pid: u32, status: &str) -> ManagedServiceRecord { + let mut record = reusable_record(paths); + record.supervisor_pid = pid; + record.status = status.to_owned(); + record + } + + #[cfg(unix)] + fn exited_child_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + child.wait().expect("reap"); + pid + } + + #[test] + #[cfg(unix)] + fn record_still_live_ends_the_wait_for_a_dead_pid_with_a_live_status() { + // The engine was killed before writing any terminal state: the status + // still says `starting`, but nothing tracked is running. + let (root, paths) = test_paths("wait-dead-pid"); + paths.ensure().expect("paths"); + let dead = record_tracking_pid(&paths, exited_child_pid(), "starting"); + assert!(!record_still_live(&dead, service_pid_is_alive)); + + let alive = record_tracking_pid(&paths, std::process::id(), "starting"); + assert!(record_still_live(&alive, service_pid_is_alive)); + + // A settled status ends the wait even while the pid is still around. + let stopped = record_tracking_pid(&paths, std::process::id(), "stopped"); + assert!(!record_still_live(&stopped, service_pid_is_alive)); + let _ = fs::remove_dir_all(root); + } + + #[test] + #[cfg(target_os = "linux")] + fn service_pid_is_alive_treats_an_unreaped_child_as_dead() { + // The serve supervisor is our own unreaped child; once it dies it is a + // zombie that `kill(pid, 0)` still reports as running. + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !process_is_zombie(pid) { + assert!(std::time::Instant::now() < deadline, "child never exited"); + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + rocm_core::process_is_running(pid), + "a zombie still answers kill(pid, 0)" + ); + assert!(!service_pid_is_alive(pid)); + child.wait().expect("reap"); + } + + #[test] + fn service_still_live_follows_the_manifest_on_disk() { + // `rocm services stop` from another terminal writes only the manifest. + let (root, paths) = test_paths("wait-manifest"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + assert!(service_still_live(&paths, &record.service_id, |_| true)); + + record.status = "stopped".to_owned(); + record.write().expect("write"); + assert!(!service_still_live(&paths, &record.service_id, |_| true)); + assert!(!service_still_live(&paths, "no-such-service", |_| true)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn settle_exited_service_demotes_a_live_manifest_and_spares_a_settled_one() { + let (root, paths) = test_paths("wait-settle"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.status, "stopped"); + + // A record that already failed keeps its reason: the guard, not the + // overwrite, is what spares it (a `stopped` -> `stopped` rewrite would + // be invisible). + record.status = "failed".to_owned(); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.status, "failed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn wait_on_existing_service_only_waits_for_one_that_is_not_serving_yet() { + // The probing path itself blocks for the readiness budget, so only the + // cases that must return at once are driven here; a manifest planted + // as `starting` would hang a wrong guard for 45 seconds and fail it. + let (root, paths) = test_paths("wait-existing"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + let report = |status: &str| ManagedLaunchReport { + service_id: record.service_id.clone(), + endpoint_url: record.endpoint_url.clone(), + status: status.to_owned(), + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + }; + let mut no_tick = |_: Duration| panic!("must not probe"); + + let untouched = wait_on_existing_service( + &paths, + report("starting"), + false, + &mut |_, _| panic!("no --wait, no banner"), + &mut no_tick, + ) + .expect("no --wait"); + assert_eq!(untouched.status, "starting"); + let ready = wait_on_existing_service( + &paths, + report("ready"), + true, + &mut |_, _| panic!("already ready, no banner"), + &mut no_tick, + ) + .expect("already ready"); + assert_eq!(ready.status, "ready"); + record.status = "starting".to_owned(); + assert_eq!( + ManagedServiceRecord::read_manifest(&paths, &record.service_id) + .unwrap() + .status, + "starting" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn died_during_wait_fails_with_a_logs_pointer_and_records_the_death() { + let (root, paths) = test_paths("wait-died"); + paths.ensure().expect("paths"); + let record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + let id = record.service_id; + let status = || { + ManagedServiceRecord::read_manifest(&paths, &id) + .unwrap() + .status + }; + + // Not waiting, already serving, or still alive: no failure, no change. + let pending = EndpointReadiness::Unreachable; + assert!(died_during_wait(&paths, &id, false, pending, |_| false).is_none()); + assert!( + died_during_wait(&paths, &id, true, EndpointReadiness::Serving, |_| false).is_none() + ); + assert!(died_during_wait(&paths, &id, true, pending, |_| true).is_none()); + assert_eq!(status(), "starting"); + + // Dead server: the error names the logs command and the record is settled. + let error = died_during_wait(&paths, &id, true, pending, |_| false).expect("died"); + let message = error.to_string(); + assert!( + message.contains("exited before it became ready"), + "{message}" + ); + assert!( + message.contains(&format!("rocm logs --service {id}")), + "{message}" + ); + assert_eq!(status(), "stopped"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn died_during_wait_adds_the_oom_guidance_for_a_vllm_launch_only() { + let (root, paths) = test_paths("wait-died-oom"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.engine = "vllm".to_owned(); + record.write().expect("write"); + fs::write( + &record.log_path, + "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.\n", + ) + .expect("log"); + let id = record.service_id; + let pending = EndpointReadiness::Unreachable; + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(message.contains("--gpu-memory-utilization"), "{message}"); + + // Same log, another engine: that engine has no such flag to suggest. + let mut record = ManagedServiceRecord::read_manifest(&paths, &id).unwrap(); + record.engine = "lemonade".to_owned(); + "starting".clone_into(&mut record.status); + record.write().expect("write"); + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(!message.contains("--gpu-memory-utilization"), "{message}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_wait_outcome_keeps_what_changed_on_disk_during_the_wait() { + let (root, paths) = test_paths("wait-outcome"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // While the wait ran, `rocmd` recorded a restart and a new supervisor. + let mut elsewhere = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + elsewhere.restart_count = 3; + elsewhere.supervisor_pid = 222; + elsewhere.write().expect("write"); + + record_wait_outcome(&paths, &mut held, "ready", true).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "ready"); + assert!(after.inference_verified_at_unix_ms.is_some()); + assert_eq!(after.restart_count, 3, "the restart count was reverted"); + assert_eq!(after.supervisor_pid, 222, "the supervisor pid was reverted"); + + record_wait_outcome(&paths, &mut held, "starting", false).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "starting"); + let _ = fs::remove_dir_all(root); + } +} diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs new file mode 100644 index 0000000000..9b3252efc0 --- /dev/null +++ b/apps/rocm/src/service_launch/spawn.rs @@ -0,0 +1,1304 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Spawning the detached managed engine child shared by the background and +//! attached serve paths: process and stdio plumbing, [`ManagedLaunchReport`] and +//! [`ManagedSpawn`], and the serialized spawn transaction itself. + +use super::endpoint_auth::ensure_public_service_has_endpoint_key; +use super::launch_transaction::{ + ManagedPortDecision, ManagedReuseRefusal, abandon_failed_managed_launch, + managed_service_reuse_refusal, no_usable_gpu_refusal, resolve_managed_port_in_transaction, +}; +use crate::endpoint_keys; +use crate::engine_ipc::builtin_engine_serve_http_args; +use crate::engine_runtime_select::managed_service_launcher_path; +#[cfg(windows)] +use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; +use crate::engines_cmd::env_root_for_service; +use crate::serve_port; +use crate::service_hints; +use crate::services_text::read_optional_tail_lines; +use crate::{device_policy_name, record_cli_audit_event}; +use anyhow::{Context, Result, bail}; +#[cfg(windows)] +use rocm_core::process_is_running; +use rocm_core::{AppPaths, ManagedServiceRecord}; +use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; +use std::fs; +#[cfg(not(windows))] +use std::io; +use std::path::{Path, PathBuf}; +#[cfg(not(windows))] +use std::process::{Command as ProcessCommand, Stdio}; +use std::thread; +use std::time::Duration; + +#[cfg(not(windows))] +pub(crate) fn detach_background_command(command: &mut ProcessCommand) { + rocm_core::detach_command_session(command); +} + +#[cfg(not(windows))] +pub(crate) fn attach_background_stdio( + command: &mut ProcessCommand, + log_path: Option<&Path>, +) -> Result<()> { + if let Some(log_path) = log_path { + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display()))?; + command + .stdout(Stdio::from(log.try_clone()?)) + .stderr(Stdio::from(log)); + } else { + command.stdout(Stdio::null()).stderr(Stdio::null()); + } + Ok(()) +} + +/// Windows spawn-liveness check: wait out the startup window, then fail with the +/// startup log when the child is already gone. The wait and the liveness probe +/// are injected so a unit test runs the exit-detected path on every platform, +/// including the Windows lane that actually calls it. +/// +/// # Errors +/// Fails with the startup-failure detail when `is_running` reports the child dead. +#[cfg_attr(not(windows), allow(dead_code))] +fn ensure_spawned_child_survives_startup( + child_pid: u32, + log_path: &Path, + settle: impl FnOnce(), + is_running: impl FnOnce(u32) -> bool, +) -> Result<()> { + settle(); + if is_running(child_pid) { + Ok(()) + } else { + bail!("{}", managed_engine_startup_failure_detail("", log_path)) + } +} + +/// `status_clause` is a pre-formatted fragment inserted right after "exited +/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real +/// `ExitStatus` is available; empty on Windows, where the liveness check +/// below only has a yes/no answer, not an exit code). +pub(crate) fn managed_engine_startup_failure_detail( + status_clause: &str, + log_path: &Path, +) -> String { + let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); + if recent_lines.is_empty() { + for _ in 0..5 { + thread::sleep(Duration::from_millis(120)); + recent_lines = read_optional_tail_lines(log_path, 80, "service log"); + if !recent_lines.is_empty() { + break; + } + } + } + if recent_lines.is_empty() { + return format!( + "managed engine exited immediately{status_clause}; inspect {}", + log_path.display() + ); + } + format!( + "managed engine exited immediately{status_clause}; inspect {}\n\nrecent startup log output:\n{}", + log_path.display(), + recent_lines.join("\n") + ) +} +#[cfg(not(windows))] +pub(crate) fn managed_service_process_command(program: &Path, args: &[String]) -> ProcessCommand { + let mut command = ProcessCommand::new(program); + command.args(args); + command +} + +/// Engine-neutral result of a managed launch. Returned rather than printed so the +/// caller can render it either as the rich deployment summary (interactive TTY) or +/// as the plain line-by-line form (piped output, chat assistant), from one code path. +pub(crate) struct ManagedLaunchReport { + pub(crate) service_id: String, + /// `http://host:port/v1`. + pub(crate) endpoint_url: String, + /// `"ready"` (inference confirmed), `"running"` (model listed but not serving + /// yet), `"starting"` (endpoint not answering), or the existing service's + /// status when nothing was spawned. + pub(crate) status: String, + /// True when an equivalent service was already live and nothing was spawned. + pub(crate) already_running: bool, + pub(crate) child_pid: Option, + pub(crate) log_path: Option, + pub(crate) manifest_path: Option, +} + +/// Either an already-live service (nothing spawned) or a freshly spawned engine +/// child that is `running` but not yet HTTP-ready. +/// +/// Split out of [`start_managed_service`](super::start_managed_service) so the +/// attached (`--verbose` / `--foreground`) serve path can spawn the very same +/// detached child and stream its log live from the first line — including +/// startup — instead of blocking on the readiness wait before any output appears. +pub(super) enum ManagedSpawn { + AlreadyRunning(ManagedLaunchReport), + // `ManagedServiceRecord` is large; box it so the two variants stay a similar + // size (clippy::large_enum_variant). + Spawned { + record: Box, + child_pid: u32, + }, +} + +/// Spawn the detached engine child shared by the managed (background) and +/// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP +/// readiness wait; callers decide whether to block on readiness +/// ([`start_managed_service`](super::start_managed_service)) or start tailing the +/// log immediately ([`run_attached_service`](super::run_attached_service)). +/// +/// The whole body is one serialized transaction under the shared allocation +/// lock: duplicate detection, live-reservation refresh, candidate leasing, +/// manifest/log creation, argv preparation, the bounded immediate-exit check, +/// and record publication. Readiness deliberately runs after the lock is +/// released. +#[allow(clippy::too_many_arguments)] +pub(super) fn spawn_managed_engine_child( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port_request: serve_port::PortRequest, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + require_api_key: bool, + gpu_refusal_waived_for_reuse: bool, +) -> Result { + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // No lock is acquired here: every production caller + // ([`start_managed_service`], [`run_attached_service`]) already holds + // [`rocm_core::AppPaths::managed_launch_lock_path`] for the whole of this + // call, so no two transactions can choose the same port or publish + // competing records for one engine+model. A second acquisition in this + // frame would self-deadlock, since the lock is exclusive per open file + // description, not per process. + // + // Idempotency guard: if a managed service for this engine+model is already + // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the + // chat assistant re-issuing the same request) is treated as satisfied, not + // an error. Keyed on engine+canonical model — the freshly generated + // `service_id` is timestamp-unique and would never match an existing one. + // Stale/dead services fall through and relaunch normally. + let requested_recipe_json = engine_recipe + .map(serde_json::to_string) + .transpose() + .context("failed to encode engine recipe hint")?; + let lease = match resolve_managed_port_in_transaction( + paths, + engine, + &resolve.canonical_model_id, + host, + port_request, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::AlreadyLive(existing) => { + let existing = *existing; + // Asked through the shared predicate rather than re-tested here, so this + // refusal and `serve()`'s reuse pre-gate cannot disagree about whether a + // service is reusable. When they disagreed, the pre-gate let a + // non-reusable service bypass the no-GPU refusal and the user was sent + // here to be told to stop a working server for nothing. + // + // The same device policy the serve path passes. This side used to pass + // `None` to keep the spawn's acceptance untouched, but that opted the + // spawn out of a term it has the input for, left its own + // `DevicePolicyTooWeak` bail unreachable, and made the serve path's + // argument droppable. Nothing observable widens: `parse_device_policy` + // is the only producer of a `DevicePolicy` on every path that reaches + // here and it rejects `cpu_only`, so no record this CLI writes can + // carry the policy this term refuses. + match managed_service_reuse_refusal( + &existing, + requested_recipe_json.as_deref(), + require_api_key, + device_policy, + ) { + Some(ManagedReuseRefusal::RecipeMismatch) => bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", + existing.service_id, + resolve.canonical_model_id + ), + Some(ManagedReuseRefusal::MissingApiKey) => bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` \ + without authentication, and a running server cannot be given a key it did not \ + start with; stop it with `{}` and run the command again to \ + serve it with `--require-api-key`", + existing.service_id, + resolve.canonical_model_id, + service_hints::service_stop_hint(&existing.service_id) + ), + // Reachable only from a record whose `device_policy` says + // `cpu_only`, which no `rocm serve` can write today + // (`parse_device_policy` refuses that policy outright) — so this is + // a guard against a record written by some future path, not a + // refusal users see now. The advice still names a command that + // clears the condition: stopping the service removes the only thing + // this term can object to, and the rerun then launches normally + // (AGENTS.md section 3). + Some(ManagedReuseRefusal::DevicePolicyTooWeak) => bail!( + "managed service `{}` is already running under a device policy that cannot \ + satisfy this request; stop it with `{}` and run the command again", + existing.service_id, + service_hints::service_stop_hint(&existing.service_id) + ), + None => {} + } + // An explicit port is a demand, not a hint: reporting success on a + // different port than the one asked for would silently drop it. + if let Some(port) = port_request.explicit() + && port != existing.port + { + bail!( + "managed service `{}` is already serving engine `{engine}` and model `{}` on port {}, so port {port} would not be used; stop it with `{}` and run the command again to serve on that port", + existing.service_id, + resolve.canonical_model_id, + existing.port, + service_hints::service_stop_hint(&existing.service_id) + ); + } + record_cli_audit_event( + paths, + "service", + "managed_service_launch_skipped", + "info", + format!( + "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", + resolve.canonical_model_id, existing.service_id, existing.status + ), + Some(&existing.service_id), + ); + return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { + service_id: existing.service_id, + endpoint_url: existing.endpoint_url, + status: existing.status, + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + })); + } + ManagedPortDecision::Lease(lease) => lease, + }; + + // Nothing live to reuse — and if the only reason this invocation got past + // `serve()`'s no-usable-GPU pre-flight was a reusable service, that premise + // has just been falsified. `serve()` decided it from a snapshot taken before + // GPU selection and the launch lock; the check above is the authoritative + // one, so a service that died in between (an OOM, say) would otherwise fall + // through to a real spawn on a host with no usable GPU, and the user would + // get whatever the engine says about having no device instead of the + // actionable refusal the pre-flight exists to give. Raise it here, where the + // premise is known to be false. + if gpu_refusal_waived_for_reuse { + bail!("{}", no_usable_gpu_refusal(device_policy)); + } + + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + requested_model, + resolve.canonical_model_id.clone(), + host, + lease.port(), + "managed", + 0, + runtime_id.map(str::to_owned), + env_id.map(str::to_owned), + Some(device_policy_name(device_policy).to_owned()), + ); + record.gpu_indices = gpu_indices.to_vec(); + record.served_model_id = resolve.served_model_id.clone(); + record.engine_recipe_json = requested_recipe_json; + // The flag the user actually passed, carried through rather than re-derived. + // + // Deriving it from key-file presence looked equivalent and was not: + // `resolve_endpoint_auth` mints a key for *every* non-loopback bind whether or + // not auth was demanded, so a plain `--host 0.0.0.0 --allow-public-bind` + // recorded `true` here. The guard below tests this field before the bind + // address, so that service was then refused with a message naming a flag it + // never used and a relaunch command that drops `--allow-public-bind` — the + // public-bind branch, which carries the right command, became unreachable. + // + // This field means "the user demanded auth on a bind that would not otherwise + // require it". A public bind needs no such record; its address still says so. + record.requires_api_key = require_api_key; + // Publishing the record inside the lock is what reserves the leased port for + // every other transaction. From here on, any failure must remove it again. + record.write()?; + + match launch_managed_engine_child_locked( + paths, + &mut record, + lease, + engine, + service_id, + resolve, + host, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + #[cfg(not(windows))] + None, + ) { + Ok(child_pid) => Ok(ManagedSpawn::Spawned { + record: Box::new(record), + child_pid, + }), + Err(error) => { + // The single cleanup choke point for every failure path below the + // record's first write: `record.engine_pid` tells it whether a child + // ever ran, so a pre-spawn failure deletes everything and a post-spawn + // one keeps the record and log (marked `status = "stopped"`) instead. + abandon_failed_managed_launch(paths, &record); + Err(error) + } + } +} + +// Test-only, thread-scoped spawn interceptor for callers that cannot take a +// `spawn_override` argument (`start_managed_service`, `run_attached_service`): +// lets a test observe state at the instant production would spawn the child. +#[cfg(all(test, not(windows)))] +thread_local! { + pub(super) static TEST_SPAWN_HOOK: std::cell::RefCell>>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(all(test, not(windows)))] +fn test_spawn_hook() -> Option>> { + TEST_SPAWN_HOOK.with(|hook| hook.borrow().clone()) +} + +#[cfg(not(windows))] +type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; + +/// The side-effecting half of the launch transaction: create the log, prepare +/// concrete argv, hand the leased port to the child, spawn it, reject a child +/// that has already exited, and publish the running record. +/// +/// Split out of [`spawn_managed_engine_child`] so that *every* failure below the +/// record's first write is funnelled through one cleanup path before the +/// launch lock is released. Returns the child pid. +#[allow(clippy::too_many_arguments)] +fn launch_managed_engine_child_locked( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + lease: rocm_core::LoopbackPortLease, + engine: &str, + service_id: &str, + resolve: &ResolveModelResponse, + host: &str, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + // Test-only seam: lets a test assert the leased port is actually bindable + // (i.e. released) at the exact point production would spawn the child, + // without needing to fake or intercept the real spawn mechanics. + #[cfg(not(windows))] spawn_override: Option<&SpawnOverride<'_>>, +) -> Result { + let port = record.port; + if let Some(parent) = record.engine_state_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + fs::File::create(&record.log_path) + .with_context(|| format!("failed to create {}", record.log_path.display()))?; + let current_exe = managed_service_launcher_path() + .context("failed to resolve current rocm executable path")?; + let serve_args = builtin_engine_serve_http_args( + engine, + service_id, + &resolve.canonical_model_id, + host, + port, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + &record.engine_state_path, + Some(&record.log_path), + )?; + let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; + // Hand the child the *path* to the endpoint key file (public bind only) via the + // environment. A path — not the secret value — is what the detached-spawn + // primitives accept as an env override, and it keeps the key off both the argv + // and the environment block. `serve()` wrote the file before spawning. + // Validity, not mere existence: the engine adapters resolve the key with + // `endpoint_api_key_from_file` and enforce nothing when it yields `None`, so + // an empty or malformed key file would otherwise satisfy the guard below and + // still produce an unauthenticated public listener. + let endpoint_key_file = endpoint_keys::endpoint_key_file_if_present(paths, service_id) + .filter(|path| rocm_engine_protocol::endpoint_api_key_from_file(path).is_some()); + // `serve()` already resolved and stored the key for a public bind, so the + // public-bind branch cannot fire on the fresh-launch path today. It is the + // shared choke point for managed spawns, so enforce the invariant here too + // rather than relying on every future caller having done so. + // + // `record.requires_api_key` is passed, not a literal, and the two arguments + // are deliberately different things: that field is the `--require-api-key` + // flag the caller passed, `endpoint_key_file` is whether a *usable* key is on + // disk. A present but empty or malformed key file is where they disagree, and + // is exactly what the `requires_api_key` branch exists to refuse. + // + // The field is threaded, never derived from the key file. Deriving it marked + // every public bind as having demanded auth — see the assignment above. + ensure_public_service_has_endpoint_key( + host, + endpoint_key_file.is_some(), + record.requires_api_key, + )?; + // Hand the port over: close the probe socket immediately before the spawn, + // still under the caller's launch lock, so no other ROCm transaction can + // take it in the meantime. On a custom host the lease never held a socket + // and this is a no-op — the engine owns that bind. + debug_assert_eq!(lease.port(), port); + lease.release(); + #[cfg(windows)] + let child_pid = { + let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); + let mut env_refs = app_path_env_var_refs(&env_values); + if let Some(key_file) = endpoint_key_file.as_deref() { + env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); + } + let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) + .context("failed to launch managed engine process")?; + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. The + // start-time token is captured here too, while the child is certainly + // ours: cleanup binds its kill to it rather than to whatever owns the + // pid by then. + record.record_engine_identity(child_pid); + ensure_spawned_child_survives_startup( + child_pid, + &record.log_path, + || thread::sleep(Duration::from_millis(200)), + process_is_running, + )?; + child_pid + }; + #[cfg(not(windows))] + let child_pid = { + let mut command = managed_service_process_command(¤t_exe, &serve_args); + command.stdin(Stdio::null()); + attach_background_stdio(&mut command, Some(&record.log_path))?; + detach_background_command(&mut command); + paths.apply_env(&mut command); + if let Some(engine_envs_root) = engine_envs_root.as_deref() { + command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); + } + if let Some(key_file) = endpoint_key_file.as_deref() { + command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); + } + #[cfg(all(test, not(windows)))] + let hook = test_spawn_hook(); + #[cfg(all(test, not(windows)))] + let spawn_override = spawn_override.or(hook.as_deref()); + let mut child = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + } + .context("failed to launch managed engine process")?; + let child_pid = child.id(); + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. The + // start-time token is captured here too, while the child is certainly + // ours: cleanup binds its kill to it rather than to whatever owns the + // pid by then. + record.record_engine_identity(child_pid); + thread::sleep(Duration::from_millis(200)); + if let Some(status) = child + .try_wait() + .context("failed to check managed engine startup state")? + { + bail!( + "{}", + managed_engine_startup_failure_detail( + &format!(" with status {status}"), + &record.log_path + ) + ); + } + child_pid + }; + // The child is both roles until the engine state file names a separate + // server process. Each role gets its own token, captured while the child is + // alive: a stop pairs every recorded PID with its own token, and the two + // PIDs diverge as soon as `rocmd` restarts the service — at which point an + // engine PID recorded without one could only be signalled blind. + // The engine's token was already captured right after the spawn (so cleanup + // can bind to it); re-capturing here could replace it with a later owner's. + record.record_supervisor_identity(child_pid); + record.status = "running".to_owned(); + if let Err(error) = record.write() { + // Cleanup is the caller's job now: `record.engine_pid` (set above) is + // already enough for `abandon_failed_managed_launch` to terminate this + // child, so there is nothing left for this inner path to do beyond + // surfacing the error. + return Err(error.context("failed to publish the managed service record")); + } + + Ok(child_pid) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::managed_service_status::load_managed_services; + use crate::service_launch::launch_transaction::live_managed_services; + #[cfg(not(windows))] + use crate::service_launch::test_support::sleeper_test_name; + use crate::service_launch::test_support::write_live_service_record; + use crate::tests::test_paths; + use rocm_core::DEFAULT_LOCAL_HOST; + + #[test] + fn a_child_that_exited_during_the_startup_window_is_reported_with_its_log() -> Result<()> { + let dir = tempfile::tempdir()?; + let log = dir.path().join("engine.log"); + fs::write(&log, "boom: engine could not start\n")?; + let settled = std::cell::Cell::new(false); + let error = ensure_spawned_child_survives_startup( + 4242, + &log, + || settled.set(true), + |pid| { + assert_eq!(pid, 4242); + assert!(settled.get(), "liveness must be probed after the wait"); + false + }, + ) + .expect_err("a dead child must fail the launch"); + assert!(settled.get(), "the startup window must elapse first"); + assert!(format!("{error:#}").contains("boom"), "{error:#}"); + ensure_spawned_child_survives_startup(4242, &log, || {}, |_| true)?; + Ok(()) + } + + #[test] + fn spawn_managed_engine_child_refuses_an_explicit_port_a_live_equivalent_does_not_serve() + -> Result<()> { + let (root, paths) = test_paths("dup-managed-explicit-port-differs"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let launch = |port_request| { + spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + port_request, + &resolve.device_policy, + &[], + None, + None, + None, + false, + false, + ) + }; + + let different = launch(serve_port::PortRequest::Explicit(11511)); + // The same port, and no port at all, still reuse the live service. + let same = launch(serve_port::PortRequest::Explicit(11510)); + let auto = launch(serve_port::PortRequest::Auto); + let _ = fs::remove_dir_all(root); + + let Err(error) = different else { + panic!("a port the live service does not serve must be refused") + }; + let message = error.to_string(); + assert!( + message.contains("on port 11510") && message.contains("port 11511 would not be used"), + "{message}" + ); + assert!(matches!(same?, ManagedSpawn::AlreadyRunning(_))); + assert!(matches!(auto?, ManagedSpawn::AlreadyRunning(_))); + Ok(()) + } + + #[test] + fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { + // A live service recorded with one recipe (e.g. a tool-call parser flag) + // must reject a relaunch requesting a different recipe rather than + // silently reusing the old server, and the error must not claim the + // mismatch is specifically about generation defaults when it could stem + // from any recipe field. + let (root, paths) = test_paths("dup-managed-recipe-mismatch"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { + contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], + ..EngineRecipeHint::default() + })?); + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let requested_recipe = EngineRecipeHint { + contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], + ..EngineRecipeHint::default() + }; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11511), + &resolve.device_policy, + &[], + None, + None, + Some(&requested_recipe), + false, + false, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("mismatched recipe on a live service must be rejected") + }; + let message = error.to_string(); + assert!( + message.contains("different serve options"), + "message should describe the mismatch generically: {message}" + ); + assert!( + message.contains("recipe hint, tool-call parser, or generation defaults"), + "message should not single out generation defaults as the sole cause: {message}" + ); + Ok(()) + } + + #[test] + fn spawn_managed_engine_child_refuses_to_reuse_an_unauthenticated_service() -> Result<()> { + // `--require-api-key` used to be accepted and dropped on this path. The + // reuse branch returns before the flag is recorded and before the + // endpoint key is checked, so a caller demanding auth got a server that + // never had it, with no error. `rocm remote serve` then published that + // endpoint onto the tailnet and printed a freshly minted key under "the + // API key above is what stops anyone else calling it" — a false + // assurance about an endpoint the whole tailnet can reach. + // + // Asserted through `spawn_managed_engine_child` rather than against the + // guard's own arguments: the defect was the early return, so only the + // real call site can fail for it. + let (root, paths) = test_paths("dup-managed-unauthenticated-reuse"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11520, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + // The state that matters: live, matching, and serving without auth. + existing.requires_api_key = false; + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-3001", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11520), + &resolve.device_policy, + &[], + None, + None, + None, + // The demand that used to be silently discarded. + true, + false, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!( + "reusing an unauthenticated service must not satisfy `--require-api-key`; a \ + satisfied reuse leaves the endpoint open while the caller is told it is not" + ) + }; + let message = error.to_string(); + assert!( + message.contains("without authentication"), + "the refusal must say why it refused: {message}" + ); + // The printed hint must be the exact, directly-runnable command: + // `main.rs`'s `service_actions_require_yes_and_render_sandbox_result` + // proves this exact form (`rocm services stop --yes`) is the one + // the approval gate itself recommends. That running this form actually + // clears *this* refusal — a service reused without the key it is now + // being asked to require — is proven by + // `@id:serve-require-api-key-refuses-unauthenticated-reuse` + // (`tests/e2e-cucumber/features/model_serving.feature`, serve-24), + // which reproduces this exact condition, runs the printed hint, and + // asserts the record is left `stopped` — not by `stop_managed_services` + // in `tests/e2e-cucumber/tests/e2e.rs`, which runs this same + // invocation as best-effort teardown and never asserts success. + assert!( + message.contains("rocm services stop lemonade-qwen-3000 --yes"), + "the refusal must name the way out, with the service to stop, directly runnable with --yes: {message}" + ); + Ok(()) + } + + #[test] + fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { + // Drives the real call site, not the guard's own arguments. The service + // was launched with `--require-api-key`, and its key file is present but + // invalid — so `requires_api_key` is true while `key_present` is false, + // which is the only way to reach the first branch. A test calling + // `ensure_public_service_has_endpoint_key` directly cannot catch a + // mis-wired call site, which is the defect that has occurred here twice. + // + // The flag is passed explicitly rather than inferred from the key file. + // Inferring it marked every public bind as having demanded auth, because + // a public bind always has a key file whether or not it asked for one. + let (root, paths) = test_paths("managed-spawn-invalid-key"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Non-empty but invalid (embedded CR/LF), so the file exists and is not + // merely empty (requires_api_key = true) but yields no usable key + // (key_present = false) — `endpoint_api_key_from_file` rejects embedded + // control characters. An empty seed here would let the post-condition + // below pass even if the real cleanup call never ran, since + // `endpoint_api_key` already reads back `None` for an empty file whether + // or not it was ever deleted. + endpoint_keys::store_endpoint_api_key( + &paths, + "lemonade-qwen-3000", + "bad-key\r\nX-Injected: value", + )?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-3000", + "qwen", + &resolve, + // Loopback on purpose: the public-bind branch must not be what + // refuses this, or the test would pass with the guard disabled. + "127.0.0.1", + serve_port::PortRequest::Explicit(11512), + &resolve.device_policy, + &[], + None, + None, + None, + true, + false, + ); + + // This guard fires before any child is spawned, so `record.engine_pid` + // is still `None` when `spawn_managed_engine_child`'s cleanup arm calls + // `abandon_failed_managed_launch` — proving its pre-spawn branch + // (delete everything) runs from the real call site, not just from the + // `abandoned_launch_leaves_no_record_log_or_endpoint_key` test that + // calls the helper directly. + let manifest_path = paths.service_manifest_path("lemonade-qwen-3000"); + let log_path = paths.service_log_path("lemonade-qwen-3000"); + let engine_state_path = paths.service_engine_state_path("lemonade", "lemonade-qwen-3000"); + let key_file_path = endpoint_keys::endpoint_key_file_path(&paths, "lemonade-qwen-3000"); + let live = live_managed_services(&paths); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("a service requiring a key must not spawn with an unusable key file") + }; + let message = error.to_string(); + assert!( + message.contains("--require-api-key"), + "the refusal must name the flag the service was launched with: {message}" + ); + assert!( + message.contains("without authentication"), + "the refusal must say what the risk is: {message}" + ); + assert!( + !manifest_path.exists(), + "a pre-spawn failure must not leave a record" + ); + assert!( + !log_path.exists(), + "a pre-spawn failure must not leave a log" + ); + assert!( + !engine_state_path.exists(), + "a pre-spawn failure must not leave engine state" + ); + assert!( + !key_file_path.exists(), + "a pre-spawn failure must delete the unusable key file from disk, not merely \ + leave it unreadable as a key" + ); + assert!( + live.is_empty(), + "a pre-spawn failure must leave no reservation" + ); + Ok(()) + } + + #[test] + fn a_failed_spawn_past_the_first_write_terminates_the_child_and_keeps_an_abandoned_record() + -> Result<()> { + // Drives the real call site's *post*-spawn cleanup branch, which + // neither `abandoned_launch_*` test reaches (they call + // `abandon_failed_managed_launch` directly). An engine name this + // binary does not build in makes the real spawned child (a second + // `rocm` process) exit almost immediately via + // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after + // `launch_managed_engine_child_locked` has already set + // `record.engine_pid` — so cleanup must terminate (already-dead) the + // child, keep the record and log for inspection, mark the record + // `stopped` (a `failed` one would be respawned by rocmd recovery), and + // clear only the endpoint key. + // + // The child always dies near-instantly, but `spawn_managed_engine_child` + // only reports that within a fixed startup-check window. On a loaded or + // antivirus-scanning Windows CI box, a fresh debug `rocm.exe` can + // occasionally be slow enough to start that the window is missed even + // though the child is already doomed — so poll by retrying a fresh + // attempt (its own service id/port, so no stale manifest collides) up + // to a deadline instead of asserting on one race-prone attempt. + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut attempt: u16 = 0; + let (root, paths, service_id, result) = loop { + attempt += 1; + let (root, paths) = + test_paths(&format!("managed-spawn-unknown-engine-cleanup-{attempt}")); + paths.ensure()?; + let service_id = format!("no-such-engine-qwen-{}", 4000 + attempt); + endpoint_keys::store_endpoint_api_key(&paths, &service_id, "test-key")?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "no-such-engine", + &service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11_514 + attempt), + &resolve.device_policy, + &[], + None, + None, + None, + false, + false, + ); + + if result.is_ok() && std::time::Instant::now() < deadline { + let _ = fs::remove_dir_all(&root); + continue; + } + break (root, paths, service_id, result); + }; + + let manifest_path = paths.service_manifest_path(&service_id); + let log_path = paths.service_log_path(&service_id); + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == service_id); + let key = endpoint_keys::endpoint_api_key(&paths, &service_id); + let live = live_managed_services(&paths); + + let Err(error) = result else { + panic!("an engine this binary does not build in must not report success") + }; + let message = error.to_string(); + assert!( + message.contains("managed engine exited immediately"), + "the failure must come from the spawned child dying, not an earlier guard: {message}" + ); + assert!( + manifest_path.exists(), + "a post-spawn failure must keep the record for inspection" + ); + assert!( + log_path.exists(), + "a post-spawn failure must keep the log for inspection" + ); + let persisted = + persisted.expect("a post-spawn failure must leave the record on disk, not delete it"); + // `stopped`, not `failed`: rocmd's recovery restarts a `failed` record + // without a stop marker, which would respawn a launch the CLI just + // reported as failed (pinned on the rocmd side by + // `abandoned_launch_record_is_not_a_recovery_candidate`). + assert_eq!( + persisted.status, "stopped", + "a post-spawn failure must mark the record abandoned, not leave it stuck" + ); + assert_eq!( + key, None, + "a post-spawn failure must still clear the endpoint key" + ); + assert!( + live.is_empty(), + "an abandoned record must not still reserve its port" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn select_gpu_indices_under_launch_lock_blocks_until_the_holder_releases() -> Result<()> { + // Exercises the real lock-acquisition function production callers use + // (`select_gpu_indices_under_launch_lock`) against a lock another thread + // actually holds, so a refactor that dropped the lock early would fail + // here. `spawn_managed_engine_child` is then run for real, but only via + // its `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no + // child process is ever launched. The holder writes nothing: this does + // not show a write made under the lock being observed. + use std::sync::{Arc, Barrier}; + use std::time::Instant; + + let (root, paths) = test_paths("launch-lock-holds-across-spawn"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-held-1", "held-canonical", 11_520, "ready")?; + + let barrier = Arc::new(Barrier::new(2)); + let hold_for = Duration::from_millis(250); + + let holder_paths = paths.clone(); + let holder_barrier = Arc::clone(&barrier); + let holder = thread::spawn(move || -> Result<()> { + let _lock = rocm_core::FileLock::acquire(holder_paths.managed_launch_lock_path())?; + holder_barrier.wait(); + thread::sleep(hold_for); + Ok(()) + }); + + barrier.wait(); + let started = Instant::now(); + let (gpu_indices, launch_lock) = crate::gpu_policy::select_gpu_indices_under_launch_lock( + &paths, + true, + None, + || None, + None, + None, + )?; + let waited = started.elapsed(); + + let resolve = ResolveModelResponse { + canonical_model_id: "held-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let spawn = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-held-2", + "held-model", + &resolve, + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Auto, + &resolve.device_policy, + &gpu_indices, + None, + None, + None, + false, + false, + ); + drop(launch_lock); + holder.join().expect("holder thread")?; + let _ = fs::remove_dir_all(root); + + // `thread::sleep` only guarantees *at least* `hold_for`, but under a + // loaded host (e.g. a full parallel `cargo test` run on WSL2) timer + // coalescing can make the measured wait a hair short of the nominal + // hold. Allow a small margin so the test still fails loudly if the + // call returns early, without being flaky on actual block duration. + let margin = Duration::from_millis(20); + assert!( + waited + margin >= hold_for, + "select_gpu_indices_under_launch_lock returned before the holder released the lock: waited {waited:?}, held {hold_for:?}" + ); + match spawn? { + ManagedSpawn::AlreadyRunning(report) => { + assert_eq!(report.service_id, "lemonade-held-1"); + } + ManagedSpawn::Spawned { .. } => { + panic!( + "expected the pre-seeded AlreadyLive record to short-circuit, not a real spawn" + ) + } + } + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn launch_releases_the_leased_port_before_spawning_the_child() -> Result<()> { + // `lease.release()` must run before the child is spawned, not after, + // or the engine is handed a port the CLI is still holding open. + // Proven via the injected `spawn_override` seam: it asserts a fresh + // bind on the leased port succeeds at the exact instant production + // would spawn, which only holds if the lease was already released. + let (root, paths) = test_paths("launch-releases-port-before-spawn"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Ephemeral, not a fixed number in the automatic range, which a real + // `rocm serve` on the developer's machine could be holding. + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let lease = rocm_core::lease_loopback_port(port)?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-release-before-spawn", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let bind_succeeded = std::cell::Cell::new(false); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + // Retry briefly: a sibling test's concurrent fork can hold a copy + // of the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + } + // A long-lived no-op child stands in for the engine: `_command` + // carries real serve args this test binary cannot execute as + // itself (see `managed_launch_cleanup_sleeper_child`). + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg(sleeper_test_name()) + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + }; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-release-before-spawn", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + if let Ok(child_pid) = result.as_ref() { + let _ = rocm_core::terminate_process(*child_pid); + } + let _ = fs::remove_dir_all(root); + + assert!( + bind_succeeded.get(), + "the port must already be bindable (released) by the time the child is spawned" + ); + result?; + Ok(()) + } +} diff --git a/apps/rocm/src/service_launch/start.rs b/apps/rocm/src/service_launch/start.rs new file mode 100644 index 0000000000..06cb93697e --- /dev/null +++ b/apps/rocm/src/service_launch/start.rs @@ -0,0 +1,487 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The background launch path: [`start_managed_service`] and the plain, +//! line-by-line rendering of its [`ManagedLaunchReport`]. + +use super::endpoint_auth::drop_orphaned_endpoint_key_on_already_running; +use super::readiness::{ + died_during_wait, record_wait_outcome, service_pid_is_alive, wait_for_record_ready, + wait_on_existing_service, +}; +#[cfg(feature = "e2e-oom-fault-injection")] +use super::readiness::{e2e_simulate_oom_launch, simulate_oom_managed_launch}; +use super::spawn::{ManagedLaunchReport, ManagedSpawn, spawn_managed_engine_child}; +use crate::serve_port; +use crate::{record_cli_audit_event, status_for_readiness}; +use anyhow::Result; +use rocm_core::format_http_base_url; +use rocm_core::{AppPaths, EndpointReadiness}; +use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; +use std::fmt::Write as _; +#[cfg(windows)] +use std::thread; +use std::time::Duration; + +/// Launch a managed service in the background: run the locked allocation and +/// spawn transaction, release the launch lock once the record is published, +/// then wait for HTTP readiness. +/// +/// Drops the endpoint key `serve()` freshly staged for this attempt if the +/// launch fails before any `ManagedServiceRecord` exists to own it (port +/// reserved/in use, range exhausted, other bind error) — such a failure +/// returns before `abandon_failed_managed_launch` ever runs, so nothing else +/// would clear it. Scoped to the spawn call only, mirroring +/// [`run_attached_service`](super::run_attached_service)'s identical guard: a later +/// failure (the readiness wait, `record.write()`) leaves a record a live child actually owns, so it +/// must not also wipe that key. +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_managed_service( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port_request: serve_port::PortRequest, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + endpoint_api_key: Option<&str>, + launch_lock: rocm_core::FileLock, + require_api_key: bool, + gpu_refusal_waived_for_reuse: bool, + wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str, u16), + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + #[cfg(feature = "e2e-oom-fault-injection")] + if e2e_simulate_oom_launch() { + return simulate_oom_managed_launch( + paths, + engine, + service_id, + requested_model, + resolve, + host, + port_request + .explicit() + .unwrap_or(rocm_core::DEFAULT_LOCAL_PORT), + device_policy, + gpu_indices, + runtime_id, + env_id, + ); + } + let (mut record, child_pid) = match spawn_managed_engine_child( + paths, + engine, + service_id, + requested_model, + resolve, + host, + port_request, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + require_api_key, + gpu_refusal_waived_for_reuse, + ) + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })? { + ManagedSpawn::AlreadyRunning(report) => { + // `--wait` applies to a service that is still starting too: the + // natural retry after `rocm serve m` printed `starting`. + drop(launch_lock); + return wait_on_existing_service( + paths, + report, + wait_until_ready, + on_wait_begin, + on_wait_tick, + ); + } + ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), + }; + // The transaction resolved the request to one concrete port; everything + // below — readiness, the endpoint URL, the audit line — uses that. + let port = record.port; + // The claiming service record is now persisted, so the selected GPU is + // visible to any concurrent auto-selection. Release the launch lock before + // the readiness wait below, which can block for many seconds — holding it + // that long would needlessly serialize unrelated serves. + drop(launch_lock); + + #[cfg(windows)] + thread::sleep(Duration::from_millis(200)); + + if wait_until_ready { + on_wait_begin(service_id, port); + } + let readiness = wait_for_record_ready( + paths, + &record, + wait_until_ready, + endpoint_api_key, + on_wait_tick, + ); + if let Some(error) = died_during_wait( + paths, + service_id, + wait_until_ready, + readiness, + service_pid_is_alive, + ) { + return Err(error); + } + let launch_status = status_for_readiness(readiness); + record_wait_outcome( + paths, + &mut record, + launch_status, + readiness == EndpointReadiness::Serving, + )?; + let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); + record_cli_audit_event( + paths, + "service", + "managed_service_launch", + "info", + format!( + "launched managed service engine={} model={} endpoint={} readiness={}", + engine, resolve.canonical_model_id, endpoint_url, launch_status + ), + Some(service_id), + ); + Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url, + status: launch_status.to_owned(), + already_running: false, + child_pid: Some(child_pid), + log_path: Some(record.log_path), + manifest_path: Some(record.manifest_path), + }) +} + +/// Reproduce the original plain, line-by-line managed-launch output. Used for +/// non-interactive output (piped, CI, the chat assistant's `serve --managed`), +/// where the animated summary is inappropriate. The interactive path renders the +/// summary table via [`crate::serve_summary`] instead. +pub(crate) fn print_managed_launch_plain( + report: &ManagedLaunchReport, + endpoint_api_key: Option<&str>, +) { + if report.already_running { + println!("managed service already running"); + println!(" service_id: {}", report.service_id); + println!(" endpoint: {}", report.endpoint_url); + println!(" status: {}", report.status); + println!(" note: existing service detected; no second process spawned"); + return; + } + println!("managed service launched"); + println!(" service_id: {}", report.service_id); + if let Some(child_pid) = report.child_pid { + println!(" process_pid: {child_pid}"); + } + println!(" endpoint: {}", report.endpoint_url); + if let Some(key) = endpoint_api_key { + // Intentional one-time display of a freshly generated API key to the + // terminal so the user can copy it — the designed delivery channel + // documented on `render_endpoint_client_config`, not a log. The tag below + // is currently inert (Rust's CodeQL pack has no AlertSuppression.ql yet — + // github/codeql#21637) but will start working once that lands, since the + // tag must be the single line immediately before the flagged code. + print!( + // codeql[rust/cleartext-logging] + "{}", + render_endpoint_client_config(&report.endpoint_url, key) + ); + } + if let Some(log_path) = report.log_path.as_deref() { + println!(" log_path: {}", log_path.display()); + } + if let Some(manifest_path) = report.manifest_path.as_deref() { + println!(" manifest_path: {}", manifest_path.display()); + } + println!(" readiness: {}", report.status); +} + +/// The client configuration to show when a `--wait` begins, so a key generated for +/// a public bind reaches the user even if the wait is interrupted or fails. +/// `None` unless the wait is on the launch that key belongs to: an attach to an +/// already-starting server (`waiting_on` differs from `service_id`) answers to +/// its own key, which is not shown here. +pub(crate) fn wait_begin_client_config( + waiting_on: &str, + service_id: &str, + host: &str, + port: u16, + api_key: Option<&str>, +) -> Option { + let key = api_key.filter(|_| waiting_on == service_id)?; + let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); + Some(render_endpoint_client_config(&endpoint_url, key)) +} + +/// Render the one-time secure client configuration for a public, authenticated +/// endpoint. This is the *intended* channel for delivering the key to the user +/// (unlike logs/status, which must never contain it) — it prints the key once at +/// launch alongside a ready-to-use example. Callers only invoke this for a +/// non-loopback bind that generated/received a key. +pub(super) fn render_endpoint_client_config(endpoint_url: &str, api_key: &str) -> String { + let mut out = String::new(); + let _ = writeln!(out, " api key: {api_key}"); + let _ = writeln!( + out, + " note: this key is shown only now - clients must send `Authorization: Bearer `" + ); + let _ = writeln!( + out, + " example: curl -H \"Authorization: Bearer {api_key}\" {endpoint_url}/models" + ); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endpoint_keys; + #[cfg(not(windows))] + use crate::service_launch::run_attached_service; + #[cfg(not(windows))] + use crate::service_launch::spawn::TEST_SPAWN_HOOK; + use crate::tests::test_paths; + use rocm_core::ManagedServiceRecord; + use std::fs; + #[cfg(not(windows))] + use std::io; + + #[test] + fn wait_begin_shows_the_key_only_for_the_launch_it_belongs_to() { + let own = wait_begin_client_config("svc-new", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .expect("own launch shows its key"); + // No `{own}` in the failure messages: the text carries the dummy key, + // and formatting it into an assert is what CodeQL flags as logging it. + assert!(own.contains("api key: k-secret"), "key line missing"); + assert!( + own.contains("http://0.0.0.0:8000/v1/models"), + "models url missing" + ); + // Attached to an already-starting server: its key is not this one. + assert!( + wait_begin_client_config("svc-old", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .is_none() + ); + // Loopback binds have no key to show. + assert!(wait_begin_client_config("svc-new", "svc-new", "127.0.0.1", 8000, None).is_none()); + } + + #[test] + fn endpoint_client_config_shows_key_once_with_bearer_guidance() { + let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); + // Dummy literal exercising the intentional one-time key display documented + // on `render_endpoint_client_config`; not a real credential or a log write. + // codeql[rust/cleartext-logging] + assert!(rendered.contains("secret-123"), "{rendered}"); + // codeql[rust/cleartext-logging]: see rationale above. + assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); + // codeql[rust/cleartext-logging]: see rationale above. + assert!(rendered.contains("shown only now"), "{rendered}"); + } + + #[test] + fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { + // serve() stores the new attempt's endpoint key before calling + // start_managed_service. If spawn_managed_engine_child fails inside + // resolve_managed_port_in_transaction (explicit port already reserved + // by another live service), it returns before any + // ManagedServiceRecord exists for this attempt, so + // abandon_failed_managed_launch never runs. Driving the real function + // end to end (not a simulated call to + // drop_orphaned_endpoint_key_on_already_running) proves + // `start_managed_service`'s own `.inspect_err` actually wires the cleanup, mirroring how + // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key + // drives run_attached_service for the foreground path. + let (root, paths) = test_paths("orphaned-key-on-reserved-port"); + paths.ensure()?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11513, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let result = start_managed_service( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11513), + &resolve.device_policy, + &[], + None, + None, + None, + Some("test-key"), + launch_lock, + false, + false, + false, + &mut |_, _| {}, + &mut |_elapsed| {}, + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "cleanup must drop the key this failed attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The launch lock must still be held when the child is spawned: it is what + /// serializes allocation against publication. Observed through the spawn + /// hook, since releasing it early changes nothing else a test can see. + #[cfg(not(windows))] + #[test] + fn start_and_attached_launches_hold_the_launch_lock_until_the_spawn() -> Result<()> { + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + for attached in [false, true] { + let (root, paths) = test_paths("launch-lock-held-until-spawn"); + paths.ensure()?; + let lock_path = paths.managed_launch_lock_path(); + let lock_was_held = std::rc::Rc::new(std::cell::Cell::new(false)); + let observed = lock_was_held.clone(); + TEST_SPAWN_HOOK.with(|hook| { + *hook.borrow_mut() = Some(std::rc::Rc::new(move |_command| { + observed.set( + rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err(), + ); + Err(io::Error::other("spawn intercepted by test")) + })); + }); + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let result = if attached { + run_attached_service( + &paths, + "lemonade", + "lemonade-lock-attached", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &[], + None, + None, + None, + launch_lock, + false, + false, + |_, _| unreachable!("the spawn fails before a Ctrl-C could stop it"), + ) + } else { + start_managed_service( + &paths, + "lemonade", + "lemonade-lock-start", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &resolve.device_policy, + &[], + None, + None, + None, + None, + launch_lock, + false, + false, + false, + &mut |_, _| {}, + &mut |_elapsed| {}, + ) + .map(|_| ()) + }; + TEST_SPAWN_HOOK.with(|hook| *hook.borrow_mut() = None); + let _ = fs::remove_dir_all(root); + assert!( + format!("{:#}", result.expect_err("the intercepted spawn fails")) + .contains("spawn intercepted by test") + ); + assert!( + lock_was_held.get(), + "the launch lock must be held at spawn (attached: {attached})" + ); + } + Ok(()) + } +} diff --git a/apps/rocm/src/service_launch/test_support.rs b/apps/rocm/src/service_launch/test_support.rs new file mode 100644 index 0000000000..70a5504e09 --- /dev/null +++ b/apps/rocm/src/service_launch/test_support.rs @@ -0,0 +1,103 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Fixtures shared by the tests of more than one launch submodule. + +use anyhow::Result; +use rocm_core::{AppPaths, ManagedServiceRecord}; +#[cfg(not(windows))] +use std::thread; +#[cfg(not(windows))] +use std::time::Duration; + +/// A record with every reuse-relevant field set to the "reusable" value, so +/// each test that uses it breaks exactly one term and nothing else. +pub(super) fn reusable_record(paths: &AppPaths) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + "vllm-qwen-1000", + "vllm", + "qwen", + "qwen-canonical".to_owned(), + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + Some("gpu_required".to_owned()), + ); + record.engine_recipe_json = None; + record.requires_api_key = true; + record +} + +pub(super) fn write_live_service_record( + paths: &AppPaths, + service_id: &str, + canonical_model_id: &str, + port: u16, + status: &str, +) -> Result { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + canonical_model_id, + canonical_model_id.to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = status.to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + Ok(record) +} + +/// The helper the cleanup tests re-run this test binary as, to get a live child +/// process that outlives the spawn they are checking. It is `#[ignore]`d so a +/// normal run skips it, and it only sleeps when the tests that spawn it set +/// `ROCM_TEST_SLEEPER_CHILD`. They select it by [`sleeper_test_name`]. +#[test] +#[cfg(not(windows))] +#[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] +fn managed_launch_cleanup_sleeper_child() { + if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { + thread::sleep(Duration::from_mins(2)); + } +} + +/// The name libtest lists [`managed_launch_cleanup_sleeper_child`] under, which +/// is what `--exact` has to be given to run it alone. +/// +/// It is read off the function item itself, so renaming the function or moving +/// this module changes it with them and the three tests that spawn the helper +/// cannot be left naming a test that no longer exists. libtest names a test by +/// its path inside the crate, while the item path starts with the crate name, +/// which is dropped. The assertions check that against the module path as well. +#[cfg(not(windows))] +pub(super) fn sleeper_test_name() -> String { + let item_path = std::any::type_name_of_val(&managed_launch_cleanup_sleeper_child); + let name = item_path + .split_once("::") + .map_or(item_path, |(_krate, rest)| rest); + let module = module_path!() + .split_once("::") + .map_or("", |(_krate, rest)| rest); + assert_eq!( + name, + format!("{module}::managed_launch_cleanup_sleeper_child"), + "the sleeper helper is no longer where its name is derived from" + ); + assert!( + name.starts_with("service_launch::"), + "libtest names tests by their path inside the crate: {name}" + ); + name.to_owned() +} diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 7be31d9579..2220426374 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -36,10 +36,11 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, ManagedServiceRecord, process_is_running}; +use rocm_core::{AppPaths, ManagedServiceRecord, ProcessProbe}; use crate::managed_service_status::{ - load_managed_services, managed_service_is_live, recorded_service_pids, + container_stop_unresolved, load_managed_services, managed_service_is_live, + recorded_pids_may_be_running, recorded_service_pids, }; use crate::services_text::{load_managed_service, validate_service_id}; use crate::{cli_progress, cli_report, endpoint_keys, service_hints}; @@ -165,11 +166,72 @@ fn remove_service_record_artifacts(artifacts: &ServiceRecordArtifacts) -> Result /// server whose engine reported failure while its process is still up reads as /// removable, and deleting it takes the log and the 0600 endpoint key out from /// under a process that may still be serving. -fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { +/// +/// A recorded process that could not be queried (on Windows, a live one this +/// process may not open) counts as running: deleting the record and its key on +/// the strength of a failed look is the mistake this guard exists to prevent. +/// +/// The read of each recorded PID is supplied by the caller, so a test drives an +/// unqueryable process on every host; production passes +/// [`rocm_core::probe_process`]. +fn managed_service_record_is_in_use_with( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { managed_service_is_live(record) - || recorded_service_pids(record) - .iter() - .any(|pid| process_is_running(*pid)) + || container_stop_unresolved(record) + || recorded_pids_may_be_running(record, probe) +} + +/// The recorded processes `--force` may remove a record over: `Some` only when +/// none of them is confirmed running, at least one could not be queried (so +/// `rocm` cannot tell whether it is still running), and a stop was requested on +/// the record, so the user has already tried to stop it. A process confirmed +/// running is never forced past. +fn forceable_unqueryable_pids( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> Option> { + let probed: Vec<(u32, ProcessProbe)> = recorded_service_pids(record) + .into_iter() + .map(|pid| (pid, probe(pid))) + .collect(); + if probed + .iter() + .any(|(_, probed)| *probed == ProcessProbe::Running) + { + return None; + } + let unqueryable: Vec = probed + .into_iter() + .filter(|(_, probed)| *probed == ProcessProbe::Unqueryable) + .map(|(pid, _)| pid) + .collect(); + (!unqueryable.is_empty() && record.stop_requested_unix_ms.is_some()).then_some(unqueryable) +} + +/// The recorded PIDs as the message names them: `process 4242`, or +/// `processes 4242 and 4243`. +fn named_processes(pids: &[u32]) -> String { + match pids { + [one] => format!("process {one}"), + many => format!( + "processes {}", + many.iter() + .map(u32::to_string) + .collect::>() + .join(" and ") + ), + } +} + +/// What [`remove_managed_service_record`] did. +pub(crate) struct RemovedRecord { + /// The text to print. + pub(crate) text: String, + /// The removal went ahead over a recorded process that could not be queried + /// (`--force`), rather than for a record that is not running. + pub(crate) forced: bool, } /// Remove one non-running local server record and everything it owns. @@ -180,28 +242,70 @@ fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { /// which has already been refreshed against the engine state file and the real /// processes — a manifest still saying `ready` for a dead PID has demoted to /// `stopped` by then, and must stay removable. +/// +/// A recorded process that could not be queried (on Windows, a live one this +/// process may not open) makes the record count as running, and nothing else +/// `rocm` has can tell a dead server whose PID was reused from a live one. For +/// that case only, `--force` removes the record anyway: when no recorded process +/// is confirmed running, at least one could not be queried, and a stop was +/// requested. The removal then says exactly that, since the process may still +/// be serving and its endpoint key is deleted with the record. +#[cfg(test)] pub(crate) fn remove_managed_service_record( paths: &AppPaths, service_id: &str, yes: bool, ) -> Result { + remove_managed_service_record_with(paths, service_id, yes, false, rocm_core::probe_process) + .map(|removed| removed.text) +} + +/// [`remove_managed_service_record`] with `--force` and the read of each +/// recorded PID supplied by the caller. +pub(crate) fn remove_managed_service_record_with( + paths: &AppPaths, + service_id: &str, + yes: bool, + force: bool, + probe: impl Fn(u32) -> ProcessProbe, +) -> Result { validate_service_id(service_id)?; let record = load_managed_service(paths, service_id)?; + let mut forced_over = None; // Checked before `--yes` on purpose: for a running server "stop it first" is // the actionable error, and repeating the command with --yes must not be the // advice a user takes away from it. - if managed_service_record_is_in_use(&record) { - bail!( - "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: {}", - record.status, - service_hints::service_stop_hint(service_id) - ); + if managed_service_record_is_in_use_with(&record, &probe) { + let forceable = forceable_unqueryable_pids(&record, &probe); + match (force, forceable) { + (true, Some(pids)) => forced_over = Some(pids), + (false, Some(pids)) => bail!( + "local server `{service_id}` cannot be removed: {} it recorded could not be queried, so `rocm` cannot tell whether it is still running.\n\nTry: {}", + named_processes(&pids), + service_hints::service_forced_remove_hint(service_id) + ), + (true, None) => bail!( + "--force removes a record only when none of its recorded processes is confirmed running, at least one could not be queried, and a stop was requested. Local server `{service_id}` is {} and does not meet that.\n\nTry: {}", + record.status, + service_hints::service_stop_hint(service_id) + ), + (false, None) => bail!( + "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: {}", + record.status, + service_hints::service_stop_hint(service_id) + ), + } } if !yes { - bail!( - "Removing local server record `{service_id}` requires --yes.\n\nTry: {}", + // The command that completes this removal, not a neighbour of it: a + // forced removal that is missing `--yes` is told to add it, not to drop + // the `--force` it asked for. + let hint = if forced_over.is_some() { + service_hints::service_forced_remove_hint(service_id) + } else { service_hints::service_action_hint("remove", service_id) - ); + }; + bail!("Removing local server record `{service_id}` requires --yes.\n\nTry: {hint}"); } let artifacts = service_record_artifacts(paths, &record.service_id, &record.engine)?; @@ -213,6 +317,15 @@ pub(crate) fn remove_managed_service_record( "Removing local server record `{service_id}` (status: {}).", record.status ); + if let Some(pids) = &forced_over { + // Said before the delete and in full: what is being risked is the user's + // to know, and after this there is no record left to ask about. + let _ = writeln!( + output, + "--force: {} could not be queried, so `rocm` cannot tell whether it is still running. If it is, it keeps running; this removes the record and the endpoint key it may still be serving with.", + named_processes(pids) + ); + } let _ = writeln!(output, " log: {}", artifacts.log.display()); let _ = writeln!( output, @@ -231,7 +344,10 @@ pub(crate) fn remove_managed_service_record( report = report.detail("removed", path.display()); } let _ = write!(output, "{}", report.render()); - Ok(output) + Ok(RemovedRecord { + text: output, + forced: forced_over.is_some(), + }) } /// One record `rocm services prune` would delete. @@ -350,7 +466,7 @@ fn describe_hours(hours: u64) -> String { /// /// "No `.json` beside it" is also exactly what a *launch in progress* looks /// like: `serve` writes the 0600 `.endpoint-key` before -/// `spawn_managed_engine_child` (in `service_launch.rs`) writes the first +/// `spawn_managed_engine_child` (in `service_launch/spawn.rs`) writes the first /// `.json`. Nothing about /// the file distinguishes the two cases — under `--any-age` there is no age gate /// left to ask — so this function does not try. It is only ever reached with the @@ -416,6 +532,7 @@ fn build_service_prune_plan( min_age: Duration, hours: u64, now: SystemTime, + probe: &impl Fn(u32) -> ProcessProbe, ) -> Result { let mut plan = ServicePrunePlan::default(); // Taken first, because `load_managed_services` below rewrites the manifest of @@ -427,14 +544,28 @@ fn build_service_prune_plan( // the real processes before returning it, so liveness below is read from the // refreshed view, never from the manifest as it was on disk. for record in load_managed_services(paths)? { - if managed_service_record_is_in_use(&record) { + if managed_service_record_is_in_use_with(&record, probe) { plan.skipped_live += 1; - plan.skipped.push(format!( - "{} is {} - stop it first with `{}`", - record.service_id, - record.status, - service_hints::service_stop_hint(&record.service_id) - )); + // A record held only by a process `rocm` cannot query, after a stop + // was requested, is the one `remove --force` exists for: say so and + // name it, instead of advising a stop that can only report the same + // thing again. `prune` itself never forces. + let skipped = match forceable_unqueryable_pids(&record, probe) { + Some(pids) => format!( + "{} is {} and {} it recorded could not be queried - remove it with `{}`", + record.service_id, + record.status, + named_processes(&pids), + service_hints::service_forced_remove_hint(&record.service_id) + ), + None => format!( + "{} is {} - stop it first with `{}`", + record.service_id, + record.status, + service_hints::service_stop_hint(&record.service_id) + ), + }; + plan.skipped.push(skipped); continue; } let artifacts = match service_record_artifacts(paths, &record.service_id, &record.engine) { @@ -579,10 +710,22 @@ pub(crate) struct ServicePruneOutcome { /// was built. That window is the only reason the liveness re-check below exists, /// and no test driving `prune` end to end can open it: one call builds the plan /// and applies it, so a record is either live for both halves or dead for both. +#[cfg(test)] fn apply_service_prune_plan( paths: &AppPaths, plan: &ServicePrunePlan, outcome: &mut ServicePruneOutcome, +) { + apply_service_prune_plan_with(paths, plan, outcome, &rocm_core::probe_process); +} + +/// [`apply_service_prune_plan`] with the read of each recorded PID supplied by +/// the caller. +fn apply_service_prune_plan_with( + paths: &AppPaths, + plan: &ServicePrunePlan, + outcome: &mut ServicePruneOutcome, + probe: &impl Fn(u32) -> ProcessProbe, ) { for entry in &plan.remove { // Liveness was snapshotted while the plan was built. A @@ -592,7 +735,7 @@ fn apply_service_prune_plan( // `remove` narrows the same window by loading the record it deletes. // Neither closes it: the check and the delete are not atomic either way. if load_managed_service(paths, &entry.service_id) - .is_ok_and(|record| managed_service_record_is_in_use(&record)) + .is_ok_and(|record| managed_service_record_is_in_use_with(&record, probe)) { outcome.skipped_live += 1; let _ = writeln!( @@ -633,9 +776,9 @@ fn apply_service_prune_plan( /// leftover, and that is also what a launch looks like mid-flight: `serve` /// writes the 0600 `.endpoint-key` at the top of the managed path, and the /// first `.json` only lands at `record.write()` inside -/// `spawn_managed_engine_child` (in `service_launch.rs`). Between those two writes a live server's -/// secret is indistinguishable from a leftover, and `--any-age` removes the age -/// gate that used to hide the window, so a concurrent +/// `spawn_managed_engine_child` (in `service_launch/spawn.rs`). Between those two +/// writes a live server's secret is indistinguishable from a leftover, and +/// `--any-age` removes the age gate that used to hide the window, so a concurrent /// `rocm services prune --any-age --yes` deleted it. /// /// `serve` already holds `managed_launch_lock_path` across the whole of that @@ -718,6 +861,18 @@ pub(crate) fn prune_managed_service_records( hours: u64, dry_run: bool, yes: bool, +) -> Result { + prune_managed_service_records_with(paths, hours, dry_run, yes, &rocm_core::probe_process) +} + +/// [`prune_managed_service_records`] with the read of each recorded PID supplied +/// by the caller. +fn prune_managed_service_records_with( + paths: &AppPaths, + hours: u64, + dry_run: bool, + yes: bool, + probe: &impl Fn(u32) -> ProcessProbe, ) -> Result { if !dry_run && !yes { bail!( @@ -742,7 +897,7 @@ pub(crate) fn prune_managed_service_records( rocm_core::FileLock::acquire(paths.managed_launch_lock_path())? }; let min_age = Duration::from_secs(hours.saturating_mul(3600)); - let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now())?; + let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now(), probe)?; let mut outcome = ServicePruneOutcome { text: render_service_prune_plan(&plan, hours, dry_run), skipped_live: plan.skipped_live, @@ -753,7 +908,7 @@ pub(crate) fn prune_managed_service_records( return Ok(outcome); } - apply_service_prune_plan(paths, &plan, &mut outcome); + apply_service_prune_plan_with(paths, &plan, &mut outcome, probe); let _ = writeln!(outcome.text); let _ = write!( @@ -1795,4 +1950,423 @@ mod tests { assert!(manifest_exists, "a refused prune must delete nothing"); Ok(()) } + + /// A record whose status is not live is still in use while a process it + /// recorded may be running, and a process that could not be queried (on + /// Windows, a live one this process may not open) may be. Only a confirmed + /// exit makes it removable. + #[test] + fn a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone() -> Result<()> { + let (root, paths) = test_paths("services-in-use-probe"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-in-use", "failed", 999_999_999)?; + let _ = fs::remove_dir_all(&root); + + assert!(managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Unqueryable + })); + assert!(managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Running + })); + assert!(!managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Gone + })); + Ok(()) + } + + /// The by-hand way out, which still works alongside `--force`, for a record + /// whose process `rocm` cannot query: the record is refused while that process may be + /// running, and deleting its `.json` by hand and running + /// `prune --any-age` leaves nothing behind (log, key and engine state all + /// swept), with the endpoint key among the files removed. + #[test] + fn a_record_held_by_an_unqueryable_process_can_be_cleared_by_hand_and_prune() -> Result<()> { + let (root, paths) = test_paths("services-unqueryable-way-out"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-stuck", "stopped", 999_999_999)?; + let refused = managed_service_record_is_in_use_with(&record, |_| ProcessProbe::Unqueryable); + let key = endpoint_keys::endpoint_key_file_path(&paths, "svc-stuck"); + assert!(key.exists(), "the planted record has its key"); + + fs::remove_file(&record.manifest_path)?; + let outcome = prune_managed_service_records(&paths, 0, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let leftovers = [&record.log_path, &record.engine_state_path, &key] + .iter() + .filter(|path| path.exists()) + .count(); + let _ = fs::remove_dir_all(&root); + + assert!( + refused, + "a record whose process cannot be queried is in use" + ); + assert_eq!(leftovers, 0, "prune swept every leftover file"); + assert_eq!(outcome.removed_files, 3); + Ok(()) + } + + /// A record the user already tried to stop, held by the test's own live PID, + /// so the real liveness read that loads it leaves it alone and only the + /// injected probe says the process cannot be queried. + fn held_record(paths: &AppPaths, service_id: &str, stop_requested: bool) -> Result<()> { + let mut record = plant_service_record(paths, service_id, "ready", std::process::id())?; + if stop_requested { + record.stop_requested_unix_ms = Some(1); + record.write()?; + } + Ok(()) + } + + fn files_left(paths: &AppPaths, service_id: &str) -> usize { + let manifest = paths.service_manifest_path(service_id).exists(); + let log = paths.service_log_path(service_id).exists(); + let key = endpoint_keys::endpoint_key_file_path(paths, service_id).exists(); + usize::from(manifest) + usize::from(log) + usize::from(key) + } + + /// Without `--force` the refusal names the forced removal, and everything the + /// record owns is still there: the advice and the state, together. + #[test] + fn a_record_held_by_an_unqueryable_process_is_refused_naming_the_forced_removal() -> Result<()> + { + let (root, paths) = test_paths("services-remove-unqueryable-refused"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + + let error = remove_managed_service_record_with(&paths, "svc-held", true, false, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record held by an unqueryable process is refused"); + let message = format!("{error:#}"); + let left = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(message.contains("could not be queried"), "{message}"); + assert!( + message.contains("rocm services remove svc-held --force --yes"), + "{message}" + ); + assert_eq!(left, 3, "nothing was removed"); + // The command the refusal names parses as printed. + let words: Vec = service_hints::service_forced_remove_hint("svc-held") + .split_whitespace() + .map(str::to_owned) + .collect(); + assert_eq!( + crate::advised_commands::rocm_verdict(&words[1..]), + crate::advised_commands::Verdict::Parses, + "{words:?}" + ); + Ok(()) + } + + /// `--force` removes it and says what it risked; the audit-relevant flag is set. + #[test] + fn force_removes_a_record_held_by_an_unqueryable_process_and_says_so() -> Result<()> { + let (root, paths) = test_paths("services-remove-unqueryable-forced"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let pid = std::process::id(); + + let removed = remove_managed_service_record_with(&paths, "svc-held", true, true, |_| { + ProcessProbe::Unqueryable + })?; + let left = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(removed.forced); + assert!( + removed.text.contains(&format!( + "--force: process {pid} could not be queried, so `rocm` cannot tell whether it is still running" + )), + "{}", + removed.text + ); + assert!(removed.text.contains("endpoint key"), "{}", removed.text); + assert_eq!(left, 0, "every file the record owned is gone"); + Ok(()) + } + + /// An `aim` stop whose engine `Stop` failed leaves the stop pending, and its + /// supervisor can be gone while the container still runs. Neither the liveness + /// refresh nor `remove` and `prune` may then treat the record as stopped and + /// delete it, with the key the container may still be enforcing: the refresh + /// leaves the status alone, `remove` refuses it, and `prune` skips it, all + /// with the supervisor confirmed gone. A process engine in the same state is + /// an ordinary stopped record that is removed. + #[test] + fn a_pending_container_stop_survives_refresh_remove_and_prune() -> Result<()> { + let gone = |_: u32| ProcessProbe::Gone; + let (root, paths) = test_paths("services-container-pending-stop"); + paths.ensure()?; + // A record already reading `stopped` (an earlier reconciliation wrote it) + // is the case only the in-use guard protects. + for (engine, status, id, protected) in [ + ("aim", "ready", "svc-container", true), + ("aim", "stopped", "svc-container-stopped", true), + ("vllm", "ready", "svc-process", false), + ] { + let mut record = plant_service_record(&paths, id, status, 999_999_999)?; + record.engine = engine.to_owned(); + record.stop_requested_unix_ms = Some(1); + record.write()?; + + let mut refreshed = record.clone(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut refreshed, + gone, + ); + let removal = remove_managed_service_record_with(&paths, id, true, false, gone); + let pruned = prune_managed_service_records_with(&paths, 0, false, true, &gone)?; + let left = files_left(&paths, id); + + if protected { + assert_eq!(refreshed.status, status, "{engine}: refresh"); + assert_eq!(refreshed.supervisor_pid, 999_999_999, "{engine}: refresh"); + assert!(removal.is_err(), "{engine}: remove must refuse"); + assert_eq!(pruned.removed_records, 0, "{engine}: prune"); + assert_eq!(left, 3, "{engine}: nothing was deleted"); + } else { + assert!(removal.is_ok() || pruned.removed_records == 1, "{engine}"); + assert_eq!(left, 0, "{engine}: an ordinary stopped record goes"); + } + } + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A `ready` record whose dead server's PID is now held by a process this + /// account cannot open, and that nobody has tried to stop: `--force` is + /// refused (it never forces a record nobody tried to stop), and the refusal + /// names `services stop`. That stop cannot verify the process, reports it + /// unverified and records the request, after which `--force` removes the + /// record. So the record is not stuck: the way out is those two commands, and + /// this is the intended shape, not a gap. + #[test] + fn a_ready_record_held_by_an_unqueryable_process_is_cleared_by_stop_then_force() -> Result<()> { + let (root, paths) = test_paths("services-remove-ready-unqueryable-chain"); + paths.ensure()?; + held_record(&paths, "svc-ready-held", false)?; + let pid = std::process::id(); + + let first = + remove_managed_service_record_with(&paths, "svc-ready-held", true, true, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record nobody tried to stop is not forced"); + let report = crate::service_control::stop_internal_managed_service_with( + &paths, + "svc-ready-held", + |_| { + vec![rocm_core::RecordedPidStop { + pid, + role: "supervisor_and_engine", + outcome: rocm_core::RecordedPidOutcome::Terminated( + rocm_core::TerminationOutcome::Unverified, + ), + }] + }, + )?; + let second = + remove_managed_service_record_with(&paths, "svc-ready-held", true, true, |_| { + ProcessProbe::Unqueryable + })?; + let left = files_left(&paths, "svc-ready-held"); + let _ = fs::remove_dir_all(root); + + assert!( + format!("{first:#}").contains("rocm services stop svc-ready-held --yes"), + "{first:#}" + ); + assert_eq!(report["stopped"], false, "{report}"); + assert!(second.forced); + assert_eq!(left, 0, "the forced removal took the record and its files"); + Ok(()) + } + + /// `--force` never goes past a process that is confirmed running, and it + /// needs a stop to have been requested: the refusal says which, and names the + /// stop that is the way forward. + #[test] + fn force_is_refused_for_a_running_process_and_without_a_requested_stop() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-refused"); + paths.ensure()?; + held_record(&paths, "svc-running", true)?; + held_record(&paths, "svc-no-stop", false)?; + + let running = remove_managed_service_record_with(&paths, "svc-running", true, true, |_| { + ProcessProbe::Running + }) + .map(|removed| removed.text) + .expect_err("a confirmed running process is never forced past"); + let no_stop = remove_managed_service_record_with(&paths, "svc-no-stop", true, true, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record nobody tried to stop is not forced"); + let left = files_left(&paths, "svc-running") + files_left(&paths, "svc-no-stop"); + let _ = fs::remove_dir_all(root); + + for error in [format!("{running:#}"), format!("{no_stop:#}")] { + assert!( + error.contains("--force removes a record only when"), + "{error}" + ); + } + assert!( + format!("{running:#}").contains("rocm services stop svc-running --yes"), + "{running:#}" + ); + assert_eq!(left, 6, "nothing was removed"); + Ok(()) + } + + /// `--force` on a record that is simply not running changes nothing: it is + /// removed as usual and not reported as forced. + #[test] + fn force_on_a_record_that_is_not_running_is_an_ordinary_removal() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-unneeded"); + paths.ensure()?; + plant_service_record(&paths, "svc-stopped", "stopped", 999_999_999)?; + + let removed = + remove_managed_service_record_with(&paths, "svc-stopped", true, true, |_| { + ProcessProbe::Gone + })?; + let _ = fs::remove_dir_all(root); + + assert!(!removed.forced); + assert!(!removed.text.contains("--force"), "{}", removed.text); + Ok(()) + } + + /// One recorded process that cannot be queried does not make a record + /// forceable while another is confirmed running: the running one is never + /// forced past, whichever role it has. + #[test] + fn force_is_refused_when_one_process_is_unqueryable_and_another_is_running() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-mixed"); + paths.ensure()?; + let own = std::process::id(); + let mut record = plant_service_record(&paths, "svc-mixed", "ready", own)?; + record.engine_pid = Some(999_999_997); + record.stop_requested_unix_ms = Some(1); + record.write()?; + + let error = remove_managed_service_record_with(&paths, "svc-mixed", true, true, |pid| { + if pid == own { + ProcessProbe::Unqueryable + } else { + ProcessProbe::Running + } + }) + .map(|removed| removed.text) + .expect_err("a record with a process confirmed running is never forced"); + let left = files_left(&paths, "svc-mixed"); + let _ = fs::remove_dir_all(root); + + assert!( + format!("{error:#}").contains("--force removes a record only when"), + "{error:#}" + ); + assert_eq!(left, 3, "nothing was removed"); + Ok(()) + } + + /// A forced removal that is missing `--yes` is told to add `--yes`, not to + /// drop the `--force` it asked for: the command it names completes the + /// removal, which is asserted by running it. + #[test] + fn a_forced_removal_missing_yes_names_the_command_that_completes_it() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-needs-yes"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let unqueryable = |_: u32| ProcessProbe::Unqueryable; + + let error = + remove_managed_service_record_with(&paths, "svc-held", false, true, unqueryable) + .map(|removed| removed.text) + .expect_err("a forced removal without --yes is refused"); + let message = format!("{error:#}"); + let left_before = files_left(&paths, "svc-held"); + // The named command, run as named, removes the record. + let named = service_hints::service_forced_remove_hint("svc-held"); + let completed = + remove_managed_service_record_with(&paths, "svc-held", true, true, unqueryable)?; + let left_after = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(message.contains("requires --yes"), "{message}"); + assert!(message.contains(&format!("Try: {named}")), "{message}"); + assert_eq!(left_before, 3, "the refusal removed nothing"); + assert!(completed.forced); + assert_eq!(left_after, 0, "the command it named cleared the record"); + // An ordinary removal still names the plain command. + let plain = service_hints::service_action_hint("remove", "svc-held"); + assert!( + !named.eq(&plain) && named.contains("--force"), + "{named} / {plain}" + ); + Ok(()) + } + + /// `prune` never forces, and says what does: a record held only by an + /// unqueryable process, after a stop was requested, is reported with the + /// exact `remove --force --yes` command, and that command, run, clears the + /// record. A record with a process confirmed running keeps the stop advice. + #[test] + fn prune_names_the_forced_removal_for_a_record_held_by_an_unqueryable_process() -> Result<()> { + let (root, paths) = test_paths("services-prune-names-force"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let unqueryable = |_: u32| ProcessProbe::Unqueryable; + + let outcome = prune_managed_service_records_with(&paths, 0, false, true, &unqueryable)?; + let left_after_prune = files_left(&paths, "svc-held"); + let named = service_hints::service_forced_remove_hint("svc-held"); + // Run exactly what prune named. + let removed = + remove_managed_service_record_with(&paths, "svc-held", true, true, unqueryable)?; + let left_after_named = files_left(&paths, "svc-held"); + + held_record(&paths, "svc-running", true)?; + let running = + prune_managed_service_records_with(&paths, 0, true, false, &|_| ProcessProbe::Running)?; + let _ = fs::remove_dir_all(root); + + assert!( + outcome.text.contains(&format!( + "svc-held is starting and process {} it recorded could not be queried - remove it with `{named}`", + std::process::id() + )), + "{}", + outcome.text + ); + assert_eq!(outcome.removed_records, 0, "prune itself never forces"); + assert_eq!(left_after_prune, 3, "and left every file alone"); + assert!(removed.forced); + assert_eq!( + left_after_named, 0, + "the command prune named clears the record" + ); + assert!( + running.text.contains( + "svc-running is starting - stop it first with `rocm services stop svc-running --yes`" + ), + "{}", + running.text + ); + assert!(!running.text.contains("--force"), "{}", running.text); + Ok(()) + } } diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index b559503110..f2cbc52623 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -38,7 +38,7 @@ use crate::service_hints::{ service_action_verb, }; use crate::service_records::{ - prune_managed_service_records, remove_managed_service_record, service_prune_min_age_hours, + prune_managed_service_records, remove_managed_service_record_with, service_prune_min_age_hours, }; use crate::services_text::{ follow_service_logs, render_service_action_result, @@ -74,17 +74,32 @@ pub(crate) fn services(command: Option) -> Result<()> { ServicesCommand::Restart { service_id, yes } => { run_approved_service_action(&paths, "restart_server", &service_id, yes) } - ServicesCommand::Remove { service_id, yes } => { - print!( - "{}", - remove_managed_service_record(&paths, &service_id, yes)? - ); + ServicesCommand::Remove { + service_id, + yes, + force, + } => { + let removed = remove_managed_service_record_with( + &paths, + &service_id, + yes, + force, + rocm_core::probe_process, + )?; + print!("{}", removed.text); + let message = if removed.forced { + format!( + "removed local server record {service_id} (--force: a recorded process could not be queried)" + ) + } else { + format!("removed local server record {service_id}") + }; record_cli_audit_event( &paths, "service", "remove_record", "info", - format!("removed local server record {service_id}"), + message, Some(&service_id), ); Ok(()) @@ -227,4 +242,31 @@ mod tests { assert!(rendered.contains("status: stopped")); assert!(rendered.contains("stopped processes: 2")); } + + /// The stop report production hands the renderer is the one + /// `stop_internal_managed_service` builds: `service_id` and `status` sit + /// directly on `result`, not under `result.service`. Scenario + /// `@id:serve-lemonade-simple-registered-direct-route` asserts the service id + /// in the stop output, so the renderer must print it for this shape. + #[test] + fn service_stop_render_prints_the_in_process_stop_report_shape() { + let rendered = crate::services_text::render_service_action_result( + "stop_server", + &serde_json::json!({ + "output": { + "result": { + "service_id": "svc-qwen", + "status": "stopped", + "engine_stop": {"attempted": true, "stopped": true, "graceful": true}, + "signaled_pids": [1234] + } + } + }), + ); + + assert!(rendered.contains("Local server stopped")); + assert!(rendered.contains("service: svc-qwen"), "{rendered}"); + assert!(rendered.contains("status: stopped"), "{rendered}"); + assert!(rendered.contains("stopped processes: 1")); + } } diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 6bae507e7f..cd6af59fb6 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -9,8 +9,10 @@ //! Mechanically relocated from `main.rs` with no behavior change, and later //! narrowed: the process-control cluster (stop, restart and the internal //! sandbox-tool runner) lives in `service_control.rs` (see -//! `docs/architecture.md`). The `--follow` tail (`LogTail`, `FollowFooter`, -//! `follow_service_logs`) is new code written here, not relocated. +//! `docs/architecture.md`). The `--follow` tail (`FollowCursor`, `FollowFooter`, +//! `follow_service_logs`) is new code written here, not relocated. A +//! `FollowCursor` follows a growing file; it is not `rocm_core::LogTail`, the +//! result of a bounded tail read that the log views here use. //! //! This module owns: the CLI/TUI logs browser (`render_logs_text`, //! `render_logs_browser_text`, `render_logs_browser_page_text`, @@ -23,7 +25,7 @@ //! `render_service_action_result`, `read_managed_service_manifest`, //! `load_managed_service`), the `--follow` tail behind `rocm logs --service //! --follow` and `rocm services logs --follow` -//! (`follow_service_logs`, `follow_service_logs_with`, the `LogTail` and +//! (`follow_service_logs`, `follow_service_logs_with`, the `FollowCursor` and //! `LogSegment` types, `write_log_segments`), the internal `rocmd`-compatible //! status surface (`render_internal_status_text`), the `stop_server` answer shape //! (`internal_stop_server_value`, `service_stop_unconfirmed`) and its renderer @@ -31,7 +33,7 @@ //! `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the //! CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, //! `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, -//! `read_tail_lines`). +//! `read_log_tail`). //! //! The `LogBrowserEntry` struct moved here with the logs browser and stays //! private. @@ -54,10 +56,10 @@ //! `service_control.rs` and from `managed_service_status.rs`'s own test module; //! `read_managed_service_manifest`, `internal_stop_server_value` and //! `process_check_command` are `pub(crate)` for `service_control.rs` (the last -//! one only for its tests); `read_tail_lines`, `render_logs_text`, +//! one only for its tests); `read_log_tail`, `render_logs_text`, //! `render_logs_browser_text`, `render_services_text` and -//! `render_service_logs_text` are also `pub(crate)` and read from `chat.rs`'s -//! deterministic tool-output summaries; `validate_service_id` is also +//! `render_service_logs_text` are also `pub(crate)` and read from +//! `chat/mcp_bridge.rs`'s read-only command runner; `validate_service_id` is also //! `pub(crate)` and read from `tool_policy.rs`'s service-tool validation, //! `services_cmd.rs` and `service_records.rs`; `read_optional_tail_lines` and //! `render_service_action_result` are also `pub(crate)` and read from @@ -77,10 +79,9 @@ //! classification respectively — stay in `main.rs`'s shared `mod tests` //! instead. -use std::collections::VecDeque; use std::fmt::Write as _; use std::fs; -use std::io::{self, BufRead, Read as _, Seek as _, SeekFrom}; +use std::io::{self, Read as _, Seek as _, SeekFrom}; use std::path::{Path, PathBuf}; use std::thread; use std::time::Duration; @@ -176,16 +177,22 @@ fn render_logs_browser_page_text_with_options( if query.is_some() { let _ = writeln!(output, "Recent activity: filtered by search"); } else { - let recent_lines = read_optional_tail_lines(&lifecycle_path, 8, "CLI lifecycle log"); - if recent_lines.is_empty() { + let recent = read_optional_log_tail(&lifecycle_path, 8, "CLI lifecycle log"); + if recent.newest_line_exceeds_the_read_bound() { + let _ = writeln!( + output, + "Recent activity: none shown; {}", + rocm_core::newest_line_exceeds_bound_notice() + ); + } else if recent.lines.is_empty() { let _ = writeln!(output, "Recent activity: no activity yet"); } else { let _ = writeln!( output, "Recent activity: last {} line(s)", - recent_lines.len() + recent.lines.len() ); - for line in recent_lines { + for line in recent.lines { let _ = writeln!(output, " {}", format_cli_lifecycle_tail_line(&line)); } } @@ -249,7 +256,15 @@ struct LogBrowserEntry { fn collect_log_browser_entries(paths: &AppPaths) -> Vec { let lifecycle_path = cli_lifecycle_log_path(paths); let mut entries = Vec::new(); - for line in read_optional_tail_lines(&lifecycle_path, 8, "CLI lifecycle log") { + let lifecycle = read_optional_log_tail(&lifecycle_path, 8, "CLI lifecycle log"); + if lifecycle.newest_line_exceeds_the_read_bound() { + // Not a lifecycle record, so it is not formatted as one. + entries.push(LogBrowserEntry { + source: "lifecycle".to_owned(), + line: omitted_output_placeholder("CLI lifecycle log"), + }); + } + for line in lifecycle.lines { entries.push(LogBrowserEntry { source: "lifecycle".to_owned(), line: format_cli_lifecycle_tail_line(&line), @@ -360,6 +375,31 @@ fn write_past_attempts_hint(output: &mut String, past_attempts: usize, newest_id let _ = writeln!(output, " Reclaim the space: rocm services prune"); } +/// The `anthropic endpoint:` row of a service listing. +/// +/// Only a service that reports `ready` is probed: the endpoint is published when +/// the engine first lists its model and is gone with the process, so for any +/// other status the answer would say nothing the status does not. The probe is +/// the same cheap `/v1/models` listing the OpenAI endpoint is judged by. +fn anthropic_endpoint_line(paths: &AppPaths, record: &ManagedServiceRecord, url: &str) -> String { + let responding = record.status != "ready" + || rocm_core::anthropic_endpoint_ready( + url, + Some(record.served_model_name()), + crate::endpoint_keys::endpoint_api_key(paths, &record.service_id).as_deref(), + Duration::from_millis(750), + ); + format_anthropic_endpoint_line(url, responding) +} + +fn format_anthropic_endpoint_line(url: &str, responding: bool) -> String { + if responding { + format!(" anthropic endpoint: {url}") + } else { + format!(" anthropic endpoint: {url} (not responding)") + } +} + pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { // Counts describe the WHOLE registry, not the rows this view keeps. The // default view hides records that are not live, and counting the filtered @@ -416,6 +456,13 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Result { let record = load_managed_service(paths, service_id)?; - let recent_lines = - read_optional_tail_lines(&record.log_path, DEFAULT_LOG_TAIL_LINES, "service log"); + let recent = read_optional_log_tail(&record.log_path, DEFAULT_LOG_TAIL_LINES, "service log"); let mut output = String::new(); let _ = writeln!(output, "Service Log"); @@ -506,11 +552,17 @@ fn render_service_logs_text_with_options( let _ = writeln!(output, " Choose Show file locations to see exact paths."); } let _ = writeln!(output); - if recent_lines.is_empty() { + if recent.newest_line_exceeds_the_read_bound() { + let _ = writeln!( + output, + "Recent output: none shown; {}", + rocm_core::newest_line_exceeds_bound_notice() + ); + } else if recent.lines.is_empty() { let _ = writeln!(output, "Recent output: no output yet"); } else { - let _ = writeln!(output, "Recent output: last {} line(s)", recent_lines.len()); - for line in recent_lines { + let _ = writeln!(output, "Recent output: last {} line(s)", recent.lines.len()); + for line in recent.lines { let _ = writeln!(output, " {line}"); } } @@ -526,7 +578,7 @@ pub(crate) fn render_service_action_result_with_helper_note( ) -> String { let mut text = render_service_action_result(tool, value); if tool == "stop_server" - && let Some(note) = crate::service_launch::background_helper_note(paths) + && let Some(note) = crate::background_helper::background_helper_note(paths) { let _ = writeln!(text, " note: {}", note.text); } @@ -539,11 +591,13 @@ const LOG_FOLLOW_POLL: Duration = Duration::from_millis(500); const LOG_FOLLOW_LIVENESS_EVERY: Duration = Duration::from_secs(2); /// Reads whatever a growing log file gained since the last [`Self::poll`]. +/// Not `rocm_core::LogTail`, which is what a bounded read of the end of a log +/// returns. /// /// Download bars (tqdm, huggingface_hub) redraw in place with `\r`, so a /// finished segment ends at either `\r` or `\n`; a trailing partial segment is /// held back until its terminator arrives. -struct LogTail { +struct FollowCursor { path: PathBuf, offset: u64, pending: Vec, @@ -556,7 +610,7 @@ struct LogSegment { redraw: bool, } -impl LogTail { +impl FollowCursor { /// Start following at the file's current end. fn at_end(path: &Path) -> Self { let offset = fs::metadata(path).map_or(0, |meta| meta.len()); @@ -851,7 +905,7 @@ fn follow_service_logs_via( // Open the tail before the snapshot: output written in between is then shown // twice (in the snapshot and by the first poll) rather than lost, and the // tail can start mid-line. - let mut tail = LogTail::at_end(&record.log_path); + let mut tail = FollowCursor::at_end(&record.log_path); write!(out, "{}", render_service_logs_text(paths, service_id)?)?; out.flush()?; if !managed_service_is_live(&record) { @@ -905,7 +959,7 @@ fn follow_service_logs_via( fn follow_loop( load: &dyn Fn() -> Result, record: &ManagedServiceRecord, - tail: &mut LogTail, + tail: &mut FollowCursor, footer: &mut Option, out: &mut impl io::Write, err: &mut impl io::Write, @@ -975,11 +1029,21 @@ pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value return render_unconfirmed_service_stop(output); } let action = service_action_past_tense(tool); - let service = output.get("service").or_else(|| { - output - .get("result") - .and_then(|result| result.get("service")) - }); + // Restart nests the record under `service`; the in-process stop report + // ([`stop_internal_managed_service`]) is itself the record's summary, with + // `service_id` and `status` directly on `result`. + let service = output + .get("service") + .or_else(|| { + output + .get("result") + .and_then(|result| result.get("service")) + }) + .or_else(|| { + output + .get("result") + .filter(|result| result.get("service_id").is_some()) + }); let mut text = String::new(); let _ = writeln!(text, "Local server {action}"); if let Some(service) = service { @@ -1018,9 +1082,20 @@ pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value /// /// - a process that was signalled and that, or a process under it, outlived /// the deadline (`timed_out`); +/// - a process that is gone, with the separate kill of the processes under it +/// not completed (`descendants_unconfirmed`, Windows): this stop saw the +/// recorded process exit, so it does not say anything outlived a deadline, but +/// one of the processes under it may still be running; +/// - a recorded process that exited by itself before the processes under it +/// could be taken down (`exited_before_tree_kill`, Windows), with no other +/// recorded process confirmed gone to vouch for what was under it: nothing +/// was signalled, and one of them may still be running; /// - a process that was not signalled because its identity could not be -/// verified (`unverified`), with how to look at it. It may have exited -/// meanwhile: the identity read can fail because the process is going away; +/// verified (`unverified`), with how to look at it. There are two routes to +/// it: a recorded start-time that can no longer be read back from a running +/// process, and, on Windows, a PID that cannot be queried at all. It may have +/// exited meanwhile: the identity read can fail because the process is going +/// away; /// - the record changed while the stop ran, typically because a restart /// recorded new processes, so the stop's verdict says nothing about what the /// record names now. @@ -1055,6 +1130,10 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { .collect() }; let timed_out = with_outcome(rocm_core::TerminationOutcome::TimedOut.as_str()); + let descendants_unconfirmed = + with_outcome(rocm_core::TerminationOutcome::DescendantsUnconfirmed.as_str()); + let exited_before_tree_kill = + with_outcome(rocm_core::TerminationOutcome::ExitedBeforeTreeKill.as_str()); let unverified = with_outcome(rocm_core::TerminationOutcome::Unverified.as_str()); let mut text = String::new(); @@ -1078,6 +1157,18 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { " still running: process {pid} ({role}) or a process under it did not exit after it was signaled" ); } + for (pid, role) in &descendants_unconfirmed { + let _ = writeln!( + text, + " not confirmed: process {pid} ({role}) exited, but a process under it may still be running, because taking the processes under it down did not complete; rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed" + ); + } + for (pid, role) in &exited_before_tree_kill { + let _ = writeln!( + text, + " not confirmed: process {pid} ({role}) was already gone when the processes under it were to be taken down, and nothing else the server recorded vouches for them, so one of them may still be running; rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed" + ); + } for (pid, role) in &unverified { let _ = writeln!( text, @@ -1140,11 +1231,24 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { /// stop left alone because it could not verify it. Not a `rocm` command: the /// CLI deliberately does not act on that process, so the user decides. /// -/// A Linux command because only Linux can produce that outcome: it needs a -/// recorded start-time that can no longer be read back, and -/// `rocm_core::process_start_ticks` records one only on Linux. +/// Linux and Windows can both produce that outcome, by two routes. On both, a +/// recorded start-time that can no longer be read back from a running process +/// (`rocm_core::process_start_ticks` records one on both). On Windows also, a PID +/// that cannot be queried at all, such as a live process this one may not open, +/// which reads as unverifiable with or without a recorded token. So the command +/// is the one the host has: `ps` on Linux, `tasklist` on Windows. pub(crate) fn process_check_command(pid: u64) -> String { - format!("ps -p {pid} -o pid,args") + process_check_command_for(cfg!(windows), pid) +} + +/// [`process_check_command`] for a chosen platform, so a test reads both forms +/// on every host. +fn process_check_command_for(windows: bool, pid: u64) -> String { + if windows { + format!("tasklist /FI \"PID eq {pid}\"") + } else { + format!("ps -p {pid} -o pid,args") + } } /// A managed service's record exactly as it is on disk, without the engine-state @@ -1248,13 +1352,42 @@ fn list_log_files(dir: &Path, limit: usize) -> Vec { paths.into_iter().take(limit).collect() } +/// The last `limit` lines of a log that may not exist. A log that has content +/// but none of it fits the read budget comes back as one +/// [`omitted_output_placeholder`] line, like a read error comes back as one +/// `` line, so a caller that lists the lines never takes a +/// log with content for an empty one. A missing or empty log is an empty list. pub(crate) fn read_optional_tail_lines(path: &Path, limit: usize, label: &str) -> Vec { + let tail = read_optional_log_tail(path, limit, label); + if tail.newest_line_exceeds_the_read_bound() { + vec![omitted_output_placeholder(label)] + } else { + tail.lines + } +} + +/// The line that stands in for a log whose newest line is longer than the +/// budget a log view reads, so nothing of it could be shown. +fn omitted_output_placeholder(label: &str) -> String { + format!( + "", + rocm_core::newest_line_exceeds_bound_notice() + ) +} + +/// [`read_optional_tail_lines`], keeping what the byte budget left unread so +/// a caller that describes an empty result can tell an empty log from one +/// whose newest line is too long to show. +pub(crate) fn read_optional_log_tail(path: &Path, limit: usize, label: &str) -> rocm_core::LogTail { if !path.is_file() { - return Vec::new(); + return rocm_core::LogTail::default(); } - match read_tail_lines(path, limit, label) { - Ok(lines) => lines, - Err(error) => vec![format!("")], + match read_log_tail(path, limit, label) { + Ok(tail) => tail, + Err(error) => rocm_core::LogTail { + lines: vec![format!("")], + skipped_bytes: 0, + }, } } @@ -1320,23 +1453,17 @@ fn lifecycle_field<'a>(line: &'a str, key: &str) -> Option<&'a str> { .filter(|value| !value.is_empty()) } -pub(crate) fn read_tail_lines(path: &Path, limit: usize, label: &str) -> Result> { - if limit == 0 { - return Ok(Vec::new()); - } - - let file = fs::File::open(path) - .with_context(|| format!("failed to open {label} {}", path.display()))?; - let reader = io::BufReader::new(file); - let mut lines = VecDeque::with_capacity(limit); - for line in reader.lines() { - let line = line.with_context(|| format!("failed to read {label} {}", path.display()))?; - if lines.len() == limit { - lines.pop_front(); - } - lines.push_back(line); - } - Ok(lines.into_iter().collect()) +/// The last `limit` lines of a log, read through the shared bounded tail +/// routine, with what the byte budget left unread; `label` names the log in the +/// error ("service log", "CLI action log"). +pub(crate) fn read_log_tail(path: &Path, limit: usize, label: &str) -> Result { + rocm_core::tail_log_decoded( + path, + limit, + rocm_core::LOG_VIEW_MAX_TAIL_READ, + rocm_core::LogDecode::PLAIN, + ) + .with_context(|| format!("failed to read {label} {}", path.display())) } #[cfg(test)] pub(crate) mod tests { @@ -1348,6 +1475,146 @@ pub(crate) mod tests { use crate::service_control::run_internal_sandbox_tool; use crate::tests::test_paths; + /// The lines of [`read_log_tail`], for the tests that only look at those. + fn read_tail_lines(path: &Path, limit: usize, label: &str) -> Result> { + Ok(read_log_tail(path, limit, label)?.lines) + } + + fn tail_fixture(name: &str, bytes: &[u8]) -> Result { + let (root, _paths) = test_paths(name); + fs::create_dir_all(&root)?; + let path = root.join("service.log"); + fs::write(&path, bytes)?; + Ok(path) + } + + /// 5 MiB: a marked first line, 5120 lines of 1 KiB, a marked last line. + fn large_log_with_head_marker() -> Vec { + let mut bytes = b"HEAD-MARKER\n".to_vec(); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..5120 { + bytes.extend_from_slice(filler.as_bytes()); + } + bytes.extend_from_slice(b"TAIL-MARKER\n"); + bytes + } + + #[test] + fn read_tail_lines_a_large_file_is_read_through_a_bounded_window() -> Result<()> { + // A line far outside the tail window is not returned however large the + // limit: the log is not streamed whole. + let log = tail_fixture("services-tail-large", &large_log_with_head_marker())?; + + let lines = read_tail_lines(&log, 10_000, "service log")?; + fs::remove_file(&log)?; + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + assert!(lines.len() < 5122, "got {} lines", lines.len()); + assert!(lines.len() > 3500, "the window holds about 4 MiB of lines"); + Ok(()) + } + + #[test] + fn read_tail_lines_shows_a_sixteen_bit_log_as_lossy_text_instead_of_failing() -> Result<()> { + // Service logs are the child's own bytes, read as UTF-8; a UTF-16 log + // is not detected here (only the ComfyUI log view does that), but it no + // longer aborts the view. + let mut bytes = vec![0xff, 0xfe]; + for unit in "[INFO] one\r\n".encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + let log = tail_fixture("services-tail-utf16", &bytes)?; + + let lines = read_tail_lines(&log, 5, "service log")?; + + assert!( + lines[0].starts_with("\u{FFFD}\u{FFFD}["), + "the byte order mark is replaced, the rest kept: {lines:?}" + ); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_ansi_colour_codes() -> Result<()> { + let log = tail_fixture("services-tail-ansi", b"\x1b[32mgreen\x1b[0m done\n")?; + + assert_eq!( + read_tail_lines(&log, 5, "service log")?, + vec!["\u{1b}[32mgreen\u{1b}[0m done"] + ); + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_invalid_eight_bit_text_lossily() -> Result<()> { + let log = tail_fixture("services-tail-invalid", b"ok\nbad\xffbyte\nlast\n")?; + + let expected = vec![ + "ok".to_owned(), + "bad\u{FFFD}byte".to_owned(), + "last".to_owned(), + ]; + assert_eq!(read_tail_lines(&log, 5, "service log")?, expected); + // The optional reader shows the same lines, not a failure placeholder. + assert_eq!(read_optional_tail_lines(&log, 5, "service log"), expected); + Ok(()) + } + + #[test] + fn read_tail_lines_missing_file_is_a_labelled_error_naming_the_path() { + let (root, _paths) = test_paths("services-tail-missing"); + let missing = root.join("absent.log"); + + let error = + read_tail_lines(&missing, 5, "service log").expect_err("a missing log must not read"); + + assert_eq!( + error.to_string(), + format!("failed to read service log {}", missing.display()) + ); + assert!( + format!("{error:#}").contains(&format!("failed to open {}", missing.display())), + "the cause names the failing step: {error:#}" + ); + // read_optional_tail_lines guards on is_file first and stays quiet. + assert!(read_optional_tail_lines(&missing, 5, "service log").is_empty()); + } + + #[test] + fn read_tail_lines_limit_zero_returns_nothing_without_opening_the_file() { + let (root, _paths) = test_paths("services-tail-zero"); + + let lines = read_tail_lines(&root.join("absent.log"), 0, "service log") + .expect("limit 0 returns before the file is opened"); + + assert!(lines.is_empty()); + } + + #[test] + fn read_tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let log = tail_fixture("services-tail-few", b"a\nb\n")?; + + assert_eq!(read_tail_lines(&log, 50, "service log")?, vec!["a", "b"]); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let log = tail_fixture("services-tail-no-newline", b"a\nb\nc")?; + + assert_eq!(read_tail_lines(&log, 2, "service log")?, vec!["b", "c"]); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_crlf_line_endings() -> Result<()> { + let log = tail_fixture("services-tail-crlf", b"a\r\nb\r\n")?; + + assert_eq!(read_tail_lines(&log, 5, "service log")?, vec!["a", "b"]); + Ok(()) + } + #[test] fn render_logs_text_preserves_directory_summary() { let (_root, paths) = test_paths("logs-summary"); @@ -1562,7 +1829,7 @@ pub(crate) mod tests { paths.ensure()?; let log = root.join("tail.log"); fs::write(&log, "old line\n")?; - let mut tail = LogTail::at_end(&log); + let mut tail = FollowCursor::at_end(&log); assert!(tail.poll().is_empty()); // tqdm-style bar redraws with `\r`; the last piece has no terminator yet. @@ -1615,7 +1882,7 @@ pub(crate) mod tests { paths.ensure()?; let log = root.join("edges.log"); fs::write(&log, "")?; - let mut tail = LogTail::at_end(&log); + let mut tail = FollowCursor::at_end(&log); // A `\r` ending the data may be half of a `\r\n`: it is not a redraw yet. fs::write(&log, " 99%|#\r")?; @@ -1761,7 +2028,7 @@ pub(crate) mod tests { fs::create_dir_all(&root)?; let log = root.join("engine.log"); fs::write(&log, "")?; - let mut tail = LogTail::at_end(&log); + let mut tail = FollowCursor::at_end(&log); fs::write(&log, "before\x1b[1Aafter\n")?; let segments = tail.poll(); assert_eq!(segments.len(), 1); @@ -1776,7 +2043,7 @@ pub(crate) mod tests { fs::create_dir_all(&root)?; let log = root.join("engine.log"); fs::write(&log, "")?; - let mut tail = LogTail::at_end(&log); + let mut tail = FollowCursor::at_end(&log); fs::write(&log, "last\x1b[1Aword")?; assert!(tail.poll().is_empty()); let segment = tail.finish().expect("unterminated output is returned"); @@ -2372,6 +2639,239 @@ pub(crate) mod tests { Ok(()) } + /// What `rocm services logs` prints for a service whose log holds `bytes` + /// (or has no file when `None`), with the log's size on disk, so a message + /// is asserted together with the state it describes. + fn render_service_log_with(name: &str, bytes: Option<&[u8]>) -> Result<(String, Option)> { + let (root, paths) = test_paths(name); + paths.ensure()?; + let record = ManagedServiceRecord::new( + &paths, + "svc_tail_notice", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_preferred".to_owned()), + ); + record.write()?; + let size = match bytes { + Some(bytes) => { + fs::write(&record.log_path, bytes)?; + Some(fs::metadata(&record.log_path)?.len()) + } + None => None, + }; + let rendered = render_service_logs_text(&paths, "svc_tail_notice")?; + let _ = fs::remove_dir_all(root); + Ok((rendered, size)) + } + + #[test] + fn service_logs_say_no_output_only_for_an_empty_log_and_name_the_bound_otherwise() -> Result<()> + { + let bound = rocm_core::LOG_VIEW_MAX_TAIL_READ; + assert_eq!( + bound, + 4 * 1024 * 1024, + "the notice below names this many MiB" + ); + let notice = + "Recent output: none shown; the newest line is longer than the 4 MiB a log view reads"; + let over_the_bound = usize::try_from(bound)? + 1024; + + // No log file and an empty one: nothing was written, and it says so. + let (rendered, size) = render_service_log_with("service-logs-none", None)?; + assert_eq!(size, None); + assert!( + rendered.contains("Recent output: no output yet"), + "{rendered}" + ); + let (rendered, size) = render_service_log_with("service-logs-empty", Some(b""))?; + assert_eq!(size, Some(0)); + assert!( + rendered.contains("Recent output: no output yet"), + "{rendered}" + ); + + // One line longer than the bound, with and without a final newline: + // the file has content, so "no output yet" would be false. + let one_line = "x".repeat(over_the_bound).into_bytes(); + let mut with_newline = one_line.clone(); + with_newline.push(b'\n'); + for (name, bytes) in [ + ("service-logs-long-line", one_line), + ("service-logs-long-line-newline", with_newline), + ] { + let (rendered, size) = render_service_log_with(name, Some(&bytes))?; + assert!(size.is_some_and(|size| size > bound), "{name}: {size:?}"); + assert!(rendered.contains(notice), "{name}: {rendered}"); + assert!(!rendered.contains("no output yet"), "{name}: {rendered}"); + assert!(!rendered.contains("last 0 line(s)"), "{name}: {rendered}"); + } + + // A big log whose newest lines are short is shown, not described. + let (rendered, size) = + render_service_log_with("service-logs-big", Some(&large_log_with_head_marker()))?; + assert!(size.is_some_and(|size| size > bound)); + assert!(rendered.contains("TAIL-MARKER"), "{rendered}"); + assert!(!rendered.contains(notice), "{rendered}"); + Ok(()) + } + + /// One newline-terminated line longer than the budget a log view reads. + fn line_longer_than_the_read_bound() -> Result> { + let mut bytes = vec![b'x'; usize::try_from(rocm_core::LOG_VIEW_MAX_TAIL_READ)? + 1024]; + bytes.push(b'\n'); + Ok(bytes) + } + + #[test] + fn read_optional_tail_lines_names_a_log_it_could_not_show_instead_of_returning_nothing() + -> Result<()> { + let over = tail_fixture( + "services-optional-over", + &line_longer_than_the_read_bound()?, + )?; + let empty = tail_fixture("services-optional-empty", b"")?; + + assert_eq!( + read_optional_tail_lines(&over, 5, "CLI action log"), + vec![ + "" + ] + ); + // An empty log has nothing to show and stays an empty list. + assert!(read_optional_tail_lines(&empty, 5, "CLI action log").is_empty()); + Ok(()) + } + + #[test] + fn the_logs_browser_does_not_call_a_log_with_an_oversized_line_absent() -> Result<()> { + let over = line_longer_than_the_read_bound()?; + // Each log holds the one oversized line and nothing else, in turn. + for (name, relative) in [ + ("logs-over-action", "logs/cli/probe-action.log"), + ("logs-over-screen", "logs/tui/probe-screen.log"), + ] { + let (root, paths) = test_paths(name); + let path = paths.data_dir.join(relative); + fs::create_dir_all(path.parent().expect("a log has a directory"))?; + fs::write(&path, &over)?; + + let rendered = render_logs_text(&paths); + let _ = fs::remove_dir_all(root); + + let label = if relative.contains("cli") { + "CLI action log" + } else { + "screen command log" + }; + assert!( + !rendered.contains("No logs found yet."), + "{name}: {rendered}" + ); + assert!( + rendered.contains(" Lines: 1 of 1 recent line(s)"), + "{name}: {rendered}" + ); + assert!( + rendered.contains(&format!( + "" + )), + "{name}: {rendered}" + ); + } + Ok(()) + } + + #[test] + fn the_recent_activity_summary_does_not_call_an_oversized_lifecycle_entry_no_activity() + -> Result<()> { + let (root, paths) = test_paths("logs-over-lifecycle"); + let lifecycle = cli_lifecycle_log_path(&paths); + fs::create_dir_all(lifecycle.parent().expect("a log has a directory"))?; + fs::write(&lifecycle, line_longer_than_the_read_bound()?)?; + + let rendered = render_logs_text(&paths); + let _ = fs::remove_dir_all(root); + + assert!(!rendered.contains("no activity yet"), "{rendered}"); + assert!( + rendered.contains(" Lines: 1 of 1 recent line(s)"), + "{rendered}" + ); + assert!( + rendered.contains( + "Recent activity: none shown; the newest line is longer than the 4 MiB a log view reads" + ), + "{rendered}" + ); + assert!(!rendered.contains("No logs found yet."), "{rendered}"); + Ok(()) + } + + #[test] + fn a_startup_failure_detail_does_not_call_a_log_with_an_oversized_line_empty() -> Result<()> { + let over = tail_fixture("startup-detail-over", &line_longer_than_the_read_bound()?)?; + let empty = tail_fixture("startup-detail-empty", b"")?; + + let detail = + crate::service_launch::managed_engine_startup_failure_detail(" with status 1", &over); + assert!( + detail.ends_with( + "recent startup log output:\n" + ), + "{detail}" + ); + // A log that really is empty still gets the short form. + let detail = + crate::service_launch::managed_engine_startup_failure_detail(" with status 1", &empty); + assert_eq!( + detail, + format!( + "managed engine exited immediately with status 1; inspect {}", + empty.display() + ) + ); + Ok(()) + } + + #[test] + fn anthropic_endpoint_row_says_when_the_endpoint_is_not_responding() { + assert_eq!( + format_anthropic_endpoint_line("http://127.0.0.1:41000", true), + " anthropic endpoint: http://127.0.0.1:41000" + ); + assert_eq!( + format_anthropic_endpoint_line("http://127.0.0.1:41000", false), + " anthropic endpoint: http://127.0.0.1:41000 (not responding)" + ); + } + + #[test] + fn services_listing_shows_the_published_anthropic_endpoint_without_probing_a_stopped_service() + -> Result<()> { + let (_guard, paths) = test_paths("services-anthropic-row"); + let mut record = crate::tests::plant_service_record(&paths, "svc-anthropic", "stopped", 0)?; + record.anthropic_endpoint_url = Some("http://127.0.0.1:1".to_owned()); + record.anthropic_endpoint_mode = Some("proxy".to_owned()); + record.write()?; + + let rendered = render_services_text(&paths, true)?; + + assert!( + rendered.contains("\n anthropic endpoint: http://127.0.0.1:1\n"), + "{rendered}" + ); + Ok(()) + } + #[test] fn render_services_text_lists_live_services_by_default_and_all_on_request() -> Result<()> { use std::io::{Read, Write}; @@ -2812,4 +3312,24 @@ pub(crate) mod tests { })); assert_eq!(answer["status"], "stop_unconfirmed", "{answer}"); } + + /// The check a stop that could not verify a process tells the user to run + /// is a command the host has: `ps` on Linux, `tasklist` on Windows, where a + /// Windows record now carries a creation time that can fail to read back. + /// The wording is pinned here; that the Windows form shows whether a PID is + /// running is what the `service-stop-07` steps rely on, which look a PID up + /// with the same `tasklist /FI "PID eq "` filter. + #[test] + fn the_process_check_command_is_the_one_the_host_has() { + assert_eq!(process_check_command_for(false, 42), "ps -p 42 -o pid,args"); + assert_eq!( + process_check_command_for(true, 42), + "tasklist /FI \"PID eq 42\"" + ); + assert_eq!( + process_check_command(42), + process_check_command_for(cfg!(windows), 42), + "production picks by the host it runs on" + ); + } } diff --git a/apps/rocm/src/setup_cmd.rs b/apps/rocm/src/setup_cmd.rs index 4fb933cb44..35a3c706a8 100644 --- a/apps/rocm/src/setup_cmd.rs +++ b/apps/rocm/src/setup_cmd.rs @@ -11,7 +11,7 @@ //! `docs/architecture.md`. `SetupCommand`, the clap enum `setup` matches on, //! lives in `cli.rs` with the rest of the `Command`/`Cli` definition. //! `dispatch()` calls `setup` through the re-import `use -//! crate::setup_cmd::{...};`, and `chat.rs`'s read-only handling of `rocm setup` +//! crate::setup_cmd::{...};`, and `chat/mcp_bridge.rs`'s read-only handling of `rocm setup` //! and `rocm setup status` reaches `render_setup_status_text` as //! `crate::render_setup_status_text` through the same re-import; both are //! `pub(crate)`. `reset_setup_prompt_state` has no caller outside this module diff --git a/apps/rocm/src/storage.rs b/apps/rocm/src/storage.rs index 3983cca0ba..d1c08eaca6 100644 --- a/apps/rocm/src/storage.rs +++ b/apps/rocm/src/storage.rs @@ -370,10 +370,94 @@ pub(crate) struct RuntimeUsage { pub hold_reason: Option, } +/// Whether Lemonade's install record points at a tree. +/// +/// `Unused` means unused right now: the record points at another tree that +/// really holds an install. It says nothing about the future. Lemonade installs +/// into the active ROCm install's folder, so a copy inside a registered ROCm +/// install is reused the moment that install becomes active, and only the data +/// folder's copy, which no ROCm install owns, is ever offered as reclaimable. A +/// record that names a folder with nothing in it, or that cannot be read, leaves +/// it unknown which copy works, so no tree is called unused. In every case the +/// advice must not tell a user to delete a copy that a later serve would reuse, +/// which would force a reinstall of roughly 25 GiB. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EngineInstallStatus { + /// The install record points here, and the install is there. + InUse, + /// The record points at another tree that holds an install, so nothing uses + /// this one. + Unused, + /// The record names a folder that holds no install, so no copy is in use. + RecordWithoutInstall, + /// A record exists but could not be read, so it is not known which copy is + /// in use. + RecordUnreadable, + /// Lemonade has no install record at all. + NotRecorded, +} + +/// Decide a tree's status from the install record. +fn classify_engine_install( + record: &rocm_engine_lemonade::InstallRecord, + tree: &Path, +) -> EngineInstallStatus { + use rocm_engine_lemonade::InstallRecord; + match record { + InstallRecord::NotRecorded => EngineInstallStatus::NotRecorded, + InstallRecord::Unreadable(_) => EngineInstallStatus::RecordUnreadable, + InstallRecord::Recorded { + holds_install: false, + .. + } => EngineInstallStatus::RecordWithoutInstall, + InstallRecord::Recorded { runtime_dir, .. } + if rocm_core::runtime_path_is_same_or_inside(runtime_dir, tree) => + { + EngineInstallStatus::InUse + } + InstallRecord::Recorded { .. } => EngineInstallStatus::Unused, + } +} + +/// One Lemonade install tree found on disk. +/// +/// A self-managed engine installs into the active ROCm runtime's own folder +/// when one is active, and into the data folder otherwise, so the same host +/// can carry a full copy in each. Before this row existed the only trace of a +/// second copy was a larger runtime size and data-folder total. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct EngineInstallUsage { + pub engine: String, + pub path: PathBuf, + /// The runtime whose folder holds this tree, `None` for the data folder. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_key: Option, + pub size_bytes: u64, + pub size_complete: bool, + pub status: EngineInstallStatus, + /// Whether the report offers this copy for deletion: unused right now and + /// not inside a registered ROCm install, where it would be reused when that + /// install becomes active. + pub reclaimable: bool, + /// Why the status could not be decided, for the two record-problem states. + #[serde(skip_serializing_if = "Option::is_none")] + pub record_note: Option, + /// Whether `rocm runtimes uninstall` deletes this tree along with its + /// runtime. Decided by the same rule uninstall applies + /// (`should_remove_runtime_install_root`): an external or imported runtime + /// is only unregistered, so its Lemonade copy stays. `false` for the data + /// folder's tree, which belongs to no runtime. + pub removed_with_runtime: bool, +} + #[derive(Debug, Clone, Serialize)] pub(crate) struct StorageReport { pub runtimes: Vec, pub runtimes_total_bytes: u64, + /// Engine installs ROCm CLI made. A tree inside a runtime folder is also + /// part of that runtime's size above. + pub engines: Vec, pub rocm_cli: Vec, /// Caches ROCm CLI uses but does not own. Reported, never deleted. pub shared_with_other_tools: Vec, @@ -419,6 +503,81 @@ fn shared_model_cache_dir() -> Option { Some(home.join(".cache").join("huggingface")) } +/// Every Lemonade tree ROCm CLI could have installed: the data folder's, and +/// one per registered runtime (where it goes while that runtime is active). +/// Only the install parts are measured (`install_tree_dirs`), not the record, +/// logs and server state that share the data folder's tree. +fn lemonade_install_usages( + paths: &AppPaths, + manifests: &[therock::InstalledRuntimeManifest], +) -> Vec { + let record = rocm_engine_lemonade::install_record(paths); + let record_note = match &record { + rocm_engine_lemonade::InstallRecord::Unreadable(error) => Some(format!( + "Lemonade's install record could not be read ({error}), so it is not known which copy is in use" + )), + rocm_engine_lemonade::InstallRecord::Recorded { + runtime_dir, + holds_install: false, + } => Some(format!( + "Lemonade's install record names {}, which holds no install, so no copy is in use", + runtime_dir.display() + )), + _ => None, + }; + let candidates = std::iter::once(( + None, + rocm_engine_lemonade::install_tree_root(paths, None), + false, + )) + .chain(manifests.iter().map(|manifest| { + ( + Some(manifest.runtime_key.clone()), + rocm_engine_lemonade::install_tree_root( + paths, + Some(&manifest.install_root.join("engines")), + ), + should_remove_runtime_install_root(manifest) + .is_ok_and(InstallRootDecision::should_remove), + ) + })); + candidates + .filter_map(|(runtime_key, path, removed_with_runtime)| { + let parts = rocm_engine_lemonade::install_tree_dirs(&path); + let present = parts + .iter() + .filter(|part| part.exists()) + .collect::>(); + if present.is_empty() { + return None; + } + let (size_bytes, size_complete) = + present + .iter() + .fold((0_u64, true), |(bytes, complete), part| { + let measured = measure_path(part); + ( + bytes.saturating_add(measured.bytes), + complete && measured.complete, + ) + }); + let status = classify_engine_install(&record, &path); + let reclaimable = status == EngineInstallStatus::Unused && runtime_key.is_none(); + Some(EngineInstallUsage { + engine: "lemonade".to_owned(), + path, + runtime_key, + size_bytes, + size_complete, + status, + reclaimable, + record_note: record_note.clone(), + removed_with_runtime, + }) + }) + .collect() +} + pub(crate) fn build_report(paths: &AppPaths, config: &RocmCliConfig) -> Result { let manifests = therock::load_runtime_manifests(paths)?; let marker = read_active_runtime_marker(paths); @@ -484,7 +643,7 @@ pub(crate) fn build_report(paths: &AppPaths, config: &RocmCliConfig) -> Result.log` beside the manifest. `ManagedServiceRecord::new` // only *computes* that path (via `AppPaths::service_log_path`) - it // writes nothing. The file is created by the launch site, - // `spawn_managed_engine_child` (in `service_launch.rs`) for + // `spawn_managed_engine_child` (in `service_launch/spawn.rs`) for // `rocm serve --managed` and `supervise_service` in `rocmd` on the // supervised/recovery path. What *fills* it differs by platform, so the // redirect cannot be stated unqualified: `supervise_service` redirects @@ -532,9 +691,12 @@ pub(crate) fn build_report(paths: &AppPaths, config: &RocmCliConfig) -> Result String { format_bytes(report.runtimes_total_bytes) ); + let _ = writeln!(output); + let _ = writeln!(output, "Engine installs:"); + if report.engines.is_empty() { + let _ = writeln!(output, " none"); + } + for usage in &report.engines { + let size = if usage.size_complete { + format_bytes(usage.size_bytes) + } else { + format!( + "{} or more (part unreadable)", + format_bytes(usage.size_bytes) + ) + }; + let location = usage.runtime_key.as_deref().map_or_else( + || "data folder".to_owned(), + |key| format!("inside ROCm install {key}"), + ); + let _ = writeln!( + output, + " - {} ({location}): {size} ({})", + usage.engine, + usage.path.display() + ); + let status = match usage.status { + EngineInstallStatus::InUse => "in use", + EngineInstallStatus::Unused => "not used right now; Lemonade runs from another install", + EngineInstallStatus::RecordWithoutInstall => { + "not known to be used; the install record names a folder with no install" + } + EngineInstallStatus::RecordUnreadable => { + "not known to be used; the install record could not be read" + } + EngineInstallStatus::NotRecorded => "not recorded as installed", + }; + let _ = writeln!(output, " status: {status}"); + if usage.reclaimable { + let _ = writeln!( + output, + " note: reclaimable; Lemonade installs into the active ROCm install's folder, so this copy is only used while no ROCm install is active. Nothing removes it automatically: delete its runtime, downloads and extract folders to free the space" + ); + } else if usage.status == EngineInstallStatus::Unused { + let _ = writeln!( + output, + " note: reused if this ROCm install becomes the active one; not marked reclaimable" + ); + } + if let Some(note) = usage.record_note.as_deref() { + let _ = writeln!(output, " note: {note}; not marked reclaimable"); + } + if usage.runtime_key.is_some() { + let note = if usage.removed_with_runtime { + "also counted in that ROCm install's size above, and removed with it" + } else { + "also counted in that ROCm install's size above; ROCm CLI does not delete that install's folder (external or imported), so uninstalling it leaves this copy" + }; + let _ = writeln!(output, " note: {note}"); + } + } + let _ = writeln!(output); let _ = writeln!(output, "ROCm CLI folders:"); for usage in &report.rocm_cli { @@ -959,6 +1181,11 @@ pub(crate) fn storage(command: Option) -> Result<()> { format_bytes(entry.size_bytes), path.display() ); + if let Some(line) = + crate::runtimes_cmd::lemonade_record_report_line(&result.lemonade_record) + { + println!(" lemonade: {line}"); + } } else { println!( "removed the registry entry for {}, but left its folder {} in place", @@ -1468,6 +1695,299 @@ mod tests { Ok(()) } + /// The release-test layout (ROCMAI-825): Lemonade installed into the data + /// folder, then a runtime activated and `rocm serve` installing a second full + /// copy inside it, with the record now pointing at the second. Both trees + /// must be listed with their sizes, and the data folder's flagged as unused. + #[test] + fn report_lists_both_lemonade_installs_and_flags_the_unused_one() -> Result<()> { + let (root, paths) = test_paths("report-lemonade"); + let runtime = install_fixture(&paths, manifest("in-use", "gfx1151", "7.14.1", 20), 4096)?; + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + let per_runtime = rocm_engine_lemonade::install_tree_root( + &paths, + Some(&runtime.install_root.join("engines")), + ); + crate::test_support::plant_lemonade_tree(&base, 3000)?; + crate::test_support::plant_lemonade_tree(&per_runtime, 5000)?; + crate::test_support::record_lemonade_install(&paths, &per_runtime)?; + // Record, logs and server state share the data folder's tree; they are + // not part of the install and must not be counted as reclaimable. + std::fs::create_dir_all(base.join("state"))?; + std::fs::write(base.join("state").join("big.log"), vec![0_u8; 100_000])?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + assert_eq!(report.engines.len(), 2, "{:?}", report.engines); + let base_row = &report.engines[0]; + assert_eq!(base_row.path, base); + assert_eq!(base_row.runtime_key, None); + assert_eq!(base_row.status, EngineInstallStatus::Unused); + assert!(base_row.size_bytes >= 3000 && base_row.size_bytes < 100_000); + let runtime_row = &report.engines[1]; + assert_eq!(runtime_row.path, per_runtime); + assert_eq!(runtime_row.runtime_key.as_deref(), Some("in-use")); + assert_eq!(runtime_row.status, EngineInstallStatus::InUse); + assert!(runtime_row.size_bytes >= 5000); + + let rendered = render_report(&report); + assert!( + rendered.contains("- lemonade (data folder): "), + "{rendered}" + ); + assert!( + rendered.contains("status: not used right now; Lemonade runs from another install"), + "{rendered}" + ); + assert!( + rendered.contains(&format!( + "- lemonade (inside ROCm install in-use): {} ({})", + format_bytes(runtime_row.size_bytes), + per_runtime.display() + )), + "{rendered}" + ); + assert!(rendered.contains("note: reclaimable"), "{rendered}"); + assert!(base_row.reclaimable && !runtime_row.reclaimable); + assert!( + rendered + .contains("also counted in that ROCm install's size above, and removed with it"), + "a runtime ROCm CLI owns is deleted with its copy:\n{rendered}" + ); + + // With no record at all, nothing is called unused: the next install may + // well reuse a current tree. + std::fs::remove_file(paths.engine_manifests_dir("lemonade").join("runtime.json"))?; + let report = build_report(&paths, &RocmCliConfig::default())?; + assert!( + report + .engines + .iter() + .all(|row| row.status == EngineInstallStatus::NotRecorded) + ); + assert!(!render_report(&report).contains("note: reclaimable")); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// Two registered runtimes that each hold a Lemonade copy, with the record + /// pointing at one. The other copy is unused right now but is reused as soon + /// as its runtime becomes active, so it must not be offered for deletion: + /// following that advice would force a ~25 GiB reinstall. Only the data + /// folder's copy, which no runtime owns, is reclaimable. + #[test] + fn a_copy_inside_another_registered_runtime_is_not_offered_for_deletion() -> Result<()> { + let (root, paths) = test_paths("report-lemonade-two-runtimes"); + let active = install_fixture(&paths, manifest("active", "gfx1151", "7.14.1", 20), 4096)?; + let idle = install_fixture(&paths, manifest("idle", "gfx1151", "7.13.0", 10), 4096)?; + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + let tree_of = |runtime: &therock::InstalledRuntimeManifest| { + rocm_engine_lemonade::install_tree_root( + &paths, + Some(&runtime.install_root.join("engines")), + ) + }; + crate::test_support::plant_lemonade_tree(&base, 3000)?; + crate::test_support::plant_lemonade_tree(&tree_of(&active), 5000)?; + crate::test_support::plant_lemonade_tree(&tree_of(&idle), 6000)?; + crate::test_support::record_lemonade_install(&paths, &tree_of(&active))?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + let row = |runtime_key: Option<&str>| { + report + .engines + .iter() + .find(|row| row.runtime_key.as_deref() == runtime_key) + .expect("the copy is listed") + }; + assert_eq!(row(Some("active")).status, EngineInstallStatus::InUse); + assert_eq!(row(Some("idle")).status, EngineInstallStatus::Unused); + assert!( + !row(Some("idle")).reclaimable, + "reused when `idle` is activated" + ); + assert!( + row(None).reclaimable, + "no runtime owns the data folder's copy" + ); + let rendered = render_report(&report); + assert_eq!( + rendered.matches("note: reclaimable").count(), + 1, + "only the data folder's copy is offered:\n{rendered}" + ); + assert!( + rendered.contains( + "note: reused if this ROCm install becomes the active one; not marked reclaimable" + ), + "{rendered}" + ); + // Reporting only: every copy is still on disk. + assert!(tree_of(&idle).join("runtime").exists() && base.join("runtime").exists()); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// A record that names a runtime folder holding no install (a runtime + /// uninstalled by a build that left the record behind) means nothing is in + /// use. The surviving data-folder copy is then the only one a later serve can + /// reuse, so the report must not call it unused or reclaimable: following that + /// advice would force a reinstall of roughly 25 GiB. + #[test] + fn a_record_naming_a_folder_with_no_install_does_not_make_the_survivor_reclaimable() + -> Result<()> { + let (root, paths) = test_paths("report-lemonade-dangling"); + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + crate::test_support::plant_lemonade_tree(&base, 3000)?; + let gone = paths + .data_dir + .join("runtimes") + .join("wheel") + .join("removed"); + crate::test_support::record_lemonade_install( + &paths, + &gone.join("engines").join("lemonade"), + )?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + assert_eq!(report.engines.len(), 1, "{:?}", report.engines); + let row = &report.engines[0]; + assert_eq!(row.status, EngineInstallStatus::RecordWithoutInstall); + let rendered = render_report(&report); + assert!(!rendered.contains("note: reclaimable"), "{rendered}"); + assert!(!rendered.contains("status: not used"), "{rendered}"); + assert!( + rendered.contains("install record names a folder with no install"), + "{rendered}" + ); + assert!( + rendered.contains("holds no install, so no copy is in use; not marked reclaimable") + ); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// An unparseable record says nothing about which copy works, so it is not + /// "no record" and no copy is called unused. + #[test] + fn an_unreadable_record_leaves_every_copy_unmarked() -> Result<()> { + let (root, paths) = test_paths("report-lemonade-unreadable"); + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + crate::test_support::plant_lemonade_tree(&base, 3000)?; + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + std::fs::create_dir_all(record.parent().expect("record has a parent"))?; + std::fs::write(&record, b"{ not json")?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + assert_eq!( + report.engines[0].status, + EngineInstallStatus::RecordUnreadable + ); + let rendered = render_report(&report); + assert!(!rendered.contains("note: reclaimable"), "{rendered}"); + assert!(rendered.contains("could not be read"), "{rendered}"); + assert!( + !rendered.contains("not recorded as installed"), + "{rendered}" + ); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// The status rule on its own, over every record state and tree position. + #[test] + fn engine_install_status_follows_the_record() { + use rocm_engine_lemonade::InstallRecord; + let tree = PathBuf::from("data").join("engines").join("lemonade"); + let inside = InstallRecord::Recorded { + runtime_dir: tree.join("runtime"), + holds_install: true, + }; + let elsewhere = InstallRecord::Recorded { + runtime_dir: PathBuf::from("elsewhere").join("runtime"), + holds_install: true, + }; + let empty = InstallRecord::Recorded { + runtime_dir: PathBuf::from("elsewhere").join("runtime"), + holds_install: false, + }; + assert_eq!( + classify_engine_install(&inside, &tree), + EngineInstallStatus::InUse + ); + assert_eq!( + classify_engine_install(&elsewhere, &tree), + EngineInstallStatus::Unused + ); + assert_eq!( + classify_engine_install(&empty, &tree), + EngineInstallStatus::RecordWithoutInstall + ); + // A record at this tree whose server binary is gone is not "in use". + let hollow = InstallRecord::Recorded { + runtime_dir: tree.join("runtime"), + holds_install: false, + }; + assert_eq!( + classify_engine_install(&hollow, &tree), + EngineInstallStatus::RecordWithoutInstall + ); + assert_eq!( + classify_engine_install(&InstallRecord::Unreadable("x".to_owned()), &tree), + EngineInstallStatus::RecordUnreadable + ); + assert_eq!( + classify_engine_install(&InstallRecord::NotRecorded, &tree), + EngineInstallStatus::NotRecorded + ); + } + + /// Uninstalling an external or imported runtime only unregisters it, so its + /// Lemonade copy stays. The row must not say it is "removed with" the + /// runtime; the claim comes from the rule uninstall itself applies. + #[test] + fn a_read_only_runtimes_lemonade_copy_is_not_said_to_be_removed_with_it() -> Result<()> { + let (root, paths) = test_paths("report-lemonade-read-only"); + let mut record = manifest("external", "gfx1151", "7.14.1", 20); + record.read_only = true; + let runtime = install_fixture(&paths, record, 4096)?; + let tree = rocm_engine_lemonade::install_tree_root( + &paths, + Some(&runtime.install_root.join("engines")), + ); + crate::test_support::plant_lemonade_tree(&tree, 5000)?; + crate::test_support::record_lemonade_install(&paths, &tree)?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + let row = report + .engines + .iter() + .find(|row| row.runtime_key.as_deref() == Some("external")) + .expect("the runtime's copy is listed"); + assert!(!row.removed_with_runtime); + let rendered = render_report(&report); + assert!(!rendered.contains("and removed with it"), "{rendered}"); + assert!( + rendered.contains("does not delete that install's folder (external or imported)"), + "{rendered}" + ); + // The claim matches what uninstall then does to that copy. + let mut config = RocmCliConfig::default(); + crate::runtimes_cmd::uninstall_runtime(&paths, &mut config, "external")?; + assert!(tree.join("runtime").exists(), "uninstall deleted the copy"); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + /// `rocm serve --managed` leaves one JSON record per launch, kept after the /// server exits, and no `rocm storage` command removes them. The report /// never named the folder, so a user asking what is on disk could not see diff --git a/apps/rocm/src/symptom_input.rs b/apps/rocm/src/symptom_input.rs new file mode 100644 index 0000000000..bca04651bc --- /dev/null +++ b/apps/rocm/src/symptom_input.rs @@ -0,0 +1,368 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `rocm diagnose --symptom-file `: reading the symptom text from a file, +//! or from standard input when the path is `-`, instead of from a shell +//! argument. +//! +//! A pasted error is hostile to every shell's argument parser: bash runs a +//! `$(...)` inside double quotes, PowerShell does the same and also treats a +//! leading `-` and a stray quote its own way, and cmd.exe has no single-quote at +//! all. Reading the text as data sidesteps all three, so the only rule this +//! module adds is how the bytes become the string `--symptom` would have +//! delivered: +//! +//! - one trailing line break (`\n` or `\r\n`) is removed. A heredoc, `echo`, +//! an editor's final newline and PowerShell's pipe to a native command each +//! add exactly one, and none of them is part of the paste. Only one: any +//! other whitespace is the user's own text and is passed through unchanged; +//! - a UTF-8 byte-order mark is dropped (PowerShell adds one when +//! `$OutputEncoding` is UTF-8 with a BOM, and `Out-File -Encoding utf8` writes +//! one), and text with a UTF-16 byte-order mark, which Windows PowerShell's +//! `>` writes, is decoded. On Windows `--symptom` arrives as UTF-16 and is +//! converted the same way, so the matcher sees the same string; +//! - anything else must be UTF-8. Input that is not, an unreadable path, and +//! input past [`MAX_SYMPTOM_BYTES`] are refused with a [`SymptomFileError`], +//! which the dispatcher turns into a usage error (exit code 2). +//! +//! When the path is `-` and standard input is a terminal, nothing has been +//! piped in, so the user is told on stderr how to paste the text and end it +//! ([`TERMINAL_PASTE_HINT`]); otherwise the command would wait silently. + +use std::fmt; +use std::io::{self, IsTerminal, Read, Write as _}; +use std::path::Path; + +/// The `--symptom-file` value that means standard input. +pub(crate) const STDIN_PATH: &str = "-"; + +/// The most a symptom may be. A whole engine log fits; a device or an endless +/// pipe passed by mistake does not get read into memory without bound. +pub(crate) const MAX_SYMPTOM_BYTES: u64 = 16 * 1024 * 1024; + +/// Printed on stderr when `--symptom-file -` finds standard input is a +/// terminal: the user is expected to paste now, and must be told how to finish. +/// Ctrl-D at the start of a line ends terminal input on Linux; on Windows the +/// console's end-of-input is Ctrl-Z followed by Enter. +pub(crate) const TERMINAL_PASTE_HINT: &str = "Paste the error text, then end it with Ctrl-D at the \ +start of a line (Ctrl-Z then Enter on Windows)."; + +/// Why a `--symptom-file` could not be turned into symptom text. +#[derive(Debug)] +pub(crate) enum SymptomFileError { + /// The file could not be opened or read. + Unreadable { source: String, error: io::Error }, + /// The bytes are not UTF-8 (or UTF-16 with a byte-order mark). + NotText { source: String, detail: String }, + /// More than [`MAX_SYMPTOM_BYTES`]. + TooLarge { source: String }, +} + +impl fmt::Display for SymptomFileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unreadable { source, error } => { + write!(f, "could not read the symptom from {source}: {error}") + } + Self::NotText { source, detail } => write!( + f, + "the symptom in {source} is not text: {detail}. Save it as UTF-8 text and try again" + ), + Self::TooLarge { source } => write!( + f, + "the symptom in {source} is larger than {} MiB; pass only the lines around the failure", + MAX_SYMPTOM_BYTES / (1024 * 1024) + ), + } + } +} + +impl std::error::Error for SymptomFileError {} + +/// `error` as the usage error clap would have raised for a bad +/// `--symptom-file` value: printed with `rocm diagnose`'s usage line, exit +/// code 2. The file is read after parsing, so clap cannot raise it itself. +pub(crate) fn usage_error(error: &SymptomFileError) -> clap::Error { + let mut command = ::command(); + // Building sets each subcommand's full `rocm diagnose` name for the usage line. + command.build(); + command + .find_subcommand_mut("diagnose") + .expect("the diagnose subcommand is defined") + .error( + clap::error::ErrorKind::ValueValidation, + format!("--symptom-file: {error}"), + ) +} + +/// The symptom text `rocm diagnose` matches on: `--symptom` as given, or the +/// contents of `--symptom-file` (clap makes the two mutually exclusive). +pub(crate) fn resolve( + symptom: Option, + symptom_file: Option<&Path>, +) -> Result, SymptomFileError> { + let Some(path) = symptom_file else { + return Ok(symptom); + }; + if path.as_os_str() == STDIN_PATH { + let stdin = io::stdin(); + if stdin.is_terminal() { + // A closed stderr must not turn the hint into a panic. + let _ = writeln!(io::stderr(), "{TERMINAL_PASTE_HINT}"); + } + return read_symptom(stdin.lock(), "standard input").map(Some); + } + let source = path.display().to_string(); + let file = std::fs::File::open(path).map_err(|error| SymptomFileError::Unreadable { + source: source.clone(), + error, + })?; + read_symptom(file, &source).map(Some) +} + +/// Read at most [`MAX_SYMPTOM_BYTES`] from `reader` and decode it as described +/// in the module docs. `source` names the input in an error. +fn read_symptom(reader: impl Read, source: &str) -> Result { + let mut bytes = Vec::new(); + reader + .take(MAX_SYMPTOM_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|error| SymptomFileError::Unreadable { + source: source.to_owned(), + error, + })?; + if bytes.len() as u64 > MAX_SYMPTOM_BYTES { + return Err(SymptomFileError::TooLarge { + source: source.to_owned(), + }); + } + let text = decode(bytes).map_err(|detail| SymptomFileError::NotText { + source: source.to_owned(), + detail, + })?; + Ok(trim_one_trailing_line_break(text)) +} + +/// UTF-8 (a leading byte-order mark dropped), or UTF-16 announced by its +/// byte-order mark. The error says where decoding failed, as an offset from the +/// start of the input. Takes the buffer by value so UTF-8 becomes the string in +/// place, without a second buffer. +fn decode(mut bytes: Vec) -> Result { + const UTF8_BOM: [u8; 3] = [0xEF, 0xBB, 0xBF]; + if bytes.starts_with(&UTF8_BOM) { + bytes.drain(..UTF8_BOM.len()); + return utf8(bytes, UTF8_BOM.len()); + } + if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) { + return utf16(rest, u16::from_le_bytes); + } + if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) { + return utf16(rest, u16::from_be_bytes); + } + utf8(bytes, 0) +} + +/// `bytes` as UTF-8. `skipped` is how many bytes of the input came before +/// them (a byte-order mark), so the offset in an error counts from the start +/// of the input, where the user can find the byte. +fn utf8(bytes: Vec, skipped: usize) -> Result { + String::from_utf8(bytes).map_err(|error| { + format!( + "it is not valid UTF-8 (invalid byte at offset {})", + skipped + error.utf8_error().valid_up_to() + ) + }) +} + +fn utf16(bytes: &[u8], unit: fn([u8; 2]) -> u16) -> Result { + let chunks = bytes.chunks_exact(2); + if !chunks.remainder().is_empty() { + return Err( + "it starts with a UTF-16 byte-order mark but has an odd number of bytes".into(), + ); + } + let units: Vec = chunks.map(|pair| unit([pair[0], pair[1]])).collect(); + String::from_utf16(&units) + .map_err(|_| "it starts with a UTF-16 byte-order mark but is not valid UTF-16".into()) +} + +/// Remove one trailing `\n` or `\r\n`, and nothing else. +fn trim_one_trailing_line_break(mut text: String) -> String { + if text.ends_with("\r\n") { + text.truncate(text.len() - 2); + } else if text.ends_with('\n') { + text.pop(); + } + text +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A paste every shell would mangle: double and single quotes, a command + /// substitution, a leading minus, a backtick and a percent sign. + const HOSTILE_PASTE: &str = + "-x \"HSA_STATUS_ERROR_INVALID_ISA\" it's $(touch pwned) `id` %PATH% & echo"; + + fn read(bytes: &[u8]) -> Result { + read_symptom(bytes, "test input") + } + + #[test] + fn a_paste_reaches_the_matcher_unchanged() { + assert_eq!(read(HOSTILE_PASTE.as_bytes()).unwrap(), HOSTILE_PASTE); + } + + #[test] + fn exactly_one_trailing_line_break_is_removed() { + assert_eq!(read(b"oom\n").unwrap(), "oom"); + assert_eq!(read(b"oom\r\n").unwrap(), "oom"); + assert_eq!(read(b"oom\n\n").unwrap(), "oom\n"); + assert_eq!(read(b"oom\r\n\r\n").unwrap(), "oom\r\n"); + // A bare `\r` is a progress-bar repaint, not a line ending a shell added. + assert_eq!(read(b"oom\r").unwrap(), "oom\r"); + assert_eq!(read(b" oom ").unwrap(), " oom "); + assert_eq!(read(b"").unwrap(), ""); + assert_eq!(read(b"a\nb\n").unwrap(), "a\nb"); + } + + #[test] + fn a_utf8_byte_order_mark_is_dropped() { + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(HOSTILE_PASTE.as_bytes()); + bytes.extend_from_slice(b"\r\n"); + assert_eq!(read(&bytes).unwrap(), HOSTILE_PASTE); + } + + #[test] + fn utf16_with_a_byte_order_mark_is_decoded() { + let text = format!("{HOSTILE_PASTE} \u{00e9}\u{1F600}\r\n"); + let mut le = vec![0xFF, 0xFE]; + let mut be = vec![0xFE, 0xFF]; + for unit in text.encode_utf16() { + le.extend_from_slice(&unit.to_le_bytes()); + be.extend_from_slice(&unit.to_be_bytes()); + } + let expected = format!("{HOSTILE_PASTE} \u{00e9}\u{1F600}"); + assert_eq!(read(&le).unwrap(), expected); + assert_eq!(read(&be).unwrap(), expected); + } + + #[test] + fn input_that_is_not_text_is_refused_with_where_it_failed() { + let error = read(b"HIP error \xff\xfe at").unwrap_err(); + assert!( + matches!(error, SymptomFileError::NotText { .. }), + "{error:?}" + ); + let message = error.to_string(); + assert!(message.contains("test input"), "{message}"); + assert!(message.contains("offset 10"), "{message}"); + assert!(message.contains("UTF-8"), "{message}"); + + // Behind a UTF-8 byte-order mark the offset still counts from the + // start of the input, so it points at the byte in the file. + let mut with_bom = vec![0xEF, 0xBB, 0xBF]; + with_bom.extend_from_slice(b"HIP error \xff at"); + let message = read(&with_bom).unwrap_err().to_string(); + assert!(message.contains("offset 13"), "{message}"); + + // A UTF-16 BOM with an odd tail, and one with a lone surrogate. + assert!(matches!( + read(&[0xFF, 0xFE, 0x41]), + Err(SymptomFileError::NotText { .. }) + )); + assert!(matches!( + read(&[0xFF, 0xFE, 0x00, 0xD8]), + Err(SymptomFileError::NotText { .. }) + )); + } + + #[test] + fn input_past_the_limit_is_refused_without_reading_it_all() { + let endless = io::repeat(b'a'); + let error = read_symptom(endless, "an endless pipe").unwrap_err(); + assert!( + matches!(error, SymptomFileError::TooLarge { .. }), + "{error:?}" + ); + assert!(error.to_string().contains("16 MiB"), "{error}"); + + let at_limit = vec![b'a'; MAX_SYMPTOM_BYTES as usize]; + assert_eq!(read(&at_limit).unwrap().len(), at_limit.len()); + } + + #[test] + fn a_missing_file_is_unreadable_and_names_the_path() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("no-such-error.txt"); + let error = resolve(None, Some(&missing)).unwrap_err(); + assert!( + matches!(error, SymptomFileError::Unreadable { .. }), + "{error:?}" + ); + assert!( + error.to_string().contains(&missing.display().to_string()), + "{error}" + ); + } + + #[test] + fn a_file_is_read_and_symptom_alone_passes_through() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("error.txt"); + std::fs::write(&path, format!("{HOSTILE_PASTE}\n")).unwrap(); + assert_eq!( + resolve(None, Some(&path)).unwrap().as_deref(), + Some(HOSTILE_PASTE) + ); + assert_eq!( + resolve(Some("as typed\n".into()), None).unwrap().as_deref(), + Some("as typed\n"), + "--symptom is never trimmed: only a file's trailing line break is" + ); + assert_eq!(resolve(None, None).unwrap(), None); + } + + // `-` alone means standard input, so a file literally named `-` is reached + // as `./-`, which is what the documentation tells the user. The two + // spellings are different paths, and a path that is not `-` is opened as a + // file whatever the working directory. + #[test] + fn a_file_named_dash_is_read_as_a_file_not_as_standard_input() { + assert_ne!(Path::new("./-").as_os_str(), STDIN_PATH); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("-"); + std::fs::write(&path, "from the file named dash\n").unwrap(); + assert_eq!( + resolve(None, Some(&path)).unwrap().as_deref(), + Some("from the file named dash") + ); + } + + #[test] + fn a_bad_file_is_a_usage_error_naming_the_command() { + let error = usage_error(&SymptomFileError::TooLarge { + source: "standard input".into(), + }); + assert_eq!(error.exit_code(), 2); + let rendered = error.render().to_string(); + assert!( + rendered.contains("--symptom-file: the symptom in standard input is larger"), + "{rendered}" + ); + assert!(rendered.contains("Usage: rocm diagnose"), "{rendered}"); + } + + #[test] + fn a_directory_is_unreadable_not_a_panic() { + let dir = tempfile::tempdir().unwrap(); + let error = resolve(None, Some(dir.path())).unwrap_err(); + assert!( + matches!(error, SymptomFileError::Unreadable { .. }), + "{error:?}" + ); + } +} diff --git a/apps/rocm/src/system_packages.rs b/apps/rocm/src/system_packages.rs index 4e3b5b78b6..22d7dcde32 100644 --- a/apps/rocm/src/system_packages.rs +++ b/apps/rocm/src/system_packages.rs @@ -13,8 +13,9 @@ //! `rocm install sdk` (`SdkInstallConsents`) stays in `runtime_install_finalize.rs`, //! which passes the plain `system_packages` bool in. That module re-exports //! `ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch` and -//! `ensure_libnuma_for_torch` for `install_cmd.rs` and `engines_cmd.rs`, and -//! `driver_install.rs` reads `real_os_release` from here directly. +//! `ensure_libnuma_for_torch` for `install_cmd.rs` and `engines_cmd.rs`. Every plan +//! here reads the real host's os-release through +//! `rocm_core::hardware_root::real_os_release`. use std::process::{Command as ProcessCommand, Stdio}; @@ -62,18 +63,6 @@ const fn system_package_install_action( } } -/// The real host's `/etc/os-release`, falling back to `/usr/lib/os-release` -/// when the former is absent (per `os-release(5)`). -/// -/// Deliberately not routed through `rocm_core::host_path`: every caller builds -/// a package-manager plan (`apt`, `dnf`, `zypper`) that then runs against the -/// real machine, so a simulated host root must not choose its distro. The descriptive distro name `examine` reports comes from -/// `ExamineSummary::gather`'s `distro` field, which does follow a simulated -/// root. -pub(crate) fn real_os_release() -> rocm_core::hardware_root::OsRelease { - rocm_core::hardware_root::read_os_release(rocm_core::hardware_root::identity_path_for_str) -} - /// Run a command given as an argv vector directly, without going through a shell. /// /// Used for [`run_system_package_install_plan`], whose commands are modeled as @@ -117,7 +106,7 @@ pub(crate) fn ensure_openmpi_for_vllm(approved: bool) -> Result<()> { } let plan = rocm_core::openmpi::plan_for_os_release_read( - &real_os_release(), + &rocm_core::hardware_root::real_os_release(), rocm_core::openmpi::build_openmpi_install_plan, ); @@ -291,7 +280,10 @@ fn ensure_torch_runtime_dep(approved: bool, dep: &TorchRuntimeDep) { return; } - let plan = rocm_core::openmpi::plan_for_os_release_read(&real_os_release(), dep.build_plan); + let plan = rocm_core::openmpi::plan_for_os_release_read( + &rocm_core::hardware_root::real_os_release(), + dep.build_plan, + ); println!("{} setup", dep.name); println!(" reason: {}", dep.reason); @@ -440,7 +432,6 @@ fn run_system_package_install_plan( #[cfg(test)] mod tests { - #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] use std::fs; use super::*; @@ -533,8 +524,10 @@ mod tests { /// The package-manager plans run against the real machine, so a simulated /// host root must not change the distro they are built for: under a root - /// whose `/etc/os-release` names a different distro, `real_os_release` - /// still returns the real file. + /// whose `/etc/os-release` names a different distro, + /// `rocm_core::hardware_root::real_os_release` still returns the real file. + /// Every caller (the driver, OpenMPI, libatomic and libnuma plans and hints) + /// reads through that one function. #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] #[test] fn real_os_release_ignores_a_simulated_host_root() { @@ -558,7 +551,7 @@ mod tests { rocm_core::hardware_root::TEST_HOST_ROOT_ENV, &root.to_string_lossy(), ); - real_os_release() + rocm_core::hardware_root::real_os_release() }; fs::remove_dir_all(&root).ok(); @@ -569,8 +562,8 @@ mod tests { ); // Independent of `hardware_root::read_os_release`: rather than call // the exact same helper with the same identity resolver `real_os_release` - // wraps one line away (which would pass even if its `.context(...)` - // call or resolver closure broke), read both candidate paths directly + // wraps (which would pass even if the resolver it passes broke), read + // both candidate paths directly // and apply the same existence-only fallback rule inline. This mirrors // which variant is correct for each case, rather than accepting // `Absent` or `Unreadable` interchangeably — see `ExpectedOsRelease`'s @@ -605,4 +598,99 @@ mod tests { ), } } + + /// Every package-manager plan this module and `driver_install.rs` build is + /// written for the real machine, so they all read its os-release through + /// `rocm_core::hardware_root::real_os_release`, whose own test + /// (`real_os_release_ignores_a_simulated_host_root`) proves it ignores a + /// simulated host root. A caller that read the file with `read_os_release` + /// and its own resolver would be a second reader that test does not cover, so + /// no source file of this binary may name `read_os_release` or the identity + /// resolver: the one place that does is `hardware_root.rs` in `rocm-core`. + #[test] + fn no_caller_in_this_binary_reads_the_real_os_release_by_hand() { + fn rust_files(dir: &std::path::Path, out: &mut Vec) { + for entry in fs::read_dir(dir).expect("a readable source directory") { + let path = entry.expect("a readable entry").path(); + if path.is_dir() { + rust_files(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } + } + + // Spelled in two halves so this file does not match its own scan. + let needles = [ + ["read_os_", "release("].concat(), + ["identity_path_", "for_str"].concat(), + ]; + let mut files = Vec::new(); + rust_files( + &std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), + &mut files, + ); + assert!(!files.is_empty(), "the scan found no source files"); + let offenders: Vec = files + .iter() + .filter(|path| { + let text = fs::read_to_string(path).expect("a readable source file"); + needles.iter().any(|needle| text.contains(needle.as_str())) + }) + .map(|path| path.display().to_string()) + .collect(); + assert!( + offenders.is_empty(), + "these files read os-release by hand instead of through \ + rocm_core::hardware_root::real_os_release: {offenders:?}" + ); + } + + /// The OpenMPI, libatomic and libnuma install hints (`rocm_core::openmpi`) + /// name package-manager commands for the machine the process runs on, like + /// the plans in this module, so a simulated host root must not choose their + /// distro either: with a root whose `/etc/os-release` names another + /// distribution, each hint is still the one the real host's file produces. + /// A hint that read through `host_path` would be the planted distro's (the + /// manual advice, as `planted-distro` maps to no package manager). + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn the_install_hints_read_the_real_host_and_ignore_a_simulated_root() { + let root = std::env::temp_dir().join(format!( + "rocm-os-release-hint-root-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + fs::create_dir_all(root.join("etc")).expect("create planted /etc"); + fs::write(root.join("etc/os-release"), "ID=planted-distro\n") + .expect("write planted os-release"); + + let hints = || { + [ + rocm_core::openmpi::install_hint(), + rocm_core::openmpi::libatomic_install_hint(), + rocm_core::openmpi::libnuma_install_hint(), + ] + }; + let (real, simulated) = { + let mut env = ScopedTestEnv::new(); + let real = hints(); + env.set( + rocm_core::hardware_root::TEST_HOST_ROOT_ENV, + &root.to_string_lossy(), + ); + (real, hints()) + }; + fs::remove_dir_all(&root).ok(); + + assert_eq!(simulated, real); + if !real[0].plan_installs { + // A host that maps to no package manager reads the same as the + // planted file, so the assertion above held without telling which + // one was read. + eprintln!("host os-release names no known distribution; nothing proven"); + } + } } diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index c724ad973f..9227ee2e41 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -9,7 +9,7 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use anyhow::Result; -use rocm_core::AppPaths; +use rocm_core::{AppPaths, AutomationRuntimeState}; use crate::runtimes_cmd::{runtime_manifest_path, runtime_registry_dir}; use crate::therock; @@ -50,39 +50,14 @@ pub(crate) fn unique_suffix() -> String { ) } -/// Case count for a property test: `PROPTEST_CASES` (proptest's own knob, -/// which an explicit `with_cases` would otherwise override) when it is set and -/// above `default`, otherwise `default`. -/// -/// Never below `default`: the counts are chosen so a property catches the -/// defects it targets with near certainty, and some of those are hit by a -/// small share of cases, so a lower count would pass with them present. `0` -/// would pass vacuously. -pub(crate) fn proptest_cases(default: u32) -> u32 { - case_count_from_env("PROPTEST_CASES", default) -} - -/// Case count read from `var`, raising `default` but never lowering it, for -/// the same reason as [`proptest_cases`]. The uninstall properties use it with -/// their own knob, `ROCM_DELETION_PROP_CASES`. A value that is ignored, -/// because it is lower or does not parse, is reported on stderr. -pub(crate) fn case_count_from_env(var: &str, default: u32) -> u32 { - let requested = std::env::var(var).ok(); - let cases = raised_case_count(requested.as_deref(), default); - if let Some(requested) = requested - && requested.parse::().ok() != Some(cases) - { - eprintln!("{var}={requested} ignored: this property runs at least {default} cases"); - } - cases -} - -/// The pure half of [`case_count_from_env`]. -fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { - requested - .and_then(|value| value.parse().ok()) - .map_or(default, |requested: u32| requested.max(default)) -} +// The case-count rule for property tests that set their own count, shared with +// `rocm-core`, its integration tests and the Lemonade engine rather than copied. +#[path = "../../../crates/rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; +// Only the Unix-only uninstall properties read their own knob. +#[cfg(unix)] +pub(crate) use proptest_cases::case_count_from_env; /// Split `bytes` at the given (unsorted, possibly duplicated) offsets, each /// taken modulo `bytes.len() + 1`. @@ -229,6 +204,243 @@ pub(crate) fn write_test_pip_runtime( Ok(manifest) } +/// A `rocm serve` argv the serve safety policy refuses, with the refusal each +/// surface reports for it. +/// +/// Two surfaces enforce the policy on a model-proposed `serve`: the chat tool +/// policy (`tool_policy::validate_chat_rocm_command_safety`) and the +/// provider-assisted planner (`freeform_planner::validate_provider_planner_tool_call`). +/// Both suites run this one table, so a rule missing from either surface fails +/// that surface's tests, and the same argv cannot be refused by one for a +/// different reason than the other without the table saying so. +pub(crate) struct ServeSafetyCase { + /// The rule the argv breaks. + pub(crate) rule: &'static str, + pub(crate) args: &'static [&'static str], + /// The full message the chat tool policy refuses it with. + pub(crate) chat_error: &'static str, + /// The full message the planner refuses it with. + pub(crate) planner_error: &'static str, +} + +/// Every refusal the serve safety policy has, each rule reached by more than one +/// spelling of its trigger. A case breaks exactly one rule, so the message names +/// the rule that fired and nothing else could have. +pub(crate) const SERVE_SAFETY_REFUSALS: &[ServeSafetyCase] = &[ + ServeSafetyCase { + rule: "public bind", + args: &[ + "serve", + "qwen", + "--engine", + "vllm", + "--allow-public-bind", + "--managed", + ], + chat_error: "local assistant cannot request public network binding", + planner_error: "provider planner cannot request public network binding", + }, + ServeSafetyCase { + rule: "public bind", + args: &["serve", "qwen", "--managed", "--allow-public-bind"], + chat_error: "local assistant cannot request public network binding", + planner_error: "provider planner cannot request public network binding", + }, + ServeSafetyCase { + rule: "cpu device", + args: &[ + "serve", + "qwen", + "--engine", + "vllm", + "--device", + "cpu", + "--managed", + ], + chat_error: "local assistant cannot request CPU execution; ROCm GPU execution is required", + planner_error: "provider planner cannot request CPU execution; ROCm GPU execution is required", + }, + ServeSafetyCase { + rule: "cpu device", + args: &["serve", "qwen", "--device=cpu", "--managed"], + chat_error: "local assistant cannot request CPU execution; ROCm GPU execution is required", + planner_error: "provider planner cannot request CPU execution; ROCm GPU execution is required", + }, + ServeSafetyCase { + rule: "cpu device", + args: &["serve", "qwen", "--device", "cpu_only", "--managed"], + chat_error: "local assistant cannot request CPU execution; ROCm GPU execution is required", + planner_error: "provider planner cannot request CPU execution; ROCm GPU execution is required", + }, + ServeSafetyCase { + rule: "non-loopback host", + args: &["serve", "qwen", "--host", "0.0.0.0", "--managed"], + chat_error: "local assistant cannot request non-local host `0.0.0.0`", + planner_error: "provider planner cannot request non-local host `0.0.0.0`", + }, + ServeSafetyCase { + rule: "non-loopback host", + args: &["serve", "qwen", "--host=example.test", "--managed"], + chat_error: "local assistant cannot request non-local host `example.test`", + planner_error: "provider planner cannot request non-local host `example.test`", + }, + ServeSafetyCase { + // Classified public on purpose: only the three exact loopback spellings + // pass (see `is_public_bind_host`). + rule: "non-loopback host", + args: &["serve", "qwen", "--host", "LOCALHOST", "--managed"], + chat_error: "local assistant cannot request non-local host `LOCALHOST`", + planner_error: "provider planner cannot request non-local host `LOCALHOST`", + }, + ServeSafetyCase { + rule: "managed required", + args: &["serve", "qwen", "--engine", "vllm"], + chat_error: "local assistant must request managed serving with --managed", + planner_error: "provider planner must request managed serving with --managed", + }, + ServeSafetyCase { + rule: "managed required", + args: &["serve", "qwen", "--engine", "vllm", "--foreground"], + chat_error: "local assistant must request managed serving with --managed", + planner_error: "provider planner must request managed serving with --managed", + }, +]; + +/// `--foreground` next to `--managed`: the safety check refuses the pair, but +/// clap does too (the flags conflict), so the planner never reaches its safety +/// check with it and only the chat check is what refuses it first. +pub(crate) const SERVE_FOREGROUND_AND_MANAGED: &[&str] = + &["serve", "qwen", "--managed", "--foreground"]; + +/// `rocm serve` argvs the serve safety policy lets through on both surfaces, and +/// that also parse as a `rocm` command, so the planner reaches its safety check +/// with them. A host other than `127.0.0.1` carries an explicit `--port`, which +/// the chat policy requires there (see [`SERVE_PORT_RULE_CHAT_ONLY`]). +pub(crate) const SERVE_SAFETY_ACCEPTED: &[&[&str]] = &[ + &["serve", "qwen", "--managed"], + &[ + "serve", + "qwen", + "--engine", + "vllm", + "--device", + "gpu_required", + "--managed", + ], + &["serve", "qwen", "--device=gpu_required", "--managed"], + &["serve", "qwen", "--host", "127.0.0.1", "--managed"], + &[ + "serve", + "qwen", + "--host=localhost", + "--port", + "8000", + "--managed", + ], + &["serve", "qwen", "--host", "::1", "--port=8000", "--managed"], +]; + +/// `rocm serve` argvs the planner lets through but the chat policy refuses: a +/// loopback host other than `127.0.0.1` without an explicit `--port`. The chat +/// policy applies `rocm serve`'s own port rule so that an approval card is never +/// offered for a command the CLI then refuses; the planner's check has no port +/// rule. Each entry is the argv and the full message the chat policy refuses it +/// with. +pub(crate) const SERVE_PORT_RULE_CHAT_ONLY: &[(&[&str], &str)] = &[ + ( + &["serve", "qwen", "--host=localhost", "--managed"], + "`rocm serve --host localhost` needs an explicit `--port <1-65535>`. Automatic port selection is supported only on the default host 127.0.0.1, where ROCm can prove a port is free before launching; on any other address the engine owns the bind.", + ), + ( + &["serve", "qwen", "--host", "::1", "--managed"], + "`rocm serve --host ::1` needs an explicit `--port <1-65535>`. Automatic port selection is supported only on the default host 127.0.0.1, where ROCm can prove a port is free before launching; on any other address the engine owns the bind.", + ), +]; + +/// `args` as the owned argv both surfaces take. +pub(crate) fn owned_argv(args: &[&str]) -> Vec { + args.iter().map(|arg| (*arg).to_owned()).collect() +} + +/// Build an `AutomationRuntimeState` for the background-helper tests (the +/// no-double-spawn guard in `background_helper.rs` and the autostart tests in +/// `service_launch/helper_autostart.rs`). +pub(crate) fn runtime_state(running: bool, daemon_pid: u32) -> AutomationRuntimeState { + AutomationRuntimeState { + running, + automations_enabled: true, + daemon_pid, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: Vec::new(), + } +} + +/// Put the on-disk facts in one of the four combinations: whether a watcher +/// is enabled, and whether the helper is running (this process stands in for +/// a live daemon). +pub(crate) fn set_helper_facts(paths: &AppPaths, watcher_enabled: bool, helper_running: bool) { + let mut config = rocm_core::RocmCliConfig::default(); + config.watcher_config_mut("server-recover").enabled = watcher_enabled; + config.save(paths).expect("save config"); + runtime_state(helper_running, std::process::id()) + .write(paths) + .expect("write runtime state"); +} + +/// Leave `paths` as `rocm setup` followed by a lost registry entry would: a ready +/// wheel runtime whose install tree holds its own `.rocm-cli-runtime.json` and +/// that `config.json` names as the setup folder, but that the runtime registry +/// has no entry for. Only `recover_setup_runtime_registration` can make it +/// visible again, so a selection that finds it has run that recovery. +pub(crate) fn write_test_setup_runtime_missing_from_registry( + paths: &AppPaths, + runtime_key: &str, +) -> Result { + let manifest = write_test_pip_runtime( + paths, + runtime_key, + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + fs::remove_file(runtime_manifest_path(paths, runtime_key))?; + let mut config = rocm_core::RocmCliConfig::default(); + config.setup.therock_venv = Some(manifest.install_root.clone()); + config.save(paths)?; + Ok(manifest) +} + +/// Make the runtime registry impossible to write for any user, an administrator +/// included: a plain file stands where its directory should be, so listing the +/// registry finds nothing and creating it fails. A read-only directory would +/// not do, because it does not stop `root`, which is who the e2e suite can run as. +/// Returns the path of that file, so a test can show it was left as it was. +pub(crate) fn make_runtime_registry_unwritable(paths: &AppPaths) -> Result { + let registry = runtime_registry_dir(paths); + let _ = fs::remove_dir_all(®istry); + if let Some(parent) = registry.parent() { + fs::create_dir_all(parent)?; + } + fs::write(®istry, "not a directory")?; + Ok(registry) +} + +/// File names in the runtime registry directory, sorted; empty when the +/// directory does not exist. Lets a test assert that a selection did, or did +/// not, write a registry entry. +pub(crate) fn runtime_registry_entry_names(paths: &AppPaths) -> Vec { + let mut names: Vec = fs::read_dir(runtime_registry_dir(paths)) + .into_iter() + .flatten() + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + mod tests { use super::*; @@ -238,13 +450,44 @@ mod tests { (0..64).map(|_| unique_suffix()).collect(); assert_eq!(suffixes.len(), 64, "two calls must never share a suffix"); } +} - #[test] - fn case_counts_are_raised_but_never_lowered() { - assert_eq!(raised_case_count(None, 256), 256); - assert_eq!(raised_case_count(Some("4096"), 256), 4096); - assert_eq!(raised_case_count(Some("16"), 256), 256); - assert_eq!(raised_case_count(Some("0"), 256), 256); - assert_eq!(raised_case_count(Some("lots"), 256), 256); - } +/// Plant an extracted Lemonade tree under `tree_root` the way an install leaves +/// one: the server binary and `payload_bytes` of backend files. Black-box on +/// purpose (plain files, not the engine's own writer), so a test reads it the +/// way the CLI reads a real install. +pub(crate) fn plant_lemonade_tree(tree_root: &std::path::Path, payload_bytes: usize) -> Result<()> { + let runtime_dir = tree_root.join("runtime"); + let backend = runtime_dir.join("bin").join("llamacpp").join("rocm-stable"); + fs::create_dir_all(&backend)?; + let lemond = if rocm_core::runtime_is_windows() { + "lemond.exe" + } else { + "lemond" + }; + fs::write(runtime_dir.join(lemond), b"lemond")?; + fs::write(backend.join("payload.bin"), vec![0_u8; payload_bytes])?; + Ok(()) +} + +/// Write Lemonade's install record pointing at `tree_root`'s runtime, in the +/// schema the engine writes (`/engines/lemonade/manifests/runtime.json`). +pub(crate) fn record_lemonade_install(paths: &AppPaths, tree_root: &std::path::Path) -> Result<()> { + let runtime_dir = tree_root.join("runtime"); + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + fs::create_dir_all(record.parent().expect("record has a parent"))?; + fs::write( + &record, + serde_json::to_vec_pretty(&serde_json::json!({ + "env_id": "lemonade-embeddable-test", + "version": "test", + "runtime_dir": runtime_dir, + "lemond": runtime_dir.join("lemond"), + "lemonade": runtime_dir.join("lemonade"), + "backend_recipe": "llamacpp", + "backend_name": "rocm", + "installed_at_unix_ms": 0, + }))?, + )?; + Ok(()) } diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index e31b255aac..804e26a093 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -13,6 +13,7 @@ mod consent; mod family; mod fetch; mod io; +mod kept_torch; mod probe; mod process; mod python; @@ -22,6 +23,7 @@ mod source; mod test_support; mod update; mod version; +mod wheel_plan; use self::consent::{ SdkInstallApproval, active_default_runtime_relation, confirm_overwrite_existing_sdk, @@ -33,35 +35,41 @@ pub(crate) use self::consent::{ use self::family::{family_override_or_recovered_arch, resolve_family}; pub(crate) use self::fetch::truthy_env; use self::fetch::{download_file, preflight_tarball_space}; -pub(crate) use self::io::write_file_atomically; use self::probe::probe_rocm_sdk_runtime_for_target; pub(crate) use self::probe::{ RuntimeDeviceProbe, TorchAlignmentProbe, install_pinned_package, probe_rocm_sdk_runtime, probe_runtime_devices, probe_torch_alignment, requirement_pinned_version, split_local_version, }; use self::process::{extract_tarball_and_discard_archive, progress_line, run_uv_progress_command}; +pub(crate) use self::python::NEXT_MANAGED_PYTHON_VERSION; use self::python::{ ensure_uv_venv, managed_python_version, python_for_install, python_requirement, + python_requirement_for_version, }; use self::resolve::{ - AggregateDeviceTarget, canonical_auto_select_needs_next_python, resolve_pip_runtime, - resolve_tarball_artifact, unverified_torchaudio_note, wheel_runtime_composition, + AggregateDeviceTarget, PipRuntimePlan, canonical_auto_select_needs_next_python, + preview_pip_runtime, resolve_pip_runtime, resolve_tarball_artifact, unverified_torchaudio_note, + wheel_runtime_composition, }; pub(crate) use self::source::RuntimeVersionSelector; #[cfg(test)] use self::source::resolve_source; use self::source::{ - SourceLayout, THEROCK_NEXT_MIN_MAJOR, TheRockChannel, next_layout_requested, - requested_source_layout, select_source_layout, + SourceLayout, THEROCK_NEXT_NIGHTLY_FIRST_BUILD_DATE, TheRockChannel, gate_install_layout, + layout_serving, requested_source_layout, }; pub(crate) use self::update::{ - maybe_refresh_startup_update_check, render_update_json, render_update_report, - runtime_update_plan, + RuntimeUpdatePlan, maybe_refresh_startup_update_check, recorded_python_version, + render_update_json, render_update_report, runtime_update_check, runtime_update_plan, }; pub(crate) use self::version::runtime_version_display; use self::version::{ parse_host_version, parse_version_for_ordering, runtime_version_build_date, version_relation, }; +use self::wheel_plan::{ + PlanStack, ResolvedStack, UNRESOLVED_PROGRESS, UnresolvedPreview, WheelPlan, + unresolved_wheel_preview, write_dry_run_tail, write_wheel_plan, +}; #[cfg(test)] pub(crate) use crate::runtime_registry::RocmSdkLibraryProbe; #[cfg(test)] @@ -71,14 +79,16 @@ pub(crate) use crate::runtime_registry::{ validate_rocm_sdk_runtime_probe, }; use crate::runtime_registry::{ - runtime_manifest_path, save_runtime_manifest, wheel_composition_includes_devel, + ManifestOverwrite, runtime_manifest_path, wheel_composition_includes_devel, + write_runtime_manifest, }; +#[cfg(test)] use crate::shell_quote::shell_quote; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, detect_host_gpu_diagnostics, detect_legacy_rocm_summary, ensure_uv_binary, interactive_terminal, runtime_is_windows, runtime_python_executable_in_env, unix_time_millis, - uv_pip_install_base, uv_venv_args, + uv_pip_install_base, }; #[cfg(test)] use rocm_core::{disk_space, managed_uv_cache_dir, normalize_therock_family}; @@ -136,6 +146,9 @@ struct InstallSourceOverride<'a> { family: Option<&'a str>, device_target: Option<&'a str>, layout: Option, + /// `--python-version`: the interpreter a ROCm 10+ install runs on in place + /// of the layout's default. + python_version: Option<&'a str>, } /// What to install, as resolved from the `rocm install sdk` arguments. @@ -150,6 +163,9 @@ pub(crate) struct SdkInstallRequest<'a> { pub prefix: Option, pub version_selector: Option, pub family_override: Option<&'a str>, + /// `--python-version`: the Python a ROCm 10+ install runs on, when not the + /// default one. Only a wheel install has a Python to choose. + pub python_version: Option<&'a str>, pub dry_run: bool, /// Install the compiler and headers alongside the runtime libraries. pub include_devel: bool, @@ -170,6 +186,7 @@ impl Default for SdkInstallRequest<'_> { prefix: None, version_selector: None, family_override: None, + python_version: None, dry_run: false, include_devel: false, // Withholding consent is the safe default: a caller that leaves it @@ -191,11 +208,15 @@ pub(crate) fn install_sdk( prefix, version_selector, family_override, + python_version, dry_run, include_devel, consent, } = request; let channel = TheRockChannel::parse(channel)?; + if python_version.is_some() && format != "wheel" { + bail!("--python-version applies only to wheel installs; a tarball install uses no Python"); + } ensure_install_format_supported(format)?; match format { "wheel" => install_wheel_runtime( @@ -204,6 +225,7 @@ pub(crate) fn install_sdk( prefix, InstallSourceOverride { family: family_override, + python_version, ..InstallSourceOverride::default() }, version_selector.as_ref(), @@ -213,14 +235,15 @@ pub(crate) fn install_sdk( ), "tarball" => { // A tarball catalog lists whole archives, not a resolvable package - // graph, so there is nothing to pin a version against — except the - // ROCm 10 catalog, which is reachable *only* by naming a version, - // because nothing else distinguishes it from the canonical one. + // graph, so there is nothing to pin a version against — except in + // the next layout's catalogs, whose archives are named by the + // version they carry. The canonical catalogs do not support a pin, + // so an earlier version is a wheel install. if let Some(selector) = version_selector.as_ref() - && !next_layout_requested(selector) + && layout_serving(channel, Some(selector)) == SourceLayout::Canonical { bail!( - "specific TheRock version selection for tarball installs is only supported for a stable ROCm {THEROCK_NEXT_MIN_MAJOR}+ pin (e.g. `--version 10.0.0`); for any other version, use `--format wheel`" + "specific TheRock version selection for tarball installs is supported only for the newer package layout: a stable ROCm 10+ release (e.g. `--version 10.0.0`) or a nightly built on or after {THEROCK_NEXT_NIGHTLY_FIRST_BUILD_DATE}; for any earlier version, use `--format wheel`" ) } install_tarball_runtime( @@ -261,6 +284,7 @@ pub(crate) fn install_sdk_for_update( device_target: Option<&str>, source_layout_generation: Option<&str>, include_devel: bool, + python_version: Option<&str>, dry_run: bool, activate_after_install: bool, ) -> Result { @@ -279,22 +303,28 @@ pub(crate) fn install_sdk_for_update( family: Some(family), device_target, layout, + python_version, }, None, dry_run, include_devel, consent, ), - "tarball" => install_tarball_runtime( - paths, - channel, - None, - Some(family), - None, - layout, - dry_run, - consent, - ), + "tarball" => { + if python_version.is_some() { + bail!("a tarball install uses no Python, so it has no --python-version"); + } + install_tarball_runtime( + paths, + channel, + None, + Some(family), + None, + layout, + dry_run, + consent, + ) + } other => bail!("unsupported install format: {other}"), } } @@ -327,6 +357,7 @@ fn install_wheel_runtime( family: family_override, device_target: device_target_override, layout: layout_override, + python_version: python_version_override, } = source_override; // `device_target_override` (an update apply's exact recorded arch) is only // otherwise consulted after `resolve_pip_runtime` returns — too late for the @@ -339,29 +370,34 @@ fn install_wheel_runtime( // channel, or a grouped family with no exact arch) before paying for Python // provisioning below, which can mean a real download for a layout this host // or channel can never actually use. - select_source_layout( + gate_install_layout( channel, &resolve_family(paths, recovered_family_override.as_deref())?, version_selector, + layout_override, )?; // The interpreter has to be picked before `resolve_pip_runtime`, which // selects wheels using that interpreter's own tags. `requested_source_layout` - // only answers from the *request*: an explicit stable ROCm 10+ pin already - // routes to `Next` and gets `cp314` that way, but a plain auto-select or a - // nightly build-date/prerelease pin always reports `Canonical`, even once - // the canonical/nightly feed's own "latest" has rotated past that major on - // its own, with no `Next` routing involved. `canonical_auto_select_needs_next_python` - // peeks the channel's own `rocm` listing to catch that case. - let python_requirement = match requested_source_layout(layout_override, version_selector) { - SourceLayout::Next => python_requirement(SourceLayout::Next), + // only answers from the *request*: a plain auto-select, a stable ROCm 10+ + // pin and a nightly built since the next layout began all route to `Next` + // and get `cp314` that way. A pin into the canonical layout can still + // resolve a ROCm 10+ build on that feed (the legacy nightlies carry ROCm 10 + // alphas), so `canonical_auto_select_needs_next_python` peeks the feed's own + // `rocm` listing to catch that case. + let python_layout = match requested_source_layout(channel, layout_override, version_selector) { + SourceLayout::Next => SourceLayout::Next, SourceLayout::Canonical => { if canonical_auto_select_needs_next_python(paths, channel, version_selector)? { - python_requirement(SourceLayout::Next) + SourceLayout::Next } else { - python_requirement(SourceLayout::Canonical) + SourceLayout::Canonical } } }; + let python_requirement = match python_version_override { + Some(version) => python_requirement_for_version(python_layout, version)?, + None => python_requirement(python_layout), + }; // A preview promises nothing here: whether a Python would be prepared, and // which, is only known once the search below has run (a version override can // make the answer a refusal), and is reported then. @@ -384,24 +420,55 @@ fn install_wheel_runtime( "Checking TheRock {} packages for this AMD GPU...", channel.as_str() )); - let resolution = resolve_pip_runtime( - paths, - channel, - recovered_family_override.as_deref(), - &wheel_compatibility, - version_selector, - layout_override, - include_devel, - )?; - let device_target = device_target_override.map_or_else( - || resolution.device_target.clone(), - |target| { - AggregateDeviceTarget::resolve( - Some(target), - &resolution.family, - &resolution.published_device_targets, - ) - }, + // A preview must not fetch `uv` either: on the ROCm 10 layout resolving the + // package versions runs `uv pip compile`, and a real install gets the managed + // `uv` for that by downloading it. A preview uses a `uv` that is already there + // and otherwise stops with what it can know without one, which says so. A real + // install and an update resolve, and so fetch, exactly as they always did. + let resolution = if dry_run { + match preview_pip_runtime( + paths, + channel, + recovered_family_override.as_deref(), + &wheel_compatibility, + version_selector, + layout_override, + include_devel, + )? { + PipRuntimePlan::Resolved(resolution) => resolution, + PipRuntimePlan::NeedsUv(unresolved) => { + progress_line(UNRESOLVED_PROGRESS); + let host_version_newer = host_rocm_version_newer_than(&unresolved.rocm_version); + return Ok(SdkInstallResult::plan(unresolved_wheel_preview( + &UnresolvedPreview { + channel, + version_selector, + unresolved: &unresolved, + device_target_override, + prefix: prefix.as_deref(), + host_version_newer: host_version_newer.as_deref(), + python: &python, + wheel_compatibility: &wheel_compatibility, + }, + ))); + } + } + } else { + resolve_pip_runtime( + paths, + channel, + recovered_family_override.as_deref(), + &wheel_compatibility, + version_selector, + layout_override, + include_devel, + )? + }; + let device_target = AggregateDeviceTarget::prefer_recorded( + &resolution.device_target, + device_target_override, + &resolution.family, + &resolution.published_device_targets, ); // The exact device payload, not the version alone, decides what this runtime // can run, so it is what identifies the runtime. A preview on a host with no @@ -417,146 +484,53 @@ fn install_wheel_runtime( let install_root = resolved_install_root(paths, "wheel", &runtime_key, prefix); let manifest_path = runtime_manifest_path(paths, &runtime_key); - let mut output = String::new(); - let _ = writeln!(output, "sdk install"); - let _ = writeln!( - output, - " summary: rocm-cli will install the ROCm SDK and matching PyTorch packages for this Python and operating system" - ); - render_canonical_provenance( - &mut output, - channel, - &resolution.index_url, - resolution.layout.generation(), - &resolution.latest_version, - ); - let _ = writeln!(output, " format: wheel"); - if let Some(selector) = version_selector { - let _ = writeln!(output, " requested: {}", selector.describe()); - } - let _ = writeln!(output, " target_family: {}", resolution.family); - let _ = writeln!( - output, - " target_family_source: {}", - resolution.family_source - ); - let _ = writeln!(output, " device_target: {}", device_target.as_str()); - if let Some(reason) = device_target.reason() { - let _ = writeln!(output, " device_target_reason: {reason}"); - } - let _ = writeln!(output, " index_url: {}", resolution.index_url); - let _ = writeln!( - output, - " latest_compatible_version: {}", - runtime_version_display(&resolution.latest_version) - ); // Probed once and reused by the progress line below: each call is a full // filesystem scan for a legacy ROCm, and the summary block and the visible // note report the same answer about the same resolved version. let host_version_newer = host_rocm_version_newer_than(&resolution.latest_version); - if let Some(host_version) = host_version_newer.as_deref() { - let _ = writeln!( - output, - " version_note: {}", - wheel_host_version_note( - host_version, - &runtime_version_display(&resolution.latest_version) - ) - ); - } - let _ = writeln!( - output, - " compatibility_key: {}", - runtime_version_display(&resolution.package_versions.compatibility_key) - ); - let _ = writeln!(output, " target: {}", install_root.display()); - let _ = writeln!(output, " runtime_key: {runtime_key}"); - let _ = writeln!(output, " python_launcher: {}", python.launcher_display()); - let _ = writeln!(output, " python_source: {}", python.source().as_str()); - if let Some(note) = python.note() { - let _ = writeln!(output, " python_note: {note}"); - } - let _ = writeln!( - output, - " python_wheel_tag: {}", - wheel_compatibility.python_tag - ); - let _ = writeln!( - output, - " platform_wheel_tags: {}", - wheel_compatibility.platform_tags.join(",") - ); - // What resolution asked for, printed beside what the install will ask for. - // The two are produced by different code paths from the same `include_devel` - // and must name the same extras; showing only the second would hide a - // resolve that constrained the version choice by a toolchain the user - // declined. Absent on the canonical layout, which resolves by scraping each - // package's index instead of compiling a requirement set. - if let Some(specs) = resolution.version_resolution_specs.as_ref() { - let _ = writeln!(output, " version_resolution_specs: {}", specs.join(" ")); - } - // Printed from the same value that chose the pin, so the line cannot claim - // a verified build the requirements above do not name. - if let Some(pin) = resolution.torchaudio_pin.as_ref() { - let _ = writeln!(output, " torchaudio_selection: {}", pin.describe()); - } - let _ = writeln!( - output, - " package_specs: {}", - wheel_composition.package_specs.join(" ") - ); - let _ = writeln!( - output, - " package_policy: resolve the pinned target-complete rocm, torch, torchvision, and torchaudio plan from published package metadata, then install it in one uv transaction" - ); - let no_wheel_warning = repo_version_without_wheels( + let no_wheel_warning = no_wheel_warning_for( resolution.newest_repo_version.as_deref(), &resolution.latest_version, - ) - .map(|newest| { - no_wheel_warning_message( - &newest, - &runtime_version_display(&resolution.latest_version), - ) + ); + // Under the alignment opt-out an environment that already holds torch keeps it: + // the install then asks `uv` for the ROCm packages alone. Decided here, from the + // environment the install will run in, so the preview and the install name the + // same requirements. + let planned_env_python = venv_python_path(&install_root); + let planned_kept = kept_torch::kept_torch_stack(rocm_core::torch_alignment_disabled(), || { + kept_torch::installed_torch_stack(&planned_env_python) }); - if let Some(warning) = no_wheel_warning.as_deref() { - let _ = writeln!(output, " warning: {warning}"); - } + let plan = WheelPlan { + channel, + version_selector, + index_url: &resolution.index_url, + layout_generation: resolution.layout.generation(), + latest_version: &resolution.latest_version, + family: &resolution.family, + family_source: &resolution.family_source, + device_target: &device_target, + host_version_newer: host_version_newer.as_deref(), + python: &python, + wheel_compatibility: &wheel_compatibility, + version_resolution_specs: resolution.version_resolution_specs.as_deref(), + torchaudio_pin: resolution.torchaudio_pin.as_ref(), + no_wheel_warning: no_wheel_warning.as_deref(), + stack: PlanStack::Resolved(ResolvedStack { + compatibility_key: &resolution.package_versions.compatibility_key, + install_root: &install_root, + runtime_key: &runtime_key, + package_specs: &wheel_composition.package_specs, + // A real install decides again once the environment is prepared (it may + // be recreated and lose its torch), and writes the one `torch_kept:` line + // then; a preview cannot know, so it prints the plan's. + kept_torch: if dry_run { planned_kept.as_ref() } else { None }, + manifest_path: &manifest_path, + }), + }; + let mut output = String::new(); + write_wheel_plan(&mut output, &plan); if dry_run { - let env_python = venv_python_path(&install_root); - let mut install_args = uv_pip_install_base(&env_python); - install_args.extend(["--index-url".to_owned(), resolution.index_url.clone()]); - if matches!(channel, TheRockChannel::Nightly) { - install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); - } - install_args.extend(wheel_composition.package_specs.iter().cloned()); - let venv_args = uv_venv_args(&python.venv_python_arg(), &install_root); - // This line is offered as the command to run, so it has to be one a - // shell reads back as the argv it previews. The install root in it - // comes from a `--prefix` the local assistant can choose - // (`install_sdk_dry_run` is a model-callable tool), so a bare or - // double-quoted value here would be a model-chosen value in a line a - // user is invited to paste. - let venv_args_display = venv_args - .iter() - .map(|arg| shell_quote(arg)) - .collect::>() - .join(" "); - let install_args_display = install_args - .iter() - .map(|arg| shell_quote(arg)) - .collect::>() - .join(" "); - let _ = writeln!(output, " mode: dry-run"); - let _ = writeln!( - output, - " command: uv {venv_args_display} && uv {install_args_display}" - ); - let _ = writeln!( - output, - " activation: use the managed venv Python; TheRock libraries are resolved from that venv by rocm_sdk.initialize_process" - ); - let _ = writeln!(output, " manifest: {}", manifest_path.display()); + write_dry_run_tail(&mut output, &plan); return Ok(SdkInstallResult::plan(output)); } @@ -667,10 +641,27 @@ fn install_wheel_runtime( )); ensure_uv_venv(paths, &uv, &python_launcher.executable, &install_root)?; let env_python = venv_python_path(&install_root); + // `ensure_uv_venv` recreates an environment built for another interpreter, and + // the recreated one holds no torch. Asked again, so that case installs the + // full stack instead of leaving the runtime without torch. + let kept = planned_kept.clone().filter(|_| { + kept_torch::kept_torch_stack(true, || kept_torch::installed_torch_stack(&env_python)) + .is_some() + }); + if let Some(kept) = &kept { + progress_line(format!( + "ROCM_CLI_DISABLE_TORCH_ALIGNMENT is set: keeping the torch this runtime already holds ({}).", + kept.describe() + )); + } + if let Some(line) = kept_torch::install_line(planned_kept.as_ref(), kept.as_ref()) { + let _ = writeln!(output, "{line}"); + } + let install_specs = kept_torch::install_specs(&wheel_composition.package_specs, kept.as_ref()); progress_line(format!( "Installing {} from {}", - wheel_composition.package_specs.join(" "), + install_specs.join(" "), resolution.index_url )); let mut install_args = uv_pip_install_base(&env_python); @@ -678,7 +669,7 @@ fn install_wheel_runtime( if matches!(channel, TheRockChannel::Nightly) { install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); } - install_args.extend(wheel_composition.package_specs.iter().cloned()); + install_args.extend(install_specs.iter().cloned()); run_uv_progress_command( paths, &uv, @@ -730,7 +721,7 @@ fn install_wheel_runtime( devel: include_devel, installed_at_unix_ms: unix_time_millis(), }; - save_runtime_manifest(paths, &manifest)?; + write_runtime_manifest(paths, &manifest, ManifestOverwrite::Replace)?; let _ = writeln!( output, @@ -817,6 +808,15 @@ fn no_wheel_warning_message(newest: &str, resolved_display: &str) -> String { ) } +/// The `warning` surfaced when the repository's newest version has no installable +/// PyTorch wheels, or `None` when there is no such gap: [`repo_version_without_wheels`] +/// decides whether there is one and [`no_wheel_warning_message`] words it. One call +/// for a plan that is resolved and one that is not, so they cannot disagree. +fn no_wheel_warning_for(newest_repo: Option<&str>, resolved_version: &str) -> Option { + repo_version_without_wheels(newest_repo, resolved_version) + .map(|newest| no_wheel_warning_message(&newest, &runtime_version_display(resolved_version))) +} + /// The repo's newest version when it is strictly newer than the version we are /// about to install — i.e. the newest release exists in the index but has no /// installable PyTorch wheels for this Python/platform, so an older one is used. @@ -955,12 +955,7 @@ fn install_tarball_runtime( fs::create_dir_all(paths.cache_dir.join("therock"))?; fs::create_dir_all(&install_root)?; - if has_nontrivial_directory_contents(&install_root)? { - bail!( - "tarball install target {} is not empty; choose a clean prefix or remove the old extraction first", - install_root.display() - ); - } + ensure_tarball_install_target_is_clean(&install_root)?; if let Some(warning) = preflight_tarball_space(&artifact.url, &cache_path, &install_root)? { let _ = writeln!(output, " {warning}"); @@ -1007,7 +1002,7 @@ fn install_tarball_runtime( devel: true, installed_at_unix_ms: unix_time_millis(), }; - save_runtime_manifest(paths, &manifest)?; + write_runtime_manifest(paths, &manifest, ManifestOverwrite::Replace)?; let _ = writeln!(output, " extracted: {}", install_root.display()); let _ = writeln!(output, " manifest: {}", manifest_path.display()); @@ -1109,21 +1104,18 @@ fn resolved_install_root( rocm_core::resolve_path_through_symlinks(&requested) } -fn has_nontrivial_directory_contents(path: &Path) -> Result { - if !path.exists() { - return Ok(false); - } - let entries = - fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))?; - for entry in entries { - let entry = entry?; - let name = entry.file_name(); - if name.to_string_lossy().starts_with('.') { - continue; - } - return Ok(true); +/// Refuses to extract a tarball into a root that already holds a payload. +/// +/// "Holds a payload" is [`rocm_core::install_root_has_payload`]: dot-entries (the +/// runtime marker) do not count, so a root that has only been prepared passes. +fn ensure_tarball_install_target_is_clean(install_root: &Path) -> Result<()> { + if rocm_core::install_root_has_payload(install_root)? { + bail!( + "tarball install target {} is not empty; choose a clean prefix or remove the old extraction first", + install_root.display() + ); } - Ok(false) + Ok(()) } fn venv_python_path(install_root: &Path) -> PathBuf { @@ -1147,6 +1139,35 @@ mod tests { use super::test_support::{test_paths, test_runtime_manifest}; use super::*; + /// `--format tarball` installs no Python, so a Python choice cannot be honoured + /// and is refused before anything is resolved or downloaded. The refusal comes + /// ahead of the platform's tarball check so it reads the same on every host. + #[test] + fn python_version_is_refused_for_a_tarball_install() { + let (_root, paths) = test_paths("tarball-python-version-refused"); + let error = install_sdk( + &paths, + SdkInstallRequest { + channel: "release", + format: "tarball", + prefix: None, + version_selector: None, + family_override: None, + python_version: Some("3.12"), + dry_run: true, + include_devel: false, + consent: SdkInstallConsent::Ask, + }, + ) + .expect_err("a tarball install has no Python to choose"); + assert!( + error + .to_string() + .contains("--python-version applies only to wheel installs"), + "{error}" + ); + } + #[test] fn download_space_requirement_includes_the_safety_margin() { let archive = 2 * 1024 * 1024 * 1024; @@ -1168,17 +1189,6 @@ mod tests { assert_eq!(estimate, archive * disk_space::EXTRACTED_SIZE_MULTIPLIER); } - #[test] - fn write_file_atomically_reports_a_full_disk_clearly() { - // Exercise the mapping the write path uses, without filling a disk. - let error = disk_space::map_write_error( - std::io::Error::from(std::io::ErrorKind::StorageFull), - Path::new("/cache/rocm.tar.gz.tmp"), - ); - let text = format!("{error:#}"); - assert!(text.contains("ran out of disk space"), "{text}"); - } - #[test] fn normalize_therock_family_maps_gfx1103_to_gfx110x_all() { assert_eq!( @@ -1703,4 +1713,41 @@ mod tests { "ROCm 7.14.0 is the newest version in this repository but has no installable PyTorch wheels for this Python and platform; installing ROCm 7.13.0 instead" ); } + + /// The extraction guard refuses a root that holds a payload and says so, and + /// it follows `rocm_core::install_root_has_payload` for what that means: a + /// root that does not exist yet (the case `install_tarball_runtime` hits when + /// `--prefix` names a new folder) and one holding only dot-entries pass. + /// Refusing must leave the existing files where they were. + #[test] + fn the_tarball_install_guard_follows_the_shared_payload_rule() { + let base = tempfile::tempdir().expect("scratch dir"); + + ensure_tarball_install_target_is_clean(&base.path().join("not-created-yet")) + .expect("a missing root is clean"); + + let marker_only = base.path().join("marker-only"); + fs::create_dir_all(&marker_only).unwrap(); + fs::write(marker_only.join(".rocm-cli-runtime.json"), b"{}").unwrap(); + ensure_tarball_install_target_is_clean(&marker_only) + .expect("a root with only the dot-marker is clean"); + + let populated = base.path().join("populated"); + fs::create_dir_all(populated.join("bin")).unwrap(); + fs::write(populated.join("bin").join("tool"), b"x").unwrap(); + let error = ensure_tarball_install_target_is_clean(&populated) + .expect_err("a root with a payload is refused") + .to_string(); + assert_eq!( + error, + format!( + "tarball install target {} is not empty; choose a clean prefix or remove the old extraction first", + populated.display() + ) + ); + assert!( + populated.join("bin").join("tool").is_file(), + "the refusal must not touch what is already there" + ); + } } diff --git a/apps/rocm/src/therock/consent.rs b/apps/rocm/src/therock/consent.rs index 60ef0f9c9b..165bddca9b 100644 --- a/apps/rocm/src/therock/consent.rs +++ b/apps/rocm/src/therock/consent.rs @@ -440,7 +440,7 @@ pub(super) fn confirm_overwrite_existing_sdk( fn prompt_yes_no(prompt: &str) -> Result { rocm_core::confirm::print_prompt(&mut std::io::stdout(), prompt) .context("failed to write confirmation prompt")?; - rocm_core::confirm::read_answer(&mut std::io::stdin().lock()) + rocm_core::confirm::read_answer(&mut std::io::stdin().lock(), &mut std::io::stdout()) .context("failed to read confirmation response") } diff --git a/apps/rocm/src/therock/fetch.rs b/apps/rocm/src/therock/fetch.rs index 1257042bc0..6829709e25 100644 --- a/apps/rocm/src/therock/fetch.rs +++ b/apps/rocm/src/therock/fetch.rs @@ -8,14 +8,14 @@ //! //! Owns the pinned metadata trust root, //! which `cargo xtask verify-pinned-keys` checks against the canonical -//! published key. Depends on the sibling `io` module for atomic writes and on -//! `slugify` in the parent for cache file names. +//! published key. Depends on `rocm_core::atomic_write` for atomic writes and +//! on `slugify` in the parent for cache file names. -#[cfg(test)] -use super::io::{publish_temp_file, write_file_atomically_with_publish}; -use super::io::{stage_file_for_atomic_publish, write_file_atomically}; use super::slugify; use anyhow::{Context, Result, bail}; +#[cfg(test)] +use rocm_core::atomic_write::{publish_temp_file, write_file_atomically_with_publish}; +use rocm_core::atomic_write::{stage_file_for_atomic_publish, write_file_atomically}; use rocm_core::{AppPaths, disk_space, unix_time_millis, verify_rsa_pkcs1_sha256_signature}; use serde::{Deserialize, Serialize}; #[cfg(test)] diff --git a/apps/rocm/src/therock/io.rs b/apps/rocm/src/therock/io.rs index b4d3d06f72..9347220aee 100644 --- a/apps/rocm/src/therock/io.rs +++ b/apps/rocm/src/therock/io.rs @@ -2,21 +2,15 @@ // // SPDX-License-Identifier: MIT -//! Filesystem helpers behind TheRock installs: unique scratch directories and -//! the stage-then-publish atomic file writes. The writes serve the metadata -//! cache entries and the staged metadata public key, the startup update-check -//! record, and (through `therock`'s re-export of `write_file_atomically`) the -//! active-runtime marker that `runtimes_cmd` maintains. Runtime registry -//! manifests do not go through these writers. +//! Unique scratch directories behind TheRock installs. The atomic file writes +//! that used to live here are `rocm_core::atomic_write`, shared with `rocmd`. //! //! A leaf of the `therock` module: nothing here depends on the rest of it. use anyhow::{Context, Result, bail}; -use rocm_core::{disk_space, unix_time_millis}; -use std::ffi::{OsStr, OsString}; +use rocm_core::unix_time_millis; use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; pub(super) fn linux_temp_dir(prefix: &str) -> Result { let root = std::env::temp_dir(); @@ -69,456 +63,3 @@ fn windows_runtime_temp_root() -> Option { } None } - -/// A unique temp path next to `path`, preserving the full file name so a -/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes -/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). -const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; - -fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { - let parent = path.parent().context("file path has no parent directory")?; - let mut file_name = path - .file_name() - .context("file path has no file name")? - .to_os_string(); - file_name.push(".tmp-"); - file_name.push(suffix); - Ok(parent.join(file_name)) -} - -pub(crate) fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - write_file_atomically_with( - path, - bytes, - |attempt| OsString::from(format!("{temp_id}-{attempt}")), - || {}, - ) -} - -fn write_file_atomically_with( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), -{ - write_file_atomically_with_publish( - path, - bytes, - suffix_for_attempt, - before_publish, - publish_temp_file, - ) -} - -pub(super) fn write_file_atomically_with_publish( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, - publish: F, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - let tmp = stage_file_for_atomic_publish_with(path, bytes, suffix_for_attempt)?; - before_publish(); - publish_staged_file_with(&tmp, path, publish) -} - -pub(super) fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - stage_file_for_atomic_publish_with(path, bytes, |attempt| { - OsString::from(format!("{temp_id}-{attempt}")) - }) -} - -fn stage_file_for_atomic_publish_with( - path: &Path, - bytes: &[u8], - mut suffix_for_attempt: S, -) -> Result -where - S: FnMut(u32) -> OsString, -{ - let parent = path.parent().context("file path has no parent directory")?; - fs::create_dir_all(parent)?; - - let mut reserved = None; - for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { - let suffix = suffix_for_attempt(attempt); - let tmp = temp_sibling_path(path, &suffix)?; - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - { - Ok(file) => { - reserved = Some((tmp, file)); - break; - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(error) => { - return Err(error).with_context(|| format!("failed to create {}", tmp.display())); - } - } - } - let Some((tmp, mut file)) = reserved else { - bail!( - "failed to reserve a temporary file next to {} after {} attempts", - path.display(), - ATOMIC_WRITE_TEMP_ATTEMPTS - ); - }; - - if let Err(error) = file.write_all(bytes) { - drop(file); - let _ = fs::remove_file(&tmp); - return Err(disk_space::map_write_error(error, &tmp)); - } - drop(file); - Ok(tmp) -} - -#[cfg(test)] -fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { - publish_staged_file_with(tmp, path, publish_temp_file) -} - -fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> -where - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - publish(tmp, path) - .inspect_err(|_| { - let _ = fs::remove_file(tmp); - }) - .with_context(|| format!("failed to publish {}", path.display())) -} - -#[cfg(not(windows))] -pub(super) fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -pub(super) fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The temp name keeps every extension, so a cleanup sweep over a cache - /// directory can still tell what a leftover was going to be. - #[test] - fn temp_sibling_path_preserves_multi_dot_file_names() { - let temp = temp_sibling_path( - Path::new("/tmp/cache/sdk.tar.gz"), - std::ffi::OsStr::new("test"), - ) - .unwrap(); - let name = temp.file_name().unwrap().to_string_lossy().into_owned(); - assert_eq!(name, "sdk.tar.gz.tmp-test"); - assert_eq!(temp.parent().unwrap(), Path::new("/tmp/cache")); - } - - #[test] - fn concurrent_atomic_writes_do_not_remove_a_published_destination() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("sdk.tar.gz"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .enumerate() - .map(|(writer, bytes)| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - write_file_atomically_with( - &destination, - bytes, - |attempt| { - if attempt == 0 { - std::ffi::OsString::from("same-millisecond") - } else { - std::ffi::OsString::from(format!( - "same-millisecond-{writer}-{attempt}" - )) - } - }, - || { - before_publish.wait(); - }, - ) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a writer must remain published"); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - } - - #[test] - fn concurrent_cached_publications_use_distinct_staging_files() { - let root = std::env::temp_dir().join(format!( - "rocm-cache-publish-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("index.body"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .map(|bytes| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - let staged = stage_file_for_atomic_publish(&destination, bytes)?; - before_publish.wait(); - publish_staged_file(&staged, &destination) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a cache writer must remain published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .filter(|name| name.to_string_lossy().contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - assert!( - leftovers.is_empty(), - "staged cache files leaked: {leftovers:?}" - ); - } - - #[test] - fn failed_cached_publication_preserves_destination_and_cleans_staging_file() { - let root = std::env::temp_dir().join(format!( - "rocm-cache-publish-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("index.body"); - fs::write(&destination, b"published").unwrap(); - let staged = stage_file_for_atomic_publish(&destination, b"replacement").unwrap(); - - publish_staged_file_with(&staged, &destination, |_, _| { - Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "simulated cache publication failure", - )) - }) - .expect_err("simulated cache publication failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - assert!( - !staged.exists(), - "failed publication leaked its staging file" - ); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn failed_atomic_replace_preserves_destination_and_cleans_temp() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-replace-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("sdk.tar.gz"); - fs::write(&destination, b"published").unwrap(); - - write_file_atomically_with_publish( - &destination, - b"replacement", - |attempt| OsString::from(format!("replace-failure-{attempt}")), - || {}, - |tmp, path| { - assert_eq!(fs::read(tmp).unwrap(), b"replacement"); - assert_eq!(path, destination); - Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "simulated atomic replacement failure", - )) - }, - ) - .expect_err("simulated replacement failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); - } - - #[cfg(unix)] - #[test] - fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - let file_name = std::ffi::OsString::from_vec(b"sdk-\xff.tar.gz".to_vec()); - let destination = Path::new("/tmp").join(&file_name); - let temp = temp_sibling_path(&destination, std::ffi::OsStr::new("collision")).unwrap(); - - let mut expected = file_name.into_vec(); - expected.extend_from_slice(b".tmp-collision"); - assert_eq!(temp.file_name().unwrap().as_bytes(), expected); - } - - /// Regression: a failed write must not leave a `.tmp-*` scratch file - /// behind. The name is unique per attempt, so before this an orphan - /// accumulated per retry — and when the failure is a full disk, those - /// orphans are exactly what keeps it full. - /// - /// Provokes the failure by pointing the destination at a non-empty - /// directory: the temp file is written, then neither the rename nor the - /// replace fallback can succeed. Portable, unlike an out-of-space test. - #[test] - fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("sdk.tar.gz"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - write_file_atomically(&occupied, b"payload") - .expect_err("renaming onto a non-empty directory should fail"); - - let leftovers: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!( - leftovers.is_empty(), - "failed write left temp files behind: {leftovers:?}" - ); - } - - /// Mirrors the `/dev/shm` reproduction from the original report: a genuine - /// ENOSPC, not a rename failure standing in for one. - /// - /// Ignored by default because it fills `/dev/shm`, which is shared with - /// anything else on the host, so it is not safe to run concurrently. Run - /// with `cargo test -p rocm -- --ignored write_file_atomically_cleans_up`. - #[test] - #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] - fn write_file_atomically_cleans_up_temp_on_write_failure() { - let shm = Path::new("/dev/shm"); - if !shm.is_dir() { - eprintln!("skipping: /dev/shm unavailable"); - return; - } - let dir = shm.join(format!("rocm-enospc-{}", std::process::id())); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - let dest = dir.join("artifact.tar.gz"); - // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. - let payload = vec![0u8; 256 * 1024 * 1024]; - - let mut failures = Vec::new(); - for _ in 0..2 { - write_file_atomically(&dest, &payload) - .expect_err("writing past the end of the filesystem should fail"); - failures.push( - fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect::>(), - ); - } - let destination_exists = dest.exists(); - let _ = fs::remove_dir_all(&dir); - - for leftovers in &failures { - assert!( - leftovers.is_empty(), - "failed write left files behind: {leftovers:?}" - ); - } - assert!( - !destination_exists, - "destination must not exist after failure" - ); - } -} diff --git a/apps/rocm/src/therock/kept_torch.rs b/apps/rocm/src/therock/kept_torch.rs new file mode 100644 index 0000000000..f50136a91d --- /dev/null +++ b/apps/rocm/src/therock/kept_torch.rs @@ -0,0 +1,375 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Keeping the torch a runtime already holds when an SDK install runs over it. +//! +//! The wheel install hands `uv` one transaction: `rocm`, `torch`, `torchvision` +//! and `torchaudio`, all from the SDK index. Run over an environment that already +//! holds a different torch (the one an engine installed, or one the user built) +//! that transaction replaces the stack, and `triton` with it. The alignment +//! opt-out, `ROCM_CLI_DISABLE_TORCH_ALIGNMENT`, only governs the step that runs +//! after the install, so without this module the opt-out was honoured everywhere +//! except in the command that rewrites torch first. +//! +//! Set, the opt-out leaves the torch stack out of the transaction when the +//! environment already has torch, and installs only the ROCm packages. An +//! environment with no torch gets the full stack, since there is nothing to keep. +//! +//! Builds on `process` for running the interpreter; nothing in `process` depends +//! on this module. + +use super::process::capture_python_stdout_with_env; +use anyhow::{Context, Result}; +use std::path::Path; + +/// The packages whose requirements the opt-out removes from the SDK install. +/// +/// `triton` is not listed because the install never names it: it arrives as a +/// dependency of `torch`, so leaving `torch` out leaves it where it is. +const TORCH_SPEC_PACKAGES: [&str; 3] = ["torch", "torchvision", "torchaudio"]; + +/// The torch stack an environment held before the install, with the versions it +/// held them at. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct KeptTorch { + installed: Vec<(String, String)>, +} + +impl KeptTorch { + /// The stack as it is named in the plan and the progress line, for example + /// `torch 2.11.0+gitd0c8b1f, torchvision 0.24.1+d801a34`. + pub(super) fn describe(&self) -> String { + self.installed + .iter() + .map(|(name, version)| format!("{name} {version}")) + .collect::>() + .join(", ") + } +} + +/// Decide what to keep: nothing unless the user opted out *and* the environment +/// has a torch to keep. +/// +/// `installed` is asked only when the opt-out is set, so an install that did not +/// opt out never launches the interpreter for an answer it would not use. A probe +/// that fails keeps nothing: the environment is then treated as the empty one the +/// install has always assumed, which is the behaviour the opt-out is a request to +/// change, not a promise to refuse the install over. +pub(super) fn kept_torch_stack( + opted_out: bool, + installed: impl FnOnce() -> Result>, +) -> Option { + if !opted_out { + return None; + } + let installed = match installed() { + Ok(installed) => installed, + Err(err) => { + eprintln!( + "warning: ROCM_CLI_DISABLE_TORCH_ALIGNMENT is set but the installed torch could not be read ({err:#}); the install will replace torch with the SDK's. Unset the variable to make that the intended behaviour." + ); + return None; + } + }; + installed + .iter() + .any(|(name, _)| name == "torch") + .then_some(KeptTorch { installed }) +} + +/// The requirements the install hands `uv`: all of them, or all but the torch +/// stack when one is being kept. +pub(super) fn install_specs(specs: &[String], kept: Option<&KeptTorch>) -> Vec { + if kept.is_none() { + return specs.to_vec(); + } + specs + .iter() + .filter(|spec| !TORCH_SPEC_PACKAGES.contains(&spec_package_name(spec))) + .cloned() + .collect() +} + +/// The distribution a requirement names: what comes before an extras bracket, a +/// version operator or an environment marker. +fn spec_package_name(spec: &str) -> &str { + let end = spec + .find(['[', '=', '<', '>', '!', '~', ';', ' ']) + .unwrap_or(spec.len()); + spec[..end].trim() +} + +/// The plan line that says the torch stack is not being installed, made from the +/// same value that filtered the requirements. +pub(super) fn plan_line(kept: &KeptTorch) -> String { + format!( + " torch_kept: ROCM_CLI_DISABLE_TORCH_ALIGNMENT is set and this runtime already holds {}; the torch, torchvision and torchaudio requirements are left out of the install", + kept.describe() + ) +} + +/// The one `torch_kept:` line a real install prints, once the environment is +/// prepared: the plan's line when the torch is kept, an explanation when the +/// environment was recreated and the torch the plan counted on is gone. +pub(super) fn install_line( + planned: Option<&KeptTorch>, + kept: Option<&KeptTorch>, +) -> Option { + match (planned, kept) { + (_, Some(kept)) => Some(plan_line(kept)), + (Some(planned), None) => Some(format!( + " torch_kept: not applied; the Python environment was recreated for another interpreter, which discarded the torch it held ({}); the full torch stack is installed", + planned.describe() + )), + (None, None) => None, + } +} + +/// Ask the environment's own interpreter which torch packages it holds. +/// +/// Distribution metadata, not an import: importing torch loads the GPU libraries +/// and can fail or stall for reasons unrelated to what is installed. An +/// environment that does not exist yet has no interpreter to ask, and yields an +/// empty stack. +pub(super) fn installed_torch_stack(env_python: &Path) -> Result> { + if !env_python.exists() { + return Ok(Vec::new()); + } + let text = capture_python_stdout_with_env( + env_python, + INSTALLED_TORCH_SCRIPT, + &[], + "read the installed torch stack", + ) + .with_context(|| { + format!( + "failed to read the installed torch stack via {}", + env_python.display() + ) + })?; + parse_installed_stack(&text) +} + +fn parse_installed_stack(output: &str) -> Result> { + let versions: std::collections::BTreeMap = + serde_json::from_str(output.trim()).context("failed to parse the installed torch stack")?; + // Fixed order, so the line printed for it reads the same on every run. + Ok(rocm_core::SDK_TORCH_STACK_PACKAGES + .iter() + .filter_map(|name| { + versions + .get(*name) + .map(|version| ((*name).to_owned(), version.clone())) + }) + .collect()) +} + +const INSTALLED_TORCH_SCRIPT: &str = r#" +import importlib.metadata as md +import json + +out = {} +for name in ("torch", "torchvision", "torchaudio", "triton"): + try: + out[name] = md.version(name) + except md.PackageNotFoundError: + pass +print(json.dumps(out)) +"#; + +#[cfg(test)] +mod tests { + use super::*; + + fn specs() -> Vec { + [ + "rocm[libraries,device-gfx942]==7.14.1", + "torch[device-gfx942]==2.11.0+rocm7.14.1", + "torchvision[device-gfx942]==0.26.0+rocm7.14.1", + "torchaudio==2.11.0+rocm7.14.1", + ] + .map(str::to_owned) + .to_vec() + } + + fn held(names: &[(&str, &str)]) -> Result> { + Ok(names + .iter() + .map(|(name, version)| ((*name).to_owned(), (*version).to_owned())) + .collect()) + } + + #[test] + fn without_the_opt_out_nothing_is_kept_and_the_environment_is_not_asked() { + let kept = kept_torch_stack(false, || panic!("the interpreter must not be launched")); + assert_eq!(kept, None); + assert_eq!(install_specs(&specs(), kept.as_ref()), specs()); + } + + #[test] + fn an_opted_out_environment_with_torch_installs_only_the_rocm_packages() { + let kept = kept_torch_stack(true, || { + held(&[("torch", "2.11.0+gitd0c8b1f"), ("triton", "3.6.0")]) + }) + .expect("a torch to keep"); + assert_eq!(kept.describe(), "torch 2.11.0+gitd0c8b1f, triton 3.6.0"); + assert_eq!( + install_specs(&specs(), Some(&kept)), + vec!["rocm[libraries,device-gfx942]==7.14.1".to_owned()], + "torch, torchvision and torchaudio are all left to the user" + ); + } + + #[test] + fn an_opted_out_environment_without_torch_gets_the_full_stack() { + for installed in [held(&[]), held(&[("triton", "3.6.0")])] { + let kept = kept_torch_stack(true, || installed); + assert_eq!(kept, None, "there is no torch to keep"); + assert_eq!(install_specs(&specs(), kept.as_ref()), specs()); + } + } + + #[test] + fn an_environment_that_cannot_be_read_keeps_nothing() { + let kept = kept_torch_stack(true, || anyhow::bail!("the interpreter would not start")); + assert_eq!(kept, None); + } + + #[test] + fn a_recreated_environment_prints_one_line_that_matches_what_is_installed() { + let planned = kept_torch_stack(true, || held(&[("torch", "2.11.0")])).expect("kept"); + assert_eq!(install_line(None, None), None); + assert_eq!( + install_line(Some(&planned), Some(&planned)), + Some(plan_line(&planned)) + ); + let line = install_line(Some(&planned), None).expect("a line"); + assert!(line.starts_with(" torch_kept: not applied;"), "{line}"); + assert!(line.contains("torch 2.11.0") && !line.contains("left out of the install")); + } + + #[test] + fn only_the_torch_requirements_are_dropped() { + let kept = KeptTorch { + installed: vec![("torch".to_owned(), "2.11.0".to_owned())], + }; + let specs = [ + "rocm[libraries]==7.14.1", + "rocm-sdk-devel==7.14.1", + "torch-geometric==2.6.0", + "torchaudio==2.11.0", + ] + .map(str::to_owned) + .to_vec(); + assert_eq!( + install_specs(&specs, Some(&kept)), + vec![ + "rocm[libraries]==7.14.1".to_owned(), + "rocm-sdk-devel==7.14.1".to_owned(), + "torch-geometric==2.6.0".to_owned(), + ], + "a package that merely starts with `torch` is not part of the stack" + ); + } + + #[test] + fn a_requirement_is_named_by_what_precedes_its_extras_and_version() { + for (spec, name) in [ + ("torch==2.11.0", "torch"), + ("torch[device-gfx942]==2.11.0+rocm7.14.1", "torch"), + ("torchvision>=0.20", "torchvision"), + ("torchaudio ; platform_system == 'Linux'", "torchaudio"), + ("rocm", "rocm"), + ] { + assert_eq!(spec_package_name(spec), name, "{spec}"); + } + } + + #[test] + fn the_installed_stack_is_read_in_a_fixed_order_and_skips_what_is_absent() { + let stack = parse_installed_stack( + r#"{"triton": "3.6.0", "torchaudio": "2.9.0+eaa9e4e", "torch": "2.11.0+gitd0c8b1f"}"#, + ) + .expect("a stack"); + assert_eq!( + stack, + vec![ + ("torch".to_owned(), "2.11.0+gitd0c8b1f".to_owned()), + ("torchaudio".to_owned(), "2.9.0+eaa9e4e".to_owned()), + ("triton".to_owned(), "3.6.0".to_owned()), + ] + ); + assert!(parse_installed_stack("not json").is_err()); + } + + #[test] + fn the_plan_line_names_what_is_kept_and_what_is_left_out() { + let kept = kept_torch_stack(true, || { + held(&[ + ("torch", "2.11.0+gitd0c8b1f"), + ("torchvision", "0.24.1+d801a34"), + ]) + }) + .expect("a torch to keep"); + let line = plan_line(&kept); + assert!(line.contains("torch 2.11.0+gitd0c8b1f, torchvision 0.24.1+d801a34")); + assert!(line.contains("ROCM_CLI_DISABLE_TORCH_ALIGNMENT")); + // The e2e step `the SDK install leaves the torch the runtime held as it was` + // reads the torch version out of this line by the words around it, and checks + // it against the runtime's interpreter. That step is what proves the torch was + // kept; this only pins the shape the step parses. + assert!( + line.contains(" already holds torch 2.11.0+gitd0c8b1f,"), + "{line}" + ); + } + + /// The interpreter really is asked: a stand-in environment whose `python` is a + /// wrapper that exposes one fake `torch` distribution, read back through the + /// same script an install runs. Parsing the script's JSON is tested above; + /// this is what ties the script to the parser. + #[test] + #[cfg(unix)] + fn the_installed_stack_is_read_from_a_real_interpreter() { + use std::os::unix::fs::PermissionsExt; + + let root = std::env::temp_dir().join(format!( + "rocm-cli-kept-torch-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let dist_info = root.join("site").join("torch-2.9.0+gittest.dist-info"); + std::fs::create_dir_all(&dist_info).unwrap(); + std::fs::write( + dist_info.join("METADATA"), + "Metadata-Version: 2.1\nName: torch\nVersion: 2.9.0+gittest\n", + ) + .unwrap(); + let python = root.join("python"); + std::fs::write( + &python, + format!( + "#!/bin/sh\nPYTHONPATH='{}' exec python3 \"$@\"\n", + root.join("site").display() + ), + ) + .unwrap(); + std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let stack = installed_torch_stack(&python).expect("the interpreter answers"); + let _ = std::fs::remove_dir_all(&root); + assert_eq!( + stack, + vec![("torch".to_owned(), "2.9.0+gittest".to_owned())], + "only the torch distribution the environment has is reported" + ); + } + + #[test] + fn a_missing_environment_holds_no_torch() { + let stack = installed_torch_stack(Path::new("/nonexistent/venv/bin/python")) + .expect("a missing environment is empty, not an error"); + assert!(stack.is_empty()); + } +} diff --git a/apps/rocm/src/therock/python.rs b/apps/rocm/src/therock/python.rs index 43b38f0261..1b5d67fd6e 100644 --- a/apps/rocm/src/therock/python.rs +++ b/apps/rocm/src/therock/python.rs @@ -35,7 +35,7 @@ const DEFAULT_MANAGED_PYTHON_VERSION: &str = "3.12"; /// Interpreter `uv` provisions for a [`SourceLayout::Next`] install. See /// [`python_requirement`] for why this differs from the canonical default. -const NEXT_MANAGED_PYTHON_VERSION: &str = "3.14"; +pub(crate) const NEXT_MANAGED_PYTHON_VERSION: &str = "3.14"; #[derive(Debug, Clone)] pub(super) struct WheelCompatibility { @@ -212,8 +212,30 @@ fn save_managed_python_manifest(paths: &AppPaths, manifest: &ManagedPythonManife .with_context(|| format!("failed to write {}", path.display())) } +/// `config.json` for the managed-Python step, failing with what was left +/// alone and how to recover (the advice comes from [`RocmCliConfig::load`]). +fn load_config_for_managed_python(paths: &AppPaths) -> Result { + RocmCliConfig::load(paths).with_context(|| { + format!( + "cannot record the managed Python in {} because that file cannot be read; it was \ + left unchanged", + paths.config_path().display() + ) + }) +} + +/// Record the managed interpreter under `tools.python` in `config.json`. +/// +/// A `config.json` that exists but cannot be read is an error here, as it is +/// for [`RocmCliConfig::load`] itself: it is not "no config yet", and saving +/// over it would replace the user's active runtime, TheRock folder, providers +/// and preferences with defaults. The callers fail rather than skip the +/// record: the install cannot finish against that file anyway (activating the +/// new runtime loads it strictly). `ensure_managed_python` checks the file +/// before it downloads anything, so this only fails there when the file +/// changed meanwhile. fn record_managed_python_config(paths: &AppPaths, python: &Path) -> Result<()> { - let mut config = RocmCliConfig::load(paths).unwrap_or_default(); + let mut config = load_config_for_managed_python(paths)?; config.tools.insert( "python".to_owned(), ManagedToolConfig { @@ -244,8 +266,8 @@ fn managed_python_bootstrap_disabled() -> bool { /// [`THEROCK_NEXT_MIN_MAJOR`]: super::source::THEROCK_NEXT_MIN_MAJOR #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct PythonRequirement { - tag: &'static str, - version: &'static str, + pub(super) tag: &'static str, + pub(super) version: &'static str, } /// The interpreter required to install from `layout`. @@ -253,11 +275,12 @@ pub(super) struct PythonRequirement { /// Keyed on the layout rather than on a resolved ROCm version because the /// interpreter has to be chosen *before* resolution: `resolve_pip_runtime` picks /// wheels using the interpreter's own tags, so the resolved version is not -/// available yet. The layout is an honest stand-in only when it was decided -/// from an explicit stable pin (`next_layout_requested`); a fresh install with -/// no pin, or a nightly build-date/prerelease pin, needs -/// [`canonical_auto_select_needs_next_python`] to check whether the channel's -/// own "latest" has rotated past [`THEROCK_NEXT_MIN_MAJOR`] on its own. +/// available yet. For a request routed to the next layout (`layout_serving`: +/// no pin, a stable ROCm 10+ pin, or a nightly built since the next layout began) +/// the layout is an honest stand-in. A pin routed to the canonical stream can +/// still resolve a ROCm 10+ build there (the earlier nightly stream carries ROCm +/// 10 alphas), so [`canonical_auto_select_needs_next_python`] checks that feed's +/// own listing for a build at or past [`THEROCK_NEXT_MIN_MAJOR`]. /// /// [`canonical_auto_select_needs_next_python`]: super::canonical_auto_select_needs_next_python /// [`THEROCK_NEXT_MIN_MAJOR`]: super::source::THEROCK_NEXT_MIN_MAJOR @@ -274,6 +297,88 @@ pub(super) const fn python_requirement(layout: SourceLayout) -> PythonRequiremen } } +/// The CPython versions a ROCm 10+ install can run on when nothing needs vLLM: +/// the SDK device wheels are `py3-none` and the torch stack is published for +/// each of these. [`NEXT_MANAGED_PYTHON_VERSION`] stays the default because +/// vLLM, flash-attn and amd-aiter for ROCm 10 exist for it alone. +const NEXT_SDK_PYTHONS: [PythonRequirement; 5] = [ + PythonRequirement { + tag: "cp310", + version: "3.10", + }, + PythonRequirement { + tag: "cp311", + version: "3.11", + }, + PythonRequirement { + tag: "cp312", + version: "3.12", + }, + PythonRequirement { + tag: "cp313", + version: "3.13", + }, + PythonRequirement { + tag: "cp314", + version: NEXT_MANAGED_PYTHON_VERSION, + }, +]; + +/// The interpreter `layout` requires when the user asked for `version` +/// (`rocm install sdk --python-version`). +/// +/// Only a [`SourceLayout::Next`] install has a choice; the canonical stream is +/// published for one interpreter, so naming anything else for it is refused +/// instead of ignored. +pub(super) fn python_requirement_for_version( + layout: SourceLayout, + version: &str, +) -> Result { + let version = version.trim(); + match layout { + SourceLayout::Next => NEXT_SDK_PYTHONS + .iter() + .copied() + .find(|candidate| candidate.version == version) + .with_context(|| { + let supported = NEXT_SDK_PYTHONS + .iter() + .map(|candidate| candidate.version) + .collect::>() + .join(", "); + format!( + "--python-version {version} is not one ROCm 10 and newer is published for; use one of {supported}" + ) + }), + SourceLayout::Canonical => { + let required = python_requirement(layout); + if version == required.version { + return Ok(required); + } + bail!( + "--python-version applies only to ROCm 10 and newer installs; this install uses Python {}", + required.version + ) + } + } +} + +/// The requirement a runtime installed from `layout` was built for, read from +/// the wheel `tag` of the interpreter it recorded. `None` for a tag the layout +/// is not published for, so the caller falls back to the layout's default. +pub(super) fn python_requirement_for_tag( + layout: SourceLayout, + tag: &str, +) -> Option { + match layout { + SourceLayout::Next => NEXT_SDK_PYTHONS + .iter() + .copied() + .find(|candidate| candidate.tag == tag), + SourceLayout::Canonical => None, + } +} + fn managed_python_version_override() -> Option { std::env::var("ROCM_CLI_MANAGED_PYTHON_VERSION").ok() } @@ -290,6 +395,10 @@ fn ensure_managed_python( requirement: PythonRequirement, version: String, ) -> Result { + // The step ends by recording the interpreter in config.json. Check that + // the file can be read before downloading uv or Python, so an unreadable + // one stops the step before it fetches anything. + load_config_for_managed_python(paths)?; progress_line(format!("Preparing Python {version}...")); let uv = ensure_uv_binary(paths)?; @@ -309,7 +418,7 @@ fn ensure_managed_python( "Using existing Python {version} at {}.", manifest.executable.display() )); - let _ = record_managed_python_config(paths, &manifest.executable); + record_managed_python_config(paths, &manifest.executable)?; return Ok(PythonLauncher { executable: manifest.executable, source: PythonSource::Managed, @@ -379,7 +488,7 @@ fn ensure_managed_python( installed_at_unix_ms: unix_time_millis(), }; save_managed_python_manifest(paths, &manifest)?; - let _ = record_managed_python_config(paths, &executable); + record_managed_python_config(paths, &executable)?; progress_line(format!( "Python {version} is ready at {}.", executable.display() @@ -525,8 +634,8 @@ pub(super) fn python_for_install( Ok(python) } -/// The interpreter a *preview* of the install reports, found without preparing -/// one. +/// The interpreter a *preview* of the install, or an update check, reports, +/// found without preparing one. /// /// [`resolve_python_launcher`] ends by downloading a managed `uv`, having it /// install Python, and saving that Python's path in the configuration when no @@ -538,7 +647,10 @@ pub(super) fn python_for_install( /// resolution raises before that step (an unusable `ROCM_CLI_PYTHON`, a /// disabled managed bootstrap) is raised here too, so a preview never succeeds /// on a host where the install would refuse. -fn plan_python_launcher(paths: &AppPaths, requirement: PythonRequirement) -> Result { +pub(super) fn plan_python_launcher( + paths: &AppPaths, + requirement: PythonRequirement, +) -> Result { let env = python_resolver_env_override().unwrap_or_else(PythonResolverEnv::from_process_env); plan_python_launcher_in(paths, &env, requirement) } @@ -655,6 +767,28 @@ impl PlannedPython { } } + /// The `python_note:` line of an update check, in the same terms as + /// [`Self::note`] but for the step an update would take. An update check + /// needs the interpreter's wheel tag to read the index, and a host with no + /// interpreter gets it from the one the layout requires, so this line is how + /// the check says it did not go and prepare one. Built from the decision, as + /// the preview's is. + pub(super) fn update_note(&self) -> Option { + match self { + Self::Ready(_) => None, + Self::WouldPrepareManaged { + version, + requirement, + tag_check, + } => { + let unchecked = unchecked_tag_caveat(*tag_check, *requirement); + Some(format!( + "no usable Python was found; updating would use uv to install managed Python {version}{unchecked} and save its path in the configuration, which this check did not do" + )) + } + } + } + /// The progress line that names the chosen interpreter: the one an install /// goes on with, or, for a preview on a host with no usable Python, what a /// real install would prepare. It is the only line that predicts the @@ -984,11 +1118,15 @@ fn python_venv_probe_temp_root() -> Result { } } +#[cfg(test)] +#[path = "python_config_props.rs"] +mod config_props; + #[cfg(test)] mod tests { use super::*; use crate::test_support::PROCESS_ENV_TEST_LOCK; - use crate::therock::test_support::test_paths; + use crate::therock::test_support::{RestoredEnv, test_paths}; #[cfg(unix)] fn write_fake_python_with_venv(dir: &Path, name: &str) -> Result { @@ -1080,6 +1218,75 @@ exit 1 assert_eq!(DEFAULT_MANAGED_PYTHON_VERSION, "3.12"); } + /// `--python-version` is a choice among the interpreters the layout is + /// published for. ROCm 10 takes 3.10 to 3.14 (each with its own wheel tag and + /// the version `uv` is asked for); the canonical layout takes only its own 3.12; + /// everything else is refused with the allowed set named, not ignored. + #[test] + fn python_version_is_a_choice_among_the_layouts_published_pythons() { + for (version, tag) in [ + ("3.10", "cp310"), + ("3.11", "cp311"), + ("3.12", "cp312"), + ("3.13", "cp313"), + ("3.14", "cp314"), + (" 3.12 ", "cp312"), + ] { + let requirement = python_requirement_for_version(SourceLayout::Next, version) + .unwrap_or_else(|error| panic!("{version} must be accepted: {error:#}")); + assert_eq!(requirement.tag, tag, "{version}"); + assert_eq!(requirement.version, version.trim(), "{version}"); + } + for version in ["3.9", "3.15", "3", "3.12.1", "cpython-3.12", ""] { + let error = python_requirement_for_version(SourceLayout::Next, version) + .expect_err("an unpublished Python must be refused"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("3.10, 3.11, 3.12, 3.13, 3.14"), + "{version}: {rendered}" + ); + } + + assert_eq!( + python_requirement_for_version(SourceLayout::Canonical, "3.12") + .expect("the canonical layout's own Python is accepted"), + python_requirement(SourceLayout::Canonical) + ); + let error = python_requirement_for_version(SourceLayout::Canonical, "3.13") + .expect_err("the canonical layout has one Python"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("applies only to ROCm 10 and newer installs") + && rendered.contains("this install uses Python 3.12"), + "{rendered}" + ); + } + + /// An update keeps the interpreter a ROCm 10 runtime was built on. The tag + /// recorded in the runtime maps back to the requirement it came from for every + /// published Python, a tag the layout is not published for maps to none (so the + /// update falls back to the layout's default), and the canonical layout never + /// reads one. + #[test] + fn a_recorded_wheel_tag_maps_back_to_the_requirement_it_came_from() { + for requirement in NEXT_SDK_PYTHONS { + assert_eq!( + python_requirement_for_tag(SourceLayout::Next, requirement.tag), + Some(requirement) + ); + } + assert_eq!(python_requirement_for_tag(SourceLayout::Next, "cp39"), None); + assert_eq!( + python_requirement_for_tag(SourceLayout::Canonical, "cp313"), + None + ); + // The default stays the one vLLM needs, so a flag-less install is unchanged. + assert_eq!( + python_requirement(SourceLayout::Next), + python_requirement_for_tag(SourceLayout::Next, "cp314").expect("cp314 is published") + ); + } + /// ROCm 10.x framework wheels (vLLM, flash-attn, amd-aiter) are published /// cp314-only, so the next layout must provision 3.14 while everything else /// stays on 3.12. @@ -1410,43 +1617,6 @@ exit 1 // ---- a dry run prepares no Python ---------------------------------------- - /// Process environment variables set for one test and restored on drop. - /// Every caller holds `PROCESS_ENV_TEST_LOCK` for as long as it holds this. - struct RestoredEnv(Vec<(&'static str, Option)>); - - impl RestoredEnv { - #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 - fn set(vars: &[(&'static str, Option<&Path>)]) -> Self { - let saved = vars - .iter() - .map(|(key, _)| (*key, std::env::var_os(key))) - .collect(); - for (key, value) in vars { - unsafe { - match value { - Some(value) => std::env::set_var(key, value), - None => std::env::remove_var(key), - } - } - } - Self(saved) - } - } - - impl Drop for RestoredEnv { - #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 - fn drop(&mut self) { - for (key, value) in self.0.drain(..) { - unsafe { - match value { - Some(value) => std::env::set_var(key, value), - None => std::env::remove_var(key), - } - } - } - } - } - /// A `uv` that records its arguments in `marker` and then fails. A host that /// reaches it has gone on to prepare a managed Python; failing keeps a test /// that does reach it from going on to the network. @@ -1541,6 +1711,16 @@ exit 1 ) } + /// What an update check goes on with, searching only this host's folder. + fn plan_for_update_check(&self) -> Result { + let _resolver = OverridePythonResolverEnv::new(PythonResolverEnv { + python_override: None, + search_dirs: vec![self.search_dir.clone()], + ..PythonResolverEnv::from_process_env() + }); + plan_python_launcher(&self.paths, python_requirement(SourceLayout::Canonical)) + } + fn uv_was_run(&self) -> bool { self.uv_marker.exists() } @@ -2300,6 +2480,81 @@ exit 1 Ok(()) } + /// An update check reads the index by the interpreter's wheel tag. On a host + /// with no interpreter that tag is the one the layout requires, which a real + /// update would prepare an interpreter to match, so the check needs nothing + /// prepared. The note it prints says so, and the folders agree; the same host + /// DOES reach `uv` for the install, so an unrun stand-in means the check + /// stayed out of it. The Gherkin scenario `update-09` drives the same case + /// through the built binary. + #[cfg(unix)] + #[test] + fn an_update_check_with_no_usable_python_reads_the_required_tag_and_prepares_nothing() + -> Result<()> { + if current_platform_wheel_tags().is_err() { + return Ok(()); + } + let host = PythonHost::new("python-update-check-none")?; + let requirement = python_requirement(SourceLayout::Canonical); + + let planned = host.plan_for_update_check()?; + + let compatibility = planned.wheel_compatibility()?; + assert_eq!(compatibility.python_tag, requirement.tag); + assert!(!compatibility.platform_tags.is_empty()); + assert_eq!( + planned.update_note().as_deref(), + Some( + "no usable Python was found; updating would use uv to install managed Python 3.12 and save its path in the configuration, which this check did not do" + ) + ); + assert!(!host.uv_was_run(), "the update check ran uv"); + assert_eq!( + snapshot(&host.paths), + Vec::::new(), + "the update check created files or folders under the config, data or cache folder" + ); + + // Control: the install on the same host does reach `uv`. + assert!(host.python_for_install(false).is_err()); + assert!( + host.uv_was_run(), + "the stand-in uv is not reachable, so the checks above prove nothing" + ); + Ok(()) + } + + /// A usable interpreter needs no note, and a managed Python whose tag cannot + /// be read off its version says so in the update note as in the preview's. + #[cfg(unix)] + #[test] + fn an_update_note_is_absent_for_a_usable_python_and_flags_an_unreadable_tag() -> Result<()> { + if current_platform_wheel_tags().is_err() { + return Ok(()); + } + let host = PythonHost::new("python-update-note")?; + { + let _unreadable = RestoredEnv::set(&[( + "ROCM_CLI_MANAGED_PYTHON_VERSION", + Some(Path::new("pypy@3.10")), + )]); + let planned = host.plan_for_update_check()?; + let note = planned.update_note().expect("nothing was prepared"); + assert!( + note.contains("managed Python pypy@3.10 (its wheel tag cannot be checked until it is installed; cp312 is required)") + && note.ends_with("which this check did not do"), + "{note}" + ); + } + + write_fake_python_with_venv(&host.search_dir, "python3")?; + let planned = host.plan_for_update_check()?; + assert!(matches!(planned, PlannedPython::Ready(_)), "{planned:?}"); + assert_eq!(planned.update_note(), None); + assert!(!host.uv_was_run()); + Ok(()) + } + /// The version-to-tag reading the preview's refusal rests on, and that it is /// the same rule `python_launcher_install_ready` applies (same message). #[test] @@ -2448,4 +2703,212 @@ exit 1 fs::remove_dir_all(root).ok(); } + + /// A stand-in `uv` that records every invocation and answers + /// `python install` / `python find` without touching the network. + /// Planted where `ensure_uv_binary` looks for the managed copy, so no + /// download and no environment override is needed. + #[cfg(unix)] + fn plant_fake_uv(paths: &AppPaths, python: &Path, log: &Path) -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + let dir = managed_tools_dir(&paths.data_dir).join("uv").join("latest"); + fs::create_dir_all(&dir)?; + let uv = dir.join(rocm_core::uv_binary_name()); + let script = format!( + "#!/bin/sh\necho \"$*\" >> '{log}'\nif [ \"$1 $2\" = \"python find\" ]; then echo '{python}'; fi\nexit 0\n", + log = log.display(), + python = python.display(), + ); + write_fake_executable(&uv, script)?; + Ok(()) + } + + /// A config.json `load` rejects, holding settings that must survive. + #[cfg(unix)] + fn plant_unreadable_config(paths: &AppPaths) -> Result> { + fs::create_dir_all(&paths.config_dir)?; + let damaged = br#"{"active_runtime_key": "kept-runtime", "onboarding_dismissed": "yes"}"#; + fs::write(paths.config_path(), damaged)?; + assert!(RocmCliConfig::load(paths).is_err()); + Ok(damaged.to_vec()) + } + + /// The error names the exact file to rename and what to rename it to. + /// The advice text itself is `AppPaths::config_aside_advice`; that + /// following it keeps the data dir is + /// `config_rename_advice_keeps_a_recorded_data_dir_when_followed` in + /// rocm-core's paths.rs. + #[cfg(unix)] + fn assert_names_the_file_to_rename(message: &str, paths: &AppPaths) { + let config = paths.config_path(); + assert!( + message.contains(&format!( + "rename \"{}\" to \"{}.broken\"", + config.display(), + config.display() + )) && message.contains(&paths.config_aside_advice()), + "the error does not name the file to rename: {message}" + ); + } + + /// The reuse path: a usable managed Python is already recorded, so + /// `ensure_managed_python` returns early. Meeting an unreadable config + /// there must fail — not report success with the record silently skipped + /// — before `uv` is run at all, and leave the file as it was. + #[cfg(unix)] + #[test] + fn ensure_managed_python_reuse_path_fails_on_an_unreadable_config() -> Result<()> { + if current_platform_wheel_tags().is_err() { + return Ok(()); + } + // Held because the uv and Python versions are read from the + // environment, which other tests in this module set. + let _guard = PROCESS_ENV_TEST_LOCK.lock().unwrap(); + let (root, paths) = test_paths("ensure-python-reuse-unreadable"); + let python_dir = managed_tools_dir(&paths.data_dir).join("python"); + fs::create_dir_all(&python_dir)?; + let python = write_fake_python_with_venv(&python_dir, "python")?; + save_managed_python_manifest( + &paths, + &ManagedPythonManifest { + executable: python.clone(), + version: managed_python_version(python_requirement(SourceLayout::Canonical)), + installed_at_unix_ms: 1, + }, + )?; + let uv_log = root.join("uv.log"); + plant_fake_uv(&paths, &python, &uv_log)?; + let damaged = plant_unreadable_config(&paths)?; + + // `PythonLauncher` is not `Debug`, so `expect_err` is unavailable. + let requirement = python_requirement(SourceLayout::Canonical); + let Err(error) = + ensure_managed_python(&paths, requirement, managed_python_version(requirement)) + else { + panic!("an unreadable config must fail the reuse path"); + }; + let uv_calls = fs::read_to_string(&uv_log).unwrap_or_default(); + let after = fs::read(paths.config_path())?; + fs::remove_dir_all(root).ok(); + + let message = format!("{error:#}"); + assert!( + message.contains("cannot record the managed Python") + && message.contains("it was left unchanged"), + "unexpected error: {message}" + ); + assert_names_the_file_to_rename(&message, &paths); + assert_eq!(after, damaged, "config.json was rewritten"); + assert!( + uv_calls.is_empty(), + "uv ran before the unreadable config stopped the step: {uv_calls}" + ); + Ok(()) + } + + /// The install path: no managed Python yet, so `uv python install` runs + /// and the new interpreter is recorded at the end. An unreadable config + /// fails the bootstrap before `uv` is run at all, with the file left as it + /// was. + #[cfg(unix)] + #[test] + fn ensure_managed_python_install_path_fails_on_an_unreadable_config() -> Result<()> { + if current_platform_wheel_tags().is_err() { + return Ok(()); + } + let _guard = PROCESS_ENV_TEST_LOCK.lock().unwrap(); + let (root, paths) = test_paths("ensure-python-install-unreadable"); + let python_dir = root.join("uv-python"); + fs::create_dir_all(&python_dir)?; + let python = write_fake_python_with_venv(&python_dir, "python")?; + let uv_log = root.join("uv.log"); + plant_fake_uv(&paths, &python, &uv_log)?; + let damaged = plant_unreadable_config(&paths)?; + + // `PythonLauncher` is not `Debug`, so `expect_err` is unavailable. + let requirement = python_requirement(SourceLayout::Canonical); + let Err(error) = + ensure_managed_python(&paths, requirement, managed_python_version(requirement)) + else { + panic!("an unreadable config must fail the install path"); + }; + let uv_calls = fs::read_to_string(&uv_log).unwrap_or_default(); + let after = fs::read(paths.config_path())?; + fs::remove_dir_all(root).ok(); + + let message = format!("{error:#}"); + assert!( + message.contains("cannot record the managed Python") + && message.contains("it was left unchanged"), + "unexpected error: {message}" + ); + assert_names_the_file_to_rename(&message, &paths); + assert_eq!(after, damaged, "config.json was rewritten"); + assert!( + uv_calls.is_empty(), + "uv ran before the unreadable config stopped the step: {uv_calls}" + ); + Ok(()) + } + + /// A config.json that reads fine but cannot be saved: its folder is made + /// read-only, so the save cannot create its temp file. Both paths must + /// report that the interpreter was not recorded, rather than finish as if + /// it had been. Skipped where the folder stays writable (running as root). + #[cfg(unix)] + #[test] + fn ensure_managed_python_fails_when_the_record_cannot_be_saved() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if current_platform_wheel_tags().is_err() { + return Ok(()); + } + let _guard = PROCESS_ENV_TEST_LOCK.lock().unwrap(); + for reuse in [true, false] { + let (root, paths) = test_paths("ensure-python-record-unsaved"); + let python_dir = if reuse { + managed_tools_dir(&paths.data_dir).join("python") + } else { + root.join("uv-python") + }; + fs::create_dir_all(&python_dir)?; + let python = write_fake_python_with_venv(&python_dir, "python")?; + if reuse { + save_managed_python_manifest( + &paths, + &ManagedPythonManifest { + executable: python.clone(), + version: managed_python_version(python_requirement( + SourceLayout::Canonical, + )), + installed_at_unix_ms: 1, + }, + )?; + } + plant_fake_uv(&paths, &python, &root.join("uv.log"))?; + RocmCliConfig::default().save(&paths)?; + fs::set_permissions(&paths.config_dir, fs::Permissions::from_mode(0o555))?; + let probe = paths.config_dir.join("probe"); + if fs::write(&probe, b"").is_ok() { + fs::remove_file(&probe).ok(); + fs::set_permissions(&paths.config_dir, fs::Permissions::from_mode(0o755))?; + fs::remove_dir_all(root).ok(); + return Ok(()); + } + + let requirement = python_requirement(SourceLayout::Canonical); + let result = + ensure_managed_python(&paths, requirement, managed_python_version(requirement)); + fs::set_permissions(&paths.config_dir, fs::Permissions::from_mode(0o755))?; + fs::remove_dir_all(root).ok(); + let Err(error) = result else { + panic!("reuse={reuse}: an unsaved record must fail the step"); + }; + assert!( + format!("{error:#}").contains("config.json"), + "reuse={reuse}: {error:#}" + ); + } + Ok(()) + } } diff --git a/apps/rocm/src/therock/python_config_props.rs b/apps/rocm/src/therock/python_config_props.rs new file mode 100644 index 0000000000..a2cb3bb92e --- /dev/null +++ b/apps/rocm/src/therock/python_config_props.rs @@ -0,0 +1,353 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for how the managed-Python step, a `config.json` writer in +//! this module, treats a `config.json` it cannot read, and for the legacy +//! rocm-dash TOML migration that could produce one. +//! +//! A child of `python` so it can drive the private +//! `record_managed_python_config`. No test here sets the process environment: +//! every `AppPaths` is built by hand under the system temp dir. The recovery +//! advice in the errors does read it (`ROCM_CLI_DATA_DIR` and the home +//! folder, through `AppPaths::config_aside_advice`), which is why the tests +//! compare against that method's output rather than a fixed string. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use proptest::prelude::*; +use proptest::test_runner::{Config as RunnerConfig, TestCaseError, TestError, TestRunner}; +use rocm_core::{AppPaths, RocmCliConfig}; + +use super::record_managed_python_config; + +static NEXT_DIR: AtomicUsize = AtomicUsize::new(0); + +fn fresh_paths(tag: &str) -> (PathBuf, AppPaths) { + let root = std::env::temp_dir().join(format!( + "rocm-props-{tag}-{}-{}", + std::process::id(), + NEXT_DIR.fetch_add(1, Ordering::Relaxed) + )); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + (root, paths) +} + +fn report(name: &str, result: Result<(), TestError>) { + match result { + Ok(()) => {} + Err(TestError::Fail(reason, value)) => { + panic!("{name}: minimal failing input {value:#?}\nreason: {reason}") + } + Err(TestError::Abort(reason)) => panic!("{name}: aborted: {reason}"), + } +} + +// --------------------------------------------------------------------------- +// config.json: a corrupt file must not be silently replaced by defaults. +// --------------------------------------------------------------------------- + +/// The user settings a later command acts on. If any of these vanish, `rocm` +/// forgets which runtime is active, where TheRock lives, and its defaults. +#[derive(Debug, Clone)] +struct UserSettings { + active_runtime_key: String, + default_runtime_id: String, + default_engine: String, + therock_venv: PathBuf, +} + +fn user_settings() -> BoxedStrategy { + ( + "[a-z0-9-]{1,16}", + "[a-z0-9:-]{1,16}", + prop_oneof![Just("vllm".to_owned()), Just("lemonade".to_owned())], + "/[a-z/]{1,16}", + ) + .prop_map(|(key, id, engine, venv)| UserSettings { + active_runtime_key: key, + default_runtime_id: id, + default_engine: engine, + therock_venv: PathBuf::from(venv), + }) + .boxed() +} + +/// Ways a real `config.json` becomes unreadable to this binary. Each is applied +/// to the bytes `RocmCliConfig::save` actually wrote, not to JSON built from +/// scratch. +#[derive(Debug, Clone)] +enum Corruption { + /// Interrupted write, as a plain `fs::write` in `RocmCliConfig::save` used to + /// leave behind (it is atomic now), or a truncating hand edit. + TruncateAt(usize), + /// What a non-finite tick serializes to (see the rocm-core property test). + NullTick, + /// A hand edit or a newer binary that changed a field's type. + RetypeOnboardingDismissed, + /// A value the loader's own validators reject. + TopPOutOfRange, +} + +fn corruption() -> BoxedStrategy { + prop_oneof![ + (1_usize..400).prop_map(Corruption::TruncateAt), + Just(Corruption::NullTick), + Just(Corruption::RetypeOnboardingDismissed), + Just(Corruption::TopPOutOfRange), + ] + .boxed() +} + +fn corrupt(bytes: &[u8], how: &Corruption) -> Vec { + let mut value: serde_json::Value = serde_json::from_slice(bytes).unwrap(); + match how { + Corruption::TruncateAt(at) => return bytes[..(*at).min(bytes.len() - 1)].to_vec(), + Corruption::NullTick => { + value["dashboard"]["daemon"]["gpu_tick_secs"] = serde_json::Value::Null; + } + Corruption::RetypeOnboardingDismissed => { + value["onboarding_dismissed"] = serde_json::json!("yes"); + } + Corruption::TopPOutOfRange => { + value["dashboard"]["tui"]["chat_top_p"] = serde_json::json!(1.5); + } + } + serde_json::to_vec_pretty(&value).unwrap() +} + +/// `RocmCliConfig::load` deliberately fails loudly on a corrupt file. A writer +/// that meets the same file must not "repair" it by saving defaults over it: +/// that turns "unreadable, fix it" into "valid but empty", and every later +/// command acts on the empty state. +#[test] +fn managed_python_recording_never_replaces_an_unreadable_config_with_defaults() { + let total = AtomicUsize::new(0); + let unreadable = AtomicUsize::new(0); + let mut runner = TestRunner::new(RunnerConfig { + cases: crate::test_support::proptest_cases(128), + ..RunnerConfig::default() + }); + let result = runner.run(&(user_settings(), corruption()), |(settings, how)| { + total.fetch_add(1, Ordering::Relaxed); + let (root, paths) = fresh_paths("config-wipe"); + let mut config = RocmCliConfig { + active_runtime_key: Some(settings.active_runtime_key.clone()), + default_runtime_id: Some(settings.default_runtime_id.clone()), + default_engine: Some(settings.default_engine.clone()), + ..RocmCliConfig::default() + }; + config.setup.therock_venv = Some(settings.therock_venv.clone()); + config.save(&paths).unwrap(); + let original = std::fs::read(paths.config_path()).unwrap(); + let damaged = corrupt(&original, &how); + std::fs::write(paths.config_path(), &damaged).unwrap(); + if RocmCliConfig::load(&paths).is_ok() { + // A truncation that happens to land on valid JSON; not this property. + let _ = std::fs::remove_dir_all(root); + return Ok(()); + } + unreadable.fetch_add(1, Ordering::Relaxed); + + let _ = record_managed_python_config(&paths, Path::new("/usr/bin/python3")); + + let after = std::fs::read(paths.config_path()).unwrap(); + let _ = std::fs::remove_dir_all(root); + if after == damaged { + return Ok(()); // left alone: the user still sees the loud error + } + let reloaded: RocmCliConfig = serde_json::from_slice(&after) + .map_err(|e| TestCaseError::fail(format!("rewritten config unreadable: {e}")))?; + prop_assert_eq!( + reloaded.active_runtime_key.as_deref(), + Some(settings.active_runtime_key.as_str()), + "config.json was rewritten and the active runtime was forgotten" + ); + prop_assert_eq!( + reloaded.setup.therock_venv.as_deref(), + Some(settings.therock_venv.as_path()) + ); + Ok(()) + }); + eprintln!( + "reach: {} of {} cases produced a config.json that load() rejects", + unreadable.load(Ordering::Relaxed), + total.load(Ordering::Relaxed) + ); + report("config wipe", result); + assert!( + unreadable.load(Ordering::Relaxed) > 0, + "no case produced a config.json that load() rejects" + ); +} + +/// The property above stops at its first (shrunk) counterexample. This walks +/// every corruption class once, and asserts what the user is told together +/// with what is left on disk: the error says the file was left unchanged, and +/// it was, byte for byte. +#[test] +fn no_corruption_class_is_wiped_by_managed_python_recording() { + let classes = [ + Corruption::TruncateAt(40), + Corruption::NullTick, + Corruption::RetypeOnboardingDismissed, + Corruption::TopPOutOfRange, + ]; + for how in classes { + let (root, paths) = fresh_paths("config-wipe-class"); + let mut config = RocmCliConfig { + active_runtime_key: Some("therock-release-gfx1151-7.13.0".to_owned()), + ..RocmCliConfig::default() + }; + config.setup.therock_venv = Some(PathBuf::from("/data/rocm")); + config.save(&paths).unwrap(); + let damaged = corrupt(&std::fs::read(paths.config_path()).unwrap(), &how); + std::fs::write(paths.config_path(), &damaged).unwrap(); + assert!( + RocmCliConfig::load(&paths).is_err(), + "{how:?} is unreadable" + ); + + let error = record_managed_python_config(&paths, Path::new("/usr/bin/python3")) + .expect_err("recording over an unreadable config must fail"); + let after = std::fs::read(paths.config_path()).unwrap(); + let _ = std::fs::remove_dir_all(&root); + + let message = format!("{error:#}"); + assert!( + message.contains("cannot record the managed Python") + && message.contains("it was left unchanged") + && message.contains("config.json"), + "{how:?}: unexpected error: {message}" + ); + assert_eq!(after, damaged, "{how:?}: config.json was rewritten"); + } +} + +/// A `config.json` that cannot be read at all — here a directory, which +/// fails the read on every platform without chmod or root — is not "no +/// settings yet" either. Recording must refuse with the same message, and +/// leave what is at that path alone. +#[test] +fn managed_python_recording_refuses_a_config_path_it_cannot_read() { + let (root, paths) = fresh_paths("config-unreadable-path"); + std::fs::create_dir_all(paths.config_path()).unwrap(); + let kept = paths.config_path().join("kept"); + std::fs::write(&kept, b"user data").unwrap(); + + let error = record_managed_python_config(&paths, Path::new("/usr/bin/python3")) + .expect_err("recording over an unreadable config path must fail"); + let still_a_dir = paths.config_path().is_dir(); + let kept_after = std::fs::read(&kept); + let _ = std::fs::remove_dir_all(&root); + + let message = format!("{error:#}"); + assert!( + message.contains("cannot record the managed Python") + && message.contains("it was left unchanged"), + "unexpected error: {message}" + ); + // Pinned here; `config_rename_advice_keeps_a_recorded_data_dir_when_followed` + // in rocm-core's paths.rs follows the advice and checks the data dir. + assert!( + message.contains(&paths.config_aside_advice()), + "the error does not name the file to rename: {message}" + ); + assert!(still_a_dir, "the path was replaced"); + assert_eq!(kept_after.unwrap(), b"user data"); +} + +/// With no `config.json` at all, recording the managed Python is the first +/// write and must still succeed: "missing" is defaults, as `load` promises. +#[test] +fn managed_python_recording_creates_a_missing_config() { + let (root, paths) = fresh_paths("config-missing"); + record_managed_python_config(&paths, Path::new("/usr/bin/python3")).unwrap(); + let loaded = RocmCliConfig::load(&paths).unwrap(); + let _ = std::fs::remove_dir_all(&root); + let tool = loaded.tools.get("python").expect("python was recorded"); + assert!(tool.managed); + assert_eq!(tool.path.as_deref(), Some(Path::new("/usr/bin/python3"))); +} + +/// The end-to-end path a user could reach without hand-editing `config.json`: +/// a legacy rocm-dash `config.toml` holding `inf`. It used to migrate into a +/// config the loader rejects, which the next managed-Python step then replaced +/// with defaults. Now the migration refuses the file, so nothing unreadable is +/// written, and the managed-Python step meets a missing config, which it may +/// create. +#[test] +fn legacy_inf_tick_never_reaches_config_json() { + let (root, paths) = fresh_paths("legacy-chain"); + std::fs::create_dir_all(&root).unwrap(); + let legacy = root.join("config.toml"); + let text = "default_engine = \"vllm\"\n[daemon]\ngpu_tick = inf\n[tui]\ntheme = \"nord\"\n"; + std::fs::write(&legacy, text).unwrap(); + + let error = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy) + .expect_err("a legacy file with an infinite tick must not migrate"); + assert!( + format!("{error:#}").contains("daemon.gpu_tick = inf"), + "unexpected error: {error:#}" + ); + assert!(!paths.config_path().exists(), "config.json was written"); + + record_managed_python_config(&paths, Path::new("/usr/bin/python3")).unwrap(); + let after = RocmCliConfig::load(&paths); + let legacy_after = std::fs::read_to_string(&legacy).unwrap(); + let _ = std::fs::remove_dir_all(&root); + assert!(after.is_ok(), "config.json is readable: {:?}", after.err()); + assert_eq!(legacy_after, text, "the legacy TOML was modified"); +} + +/// rocm-dash's reader and the migration into `config.json` read the same +/// legacy file. They must agree on whether it is readable: a file the +/// dashboard rejects must not be migrated, and one it accepts must migrate +/// into a `config.json` that loads. +#[test] +fn legacy_tick_readers_agree_on_which_files_are_readable() { + let ticks = [ + "0", + "0.5", + "120", + "nan", + "inf", + "-inf", + "-1.0", + "-0.0", + "1.7976931348623157e308", + ]; + let mut disagreements = Vec::new(); + for tick in ticks { + let (root, paths) = fresh_paths("legacy-agree"); + std::fs::create_dir_all(&root).unwrap(); + let legacy = root.join("config.toml"); + // rocm-dash requires the whole `[daemon]` table, so every other key + // holds a value both readers accept; only `instance_tick` varies. + std::fs::write( + &legacy, + format!( + "[daemon]\nlisten = \"unix:/tmp/rocm-dash.sock\"\ngpu_tick = 1.0\n\ + discovery_tick = 5.0\ninstance_tick = {tick}\n" + ), + ) + .unwrap(); + let dash_accepts = rocm_dash_core::config::Config::load(&legacy).is_ok(); + let migrated = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy); + let migration_accepts = matches!(migrated, Ok(Some(_))); + let loadable = RocmCliConfig::load(&paths).is_ok(); + let _ = std::fs::remove_dir_all(&root); + if dash_accepts != migration_accepts || !loadable { + disagreements.push(format!( + "{tick}: dashboard accepts={dash_accepts}, migration accepts={migration_accepts}, \ + config.json loadable={loadable}" + )); + } + } + assert!(disagreements.is_empty(), "{disagreements:#?}"); +} diff --git a/apps/rocm/src/therock/resolve.rs b/apps/rocm/src/therock/resolve.rs index 9efff586db..c6b51dbe17 100644 --- a/apps/rocm/src/therock/resolve.rs +++ b/apps/rocm/src/therock/resolve.rs @@ -5,15 +5,21 @@ //! How a request is turned into the artifact an install fetches. Only module //! declarations and re-exports live here, no logic: `tarball` resolves the SDK //! tarball from the catalog's index, `wheel` resolves the wheel stack from the -//! wheel indexes, and the rest of `therock` reads the names below. +//! wheel indexes (with `uv_access` deciding who may fetch `uv` while it does, +//! and `torchaudio` choosing the torchaudio build a ROCm 10 or newer install +//! pins), and the rest of `therock` reads the names below. mod tarball; +mod torchaudio; +mod uv_access; mod wheel; pub(super) use self::tarball::{resolve_tarball_artifact, resolve_tarball_artifact_with_timeout}; +pub(super) use self::torchaudio::{TorchaudioPin, unverified_torchaudio_note}; #[cfg(test)] pub(super) use self::wheel::select_rocm_version; pub(super) use self::wheel::{ - AggregateDeviceTarget, canonical_auto_select_needs_next_python, resolve_pip_runtime, - resolve_pip_runtime_with_timeout, unverified_torchaudio_note, wheel_runtime_composition, + AggregateDeviceTarget, PipRuntimePlan, UnresolvedPipRuntime, + canonical_auto_select_needs_next_python, preview_pip_runtime, resolve_pip_runtime, + resolve_pip_runtime_with_timeout, wheel_runtime_composition, }; diff --git a/apps/rocm/src/therock/resolve/tarball.rs b/apps/rocm/src/therock/resolve/tarball.rs index 453b8969ed..6a6d093aee 100644 --- a/apps/rocm/src/therock/resolve/tarball.rs +++ b/apps/rocm/src/therock/resolve/tarball.rs @@ -17,9 +17,11 @@ use super::super::family::{family_resolution_hint, resolve_family}; use super::super::fetch::download_text_cached; use super::super::platform_tarball_token; use super::super::source::{ - RuntimeVersionSelector, SourceLayout, TheRockChannel, resolve_source, select_source_layout, + RuntimeVersionSelector, SourceLayout, TheRockChannel, gate_install_layout, resolve_source, +}; +use super::super::version::{ + compare_version_strings, is_parseable_runtime_version, is_stable_runtime_version, }; -use super::super::version::{compare_version_strings, is_stable_runtime_version}; use anyhow::{Context, Result, bail}; use rocm_core::AppPaths; use serde::Deserialize; @@ -72,10 +74,12 @@ pub(in crate::therock) fn resolve_tarball_artifact_with_timeout( download_timeout_secs: Option, ) -> Result { let family_resolution = resolve_family(paths, family_override)?; - let layout = match layout_override { - Some(layout) => layout, - None => select_source_layout(channel, &family_resolution, version_selector)?, - }; + let layout = gate_install_layout( + channel, + &family_resolution, + version_selector, + layout_override, + )?; let source = resolve_source(channel, layout); let catalog_cache_key = match layout { SourceLayout::Canonical => format!("tarball-index-{}", channel.as_str()), @@ -146,13 +150,14 @@ fn tarball_family_token(layout: SourceLayout, family: &str) -> &str { /// The newest archive in `files` that this request can actually install. /// -/// The release channel's stable-version filter does double duty on the next -/// catalog: that catalog publishes non-release siblings beside the real dist -/// archive (`...-tests-10.0.0.tar.gz`) whose leftover suffix is not a version at -/// all, and whose mtime is *later* than the archive they shadow, so an unfiltered -/// "newest wins" would pick the wrong file. Requiring a parseable stable version -/// excludes them without inventing a second grammar for the canonical catalogs, -/// which publish no such siblings and have never been held to one. +/// The next catalogs (release and nightly) publish non-release siblings beside +/// the real dist archive (`...-tests-10.0.0.tar.gz`) whose leftover suffix is not +/// a version at all, and whose mtime is *later* than the archive they shadow, so +/// an unfiltered "newest wins" would pick the wrong file. The release channel's +/// stable-version filter excludes them there; the next layout additionally +/// requires a parseable version on every channel, which covers the nightly +/// catalog's prerelease names. The canonical catalogs publish no such siblings +/// and have never been held to that grammar. fn select_tarball_candidate( files: &[TarballIndexFile], channel: TheRockChannel, @@ -176,6 +181,12 @@ fn select_tarball_candidate( if matches!(channel, TheRockChannel::Release) && !is_stable_runtime_version(&version) { return None; } + // The next nightly catalog lists the same `-tests-` siblings, and its + // real versions are prereleases, so the stable check above cannot be + // applied there; a leftover that is not a version at all is dropped. + if matches!(layout, SourceLayout::Next) && !is_parseable_runtime_version(&version) { + return None; + } if version_selector.is_some_and(|selector| !selector.matches_version(&version)) { return None; } @@ -437,4 +448,33 @@ mod tests { Some("10.1.0a20260822".to_owned()) ); } + + /// The next nightly catalog lists `-tests-` beside the real archive with a + /// later mtime; its prerelease version must still select the real one. + #[test] + fn next_nightly_tarball_selection_skips_tests_sibling() { + let platform = platform_tarball_token(); + let prefix = format!("therock-dist-{platform}-gfx120X-all-"); + let files = vec![ + TarballIndexFile { + name: format!("{prefix}10.3.0a20261111.tar.gz"), + mtime: 1.0, + }, + TarballIndexFile { + name: format!("{prefix}tests-10.3.0a20261111.tar.gz"), + mtime: 2.0, + }, + ]; + + let (file, version) = select_tarball_candidate( + &files, + TheRockChannel::Nightly, + SourceLayout::Next, + "gfx120X-all", + None, + ) + .expect("the real nightly archive must be selected"); + assert_eq!(version, "10.3.0a20261111"); + assert_eq!(file.name, format!("{prefix}10.3.0a20261111.tar.gz")); + } } diff --git a/apps/rocm/src/therock/resolve/torchaudio.rs b/apps/rocm/src/therock/resolve/torchaudio.rs new file mode 100644 index 0000000000..9abc0a2b5a --- /dev/null +++ b/apps/rocm/src/therock/resolve/torchaudio.rs @@ -0,0 +1,507 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Which torchaudio build a ROCm 10 or newer install pins, and why. The wheel +//! indexes publish torchaudio rebuilds under one `+rocm` label that +//! nothing in their metadata tells apart, so `wheel` asks `torchaudio_pin` for +//! the build instead of leaving it to `uv`: the hardware-verified release when +//! the index publishes it, the newest build for that ROCm build otherwise. The +//! `TorchaudioPin` it returns carries the reason, and the plan line and the +//! install-log note are both derived from it. Depends on `wheel` for the PEP 440 +//! ordering of package versions. + +use super::wheel::{package_release_key, package_rocm_suffix, package_versions_matching_rocm}; +use anyhow::{Context, Result, bail}; + +/// The torchaudio build pinned for a ROCm 10 install, and why it was chosen. +/// The plan line and install-log note are both [`TorchaudioPin::describe`], so +/// what is printed is derived from the decision rather than restated beside it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(in crate::therock) struct TorchaudioPin { + /// The published version pinned, `+rocm…` label included. + pub(super) version: String, + /// The ROCm build `version` was chosen for. + rocm_version: String, + basis: TorchaudioPinBasis, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum TorchaudioPinBasis { + /// `version` is the hardware-verified release for this ROCm line. + Verified, + /// A verified release is recorded for this ROCm line, but the index + /// publishes no build of it labelled for this exact ROCm build (`+rocm` + /// label) for this Python and platform, so `version` is the newest build + /// instead. + VerifiedNotPublished { + release: &'static str, + /// A build of `release` this Python and platform can install, labelled + /// for another ROCm build, when one is published. The verified release + /// is matched by ROCm line, so a `10.1.1` request can find it published + /// only as `+rocm10.1.0`; the reason must then name the ROCm build, not + /// the Python or platform. + other_build: Option, + }, + /// No verified release is recorded for this ROCm line, so `version` is the + /// newest build. + NoVerifiedRelease, +} + +impl TorchaudioPin { + const fn is_verified(&self) -> bool { + matches!(self.basis, TorchaudioPinBasis::Verified) + } + + pub(in crate::therock) fn describe(&self) -> String { + let Self { + version, + rocm_version, + basis, + } = self; + match basis { + TorchaudioPinBasis::Verified => format!( + "torchaudio {version}, the build verified on hardware for ROCm {rocm_version}" + ), + TorchaudioPinBasis::VerifiedNotPublished { + release, + other_build: None, + } => format!( + "torchaudio {version}, the newest build published for ROCm {rocm_version}; it is not hardware-verified, because the verified release, {release}, is not published for this Python and platform" + ), + TorchaudioPinBasis::VerifiedNotPublished { + release, + other_build: Some(other_build), + } => format!( + "torchaudio {version}, the newest build published for ROCm {rocm_version}; it is not hardware-verified, because the verified release, {release}, is published for this Python and platform only for another ROCm build ({other_build}), not for ROCm {rocm_version}" + ), + TorchaudioPinBasis::NoVerifiedRelease => format!( + "torchaudio {version}, the newest build published for ROCm {rocm_version}; it is not hardware-verified, because no torchaudio release is recorded as verified for this ROCm line" + ), + } + } +} + +/// The install-log line for a torchaudio pin that is not hardware-verified, or +/// `None` for one that is. +/// +/// An unverified torchaudio is the one stack choice ROCm CLI makes on its own +/// judgement, so a real install says so in the install log, not only in the +/// plan's `torchaudio_selection:` line. A verified pin needs no such warning. +/// The text is [`TorchaudioPin::describe`], so it cannot disagree with the pin. +pub(in crate::therock) fn unverified_torchaudio_note(pin: &TorchaudioPin) -> Option { + (!pin.is_verified()).then(|| format!("Note: {}", pin.describe())) +} + +/// The torchaudio build to pin for `rocm_version`, from `published` (this +/// Python and platform's builds on the index). +/// +/// `verified_release` comes first: it is the release confirmed on hardware for +/// this ROCm line ([`rocm_engine_vllm::verified_torchaudio_release`], read from +/// the vLLM ROCm 10.x discover-build table so the SDK and its vLLM install pin +/// the same build). Recency alone is not a safe choice among builds sharing a +/// `+rocm` label: `whl-next` publishes `2.11.0`, `2.11.0.2` and +/// `2.11.0.3` all as `+rocm10.1.0`, and `2.11.0` is a CPU-only rebuild with no +/// HIP kernels, which nothing in the version or label distinguishes. Only when +/// no verified release is recorded, or it is not published here, does this fall +/// back to [`latest_build_for_rocm`], and the returned basis says so. +/// +/// Refuses when no build for `rocm_version` is published at all, or when none of +/// the published builds' versions can be read — the second is not the first, +/// and reporting it as "none published" would send the user to the wrong place. +pub(super) fn torchaudio_pin( + published: &[String], + rocm_version: &str, + verified_release: Option<&'static str>, + index_url: &str, +) -> Result { + let built_for_rocm = package_versions_matching_rocm(published, rocm_version); + if built_for_rocm.is_empty() { + bail!( + "no torchaudio built for ROCm {rocm_version} is published for this Python and platform in {index_url}" + ); + } + let pin = |version, basis| TorchaudioPin { + version, + rocm_version: rocm_version.to_owned(), + basis, + }; + let basis = match verified_release { + Some(release) => match published_build_of(&built_for_rocm, release) { + Some(version) => return Ok(pin(version, TorchaudioPinBasis::Verified)), + // Not among `built_for_rocm`, so a labelled build of it in the + // whole listing is one for another ROCm build. + None => TorchaudioPinBasis::VerifiedNotPublished { + release, + other_build: published_build_of( + &published + .iter() + .filter(|version| package_rocm_suffix(version).is_some()) + .cloned() + .collect::>(), + release, + ), + }, + }, + None => TorchaudioPinBasis::NoVerifiedRelease, + }; + let newest = latest_build_for_rocm(published, rocm_version).with_context(|| { + format!( + "torchaudio builds for ROCm {rocm_version} are published in {index_url}, but none has a PEP 440 version ROCm CLI can order: {}", + built_for_rocm.join(", ") + ) + })?; + Ok(pin(newest, basis)) +} + +/// The build in `versions` whose public version is `release` under PEP 440 +/// (local label ignored, trailing zeros insignificant), as `uv` would match +/// `torchaudio=={release}`. Not a prefix match: `2.11.0` must not find +/// `2.11.0.2`. +fn published_build_of(versions: &[String], release: &str) -> Option { + let wanted = package_release_key(release)?; + versions + .iter() + .find(|version| package_release_key(version).as_ref() == Some(&wanted)) + .cloned() +} + +/// The newest of `versions` built for exactly `rocm_version`, chosen the way +/// `uv` would choose among them: in PEP 440 order, with final releases ahead of +/// any pre-release. +/// +/// The fallback for [`torchaudio_pin`] only, never the first choice: newest is +/// not a safe discriminator among builds sharing a `+rocm` label (see +/// there for the CPU-only `2.11.0+rocm10.1.0` precedent, and the vLLM engine's +/// ROCm 10.x discover-build table, which pins `2.11.0.2` exactly for the same +/// reason). Nothing on the index pairs a torchaudio build with a torch build, +/// so this assumes the newest torchaudio of a ROCm build is the one built +/// against that build's torch — the same choice `uv` made unaided when the +/// pinned build was also the newest one on the index. +/// +/// Not `select_latest_stack_package` (in `wheel.rs`): `parse_version` reads only +/// `major.minor.patch` and rejects the rest, and torchaudio publishes rebuilds +/// as extra release segments (`2.11.0.2`, `2.11.0.3`), so that would drop every +/// rebuild and settle on the original `2.11.0` build. +fn latest_build_for_rocm(versions: &[String], rocm_version: &str) -> Option { + package_versions_matching_rocm(versions, rocm_version) + .into_iter() + .filter_map(|version| package_release_key(&version).map(|key| (key, version))) + .max_by(|(left, _), (right, _)| (left.is_final(), left).cmp(&(right.is_final(), right))) + .map(|(_, version)| version) +} + +#[cfg(test)] +mod tests { + use super::super::wheel::published_pip_requirements; + use super::*; + + /// The live ROCm 10 index publishes torchaudio rebuilds as a fourth release + /// segment beside the original build, for more than one ROCm build at once. + #[test] + fn torchaudio_pin_is_the_newest_rebuild_of_the_requested_rocm_build() { + let published = [ + "2.11.0+rocm10.0.0", + "2.11.0.2+rocm10.0.0", + "2.11.0+rocm10.1.0", + "2.11.0.2+rocm10.1.0", + "2.11.0.3+rocm10.1.0", + ] + .map(str::to_owned); + + assert_eq!( + latest_build_for_rocm(&published, "10.0.0").as_deref(), + Some("2.11.0.2+rocm10.0.0") + ); + assert_eq!( + latest_build_for_rocm(&published, "10.1.0").as_deref(), + Some("2.11.0.3+rocm10.1.0") + ); + assert_eq!(latest_build_for_rocm(&published, "10.2.0"), None); + } + + /// Mirrors uv's default: a pre-release is only chosen when no final + /// release exists for that build, and a post-release counts as final. + #[test] + fn torchaudio_pin_prefers_final_releases_like_uv() { + let published = [ + "2.12.0a0+rocm10.0.0", + "2.11.0.2+rocm10.0.0", + "2.11.0+rocm10.0.0", + ] + .map(str::to_owned); + assert_eq!( + latest_build_for_rocm(&published, "10.0.0").as_deref(), + Some("2.11.0.2+rocm10.0.0") + ); + + let only_pre = ["2.12.0a0+rocm10.0.0", "2.12.0a1+rocm10.0.0"].map(str::to_owned); + assert_eq!( + latest_build_for_rocm(&only_pre, "10.0.0").as_deref(), + Some("2.12.0a1+rocm10.0.0") + ); + + let post = ["2.11.0+rocm10.0.0", "2.11.0.post1+rocm10.0.0"].map(str::to_owned); + assert_eq!( + latest_build_for_rocm(&post, "10.0.0").as_deref(), + Some("2.11.0.post1+rocm10.0.0") + ); + } + + /// The live `whl-next` shape for `+rocm10.1.0`: a CPU-only `2.11.0`, the + /// hardware-verified `2.11.0.2`, and a newer `2.11.0.3` sharing the label. + fn rocm10_1_torchaudio_builds() -> Vec { + [ + "2.11.0+rocm10.1.0", + "2.11.0.2+rocm10.1.0", + "2.11.0.3+rocm10.1.0", + "2.11.0.2+rocm10.0.0", + ] + .map(str::to_owned) + .to_vec() + } + + /// A verified release that is published wins over a newer build with the + /// same `+rocm` label, and the printed line says it is the verified one. + #[test] + fn torchaudio_pin_takes_the_published_verified_release_over_a_newer_build() -> Result<()> { + let pin = torchaudio_pin( + &rocm10_1_torchaudio_builds(), + "10.1.0", + Some("2.11.0.2"), + "https://index.test", + )?; + assert_eq!(pin.version, "2.11.0.2+rocm10.1.0"); + assert_eq!(pin.basis, TorchaudioPinBasis::Verified); + assert!(pin.is_verified()); + let line = pin.describe(); + assert!( + line.contains("torchaudio 2.11.0.2+rocm10.1.0, the build verified on hardware") + && !line.contains("not hardware-verified"), + "{line}" + ); + // The requirement handed to `uv` names the same build the line does. + assert_eq!( + published_pip_requirements("10.1.0", &pin.version, "gfx1200", false)[3], + "torchaudio==2.11.0.2+rocm10.1.0" + ); + Ok(()) + } + + /// The verified release is matched as a whole PEP 440 version, never a + /// prefix: a verified `2.11.0` must not be satisfied by `2.11.0.2`. + #[test] + fn torchaudio_pin_matches_the_verified_release_exactly() -> Result<()> { + let published = ["2.11.0.2+rocm10.1.0", "2.11.0.3+rocm10.1.0"].map(str::to_owned); + let pin = torchaudio_pin(&published, "10.1.0", Some("2.11.0"), "https://index.test")?; + assert_eq!( + pin.basis, + TorchaudioPinBasis::VerifiedNotPublished { + release: "2.11.0", + other_build: None, + } + ); + assert_eq!(pin.version, "2.11.0.3+rocm10.1.0"); + Ok(()) + } + + /// A verified release published only for another ROCm build falls back to + /// the newest build for the requested one, and the printed reason names the + /// ROCm build as the cause, not the Python or platform: the verified release + /// is matched by ROCm line, so it can be published here under another label + /// of the same line. + #[test] + fn torchaudio_pin_falls_back_to_newest_when_the_verified_release_is_published_for_another_rocm_build() + -> Result<()> { + let published = [ + "2.11.0+rocm10.1.1", + "2.11.0.3+rocm10.1.1", + "2.11.0.2+rocm10.1.0", + ] + .map(str::to_owned); + let pin = torchaudio_pin(&published, "10.1.1", Some("2.11.0.2"), "https://index.test")?; + assert_eq!(pin.version, "2.11.0.3+rocm10.1.1"); + assert_eq!( + pin.basis, + TorchaudioPinBasis::VerifiedNotPublished { + release: "2.11.0.2", + other_build: Some("2.11.0.2+rocm10.1.0".to_owned()), + } + ); + assert!(!pin.is_verified()); + let line = pin.describe(); + assert!( + line.contains( + "torchaudio 2.11.0.3+rocm10.1.1, the newest build published for ROCm 10.1.1" + ) && line.contains("not hardware-verified") + && line.contains( + "the verified release, 2.11.0.2, is published for this Python and platform only for another ROCm build (2.11.0.2+rocm10.1.0), not for ROCm 10.1.1" + ) + && !line.contains("is not published for this Python and platform"), + "{line}" + ); + Ok(()) + } + + /// A verified release with no build at all for this Python and platform + /// falls back to the newest build, and the printed line says the release is + /// not published here. An unlabelled build is not a ROCm build, so it does + /// not count as one "for another ROCm build". + #[test] + fn torchaudio_pin_falls_back_to_newest_when_the_verified_release_is_unpublished() -> Result<()> + { + let published = ["2.11.0+rocm10.1.0", "2.11.0.3+rocm10.1.0", "2.11.0.2"].map(str::to_owned); + let pin = torchaudio_pin(&published, "10.1.0", Some("2.11.0.2"), "https://index.test")?; + assert_eq!(pin.version, "2.11.0.3+rocm10.1.0"); + assert_eq!( + pin.basis, + TorchaudioPinBasis::VerifiedNotPublished { + release: "2.11.0.2", + other_build: None, + } + ); + assert!(!pin.is_verified()); + let line = pin.describe(); + assert!( + line.contains( + "torchaudio 2.11.0.3+rocm10.1.0, the newest build published for ROCm 10.1.0" + ) && line.contains("not hardware-verified") + && line.contains( + "the verified release, 2.11.0.2, is not published for this Python and platform" + ), + "{line}" + ); + Ok(()) + } + + /// A real install notes an unverified torchaudio in its install log, for + /// both reasons a pin can be unverified, and stays quiet for a verified + /// one. The note is the pin's own `describe()` line, so it states the + /// decision that was actually taken. + #[test] + fn unverified_torchaudio_note_is_logged_only_for_an_unverified_pin() -> Result<()> { + let builds = rocm10_1_torchaudio_builds(); + let verified = torchaudio_pin(&builds, "10.1.0", Some("2.11.0.2"), "https://index.test")?; + let not_published = + torchaudio_pin(&builds, "10.1.0", Some("2.11.0.9"), "https://index.test")?; + let no_verified_release = torchaudio_pin(&builds, "10.1.0", None, "https://index.test")?; + assert_eq!(verified.basis, TorchaudioPinBasis::Verified); + assert!(matches!( + not_published.basis, + TorchaudioPinBasis::VerifiedNotPublished { .. } + )); + assert_eq!( + no_verified_release.basis, + TorchaudioPinBasis::NoVerifiedRelease + ); + + assert_eq!(unverified_torchaudio_note(&verified), None); + for pin in [¬_published, &no_verified_release] { + let note = unverified_torchaudio_note(pin); + assert_eq!(note, Some(format!("Note: {}", pin.describe()))); + assert!( + note.as_deref() + .is_some_and(|note| note.contains("torchaudio 2.11.0.3+rocm10.1.0") + && note.contains("not hardware-verified")), + "{note:?}" + ); + } + Ok(()) + } + + /// With no verified release recorded for the line, the newest build is + /// pinned, and the printed line says it is not hardware-verified. + #[test] + fn torchaudio_pin_falls_back_to_newest_when_no_release_is_verified() -> Result<()> { + let pin = torchaudio_pin( + &rocm10_1_torchaudio_builds(), + "10.1.0", + None, + "https://index.test", + )?; + assert_eq!(pin.version, "2.11.0.3+rocm10.1.0"); + assert_eq!(pin.basis, TorchaudioPinBasis::NoVerifiedRelease); + assert!(!pin.is_verified()); + let line = pin.describe(); + assert!( + line.contains( + "torchaudio 2.11.0.3+rocm10.1.0, the newest build published for ROCm 10.1.0" + ) && line.contains("not hardware-verified") + && line.contains("no torchaudio release is recorded as verified"), + "{line}" + ); + Ok(()) + } + + /// The verified release the SDK reads is the vLLM engine's: 10.1 has one, + /// 10.0 has none yet. Pins the wiring, not the table (which the engine's + /// own tests own), so a change to it is seen here as a deliberate choice. + #[test] + fn torchaudio_verified_release_comes_from_the_vllm_discover_table() { + assert_eq!( + rocm_engine_vllm::verified_torchaudio_release("10.1.0"), + Some("2.11.0.2") + ); + assert_eq!( + rocm_engine_vllm::verified_torchaudio_release("10.0.0"), + None + ); + } + + /// A version the parser cannot read is left out, never guessed at: a newer + /// unreadable build must not win the pin. + #[test] + fn torchaudio_pin_skips_versions_it_cannot_read() -> Result<()> { + let published = [ + "2.11.0+rocm10.0.0", + "2.12.0-local-hack+rocm10.0.0", + "2.11.0.2+rocm10.0.0", + ] + .map(str::to_owned); + assert_eq!( + torchaudio_pin(&published, "10.0.0", None, "https://index.test")?.version, + "2.11.0.2+rocm10.0.0" + ); + Ok(()) + } + + /// The two refusals name different problems, so each says which one it is. + #[test] + fn torchaudio_pin_refusal_says_whether_the_build_is_missing_or_unreadable() { + let other_build = ["2.11.0.3+rocm10.1.0"].map(str::to_owned); + let missing = torchaudio_pin( + &other_build, + "10.0.0", + Some("2.11.0.2"), + "https://index.test", + ) + .expect_err("no torchaudio for the pinned build must refuse") + .to_string(); + assert!( + missing.contains("no torchaudio built for ROCm 10.0.0 is published"), + "{missing}" + ); + + let unreadable = ["2.12.0-local-hack+rocm10.0.0"].map(str::to_owned); + let unreadable = torchaudio_pin(&unreadable, "10.0.0", None, "https://index.test") + .expect_err("an unreadable-only build must refuse") + .to_string(); + assert!( + unreadable.contains("none has a PEP 440 version") + && unreadable.contains("2.12.0-local-hack+rocm10.0.0") + && !unreadable.contains("no torchaudio built"), + "{unreadable}" + ); + } + + /// A post-release that is itself a dev build is still a pre-release to + /// `uv`, so a final release wins over it. + #[test] + fn torchaudio_pin_treats_a_dev_post_release_as_a_pre_release() { + let published = ["2.11.0+rocm10.0.0", "2.11.0.post1.dev0+rocm10.0.0"].map(str::to_owned); + assert_eq!( + latest_build_for_rocm(&published, "10.0.0").as_deref(), + Some("2.11.0+rocm10.0.0") + ); + } +} diff --git a/apps/rocm/src/therock/resolve/uv_access.rs b/apps/rocm/src/therock/resolve/uv_access.rs new file mode 100644 index 0000000000..3ad8dbdb42 --- /dev/null +++ b/apps/rocm/src/therock/resolve/uv_access.rs @@ -0,0 +1,215 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Who may fetch `uv` while a wheel stack is resolved. +//! +//! On the ROCm 10 layout `uv pip compile` is the one step of resolution that can +//! download the managed `uv` and create the data folder's `tools` folder. A real +//! install and an update ([`UvAccess::Acquire`]) take it as they always did; a +//! preview ([`UvAccess::PresentOnly`]) runs a `uv` that is already there and +//! otherwise skips the step. [`plan_version_resolution`] decides which, as a pure +//! function of the layout, the access and whether a `uv` is present, so that every +//! combination is tested without a network. `wheel.rs` acts on the answer. + +use super::super::source::SourceLayout; +use anyhow::{Context, Result}; +use std::path::PathBuf; + +/// Whether a resolution may fetch `uv` to choose the ROCm 10 package versions. +/// +/// Only the ROCm 10 (`next`) layout needs `uv`: it has `uv pip compile` pick the +/// `torch` and `torchvision` builds. That is the one step of resolution that can +/// download the managed `uv` and create the data folder's `tools` folder. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum UvAccess { + /// A real install or an update: take the managed `uv`, downloading it if no + /// copy is installed yet. This is what resolution has always done. + Acquire, + /// A preview: use a `uv` that is already there and never fetch one. + PresentOnly, +} + +/// What a resolution may do about `uv`: how far it may go ([`UvAccess`]) and the two +/// things it can ask for. The real ones, `find_uv_binary` and `ensure_uv_binary`, +/// are built where a resolution starts; they are fields so that a test can count +/// the calls instead of reaching the network. +pub(super) struct UvPolicy<'a> { + pub(super) access: UvAccess, + /// A `uv` that is already there, without acquiring one. + pub(super) find_present: &'a dyn Fn() -> Option, + /// The managed `uv`, downloading it when none is installed. + pub(super) acquire: &'a dyn Fn() -> Result, +} + +/// How a resolution chooses the package versions, given [`UvAccess`] and whether a +/// `uv` is present. A pure function of those inputs: see [`plan_version_resolution`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum VersionResolution { + /// The canonical layout: read each package's simple index. No `uv`. + ScrapeIndexes, + /// The ROCm 10 layout: run `uv pip compile` with this `uv`. + Compile(UvChoice), + /// The ROCm 10 layout with no `uv` to run and none to fetch: stop with what is + /// known without one, and leave the versions `uv` would choose unresolved. + Skip, +} + +/// Which `uv` the compile step runs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum UvChoice { + /// Ask `ensure_uv_binary`, which downloads the managed `uv` when none is installed. + Acquire, + /// A `uv` that was already there. + Existing(PathBuf), +} + +/// Decide how the versions are chosen, without doing it. +/// +/// `find_present` looks for a `uv` that is already there. It runs only when the +/// answer depends on it: never for the canonical layout, which needs no `uv`, and +/// never for [`UvAccess::Acquire`], whose `ensure_uv_binary` does its own looking. +/// So a real install and an update decide, and call, exactly what they did before a +/// preview could skip the step, and no answer for them is [`VersionResolution::Skip`]. +pub(super) fn plan_version_resolution( + layout: SourceLayout, + access: UvAccess, + find_present: impl FnOnce() -> Option, +) -> VersionResolution { + match (layout, access) { + (SourceLayout::Canonical, _) => VersionResolution::ScrapeIndexes, + (SourceLayout::Next, UvAccess::Acquire) => VersionResolution::Compile(UvChoice::Acquire), + (SourceLayout::Next, UvAccess::PresentOnly) => find_present() + .map_or(VersionResolution::Skip, |uv| { + VersionResolution::Compile(UvChoice::Existing(uv)) + }), + } +} + +/// The `uv` the compile step runs: the one already there, or the managed `uv` that +/// `acquire` (`ensure_uv_binary`) fetches when none is installed. +pub(super) fn uv_for_compile( + choice: UvChoice, + acquire: impl FnOnce() -> Result, +) -> Result { + match choice { + UvChoice::Existing(uv) => Ok(uv), + UvChoice::Acquire => { + acquire().context("failed to acquire uv for ROCm X metadata resolution") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::bail; + + /// Asks [`plan_version_resolution`] for one cell of the matrix and reports + /// the answer together with how many times a present `uv` was looked for. + fn decide( + layout: SourceLayout, + access: UvAccess, + uv_present: bool, + ) -> (VersionResolution, usize) { + let looked = std::cell::Cell::new(0); + let decision = plan_version_resolution(layout, access, || { + looked.set(looked.get() + 1); + uv_present.then(|| PathBuf::from("/opt/uv")) + }); + (decision, looked.get()) + } + + /// The state matrix of the decision: both layouts, both accesses, a `uv` + /// present or not. Only the ROCm 10 layout needs `uv`; a real install and an + /// update (`Acquire`) always go on to acquire it and never ask whether one is + /// present, because `ensure_uv_binary` does its own looking; a preview + /// (`PresentOnly`) asks, and with none present stops. `Skip` therefore exists + /// for exactly one row. + #[test] + fn the_version_step_is_decided_by_layout_access_and_a_present_uv_alone() { + use SourceLayout::{Canonical, Next}; + use UvAccess::{Acquire, PresentOnly}; + let existing = || VersionResolution::Compile(UvChoice::Existing(PathBuf::from("/opt/uv"))); + let rows = [ + // (layout, access, uv present, decision, times a present uv was looked for) + ( + Canonical, + Acquire, + false, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + Acquire, + true, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + PresentOnly, + false, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + PresentOnly, + true, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Next, + Acquire, + false, + VersionResolution::Compile(UvChoice::Acquire), + 0, + ), + ( + Next, + Acquire, + true, + VersionResolution::Compile(UvChoice::Acquire), + 0, + ), + (Next, PresentOnly, false, VersionResolution::Skip, 1), + (Next, PresentOnly, true, existing(), 1), + ]; + for (layout, access, uv_present, expected, looked) in rows { + let (decision, calls) = decide(layout, access, uv_present); + let row = format!("{layout:?}, {access:?}, uv present: {uv_present}"); + assert_eq!(decision, expected, "decision for {row}"); + assert_eq!(calls, looked, "times a present uv was looked for in {row}"); + } + } + + /// The compile step runs the `uv` it was given, and asks for one only on + /// `Acquire`, with the context a failed fetch has always carried. + #[test] + fn the_compile_step_acquires_uv_only_when_told_to() { + let acquired = std::cell::Cell::new(0); + let acquire = || { + acquired.set(acquired.get() + 1); + Ok(PathBuf::from("/managed/uv")) + }; + assert_eq!( + uv_for_compile(UvChoice::Acquire, acquire).unwrap(), + PathBuf::from("/managed/uv") + ); + assert_eq!(acquired.get(), 1); + assert_eq!( + uv_for_compile(UvChoice::Existing(PathBuf::from("/opt/uv")), acquire).unwrap(), + PathBuf::from("/opt/uv") + ); + assert_eq!(acquired.get(), 1, "a present uv is not acquired again"); + + let error = uv_for_compile(UvChoice::Acquire, || bail!("no network")).unwrap_err(); + assert_eq!( + format!("{error:#}"), + "failed to acquire uv for ROCm X metadata resolution: no network" + ); + } +} diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index cf5c1f1a92..381929ba0f 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -13,9 +13,14 @@ //! chosen in `resolve_pip_runtime_from_index`: on the canonical layout //! `select_matching_pip_package_versions` takes the newest `rocm` whose //! `torch`, `torchvision` and `torchaudio` listings agree; on the next layout -//! `torchaudio_pin` pins a torchaudio first (the hardware-verified release when -//! it is published) and `resolve_published_pip_package_versions` has -//! `uv pip compile` choose the rest. `wheel_runtime_composition` turns a +//! `torchaudio::torchaudio_pin` pins a torchaudio first (the hardware-verified +//! release when it is published), `rocm_versions_with_full_stack` first drops +//! any `rocm` build that `torch`, `torchvision` or `torchaudio` do not all +//! publish yet, and `resolve_published_pip_package_versions` has +//! `uv pip compile` choose the rest. That compile is the one step that may fetch +//! `uv`: a real install and an update acquire it; a preview +//! (`preview_pip_runtime`) runs a present one or stops with an +//! `UnresolvedPipRuntime`, as `uv_access` decides. `wheel_runtime_composition` turns a //! resolution into the exact package specs (`therock_pip_package_specs`) an //! install gives `uv` and records in the manifest, and //! `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` @@ -40,18 +45,23 @@ use super::super::python::WheelCompatibility; use super::super::slugify; use super::super::source::{ FamilyResolution, RuntimeVersionSelector, SourceLayout, THEROCK_NEXT_MIN_MAJOR, TheRockChannel, - resolve_source, select_source_layout, + gate_install_layout, resolve_source, }; use super::super::version::{ ParsedVersion, compare_version_strings, is_stable_runtime_version, parse_version, }; +use super::torchaudio::{TorchaudioPin, torchaudio_pin}; +use super::uv_access::{ + UvAccess, UvPolicy, VersionResolution, plan_version_resolution, uv_for_compile, +}; use crate::runtime_registry::WheelRuntimeComposition; use anyhow::{Context, Result, bail}; use rocm_core::{ - AppPaths, detect_host_gfx_target, ensure_uv_binary, extract_first_gfx_token, + AppPaths, detect_host_gfx_target, ensure_uv_binary, extract_first_gfx_token, find_uv_binary, normalize_therock_family, runtime_is_windows, uv_command_env, }; use std::io::Write; +use std::path::Path; use std::process::{Command, Stdio}; use std::time::Duration; @@ -192,6 +202,22 @@ impl AggregateDeviceTarget { Self::Exact(target) } + /// A recorded target (an update apply's, or a runtime manifest's) wins over + /// `detected` and is checked against what the source publishes; with none the + /// detected answer stands, `Undetermined` included. The one place this rule is + /// written, so a plan, a preview and an update's plan cannot disagree. + pub(in crate::therock) fn prefer_recorded( + detected: &Self, + recorded: Option<&str>, + family: &str, + published: &[String], + ) -> Self { + recorded.map_or_else( + || detected.clone(), + |target| Self::resolve(Some(target), family, published), + ) + } + pub(in crate::therock) fn as_str(&self) -> &str { match self { Self::Exact(target) => target, @@ -211,7 +237,7 @@ impl AggregateDeviceTarget { /// PEP 440's: field order is comparison order, and each enum's variants are /// declared lowest first. #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -struct PackageReleaseKey { +pub(super) struct PackageReleaseKey { epoch: u64, /// Trailing zeros dropped, so `2.11` and `2.11.0` compare equal. release: Vec, @@ -238,7 +264,7 @@ enum DevRelease { impl PackageReleaseKey { /// `uv` only picks a pre-release (including any `.devN`) when no final /// release is available; a post-release counts as final. - fn is_final(&self) -> bool { + pub(super) fn is_final(&self) -> bool { self.pre == PreRelease::Final && self.dev == DevRelease::Absent } } @@ -247,7 +273,7 @@ impl PackageReleaseKey { /// (`[N!]N(.N)*[{a|b|rc}N][.postN][.devN][+local]`), already percent-decoded /// by [`parse_simple_index_versions`]. Anything else is `None`, and is skipped /// rather than guessed at. -fn package_release_key(version: &str) -> Option { +pub(super) fn package_release_key(version: &str) -> Option { fn number(rest: &mut &str) -> Option { let len = rest.chars().take_while(char::is_ascii_digit).count(); let (digits, tail) = rest.split_at(len); @@ -311,7 +337,7 @@ fn package_release_key(version: &str) -> Option { }) } -fn package_rocm_suffix(version: &str) -> Option { +pub(super) fn package_rocm_suffix(version: &str) -> Option { let decoded = decode_simple_index_version(version); let lower = decoded.to_ascii_lowercase(); let marker = "+rocm"; @@ -480,6 +506,37 @@ pub(in crate::therock) struct PipRuntimeResolution { pub(in crate::therock) published_device_targets: Vec, } +/// What a resolution knows once it has been asked for with a given [`UvAccess`]. +#[derive(Debug, Clone)] +pub(in crate::therock) enum PipRuntimePlan { + Resolved(PipRuntimeResolution), + /// A preview of the ROCm 10 layout on a host with no `uv`. Only + /// [`UvAccess::PresentOnly`] yields it. + NeedsUv(UnresolvedPipRuntime), +} + +/// A ROCm 10 resolution stopped before `uv pip compile`: what was chosen without +/// `uv` (family, device target, index, `rocm` version, torchaudio pin, the +/// requirements `uv` would be given) and nothing made from the `torch` and +/// `torchvision` versions only `uv` produces. It carries no +/// [`TheRockPipPackageVersions`], so such a value cannot be mistaken for one. +#[derive(Debug, Clone)] +pub(in crate::therock) struct UnresolvedPipRuntime { + pub(in crate::therock) family: String, + pub(in crate::therock) family_source: String, + pub(in crate::therock) index_url: String, + pub(in crate::therock) layout: SourceLayout, + /// The `rocm` version the install would be for. + pub(in crate::therock) rocm_version: String, + pub(in crate::therock) newest_repo_version: Option, + /// The requirement lines `uv pip compile` would be given, as + /// [`PipRuntimeResolution::version_resolution_specs`] holds them once it ran. + pub(in crate::therock) version_resolution_specs: Vec, + pub(in crate::therock) torchaudio_pin: TorchaudioPin, + pub(in crate::therock) device_target: AggregateDeviceTarget, + pub(in crate::therock) published_device_targets: Vec, +} + #[derive(Debug, Clone, Eq, PartialEq)] pub(in crate::therock) struct TheRockPipPackageVersions { rocm: String, @@ -520,9 +577,13 @@ const fn therock_sdk_extras(include_devel: bool) -> &'static str { /// The requirement lines handed to `uv pip compile` to pick a mutually /// installable version set on the ROCm 10 (`next`) layout. /// -/// torch and torchvision are unversioned on purpose: choosing those versions is -/// what the resolve is for, and their published metadata depends on the exact -/// `rocm` build, so the `rocm` pin carries them to the matching `+rocm`. +/// torch and torchvision carry no exact version on purpose: choosing those +/// versions is what the resolve is for, and their published metadata depends on +/// the exact `rocm` build, so the `rocm` pin carries them to the matching +/// `+rocm`. A line with a vLLM discover row additionally bounds both to +/// that row's series (`torch==2.13.*`), the pairing the discover table records. +/// Only a row whose own comments cite hardware evidence (10.1) carries that +/// guarantee; the 10.0 row's pairing is unverified there. /// torchaudio's metadata names no `rocm` build, so left unversioned it resolves /// to the newest build on the index — which is a different ROCm build for every /// pin but the newest — and the caller pins it to `torchaudio_version`, already @@ -533,20 +594,30 @@ const fn therock_sdk_extras(include_devel: bool) -> &'static str { /// constrains the chosen versions by a toolchain they declined, and can fail the /// whole install with a toolchain-resolution error on a default /// `rocm install sdk`. -fn published_pip_requirements( +pub(super) fn published_pip_requirements( rocm_version: &str, torchaudio_version: &str, device_target: &str, include_devel: bool, ) -> Vec { let device_extra = format!("device-{device_target}"); + // Left to `uv`, torch resolves to the newest build on the index while + // torchvision's metadata names no exact torch, so the pair can be one whose + // torchvision cannot load (`operator torchvision::nms does not exist`). The + // vLLM discover table already records which series pair for this line; pin + // to it so the SDK and the engine install settle on the same torch. + let (torch_series, torchvision_series) = + match rocm_engine_vllm::verified_torch_stack_series(rocm_version) { + Some((torch, torchvision)) => (format!("=={torch}.*"), format!("=={torchvision}.*")), + None => (String::new(), String::new()), + }; vec![ format!( "rocm[{},{device_extra}]=={rocm_version}", therock_sdk_extras(include_devel) ), - format!("torch[{device_extra}]"), - format!("torchvision[{device_extra}]"), + format!("torch[{device_extra}]{torch_series}"), + format!("torchvision[{device_extra}]{torchvision_series}"), format!("torchaudio=={torchaudio_version}"), ] } @@ -618,6 +689,9 @@ pub(in crate::therock) fn resolve_pip_runtime( /// `layout_override` is how an update keeps a runtime on the stream it was /// installed from; a fresh install passes `None` and lets the request decide. +/// +/// Always [`UvAccess::Acquire`], as a real install and an update always were; a +/// preview asks [`preview_pip_runtime`]. #[allow(clippy::too_many_arguments)] pub(in crate::therock) fn resolve_pip_runtime_with_timeout( paths: &AppPaths, @@ -629,11 +703,74 @@ pub(in crate::therock) fn resolve_pip_runtime_with_timeout( include_devel: bool, download_timeout_secs: Option, ) -> Result { - let family_resolution = resolve_family(paths, family_override)?; - let layout = match layout_override { - Some(layout) => layout, - None => select_source_layout(channel, &family_resolution, version_selector)?, + match resolve_pip_runtime_plan( + paths, + channel, + family_override, + wheel_compatibility, + version_selector, + layout_override, + include_devel, + download_timeout_secs, + UvAccess::Acquire, + )? { + PipRuntimePlan::Resolved(resolution) => Ok(resolution), + // `Skip` is only ever answered to `PresentOnly` (a test holds it to that). + PipRuntimePlan::NeedsUv(_) => { + bail!("internal error: a resolution that may acquire uv stopped for want of one") + } + } +} + +/// [`resolve_pip_runtime`] for a preview: never downloads `uv` or creates the +/// folders that hold it. A `uv` that is already there is used; with none, a ROCm 10 +/// request comes back as [`PipRuntimePlan::NeedsUv`]. +pub(in crate::therock) fn preview_pip_runtime( + paths: &AppPaths, + channel: TheRockChannel, + family_override: Option<&str>, + wheel_compatibility: &WheelCompatibility, + version_selector: Option<&RuntimeVersionSelector>, + layout_override: Option, + include_devel: bool, +) -> Result { + resolve_pip_runtime_plan( + paths, + channel, + family_override, + wheel_compatibility, + version_selector, + layout_override, + include_devel, + None, + UvAccess::PresentOnly, + ) +} + +#[allow(clippy::too_many_arguments)] +fn resolve_pip_runtime_plan( + paths: &AppPaths, + channel: TheRockChannel, + family_override: Option<&str>, + wheel_compatibility: &WheelCompatibility, + version_selector: Option<&RuntimeVersionSelector>, + layout_override: Option, + include_devel: bool, + download_timeout_secs: Option, + uv_access: UvAccess, +) -> Result { + let uv = UvPolicy { + access: uv_access, + find_present: &|| find_uv_binary(paths), + acquire: &|| ensure_uv_binary(paths), }; + let family_resolution = resolve_family(paths, family_override)?; + let layout = gate_install_layout( + channel, + &family_resolution, + version_selector, + layout_override, + )?; let source = resolve_source(channel, layout); let root_url = format!("{}/", source.wheel_index.trim_end_matches('/')); // Distinct cache keys per layout: the two streams answer the same request @@ -680,6 +817,7 @@ pub(in crate::therock) fn resolve_pip_runtime_with_timeout( version_selector, include_devel, download_timeout_secs, + &uv, ) .with_context(|| { format!( @@ -704,7 +842,8 @@ fn resolve_pip_runtime_from_index( version_selector: Option<&RuntimeVersionSelector>, include_devel: bool, download_timeout_secs: Option, -) -> Result { + uv: &UvPolicy<'_>, +) -> Result { let index_url = source.index_url.as_str(); let mut version_resolution_specs = None; let mut torchaudio_selection = None; @@ -720,99 +859,6 @@ fn resolve_pip_runtime_from_index( "release channel only installs stable TheRock wheel versions, but no stable `rocm` package versions were found in {index_url}; try `rocm install sdk --channel release --format tarball` for stable release artifacts, or use `--channel nightly --format wheel` for preview builds" ); } - let package_versions = if matches!(source.layout, SourceLayout::Next) { - let device_target = match &source.device_target { - AggregateDeviceTarget::Exact(target) => target.as_str(), - AggregateDeviceTarget::Undetermined(reason) => { - bail!( - "cannot resolve ROCm X package metadata without an exact device target: {reason}" - ) - } - }; - let rocm_version = select_rocm_version(channel, &rocm_versions, version_selector) - .with_context(|| { - let requested = version_selector.map_or_else( - || "latest compatible version".to_owned(), - RuntimeVersionSelector::describe, - ); - format!("no TheRock rocm package was found for {requested} in {index_url}") - })?; - // Chosen here, not by `uv`: see `published_pip_requirements`. - let torchaudio_versions = load_simple_index_versions( - paths, - index_url, - "torchaudio", - Some(wheel_compatibility), - download_timeout_secs, - )?; - let pin = torchaudio_pin( - &torchaudio_versions, - &rocm_version, - rocm_engine_vllm::verified_torchaudio_release(&rocm_version), - index_url, - )?; - // The canonical branch below budgets one `download_timeout_secs` per - // package (rocm, then torch, torchvision, torchaudio: 4 sequential HTTP - // fetches). `uv pip compile` still fetches metadata for all four - // packages in one subprocess call — the torchaudio listing above only - // chose its version — so it needs a comparable aggregate budget, not - // the single-fetch one: reusing the bare per-fetch value here would make - // a startup check that budgets 2s per fetch reliably time out a call - // doing 4 fetches' worth of work. That listing is one more fetch on top, - // with its own single-fetch budget. - let requirements = - published_pip_requirements(&rocm_version, &pin.version, device_target, include_devel); - let versions = resolve_published_pip_package_versions( - paths, - index_url, - &requirements, - wheel_compatibility, - download_timeout_secs.map(|secs| secs.saturating_mul(4)), - )?; - version_resolution_specs = Some(requirements); - torchaudio_selection = Some(pin); - versions - } else { - let torch_versions = load_simple_index_versions( - paths, - index_url, - "torch", - Some(wheel_compatibility), - download_timeout_secs, - )?; - let torchvision_versions = load_simple_index_versions( - paths, - index_url, - "torchvision", - Some(wheel_compatibility), - download_timeout_secs, - )?; - let torchaudio_versions = load_simple_index_versions( - paths, - index_url, - "torchaudio", - Some(wheel_compatibility), - download_timeout_secs, - )?; - select_matching_pip_package_versions( - channel, - &rocm_versions, - &torch_versions, - &torchvision_versions, - &torchaudio_versions, - version_selector, - ) - .with_context(|| { - let requested = version_selector.map_or_else( - || "latest compatible version".to_owned(), - RuntimeVersionSelector::describe, - ); - format!( - "no mutually compatible TheRock rocm, torch, torchvision, and torchaudio versions were found for {requested} in {index_url}" - ) - })? - }; - let latest_version = package_versions.rocm.clone(); // The repo's newest version for this channel, ignoring wheel availability. // Only meaningful when we auto-selected "latest" (no explicit request), so a // caller can warn when that newest version has no installable wheels. @@ -823,7 +869,166 @@ fn resolve_pip_runtime_from_index( } else { None }; - Ok(PipRuntimeResolution { + let package_versions = match plan_version_resolution(source.layout, uv.access, uv.find_present) + { + VersionResolution::ScrapeIndexes => { + let torch_versions = load_simple_index_versions( + paths, + index_url, + "torch", + Some(wheel_compatibility), + download_timeout_secs, + )?; + let torchvision_versions = load_simple_index_versions( + paths, + index_url, + "torchvision", + Some(wheel_compatibility), + download_timeout_secs, + )?; + let torchaudio_versions = load_simple_index_versions( + paths, + index_url, + "torchaudio", + Some(wheel_compatibility), + download_timeout_secs, + )?; + select_matching_pip_package_versions( + channel, + &rocm_versions, + &torch_versions, + &torchvision_versions, + &torchaudio_versions, + version_selector, + ) + .with_context(|| { + let requested = version_selector.map_or_else( + || "latest compatible version".to_owned(), + RuntimeVersionSelector::describe, + ); + format!( + "no mutually compatible TheRock rocm, torch, torchvision, and torchaudio versions were found for {requested} in {index_url}" + ) + })? + } + resolution @ (VersionResolution::Compile(_) | VersionResolution::Skip) => { + let device_target = match &source.device_target { + AggregateDeviceTarget::Exact(target) => target.as_str(), + AggregateDeviceTarget::Undetermined(reason) => { + bail!( + "cannot resolve ROCm X package metadata without an exact device target: {reason}" + ) + } + }; + // Chosen here, not by `uv`: see `published_pip_requirements`. + let torchaudio_versions = load_simple_index_versions( + paths, + index_url, + "torchaudio", + Some(wheel_compatibility), + download_timeout_secs, + )?; + // The newest `rocm` build is not necessarily installable: a nightly + // `rocm` build is published before its framework wheels, so the + // newest one can have no torch stack yet. With no pin the install + // takes the newest build that has one, and `newest_repo_version` + // (set above for exactly this case) lets the caller warn about the + // gap. A pin is taken as named, so its absence stays an error. + let with_full_stack = if version_selector.is_none() { + let torch_versions = load_simple_index_versions( + paths, + index_url, + "torch", + Some(wheel_compatibility), + download_timeout_secs, + )?; + let torchvision_versions = load_simple_index_versions( + paths, + index_url, + "torchvision", + Some(wheel_compatibility), + download_timeout_secs, + )?; + rocm_versions_with_full_stack( + &rocm_versions, + [&torch_versions, &torchvision_versions, &torchaudio_versions], + ) + } else { + Vec::new() + }; + // None with a full stack (or a pin): keep the full list, so the + // newest build is the one chosen and the failure that follows is + // about it. The torchaudio pin below refuses a build with no + // torchaudio by name; a missing torch or torchvision fails in + // `uv pip compile`. + let installable_rocm_versions = if with_full_stack.is_empty() { + &rocm_versions + } else { + &with_full_stack + }; + let rocm_version = + select_rocm_version(channel, installable_rocm_versions, version_selector) + .with_context(|| { + let requested = version_selector.map_or_else( + || "latest compatible version".to_owned(), + RuntimeVersionSelector::describe, + ); + format!("no TheRock rocm package was found for {requested} in {index_url}") + })?; + let pin = torchaudio_pin( + &torchaudio_versions, + &rocm_version, + rocm_engine_vllm::verified_torchaudio_release(&rocm_version), + index_url, + )?; + let requirements = published_pip_requirements( + &rocm_version, + &pin.version, + device_target, + include_devel, + ); + let VersionResolution::Compile(choice) = resolution else { + // No `uv` to run and none to fetch: report what is known, and + // leave the versions `uv` would choose to the install. + return Ok(PipRuntimePlan::NeedsUv(UnresolvedPipRuntime { + family: family_resolution.family.clone(), + family_source: family_resolution.source.clone(), + index_url: index_url.to_owned(), + layout: source.layout, + rocm_version, + newest_repo_version, + version_resolution_specs: requirements, + torchaudio_pin: pin, + device_target: source.device_target.clone(), + published_device_targets: source.published_device_targets.clone(), + })); + }; + // The canonical branch above budgets one `download_timeout_secs` per + // package (rocm, then torch, torchvision, torchaudio: 4 sequential HTTP + // fetches). `uv pip compile` still fetches metadata for all four + // packages in one subprocess call — the torchaudio listing above only + // chose its version — so it needs a comparable aggregate budget, not + // the single-fetch one: reusing the bare per-fetch value here would make + // a startup check that budgets 2s per fetch reliably time out a call + // doing 4 fetches' worth of work. The listings read above (torchaudio, + // and torch and torchvision for an unpinned install) are further + // fetches on top, each with its own single-fetch budget. + let compile_uv = uv_for_compile(choice, uv.acquire)?; + let versions = resolve_published_pip_package_versions( + paths, + &compile_uv, + index_url, + &requirements, + wheel_compatibility, + download_timeout_secs.map(|secs| secs.saturating_mul(4)), + )?; + version_resolution_specs = Some(requirements); + torchaudio_selection = Some(pin); + versions + } + }; + let latest_version = package_versions.rocm.clone(); + Ok(PipRuntimePlan::Resolved(PipRuntimeResolution { family: family_resolution.family.clone(), family_source: family_resolution.source.clone(), index_url: index_url.to_owned(), @@ -835,7 +1040,7 @@ fn resolve_pip_runtime_from_index( package_versions, device_target: source.device_target.clone(), published_device_targets: source.published_device_targets.clone(), - }) + })) } pub(in crate::therock) fn select_rocm_version( @@ -937,19 +1142,22 @@ fn parse_uv_compiled_package_versions(output: &str) -> Result, ) -> Result { - let uv = - ensure_uv_binary(paths).context("failed to acquire uv for ROCm X metadata resolution")?; let python_version = uv_python_version(compatibility)?; let python_platform = uv_python_platform(compatibility)?; let requirements = format!("{}\n", requirements.join("\n")); - let mut child = Command::new(&uv) + let mut child = Command::new(uv) .args([ "pip", "compile", @@ -1073,184 +1281,27 @@ fn select_latest_stack_package( candidates.pop() } -/// The torchaudio build pinned for a ROCm 10 install, and why it was chosen. -/// The plan line and install-log note are both [`TorchaudioPin::describe`], so -/// what is printed is derived from the decision rather than restated beside it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(in crate::therock) struct TorchaudioPin { - /// The published version pinned, `+rocm…` label included. - version: String, - /// The ROCm build `version` was chosen for. - rocm_version: String, - basis: TorchaudioPinBasis, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum TorchaudioPinBasis { - /// `version` is the hardware-verified release for this ROCm line. - Verified, - /// A verified release is recorded for this ROCm line, but the index - /// publishes no build of it labelled for this exact ROCm build (`+rocm` - /// label) for this Python and platform, so `version` is the newest build - /// instead. - VerifiedNotPublished { - release: &'static str, - /// A build of `release` this Python and platform can install, labelled - /// for another ROCm build, when one is published. The verified release - /// is matched by ROCm line, so a `10.1.1` request can find it published - /// only as `+rocm10.1.0`; the reason must then name the ROCm build, not - /// the Python or platform. - other_build: Option, - }, - /// No verified release is recorded for this ROCm line, so `version` is the - /// newest build. - NoVerifiedRelease, -} - -impl TorchaudioPin { - const fn is_verified(&self) -> bool { - matches!(self.basis, TorchaudioPinBasis::Verified) - } - - pub(in crate::therock) fn describe(&self) -> String { - let Self { - version, - rocm_version, - basis, - } = self; - match basis { - TorchaudioPinBasis::Verified => format!( - "torchaudio {version}, the build verified on hardware for ROCm {rocm_version}" - ), - TorchaudioPinBasis::VerifiedNotPublished { - release, - other_build: None, - } => format!( - "torchaudio {version}, the newest build published for ROCm {rocm_version}; it is not hardware-verified, because the verified release, {release}, is not published for this Python and platform" - ), - TorchaudioPinBasis::VerifiedNotPublished { - release, - other_build: Some(other_build), - } => format!( - "torchaudio {version}, the newest build published for ROCm {rocm_version}; it is not hardware-verified, because the verified release, {release}, is published for this Python and platform only for another ROCm build ({other_build}), not for ROCm {rocm_version}" - ), - TorchaudioPinBasis::NoVerifiedRelease => format!( - "torchaudio {version}, the newest build published for ROCm {rocm_version}; it is not hardware-verified, because no torchaudio release is recorded as verified for this ROCm line" - ), - } - } -} - -/// The install-log line for a torchaudio pin that is not hardware-verified, or -/// `None` for one that is. -/// -/// An unverified torchaudio is the one stack choice ROCm CLI makes on its own -/// judgement, so a real install says so in the install log, not only in the -/// plan's `torchaudio_selection:` line. A verified pin needs no such warning. -/// The text is [`TorchaudioPin::describe`], so it cannot disagree with the pin. -pub(in crate::therock) fn unverified_torchaudio_note(pin: &TorchaudioPin) -> Option { - (!pin.is_verified()).then(|| format!("Note: {}", pin.describe())) -} - -/// The torchaudio build to pin for `rocm_version`, from `published` (this -/// Python and platform's builds on the index). -/// -/// `verified_release` comes first: it is the release confirmed on hardware for -/// this ROCm line ([`rocm_engine_vllm::verified_torchaudio_release`], read from -/// the vLLM ROCm 10.x discover-build table so the SDK and its vLLM install pin -/// the same build). Recency alone is not a safe choice among builds sharing a -/// `+rocm` label: `whl-next` publishes `2.11.0`, `2.11.0.2` and -/// `2.11.0.3` all as `+rocm10.1.0`, and `2.11.0` is a CPU-only rebuild with no -/// HIP kernels, which nothing in the version or label distinguishes. Only when -/// no verified release is recorded, or it is not published here, does this fall -/// back to [`latest_build_for_rocm`], and the returned basis says so. -/// -/// Refuses when no build for `rocm_version` is published at all, or when none of -/// the published builds' versions can be read — the second is not the first, -/// and reporting it as "none published" would send the user to the wrong place. -fn torchaudio_pin( - published: &[String], - rocm_version: &str, - verified_release: Option<&'static str>, - index_url: &str, -) -> Result { - let built_for_rocm = package_versions_matching_rocm(published, rocm_version); - if built_for_rocm.is_empty() { - bail!( - "no torchaudio built for ROCm {rocm_version} is published for this Python and platform in {index_url}" - ); - } - let pin = |version, basis| TorchaudioPin { - version, - rocm_version: rocm_version.to_owned(), - basis, - }; - let basis = match verified_release { - Some(release) => match published_build_of(&built_for_rocm, release) { - Some(version) => return Ok(pin(version, TorchaudioPinBasis::Verified)), - // Not among `built_for_rocm`, so a labelled build of it in the - // whole listing is one for another ROCm build. - None => TorchaudioPinBasis::VerifiedNotPublished { - release, - other_build: published_build_of( - &published - .iter() - .filter(|version| package_rocm_suffix(version).is_some()) - .cloned() - .collect::>(), - release, - ), - }, - }, - None => TorchaudioPinBasis::NoVerifiedRelease, - }; - let newest = latest_build_for_rocm(published, rocm_version).with_context(|| { - format!( - "torchaudio builds for ROCm {rocm_version} are published in {index_url}, but none has a PEP 440 version ROCm CLI can order: {}", - built_for_rocm.join(", ") - ) - })?; - Ok(pin(newest, basis)) -} - -/// The build in `versions` whose public version is `release` under PEP 440 -/// (local label ignored, trailing zeros insignificant), as `uv` would match -/// `torchaudio=={release}`. Not a prefix match: `2.11.0` must not find -/// `2.11.0.2`. -fn published_build_of(versions: &[String], release: &str) -> Option { - let wanted = package_release_key(release)?; - versions +/// The `rocm` versions that every package in `framework_listings` publishes a +/// build for, in their original order. +fn rocm_versions_with_full_stack( + rocm_versions: &[String], + framework_listings: [&Vec; 3], +) -> Vec { + rocm_versions .iter() - .find(|version| package_release_key(version).as_ref() == Some(&wanted)) + .filter(|rocm_version| { + framework_listings + .iter() + .all(|listing| !package_versions_matching_rocm(listing, rocm_version).is_empty()) + }) .cloned() + .collect() } -/// The newest of `versions` built for exactly `rocm_version`, chosen the way -/// `uv` would choose among them: in PEP 440 order, with final releases ahead of -/// any pre-release. -/// -/// The fallback for [`torchaudio_pin`] only, never the first choice: newest is -/// not a safe discriminator among builds sharing a `+rocm` label (see -/// there for the CPU-only `2.11.0+rocm10.1.0` precedent, and the vLLM engine's -/// ROCm 10.x discover-build table, which pins `2.11.0.2` exactly for the same -/// reason). Nothing on the index pairs a torchaudio build with a torch build, -/// so this assumes the newest torchaudio of a ROCm build is the one built -/// against that build's torch — the same choice `uv` made unaided when the -/// pinned build was also the newest one on the index. -/// -/// Not [`select_latest_stack_package`]: [`parse_version`] reads only -/// `major.minor.patch` and rejects the rest, and torchaudio publishes rebuilds -/// as extra release segments (`2.11.0.2`, `2.11.0.3`), so that would drop every -/// rebuild and settle on the original `2.11.0` build. -fn latest_build_for_rocm(versions: &[String], rocm_version: &str) -> Option { - package_versions_matching_rocm(versions, rocm_version) - .into_iter() - .filter_map(|version| package_release_key(&version).map(|key| (key, version))) - .max_by(|(left, _), (right, _)| (left.is_final(), left).cmp(&(right.is_final(), right))) - .map(|(_, version)| version) -} - -fn package_versions_matching_rocm(versions: &[String], rocm_version: &str) -> Vec { +pub(super) fn package_versions_matching_rocm( + versions: &[String], + rocm_version: &str, +) -> Vec { versions .iter() .filter(|version| package_rocm_suffix(version).as_deref() == Some(rocm_version)) @@ -1294,16 +1345,17 @@ fn select_latest_version(versions: &[String], channel: TheRockChannel) -> Option } } -/// Whether a `Canonical`-layout install (auto-select, or a nightly -/// build-date/prerelease pin) will actually resolve a ROCm build at major >= -/// [`THEROCK_NEXT_MIN_MAJOR`], regardless of [`VersionStage`](super::super::version::VersionStage). +/// Whether a `Canonical`-layout install (an earlier release pin, or a nightly +/// build-date/prerelease pin from before the next layout began) will actually +/// resolve a ROCm build at major >= [`THEROCK_NEXT_MIN_MAJOR`], regardless of +/// [`VersionStage`](super::super::version::VersionStage). /// -/// `requested_source_layout` only answers from the shape of the request: an -/// explicit *stable* pin at ROCm >= `THEROCK_NEXT_MIN_MAJOR` routes to `Next` -/// and gets `cp314` that way, but the canonical/nightly feed can resolve a -/// ROCm >= `THEROCK_NEXT_MIN_MAJOR` build (stable or prerelease) on its own, -/// with no `Next` routing involved — and the `cp314` requirement is a property -/// of the build itself, not of which index served it. Fetching the channel's +/// `requested_source_layout` only answers from the shape of the request: a +/// request with no pin, or a stable pin at ROCm >= `THEROCK_NEXT_MIN_MAJOR`, +/// routes to `Next` and gets `cp314` that way, but the earlier nightly feed +/// carries ROCm >= `THEROCK_NEXT_MIN_MAJOR` alpha builds (up to 2026-08-22) that +/// a build-date pin resolves with no `Next` routing involved — and the `cp314` +/// requirement is a property of the build itself, not of which index served it. Fetching the channel's /// own interpreter-agnostic `rocm` listing ahead of interpreter selection is /// the only way to know; `resolve_pip_runtime` re-fetches the same URL for the /// real resolution afterwards and shares this call's cache entry, so this @@ -1369,6 +1421,7 @@ mod tests { use super::super::super::test_support::test_runtime_manifest; use super::*; use crate::runtime_registry::{InstalledRuntimeManifest, wheel_composition_includes_devel}; + use std::path::PathBuf; /// A representative slice of what the canonical aggregate index publishes. /// Notably it has no `gfx943`: MI300 steppings other than `gfx942` have no @@ -1553,6 +1606,51 @@ mod tests { ); } + /// A nightly `rocm` build is published before its framework wheels (the + /// next layout's newest build had no torch stack on 2026-11-11), so the + /// newest `rocm` build is not necessarily the newest installable one. The + /// unpinned choice must skip a build that any of the three packages lacks, + /// not just one that torchaudio lacks. + #[test] + fn rocm_versions_with_full_stack_skips_a_build_any_package_lacks() { + let rocm = vec![ + "10.3.0a20261109".to_owned(), + "10.3.0a20261110".to_owned(), + "10.3.0a20261111".to_owned(), + ]; + let torch = vec![ + "2.15.0+rocm10.3.0a20261109".to_owned(), + "2.15.0+rocm10.3.0a20261110".to_owned(), + ]; + let torchvision = vec![ + "0.29.2+rocm10.3.0a20261109".to_owned(), + "0.29.2+rocm10.3.0a20261110".to_owned(), + ]; + // torchaudio lacks the 20261110 build only. + let torchaudio = vec![ + "2.11.1+rocm10.3.0a20261109".to_owned(), + "2.11.1+rocm10.3.0a20261111".to_owned(), + ]; + + assert_eq!( + rocm_versions_with_full_stack(&rocm, [&torch, &torchvision, &torchaudio]), + vec!["10.3.0a20261109".to_owned()], + "20261111 lacks torch and torchvision, 20261110 lacks torchaudio" + ); + let torchaudio_complete = vec![ + "2.11.1+rocm10.3.0a20261109".to_owned(), + "2.11.1+rocm10.3.0a20261110".to_owned(), + ]; + assert_eq!( + select_rocm_version( + TheRockChannel::Nightly, + &rocm_versions_with_full_stack(&rocm, [&torch, &torchvision, &torchaudio_complete]), + None + ), + Some("10.3.0a20261110".to_owned()) + ); + } + #[test] fn release_channel_prefers_stable_versions() { let versions = vec![ @@ -1644,315 +1742,6 @@ mod tests { Ok(()) } - /// The live ROCm 10 index publishes torchaudio rebuilds as a fourth release - /// segment beside the original build, for more than one ROCm build at once. - #[test] - fn torchaudio_pin_is_the_newest_rebuild_of_the_requested_rocm_build() { - let published = [ - "2.11.0+rocm10.0.0", - "2.11.0.2+rocm10.0.0", - "2.11.0+rocm10.1.0", - "2.11.0.2+rocm10.1.0", - "2.11.0.3+rocm10.1.0", - ] - .map(str::to_owned); - - assert_eq!( - latest_build_for_rocm(&published, "10.0.0").as_deref(), - Some("2.11.0.2+rocm10.0.0") - ); - assert_eq!( - latest_build_for_rocm(&published, "10.1.0").as_deref(), - Some("2.11.0.3+rocm10.1.0") - ); - assert_eq!(latest_build_for_rocm(&published, "10.2.0"), None); - } - - /// Mirrors uv's default: a pre-release is only chosen when no final - /// release exists for that build, and a post-release counts as final. - #[test] - fn torchaudio_pin_prefers_final_releases_like_uv() { - let published = [ - "2.12.0a0+rocm10.0.0", - "2.11.0.2+rocm10.0.0", - "2.11.0+rocm10.0.0", - ] - .map(str::to_owned); - assert_eq!( - latest_build_for_rocm(&published, "10.0.0").as_deref(), - Some("2.11.0.2+rocm10.0.0") - ); - - let only_pre = ["2.12.0a0+rocm10.0.0", "2.12.0a1+rocm10.0.0"].map(str::to_owned); - assert_eq!( - latest_build_for_rocm(&only_pre, "10.0.0").as_deref(), - Some("2.12.0a1+rocm10.0.0") - ); - - let post = ["2.11.0+rocm10.0.0", "2.11.0.post1+rocm10.0.0"].map(str::to_owned); - assert_eq!( - latest_build_for_rocm(&post, "10.0.0").as_deref(), - Some("2.11.0.post1+rocm10.0.0") - ); - } - - /// The live `whl-next` shape for `+rocm10.1.0`: a CPU-only `2.11.0`, the - /// hardware-verified `2.11.0.2`, and a newer `2.11.0.3` sharing the label. - fn rocm10_1_torchaudio_builds() -> Vec { - [ - "2.11.0+rocm10.1.0", - "2.11.0.2+rocm10.1.0", - "2.11.0.3+rocm10.1.0", - "2.11.0.2+rocm10.0.0", - ] - .map(str::to_owned) - .to_vec() - } - - /// A verified release that is published wins over a newer build with the - /// same `+rocm` label, and the printed line says it is the verified one. - #[test] - fn torchaudio_pin_takes_the_published_verified_release_over_a_newer_build() -> Result<()> { - let pin = torchaudio_pin( - &rocm10_1_torchaudio_builds(), - "10.1.0", - Some("2.11.0.2"), - "https://index.test", - )?; - assert_eq!(pin.version, "2.11.0.2+rocm10.1.0"); - assert_eq!(pin.basis, TorchaudioPinBasis::Verified); - assert!(pin.is_verified()); - let line = pin.describe(); - assert!( - line.contains("torchaudio 2.11.0.2+rocm10.1.0, the build verified on hardware") - && !line.contains("not hardware-verified"), - "{line}" - ); - // The requirement handed to `uv` names the same build the line does. - assert_eq!( - published_pip_requirements("10.1.0", &pin.version, "gfx1200", false)[3], - "torchaudio==2.11.0.2+rocm10.1.0" - ); - Ok(()) - } - - /// The verified release is matched as a whole PEP 440 version, never a - /// prefix: a verified `2.11.0` must not be satisfied by `2.11.0.2`. - #[test] - fn torchaudio_pin_matches_the_verified_release_exactly() -> Result<()> { - let published = ["2.11.0.2+rocm10.1.0", "2.11.0.3+rocm10.1.0"].map(str::to_owned); - let pin = torchaudio_pin(&published, "10.1.0", Some("2.11.0"), "https://index.test")?; - assert_eq!( - pin.basis, - TorchaudioPinBasis::VerifiedNotPublished { - release: "2.11.0", - other_build: None, - } - ); - assert_eq!(pin.version, "2.11.0.3+rocm10.1.0"); - Ok(()) - } - - /// A verified release published only for another ROCm build falls back to - /// the newest build for the requested one, and the printed reason names the - /// ROCm build as the cause, not the Python or platform: the verified release - /// is matched by ROCm line, so it can be published here under another label - /// of the same line. - #[test] - fn torchaudio_pin_falls_back_to_newest_when_the_verified_release_is_published_for_another_rocm_build() - -> Result<()> { - let published = [ - "2.11.0+rocm10.1.1", - "2.11.0.3+rocm10.1.1", - "2.11.0.2+rocm10.1.0", - ] - .map(str::to_owned); - let pin = torchaudio_pin(&published, "10.1.1", Some("2.11.0.2"), "https://index.test")?; - assert_eq!(pin.version, "2.11.0.3+rocm10.1.1"); - assert_eq!( - pin.basis, - TorchaudioPinBasis::VerifiedNotPublished { - release: "2.11.0.2", - other_build: Some("2.11.0.2+rocm10.1.0".to_owned()), - } - ); - assert!(!pin.is_verified()); - let line = pin.describe(); - assert!( - line.contains( - "torchaudio 2.11.0.3+rocm10.1.1, the newest build published for ROCm 10.1.1" - ) && line.contains("not hardware-verified") - && line.contains( - "the verified release, 2.11.0.2, is published for this Python and platform only for another ROCm build (2.11.0.2+rocm10.1.0), not for ROCm 10.1.1" - ) - && !line.contains("is not published for this Python and platform"), - "{line}" - ); - Ok(()) - } - - /// A verified release with no build at all for this Python and platform - /// falls back to the newest build, and the printed line says the release is - /// not published here. An unlabelled build is not a ROCm build, so it does - /// not count as one "for another ROCm build". - #[test] - fn torchaudio_pin_falls_back_to_newest_when_the_verified_release_is_unpublished() -> Result<()> - { - let published = ["2.11.0+rocm10.1.0", "2.11.0.3+rocm10.1.0", "2.11.0.2"].map(str::to_owned); - let pin = torchaudio_pin(&published, "10.1.0", Some("2.11.0.2"), "https://index.test")?; - assert_eq!(pin.version, "2.11.0.3+rocm10.1.0"); - assert_eq!( - pin.basis, - TorchaudioPinBasis::VerifiedNotPublished { - release: "2.11.0.2", - other_build: None, - } - ); - assert!(!pin.is_verified()); - let line = pin.describe(); - assert!( - line.contains( - "torchaudio 2.11.0.3+rocm10.1.0, the newest build published for ROCm 10.1.0" - ) && line.contains("not hardware-verified") - && line.contains( - "the verified release, 2.11.0.2, is not published for this Python and platform" - ), - "{line}" - ); - Ok(()) - } - - /// A real install notes an unverified torchaudio in its install log, for - /// both reasons a pin can be unverified, and stays quiet for a verified - /// one. The note is the pin's own `describe()` line, so it states the - /// decision that was actually taken. - #[test] - fn unverified_torchaudio_note_is_logged_only_for_an_unverified_pin() -> Result<()> { - let builds = rocm10_1_torchaudio_builds(); - let verified = torchaudio_pin(&builds, "10.1.0", Some("2.11.0.2"), "https://index.test")?; - let not_published = - torchaudio_pin(&builds, "10.1.0", Some("2.11.0.9"), "https://index.test")?; - let no_verified_release = torchaudio_pin(&builds, "10.1.0", None, "https://index.test")?; - assert_eq!(verified.basis, TorchaudioPinBasis::Verified); - assert!(matches!( - not_published.basis, - TorchaudioPinBasis::VerifiedNotPublished { .. } - )); - assert_eq!( - no_verified_release.basis, - TorchaudioPinBasis::NoVerifiedRelease - ); - - assert_eq!(unverified_torchaudio_note(&verified), None); - for pin in [¬_published, &no_verified_release] { - let note = unverified_torchaudio_note(pin); - assert_eq!(note, Some(format!("Note: {}", pin.describe()))); - assert!( - note.as_deref() - .is_some_and(|note| note.contains("torchaudio 2.11.0.3+rocm10.1.0") - && note.contains("not hardware-verified")), - "{note:?}" - ); - } - Ok(()) - } - - /// With no verified release recorded for the line, the newest build is - /// pinned, and the printed line says it is not hardware-verified. - #[test] - fn torchaudio_pin_falls_back_to_newest_when_no_release_is_verified() -> Result<()> { - let pin = torchaudio_pin( - &rocm10_1_torchaudio_builds(), - "10.1.0", - None, - "https://index.test", - )?; - assert_eq!(pin.version, "2.11.0.3+rocm10.1.0"); - assert_eq!(pin.basis, TorchaudioPinBasis::NoVerifiedRelease); - assert!(!pin.is_verified()); - let line = pin.describe(); - assert!( - line.contains( - "torchaudio 2.11.0.3+rocm10.1.0, the newest build published for ROCm 10.1.0" - ) && line.contains("not hardware-verified") - && line.contains("no torchaudio release is recorded as verified"), - "{line}" - ); - Ok(()) - } - - /// The verified release the SDK reads is the vLLM engine's: 10.1 has one, - /// 10.0 has none yet. Pins the wiring, not the table (which the engine's - /// own tests own), so a change to it is seen here as a deliberate choice. - #[test] - fn torchaudio_verified_release_comes_from_the_vllm_discover_table() { - assert_eq!( - rocm_engine_vllm::verified_torchaudio_release("10.1.0"), - Some("2.11.0.2") - ); - assert_eq!( - rocm_engine_vllm::verified_torchaudio_release("10.0.0"), - None - ); - } - - /// A version the parser cannot read is left out, never guessed at: a newer - /// unreadable build must not win the pin. - #[test] - fn torchaudio_pin_skips_versions_it_cannot_read() -> Result<()> { - let published = [ - "2.11.0+rocm10.0.0", - "2.12.0-local-hack+rocm10.0.0", - "2.11.0.2+rocm10.0.0", - ] - .map(str::to_owned); - assert_eq!( - torchaudio_pin(&published, "10.0.0", None, "https://index.test")?.version, - "2.11.0.2+rocm10.0.0" - ); - Ok(()) - } - - /// The two refusals name different problems, so each says which one it is. - #[test] - fn torchaudio_pin_refusal_says_whether_the_build_is_missing_or_unreadable() { - let other_build = ["2.11.0.3+rocm10.1.0"].map(str::to_owned); - let missing = torchaudio_pin( - &other_build, - "10.0.0", - Some("2.11.0.2"), - "https://index.test", - ) - .expect_err("no torchaudio for the pinned build must refuse") - .to_string(); - assert!( - missing.contains("no torchaudio built for ROCm 10.0.0 is published"), - "{missing}" - ); - - let unreadable = ["2.12.0-local-hack+rocm10.0.0"].map(str::to_owned); - let unreadable = torchaudio_pin(&unreadable, "10.0.0", None, "https://index.test") - .expect_err("an unreadable-only build must refuse") - .to_string(); - assert!( - unreadable.contains("none has a PEP 440 version") - && unreadable.contains("2.12.0-local-hack+rocm10.0.0") - && !unreadable.contains("no torchaudio built"), - "{unreadable}" - ); - } - - /// A post-release that is itself a dev build is still a pre-release to - /// `uv`, so a final release wins over it. - #[test] - fn torchaudio_pin_treats_a_dev_post_release_as_a_pre_release() { - let published = ["2.11.0+rocm10.0.0", "2.11.0.post1.dev0+rocm10.0.0"].map(str::to_owned); - assert_eq!( - latest_build_for_rocm(&published, "10.0.0").as_deref(), - Some("2.11.0+rocm10.0.0") - ); - } - #[test] fn uv_metadata_plan_rejects_mixed_rocm_builds() { let error = parse_uv_compiled_package_versions( @@ -2109,13 +1898,40 @@ mod tests { assert_eq!( runtime_only[1..], [ - "torch[device-gfx1200]".to_owned(), - "torchvision[device-gfx1200]".to_owned(), + "torch[device-gfx1200]==2.12.*".to_owned(), + "torchvision[device-gfx1200]==0.27.*".to_owned(), "torchaudio==2.11.0.2+rocm10.0.0".to_owned(), ] ); } + /// Regression: with torch unconstrained, `uv` took the newest torch on the + /// index (2.14.0) next to torchvision 0.28.0, whose metadata accepts any + /// torch. That pair cannot register `torchvision::nms`, so every vLLM serve + /// on the runtime failed at import. The vLLM discover table pairs torch 2.13 + /// with torchvision 0.28 for ROCm 10.1; the SDK has to ask for that pair, and + /// a line with no row stays unconstrained. + #[test] + fn version_resolution_pins_the_torch_pair_the_discover_table_verified() { + let rocm_10_1 = + published_pip_requirements("10.1.0", "2.11.0.2+rocm10.1.0", "gfx942", false); + assert_eq!(rocm_10_1[1], "torch[device-gfx942]==2.13.*"); + assert_eq!(rocm_10_1[2], "torchvision[device-gfx942]==0.28.*"); + + // A nightly build of the same line resolves through the same row. + let nightly = published_pip_requirements( + "10.1.0a20260822", + "2.11.0.2+rocm10.1.0a20260822", + "gfx942", + false, + ); + assert_eq!(nightly[1], "torch[device-gfx942]==2.13.*"); + + let no_row = published_pip_requirements("10.2.0", "2.12.0+rocm10.2.0", "gfx942", false); + assert_eq!(no_row[1], "torch[device-gfx942]"); + assert_eq!(no_row[2], "torchvision[device-gfx942]"); + } + /// Resolution and composition are two code paths reading one flag, and the /// bug this pins is them disagreeing: the requirements handed to `uv` were /// hardcoded to `devel` while the plan printed and installed `libraries`. @@ -2322,4 +2138,777 @@ mod tests { assert!(hint.contains("--format tarball")); } } + + // ---- who may fetch `uv`: the real resolution --------------------------- + + /// Serves fixed pages over HTTP until dropped. A request for a path it does + /// not hold is a 404. + #[cfg(unix)] + struct IndexServer { + base: String, + stop: std::sync::Arc, + thread: Option>, + } + + #[cfg(unix)] + impl IndexServer { + fn start(pages: Vec<(String, String)>) -> Self { + use std::io::{Read, Write}; + use std::sync::atomic::Ordering; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("local addr").port(); + listener.set_nonblocking(true).expect("nonblocking"); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stopped = std::sync::Arc::clone(&stop); + let thread = std::thread::spawn(move || { + while !stopped.load(Ordering::SeqCst) { + let Ok((mut stream, _)) = listener.accept() else { + std::thread::sleep(Duration::from_millis(5)); + continue; + }; + let _ = stream.set_nonblocking(false); + let _ = stream.set_read_timeout(Some(Duration::from_secs(5))); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + match stream.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(read) => request.extend_from_slice(&buffer[..read]), + } + } + let request = String::from_utf8_lossy(&request); + let path = request.split_whitespace().nth(1).unwrap_or_default(); + let page = pages.iter().find(|(served, _)| served == path); + let (status, body) = match page { + Some((_, body)) => ("200 OK", body.as_str()), + None => ("404 Not Found", ""), + }; + let _ = write!( + stream, + "HTTP/1.1 {status}\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.flush(); + } + }); + Self { + base: format!("http://127.0.0.1:{port}"), + stop, + thread: Some(thread), + } + } + } + + #[cfg(unix)] + impl Drop for IndexServer { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } + } + + #[cfg(unix)] + fn wheel_links(package: &str, versions: &[&str]) -> String { + use std::fmt::Write as _; + + versions.iter().fold(String::new(), |mut html, version| { + let file = format!("{package}-{version}-py3-none-any.whl"); + let _ = writeln!(html, "{file}"); + html + }) + } + + /// One canonical and one ROCm 10 index, as the preview's resolution reads them. + #[cfg(unix)] + fn two_layout_index() -> IndexServer { + let mut pages = Vec::new(); + for (layout, stack) in [ + ( + "current", + [ + ("rocm", "7.10.0"), + ("torch", "2.9.0+rocm7.10.0"), + ("torchvision", "0.24.0+rocm7.10.0"), + ("torchaudio", "2.9.0+rocm7.10.0"), + ], + ), + ( + "next", + [ + ("rocm", "10.0.0"), + ("torch", "2.10.0+rocm10.0.0"), + ("torchvision", "0.25.0+rocm10.0.0"), + ("torchaudio", "2.9.1+rocm10.0.0"), + ], + ), + ] { + for (package, version) in stack { + pages.push(( + format!("/{layout}/{package}/"), + wheel_links(package, &[version]), + )); + } + } + IndexServer::start(pages) + } + + /// A `uv` that logs its arguments, reads the requirements it is piped, and + /// answers `uv pip compile` with the ROCm 10 stack above. + #[cfg(unix)] + fn write_logging_uv(dir: &Path) -> (PathBuf, PathBuf) { + let log = dir.join("uv-runs.log"); + let uv = dir.join("uv"); + std::fs::create_dir_all(dir).expect("create the uv folder"); + let script = format!( + "#!/bin/sh\necho \"$@\" >> '{}'\ncat > /dev/null\nprintf '%s\\n' 'rocm==10.0.0' 'torch==2.10.0+rocm10.0.0' 'torchvision==0.25.0+rocm10.0.0' 'torchaudio==2.9.1+rocm10.0.0'\n", + log.display() + ); + rocm_core::fake_executable::write_fake_executable(&uv, script) + .expect("plant the stand-in uv"); + (uv, log) + } + + /// What one cell of the matrix did: the answer, and what it touched. + #[cfg(unix)] + struct UvAccessRun { + plan: Result, + /// Times a present `uv` was looked for. + looked: usize, + /// Times the managed `uv` was acquired (downloaded, for a real one). + acquired: usize, + /// The arguments `uv` was run with, one run per line. + uv_runs: String, + /// Whether the managed tools folder, where a download goes, exists. + tools_folder: bool, + } + + /// Runs the real resolution against the served index for `layout`, with `uv` + /// "present" at the logging stand-in or not, and `acquire` answering for the + /// managed `uv` with that stand-in or with a failure. + #[cfg(unix)] + fn resolve_through_policy( + layout: SourceLayout, + access: UvAccess, + uv_present: bool, + acquire_succeeds: bool, + ) -> UvAccessRun { + let (root, paths) = super::super::super::test_support::test_paths("uv-access"); + let (uv, log) = write_logging_uv(&root.join("bin")); + let server = two_layout_index(); + let (index, python_tag) = match layout { + SourceLayout::Canonical => (format!("{}/current", server.base), "cp312"), + SourceLayout::Next => (format!("{}/next", server.base), "cp314"), + }; + let source = ResolvedAggregateWheelSource { + index_url: index, + layout, + device_target: AggregateDeviceTarget::Exact("gfx1200".to_owned()), + published_device_targets: vec!["gfx1200".to_owned()], + }; + let compatibility = WheelCompatibility { + python_tag: python_tag.to_owned(), + platform_tags: vec!["linux_x86_64".to_owned(), "any".to_owned()], + }; + let looked = std::cell::Cell::new(0); + let acquired = std::cell::Cell::new(0); + let find_present = || { + looked.set(looked.get() + 1); + uv_present.then(|| uv.clone()) + }; + let acquire = || { + acquired.set(acquired.get() + 1); + if acquire_succeeds { + Ok(uv.clone()) + } else { + bail!("the managed uv could not be downloaded") + } + }; + let policy = UvPolicy { + access, + find_present: &find_present, + acquire: &acquire, + }; + let plan = resolve_pip_runtime_from_index( + &paths, + TheRockChannel::Release, + &super::super::super::test_support::family_for_test("gfx1200", Some("gfx1200")), + &source, + &compatibility, + None, + false, + None, + &policy, + ); + let run = UvAccessRun { + plan, + looked: looked.get(), + acquired: acquired.get(), + uv_runs: std::fs::read_to_string(&log).unwrap_or_default(), + tools_folder: rocm_core::managed_tools_dir(&paths.data_dir).exists(), + }; + let _ = std::fs::remove_dir_all(&root); + run + } + + #[cfg(unix)] + fn stack_of(plan: &PipRuntimePlan) -> &TheRockPipPackageVersions { + match plan { + PipRuntimePlan::Resolved(resolution) => &resolution.package_versions, + PipRuntimePlan::NeedsUv(_) => { + panic!("the plan stopped before the versions were chosen") + } + } + } + + /// The resolution itself, every layout and access with a `uv` present or not, + /// each row asserting the outcome, what was touched, and for the row that stops + /// the message the preview prints about it. It runs the real resolution against + /// a served index, so a preview that fetched `uv` would show up as an acquire + /// or a tools folder, and a real install that stopped fetching it would show up + /// as a plan with no versions. + #[cfg(unix)] + #[test] + fn a_preview_never_acquires_uv_and_a_real_resolution_still_does() { + use SourceLayout::{Canonical, Next}; + use UvAccess::{Acquire, PresentOnly}; + + // ROCm 10, a preview, no uv present: the one row that stops. It asks + // whether one is present and touches nothing, so no `uv` ran, none was + // acquired and no tools folder was made. + let run = resolve_through_policy(Next, PresentOnly, false, false); + let Ok(PipRuntimePlan::NeedsUv(unresolved)) = &run.plan else { + panic!("expected a plan that stopped before uv, got {:?}", run.plan); + }; + assert_eq!( + ( + run.looked, + run.acquired, + run.uv_runs.as_str(), + run.tools_folder + ), + (1, 0, "", false), + "a preview with no uv present must acquire nothing and run nothing" + ); + // What it knows without `uv`: the ROCm version, the pinned torchaudio and + // the requirement lines `uv` would be given. + assert_eq!(unresolved.rocm_version, "10.0.0"); + assert_eq!(unresolved.torchaudio_pin.version, "2.9.1+rocm10.0.0"); + assert_eq!( + unresolved.version_resolution_specs, + [ + "rocm[libraries,device-gfx1200]==10.0.0", + "torch[device-gfx1200]==2.12.*", + "torchvision[device-gfx1200]==0.27.*", + "torchaudio==2.9.1+rocm10.0.0", + ] + ); + // And the message says what it did not do, in the same row. + let text = preview_text_with_prefix(unresolved, None); + for line in [ + " runtime_key: not resolved by this preview", + " package_specs: not resolved by this preview", + " command: not resolved by this preview", + " manifest: not resolved by this preview", + ] { + assert!( + text.lines().any(|l| l == line), + "missing `{line}` in:\n{text}" + ); + } + assert!( + text.contains("resolution_note: no uv is installed"), + "the preview must say why:\n{text}" + ); + assert!( + !text.contains("2.10.0+rocm10.0.0") && !text.contains("0.25.0+rocm10.0.0"), + "no version only uv could have chosen may be printed:\n{text}" + ); + + // ROCm 10, a preview, a uv present: it runs, once, and the plan is the + // one a real install reaches. Nothing is acquired. + let run = resolve_through_policy(Next, PresentOnly, true, false); + let stack = stack_of(run.plan.as_ref().expect("resolves with the present uv")); + assert_eq!( + (stack.torch.as_str(), stack.compatibility_key.as_str()), + ("2.10.0+rocm10.0.0", "10.0.0") + ); + assert_eq!( + (run.looked, run.acquired, run.tools_folder), + (1, 0, false), + "a preview with a uv present must not acquire another" + ); + assert_eq!(run.uv_runs.lines().count(), 1, "{}", run.uv_runs); + assert!(run.uv_runs.starts_with("pip compile "), "{}", run.uv_runs); + + // ROCm 10, a real install or an update: the managed uv is acquired, once, + // whether or not one is present (acquiring is what finds an installed + // one), and never looked for first. This is what must not change. + for uv_present in [false, true] { + let run = resolve_through_policy(Next, Acquire, uv_present, true); + let stack = stack_of(run.plan.as_ref().expect("resolves with the acquired uv")); + assert_eq!(stack.torch, "2.10.0+rocm10.0.0"); + assert_eq!( + (run.looked, run.acquired), + (0, 1), + "an install acquires uv, with one present: {uv_present}" + ); + assert_eq!(run.uv_runs.lines().count(), 1, "{}", run.uv_runs); + } + // A failed acquisition fails the resolution with the context it always had, + // and no plan stands in for it. + let run = resolve_through_policy(Next, Acquire, false, false); + let error = format!( + "{:#}", + run.plan.expect_err("an install cannot go without uv") + ); + assert!( + error.contains("failed to acquire uv for ROCm X metadata resolution: the managed uv could not be downloaded"), + "{error}" + ); + assert_eq!((run.looked, run.acquired), (0, 1)); + + // The canonical layout needs no uv for either access, present or not: the + // versions are read off the simple indexes, and nothing is looked for, + // acquired or run. + for access in [Acquire, PresentOnly] { + for uv_present in [false, true] { + let run = resolve_through_policy(Canonical, access, uv_present, true); + let stack = stack_of(run.plan.as_ref().expect("resolves from the indexes")); + assert_eq!( + (stack.rocm.as_str(), stack.torch.as_str()), + ("7.10.0", "2.9.0+rocm7.10.0") + ); + assert_eq!( + ( + run.looked, + run.acquired, + run.uv_runs.as_str(), + run.tools_folder + ), + (0, 0, "", false), + "{access:?}, uv present: {uv_present}" + ); + } + } + } + + // ---- the plan a preview without uv prints -------------------------------- + // + // Built by hand rather than from a served index, so that these run on every + // platform, the Windows CI job included: the wording and the shape of the plan + // do not depend on a stand-in `uv`. + + const PLAN_INDEX: &str = "http://index.test/next"; + + fn rocm10_specs() -> Vec { + published_pip_requirements("10.0.0", "2.9.1+rocm10.0.0", "gfx1200", false) + } + + fn rocm10_pin() -> TorchaudioPin { + torchaudio_pin(&["2.9.1+rocm10.0.0".to_owned()], "10.0.0", None, PLAN_INDEX) + .expect("the one published torchaudio is pinned") + } + + /// What a ROCm 10 resolution knows before `uv` has run. + fn unresolved_runtime() -> UnresolvedPipRuntime { + UnresolvedPipRuntime { + family: "gfx1200".to_owned(), + family_source: "manifest".to_owned(), + index_url: PLAN_INDEX.to_owned(), + layout: SourceLayout::Next, + rocm_version: "10.0.0".to_owned(), + newest_repo_version: None, + version_resolution_specs: rocm10_specs(), + torchaudio_pin: rocm10_pin(), + device_target: AggregateDeviceTarget::Exact("gfx1200".to_owned()), + published_device_targets: vec!["gfx1200".to_owned()], + } + } + + /// The same resolution once `uv` has chosen the versions. + fn resolved_runtime() -> PipRuntimeResolution { + let unresolved = unresolved_runtime(); + PipRuntimeResolution { + family: unresolved.family, + family_source: unresolved.family_source, + index_url: unresolved.index_url, + layout: unresolved.layout, + latest_version: "10.0.0".to_owned(), + newest_repo_version: None, + version_resolution_specs: Some(unresolved.version_resolution_specs), + torchaudio_pin: Some(unresolved.torchaudio_pin), + package_versions: TheRockPipPackageVersions { + rocm: "10.0.0".to_owned(), + torch: "2.10.0+rocm10.0.0".to_owned(), + torchvision: "0.25.0+rocm10.0.0".to_owned(), + torchaudio: "2.9.1+rocm10.0.0".to_owned(), + compatibility_key: "10.0.0".to_owned(), + }, + device_target: unresolved.device_target, + published_device_targets: unresolved.published_device_targets, + } + } + + /// The Python a clean host would prepare, and the wheel tags that follow from it. + fn clean_host_python() -> ( + super::super::super::python::PlannedPython, + WheelCompatibility, + ) { + use super::super::super::python::{PlannedPython, TagCheck, python_requirement}; + ( + PlannedPython::WouldPrepareManaged { + version: "3.14".to_owned(), + requirement: python_requirement(SourceLayout::Next), + tag_check: TagCheck::Checked, + }, + WheelCompatibility { + python_tag: "cp314".to_owned(), + platform_tags: vec!["linux_x86_64".to_owned(), "any".to_owned()], + }, + ) + } + + fn preview_text_with_prefix( + unresolved: &UnresolvedPipRuntime, + prefix: Option<&Path>, + ) -> String { + let (python, wheel_compatibility) = clean_host_python(); + super::super::super::wheel_plan::unresolved_wheel_preview( + &super::super::super::wheel_plan::UnresolvedPreview { + channel: TheRockChannel::Release, + version_selector: None, + unresolved, + device_target_override: None, + prefix, + host_version_newer: None, + python: &python, + wheel_compatibility: &wheel_compatibility, + }, + ) + } + + /// The whole text, so that no line can be added, dropped or reworded without + /// this test saying so. Everything made from the torch and torchvision + /// versions reads `not resolved by this preview`, and `resolution_note` says + /// why, in the voice of `python_note` beside it. + #[test] + fn the_plan_a_preview_without_uv_prints_is_exactly_this() { + let unresolved = unresolved_runtime(); + let text = preview_text_with_prefix(&unresolved, None); + let generation = unresolved.layout.generation(); + let pin = unresolved.torchaudio_pin.describe(); + let expected = format!( + "\ +sdk install + summary: rocm-cli will install the ROCm SDK and matching PyTorch packages for this Python and operating system + channel: release + canonical_source: {PLAN_INDEX} + selected_rocm_version: 10.0.0 + build_date: not encoded in stable version + source_layout_generation: {generation} + format: wheel + target_family: gfx1200 + target_family_source: manifest + device_target: gfx1200 + index_url: {PLAN_INDEX} + latest_compatible_version: 10.0.0 + compatibility_key: 10.0.0 + target: not resolved by this preview + runtime_key: not resolved by this preview + python_launcher: not prepared by this preview + python_source: managed + python_note: no usable Python was found; a real install would use uv to install managed Python 3.14 and save its path in the configuration, which this preview did not do + resolution_note: no uv is installed, so a real install would fetch it and run `uv pip compile` to choose the torch and torchvision versions and to check that the published packages install together, which this preview did not do; the package specs, runtime key, install folder, manifest and command depend on those versions and are not resolved here + python_wheel_tag: cp314 + platform_wheel_tags: linux_x86_64,any + version_resolution_specs: rocm[libraries,device-gfx1200]==10.0.0 torch[device-gfx1200]==2.12.* torchvision[device-gfx1200]==0.27.* torchaudio==2.9.1+rocm10.0.0 + torchaudio_selection: {pin} + package_specs: not resolved by this preview + package_policy: resolve the pinned target-complete rocm, torch, torchvision, and torchaudio plan from published package metadata, then install it in one uv transaction + mode: dry-run + command: not resolved by this preview + activation: use the managed venv Python; TheRock libraries are resolved from that venv by rocm_sdk.initialize_process + manifest: not resolved by this preview +" + ); + assert_eq!(text, expected); + } + + /// A `--prefix` does not depend on the versions, so it is the plan's target + /// even though the runtime key is not known; the key is what would otherwise + /// name the folder. + #[test] + fn an_explicit_prefix_is_the_target_of_a_preview_without_uv() { + let (root, _) = super::super::super::test_support::test_paths("uv-access-prefix"); + std::fs::create_dir_all(&root).expect("create the prefix parent"); + let prefix = root.join("my runtime"); + let text = preview_text_with_prefix(&unresolved_runtime(), Some(&prefix)); + let resolved = rocm_core::resolve_path_through_symlinks(&prefix); + let _ = std::fs::remove_dir_all(&root); + assert!( + text.lines() + .any(|line| line == format!(" target: {}", resolved.display())), + "{text}" + ); + assert!( + text.lines() + .any(|line| line == " runtime_key: not resolved by this preview"), + "{text}" + ); + } + + /// The keys a plan prints, in order. + fn plan_keys(text: &str) -> Vec<&str> { + text.lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter_map(|line| line.split_once(": ").map(|(key, _)| key)) + .collect() + } + + /// One writer serves both plans, so a line the resolved plan gains cannot be + /// missing from the other: the two print the same keys in the same order, and + /// the unresolved one adds only `resolution_note`. + #[test] + fn a_resolved_and_an_unresolved_plan_print_the_same_keys() { + use super::super::super::wheel_plan::{ + PlanStack, ResolvedStack, WheelPlan, write_dry_run_tail, write_wheel_plan, + }; + + let unresolved = preview_text_with_prefix(&unresolved_runtime(), None); + let resolution = resolved_runtime(); + let composition = wheel_runtime_composition(&resolution, &resolution.device_target, false); + let runtime_key = super::super::super::wheel_runtime_key( + TheRockChannel::Release, + &resolution.latest_version, + &composition, + ); + let install_root = PathBuf::from("/runtimes").join(&runtime_key); + let manifest_path = PathBuf::from("/manifests").join(&runtime_key); + let (python, compatibility) = clean_host_python(); + let plan = WheelPlan { + channel: TheRockChannel::Release, + version_selector: None, + index_url: &resolution.index_url, + layout_generation: resolution.layout.generation(), + latest_version: &resolution.latest_version, + family: &resolution.family, + family_source: &resolution.family_source, + device_target: &resolution.device_target, + host_version_newer: None, + python: &python, + wheel_compatibility: &compatibility, + version_resolution_specs: resolution.version_resolution_specs.as_deref(), + torchaudio_pin: resolution.torchaudio_pin.as_ref(), + no_wheel_warning: None, + stack: PlanStack::Resolved(ResolvedStack { + compatibility_key: &resolution.package_versions.compatibility_key, + install_root: &install_root, + runtime_key: &runtime_key, + package_specs: &composition.package_specs, + kept_torch: None, + manifest_path: &manifest_path, + }), + }; + let mut resolved = String::new(); + write_wheel_plan(&mut resolved, &plan); + write_dry_run_tail(&mut resolved, &plan); + + let mut expected = plan_keys(&resolved); + let at = expected + .iter() + .position(|key| *key == "python_note") + .expect("a python_note: this host would prepare Python") + + 1; + expected.insert(at, "resolution_note"); + assert_eq!(plan_keys(&unresolved), expected); + // And the resolved plan carries what the unresolved one cannot: the + // versions `uv` chose, in the package specs, and a real folder and key. + assert!( + resolved.contains("torch[device-gfx1200]==2.10.0+rocm10.0.0"), + "{resolved}" + ); + assert!(!resolved.contains("not resolved"), "{resolved}"); + assert!(!resolved.contains("resolution_note"), "{resolved}"); + } + + /// A kept torch changes what `uv` is handed and nothing the runtime is recorded as. + /// The preview's `command:` is the line a user is invited to run, so it has to + /// leave the torch stack out exactly as the install does, while `package_specs:` + /// stays the composition the manifest and the runtime key are made from. + #[test] + fn a_kept_torch_is_left_out_of_the_command_but_not_out_of_the_composition() { + use super::super::super::kept_torch::{KeptTorch, kept_torch_stack}; + use super::super::super::wheel_plan::{ + PlanStack, ResolvedStack, WheelPlan, write_dry_run_tail, write_wheel_plan, + }; + + let resolution = resolved_runtime(); + let composition = wheel_runtime_composition(&resolution, &resolution.device_target, false); + let install_root = PathBuf::from("/runtimes/kept"); + let manifest_path = PathBuf::from("/manifests/kept"); + let (python, compatibility) = clean_host_python(); + let kept = kept_torch_stack(true, || { + Ok(vec![("torch".to_owned(), "2.9.0+gitd0c8b1f".to_owned())]) + }); + let render = |kept: Option<&KeptTorch>| { + let plan = WheelPlan { + channel: TheRockChannel::Release, + version_selector: None, + index_url: &resolution.index_url, + layout_generation: resolution.layout.generation(), + latest_version: &resolution.latest_version, + family: &resolution.family, + family_source: &resolution.family_source, + device_target: &resolution.device_target, + host_version_newer: None, + python: &python, + wheel_compatibility: &compatibility, + version_resolution_specs: resolution.version_resolution_specs.as_deref(), + torchaudio_pin: resolution.torchaudio_pin.as_ref(), + no_wheel_warning: None, + stack: PlanStack::Resolved(ResolvedStack { + compatibility_key: &resolution.package_versions.compatibility_key, + install_root: &install_root, + runtime_key: "kept", + package_specs: &composition.package_specs, + kept_torch: kept, + manifest_path: &manifest_path, + }), + }; + let mut text = String::new(); + write_wheel_plan(&mut text, &plan); + write_dry_run_tail(&mut text, &plan); + text + }; + let line = |text: &str, key: &str| { + text.lines() + .find(|line| line.trim_start().starts_with(key)) + .unwrap_or_else(|| panic!("no `{key}` line in:\n{text}")) + .to_owned() + }; + + let plain = render(None); + assert!(!plain.contains("torch_kept:"), "{plain}"); + assert!( + line(&plain, "command:").contains("torch[device-gfx1200]==2.10.0+rocm10.0.0"), + "without a kept torch the command installs it: {plain}" + ); + + let kept_plan = render(kept.as_ref()); + assert!( + line(&kept_plan, "torch_kept:").contains("torch 2.9.0+gitd0c8b1f"), + "{kept_plan}" + ); + let command = line(&kept_plan, "command:"); + assert!( + command.contains("rocm["), + "the SDK is still installed: {command}" + ); + for left_out in ["torch[", "torchvision", "torchaudio"] { + assert!( + !command.contains(left_out), + "the command still installs `{left_out}`: {command}" + ); + } + assert_eq!( + line(&kept_plan, "package_specs:"), + line(&plain, "package_specs:"), + "the composition the manifest records does not change" + ); + } + + // ---- the recorded device target ------------------------------------------ + + /// The rule in one place: nothing recorded leaves the detected answer as it + /// was, a recorded target is judged against what the source publishes, and an + /// unpublished one is `Undetermined` with the reason rather than a guess. + #[test] + fn a_recorded_device_target_is_judged_against_what_the_source_publishes() { + let published = vec!["gfx1200".to_owned()]; + let undetected = AggregateDeviceTarget::Undetermined("no GPU".to_owned()); + let exact = AggregateDeviceTarget::Exact("gfx1200".to_owned()); + // Nothing recorded: the detected answer stands, undetermined included. + assert_eq!( + AggregateDeviceTarget::prefer_recorded(&undetected, None, "gfx120X-all", &published), + undetected + ); + assert_eq!( + AggregateDeviceTarget::prefer_recorded(&exact, None, "gfx120X-all", &published), + exact + ); + // Recorded and published: it wins over an undetermined host probe. + assert_eq!( + AggregateDeviceTarget::prefer_recorded( + &undetected, + Some("gfx1200"), + "gfx120X-all", + &published + ), + exact + ); + // Recorded but not published: undetermined, whatever the host probe said. + let refused = AggregateDeviceTarget::prefer_recorded( + &exact, + Some("gfx1201"), + "gfx120X-all", + &published, + ); + assert!( + matches!(&refused, AggregateDeviceTarget::Undetermined(reason) if reason.contains("gfx1201")), + "{refused:?}" + ); + } + + /// The same rule reaches a preview that stopped before `uv`: an override the + /// source does not publish shows as `` with its reason, not as the + /// target the resolution detected. Red if `unresolved_wheel_preview` stops + /// using the helper and keeps the detected target. + #[test] + fn a_preview_without_uv_applies_a_recorded_device_target_through_the_helper() { + let unresolved = unresolved_runtime(); + let (python, wheel_compatibility) = clean_host_python(); + let text = super::super::super::wheel_plan::unresolved_wheel_preview( + &super::super::super::wheel_plan::UnresolvedPreview { + channel: TheRockChannel::Release, + version_selector: None, + unresolved: &unresolved, + device_target_override: Some("gfx1201"), + prefix: None, + host_version_newer: None, + python: &python, + wheel_compatibility: &wheel_compatibility, + }, + ); + assert!( + text.contains("\n device_target: \n"), + "{text}" + ); + assert!(text.contains("\n device_target_reason: "), "{text}"); + assert!(!text.contains("device_target: gfx1200"), "{text}"); + } + + /// No call site writes the rule by hand again: every one goes through + /// `AggregateDeviceTarget::prefer_recorded`, so a change to the rule reaches + /// the install plan, the preview and an update's plan together. A scan of the + /// three files that call it, red if one resolves a recorded target itself. + #[test] + fn the_recorded_device_target_rule_is_written_once() { + for (name, source) in [ + ("therock.rs", include_str!("../../therock.rs")), + ("therock/update.rs", include_str!("../update.rs")), + ("therock/wheel_plan.rs", include_str!("../wheel_plan.rs")), + ] { + assert!( + source.contains("AggregateDeviceTarget::prefer_recorded("), + "{name} no longer resolves its device target through the helper" + ); + assert!( + !source.contains("AggregateDeviceTarget::resolve("), + "{name} resolves a device target by hand; use AggregateDeviceTarget::prefer_recorded" + ); + } + } } diff --git a/apps/rocm/src/therock/source.rs b/apps/rocm/src/therock/source.rs index f0946679e8..29147bfffd 100644 --- a/apps/rocm/src/therock/source.rs +++ b/apps/rocm/src/therock/source.rs @@ -22,22 +22,34 @@ const THEROCK_NIGHTLY_PIP_INDEX_BASE: &str = "https://rocm.nightlies.amd.com/whl const THEROCK_RELEASE_PIP_INDEX_BASE: &str = "https://repo.amd.com/rocm/whl-multi-arch"; const THEROCK_RELEASE_TARBALL_BASE: &str = "https://repo.amd.com/rocm/tarball/"; const THEROCK_NIGHTLY_TARBALL_BASE: &str = "https://rocm.nightlies.amd.com/tarball/"; -/// ROCm 10's aggregate pip index. Same simple-index shape the canonical -/// multi-arch stream publishes, on a separate host, with its own package -/// generation; it is not a mirror of, or a fallback for, the canonical index. +/// The next layout's stable aggregate pip index. Same simple-index shape the +/// canonical multi-arch stream publishes, on a separate host, with its own +/// package generation; it is not a mirror of, or a fallback for, the canonical +/// index. const THEROCK_NEXT_PIP_INDEX_BASE: &str = "https://stable.repo.amd.com/rocm/whl-next"; -/// ROCm 10's tarball catalog. Same scrapeable listing the canonical catalog -/// uses, but it publishes non-release sibling archives beside the real dist -/// archive (see `select_tarball_candidate` in `resolve/tarball.rs`). +/// The next layout's stable tarball catalog. Same scrapeable listing the +/// canonical catalog uses, but it publishes non-release sibling archives beside +/// the real dist archive (see `select_tarball_candidate` in `resolve/tarball.rs`). const THEROCK_NEXT_TARBALL_BASE: &str = "https://stable.repo.amd.com/rocm/core/tarball/"; +/// The next layout's nightly aggregate pip index: the same package shape as +/// [`THEROCK_NEXT_PIP_INDEX_BASE`] on the nightly host. +const THEROCK_NEXT_NIGHTLY_PIP_INDEX_BASE: &str = "https://nightly.repo.amd.com/rocm/whl-next"; +/// The next layout's nightly tarball catalog, beside +/// [`THEROCK_NEXT_TARBALL_BASE`] on the nightly host. +const THEROCK_NEXT_NIGHTLY_TARBALL_BASE: &str = "https://nightly.repo.amd.com/rocm/core/tarball/"; +/// The first nightly build date the next layout publishes. The canonical +/// nightly stream's last build is 2026-08-22 and the next layout's first is +/// 2026-08-23, so the two never overlap: an earlier build can only be fetched +/// from the canonical hosts and a later one only from the next layout's. +pub(super) const THEROCK_NEXT_NIGHTLY_FIRST_BUILD_DATE: &str = "2026-08-23"; pub(super) const THEROCK_SOURCE_LAYOUT_GENERATION: &str = "multi-arch-v2"; /// Recorded in a [`WheelRuntimeComposition`](crate::runtime_registry::WheelRuntimeComposition) /// installed from the ROCm 10 layout, so a later update reads back the layout /// that produced the runtime instead of guessing it from the version. const THEROCK_NEXT_LAYOUT_GENERATION: &str = "next-v1"; -/// The first ROCm major version published only in the next layout. A pin at or -/// past it is the one request the canonical stream provably cannot serve, and -/// therefore the only thing that selects [`SourceLayout::Next`]. +/// The first stable ROCm major version published only in the next layout. A +/// stable release pin at or past it is a request the canonical stream provably +/// cannot serve. pub(super) const THEROCK_NEXT_MIN_MAJOR: u32 = 10; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -48,12 +60,15 @@ pub(super) enum TheRockChannel { /// Which published hosting layout a resolution reads from. /// -/// Orthogonal to [`TheRockChannel`]. `Canonical` is the release/nightly stream -/// the CLI has always installed and remains the answer for every request that -/// does not explicitly ask for something else. `Next` is ROCm 10's separately -/// hosted layout — an extension, never a fallback: nothing degrades into it and -/// nothing degrades out of it, so a canonical install resolves exactly the URLs, -/// device target, and package specs it resolved before the layout existed. +/// Orthogonal to [`TheRockChannel`]. `Next` is the layout AMD now publishes +/// every channel to (`stable.repo.amd.com` and `nightly.repo.amd.com`), and +/// what a request with no pin installs. `Canonical` is the earlier release and +/// nightly stream (`repo.amd.com` and `rocm.nightlies.amd.com`), which no longer +/// receives new builds but still serves every ROCm 7 release and the nightlies +/// up to 2026-08-22, so a pin of one of those reads it. Nothing degrades from +/// one layout into the other: [`layout_serving`] picks exactly one from the +/// request, and an install resolves only the URLs, device target and package +/// specs of that layout. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum SourceLayout { Canonical, @@ -193,98 +208,122 @@ fn env_override_base(env_var: &str, default: &str) -> String { } /// The bases `channel` publishes under `layout`. -/// -/// The next layout is hosted per-layout rather than per-channel: it has no -/// nightly stream, and [`select_source_layout`] refuses to reach it from one, so -/// there is nothing here to branch on. pub(super) fn resolve_source(channel: TheRockChannel, layout: SourceLayout) -> ResolvedSource { - match layout { - SourceLayout::Canonical => match channel { - TheRockChannel::Release => ResolvedSource { - wheel_index: env_override_base( - "ROCM_CLI_THEROCK_RELEASE_PIP_BASE", - THEROCK_RELEASE_PIP_INDEX_BASE, - ), - tarball_catalog: env_override_base( - "ROCM_CLI_THEROCK_RELEASE_TARBALL_BASE", - THEROCK_RELEASE_TARBALL_BASE, - ), - layout, - }, - TheRockChannel::Nightly => ResolvedSource { - wheel_index: env_override_base( - "ROCM_CLI_THEROCK_NIGHTLY_PIP_BASE", - THEROCK_NIGHTLY_PIP_INDEX_BASE, - ), - tarball_catalog: env_override_base( - "ROCM_CLI_THEROCK_NIGHTLY_TARBALL_BASE", - THEROCK_NIGHTLY_TARBALL_BASE, - ), - layout, - }, - }, - SourceLayout::Next => ResolvedSource { - wheel_index: env_override_base( - "ROCM_CLI_THEROCK_NEXT_PIP_BASE", - THEROCK_NEXT_PIP_INDEX_BASE, - ), - tarball_catalog: env_override_base( - "ROCM_CLI_THEROCK_NEXT_TARBALL_BASE", - THEROCK_NEXT_TARBALL_BASE, - ), - layout, - }, + let (wheel_var, wheel_default, tarball_var, tarball_default) = match (layout, channel) { + (SourceLayout::Canonical, TheRockChannel::Release) => ( + "ROCM_CLI_THEROCK_RELEASE_PIP_BASE", + THEROCK_RELEASE_PIP_INDEX_BASE, + "ROCM_CLI_THEROCK_RELEASE_TARBALL_BASE", + THEROCK_RELEASE_TARBALL_BASE, + ), + (SourceLayout::Canonical, TheRockChannel::Nightly) => ( + "ROCM_CLI_THEROCK_NIGHTLY_PIP_BASE", + THEROCK_NIGHTLY_PIP_INDEX_BASE, + "ROCM_CLI_THEROCK_NIGHTLY_TARBALL_BASE", + THEROCK_NIGHTLY_TARBALL_BASE, + ), + (SourceLayout::Next, TheRockChannel::Release) => ( + "ROCM_CLI_THEROCK_NEXT_PIP_BASE", + THEROCK_NEXT_PIP_INDEX_BASE, + "ROCM_CLI_THEROCK_NEXT_TARBALL_BASE", + THEROCK_NEXT_TARBALL_BASE, + ), + (SourceLayout::Next, TheRockChannel::Nightly) => ( + "ROCM_CLI_THEROCK_NEXT_NIGHTLY_PIP_BASE", + THEROCK_NEXT_NIGHTLY_PIP_INDEX_BASE, + "ROCM_CLI_THEROCK_NEXT_NIGHTLY_TARBALL_BASE", + THEROCK_NEXT_NIGHTLY_TARBALL_BASE, + ), + }; + ResolvedSource { + wheel_index: env_override_base(wheel_var, wheel_default), + tarball_catalog: env_override_base(tarball_var, tarball_default), + layout, } } -/// Whether `selector` names a ROCm release that only the next layout publishes. +/// The layout that publishes what `selector` asks for on `channel`. /// -/// An exact *stable* pin is the only signal that qualifies. A build date names -/// a nightly build, an unparseable string names nothing this CLI can reason -/// about, and neither is evidence that the canonical stream cannot serve the -/// request. Nor is a pinned prerelease of a future major: the canonical -/// nightly stream already serves those (see -/// `nightly_accepts_future_prerelease_major_without_cli_changes`), so gating -/// on major alone would route a nightly alpha pin like `10.1.0a20260822` into -/// a refusal the release-channel retry it suggests could never satisfy. -pub(super) fn next_layout_requested(selector: &RuntimeVersionSelector) -> bool { - let RuntimeVersionSelector::Version(version) = selector else { - return false; +/// No pin installs the channel's newest build, and every channel's newest +/// builds are in the next layout. A pin goes where that exact build lives: +/// +/// - release: a stable version at or past [`THEROCK_NEXT_MIN_MAJOR`] is +/// next-only, an earlier one is canonical-only; +/// - nightly: a build date (given directly or carried by a prerelease version) +/// on or after [`THEROCK_NEXT_NIGHTLY_FIRST_BUILD_DATE`] is next-only, an +/// earlier one is canonical-only. A *stable* ROCm 10+ version on the nightly +/// channel names a release, so it routes to the next layout, where +/// [`select_source_layout`] refuses it. +/// +/// A pin this CLI cannot read stays canonical: nothing can be said about where +/// it lives, and the canonical lookup fails on it as it always has. +pub(super) fn layout_serving( + channel: TheRockChannel, + selector: Option<&RuntimeVersionSelector>, +) -> SourceLayout { + let Some(selector) = selector else { + return SourceLayout::Next; }; - parse_version(version).is_some_and(|parsed| { - parsed.stage == VersionStage::Stable && parsed.major >= THEROCK_NEXT_MIN_MAJOR - }) + let by_build_date = |build_date: &str| { + if build_date >= THEROCK_NEXT_NIGHTLY_FIRST_BUILD_DATE { + SourceLayout::Next + } else { + SourceLayout::Canonical + } + }; + match (channel, selector) { + (TheRockChannel::Nightly, RuntimeVersionSelector::BuildDate(date)) => by_build_date(date), + (TheRockChannel::Release, RuntimeVersionSelector::BuildDate(_)) => SourceLayout::Canonical, + (_, RuntimeVersionSelector::Version(version)) => { + let Some(parsed) = parse_version(version) else { + return SourceLayout::Canonical; + }; + let stable_next = + parsed.stage == VersionStage::Stable && parsed.major >= THEROCK_NEXT_MIN_MAJOR; + match channel { + TheRockChannel::Release if stable_next => SourceLayout::Next, + TheRockChannel::Release => SourceLayout::Canonical, + TheRockChannel::Nightly => match runtime_version_build_date(version) { + Some(date) => by_build_date(&date), + None if stable_next => SourceLayout::Next, + None => SourceLayout::Canonical, + }, + } + } + } } /// The layout an install must read from, and the refusal when it cannot. /// -/// Only an explicit pin at ROCm >= [`THEROCK_NEXT_MIN_MAJOR`] selects `Next`, -/// because that is the one request the canonical stream provably cannot serve. -/// No selector, a build date, an unparseable pin, or an older pin all stay -/// canonical, so nothing that resolves today starts resolving somewhere else. -/// A pin that does reach `Next` still needs an exact arch, because that layout -/// picks its device payload by exact arch and a grouped family names none. +/// [`layout_serving`] decides the layout. The next layout additionally needs an +/// exact arch, because it picks its device payload by exact arch and a grouped +/// family names none, and it publishes no stable ROCm release on the nightly +/// channel. pub(super) fn select_source_layout( channel: TheRockChannel, family_resolution: &FamilyResolution, version_selector: Option<&RuntimeVersionSelector>, ) -> Result { - let Some(selector) = version_selector.filter(|selector| next_layout_requested(selector)) else { - return Ok(SourceLayout::Canonical); - }; - let RuntimeVersionSelector::Version(version) = selector else { - return Ok(SourceLayout::Canonical); - }; - let major = parse_version(version).map_or(THEROCK_NEXT_MIN_MAJOR, |parsed| parsed.major); - if !matches!(channel, TheRockChannel::Release) { + let layout = layout_serving(channel, version_selector); + if layout == SourceLayout::Canonical { + return Ok(layout); + } + if let Some(RuntimeVersionSelector::Version(version)) = version_selector + && matches!(channel, TheRockChannel::Nightly) + && let Some(parsed) = parse_version(version) + && parsed.stage == VersionStage::Stable + { bail!( - "ROCm {major} and newer is published only on the release channel; re-run `rocm install sdk --channel release --version {version}`" + "ROCm {} and newer stable releases are published only on the release channel; re-run `rocm install sdk --channel release --version {version}`", + parsed.major ); } if family_resolution.raw_arch.is_none() { + let arch = example_arch_for_family(&family_resolution.family).unwrap_or(""); bail!( - "installing ROCm {major} requires an exact GPU arch, but the resolved target family `{}` names a group of them.\n\ - ROCm {major} selects its device payload by exact arch, so re-run with the one this host has, for example `rocm install sdk --version {version} --family gfx1200`.\n\n{}", + "this install reads the ROCm {THEROCK_NEXT_MIN_MAJOR} package layout, which requires an exact GPU arch, but the resolved target family `{}` names a group of them.\n\ + That layout selects its device payload by exact arch, so re-run with the one this host has, for example `rocm install sdk --family {arch}`.\n\ + An earlier build, which still publishes grouped families, is installed by pinning it in a wheel install (`--format wheel`; a tarball install takes no pin): `--version 7.14.1` on the release channel, or `--build-date YYYY-MM-DD` with a date before {THEROCK_NEXT_NIGHTLY_FIRST_BUILD_DATE} for a nightly.\n\n{}", family_resolution.family, detect_host_gpu_diagnostics() ); @@ -292,6 +331,39 @@ pub(super) fn select_source_layout( Ok(SourceLayout::Next) } +/// One real arch that `family` groups, for the remediation text. `None` for a +/// family with no entry here, which the caller words as a placeholder rather +/// than naming an arch from a different family. +fn example_arch_for_family(family: &str) -> Option<&'static str> { + match family { + "gfx101X-dgpu" => Some("gfx1010"), + "gfx103X-dgpu" => Some("gfx1030"), + "gfx110X-all" => Some("gfx1100"), + "gfx120X-all" => Some("gfx1200"), + "gfx125X-dcgpu" => Some("gfx1250"), + "gfx94X-dcgpu" => Some("gfx942"), + _ => None, + } +} + +/// The pre-install gate: [`select_source_layout`] for a fresh request, and the +/// recorded layout untouched for an update, which names no pin and installs from +/// the layout its runtime was built from. Gating an update on the request it +/// does not make would route it to the newest layout and refuse a runtime of the +/// earlier stream that has only a family label, though its own resolution reads +/// the layout it recorded and never needed an exact arch. +pub(super) fn gate_install_layout( + channel: TheRockChannel, + family_resolution: &FamilyResolution, + version_selector: Option<&RuntimeVersionSelector>, + layout_override: Option, +) -> Result { + match layout_override { + Some(layout) => Ok(layout), + None => select_source_layout(channel, family_resolution, version_selector), + } +} + #[derive(Debug, Clone)] pub(super) struct FamilyResolution { pub(super) family: String, @@ -309,14 +381,11 @@ pub(super) struct FamilyResolution { /// an interpreter has been chosen. An update passes its manifest's recorded /// layout; a fresh install passes its request and lets the pin decide. pub(super) fn requested_source_layout( + channel: TheRockChannel, layout_override: Option, version_selector: Option<&RuntimeVersionSelector>, ) -> SourceLayout { - match layout_override { - Some(layout) => layout, - None if version_selector.is_some_and(next_layout_requested) => SourceLayout::Next, - None => SourceLayout::Canonical, - } + layout_override.unwrap_or_else(|| layout_serving(channel, version_selector)) } #[cfg(test)] @@ -325,36 +394,109 @@ mod tests { use crate::test_support::PROCESS_ENV_TEST_LOCK; use crate::therock::test_support::family_for_test; - /// Everything a user can ask for today must keep resolving the canonical - /// stream. The next layout is an extension, so no existing request may start - /// answering from somewhere else. + /// Which layout serves a request, for every shape of request on both + /// channels. A request with no pin reads the newest builds, which only the + /// next layout publishes; a pin reads whichever layout holds that build, so + /// an earlier ROCm 7 release or nightly stays installable. #[test] - fn only_a_rocm_10_pin_leaves_the_canonical_layout() { - let exact = family_for_test("gfx120X-all", Some("gfx1200")); - let build_date = RuntimeVersionSelector::build_date("2026-06-05").unwrap(); - let older_pin = RuntimeVersionSelector::version("7.14.0").unwrap(); - let unparseable = RuntimeVersionSelector::version("main").unwrap(); - - for selector in [ - None, - Some(&build_date), - Some(&older_pin), - Some(&unparseable), - ] { + fn a_request_reads_the_layout_that_publishes_it() { + use RuntimeVersionSelector::{BuildDate, Version}; + use SourceLayout::{Canonical, Next}; + let v = |value: &str| Version(value.to_owned()); + let d = |value: &str| BuildDate(value.to_owned()); + let rows: Vec<(TheRockChannel, Option, SourceLayout)> = vec![ + // No pin: the newest build of either channel. + (TheRockChannel::Release, None, Next), + (TheRockChannel::Nightly, None, Next), + // Release pins: stable 10+ is next-only, earlier stable is canonical-only. + (TheRockChannel::Release, Some(v("10.1.0")), Next), + (TheRockChannel::Release, Some(v("10.0.0")), Next), + (TheRockChannel::Release, Some(v("7.14.1")), Canonical), + (TheRockChannel::Release, Some(v("7.9.0")), Canonical), + // Nightly pins: the last canonical build and the first next build + // are adjacent days, so either side of that boundary is a row. + ( + TheRockChannel::Nightly, + Some(v("10.1.0a20260822")), + Canonical, + ), + (TheRockChannel::Nightly, Some(v("10.1.0a20260823")), Next), + (TheRockChannel::Nightly, Some(v("10.3.0a20261110")), Next), + ( + TheRockChannel::Nightly, + Some(v("7.14.0a20260612")), + Canonical, + ), + (TheRockChannel::Nightly, Some(d("2026-08-22")), Canonical), + (TheRockChannel::Nightly, Some(d("2026-08-23")), Next), + (TheRockChannel::Nightly, Some(d("2026-11-10")), Next), + // A stable ROCm 10+ version on the nightly channel names a + // release; it reaches the next layout so the refusal can say so. + (TheRockChannel::Nightly, Some(v("10.0.0")), Next), + // A pin this CLI cannot read, or a build date on the release + // channel (which has no dated builds), says nothing about where it + // lives: it stays canonical and fails there as it always has. + (TheRockChannel::Release, Some(v("main")), Canonical), + (TheRockChannel::Nightly, Some(v("main")), Canonical), + (TheRockChannel::Release, Some(d("2026-11-10")), Canonical), + ]; + for (channel, selector, expected) in rows { assert_eq!( - select_source_layout(TheRockChannel::Release, &exact, selector).unwrap(), - SourceLayout::Canonical, - "selector {selector:?} must stay canonical" + layout_serving(channel, selector.as_ref()), + expected, + "{channel:?} {selector:?}" ); } + } - let pin = RuntimeVersionSelector::version("10.0.0").unwrap(); + /// The routing rule on real values: the last build the legacy host publishes + /// is 2026-08-22 and the first the new host publishes is 2026-08-23, so the + /// boundary constant must sit exactly between them. + #[test] + fn the_nightly_boundary_sits_between_the_last_canonical_and_first_next_build() { + assert_eq!(THEROCK_NEXT_NIGHTLY_FIRST_BUILD_DATE, "2026-08-23"); + let last_canonical = RuntimeVersionSelector::version("10.1.0a20260822").unwrap(); + let first_next = RuntimeVersionSelector::version("10.1.0a20260823").unwrap(); assert_eq!( - select_source_layout(TheRockChannel::Release, &exact, Some(&pin)).unwrap(), + layout_serving(TheRockChannel::Nightly, Some(&last_canonical)), + SourceLayout::Canonical + ); + assert_eq!( + layout_serving(TheRockChannel::Nightly, Some(&first_next)), SourceLayout::Next ); } + /// The selection a install uses is the routing rule plus its refusals, so a + /// request that routes to the next layout with an exact arch selects it and + /// an earlier pin needs no arch at all. + #[test] + fn selecting_a_layout_applies_the_routing_rule() { + let exact = family_for_test("gfx120X-all", Some("gfx1200")); + let grouped = family_for_test("gfx120X-all", None); + let old_pin = RuntimeVersionSelector::version("7.14.1").unwrap(); + let new_pin = RuntimeVersionSelector::version("10.1.0").unwrap(); + + assert_eq!( + select_source_layout(TheRockChannel::Release, &exact, None).unwrap(), + SourceLayout::Next + ); + assert_eq!( + select_source_layout(TheRockChannel::Nightly, &exact, None).unwrap(), + SourceLayout::Next + ); + assert_eq!( + select_source_layout(TheRockChannel::Release, &exact, Some(&new_pin)).unwrap(), + SourceLayout::Next + ); + // An earlier release needs no exact arch: the canonical layout serves + // grouped families. + assert_eq!( + select_source_layout(TheRockChannel::Release, &grouped, Some(&old_pin)).unwrap(), + SourceLayout::Canonical + ); + } + /// A grouped family names no single arch, and the next layout has no /// grouped device payload to fall back to, so the refusal has to name the /// flag and a value that would work. @@ -369,6 +511,74 @@ mod tests { assert!(error.contains("requires an exact GPU arch"), "{error}"); assert!(error.contains("--family gfx1200"), "{error}"); + // The way out for a grouped family is naming an earlier release, which + // must itself be a pin the router sends to the canonical layout. + assert!(error.contains("--version 7.14.1"), "{error}"); + let escape = RuntimeVersionSelector::version("7.14.1").unwrap(); + assert_eq!( + select_source_layout(TheRockChannel::Release, &grouped, Some(&escape)).unwrap(), + SourceLayout::Canonical + ); + } + + /// The example arch in the refusal must belong to the family being refused: + /// naming `gfx1200` for an MI300X host would install the wrong kernels. + #[test] + fn the_refusal_example_arch_belongs_to_the_refused_family() { + for family in [ + "gfx101X-dgpu", + "gfx103X-dgpu", + "gfx110X-all", + "gfx120X-all", + "gfx125X-dcgpu", + "gfx94X-dcgpu", + ] { + let arch = example_arch_for_family(family).expect(family); + assert_eq!( + rocm_core::normalize_therock_family(arch).as_deref(), + Some(family), + "{arch} is not a member of {family}" + ); + let error = select_source_layout( + TheRockChannel::Release, + &family_for_test(family, None), + None, + ) + .expect_err("grouped family") + .to_string(); + assert!(error.contains(&format!("--family {arch}`")), "{error}"); + assert!( + !error.contains("--family gfx1200") || family == "gfx120X-all", + "{error}" + ); + } + let error = select_source_layout( + TheRockChannel::Release, + &family_for_test("gfx90X-dcgpu", None), + None, + ) + .expect_err("grouped family") + .to_string(); + assert!(error.contains("--family "), "{error}"); + assert!( + error.contains("--build-date YYYY-MM-DD` with a date before 2026-08-23"), + "{error}" + ); + } + + /// With no pin, the newest build is only in the next layout, so a grouped + /// family must be refused with the same actionable message rather than + /// silently installing from the stale canonical stream. + #[test] + fn an_unpinned_install_with_a_grouped_family_refuses_instead_of_going_canonical() { + let grouped = family_for_test("gfx120X-all", None); + + for channel in [TheRockChannel::Release, TheRockChannel::Nightly] { + let error = select_source_layout(channel, &grouped, None) + .expect_err("a grouped family cannot select a next-layout payload") + .to_string(); + assert!(error.contains("requires an exact GPU arch"), "{error}"); + } } #[test] @@ -384,12 +594,12 @@ mod tests { } /// A pinned nightly prerelease of a future major (e.g. a ROCm 10 alpha - /// build) is not evidence the canonical stream can't serve it — only a - /// *stable* pin is. Gating on major alone would refuse this pin and then - /// suggest a release-channel retry that the stable-version filter would - /// also reject, leaving no working path. + /// build) is routed by its build date, not its major: the legacy nightlies + /// carry ROCm 10 alphas up to 2026-08-22. Gating on major alone would refuse + /// this pin and then suggest a release-channel retry that the + /// stable-version filter would also reject, leaving no working path. #[test] - fn nightly_prerelease_pin_of_a_future_major_stays_canonical() { + fn nightly_prerelease_pin_of_a_future_major_is_routed_by_build_date() { let exact = family_for_test("gfx120X-all", Some("gfx1200")); let pin = RuntimeVersionSelector::version("10.1.0a20260822").unwrap(); @@ -397,6 +607,61 @@ mod tests { select_source_layout(TheRockChannel::Nightly, &exact, Some(&pin)).unwrap(), SourceLayout::Canonical ); + let later = RuntimeVersionSelector::version("10.1.0a20260901").unwrap(); + assert_eq!( + select_source_layout(TheRockChannel::Nightly, &exact, Some(&later)).unwrap(), + SourceLayout::Next + ); + } + + /// A ROCm major newer than any this CLI has seen must resolve without a CLI + /// release: the layout is chosen by `major >= THEROCK_NEXT_MIN_MAJOR`, not by + /// a list of known majors. A stable pin of a later major takes the next + /// layout, and a pinned nightly prerelease of one is routed by its build date. + #[test] + fn a_future_rocm_major_selects_its_layout_without_a_cli_change() { + let exact = family_for_test("gfx120X-all", Some("gfx1200")); + + for stable in ["11.0.0", "12.3.4", "99.0.0"] { + let pin = RuntimeVersionSelector::version(stable).unwrap(); + assert_eq!( + select_source_layout(TheRockChannel::Release, &exact, Some(&pin)).unwrap(), + SourceLayout::Next, + "stable pin {stable}" + ); + } + + // Built after the next layout began, so it can only be there; the major + // plays no part. + let alpha = RuntimeVersionSelector::version("11.0.0a20270101").unwrap(); + assert_eq!( + select_source_layout(TheRockChannel::Nightly, &exact, Some(&alpha)).unwrap(), + SourceLayout::Next + ); + } + + /// An update names no pin, so on its own it would be routed to the newest + /// layout and, with only a family label, refused; the layout it recorded is + /// what it installs from, so that is what the gate lets through. + #[test] + fn an_update_is_gated_on_its_recorded_layout_not_on_the_request_it_does_not_make() { + let grouped = family_for_test("gfx120X-all", None); + + for channel in [TheRockChannel::Release, TheRockChannel::Nightly] { + assert_eq!( + gate_install_layout(channel, &grouped, None, Some(SourceLayout::Canonical)) + .unwrap(), + SourceLayout::Canonical, + "{channel:?}" + ); + assert_eq!( + gate_install_layout(channel, &grouped, None, Some(SourceLayout::Next)).unwrap(), + SourceLayout::Next, + "{channel:?}" + ); + // A fresh request with no recorded layout is still gated. + assert!(gate_install_layout(channel, &grouped, None, None).is_err()); + } } #[test] @@ -413,6 +678,17 @@ mod tests { "https://stable.repo.amd.com/rocm/core/tarball/" ); assert_eq!(next.layout.generation(), "next-v1"); + + let nightly = resolve_source(TheRockChannel::Nightly, SourceLayout::Next); + assert_eq!( + nightly.wheel_index, + "https://nightly.repo.amd.com/rocm/whl-next" + ); + assert_eq!( + nightly.tarball_catalog, + "https://nightly.repo.amd.com/rocm/core/tarball/" + ); + assert_eq!(nightly.layout.generation(), "next-v1"); } /// A base override is a redirect to an arbitrary host, so naming the @@ -472,29 +748,38 @@ mod tests { } /// The requirement is read before `resolve_pip_runtime` runs, so it has to - /// be derivable from the request alone. Only an explicit stable ROCm >= 10 - /// pin reaches the next layout; a build-date pin, an older pin, or no - /// selector at all must stay canonical so nothing that installs today - /// starts demanding a different interpreter. + /// be derivable from the request alone, and it has to agree with the layout + /// resolution will then use. #[test] fn requested_source_layout_reads_the_request_not_the_resolution() { let next_pin = RuntimeVersionSelector::Version("10.1.0".to_owned()); assert_eq!( - requested_source_layout(None, Some(&next_pin)), + requested_source_layout(TheRockChannel::Release, None, Some(&next_pin)), SourceLayout::Next ); let old_pin = RuntimeVersionSelector::Version("7.2.3".to_owned()); assert_eq!( - requested_source_layout(None, Some(&old_pin)), + requested_source_layout(TheRockChannel::Release, None, Some(&old_pin)), SourceLayout::Canonical ); - assert_eq!(requested_source_layout(None, None), SourceLayout::Canonical); + assert_eq!( + requested_source_layout(TheRockChannel::Release, None, None), + SourceLayout::Next + ); + assert_eq!( + requested_source_layout(TheRockChannel::Nightly, None, None), + SourceLayout::Next + ); // An update supplies its manifest's layout, which wins over the (absent) // selector so the runtime keeps the interpreter its stream needs. assert_eq!( - requested_source_layout(Some(SourceLayout::Next), None), + requested_source_layout(TheRockChannel::Nightly, Some(SourceLayout::Canonical), None), + SourceLayout::Canonical + ); + assert_eq!( + requested_source_layout(TheRockChannel::Release, Some(SourceLayout::Next), None), SourceLayout::Next ); } diff --git a/apps/rocm/src/therock/test_support.rs b/apps/rocm/src/therock/test_support.rs index e0b41a46d0..f27c50dd8e 100644 --- a/apps/rocm/src/therock/test_support.rs +++ b/apps/rocm/src/therock/test_support.rs @@ -11,7 +11,45 @@ use crate::runtime_registry::{InstalledRuntimeManifest, runtime_manifest_path}; use anyhow::Result; use rocm_core::AppPaths; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; + +/// Process environment variables set for one test and restored on drop. Every +/// caller holds `PROCESS_ENV_TEST_LOCK` for as long as it holds this. +pub(super) struct RestoredEnv(Vec<(&'static str, Option)>); + +impl RestoredEnv { + /// Sets (`Some`) or removes (`None`) each variable, remembering what it was. + #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 + pub(super) fn set(vars: &[(&'static str, Option<&Path>)]) -> Self { + let saved = vars + .iter() + .map(|(key, _)| (*key, std::env::var_os(key))) + .collect(); + for (key, value) in vars { + unsafe { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + } + Self(saved) + } +} + +impl Drop for RestoredEnv { + #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 + fn drop(&mut self) { + for (key, value) in self.0.drain(..) { + unsafe { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + } + } +} pub(super) fn family_for_test(family: &str, raw_arch: Option<&str>) -> FamilyResolution { FamilyResolution { diff --git a/apps/rocm/src/therock/update.rs b/apps/rocm/src/therock/update.rs index 3b203c3e2d..1f0972ba3b 100644 --- a/apps/rocm/src/therock/update.rs +++ b/apps/rocm/src/therock/update.rs @@ -4,42 +4,50 @@ //! Whether an installed runtime has something newer to offer, and the bounded //! startup check that asks. `runtime_update_plan` resolves the latest source for -//! one runtime (`resolve_latest_for_manifest`, which re-resolves a wheel -//! runtime through `resolve_pip_runtime_with_timeout` and a tarball one through +//! one runtime, and `runtime_update_check` does the same for a caller that +//! changes nothing (the report, the JSON, the startup check and an apply's +//! preview). They differ in `PythonAccess`: given a wheel runtime that records +//! no usable Python, the plan prepares a managed Python (an apply needs one), +//! while the check works the wheel tag out from the interpreter the layout +//! requires and carries a `python_note` saying the preparation was left undone +//! (`resolve_latest_for_manifest` re-resolves a wheel runtime through +//! `resolve_pip_runtime_with_timeout` and a tarball one through //! `resolve_tarball_artifact_with_timeout`, on the layout the runtime was //! installed from) and `runtime_freshness` classifies it (`RuntimeFreshness`: //! up to date, update available, repair available, or ahead of the index) into //! a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` show //! the plans for `rocm update` (the JSON form is what the dashboard's update -//! check parses), and `maybe_refresh_startup_update_check` keeps one +//! check parses and carries no `python_note`), and `maybe_refresh_startup_update_check` keeps one //! `StartupUpdateCheckRecord` in the cache so the check runs at most once an //! interval and is bounded by a short download timeout. //! //! Builds on `family`, `io`, `process`, `python`, `resolve`, `source` and //! `version`, and on `runtime_registry` for the manifests; `runtime_key`, -//! `wheel_runtime_key` and `has_nontrivial_directory_contents`, which name and -//! inspect the install a plan targets, stay in `therock.rs` with the other keys -//! and roots helpers. +//! `wheel_runtime_key`, which name the install a plan targets, stay in +//! `therock.rs` with the other keys and roots helpers. Whether an install root +//! holds anything is `rocm_core::install_root_has_payload`, the one rule the +//! install and activation paths share. use super::family::family_override_or_recovered_arch; -use super::io::write_file_atomically; use super::process::SuppressProgressOutput; use super::python::{ - PythonLauncher, PythonSource, python_launcher_install_ready, python_requirement, + PythonLauncher, PythonRequirement, PythonSource, plan_python_launcher, + python_launcher_install_ready, python_requirement, python_requirement_for_tag, resolve_python_launcher, wheel_compatibility_for_python, }; use super::resolve::{ AggregateDeviceTarget, resolve_pip_runtime_with_timeout, resolve_tarball_artifact_with_timeout, wheel_runtime_composition, }; -use super::source::{SourceLayout, TheRockChannel}; -use super::version::{runtime_version_display, version_relation}; -use super::{has_nontrivial_directory_contents, runtime_key, wheel_runtime_key}; +use super::source::{SourceLayout, THEROCK_NEXT_MIN_MAJOR, TheRockChannel}; +use super::version::{parse_version, runtime_version_display, version_relation}; +use super::{runtime_key, wheel_runtime_key}; use crate::runtime_registry::{ InstalledRuntimeManifest, WheelRuntimeComposition, load_runtime_manifests, wheel_composition_device_target, }; use anyhow::{Context, Result, bail}; +use rocm_core::atomic_write::write_file_atomically; use rocm_core::{AppPaths, unix_time_millis}; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; @@ -83,6 +91,22 @@ pub(crate) struct RuntimeUpdatePlan { pub source_layout_generation: Option, pub repair_required: bool, pub update_available: bool, + /// Set when the runtime records no usable Python and the check left one + /// unprepared: what an update would do about it. + pub python_note: Option, +} + +/// Whether resolving a wheel runtime's latest version may prepare a Python. +/// +/// Reading the index needs the interpreter's wheel tag. A runtime that records +/// a usable Python supplies it. For one that does not, an apply goes on to +/// prepare a managed Python (it needs one for the install that follows), but a +/// check changes nothing, so it works the tag out from the interpreter the +/// layout requires and says it left the preparation undone. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum PythonAccess { + MayPrepare, + SearchOnly, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -116,6 +140,7 @@ struct ResolvedRuntimeUpdate { format: String, source_layout_generation: String, wheel_composition: Option, + python_note: Option, } pub(crate) fn render_update_report(paths: &AppPaths) -> Result { @@ -144,7 +169,7 @@ pub(crate) fn render_update_report(paths: &AppPaths) -> Result { } for manifest in &manifests { - let plan = match runtime_update_plan(paths, manifest, &manifests, None) { + let plan = match runtime_update_check(paths, manifest, &manifests, None) { Ok(plan) => Some(plan), Err(error) => { let _ = writeln!( @@ -181,6 +206,9 @@ pub(crate) fn render_update_report(paths: &AppPaths) -> Result { manifest.install_root.display() ); let _ = writeln!(output, " source: {}", plan.latest_source); + if let Some(note) = &plan.python_note { + let _ = writeln!(output, " python_note: {note}"); + } if plan.update_available { let next_step = if plan.repair_required { format!( @@ -247,7 +275,7 @@ pub(crate) fn render_update_json( let manifests = load_runtime_manifests(paths)?; let mut runtimes = Vec::with_capacity(manifests.len()); for manifest in &manifests { - match runtime_update_plan(paths, manifest, &manifests, download_timeout_secs) { + match runtime_update_check(paths, manifest, &manifests, download_timeout_secs) { Ok(plan) => runtimes.push(UpdateJsonRuntime { runtime_key: manifest.runtime_key.clone(), channel: manifest.channel.clone(), @@ -295,7 +323,7 @@ fn replacement_runtime_is_installed( && candidate.format == source.format && candidate.family == source.family && candidate.wheel_composition.as_ref() == Some(required_composition) - && has_nontrivial_directory_contents(&candidate.install_root).unwrap_or(false) + && rocm_core::install_root_has_payload(&candidate.install_root).unwrap_or(false) }) } @@ -356,13 +384,49 @@ fn runtime_freshness_with_manifests( } } +/// The plan an apply works from. A wheel runtime that records no usable Python +/// has one prepared here, since the install that follows needs it. pub(crate) fn runtime_update_plan( paths: &AppPaths, manifest: &InstalledRuntimeManifest, manifests: &[InstalledRuntimeManifest], download_timeout_secs: Option, ) -> Result { - let latest = resolve_latest_for_manifest(paths, manifest, download_timeout_secs)?; + plan_runtime_update( + paths, + manifest, + manifests, + download_timeout_secs, + PythonAccess::MayPrepare, + ) +} + +/// The same plan for a caller that changes nothing: the update report, the JSON +/// the dashboard reads, and the preview of an apply. It never prepares a Python. +pub(crate) fn runtime_update_check( + paths: &AppPaths, + manifest: &InstalledRuntimeManifest, + manifests: &[InstalledRuntimeManifest], + download_timeout_secs: Option, +) -> Result { + plan_runtime_update( + paths, + manifest, + manifests, + download_timeout_secs, + PythonAccess::SearchOnly, + ) +} + +fn plan_runtime_update( + paths: &AppPaths, + manifest: &InstalledRuntimeManifest, + manifests: &[InstalledRuntimeManifest], + download_timeout_secs: Option, + python_access: PythonAccess, +) -> Result { + let latest = + resolve_latest_for_manifest(paths, manifest, download_timeout_secs, python_access)?; let freshness = runtime_freshness_with_manifests( manifests, manifest, @@ -385,6 +449,7 @@ pub(crate) fn runtime_update_plan( source_layout_generation: Some(latest.source_layout_generation), repair_required: freshness == RuntimeFreshness::RepairAvailable, update_available: freshness.update_available(), + python_note: latest.python_note, }) } @@ -412,10 +477,71 @@ fn manifest_wheel_family_override(manifest: &InstalledRuntimeManifest) -> String family_override_or_recovered_arch(&manifest.family, recorded_arch) } +/// The layout whose interpreters `manifest`'s runtime was built against. +/// +/// The recorded source layout is not enough: an unpinned or build-date nightly +/// install records the canonical layout even when the channel's newest build is +/// ROCm 10 and the install went on the ROCm 10 Python (see +/// `canonical_auto_select_needs_next_python`). The recorded ROCm version is the +/// other half of the answer, so a ROCm 10+ runtime counts as `Next` whichever +/// layout it was read from. +fn python_layout_of(manifest: &InstalledRuntimeManifest, layout: SourceLayout) -> SourceLayout { + let rocm_10_or_newer = parse_version(&manifest.version) + .is_some_and(|parsed| parsed.major >= THEROCK_NEXT_MIN_MAJOR); + if layout == SourceLayout::Next || rocm_10_or_newer { + SourceLayout::Next + } else { + SourceLayout::Canonical + } +} + +/// The interpreter an update of `manifest` goes on with. +/// +/// A ROCm 10+ runtime installed with `--python-version` keeps the Python it was +/// built on, so an update does not silently move it to the layout's default +/// (which would also leave it unable to carry the packages it was chosen for). +/// Anything else follows the layout. +fn recorded_python_requirement( + manifest: &InstalledRuntimeManifest, + layout: SourceLayout, +) -> PythonRequirement { + let tag = manifest + .python_executable + .as_deref() + .map(PathBuf::from) + .filter(|path| path.is_file()) + .and_then(|path| wheel_compatibility_for_python(&path).ok()) + .map(|compatibility| compatibility.python_tag); + requirement_for_recorded_tag(manifest, layout, tag.as_deref()) +} + +/// [`recorded_python_requirement`] once the recorded interpreter's wheel tag has +/// been read (`None` when it is missing or unreadable), so the decision runs +/// without a Python on disk. +fn requirement_for_recorded_tag( + manifest: &InstalledRuntimeManifest, + layout: SourceLayout, + tag: Option<&str>, +) -> PythonRequirement { + let layout = python_layout_of(manifest, layout); + tag.and_then(|tag| python_requirement_for_tag(layout, tag)) + .unwrap_or_else(|| python_requirement(layout)) +} + +/// The `--python-version` an update of `manifest` reinstalls with: the one the +/// runtime was built on, for a ROCm 10+ wheel runtime, and none for any other (a +/// tarball runtime has no Python of its own to carry over). +pub(crate) fn recorded_python_version(manifest: &InstalledRuntimeManifest) -> Option<&'static str> { + let layout = manifest_source_layout(manifest).ok()?; + (manifest.format == "wheel" && python_layout_of(manifest, layout) == SourceLayout::Next) + .then(|| recorded_python_requirement(manifest, layout).version) +} + fn resolve_latest_for_manifest( paths: &AppPaths, manifest: &InstalledRuntimeManifest, download_timeout_secs: Option, + python_access: PythonAccess, ) -> Result { let channel = TheRockChannel::parse(&manifest.channel)?; let layout = manifest_source_layout(manifest)?; @@ -425,7 +551,7 @@ fn resolve_latest_for_manifest( // interpreter, so the recorded one is re-checked rather than trusted // on `is_file()` alone; otherwise a canonical cp312 venv would carry // into a next-layout update and resolve nothing. - let requirement = python_requirement(layout); + let requirement = recorded_python_requirement(manifest, layout); let manifest_python = manifest .python_executable .as_deref() @@ -436,12 +562,17 @@ fn resolve_latest_for_manifest( executable, source: PythonSource::Manifest, }); - let python_executable = match manifest_python { - Some(python) => python, - None => resolve_python_launcher(paths, requirement)?, + let (wheel_compatibility, python_note) = match (manifest_python, python_access) { + (Some(python), _) => (wheel_compatibility_for_python(&python.executable)?, None), + (None, PythonAccess::MayPrepare) => { + let python = resolve_python_launcher(paths, requirement)?; + (wheel_compatibility_for_python(&python.executable)?, None) + } + (None, PythonAccess::SearchOnly) => { + let planned = plan_python_launcher(paths, requirement)?; + (planned.wheel_compatibility()?, planned.update_note()) + } }; - let wheel_compatibility = - wheel_compatibility_for_python(&python_executable.executable)?; let family_override = manifest_wheel_family_override(manifest); let resolution = resolve_pip_runtime_with_timeout( paths, @@ -459,17 +590,12 @@ fn resolve_latest_for_manifest( // a fresh host probe. Planning must predict the key an apply will // produce, and re-probing would disagree with the installed runtime on // any host whose GPU is absent, hidden, or simply a second card. - let device_target = - wheel_composition_device_target(manifest.wheel_composition.as_ref()).map_or_else( - || resolution.device_target.clone(), - |target| { - AggregateDeviceTarget::resolve( - Some(target), - &resolution.family, - &resolution.published_device_targets, - ) - }, - ); + let device_target = AggregateDeviceTarget::prefer_recorded( + &resolution.device_target, + wheel_composition_device_target(manifest.wheel_composition.as_ref()), + &resolution.family, + &resolution.published_device_targets, + ); // No exact target means no reproducible composition, so freshness // falls back to the version comparison rather than demanding a repair // this host could not perform. @@ -492,6 +618,7 @@ fn resolve_latest_for_manifest( format: "wheel".to_owned(), source_layout_generation: resolution.layout.generation().to_owned(), wheel_composition, + python_note, }) } "tarball" => { @@ -516,6 +643,7 @@ fn resolve_latest_for_manifest( format: "tarball".to_owned(), source_layout_generation: artifact.layout.generation().to_owned(), wheel_composition: None, + python_note: None, }) } other => bail!("unknown manifest format `{other}`"), @@ -589,7 +717,12 @@ fn build_startup_update_check_record( now_unix_ms: u128, download_timeout_secs: Option, ) -> StartupUpdateCheckRecord { - match resolve_latest_for_manifest(paths, manifest, download_timeout_secs) { + match resolve_latest_for_manifest( + paths, + manifest, + download_timeout_secs, + PythonAccess::SearchOnly, + ) { Ok(latest) => { let freshness = runtime_freshness_with_manifests( manifests, @@ -658,11 +791,15 @@ mod tests { use super::super::process::{PROGRESS_LINE_SINK, progress_output_suppressed}; use super::super::python::{OverridePythonResolverEnv, PythonResolverEnv}; use super::super::source::THEROCK_SOURCE_LAYOUT_GENERATION; + #[cfg(unix)] + use super::super::test_support::RestoredEnv; use super::super::test_support::{ test_paths, test_runtime_manifest, write_test_runtime_manifest, }; use super::*; use crate::test_support::PROCESS_ENV_TEST_LOCK; + #[cfg(unix)] + use std::path::Path; /// Updating a next-layout manifest whose family is a group label (as /// `gfx125X-dcgpu` always is) must recover the exact arch this runtime was @@ -695,6 +832,66 @@ mod tests { assert_eq!(manifest_wheel_family_override(&manifest), "gfx120X-all"); } + /// The engine auto-install skip reads this: a ROCm 7 (canonical) runtime has no + /// recorded Python choice even when it was installed with a redundant + /// `--python-version 3.12`, so vLLM is never skipped on it. + #[test] + fn recorded_python_version_is_only_reported_for_a_rocm_10_runtime() { + let mut manifest = test_runtime_manifest("runtime-key", "therock-release:gfx94X-dcgpu", 0); + assert_eq!(recorded_python_version(&manifest), None); + + manifest.source_layout_generation = Some("next-v1".to_owned()); + manifest.format = "wheel".to_owned(); + assert_eq!( + recorded_python_version(&manifest), + Some(super::super::NEXT_MANAGED_PYTHON_VERSION) + ); + + manifest.format = "tarball".to_owned(); + assert_eq!(recorded_python_version(&manifest), None); + } + + /// An unpinned nightly install that resolved to ROCm 10 records the canonical + /// layout, yet it runs on the ROCm 10 Python; the chosen interpreter must still + /// be carried over, and a ROCm 7 runtime on the same layout must not. + #[test] + fn nightly_rocm_10_runtime_recorded_as_canonical_keeps_its_chosen_python() { + let mut manifest = test_runtime_manifest("runtime-key", "therock-nightly:gfx94X-dcgpu", 0); + manifest.channel = "nightly".to_owned(); + manifest.version = "10.0.0a20261105".to_owned(); + manifest.source_layout_generation = Some("multi-arch-v2".to_owned()); + assert_eq!( + manifest_source_layout(&manifest).unwrap(), + SourceLayout::Canonical + ); + + let layout = SourceLayout::Canonical; + assert_eq!( + requirement_for_recorded_tag(&manifest, layout, Some("cp312")).version, + "3.12" + ); + assert_eq!( + requirement_for_recorded_tag(&manifest, layout, Some("cp314")).version, + super::super::NEXT_MANAGED_PYTHON_VERSION + ); + // Interpreter missing or unreadable: the ROCm 10 default, not cp312. + assert_eq!( + requirement_for_recorded_tag(&manifest, layout, None).version, + super::super::NEXT_MANAGED_PYTHON_VERSION + ); + assert_eq!( + recorded_python_version(&manifest), + Some(super::super::NEXT_MANAGED_PYTHON_VERSION) + ); + + manifest.version = "7.13.0a20260416".to_owned(); + assert_eq!(recorded_python_version(&manifest), None); + assert_eq!( + requirement_for_recorded_tag(&manifest, layout, Some("cp312")).tag, + "cp312" + ); + } + /// An installed runtime must keep resolving the stream it came from, and a /// manifest written before the field existed must still be readable. #[test] @@ -1072,6 +1269,55 @@ mod tests { fs::remove_dir_all(root).ok(); } + /// A replacement counts as installed by `rocm_core::install_root_has_payload`: + /// a root holding only the runtime's own dot-marker was prepared but never + /// filled, so the repair is still on offer; one entry beside the marker is a + /// real install, so it is not. The runtime directory vanishing is covered by + /// `an_installed_replacement_sibling_suppresses_repeat_repair`. + #[test] + fn a_replacement_counts_as_installed_by_the_shared_payload_rule() { + let required = test_wheel_composition("gfx942"); + let target_runtime_key = wheel_runtime_key(TheRockChannel::Release, "7.14.0", &required); + let mut source = test_runtime_manifest( + "release-wheel-multi-arch-7-14-0", + "therock-release:gfx94X-dcgpu", + 1, + ); + source.version = "7.14.0".to_owned(); + let mut replacement = + test_runtime_manifest(&target_runtime_key, "therock-release:gfx94X-dcgpu", 2); + replacement.version = "7.14.0".to_owned(); + replacement.wheel_composition = Some(required.clone()); + let (root, _) = test_paths("replacement-shared-payload-rule"); + replacement.install_root = root.join("replacement"); + fs::create_dir_all(&replacement.install_root).unwrap(); + fs::write( + replacement.install_root.join(".rocm-cli-runtime.json"), + b"{}", + ) + .unwrap(); + let manifests = vec![source.clone(), replacement.clone()]; + let freshness = |manifests: &[InstalledRuntimeManifest]| { + runtime_freshness_with_manifests( + manifests, + &source, + "7.14.0", + Some(&required), + &target_runtime_key, + ) + }; + + assert_eq!( + freshness(&manifests), + RuntimeFreshness::RepairAvailable, + "a root with only the dot-marker has no payload, so it is not a replacement" + ); + + fs::write(replacement.install_root.join("installed.marker"), b"ok").unwrap(); + assert_eq!(freshness(&manifests), RuntimeFreshness::UpToDate); + fs::remove_dir_all(root).ok(); + } + #[test] fn update_json_reports_no_managed_runtimes_as_empty_list() -> Result<()> { let (root, paths) = test_paths("update-json-empty"); @@ -1197,4 +1443,141 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// The bug: a read-only look at a wheel runtime that records no usable Python, + /// on a host with none, went on to prepare one (download a managed `uv`, have + /// it install Python, save the path in the configuration). Driven here, each + /// against the same host: the update report, the JSON the dashboard reads, + /// the startup check, `runtime_update_check` itself, and + /// `apply_runtime_update` with `dry_run` (the preview of an apply, which takes + /// the check). Each must leave `uv` unrun and the configuration and managed + /// tools absent, and each must still reach the index (the connection to the + /// unreachable index is what each error names), so the absences are not a + /// check that stopped early. `runtime_update_plan` and a real + /// `apply_runtime_update` do prepare one on the same host, which is also what + /// shows the stand-in `uv` is reachable. That a `--dry-run` takes the check and + /// an apply the plan is pinned on every platform by + /// `a_dry_run_takes_the_check_and_an_apply_takes_the_plan` in `update_cmd.rs`; + /// which function is the check is pinned only here and by scenario `update-10` + /// on Linux, and no test on Windows pins it. The Gherkin scenarios `update-09` + /// and `update-10` drive the report and the preview through the built binary. + #[cfg(unix)] + #[test] + fn the_update_checks_prepare_no_python_and_the_apply_plan_does() -> Result<()> { + use rocm_core::fake_executable::write_fake_executable; + + let _env_lock = PROCESS_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (root, paths) = test_paths("update-checks-prepare-no-python"); + + let marker = root.join("uv-was-run"); + let tools = root.join("tools"); + fs::create_dir_all(&tools)?; + let uv = tools.join("uv"); + write_fake_executable( + &uv, + format!("#!/bin/sh\necho \"$@\" >> '{}'\nexit 1\n", marker.display()), + )?; + let search = root.join("search"); + fs::create_dir_all(&search)?; + // Port 9 refuses the connection at once, so a check that gets past + // Python fails at the index without touching the network. + let unreachable_index = "http://127.0.0.1:9/"; + let _env = RestoredEnv::set(&[ + ("ROCM_CLI_UV_BINARY", Some(uv.as_path())), + ("ROCM_CLI_PYTHON", None), + ("ROCM_CLI_DISABLE_MANAGED_PYTHON_BOOTSTRAP", None), + ("ROCM_CLI_MANAGED_PYTHON_VERSION", None), + ("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", None), + ("ROCM_CLI_THEROCK_ALLOW_BASE_OVERRIDE", Some(Path::new("1"))), + ( + "ROCM_CLI_THEROCK_RELEASE_PIP_BASE", + Some(Path::new(unreachable_index)), + ), + ]); + let _resolver = OverridePythonResolverEnv::new(PythonResolverEnv { + python_override: None, + search_dirs: vec![search], + ..PythonResolverEnv::from_process_env() + }); + + // The fixture's recorded interpreter does not exist, so the runtime + // records no usable Python. + let manifest = test_runtime_manifest("wheel-no-python", "therock-release:gfx120X-all", 1); + write_test_runtime_manifest(&paths, &manifest)?; + let manifests = load_runtime_manifests(&paths)?; + + let prepared_nothing = |what: &str| { + assert!(!marker.exists(), "{what} ran uv"); + assert!( + !paths.config_path().exists(), + "{what} saved a Python path in the configuration" + ); + assert!( + !rocm_core::managed_tools_dir(&paths.data_dir).exists(), + "{what} created the managed tools folder" + ); + }; + + let report = render_update_report(&paths)?; + assert!(report.contains("127.0.0.1:9"), "{report}"); + prepared_nothing("the update report"); + + let document = render_update_json(&paths, Some(1))?; + assert_eq!(document.runtimes[0].status, "error"); + assert!( + document.runtimes[0] + .message + .as_deref() + .is_some_and(|message| message.contains("127.0.0.1:9")), + "{document:?}" + ); + prepared_nothing("the update JSON"); + + let record = maybe_refresh_startup_update_check_at(&paths, None, 0)? + .expect("a registered runtime is checked at startup"); + assert!( + record + .message + .as_deref() + .is_some_and(|message| message.contains("127.0.0.1:9")), + "{record:?}" + ); + prepared_nothing("the startup check"); + + let error = runtime_update_check(&paths, &manifest, &manifests, Some(1)) + .expect_err("the index is unreachable") + .to_string(); + assert!(error.contains("127.0.0.1:9"), "{error}"); + prepared_nothing("the update check"); + + let mut config = rocm_core::RocmCliConfig::default(); + let preview = + crate::update_cmd::apply_runtime_update(&paths, &mut config, None, false, true) + .expect_err("the index is unreachable") + .to_string(); + assert!(preview.contains("127.0.0.1:9"), "{preview}"); + prepared_nothing("the --dry-run preview"); + + // Control: the plan an apply works from does reach `uv` on this host, and + // so does a real apply, which takes that plan. + assert!(runtime_update_plan(&paths, &manifest, &manifests, Some(1)).is_err()); + assert!( + marker.exists(), + "the stand-in uv is not reachable, so the checks above prove nothing" + ); + fs::remove_file(&marker)?; + assert!( + crate::update_cmd::apply_runtime_update(&paths, &mut config, None, false, false) + .is_err() + ); + assert!( + marker.exists(), + "a real apply did not take the plan that prepares a Python" + ); + + fs::remove_dir_all(&root).ok(); + Ok(()) + } } diff --git a/apps/rocm/src/therock/version.rs b/apps/rocm/src/therock/version.rs index cefbef9b99..33e086fd08 100644 --- a/apps/rocm/src/therock/version.rs +++ b/apps/rocm/src/therock/version.rs @@ -383,6 +383,12 @@ pub(super) fn is_stable_runtime_version(version: &str) -> bool { parse_version(version).is_some_and(|parsed| parsed.stage == VersionStage::Stable) } +/// Whether `version` parses as a version at all, whatever its stage. Unlike +/// [`is_stable_runtime_version`] this accepts prereleases such as `10.3.0a20261111`. +pub(super) fn is_parseable_runtime_version(version: &str) -> bool { + parse_version(version).is_some() +} + pub(crate) fn runtime_version_display(version: &str) -> String { if let Some(date) = runtime_version_build_date(version) { format!("{version} (build {date})") diff --git a/apps/rocm/src/therock/wheel_plan.rs b/apps/rocm/src/therock/wheel_plan.rs new file mode 100644 index 0000000000..a7ddbe0322 --- /dev/null +++ b/apps/rocm/src/therock/wheel_plan.rs @@ -0,0 +1,354 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `sdk install` plan a wheel install prints, key by key. +//! +//! A preview prints it and stops; a real install prints it as the head of its +//! own output. [`write_wheel_plan`] writes the lines both share, and +//! [`write_dry_run_tail`] the lines only a preview adds. +//! +//! One writer serves a resolved plan and one that could not be resolved, so the +//! two cannot drift apart: a line the plan gains appears in both. They differ +//! only in the lines made from the package versions `uv pip compile` chooses on +//! the ROCm 10 layout (`package_specs`, `runtime_key`, `target`, `manifest` and +//! `command`). [`PlanStack::NeedsUv`] says those are not resolved, and why, +//! where [`PlanStack::Resolved`] carries their values. + +use super::kept_torch::{self, KeptTorch}; +use super::python::{PlannedPython, WheelCompatibility}; +use super::resolve::{AggregateDeviceTarget, TorchaudioPin, UnresolvedPipRuntime}; +use super::source::{RuntimeVersionSelector, TheRockChannel}; +use super::{ + no_wheel_warning_for, render_canonical_provenance, runtime_version_display, venv_python_path, + wheel_host_version_note, +}; +use crate::shell_quote::shell_quote; +use rocm_core::{uv_pip_install_base, uv_venv_args}; +use std::fmt::Write as _; +use std::path::Path; + +/// The value printed for a line that depends on the package versions `uv` has not +/// chosen. +pub(super) const NOT_RESOLVED: &str = "not resolved by this preview"; + +/// The facts the plan prints, from a resolution and the Python decision. +pub(super) struct WheelPlan<'a> { + pub(super) channel: TheRockChannel, + pub(super) version_selector: Option<&'a RuntimeVersionSelector>, + pub(super) index_url: &'a str, + pub(super) layout_generation: &'a str, + /// The `rocm` version the plan is for. + pub(super) latest_version: &'a str, + pub(super) family: &'a str, + pub(super) family_source: &'a str, + pub(super) device_target: &'a AggregateDeviceTarget, + /// The host's ROCm when it is newer than `latest_version`. + pub(super) host_version_newer: Option<&'a str>, + pub(super) python: &'a PlannedPython, + pub(super) wheel_compatibility: &'a WheelCompatibility, + /// The requirement lines version resolution asked `uv` about. `None` on the + /// canonical layout, which resolves without `uv`. + pub(super) version_resolution_specs: Option<&'a [String]>, + pub(super) torchaudio_pin: Option<&'a TorchaudioPin>, + pub(super) no_wheel_warning: Option<&'a str>, + pub(super) stack: PlanStack<'a>, +} + +/// Everything in the plan that is made from the package versions. +pub(super) enum PlanStack<'a> { + /// The versions are chosen; these are what they produce. + Resolved(ResolvedStack<'a>), + /// A preview of the ROCm 10 layout with no `uv`: the versions were not chosen, + /// so nothing made from them can be printed. + NeedsUv { + /// The install folder when the user named one with `--prefix`: it does not + /// depend on the versions. Otherwise the runtime key names it. + explicit_install_root: Option<&'a Path>, + }, +} + +pub(super) struct ResolvedStack<'a> { + pub(super) compatibility_key: &'a str, + pub(super) install_root: &'a Path, + pub(super) runtime_key: &'a str, + pub(super) package_specs: &'a [String], + /// The torch stack the install leaves alone, when the opt-out asked for that and + /// the environment has one. The specs above stay the runtime's composition, as + /// recorded in its manifest; what `uv` is handed is [`kept_torch::install_specs`] + /// of them. + pub(super) kept_torch: Option<&'a KeptTorch>, + pub(super) manifest_path: &'a Path, +} + +/// The `compatibility_key` the install is recorded under is the `rocm` build every +/// package in the stack shares, so a plan that has not chosen the stack still +/// knows it: the `rocm` version. +const fn compatibility_key<'a>(plan: &'a WheelPlan<'a>) -> &'a str { + match &plan.stack { + PlanStack::Resolved(stack) => stack.compatibility_key, + PlanStack::NeedsUv { .. } => plan.latest_version, + } +} + +/// Writes the plan from `sdk install` to `package_policy` and the repository +/// warning. A real install carries on from here; a preview adds +/// [`write_dry_run_tail`]. +pub(super) fn write_wheel_plan(output: &mut String, plan: &WheelPlan<'_>) { + let _ = writeln!(output, "sdk install"); + let _ = writeln!( + output, + " summary: rocm-cli will install the ROCm SDK and matching PyTorch packages for this Python and operating system" + ); + render_canonical_provenance( + output, + plan.channel, + plan.index_url, + plan.layout_generation, + plan.latest_version, + ); + let _ = writeln!(output, " format: wheel"); + if let Some(selector) = plan.version_selector { + let _ = writeln!(output, " requested: {}", selector.describe()); + } + let _ = writeln!(output, " target_family: {}", plan.family); + let _ = writeln!(output, " target_family_source: {}", plan.family_source); + let _ = writeln!(output, " device_target: {}", plan.device_target.as_str()); + if let Some(reason) = plan.device_target.reason() { + let _ = writeln!(output, " device_target_reason: {reason}"); + } + let _ = writeln!(output, " index_url: {}", plan.index_url); + let _ = writeln!( + output, + " latest_compatible_version: {}", + runtime_version_display(plan.latest_version) + ); + if let Some(host_version) = plan.host_version_newer { + let _ = writeln!( + output, + " version_note: {}", + wheel_host_version_note(host_version, &runtime_version_display(plan.latest_version)) + ); + } + let _ = writeln!( + output, + " compatibility_key: {}", + runtime_version_display(compatibility_key(plan)) + ); + match &plan.stack { + PlanStack::Resolved(stack) => { + let _ = writeln!(output, " target: {}", stack.install_root.display()); + let _ = writeln!(output, " runtime_key: {}", stack.runtime_key); + } + PlanStack::NeedsUv { + explicit_install_root, + } => { + match explicit_install_root { + Some(root) => { + let _ = writeln!(output, " target: {}", root.display()); + } + None => { + let _ = writeln!(output, " target: {NOT_RESOLVED}"); + } + } + let _ = writeln!(output, " runtime_key: {NOT_RESOLVED}"); + } + } + let _ = writeln!( + output, + " python_launcher: {}", + plan.python.launcher_display() + ); + let _ = writeln!(output, " python_source: {}", plan.python.source().as_str()); + if let Some(note) = plan.python.note() { + let _ = writeln!(output, " python_note: {note}"); + } + if matches!(plan.stack, PlanStack::NeedsUv { .. }) { + let _ = writeln!(output, " resolution_note: {UNRESOLVED_NOTE}"); + } + let _ = writeln!( + output, + " python_wheel_tag: {}", + plan.wheel_compatibility.python_tag + ); + let _ = writeln!( + output, + " platform_wheel_tags: {}", + plan.wheel_compatibility.platform_tags.join(",") + ); + // What resolution asked for, printed beside what the install will ask for. + // The two are produced by different code paths from the same `include_devel` + // and must name the same extras; showing only the second would hide a + // resolve that constrained the version choice by a toolchain the user + // declined. Absent on the canonical layout, which resolves by scraping each + // package's index instead of compiling a requirement set. + if let Some(specs) = plan.version_resolution_specs { + let _ = writeln!(output, " version_resolution_specs: {}", specs.join(" ")); + } + // Printed from the same value that chose the pin, so the line cannot claim + // a verified build the requirements above do not name. + if let Some(pin) = plan.torchaudio_pin { + let _ = writeln!(output, " torchaudio_selection: {}", pin.describe()); + } + match &plan.stack { + PlanStack::Resolved(stack) => { + let _ = writeln!(output, " package_specs: {}", stack.package_specs.join(" ")); + } + PlanStack::NeedsUv { .. } => { + let _ = writeln!(output, " package_specs: {NOT_RESOLVED}"); + } + } + let kept = match &plan.stack { + PlanStack::Resolved(ResolvedStack { kept_torch, .. }) => *kept_torch, + PlanStack::NeedsUv { .. } => None, + }; + let _ = writeln!( + output, + " package_policy: resolve the pinned target-complete rocm, torch, torchvision, and torchaudio plan from published package metadata, then install it in one uv transaction{}", + if kept.is_some() { + " (the torch, torchvision and torchaudio requirements are resolved but not installed; see torch_kept)" + } else { + "" + } + ); + if let Some(kept) = kept { + let _ = writeln!(output, "{}", kept_torch::plan_line(kept)); + } + if let Some(warning) = plan.no_wheel_warning { + let _ = writeln!(output, " warning: {warning}"); + } +} + +/// What a preview says about the versions it did not choose. Written beside +/// `python_note`, in the same voice: what a real install would do, and that this +/// preview did not. +pub(super) const UNRESOLVED_NOTE: &str = "no uv is installed, so a real install would fetch it and run `uv pip compile` to choose the torch and torchvision versions and to check that the published packages install together, which this preview did not do; the package specs, runtime key, install folder, manifest and command depend on those versions and are not resolved here"; + +/// The progress line for the same decision as [`UNRESOLVED_NOTE`], in the voice of +/// the Python line (`... a real install would prepare Python X. This preview does +/// not.`). +pub(super) const UNRESOLVED_PROGRESS: &str = "No uv is installed; a real install would fetch it and resolve the torch and torchvision versions with `uv pip compile`. This preview does not."; + +/// The lines a preview adds after [`write_wheel_plan`]: the commands a real install +/// would run, and where it would record the runtime. +pub(super) fn write_dry_run_tail(output: &mut String, plan: &WheelPlan<'_>) { + let _ = writeln!(output, " mode: dry-run"); + match &plan.stack { + PlanStack::Resolved(stack) => { + let env_python = venv_python_path(stack.install_root); + let mut install_args = uv_pip_install_base(&env_python); + install_args.extend(["--index-url".to_owned(), plan.index_url.to_owned()]); + if matches!(plan.channel, TheRockChannel::Nightly) { + install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); + } + install_args.extend(kept_torch::install_specs( + stack.package_specs, + stack.kept_torch, + )); + let venv_args = uv_venv_args(&plan.python.venv_python_arg(), stack.install_root); + // This line is offered as the command to run, so it has to be one a + // shell reads back as the argv it previews. The install root in it + // comes from a `--prefix` the local assistant can choose + // (`install_sdk_dry_run` is a model-callable tool), so a bare or + // double-quoted value here would be a model-chosen value in a line a + // user is invited to paste. + let venv_args_display = venv_args + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" "); + let install_args_display = install_args + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" "); + let _ = writeln!( + output, + " command: uv {venv_args_display} && uv {install_args_display}" + ); + } + PlanStack::NeedsUv { .. } => { + let _ = writeln!(output, " command: {NOT_RESOLVED}"); + } + } + let _ = writeln!( + output, + " activation: use the managed venv Python; TheRock libraries are resolved from that venv by rocm_sdk.initialize_process" + ); + match &plan.stack { + PlanStack::Resolved(stack) => { + let _ = writeln!(output, " manifest: {}", stack.manifest_path.display()); + } + PlanStack::NeedsUv { .. } => { + let _ = writeln!(output, " manifest: {NOT_RESOLVED}"); + } + } +} + +/// What [`unresolved_wheel_preview`] is made from. +pub(super) struct UnresolvedPreview<'a> { + pub(super) channel: TheRockChannel, + pub(super) version_selector: Option<&'a RuntimeVersionSelector>, + pub(super) unresolved: &'a UnresolvedPipRuntime, + /// An update apply's recorded exact arch, resolved against the targets the + /// source publishes exactly as a resolved plan does. + pub(super) device_target_override: Option<&'a str>, + /// The `--prefix` the user named. It does not depend on the versions, so it is + /// the plan's `target`; without one the runtime key names the install folder + /// and the `target` is not resolved. + pub(super) prefix: Option<&'a Path>, + /// The host's ROCm when it is newer than the version the plan is for. + pub(super) host_version_newer: Option<&'a str>, + pub(super) python: &'a PlannedPython, + pub(super) wheel_compatibility: &'a WheelCompatibility, +} + +/// The whole plan for a preview of the ROCm 10 layout that stopped before +/// `uv pip compile`: what was chosen without `uv`, written by the same writer as +/// every other plan, with the lines made from the unchosen versions marked. +pub(super) fn unresolved_wheel_preview(preview: &UnresolvedPreview<'_>) -> String { + let UnresolvedPreview { + channel, + version_selector, + unresolved, + device_target_override, + prefix, + host_version_newer, + python, + wheel_compatibility, + } = *preview; + let device_target = AggregateDeviceTarget::prefer_recorded( + &unresolved.device_target, + device_target_override, + &unresolved.family, + &unresolved.published_device_targets, + ); + let explicit_install_root = prefix.map(rocm_core::resolve_path_through_symlinks); + let no_wheel_warning = no_wheel_warning_for( + unresolved.newest_repo_version.as_deref(), + &unresolved.rocm_version, + ); + let plan = WheelPlan { + channel, + version_selector, + index_url: &unresolved.index_url, + layout_generation: unresolved.layout.generation(), + latest_version: &unresolved.rocm_version, + family: &unresolved.family, + family_source: &unresolved.family_source, + device_target: &device_target, + host_version_newer, + python, + wheel_compatibility, + version_resolution_specs: Some(&unresolved.version_resolution_specs), + torchaudio_pin: Some(&unresolved.torchaudio_pin), + no_wheel_warning: no_wheel_warning.as_deref(), + stack: PlanStack::NeedsUv { + explicit_install_root: explicit_install_root.as_deref(), + }, + }; + let mut output = String::new(); + write_wheel_plan(&mut output, &plan); + write_dry_run_tail(&mut output, &plan); + output +} diff --git a/apps/rocm/src/tool_policy.rs b/apps/rocm/src/tool_policy.rs index 678dbdaec8..9634985e45 100644 --- a/apps/rocm/src/tool_policy.rs +++ b/apps/rocm/src/tool_policy.rs @@ -18,8 +18,8 @@ //! `rocm_chat_tool_requested_command` turn a tool call into the argv and the //! one-line command it stands for. //! -//! The module was relocated verbatim from `chat.rs`, which calls into it and -//! re-exports the items other modules reach as `chat::`. It calls +//! The module was relocated verbatim from `chat.rs`, whose child modules call +//! into it and which re-exports the items other modules reach as `chat::`. It calls //! nothing in `chat.rs` and nothing in `freeform_planner.rs`, which is what //! lets the planner call into it without importing `chat`. @@ -254,7 +254,7 @@ const CHAT_TOOLS_SCREENING_THEIR_OWN_ARGUMENTS: [&str; 1] = ["rocm_command"]; /// /// Scope note, so the guarantee is not overread: this covers model-supplied /// *arguments*, not model-supplied prose. An assistant's reply text is still -/// printed as it comes; `strip_ansi_sequences` in `comfyui.rs` is the tool for +/// printed as it comes; `rocm_core::strip_ansi_sequences` is the tool for /// that surface, and is already used on ComfyUI log output. /// /// Walks the whole arguments object rather than naming fields, so a new @@ -849,23 +849,44 @@ fn canonicalize_chat_rocm_command(args: &mut [String]) -> Result<()> { Ok(()) } +/// The rules a model-proposed `rocm serve` must satisfy, whoever proposes it. +/// +/// Four, checked in this order and reported first-failure-wins: no +/// `--allow-public-bind`, no CPU device, a loopback `--host` if one is given, +/// and `--managed` without `--foreground`. `args` is the whole argv, starting +/// at `serve`; the caller has already decided it is one. +/// +/// `actor` names who proposed the call and opens every refusal (`{actor} +/// cannot request ...`), so each surface keeps saying who was refused while the +/// rules, their order and the rest of the wording stay in this one place. The +/// chat policy passes `local assistant`; the provider-assisted planner passes +/// `provider planner`. Deleting a rule here removes it from both. +pub(crate) fn validate_serve_command_safety(args: &[String], actor: &str) -> Result<()> { + if chat_cli_has_flag(args, "--allow-public-bind") { + bail!("{actor} cannot request public network binding"); + } + if serve_args_request_cpu_device(args) { + bail!("{actor} cannot request CPU execution; ROCm GPU execution is required"); + } + if let Some(host) = chat_cli_arg_value(args, "--host") + && !is_loopback_host(host) + { + bail!("{actor} cannot request non-local host `{host}`"); + } + if chat_cli_has_flag(args, "--foreground") || !chat_cli_has_flag(args, "--managed") { + bail!("{actor} must request managed serving with --managed"); + } + Ok(()) +} + pub(crate) fn validate_chat_rocm_command_safety(args: &[String]) -> Result<()> { if let Some(first) = args.first() && first.eq_ignore_ascii_case("serve") { - if chat_cli_has_flag(args, "--allow-public-bind") { - bail!("local assistant cannot request public network binding"); - } - if serve_args_request_cpu_device(args) { - bail!("local assistant cannot request CPU execution; ROCm GPU execution is required"); - } - if let Some(host) = chat_cli_arg_value(args, "--host") - && !is_loopback_host(host) - { - bail!("local assistant cannot request non-local host `{host}`"); - } + validate_serve_command_safety(args, "local assistant")?; // The same rule `rocm serve` applies, so an approval card is never offered - // for a command the CLI then refuses. + // for a command the CLI then refuses. Only the chat policy applies it; the + // provider-assisted planner's check has no port rule. let port = chat_cli_arg_value(args, "--port") .map(crate::serve_port::parse_serve_port) .transpose() @@ -874,9 +895,6 @@ pub(crate) fn validate_chat_rocm_command_safety(args: &[String]) -> Result<()> { chat_cli_arg_value(args, "--host").unwrap_or(rocm_core::DEFAULT_LOCAL_HOST), crate::serve_port::PortRequest::from_flag(port), )?; - if chat_cli_has_flag(args, "--foreground") || !chat_cli_has_flag(args, "--managed") { - bail!("local assistant must request managed serving with --managed"); - } } if args .first() @@ -2629,4 +2647,188 @@ mod tests { "accepted as a user install folder: {accepted:?}" ); } + + /// The serve safety rules as the chat assistant meets them: every refusal in + /// the shared table, through the safety check itself and through the two + /// entry points a model's `rocm_command` call goes by. A refused call yields + /// no `ChatRocmCommandAction`, so no approval card is built and nothing runs. + /// + /// The planner suite (`freeform_planner`) runs the same table. + #[test] + fn chat_refuses_each_unsafe_serve_and_proposes_nothing() { + use crate::test_support::{SERVE_SAFETY_REFUSALS, owned_argv}; + + for case in SERVE_SAFETY_REFUSALS { + let context = format!("{}: {:?}", case.rule, case.args); + let error = validate_chat_rocm_command_safety(&owned_argv(case.args)) + .expect_err(&context) + .to_string(); + assert_eq!(error, case.chat_error, "{context}"); + + let call = rocm_command_call(case.args); + let error = chat_rocm_command_action(&call) + .expect_err(&context) + .to_string(); + assert_eq!(error, case.chat_error, "{context}"); + let error = validate_chat_tool_call(&call) + .expect_err(&context) + .to_string(); + assert_eq!(error, case.chat_error, "{context}"); + } + + // `--foreground` refuses even next to `--managed`. The chat check is the + // only one that sees this pair; the planner's clap parse refuses it first. + let error = validate_chat_rocm_command_safety(&owned_argv( + crate::test_support::SERVE_FOREGROUND_AND_MANAGED, + )) + .expect_err("foreground serving is refused even with --managed") + .to_string(); + assert_eq!( + error, + "local assistant must request managed serving with --managed" + ); + } + + #[test] + fn chat_proposes_a_safe_serve_for_approval_with_its_argv_unchanged() { + use crate::test_support::{SERVE_SAFETY_ACCEPTED, owned_argv}; + + for args in SERVE_SAFETY_ACCEPTED { + let argv = owned_argv(args); + validate_chat_rocm_command_safety(&argv) + .unwrap_or_else(|error| panic!("{args:?} must pass the safety check: {error}")); + let action = chat_rocm_command_action(&rocm_command_call(args)) + .unwrap_or_else(|error| panic!("{args:?} must be proposed: {error}")); + // A serve starts a server, so it is never read-only. + assert_eq!( + action, + ChatRocmCommandAction::Approval { + args: argv, + pending_title: "Start local model server".to_owned(), + command_title: "Serve".to_owned(), + }, + "{args:?}" + ); + } + } + + /// The chat policy also applies `rocm serve`'s port rule, which the planner's + /// check does not: a loopback host other than `127.0.0.1` needs an explicit + /// `--port`. Each argv is refused with the exact message and proposes nothing, + /// and the same argv with a port is accepted (the shared accepted table). + /// `planner_accepts_a_loopback_host_without_a_port` holds the planner side. + #[test] + fn chat_requires_a_port_on_a_loopback_host_other_than_the_default() { + use crate::test_support::{SERVE_PORT_RULE_CHAT_ONLY, owned_argv}; + + for (args, expected) in SERVE_PORT_RULE_CHAT_ONLY { + let error = validate_chat_rocm_command_safety(&owned_argv(args)) + .expect_err("a non-default host needs a port") + .to_string(); + assert_eq!(error, *expected, "{args:?}"); + let call = rocm_command_call(args); + let error = chat_rocm_command_action(&call) + .expect_err("no approval card for a command the CLI refuses") + .to_string(); + assert_eq!(error, *expected, "{args:?}"); + let error = validate_chat_tool_call(&call) + .expect_err("the tool-call entry refuses it too") + .to_string(); + assert_eq!(error, *expected, "{args:?}"); + } + } + + /// The rules fire in a fixed order, so an argv that breaks several reports + /// the first one: public bind, then CPU, then host, then managed. + #[test] + fn chat_reports_the_first_broken_serve_rule() { + use crate::test_support::owned_argv; + + for (args, expected) in [ + ( + &[ + "serve", + "qwen", + "--allow-public-bind", + "--device", + "cpu", + "--host", + "0.0.0.0", + "--foreground", + ][..], + "local assistant cannot request public network binding", + ), + ( + &[ + "serve", + "qwen", + "--device", + "cpu", + "--host", + "0.0.0.0", + "--foreground", + ][..], + "local assistant cannot request CPU execution; ROCm GPU execution is required", + ), + ( + &["serve", "qwen", "--host", "0.0.0.0", "--foreground"][..], + "local assistant cannot request non-local host `0.0.0.0`", + ), + ( + &["serve", "qwen", "--foreground"][..], + "local assistant must request managed serving with --managed", + ), + ] { + let error = validate_chat_rocm_command_safety(&owned_argv(args)) + .expect_err("unsafe") + .to_string(); + assert_eq!(error, expected, "{args:?}"); + } + } + + /// The shared serve check opens every refusal with the actor its caller + /// names, so each surface keeps saying who it refused. + #[test] + fn serve_safety_refusals_open_with_the_actor_the_caller_names() { + use crate::test_support::{SERVE_SAFETY_REFUSALS, owned_argv}; + + for case in SERVE_SAFETY_REFUSALS { + let error = validate_serve_command_safety(&owned_argv(case.args), "some caller") + .expect_err(case.rule) + .to_string(); + assert_eq!( + error, + case.chat_error + .replacen("local assistant", "some caller", 1), + "{}: {:?}", + case.rule, + case.args + ); + } + } + + /// The chat check matches the `serve` word without regard to case, so a + /// model that writes `SERVE` does not step around the rules. + #[test] + fn chat_enforces_the_serve_rules_whatever_the_case_of_the_serve_word() { + use crate::test_support::owned_argv; + + for word in ["Serve", "SERVE"] { + let error = validate_chat_rocm_command_safety(&owned_argv(&[ + word, + "qwen", + "--device", + "cpu", + "--managed", + ])) + .expect_err("a CPU serve is refused in any case") + .to_string(); + assert_eq!( + error, + "local assistant cannot request CPU execution; ROCm GPU execution is required" + ); + validate_chat_rocm_command_safety(&owned_argv(&[word, "qwen", "--managed"])) + .expect("a managed serve passes in any case"); + } + } } diff --git a/apps/rocm/src/uninstall.rs b/apps/rocm/src/uninstall.rs index aec8c1f57e..28007a0c46 100644 --- a/apps/rocm/src/uninstall.rs +++ b/apps/rocm/src/uninstall.rs @@ -9,6 +9,8 @@ //! owns what only uninstall does on top of removing files: //! //! - undoing the PATH change the installer recorded making; +//! - removing the fish completion script the Linux installer recorded writing, +//! when it is still the installer's own file; //! - deleting the running `rocm.exe` on Windows, which cannot unlink itself. use std::ffi::OsString; @@ -18,17 +20,18 @@ use std::io::Write as _; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, require_yes_or_interactive}; +use rocm_core::{AppPaths, interactive_terminal, require_yes_or_interactive}; use crate::uninstall_plan::{ - UninstallOptions, UninstallPlan, build_uninstall_plan, confirm_uninstall, - is_rocm_install_entry_name, remove_path, render_uninstall_plan, + PlanAudience, UninstallOptions, UninstallPlan, UninstallPlanEntry, build_uninstall_plan, + confirm_uninstall, is_rocm_install_entry_name, remove_path, render_uninstall_plan, }; /// The file, next to the installed binaries, in which `install.sh` and /// `install.ps1` record the PATH change they made, one per line: -/// `profile ` and `path-expr ` -/// from `install.sh`, `user-path ` from `install.ps1`. +/// `profile `, `path-expr `, +/// and `completions ` from `install.sh`, +/// `user-path ` from `install.ps1`. pub(crate) const PATH_EDIT_RECORD: &str = ".rocm-cli-path-edit"; const PROFILE_BLOCK_START: &str = "# >>> rocm-cli path >>>"; @@ -63,10 +66,31 @@ impl PathEdit { } } +/// `rocm uninstall` has no `--force`; the flag is declared (hidden) only so this +/// can answer it. Left undeclared, clap's nearest-name guess for `--force` is +/// `--force-dev-binaries`, which removes development binaries, and a near miss +/// such as `--forc` gets that same suggestion. Declared, those guesses land here +/// instead, before anything is planned or removed. +pub(crate) fn reject_force(force: bool) -> Result<()> { + if force { + bail!( + "`--force` is not an option of `rocm uninstall`; use `--yes` to skip the confirmation" + ); + } + Ok(()) +} + pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { let paths = AppPaths::discover()?; let plan = build_uninstall_plan(&paths, &options)?; - print!("{}", render_uninstall_plan(&plan, &options)); + print!( + "{}", + render_uninstall_plan( + &plan, + &options, + PlanAudience::for_command(interactive_terminal()) + ) + ); if plan_is_empty(&plan) || options.dry_run { return Ok(()); @@ -91,6 +115,8 @@ pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { /// the record of the change are all still there to retry with. Then the /// config, data, and cache go, while `rocm` is still there to retry a failure /// with; then the binaries, and the record only once every change is undone. +/// A fish completion script that cannot be removed is reported as a warning +/// and the rest goes on. fn execute_plan(plan: &UninstallPlan, report: &mut dyn FnMut(String)) -> Result<()> { for edit in &plan.path_edits { match apply_path_edit(edit) { @@ -107,9 +133,33 @@ fn execute_plan(plan: &UninstallPlan, report: &mut dyn FnMut(String)) -> Result< let non_binaries = plan.actions.iter().filter(|entry| !is_binary(entry.kind)); let binaries = plan.actions.iter().filter(|entry| is_binary(entry.kind)); for entry in non_binaries.chain(binaries) { - remove_path(&entry.path) - .with_context(|| format!("failed to remove {}", entry.path.display()))?; - report(format!("removed {} {}", entry.kind, entry.path.display())); + // The review may have sat at a prompt, so a completion script is + // looked at again: only the regular file that was planned is removed. + if entry.kind == SHELL_COMPLETIONS_KIND && is_not_a_regular_file(&entry.path) { + report(format!( + "warning: left {} {} in place: it is no longer the regular file that was \ + reviewed", + entry.kind, + entry.path.display() + )); + continue; + } + match remove_path(&entry.path) { + Ok(()) => report(format!("removed {} {}", entry.kind, entry.path.display())), + // Completions are best effort when installed, so a script that + // cannot be removed does not stop the uninstall either. + Err(error) if entry.kind == SHELL_COMPLETIONS_KIND => report(format!( + "warning: could not remove {} {}: {error:#}\n \ + The uninstall went on without it; delete that file yourself so fish \ + stops completing rocm.", + entry.kind, + entry.path.display() + )), + Err(error) => { + return Err(error) + .with_context(|| format!("failed to remove {}", entry.path.display())); + } + } } if let Some(record) = &plan.path_record { remove_path(record).with_context(|| format!("failed to remove {}", record.display()))?; @@ -166,7 +216,8 @@ pub(crate) fn render_extra_plan(output: &mut String, plan: &UninstallPlan) { // --------------------------------------------------------------------------- /// Plan undoing the PATH change the installer made for `bin_dir`, the folder -/// whose ROCm CLI entries `removed` lists. +/// whose ROCm CLI entries `removed` lists, and (on Linux) removing the fish +/// completion script it recorded writing for them. /// /// Only a change the installer recorded (or the shell-profile block it marks as /// its own) is undone, never a PATH line the user wrote. And it is left in place @@ -219,6 +270,9 @@ pub(crate) fn plan_path_edits( )); } } else { + if plan_completion_removals(plan, &entries, bin_dir) { + unreadable = true; + } let mut path_exprs = profile_path_exprs(bin_dir, home); for (kind, target) in &entries { if *kind == "path-expr" && !path_exprs.contains(target) { @@ -314,6 +368,167 @@ pub(crate) fn plan_path_edits( } } +/// The plan entry kind for a fish completion script `install.sh` wrote. A +/// failure to remove one is a warning, not a stop: the installer treats +/// completions as best effort, so uninstall does too. +pub(crate) const SHELL_COMPLETIONS_KIND: &str = "shell completions"; + +/// The first line of the fish completion script `install.sh` writes, followed +/// by the folder it installed into. Kept in step with `FISH_COMPLETIONS_MARKER` +/// in `install.sh`, which a test below checks. +const FISH_COMPLETIONS_MARKER: &[u8] = b"# rocm-cli fish completions for "; + +/// Plan removing the fish completion scripts `install.sh` recorded writing. +/// +/// They only describe the binaries being removed, so they go with them even +/// when the PATH change stays for other programs in the folder. `rocm.fish` is +/// one fixed path that the user, or another install folder, may have written +/// since, so a script is removed only when it is still a regular file whose +/// first line is the installer's marker naming `bin_dir`. Any other script the +/// record names is left in place, and the review says why. +/// +/// Returns whether a script could not be checked, in which case the record has +/// to outlive this uninstall so a later one can look again. +fn plan_completion_removals( + plan: &mut UninstallPlan, + entries: &[(&str, OsString)], + bin_dir: &Path, +) -> bool { + plan_completion_removals_with(plan, entries, bin_dir, fish_completions_owner) +} + +/// [`plan_completion_removals`] with the reading of a script's marker line +/// supplied, so a script that cannot be read is testable without relying on +/// file permissions, which root ignores. +fn plan_completion_removals_with( + plan: &mut UninstallPlan, + entries: &[(&str, OsString)], + bin_dir: &Path, + read_owner: impl Fn(&Path) -> std::io::Result>, +) -> bool { + let mut unchecked = false; + for (kind, target) in entries { + if *kind != "completions" { + continue; + } + let path = PathBuf::from(target); + if !is_recorded_fish_script(&path) { + plan.skipped.push(format!( + "the install record names {} as fish completions, which is not an absolute \ + path ending in completions/rocm.fish without `..`, so it is left in place", + path.display() + )); + continue; + } + let meta = match fs::symlink_metadata(&path) { + Ok(meta) => meta, + // Already gone: nothing to remove or to say. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + plan.warnings.push(format!( + "could not look at {} to check that the installer wrote it, so it is left \ + in place: {error}", + path.display() + )); + unchecked = true; + continue; + } + }; + if meta.file_type().is_symlink() { + plan.skipped.push(format!( + "fish completions {} left in place: it is a link, and the installer writes a file", + path.display() + )); + continue; + } + if !meta.is_file() { + plan.skipped.push(format!( + "fish completions {} left in place: it is not a regular file, and the installer \ + writes a file", + path.display() + )); + continue; + } + match read_owner(&path) { + Ok(Some(owner)) if same_folder(&owner, bin_dir) => { + plan.actions.push(UninstallPlanEntry { + kind: SHELL_COMPLETIONS_KIND, + path, + }); + } + Ok(Some(owner)) => plan.skipped.push(format!( + "fish completions {} left in place: the rocm-cli install in {} wrote it, not \ + this one in {}", + path.display(), + owner.display(), + bin_dir.display() + )), + Ok(None) => plan.skipped.push(format!( + "fish completions {} left in place: it does not start with the rocm-cli \ + installer's marker, so the installer did not write it", + path.display() + )), + Err(error) => { + plan.warnings.push(format!( + "could not read {} to check that the installer wrote it, so it is left in \ + place; delete it yourself if you no longer want it: {error}", + path.display() + )); + unchecked = true; + } + } + } + unchecked +} + +/// Whether `path` is now something other than a regular file, such as a link +/// or a folder. A path that is gone, or that cannot be looked at, is not: the +/// removal that follows then reports that itself. +fn is_not_a_regular_file(path: &Path) -> bool { + matches!(fs::symlink_metadata(path), Ok(meta) if !meta.is_file()) +} + +/// Whether a `completions` line in the record names a path `install.sh` +/// writes: absolute, with no `..`, and ending in `completions/rocm.fish`. The +/// record sits in a user-writable folder, so a line naming anything else is not +/// one the installer wrote. +fn is_recorded_fish_script(path: &Path) -> bool { + path.is_absolute() + && !path + .components() + .any(|part| part == std::path::Component::ParentDir) + && path.file_name().is_some_and(|name| name == "rocm.fish") + && path + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name == "completions") +} + +/// The install folder named by the marker on the first line of `path`, or +/// `None` when that line is not the installer's marker. +fn fish_completions_owner(path: &Path) -> std::io::Result> { + use std::io::{BufRead as _, BufReader}; + let mut first = Vec::new(); + BufReader::new(fs::File::open(path)?).read_until(b'\n', &mut first)?; + let line = first.strip_suffix(b"\n").unwrap_or(&first); + let line = line.strip_suffix(b"\r").unwrap_or(line); + Ok(line + .strip_prefix(FISH_COMPLETIONS_MARKER) + .filter(|dir| !dir.is_empty()) + .map(|dir| PathBuf::from(os_string_from_bytes(dir)))) +} + +/// Whether `a` and `b` name the same folder, as spelled or with links +/// resolved: the installer names the folder as `$HOME` spelled it, while the +/// running binary reports it resolved. +fn same_folder(a: &Path, b: &Path) -> bool { + a == b + || matches!( + (fs::canonicalize(a), fs::canonicalize(b)), + (Ok(a), Ok(b)) if a == b + ) +} + /// The `kind target` lines of a PATH-edit record. A target is kept as the /// bytes the installer wrote, so a path that is not UTF-8 still names its file. fn parse_path_edit_record(record: &[u8]) -> Vec<(&str, OsString)> { @@ -837,7 +1052,8 @@ mod tests { self_delete: Some(PathBuf::from("/opt/bin/rocm.exe")), ..UninstallPlan::default() }; - let output = render_uninstall_plan(&plan, &UninstallOptions::default()); + let output = + render_uninstall_plan(&plan, &UninstallOptions::default(), PlanAudience::Captured); for expected in [ "2 item(s) would be removed:", " - install record: /opt/bin/.rocm-cli-path-edit", @@ -1239,15 +1455,24 @@ mod tests { Ok(()) } - /// `--keep-binaries` keeps the installer's PATH change and its record - /// with the binaries, and says so. Run against an installed folder that - /// holds both, so the same plan without the flag would undo them. + /// `--keep-binaries` keeps the installer's PATH change, its fish + /// completions, and its record with the binaries, and says so. Run + /// against an installed folder that holds all of them, so the same plan + /// without the flag would remove them. #[test] fn keeping_the_binaries_keeps_the_path_change_and_says_so() -> Result<()> { let layout = layout()?; let profile = layout.home.join(".bashrc"); fs::write(&profile, POSIX_BLOCK)?; - let record = record(&layout, &format!("profile {}\n", profile.display()))?; + let completions = fish_completions(&layout)?; + let record = record( + &layout, + &format!( + "profile {}\ncompletions {}\n", + profile.display(), + completions.display() + ), + )?; let plan_with = |keep_binaries| -> Result { let mut plan = UninstallPlan::default(); let options = UninstallOptions { @@ -1266,6 +1491,11 @@ mod tests { let plan = plan_with(false)?; assert_eq!(profiles(&plan), vec![profile.clone()], "{plan:?}"); assert_eq!(plan.path_record.as_ref(), Some(&record)); + assert_eq!( + completion_actions(&plan), + vec![completions.clone()], + "{plan:?}" + ); let plan = plan_with(true)?; assert!( @@ -1277,14 +1507,20 @@ mod tests { plan.skipped .iter() .any(|note| note.contains("--keep-binaries") - && note.contains("PATH change the installer made stays too")), + && note.contains( + "PATH change and fish completions the installer made stay too" + )), "{plan:?}" ); - // Carrying the plan out leaves the change, its record, and the - // binaries where they are. + // Carrying the plan out leaves the change, the completions, their + // record, and the binaries where they are. run(&plan).1?; assert_eq!(fs::read_to_string(&profile)?, POSIX_BLOCK); assert!(record.exists() && layout.removed.iter().all(|path| path.exists())); + assert!( + completions.exists(), + "the completions stay with the binaries" + ); Ok(()) } @@ -1310,5 +1546,483 @@ mod tests { assert_eq!(fs::metadata(&real)?.permissions().mode() & 0o777, 0o640); Ok(()) } + + /// The fish completion script `install.sh` writes for the install in + /// `bin`: its marker line naming that folder, then the script. + fn marked_script(bin: &Path) -> String { + format!( + "{}{}\ncomplete -c rocm\n", + String::from_utf8_lossy(FISH_COMPLETIONS_MARKER), + bin.display() + ) + } + + /// Write `contents` to `path`, creating its folder. + fn write_script(path: &Path, contents: &str) -> Result<()> { + fs::create_dir_all(path.parent().expect("has a folder"))?; + fs::write(path, contents)?; + Ok(()) + } + + /// Where `install.sh` writes fish completions under `layout`'s home, + /// written there as it writes them. + fn fish_completions(layout: &Layout) -> Result { + let path = layout + .home + .join(".config") + .join("fish") + .join("completions") + .join("rocm.fish"); + write_script(&path, &marked_script(&layout.bin))?; + Ok(path) + } + + fn completion_actions(plan: &UninstallPlan) -> Vec { + plan.actions + .iter() + .filter(|entry| entry.kind == SHELL_COMPLETIONS_KIND) + .map(|entry| entry.path.clone()) + .collect() + } + + /// The plan for `layout` with its binaries, after checking that planning + /// itself chose no completion script to remove. `with_binaries` replaces + /// the plan's actions, so asserting on the result alone could not fail. + fn binaries_with_completions_planned_for_none(layout: &Layout) -> UninstallPlan { + let planned = plan_for(layout); + assert_eq!( + completion_actions(&planned), + Vec::::new(), + "{planned:?}" + ); + with_binaries(layout, planned) + } + + fn completions_record(layout: &Layout, completions: &Path) -> Result { + record(layout, &format!("completions {}\n", completions.display())) + } + + #[test] + fn recorded_fish_completions_are_removed_with_the_binaries() -> Result<()> { + let layout = layout()?; + let profile = layout.home.join(".bashrc"); + fs::write(&profile, format!("export A=1\n{POSIX_BLOCK}"))?; + let completions = fish_completions(&layout)?; + let record = record( + &layout, + &format!( + "profile {}\ncompletions {}\n", + profile.display(), + completions.display() + ), + )?; + let mut plan = with_binaries(&layout, UninstallPlan::default()); + plan_path_edits(&mut plan, &layout.bin, &layout.removed, Some(&layout.home)); + assert_eq!( + completion_actions(&plan), + vec![completions.clone()], + "{plan:?}" + ); + + let (lines, result) = run(&plan); + result?; + assert!(!completions.exists(), "the completion script is gone"); + assert!( + completions.parent().is_some_and(Path::exists), + "only the script goes, not the user's completions folder" + ); + assert!(!record.exists() && !layout.removed[0].exists()); + assert!( + lines + .iter() + .any(|line| line + == &format!("removed shell completions {}", completions.display())), + "{lines:?}" + ); + Ok(()) + } + + #[test] + fn fish_completions_go_even_when_the_path_change_stays_for_other_programs() -> Result<()> { + let layout = layout()?; + let profile = layout.home.join(".bashrc"); + fs::write(&profile, POSIX_BLOCK)?; + fs::write(layout.bin.join("uv"), "")?; + let completions = fish_completions(&layout)?; + record( + &layout, + &format!( + "profile {}\ncompletions {}\n", + profile.display(), + completions.display() + ), + )?; + let plan = plan_for(&layout); + assert!(plan.path_edits.is_empty(), "{plan:?}"); + assert_eq!(completion_actions(&plan), vec![completions], "{plan:?}"); + Ok(()) + } + + /// Every file below carries this install's marker, so what stops each + /// one is the check on the recorded path or on the kind of file. + #[test] + fn a_completions_line_naming_another_file_is_not_trusted() -> Result<()> { + let layout = layout()?; + let marked = marked_script(&layout.bin); + let elsewhere = layout.home.join("completions").join("other.fish"); + write_script(&elsewhere, &marked)?; + let folder = layout + .home + .join("dir") + .join("completions") + .join("rocm.fish"); + fs::create_dir_all(folder.join("keep"))?; + let not_completions = layout.home.join("notcompletions").join("rocm.fish"); + write_script(¬_completions, &marked)?; + // `..` that lands on a real `completions/rocm.fish` file. + let dotted = layout + .home + .join("x") + .join("completions") + .join("..") + .join("completions") + .join("rocm.fish"); + write_script( + &layout.home.join("x").join("completions").join("rocm.fish"), + &marked, + )?; + let missing = layout + .home + .join("gone") + .join("completions") + .join("rocm.fish"); + // A line of another kind naming a real script this install wrote. + let other_kind = fish_completions(&layout)?; + record( + &layout, + &format!( + "completions {}\ncompletions {}\ncompletions {}\ncompletions {}\n\ + completions {}\nprofile {}\npath-expr {}\n", + elsewhere.display(), + folder.display(), + missing.display(), + not_completions.display(), + dotted.display(), + other_kind.display(), + other_kind.display() + ), + )?; + let plan = plan_for(&layout); + assert_eq!(completion_actions(&plan), Vec::::new(), "{plan:?}"); + // Each refused line is named in the review, except the missing + // file, which is simply gone. + for refused in [&elsewhere, ¬_completions, &dotted] { + assert!( + plan.skipped.iter().any(|note| note.starts_with(&format!( + "the install record names {} as fish completions", + refused.display() + ))), + "{plan:?}" + ); + } + assert!( + plan.skipped.iter().any(|note| note + == &format!( + "fish completions {} left in place: it is not a regular file, and the \ + installer writes a file", + folder.display() + )), + "{plan:?}" + ); + assert!( + !plan + .skipped + .iter() + .chain(&plan.warnings) + .any(|note| note.contains(&missing.display().to_string())), + "{plan:?}" + ); + run(&plan).1?; + assert!(other_kind.exists(), "a line of another kind removed it"); + assert!(folder.join("keep").exists(), "a folder was removed"); + Ok(()) + } + + /// A relative path cannot be tried against the filesystem without + /// depending on the test's working folder, so the check is pinned + /// directly. + #[test] + fn only_an_absolute_completions_rocm_fish_path_is_the_installers() { + assert!(is_recorded_fish_script(Path::new( + "/home/me/.config/fish/completions/rocm.fish" + ))); + for path in [ + ".config/fish/completions/rocm.fish", + "completions/rocm.fish", + "/home/me/.config/fish/completions/../completions/rocm.fish", + "/home/me/.config/fish/completions/other.fish", + "/home/me/.config/fish/rocm.fish", + ] { + assert!(!is_recorded_fish_script(Path::new(path)), "{path}"); + } + } + + /// A script the installer did not write, such as one the user made + /// with `rocm completions fish > ...`, stays, and the review shown + /// before the uninstall runs says so. + #[test] + fn unmarked_fish_completions_are_left_in_place_and_the_review_says_so() -> Result<()> { + let layout = layout()?; + let completions = fish_completions(&layout)?; + fs::write(&completions, "complete -c rocm\n")?; + completions_record(&layout, &completions)?; + let plan = binaries_with_completions_planned_for_none(&layout); + assert_eq!(completion_actions(&plan), Vec::::new(), "{plan:?}"); + + let review = + render_uninstall_plan(&plan, &UninstallOptions::default(), PlanAudience::Captured); + let note = format!( + " - fish completions {} left in place: it does not start with the rocm-cli \ + installer's marker, so the installer did not write it", + completions.display() + ); + assert!(review.contains(&format!("Left alone:\n{note}")), "{review}"); + let (lines, result) = run(&plan); + result?; + assert_eq!(fs::read_to_string(&completions)?, "complete -c rocm\n"); + assert!(!layout.removed[0].exists(), "the binaries still go"); + assert!( + !lines.iter().any(|line| line.contains("rocm.fish")), + "{lines:?}" + ); + Ok(()) + } + + /// `rocm.fish` is one path for every install folder, so the one that + /// wrote it last owns it: uninstalling another leaves it. + #[test] + fn fish_completions_another_install_folder_wrote_are_left_in_place() -> Result<()> { + let layout = layout()?; + let other_bin = layout.home.join("opt").join("rocm").join("bin"); + fs::create_dir_all(&other_bin)?; + let completions = fish_completions(&layout)?; + fs::write(&completions, marked_script(&other_bin))?; + completions_record(&layout, &completions)?; + let plan = binaries_with_completions_planned_for_none(&layout); + assert_eq!(completion_actions(&plan), Vec::::new(), "{plan:?}"); + let review = + render_uninstall_plan(&plan, &UninstallOptions::default(), PlanAudience::Captured); + assert!( + review.contains(&format!( + "fish completions {} left in place: the rocm-cli install in {} wrote it, \ + not this one in {}", + completions.display(), + other_bin.display(), + layout.bin.display() + )), + "{review}" + ); + run(&plan).1?; + assert_eq!(fs::read_to_string(&completions)?, marked_script(&other_bin)); + Ok(()) + } + + /// A script that cannot be checked is left in place with a warning, and + /// the install record stays so a later uninstall can look again. + #[test] + fn fish_completions_that_cannot_be_checked_are_left_and_keep_the_record() -> Result<()> { + let layout = layout()?; + // A file where a folder is needed: looking at the path fails with + // something other than "not found", even for root. + let blocker = layout.home.join("afile"); + fs::write(&blocker, "")?; + let script = blocker.join("completions").join("rocm.fish"); + let record = completions_record(&layout, &script)?; + let plan = binaries_with_completions_planned_for_none(&layout); + assert_eq!(completion_actions(&plan), Vec::::new(), "{plan:?}"); + assert!( + plan.warnings.iter().any(|note| note.starts_with(&format!( + "could not look at {} to check that the installer wrote it, so it is left \ + in place: ", + script.display() + ))), + "{plan:?}" + ); + assert_eq!( + plan.path_record, None, + "the record must outlive this uninstall" + ); + run(&plan).1?; + assert!(record.exists(), "the record stays for a later uninstall"); + Ok(()) + } + + /// The same for a script whose first line cannot be read, which no file + /// permission can stage for root, so the reader is supplied. + #[test] + fn a_marker_that_cannot_be_read_is_a_warning_and_the_script_stays() -> Result<()> { + let layout = layout()?; + let script = fish_completions(&layout)?; + let entries = vec![("completions", script.clone().into_os_string())]; + let mut plan = UninstallPlan::default(); + let unchecked = plan_completion_removals_with(&mut plan, &entries, &layout.bin, |_| { + Err(std::io::Error::other("no access")) + }); + assert!(unchecked, "{plan:?}"); + assert_eq!(completion_actions(&plan), Vec::::new(), "{plan:?}"); + assert_eq!( + plan.warnings, + vec![format!( + "could not read {} to check that the installer wrote it, so it is left in \ + place; delete it yourself if you no longer want it: no access", + script.display() + )] + ); + assert!(script.exists()); + Ok(()) + } + + /// The installer names the folder as `$HOME` spells it, while the + /// running binary reports it with links resolved. + #[test] + fn a_marker_naming_this_folder_through_a_link_still_counts() -> Result<()> { + let layout = layout()?; + let linked_bin = layout.home.join("linked-bin"); + std::os::unix::fs::symlink(&layout.bin, &linked_bin)?; + let completions = fish_completions(&layout)?; + fs::write(&completions, marked_script(&linked_bin))?; + completions_record(&layout, &completions)?; + let plan = plan_for(&layout); + assert_eq!(completion_actions(&plan), vec![completions], "{plan:?}"); + Ok(()) + } + + #[test] + fn linked_fish_completions_are_left_in_place() -> Result<()> { + let layout = layout()?; + let target = layout.home.join("dotfiles").join("rocm.fish"); + write_script(&target, &marked_script(&layout.bin))?; + let dir = layout.home.join(".config").join("fish").join("completions"); + fs::create_dir_all(&dir)?; + let link = dir.join("rocm.fish"); + std::os::unix::fs::symlink(&target, &link)?; + completions_record(&layout, &link)?; + let plan = binaries_with_completions_planned_for_none(&layout); + assert_eq!(completion_actions(&plan), Vec::::new(), "{plan:?}"); + assert!( + plan.skipped.iter().any(|note| note + == &format!( + "fish completions {} left in place: it is a link, and the installer \ + writes a file", + link.display() + )), + "{plan:?}" + ); + run(&plan).1?; + assert!(fs::symlink_metadata(&link)?.is_symlink(), "the link stays"); + assert!(target.exists(), "what the link points at stays"); + Ok(()) + } + + /// The review may sit at a prompt: a script swapped for a folder before + /// the removal runs is left alone, with a warning, and the rest goes on. + #[test] + fn a_completions_script_swapped_for_a_folder_after_the_review_is_left_alone() -> Result<()> + { + let layout = layout()?; + let script = fish_completions(&layout)?; + completions_record(&layout, &script)?; + let plan = plan_for(&layout); + assert_eq!(completion_actions(&plan), vec![script.clone()], "{plan:?}"); + fs::remove_file(&script)?; + fs::create_dir_all(script.join("precious"))?; + + let (lines, result) = run(&plan); + result?; + assert!(script.join("precious").exists(), "the folder was removed"); + assert!( + lines.iter().any(|line| line + == &format!( + "warning: left shell completions {} in place: it is no longer the \ + regular file that was reviewed", + script.display() + )), + "{lines:?}" + ); + Ok(()) + } + + /// The installer treats completions as best effort, so a script that + /// cannot be removed is a warning: the binaries and the record still + /// go, and the uninstall reports success. + #[test] + fn fish_completions_that_cannot_be_removed_are_a_warning_and_the_rest_goes_on() -> Result<()> + { + let layout = layout()?; + let record = record(&layout, "")?; + let mut plan = with_binaries(&layout, plan_for(&layout)); + plan.path_record = Some(record.clone()); + // A name too long for any filesystem fails to remove, even for root. + let stuck = layout.home.join("x".repeat(300)); + plan.actions.insert( + 0, + crate::uninstall_plan::UninstallPlanEntry { + kind: SHELL_COMPLETIONS_KIND, + path: stuck.clone(), + }, + ); + + let (lines, result) = run(&plan); + result?; + let warning = format!( + "warning: could not remove shell completions {}: ", + stuck.display() + ); + assert!( + lines.iter().any(|line| line.starts_with(&warning) + && line.contains("The uninstall went on without it")), + "{lines:?}" + ); + assert!(layout.removed.iter().all(|path| !path.exists())); + assert!(!record.exists(), "the record still goes last"); + Ok(()) + } + + #[test] + fn a_record_from_before_completions_were_recorded_still_undoes_the_path_change() + -> Result<()> { + let layout = layout()?; + let profile = layout.home.join(".bashrc"); + fs::write(&profile, format!("export A=1\n{POSIX_BLOCK}"))?; + // An earlier installer wrote the script but no `completions` line, + // so uninstall has no record of it and leaves it alone, marker or + // not. + let completions = fish_completions(&layout)?; + let record = record( + &layout, + &format!( + "profile {}\npath-expr $HOME/.local/bin\n", + profile.display() + ), + )?; + let plan = plan_for(&layout); + assert_eq!(completion_actions(&plan), Vec::::new(), "{plan:?}"); + assert_eq!(profiles(&plan), vec![profile.clone()]); + run(&with_binaries(&layout, plan)).1?; + assert_eq!(fs::read_to_string(&profile)?, "export A=1\n"); + assert!(!record.exists() && completions.exists()); + Ok(()) + } + } + + /// `install.sh` writes the marker this module looks for. + #[test] + fn the_installer_writes_the_fish_completions_marker_uninstall_reads() { + let installer = include_str!("../../../install.sh"); + let marker = std::str::from_utf8(FISH_COMPLETIONS_MARKER).expect("ASCII"); + assert!( + installer.contains(&format!("FISH_COMPLETIONS_MARKER='{marker}'")), + "install.sh no longer writes {marker:?} as its fish completions marker" + ); } } diff --git a/apps/rocm/src/uninstall_plan.rs b/apps/rocm/src/uninstall_plan.rs index e450fde189..3c3da4e6d2 100644 --- a/apps/rocm/src/uninstall_plan.rs +++ b/apps/rocm/src/uninstall_plan.rs @@ -5,8 +5,8 @@ //! The uninstall plan: what would be removed, how that plan is rendered for //! review, and the filesystem-removal primitive used to carry it out. //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-212) of the `main.rs` modularization (see `docs/architecture.md`). +//! Mechanically relocated from `main.rs` with no behavior change (the +//! `relocation` class in `docs/architecture.md`). //! `dispatch()`'s call sites are unaffected: none of this module's functions //! are called directly from `dispatch()`. //! @@ -21,22 +21,26 @@ //! describe the `uninstall` command's own, exclusive logic. //! //! This module owns: the plan types (`UninstallOptions`, `UninstallPlan`, -//! `UninstallPlanEntry`), building the plan (`build_uninstall_plan`, +//! `UninstallPlanEntry`), the `PlanAudience` a rendered plan is written for +//! (a person at a terminal, a captured caller, or the assistant's preview), +//! which decides what the dry-run footer says comes next, building the plan (`build_uninstall_plan`, //! `plan_installed_binaries`, and the installed-binary/shared-cache //! discovery helpers behind them), rendering it (`render_uninstall_dry_run`, //! `render_uninstall_plan`), the `[y/N]` confirmation prompt //! (`confirm_uninstall`), and the filesystem removal primitive both //! `uninstall.rs` and `storage.rs` call to carry a plan out (`remove_path`, //! and the entry-resolution helpers behind it). `uninstall.rs` calls into it -//! for `UninstallOptions`/`UninstallPlan`/`build_uninstall_plan`/ +//! for `PlanAudience`/`UninstallOptions`/`UninstallPlan`/`build_uninstall_plan`/ //! `confirm_uninstall`/`is_rocm_install_entry_name`/`remove_path`/ //! `render_uninstall_plan`; `storage.rs` calls into it for //! `UninstallPlan`/`UninstallPlanEntry`/`confirm_uninstall`; `runtimes_cmd.rs` //! calls into it for `confirm_uninstall` (the same `[y/N]` prompt, reused for -//! `rocm runtimes uninstall`); `chat.rs` calls into it for +//! `rocm runtimes uninstall`); `chat/mcp_bridge.rs` calls into it for //! `render_uninstall_dry_run` (the deterministic uninstall-preview tool -//! result); `deletion_properties.rs`'s recursive-deletion property tests call -//! into it for `UninstallOptions`/`build_uninstall_plan`/`remove_path`/ +//! result, which renders for `PlanAudience::Assistant` itself and so does not +//! import the type); `deletion_properties.rs`'s recursive-deletion property +//! tests call into it for `PlanAudience` (the `uninstall_case` property renders +//! as `Captured`)/`UninstallOptions`/`build_uninstall_plan`/`remove_path`/ //! `render_uninstall_plan`. use std::fmt::Write as _; @@ -137,7 +141,7 @@ pub(crate) fn build_uninstall_plan( // Only a helper that is running can be left behind. "Uninstall does not stop // it" is said only where that is the whole story: with a watcher enabled the // helper stays up; an idle one is already on its way out. - if let Some(note) = crate::service_launch::background_helper_note(paths) + if let Some(note) = crate::background_helper::background_helper_note(paths) && note.running { plan.warnings.push(if note.outlives { @@ -193,8 +197,9 @@ pub(crate) fn build_uninstall_plan( } /// Plan removing the installed binaries beside the running one, which -/// `current_exe` names, and undoing the installer's PATH change for their -/// folder. `--keep-binaries` keeps both, and so does a cargo build's folder. +/// `current_exe` names, undoing the installer's PATH change for their folder, +/// and removing the fish completion script it recorded writing for them. +/// `--keep-binaries` keeps all three, and so does a cargo build's folder. pub(crate) fn plan_installed_binaries( plan: &mut UninstallPlan, options: &UninstallOptions, @@ -202,10 +207,11 @@ pub(crate) fn plan_installed_binaries( home: Option<&Path>, ) -> Result<()> { if options.keep_binaries { - // The PATH change only serves the binaries, so it stays with them. + // The PATH change and the completions only serve the binaries, so + // they stay with them. plan.skipped.push( - "binary removal disabled by --keep-binaries, so any PATH change the installer made \ - stays too" + "binary removal disabled by --keep-binaries, so any PATH change and fish completions \ + the installer made stay too" .to_owned(), ); return Ok(()); @@ -268,10 +274,61 @@ pub(crate) fn render_uninstall_dry_run(paths: &AppPaths) -> Result { ..UninstallOptions::default() }; let plan = build_uninstall_plan(paths, &options)?; - Ok(render_uninstall_plan(&plan, &options)) + Ok(render_uninstall_plan( + &plan, + &options, + PlanAudience::Assistant, + )) } -pub(crate) fn render_uninstall_plan(plan: &UninstallPlan, options: &UninstallOptions) -> String { +/// Who reads a rendered plan, which decides what its dry-run footer can truthfully +/// say comes next. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PlanAudience { + /// A person at a terminal ran `rocm uninstall --dry-run` themselves: running + /// the same command again without `--dry-run` is the next step. + Terminal, + /// The output is captured by a caller (a script, the dashboard, the daemon's + /// tool). Whatever drives that caller has its own way to proceed, so there + /// is no next step to name; "run it again" would address the wrong reader. + Captured, + /// The assistant's read-only preview tool: it has no flag to offer, and the + /// removal itself goes through an approval step. + Assistant, +} + +impl PlanAudience { + /// The audience for `rocm uninstall` itself, from whether `interactive_terminal` + /// holds: stdin and stdout are both terminals. A dry run with either one + /// redirected, `--dry-run < /dev/null` at a terminal for one, is `Captured` + /// and prints no footer. + pub(crate) const fn for_command(interactive_terminal: bool) -> Self { + if interactive_terminal { + Self::Terminal + } else { + Self::Captured + } + } + + /// What a dry run says to do next, if anything. + const fn dry_run_footer(self) -> Option<&'static str> { + match self { + Self::Terminal => { + Some("Nothing was removed. To remove these items, run it again without --dry-run.") + } + Self::Captured => None, + // The assistant's own uninstall is an approval action; see + // `lifecycle_read_mutate_split_is_honest` in `tool_policy.rs`. + Self::Assistant => Some("Nothing was removed. Removal needs your approval."), + } + } +} + +pub(crate) fn render_uninstall_plan( + plan: &UninstallPlan, + options: &UninstallOptions, + audience: PlanAudience, +) -> String { let mut output = String::new(); let _ = writeln!(output, "Uninstall review"); let _ = writeln!(output); @@ -299,9 +356,13 @@ pub(crate) fn render_uninstall_plan(plan: &UninstallPlan, options: &UninstallOpt for skipped in &plan.skipped { let _ = writeln!(output, " - {skipped}"); } - if options.dry_run { + // An empty plan has nothing to approve, so a dry run says nothing about a next step. + if options.dry_run + && !uninstall::plan_is_empty(plan) + && let Some(footer) = audience.dry_run_footer() + { let _ = writeln!(output); - let _ = writeln!(output, "Choose Review uninstall to approve removal."); + let _ = writeln!(output, "{footer}"); } output } @@ -379,7 +440,7 @@ fn env_path(name: &str) -> Option { pub(crate) fn confirm_uninstall() -> Result { rocm_core::confirm::print_prompt(&mut io::stdout(), "Proceed with uninstall? [y/N]: ") .context("failed to write uninstall prompt")?; - rocm_core::confirm::read_answer(&mut io::stdin().lock()) + rocm_core::confirm::read_answer(&mut io::stdin().lock(), &mut io::stdout()) .context("failed to read uninstall confirmation") } @@ -521,6 +582,95 @@ mod tests { use super::*; use crate::tests::test_paths; + fn plan_with_a_binary() -> UninstallPlan { + UninstallPlan { + self_delete: Some(PathBuf::from("/opt/bin/rocm")), + ..UninstallPlan::default() + } + } + + fn dry_run() -> UninstallOptions { + UninstallOptions { + dry_run: true, + ..UninstallOptions::default() + } + } + + /// A person at a terminal who dry-runs is told to run the same command again + /// without `--dry-run`: the next step is a rerun, not a control the command + /// line lacks. The scenario `uninstall-07` runs it again at a terminal and + /// checks that the listed folder goes. + #[test] + fn terminal_dry_run_footer_says_to_run_it_again_without_dry_run() { + let output = + render_uninstall_plan(&plan_with_a_binary(), &dry_run(), PlanAudience::Terminal); + assert!( + output.ends_with( + "\nNothing was removed. To remove these items, run it again without --dry-run.\n" + ), + "unexpected footer in:\n{output}" + ); + assert!(!output.contains("Review uninstall"), "{output}"); + } + + /// Output a caller captures is not addressed to someone who can rerun it, so + /// it carries no next step (the dashboard's `/uninstall` and the daemon's + /// `rocm_command` tool run the dry run as a subprocess, where the real step + /// is theirs to offer). + #[test] + fn captured_dry_run_names_no_next_step() { + let output = + render_uninstall_plan(&plan_with_a_binary(), &dry_run(), PlanAudience::Captured); + assert!(!output.contains("Nothing was removed."), "{output}"); + assert!(!output.contains("run it again"), "{output}"); + assert!(output.contains("1 item(s) would be removed:"), "{output}"); + } + + /// The command picks its audience from whether it has a terminal. + #[test] + fn the_command_is_addressed_to_a_terminal_only_when_it_has_one() { + assert_eq!(PlanAudience::for_command(true), PlanAudience::Terminal); + assert_eq!(PlanAudience::for_command(false), PlanAudience::Captured); + } + + /// An empty plan has nothing to approve, so a dry run adds no next step for + /// anyone; a real run adds none either, since its confirmation prompt follows. + #[test] + fn footer_is_only_for_a_dry_run_that_has_something_to_remove() { + for audience in [ + PlanAudience::Terminal, + PlanAudience::Captured, + PlanAudience::Assistant, + ] { + let empty = render_uninstall_plan(&UninstallPlan::default(), &dry_run(), audience); + assert!(!empty.contains("Nothing was removed."), "{empty}"); + let real_run = render_uninstall_plan( + &plan_with_a_binary(), + &UninstallOptions::default(), + audience, + ); + assert!(!real_run.contains("Nothing was removed."), "{real_run}"); + } + } + + /// The assistant's preview has no flag to offer, so it must not point at a + /// command line it cannot run; what it can truthfully say is that removal + /// needs approval (`lifecycle_read_mutate_split_is_honest` in `tool_policy.rs` pins + /// that its own `uninstall` is an approval action). It does not say "these + /// items": the preview always plans with the default options, while the + /// approval carries the user's own. + #[test] + fn assistant_dry_run_footer_names_no_command() { + let output = + render_uninstall_plan(&plan_with_a_binary(), &dry_run(), PlanAudience::Assistant); + assert!( + output.ends_with("\nNothing was removed. Removal needs your approval.\n"), + "unexpected footer in:\n{output}" + ); + assert!(!output.contains("rocm uninstall"), "{output}"); + assert!(!output.contains("these items"), "{output}"); + } + /// A cache that has moved inside a directory uninstall already removes must /// not be reported as "not removed" — the note would be false. #[test] diff --git a/apps/rocm/src/update_cmd.rs b/apps/rocm/src/update_cmd.rs index 49cf65942c..10fac53f17 100644 --- a/apps/rocm/src/update_cmd.rs +++ b/apps/rocm/src/update_cmd.rs @@ -11,14 +11,14 @@ //! update-source/installed-runtime pickers behind it //! (`select_runtime_update_source`, `select_installed_update_runtime`). //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-210) of the `main.rs` modularization (see `docs/architecture.md`). +//! Mechanically relocated from `main.rs` with no behavior change (the +//! `relocation` class in `docs/architecture.md`). //! The `update` command arm of `dispatch()` calls `update`, the handler whose //! body was that arm's own and which was relocated here verbatim, re-imported //! via `use crate::update_cmd::update;`; `update` in turn calls //! `update_should_preview_or_apply(apply, dry_run)`, -//! `apply_runtime_update(...)`, and `render_update_text(&paths)`. `chat.rs`'s -//! deterministic `rocm update` tool summary also calls `render_update_text`, +//! `apply_runtime_update(...)`, and `render_update_text(&paths)`. `chat/mcp_bridge.rs`'s +//! read-only `rocm update` handling also calls `render_update_text`, //! outside `dispatch()`. //! //! One item physically interleaved in this span in `main.rs` stayed there @@ -31,7 +31,7 @@ //! from here the same way via `use crate::ROLLBACK_RECOVERY_HINT;`. //! //! `update` is `pub(crate)` for `dispatch()` and `render_update_text` for -//! `chat.rs`; `apply_runtime_update` and `update_should_preview_or_apply` keep +//! `chat/mcp_bridge.rs`; `apply_runtime_update` and `update_should_preview_or_apply` keep //! the `pub(crate)` they had when `dispatch()` called them directly, though //! only `update` calls them now. `select_runtime_update_source` is also //! `pub(crate)`: it is @@ -186,7 +186,7 @@ fn append_update_surfaces(output: &mut String) { let _ = writeln!( output, " cli: installed={} status=not_configured reason=repository-owned CLI update feed is not published yet", - env!("CARGO_PKG_VERSION") + crate::cli_version_number() ); let engine_ids = engine_inventory() .iter() @@ -254,16 +254,61 @@ pub(crate) const fn update_should_preview_or_apply(apply: bool, dry_run: bool) - apply || dry_run } +/// Which planner each path takes is pinned in two places. The `--dry-run` choice, +/// the check and not the plan, is pinned on every platform by +/// `a_dry_run_takes_the_check_and_an_apply_takes_the_plan` below, which runs +/// `apply_runtime_update_with`. That `runtime_update_check` is the check and +/// `runtime_update_plan` the plan, which the two arguments below cannot tell +/// apart by type, is pinned only on Unix: by +/// `the_update_checks_prepare_no_python_and_the_apply_plan_does` in +/// `therock/update.rs` and by scenario `update-10` on Linux. No test on Windows +/// pins that pairing. pub(crate) fn apply_runtime_update( paths: &AppPaths, config: &mut RocmCliConfig, runtime_selector: Option<&str>, activate: bool, dry_run: bool, +) -> Result { + apply_runtime_update_with( + paths, + config, + runtime_selector, + activate, + dry_run, + |source, manifests| therock::runtime_update_check(paths, source, manifests, None), + |source, manifests| therock::runtime_update_plan(paths, source, manifests, None), + ) +} + +/// The update flow with its two planners passed in. A `--dry-run` only previews, +/// so it takes `check`, which prepares nothing; a real apply takes `apply_plan`, +/// which prepares the Python the install that follows needs. Taking them as +/// arguments lets a test run the choice on every platform without a host that +/// has no Python. +fn apply_runtime_update_with( + paths: &AppPaths, + config: &mut RocmCliConfig, + runtime_selector: Option<&str>, + activate: bool, + dry_run: bool, + check: impl FnOnce( + &therock::InstalledRuntimeManifest, + &[therock::InstalledRuntimeManifest], + ) -> Result, + apply_plan: impl FnOnce( + &therock::InstalledRuntimeManifest, + &[therock::InstalledRuntimeManifest], + ) -> Result, ) -> Result { let manifests = therock::load_runtime_manifests(paths)?; let source = select_runtime_update_source(&manifests, config, runtime_selector)?; - let plan = therock::runtime_update_plan(paths, source, &manifests, None)?; + let planned = if dry_run { + check(source, &manifests) + } else { + apply_plan(source, &manifests) + }; + let plan = planned?; let mut output = String::new(); let _ = writeln!(output, "runtime update"); let _ = writeln!(output, " source_runtime_key: {}", source.runtime_key); @@ -289,6 +334,11 @@ pub(crate) fn apply_runtime_update( return Ok(output); } + // The plan's `python_note` is not printed here: the preview below prints its + // own `python_note:` from the decision a real install takes (scenario + // `update-10` asserts it), so a second line would say the same thing twice. + // A dry run that finds nothing newer returned above and prints neither; the + // plain `rocm update` report is where the check's note shows. if dry_run { let _ = writeln!(output, " mode: dry-run"); let install_plan = therock::install_sdk_for_update( @@ -299,6 +349,7 @@ pub(crate) fn apply_runtime_update( plan.device_target.as_deref(), plan.source_layout_generation.as_deref(), source.includes_devel(), + therock::recorded_python_version(source), true, activate, )?; @@ -324,6 +375,7 @@ pub(crate) fn apply_runtime_update( // Reinstall what the user originally chose rather than silently // adding or dropping the compiler toolchain on update. source.includes_devel(), + therock::recorded_python_version(source), false, activate, )?; @@ -440,6 +492,73 @@ mod tests { use crate::test_support::write_test_pip_runtime; use crate::tests::{test_paths, test_runtime_manifest_for_update}; + /// A `--dry-run` takes its plan from the check and an apply from the plan, + /// and calls only that one: the check must never be the one that prepares a + /// Python, and an apply must still get the plan that does. Both planners + /// report nothing newer, so the flow returns before it installs or previews + /// anything and the test needs no host without a Python, which runs it on + /// every platform. Which `therock` function is the check is pinned only on + /// Unix, by `the_update_checks_prepare_no_python_and_the_apply_plan_does` in + /// `therock/update.rs`, and by scenario `update-10`. + #[test] + fn a_dry_run_takes_the_check_and_an_apply_takes_the_plan() -> Result<()> { + let (root, paths) = test_paths("update-takes-check-or-plan"); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0a20260416", + 1, + )?; + let calls = std::cell::RefCell::new(Vec::new()); + let planner = |name: &'static str| { + let calls = &calls; + move |source: &therock::InstalledRuntimeManifest, + _: &[therock::InstalledRuntimeManifest]| { + calls.borrow_mut().push(name); + Ok(therock::RuntimeUpdatePlan { + latest_version: source.version.clone(), + latest_source: String::new(), + format: source.format.clone(), + status: "up_to_date".to_owned(), + target_runtime_key: source.runtime_key.clone(), + device_target: None, + source_layout_generation: None, + repair_required: false, + update_available: false, + python_note: None, + }) + } + }; + let take = |dry_run: bool| -> Result<(String, Vec<&'static str>)> { + calls.borrow_mut().clear(); + let output = apply_runtime_update_with( + &paths, + &mut RocmCliConfig::default(), + None, + false, + dry_run, + planner("check"), + planner("plan"), + )?; + Ok((output, calls.borrow().clone())) + }; + let (output, taken) = take(true)?; + assert_eq!(taken, ["check"], "a --dry-run took the plan: {output}"); + assert!( + output.contains("result: no newer runtime found"), + "{output}" + ); + let (output, taken) = take(false)?; + assert_eq!(taken, ["plan"], "an apply took the check: {output}"); + assert!( + output.contains("result: no newer runtime found"), + "{output}" + ); + fs::remove_dir_all(root).ok(); + Ok(()) + } + #[test] fn render_update_text_reports_all_update_surfaces() -> Result<()> { let (root, paths) = test_paths("update-surfaces"); @@ -448,7 +567,16 @@ mod tests { fs::remove_dir_all(root).ok(); assert!(rendered.contains("update_surfaces:")); - assert!(rendered.contains("cli: installed=")); + // One token, taken from the same string `rocm version` prints. Which + // leading version a tag, `main`, or an unknown ref produces is the + // `format_cli_version_*` tests in `build.rs`. + let installed = crate::cli_version_number(); + assert!(!installed.contains(' '), "{installed}"); + assert!(crate::cli_version_string().starts_with(&installed)); + assert!( + rendered.contains(&format!("cli: installed={installed} status=not_configured")), + "{rendered}" + ); assert!(rendered.contains("status=not_configured")); assert!(rendered.contains("engines: status=package_managed")); assert!(rendered.contains("model_recipes: status=")); diff --git a/apps/rocm/tests/build_metadata.rs b/apps/rocm/tests/build_metadata.rs index 33c43052bd..da12e7eb46 100644 --- a/apps/rocm/tests/build_metadata.rs +++ b/apps/rocm/tests/build_metadata.rs @@ -2,8 +2,9 @@ // // SPDX-License-Identifier: MIT -// The included build-script module exposes its internal path calculation to -// this integration test; its build entry point is intentionally unused here. +// The included build-script module exposes its internal path calculation and +// `format_cli_version` (tag, branch, and unknown cases) to this integration +// test; its build entry point is intentionally unused here. #[allow(dead_code)] #[path = "../build.rs"] mod build_script; diff --git a/apps/rocm/tests/diagnose_model_sends_no_token.rs b/apps/rocm/tests/diagnose_model_sends_no_token.rs new file mode 100644 index 0000000000..d64fa89193 --- /dev/null +++ b/apps/rocm/tests/diagnose_model_sends_no_token.rs @@ -0,0 +1,220 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `rocm diagnose --model owner/repo:tag` reads a GGUF header from the hub and +//! says it sends no token. That is a promise that something does not happen, so +//! it is checked on the wire: with every Hugging Face token variable set in the +//! command's environment, the listing, the `HEAD` and the range requests that +//! reach a stand-in hub carry no `Authorization` and no `Cookie` header. + +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::thread; + +/// A small but complete GGUF header: the shape keys an estimate reads and one +/// tensor record, so the read ends in a sized estimate. 2 layers of 4 KV heads, +/// key and value width 64 each: 1024 cache elements, 2048 bytes at f16, per +/// token of context. +fn gguf_header() -> Vec { + fn string(out: &mut Vec, value: &str) { + out.extend_from_slice(&(value.len() as u64).to_le_bytes()); + out.extend_from_slice(value.as_bytes()); + } + fn uint(out: &mut Vec, key: &str, value: u32) { + string(out, key); + out.extend_from_slice(&4_u32.to_le_bytes()); + out.extend_from_slice(&value.to_le_bytes()); + } + let mut metadata = Vec::new(); + string(&mut metadata, "general.architecture"); + metadata.extend_from_slice(&8_u32.to_le_bytes()); + string(&mut metadata, "llama"); + for (key, value) in [ + ("llama.block_count", 2), + ("llama.embedding_length", 256), + ("llama.feed_forward_length", 512), + ("llama.attention.head_count", 4), + ("llama.attention.head_count_kv", 4), + ("llama.attention.key_length", 64), + ("llama.attention.value_length", 64), + ] { + uint(&mut metadata, key, value); + } + let mut bytes = b"GGUF".to_vec(); + bytes.extend_from_slice(&3_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u64.to_le_bytes()); // tensors + bytes.extend_from_slice(&8_u64.to_le_bytes()); // metadata entries + bytes.extend(metadata); + string(&mut bytes, "blk.0.attn_q.weight"); + bytes.extend_from_slice(&2_u32.to_le_bytes()); + bytes.extend_from_slice(&256_u64.to_le_bytes()); + bytes.extend_from_slice(&256_u64.to_le_bytes()); + bytes.extend_from_slice(&0_u32.to_le_bytes()); // f32 + bytes.extend_from_slice(&0_u64.to_le_bytes()); // offset + bytes +} + +/// The KV cache the header above implies at the default 4096 tokens, f16. +const EXPECTED_KV_CACHE_BYTES: u64 = 2 * 4 * (64 + 64) * 2 * 4096; + +/// What the file is said to weigh: larger than the header, so the reader asks +/// for a range and never the whole file. +const FILE_SIZE: u64 = 1 << 20; + +fn answer(mut stream: TcpStream, requests: &Mutex>, header: &[u8]) { + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).unwrap_or(0) == 0 { + return; + } + request.push(byte[0]); + } + let text = String::from_utf8_lossy(&request).into_owned(); + requests.lock().unwrap().push(text.clone()); + let mut words = text.split_whitespace(); + let method = words.next().unwrap_or(""); + let path = words.next().unwrap_or(""); + let range = text.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("range") + .then(|| value.trim().strip_prefix("bytes=").map(str::to_owned)) + .flatten() + }); + if path.starts_with("/api/models/o/r") { + // No size in the listing, so the file's size is asked for by `HEAD`. + let body = r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf"}]}"#; + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\ + Connection: close\r\n\r\n{body}", + body.len() + ); + } else if method == "HEAD" { + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {FILE_SIZE}\r\nConnection: close\r\n\r\n" + ); + } else if let Some((start, end)) = range.as_deref().and_then(|r| r.split_once('-')) { + let start: u64 = start.parse().unwrap_or(0); + let end: u64 = end.parse().unwrap_or(0).min(FILE_SIZE - 1); + let body: Vec = (start..=end) + .map(|at| { + usize::try_from(at) + .ok() + .and_then(|at| header.get(at)) + .copied() + .unwrap_or(0) + }) + .collect(); + let _ = write!( + stream, + "HTTP/1.1 206 Partial Content\r\nContent-Range: bytes {start}-{end}/{FILE_SIZE}\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(&body); + } else { + let _ = write!( + stream, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + } +} + +#[test] +fn a_hub_header_read_sends_no_token_even_when_one_is_set() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let hub = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = Arc::clone(&requests); + thread::spawn(move || { + let header = gguf_header(); + for stream in listener.incoming() { + let Ok(stream) = stream else { break }; + answer(stream, &seen, &header); + } + }); + + let home = tempfile::tempdir().unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_rocm")) + .args(["diagnose", "--model", "o/r:Q4_K_M", "--json"]) + .env("HF_ENDPOINT", &hub) + .env("HF_TOKEN", "hf_secret_token") + .env("HUGGING_FACE_HUB_TOKEN", "hf_secret_token") + .env("ROCM_CLI_HUGGINGFACE_TOKEN", "hf_secret_token") + .env("ROCM_CLI_CONFIG_DIR", home.path().join("config")) + .env("ROCM_CLI_DATA_DIR", home.path().join("data")) + .env("ROCM_CLI_CACHE_DIR", home.path().join("cache")) + .stdin(Stdio::null()) + .output() + .expect("run rocm diagnose --model"); + + let requests = requests.lock().unwrap().clone(); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let first_line = |request: &String| request.lines().next().unwrap_or("").to_owned(); + let lines: Vec = requests.iter().map(first_line).collect(); + // Each kind of request reached the hub, so the check below is not vacuous. + for (kind, found) in [ + ( + "the listing", + lines.iter().any(|l| l.starts_with("GET /api/models/o/r")), + ), + ( + "the HEAD", + lines.iter().any(|l| l.starts_with("HEAD /o/r/resolve/")), + ), + ( + "a range request", + requests.iter().any(|request| { + request.starts_with("GET /o/r/resolve/") + && request.to_ascii_lowercase().contains("\r\nrange: bytes=") + }), + ), + ] { + assert!( + found, + "{kind} never reached the hub: {lines:?}\nstdout: {stdout}\nstderr: {stderr}" + ); + } + for request in &requests { + let lower = request.to_ascii_lowercase(); + assert!( + !lower.contains("\r\nauthorization:") && !lower.contains("\r\ncookie:"), + "a credential went to the hub:\n{request}" + ); + assert!(!request.contains("hf_secret_token"), "{request}"); + assert!( + request + .to_ascii_lowercase() + .contains("\r\nuser-agent: rocm-cli"), + "{request}" + ); + } + + // And the read is the estimate's: the answer is sized from that header. + let report: serde_json::Value = serde_json::from_str(&stdout) + .unwrap_or_else(|error| panic!("{error}\nstdout: {stdout}\nstderr: {stderr}")); + let estimate = &report["model"]["estimate"]; + assert_eq!( + estimate["source"].as_str(), + Some("o/r: m-Q4_K_M.gguf"), + "{report:#}" + ); + assert_eq!( + estimate["architecture"].as_str(), + Some("llama"), + "{report:#}" + ); + assert_eq!(estimate["layers"].as_u64(), Some(2), "{report:#}"); + assert_eq!( + estimate["kv_cache_bytes"].as_u64(), + Some(EXPECTED_KV_CACHE_BYTES), + "{report:#}" + ); + assert!(estimate["total_bytes"].as_u64().is_some(), "{report:#}"); +} diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..70587d7456 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -15,6 +15,7 @@ e2e-test-hooks = ["rocm-core/e2e-test-hooks"] [dependencies] anyhow.workspace = true +async-stream.workspace = true axum.workspace = true clap.workspace = true rocm-core = { path = "../../crates/rocm-core" } @@ -25,5 +26,10 @@ sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } +[dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test restores in a re-executed child. +libc.workspace = true diff --git a/apps/rocmd/src/anthropic_proxy.rs b/apps/rocmd/src/anthropic_proxy.rs new file mode 100644 index 0000000000..76486a9153 --- /dev/null +++ b/apps/rocmd/src/anthropic_proxy.rs @@ -0,0 +1,2294 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Anthropic Messages front for a served model. +//! +//! The engines speak the OpenAI API only. This module gives a managed service an +//! Anthropic-compatible `POST /v1/messages` by translating each request to the +//! engine's `/v1/chat/completions` and the reply back. It runs inside the process +//! that already owns the engine (`rocm`'s hidden `__engine-serve-http` entry), +//! as a background thread, so it starts, stops and restarts with the service and +//! adds no process of its own to track. +//! +//! [`spawn_alongside_engine`] first waits for the engine to list its model, then +//! binds a port of its own and publishes that address in the engine state file +//! under the keys in `rocm_core::anthropic_endpoint`, which the service record +//! adopts. The proxy fronts the engine even when the engine has a `/v1/messages` +//! route of its own (vLLM, Lemonade): that route only accepts the served model's +//! name, and an Anthropic client sends a Claude one. +//! +//! The translation covers system prompts and text turns, sampling parameters, +//! stop sequences, usage and stop reasons, and tool use in both directions: +//! `tools` and `tool_choice` on the request, `tool_use` and `tool_result` blocks +//! in the conversation, and `tool_use` blocks with the `tool_use` stop reason in +//! the reply. A `stream: true` request is answered with Anthropic server-sent +//! events (`message_start` through `message_stop`) built from the engine's +//! streamed chat-completion chunks as they arrive; streaming a request that +//! defines `tools` is not translated yet. Streaming with tools, extended +//! thinking, server tools and other non-text content blocks (images) are +//! refused with an Anthropic `invalid_request_error` that says so, rather than +//! being dropped silently. The engine reports token usage only at the end of a +//! stream, so `message_start` carries `input_tokens: 0` and the real figures +//! arrive in `message_delta`. Fields that only annotate a request (`metadata`) +//! are ignored. + +use std::convert::Infallible; +use std::io::{BufRead, BufReader, Read}; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::time::Duration; + +use axum::body::{Body, Bytes}; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use rocm_core::{ + ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AnthropicEndpointMode, + format_http_base_url, http_get_with_auth, http_post_json_with_auth, merge_json_state_file, + openai_models_endpoint_has_model, +}; +use serde_json::{Value, json}; + +/// How long to wait for the engine to list its model before giving up on +/// publishing an Anthropic endpoint. Model loads can take many minutes. +const ENGINE_LISTING_TIMEOUT: Duration = Duration::from_mins(30); + +/// Budget for one proxied generation. +const UPSTREAM_REQUEST_TIMEOUT: Duration = Duration::from_mins(10); + +/// What the proxy needs to know about the engine it fronts. +#[derive(Debug, Clone)] +pub struct AnthropicProxyConfig { + /// Host the engine listens on, as given to `rocm serve`. The proxy binds the + /// same host, so it is never reachable from further away than the engine. + pub host: String, + pub engine_port: u16, + /// The model the engine was asked to serve. Every request is sent under the + /// name the engine actually serves it as — `served_model_id` in the state + /// file when the engine reported one, else this — whatever model the client + /// asked for: an Anthropic client names a Claude model, no local engine's. + pub served_model: String, + /// The engine state file the endpoint is published to. + pub state_path: PathBuf, + /// The service's endpoint key, used only for the proxy's own probes. Client + /// requests are never answered with it: a client brings its own. + pub probe_api_key: Option, +} + +impl AnthropicProxyConfig { + /// Where to reach the engine from this machine, even when it listens on a + /// wildcard address. + fn upstream_url(&self) -> String { + let host = match self.host.as_str() { + "0.0.0.0" => "127.0.0.1", + "::" => "::1", + other => other, + }; + format_http_base_url(host, self.engine_port) + } +} + +/// Start publishing an Anthropic-compatible endpoint for the engine described by +/// `config`, on a background thread that lives as long as the process. +pub fn spawn_alongside_engine(config: AnthropicProxyConfig) { + let _ = std::thread::Builder::new() + .name("anthropic-endpoint".to_owned()) + .spawn(move || { + if let Err(error) = run(&config) { + eprintln!("anthropic endpoint unavailable: {error:#}"); + } + }); +} + +fn run(config: &AnthropicProxyConfig) -> anyhow::Result<()> { + let upstream = config.upstream_url(); + wait_for_engine_listing(config, &upstream)?; + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build()?; + runtime.block_on(async { + let listener = tokio::net::TcpListener::bind((config.host.as_str(), 0)).await?; + let address: SocketAddr = listener.local_addr()?; + // Published only once the socket is bound, so a reader never sees a URL + // nothing is listening on. + publish( + config, + &format_http_base_url(&config.host, address.port()), + AnthropicEndpointMode::Proxy, + )?; + axum::serve(listener, router(ProxyState::from_config(config))).await?; + Ok(()) + }) +} + +fn wait_for_engine_listing(config: &AnthropicProxyConfig, upstream: &str) -> anyhow::Result<()> { + let started = std::time::Instant::now(); + while started.elapsed() < ENGINE_LISTING_TIMEOUT { + if openai_models_endpoint_has_model( + upstream, + // Any listed model counts: a container engine lists its own name, + // not the reference it was asked to serve. + None, + config.probe_api_key.as_deref(), + Duration::from_secs(3), + ) + .unwrap_or(false) + { + return Ok(()); + } + std::thread::sleep(Duration::from_secs(2)); + } + anyhow::bail!("the engine did not list a model in time") +} + +fn publish( + config: &AnthropicProxyConfig, + url: &str, + mode: AnthropicEndpointMode, +) -> anyhow::Result<()> { + // The listener URL carries no `/v1`: Anthropic SDKs append `/v1/messages`. + let base = url.trim_end_matches('/').trim_end_matches("/v1"); + merge_json_state_file( + &config.state_path, + &json!({ + ANTHROPIC_ENDPOINT_URL_STATE_KEY: base, + ANTHROPIC_ENDPOINT_MODE_STATE_KEY: mode.as_str(), + }), + ) +} + +#[derive(Clone)] +struct ProxyState { + upstream: String, + served_model: String, +} + +impl ProxyState { + fn from_config(config: &AnthropicProxyConfig) -> Self { + let reported = std::fs::read(&config.state_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(|state| { + state + .get("served_model_id") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .filter(|name| !name.trim().is_empty()); + Self { + upstream: config.upstream_url(), + served_model: reported.unwrap_or_else(|| config.served_model.clone()), + } + } +} + +fn router(state: ProxyState) -> Router { + Router::new() + .route("/v1/messages", post(messages)) + .route("/v1/models", get(models)) + .with_state(state) +} + +/// What went wrong, in the terms of the Anthropic error envelope. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProxyError { + status: u16, + kind: &'static str, + message: String, +} + +impl ProxyError { + fn invalid(message: impl Into) -> Self { + Self { + status: 400, + kind: "invalid_request_error", + message: message.into(), + } + } + + fn unsupported(what: &str) -> Self { + Self::invalid(format!( + "{what} is not supported by the rocm Anthropic-compatible endpoint yet" + )) + } + + /// An error the engine returned, relabelled with Anthropic's error types. + fn from_upstream(status: u16, body: &str) -> Self { + let message = serde_json::from_str::(body) + .ok() + .and_then(|value| { + // OpenAI-style `{"error": {"message": ...}}` or `{"error": "..."}`, + // and FastAPI-style `{"detail": "..."}`. + let error = value.get("error").or_else(|| value.get("detail"))?; + error + .get("message") + .or(Some(error)) + .and_then(Value::as_str) + .map(str::to_owned) + }) + .unwrap_or_else(|| format!("the engine answered HTTP {status}")); + let kind = match status { + 400 | 422 => "invalid_request_error", + 401 => "authentication_error", + 403 => "permission_error", + 404 => "not_found_error", + 413 => "request_too_large", + 429 => "rate_limit_error", + _ => "api_error", + }; + Self { + status, + kind, + message, + } + } + + fn task_failed(error: &tokio::task::JoinError) -> Self { + Self { + status: 500, + kind: "api_error", + message: format!("the proxy task failed: {error}"), + } + } + + fn unreachable(error: &anyhow::Error) -> Self { + let timed_out = error.chain().any(|cause| { + cause.downcast_ref::().is_some_and(|io| { + matches!( + io.kind(), + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock + ) + }) + }); + if timed_out { + return Self { + status: 504, + kind: "api_error", + message: format!("the engine did not answer in time: {error:#}"), + }; + } + Self { + status: 502, + kind: "api_error", + message: format!("the engine could not be reached: {error:#}"), + } + } +} + +impl IntoResponse for ProxyError { + fn into_response(self) -> Response { + let body = json!({ + "type": "error", + "error": { "type": self.kind, "message": self.message }, + }); + ( + StatusCode::from_u16(self.status).unwrap_or(StatusCode::BAD_GATEWAY), + Json(body), + ) + .into_response() + } +} + +/// The key a client presented, in either of the two places Anthropic clients put +/// it, as the bearer token the engine expects. +fn client_api_key(headers: &HeaderMap) -> Option { + let header = |name: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + }; + header("x-api-key") + .or_else(|| { + header("authorization").and_then(|value| match value.split_once(' ') { + Some((scheme, token)) if scheme.eq_ignore_ascii_case("bearer") => Some(token), + // Another scheme (`Basic ...`) carries no key the engine accepts. + Some(_) => None, + None => Some(value), + }) + }) + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_owned) +} + +async fn messages(State(state): State, headers: HeaderMap, body: Bytes) -> Response { + let key = client_api_key(&headers); + let upstream_request = match parse_request(&state, &body) { + Ok(request) => request, + Err(error) => return error.into_response(), + }; + if upstream_request.get("stream").and_then(Value::as_bool) == Some(true) { + return stream_messages(state, key, upstream_request).await; + } + let result = tokio::task::spawn_blocking(move || { + forward_messages(&state, key.as_deref(), &upstream_request) + }) + .await + .unwrap_or_else(|error| Err(ProxyError::task_failed(&error))); + match result { + Ok(message) => Json(message).into_response(), + Err(error) => error.into_response(), + } +} + +/// The chat-completions request an Anthropic request body translates to. +fn parse_request(state: &ProxyState, body: &[u8]) -> Result { + let request: Value = serde_json::from_slice(body) + .map_err(|error| ProxyError::invalid(format!("the request body is not JSON: {error}")))?; + anthropic_to_openai(&request, &state.served_model) +} + +fn forward_messages( + state: &ProxyState, + key: Option<&str>, + upstream_request: &Value, +) -> Result { + let response = http_post_json_with_auth( + &state.upstream, + "/v1/chat/completions", + upstream_request, + key, + UPSTREAM_REQUEST_TIMEOUT, + ) + .map_err(|error| ProxyError::unreachable(&error))?; + if response.status != 200 { + return Err(ProxyError::from_upstream(response.status, &response.body)); + } + let reply: Value = serde_json::from_str(response.body.trim()).map_err(|error| ProxyError { + status: 502, + kind: "api_error", + message: format!("the engine's reply was not JSON: {error}"), + })?; + openai_to_anthropic(&reply, &state.served_model) +} + +/// Answer a streaming request. +/// +/// The engine is contacted before any response byte is sent, so an engine that +/// refuses the request (a bad key, an unknown model) still gets a plain Anthropic +/// error with its real status. Once the engine has accepted it the answer is a +/// `text/event-stream`, fed by a blocking task that translates the engine's +/// chunks as they arrive; a failure after that point can only be reported as an +/// in-stream `error` event. +async fn stream_messages(state: ProxyState, key: Option, request: Value) -> Response { + let served_model = state.served_model.clone(); + let opened = tokio::task::spawn_blocking(move || { + open_upstream_stream(&state.upstream, key.as_deref(), &request) + }) + .await + .unwrap_or_else(|error| Err(ProxyError::task_failed(&error))); + let upstream = match opened { + Ok(upstream) => upstream, + Err(error) => return error.into_response(), + }; + let (sender, mut receiver) = tokio::sync::mpsc::channel::(32); + tokio::task::spawn_blocking(move || pump_stream(upstream, &served_model, &sender)); + let events = async_stream::stream! { + while let Some(chunk) = receiver.recv().await { + yield Ok::(chunk); + } + }; + let mut response = Response::new(Body::from_stream(events)); + let headers = response.headers_mut(); + headers.insert("content-type", "text/event-stream".parse().expect("static")); + headers.insert("cache-control", "no-cache".parse().expect("static")); + response +} + +/// POST `request` to the engine and hand back its (streaming) response body. +fn open_upstream_stream( + upstream: &str, + key: Option<&str>, + request: &Value, +) -> Result, ProxyError> { + // The read timeout applies to each read, not to the whole generation: a + // stream may run for as long as the engine keeps producing chunks. + let agent = ureq::AgentBuilder::new() + .timeout_connect(Duration::from_secs(10)) + .timeout_read(UPSTREAM_REQUEST_TIMEOUT) + .build(); + let mut upstream_request = agent + .post(&format!("{upstream}/v1/chat/completions")) + .set("Content-Type", "application/json") + .set("Accept", "text/event-stream"); + if let Some(key) = key { + upstream_request = upstream_request.set("Authorization", &format!("Bearer {key}")); + } + match upstream_request.send_string(&request.to_string()) { + Ok(response) => Ok(Box::new(response.into_reader())), + Err(ureq::Error::Status(status, response)) => Err(ProxyError::from_upstream( + status, + &response.into_string().unwrap_or_default(), + )), + Err(error) => Err(ProxyError::unreachable(&anyhow::Error::new(error))), + } +} + +/// Read the engine's server-sent events and send their Anthropic translation +/// through `sender`, until the engine finishes or the client goes away (which +/// drops `upstream` and with it the engine connection). +fn pump_stream( + upstream: Box, + served_model: &str, + sender: &tokio::sync::mpsc::Sender, +) { + let mut translator = StreamTranslator::new(served_model); + let send = |events: Vec| { + events + .into_iter() + .all(|event| sender.blocking_send(Bytes::from(event)).is_ok()) + }; + for line in BufReader::new(upstream).lines() { + // Events are only sent when a chunk carries text, so a client that left + // during a long reasoning phase would otherwise go unnoticed. + if sender.is_closed() { + return; + } + let events = match line { + Ok(line) => match line.strip_prefix("data:") { + Some(data) => translator.feed(data.trim()), + // A 200 reply whose body is a plain JSON error rather than + // `data:` lines: surface the engine's message. + None if !translator.has_started() && line.trim_start().starts_with('{') => { + translator.feed(line.trim()) + } + None => continue, + }, + Err(error) => translator.fail(&format!("reading the engine's stream failed: {error}")), + }; + if !send(events) || translator.is_finished() { + return; + } + } + send(translator.end_of_stream()); +} + +async fn models(State(state): State, headers: HeaderMap) -> Response { + let key = client_api_key(&headers); + let result = tokio::task::spawn_blocking(move || { + http_get_with_auth( + &state.upstream, + "/v1/models", + key.as_deref(), + Duration::from_secs(10), + ) + }) + .await; + match result { + Ok(Ok(response)) if response.status == 200 => ( + StatusCode::OK, + [("content-type", "application/json")], + response.body, + ) + .into_response(), + Ok(Ok(response)) => { + ProxyError::from_upstream(response.status, &response.body).into_response() + } + Ok(Err(error)) => ProxyError::unreachable(&error).into_response(), + Err(error) => ProxyError::task_failed(&error).into_response(), + } +} + +/// The text of an Anthropic `system` or message `content` value. +/// +/// A string is itself; an array must hold only `text` blocks. +fn text_of(content: &Value, field: &str) -> Result { + match content { + Value::String(text) => Ok(text.clone()), + Value::Array(blocks) => { + let mut parts = Vec::with_capacity(blocks.len()); + for block in blocks { + match block.get("type").and_then(Value::as_str) { + Some("text") => parts.push( + block + .get("text") + .and_then(Value::as_str) + .ok_or_else(|| { + ProxyError::invalid(format!( + "a `text` content block in `{field}` needs a string `text`" + )) + })? + .to_owned(), + ), + Some(other) => { + return Err(ProxyError::unsupported(&format!( + "a `{other}` content block in `{field}`" + ))); + } + None => { + return Err(ProxyError::invalid(format!( + "a content block in `{field}` has no `type`" + ))); + } + } + } + Ok(parts.join("\n\n")) + } + _ => Err(ProxyError::invalid(format!( + "`{field}` must be a string or an array of content blocks" + ))), + } +} + +/// The text of a `tool_result` block's `content`: a string, or `text` blocks. +fn tool_result_text(block: &Value) -> Result { + let text = match block.get("content") { + None | Some(Value::Null) => String::new(), + Some(content) => text_of(content, "tool_result content")?, + }; + Ok( + if block.get("is_error").and_then(Value::as_bool) == Some(true) { + // OpenAI's tool message has no error flag; say it in the text instead. + format!("Error: {text}") + } else { + text + }, + ) +} + +/// Append one Anthropic turn to `messages` as OpenAI chat messages. +/// +/// A string, or an array of `text` blocks, is one message. An assistant +/// `tool_use` block becomes a `tool_calls` entry on that message. A user +/// `tool_result` block becomes a `tool` message of its own, placed before the +/// turn's text: OpenAI wants tool results to follow the assistant's calls +/// directly, and Anthropic requires them first in the user turn. +fn push_turn(messages: &mut Vec, role: &str, content: &Value) -> Result<(), ProxyError> { + let Value::Array(blocks) = content else { + messages.push(json!({ "role": role, "content": text_of(content, "messages")? })); + return Ok(()); + }; + let mut text = Vec::new(); + let mut calls = Vec::new(); + let mut results = Vec::new(); + for block in blocks { + let kind = block.get("type").and_then(Value::as_str); + match (kind, role) { + (Some("text"), _) => text.push(block.clone()), + (Some("tool_use"), "assistant") => { + let field = |name: &str| { + block + .get(name) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + ProxyError::invalid(format!( + "a `tool_use` content block needs a string `{name}`" + )) + }) + }; + let input = block.get("input").cloned().unwrap_or_else(|| json!({})); + calls.push(json!({ + "id": field("id")?, + "type": "function", + "function": { "name": field("name")?, "arguments": input.to_string() }, + })); + } + (Some("tool_result"), "user") => { + let id = block + .get("tool_use_id") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + ProxyError::invalid("a `tool_result` content block needs a `tool_use_id`") + })?; + results.push(json!({ + "role": "tool", + "tool_call_id": id, + "content": tool_result_text(block)?, + })); + } + (Some(kind @ ("tool_use" | "tool_result")), _) => { + let allowed = if kind == "tool_use" { + "assistant" + } else { + "user" + }; + return Err(ProxyError::invalid(format!( + "messages: a `{kind}` content block belongs in a {allowed} message, not {role}" + ))); + } + (Some(other), _) => { + return Err(ProxyError::unsupported(&format!( + "a `{other}` content block in `messages`" + ))); + } + (None, _) => { + return Err(ProxyError::invalid( + "a content block in `messages` has no `type`", + )); + } + } + } + messages.extend(results); + let text = text_of(&Value::Array(text), "messages")?; + if calls.is_empty() { + if !text.is_empty() || blocks.is_empty() { + messages.push(json!({ "role": role, "content": text })); + } + } else { + let content = if text.is_empty() { + Value::Null + } else { + json!(text) + }; + messages.push(json!({ "role": role, "content": content, "tool_calls": calls })); + } + Ok(()) +} + +/// Anthropic `tools` as OpenAI function tools; `None` when none are given. +/// +/// Server tools (web search, code execution, ...) run on Anthropic's side and +/// have no engine counterpart, so a tool with a `type` other than `custom` is +/// refused. +fn translate_tools(tools: Option<&Value>) -> Result, ProxyError> { + let Some(tools) = tools.filter(|value| !value.is_null()) else { + return Ok(None); + }; + let tools = tools + .as_array() + .ok_or_else(|| ProxyError::invalid("tools: must be an array"))?; + if tools.is_empty() { + return Ok(None); + } + let mut out = Vec::with_capacity(tools.len()); + for tool in tools { + if let Some(kind) = tool + .get("type") + .and_then(Value::as_str) + .filter(|kind| *kind != "custom") + { + return Err(ProxyError::unsupported(&format!( + "a tool of type `{kind}` (only `custom` function tools are accepted)" + ))); + } + let name = tool + .get("name") + .and_then(Value::as_str) + .filter(|name| !name.is_empty()) + .ok_or_else(|| ProxyError::invalid("tools: each tool needs a `name`"))?; + let mut function = json!({ + "name": name, + "parameters": tool + .get("input_schema") + .cloned() + .unwrap_or_else(|| json!({"type": "object", "properties": {}})), + }); + if let Some(description) = tool.get("description").filter(|value| value.is_string()) { + function["description"] = description.clone(); + } + out.push(json!({ "type": "function", "function": function })); + } + Ok(Some(Value::Array(out))) +} + +/// Anthropic `tool_choice` as OpenAI's, plus whether parallel calls stay allowed. +fn translate_tool_choice(choice: &Value) -> Result<(Value, bool), ProxyError> { + let parallel = choice + .get("disable_parallel_tool_use") + .and_then(Value::as_bool) + != Some(true); + let translated = match choice.get("type").and_then(Value::as_str) { + Some("auto") => json!("auto"), + Some("any") => json!("required"), + Some("none") => json!("none"), + Some("tool") => { + let name = choice + .get("name") + .and_then(Value::as_str) + .filter(|name| !name.is_empty()) + .ok_or_else(|| ProxyError::invalid("tool_choice: `tool` needs a `name`"))?; + json!({ "type": "function", "function": { "name": name } }) + } + _ => { + return Err(ProxyError::invalid( + "tool_choice: `type` must be auto, any, tool or none", + )); + } + }; + Ok((translated, parallel)) +} + +/// The engine's `tool_calls` as Anthropic `tool_use` blocks. +fn tool_use_blocks(calls: &[Value]) -> Result, ProxyError> { + let bad_gateway = |message: String| ProxyError { + status: 502, + kind: "api_error", + message, + }; + calls + .iter() + .enumerate() + .map(|(index, call)| { + let name = call + .pointer("/function/name") + .and_then(Value::as_str) + .filter(|name| !name.is_empty()) + .ok_or_else(|| bad_gateway("the engine's tool call had no name".to_owned()))?; + let input = match call.pointer("/function/arguments") { + None | Some(Value::Null) => json!({}), + Some(Value::String(raw)) if raw.trim().is_empty() => json!({}), + Some(Value::String(raw)) => serde_json::from_str::(raw) + .ok() + .filter(Value::is_object) + .ok_or_else(|| { + bad_gateway(format!( + "the model's arguments for tool `{name}` were not a JSON object: {raw}" + )) + })?, + Some(object @ Value::Object(_)) => object.clone(), + Some(_) => { + return Err(bad_gateway(format!( + "the engine's arguments for tool `{name}` were not a JSON object" + ))); + } + }; + let id = call + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map_or_else(|| format!("toolu_rocm_{index}"), str::to_owned); + Ok(json!({ "type": "tool_use", "id": id, "name": name, "input": input })) + }) + .collect() +} + +/// Translate an Anthropic Messages request into an OpenAI chat-completions one. +fn anthropic_to_openai(request: &Value, served_model: &str) -> Result { + let object = request + .as_object() + .ok_or_else(|| ProxyError::invalid("the request body must be a JSON object"))?; + let stream = object.get("stream").and_then(Value::as_bool) == Some(true); + if object + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + .is_some_and(|kind| kind != "disabled") + { + return Err(ProxyError::unsupported("extended thinking (`thinking`)")); + } + let max_tokens = object + .get("max_tokens") + .and_then(Value::as_u64) + .filter(|value| *value > 0) + .ok_or_else(|| ProxyError::invalid("max_tokens: a positive integer is required"))?; + let turns = object + .get("messages") + .and_then(Value::as_array) + .filter(|turns| !turns.is_empty()) + .ok_or_else(|| ProxyError::invalid("messages: at least one message is required"))?; + + let mut messages = Vec::with_capacity(turns.len() + 1); + if let Some(system) = object.get("system").filter(|value| !value.is_null()) { + let text = text_of(system, "system")?; + if !text.is_empty() { + messages.push(json!({ "role": "system", "content": text })); + } + } + for turn in turns { + let role = turn + .get("role") + .and_then(Value::as_str) + .filter(|role| matches!(*role, "user" | "assistant")) + .ok_or_else(|| { + ProxyError::invalid("messages: each message needs a `role` of user or assistant") + })?; + let content = turn + .get("content") + .ok_or_else(|| ProxyError::invalid("messages: each message needs `content`"))?; + push_turn(&mut messages, role, content)?; + } + + let tools = translate_tools(object.get("tools"))?; + if stream && tools.is_some() { + // The stream translator only emits text blocks; tool-call deltas would + // be dropped silently. + return Err(ProxyError::unsupported("streaming a request with `tools`")); + } + let mut out = json!({ + "model": served_model, + "messages": messages, + "max_tokens": max_tokens, + "stream": stream, + }); + if stream { + // Without this the engine omits token counts from a stream, and the + // closing `message_delta` would report zero output tokens. + out["stream_options"] = json!({ "include_usage": true }); + } + for key in ["temperature", "top_p", "top_k"] { + if let Some(value) = object.get(key).filter(|value| value.is_number()) { + out[key] = value.clone(); + } + } + if let Some(tools) = tools { + out["tools"] = tools; + if let Some(choice) = object.get("tool_choice").filter(|value| !value.is_null()) { + let (choice, parallel) = translate_tool_choice(choice)?; + out["tool_choice"] = choice; + if !parallel { + out["parallel_tool_calls"] = json!(false); + } + } + } else if object + .get("tool_choice") + .is_some_and(|choice| !choice.is_null()) + { + return Err(ProxyError::invalid( + "tool_choice: `tools` must list at least one tool to choose from", + )); + } + if let Some(stops) = object.get("stop_sequences").and_then(Value::as_array) + && !stops.is_empty() + { + out["stop"] = Value::Array(stops.clone()); + } + Ok(out) +} + +/// Translate an OpenAI chat-completions reply into an Anthropic Message. +fn openai_to_anthropic(reply: &Value, served_model: &str) -> Result { + let choice = reply + .get("choices") + .and_then(Value::as_array) + .and_then(|choices| choices.first()) + .ok_or_else(|| ProxyError { + status: 502, + kind: "api_error", + message: "the engine's reply had no choices".to_owned(), + })?; + let text = choice + .pointer("/message/content") + .and_then(Value::as_str) + .unwrap_or_default(); + let calls = choice + .pointer("/message/tool_calls") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + let truncated = choice.get("finish_reason").and_then(Value::as_str) == Some("length"); + let mut tool_uses = tool_use_blocks(calls); + if truncated && tool_uses.is_err() { + // The token limit cut the arguments off mid-object; keep the text and + // report `max_tokens` rather than failing the whole reply. + tool_uses = Ok(Vec::new()); + } + let tool_uses = tool_uses?; + // Some engines report `stop` beside tool calls, so the calls decide. + let stop_reason = if !tool_uses.is_empty() { + "tool_use" + } else if truncated { + "max_tokens" + } else { + "end_turn" + }; + let mut content = Vec::new(); + if !text.is_empty() || tool_uses.is_empty() { + content.push(json!({ "type": "text", "text": text })); + } + content.extend(tool_uses); + let id = reply + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map_or_else( + || "msg_rocm".to_owned(), + |id| format!("msg_{}", id.trim_start_matches("chatcmpl-")), + ); + let token_count = |key: &str| { + reply + .pointer(&format!("/usage/{key}")) + .and_then(Value::as_u64) + .unwrap_or(0) + }; + Ok(json!({ + "id": id, + "type": "message", + "role": "assistant", + "model": reply + .get("model") + .and_then(Value::as_str) + .filter(|model| !model.is_empty()) + .unwrap_or(served_model), + "content": content, + "stop_reason": stop_reason, + "stop_sequence": null, + "usage": { + "input_tokens": token_count("prompt_tokens"), + "output_tokens": token_count("completion_tokens"), + }, + })) +} + +/// One Anthropic server-sent event. +fn sse_event(event: &str, data: &Value) -> String { + format!("event: {event}\ndata: {data}\n\n") +} + +/// Turns an engine's streamed chat-completion chunks into the Anthropic +/// streaming event sequence: `message_start`, one text `content_block_start`, +/// a `content_block_delta` per text fragment, then `content_block_stop`, +/// `message_delta` and `message_stop` once the engine is done. +/// +/// The closing events wait for the engine's `[DONE]` (or end of stream) rather +/// than its `finish_reason` chunk, because the usage totals arrive in a chunk +/// after it. +struct StreamTranslator { + served_model: String, + started: bool, + block_open: bool, + finished: bool, + stop_reason: Option<&'static str>, + input_tokens: u64, + output_tokens: u64, +} + +impl StreamTranslator { + fn new(served_model: &str) -> Self { + Self { + served_model: served_model.to_owned(), + started: false, + block_open: false, + finished: false, + stop_reason: None, + input_tokens: 0, + output_tokens: 0, + } + } + + const fn has_started(&self) -> bool { + self.started + } + + /// Whether the closing event (`message_stop` or `error`) has been produced. + const fn is_finished(&self) -> bool { + self.finished + } + + /// Translate the payload of one engine `data:` line. + fn feed(&mut self, data: &str) -> Vec { + if self.finished || data.is_empty() { + return Vec::new(); + } + if data == "[DONE]" { + // `[DONE]` with no stop reason seen is an early end, like a cut stream. + return self.end_of_stream(); + } + let Ok(chunk) = serde_json::from_str::(data) else { + return self.fail("the engine sent a stream chunk that was not JSON"); + }; + if let Some(error) = chunk.get("error") { + let message = error + .get("message") + .or(Some(error)) + .and_then(Value::as_str) + .unwrap_or("the engine reported an error"); + return self.fail(message); + } + let mut events = Vec::new(); + if !self.started { + let id = chunk + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map_or_else( + || "msg_rocm".to_owned(), + |id| format!("msg_{}", id.trim_start_matches("chatcmpl-")), + ); + let model = chunk + .get("model") + .and_then(Value::as_str) + .filter(|model| !model.is_empty()) + .unwrap_or(&self.served_model) + .to_owned(); + events.push(self.start(&id, &model)); + } + let token_count = |key: &str| { + chunk + .pointer(&format!("/usage/{key}")) + .and_then(Value::as_u64) + }; + self.input_tokens = token_count("prompt_tokens").unwrap_or(self.input_tokens); + self.output_tokens = token_count("completion_tokens").unwrap_or(self.output_tokens); + if let Some(choice) = chunk + .get("choices") + .and_then(Value::as_array) + .and_then(|choices| choices.first()) + { + if let Some(text) = choice + .pointer("/delta/content") + .and_then(Value::as_str) + .filter(|text| !text.is_empty()) + { + events.extend(self.open_block()); + events.push(sse_event( + "content_block_delta", + &json!({ + "type": "content_block_delta", + "index": 0, + "delta": { "type": "text_delta", "text": text }, + }), + )); + } + match choice.get("finish_reason").and_then(Value::as_str) { + Some("length") => self.stop_reason = Some("max_tokens"), + Some(_) => self.stop_reason = Some("end_turn"), + None => {} + } + } + events + } + + /// The events for an engine stream that is over: complete if a stop reason + /// arrived, an `error` otherwise. + fn end_of_stream(&mut self) -> Vec { + if self.finished { + return Vec::new(); + } + if self.stop_reason.is_some() { + return self.complete(); + } + self.fail("the engine's stream ended before the reply was complete") + } + + /// End the stream with an Anthropic `error` event. + fn fail(&mut self, message: &str) -> Vec { + if self.finished { + return Vec::new(); + } + self.finished = true; + vec![sse_event( + "error", + &json!({ + "type": "error", + "error": { "type": "api_error", "message": message }, + }), + )] + } + + fn start(&mut self, id: &str, model: &str) -> String { + self.started = true; + sse_event( + "message_start", + &json!({ + "type": "message_start", + "message": { + "id": id, + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": null, + "stop_sequence": null, + "usage": { "input_tokens": 0, "output_tokens": 0 }, + }, + }), + ) + } + + fn open_block(&mut self) -> Option { + if self.block_open { + return None; + } + self.block_open = true; + Some(sse_event( + "content_block_start", + &json!({ + "type": "content_block_start", + "index": 0, + "content_block": { "type": "text", "text": "" }, + }), + )) + } + + /// The closing events. An engine that produced no text still gets an empty + /// text block, matching the non-streaming reply. + fn complete(&mut self) -> Vec { + let mut events = Vec::new(); + if !self.started { + let model = self.served_model.clone(); + events.push(self.start("msg_rocm", &model)); + } + events.extend(self.open_block()); + events.push(sse_event( + "content_block_stop", + &json!({ "type": "content_block_stop", "index": 0 }), + )); + events.push(sse_event( + "message_delta", + &json!({ + "type": "message_delta", + "delta": { + "stop_reason": self.stop_reason.unwrap_or("end_turn"), + "stop_sequence": null, + }, + "usage": { + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + }, + }), + )); + events.push(sse_event( + "message_stop", + &json!({ "type": "message_stop" }), + )); + self.finished = true; + events + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::{Arc, Mutex}; + + const MODEL: &str = "Org/Served-Model"; + + /// A directory no other test in this process shares. Process id plus a + /// millisecond clock is not enough: tests run in parallel threads and two can + /// start in the same millisecond, then overwrite each other's state file. + fn unique_temp_dir(label: &str) -> std::path::PathBuf { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + std::env::temp_dir().join(format!( + "rocmd-anthropic-{label}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )) + } + + fn translate(request: &Value) -> Result { + anthropic_to_openai(request, MODEL) + } + + #[test] + fn request_maps_system_turns_and_sampling_to_chat_completions() { + let out = translate(&json!({ + "model": "claude-sonnet-4", + "max_tokens": 64, + "system": [{"type": "text", "text": "Be brief."}, {"type": "text", "text": "Be kind."}], + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": "hello"}]}, + {"role": "user", "content": "again"}, + ], + "temperature": 0.2, + "top_p": 0.9, + "top_k": 40, + "stop_sequences": ["END"], + })) + .unwrap(); + assert_eq!( + out, + json!({ + "model": MODEL, + "messages": [ + {"role": "system", "content": "Be brief.\n\nBe kind."}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "again"}, + ], + "max_tokens": 64, + "stream": false, + "temperature": 0.2, + "top_p": 0.9, + "top_k": 40, + "stop": ["END"], + }) + ); + } + + #[test] + fn a_streaming_request_asks_the_engine_to_stream_with_usage() { + let out = translate(&json!({ + "max_tokens": 8, + "stream": true, + "messages": [{"role": "user", "content": "x"}], + })) + .unwrap(); + assert_eq!(out["stream"], true); + assert_eq!(out["stream_options"], json!({"include_usage": true})); + let out = translate(&json!({ + "max_tokens": 8, + "stream": false, + "messages": [{"role": "user", "content": "x"}], + })) + .unwrap(); + assert_eq!(out["stream"], false); + assert!(out.get("stream_options").is_none()); + } + + #[test] + fn request_always_names_the_served_model_not_the_clients() { + let out = translate(&json!({ + "model": "claude-opus-4", + "max_tokens": 1, + "messages": [{"role": "user", "content": "x"}], + })) + .unwrap(); + assert_eq!(out["model"], MODEL); + } + + #[test] + fn request_refuses_what_it_cannot_translate_instead_of_dropping_it() { + let base = || json!({"max_tokens": 8, "messages": [{"role": "user", "content": "x"}]}); + let mut streaming_tools = base(); + streaming_tools["stream"] = json!(true); + streaming_tools["tools"] = + json!([{"name": "get_weather", "input_schema": {"type": "object"}}]); + let mut server_tool = base(); + server_tool["tools"] = json!([{"type": "web_search_20250305", "name": "web_search"}]); + let image = json!({"max_tokens": 8, "messages": [{"role": "user", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}} + ]}]}); + for (request, needle) in [ + (streaming_tools, "streaming"), + (server_tool, "a tool of type `web_search_20250305`"), + (image, "`image` content block"), + ] { + let error = translate(&request).unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!(error.kind, "invalid_request_error"); + assert!(error.message.contains(needle), "{}", error.message); + } + } + + #[test] + fn request_rejects_malformed_input_with_invalid_request_errors() { + for request in [ + json!([]), + json!({"messages": [{"role": "user", "content": "x"}]}), + json!({"max_tokens": 0, "messages": [{"role": "user", "content": "x"}]}), + json!({"max_tokens": 8, "messages": []}), + json!({"max_tokens": 8}), + json!({"max_tokens": 8, "messages": [{"role": "system", "content": "x"}]}), + json!({"max_tokens": 8, "messages": [{"role": "user"}]}), + ] { + let error = translate(&request).unwrap_err(); + assert_eq!(error.kind, "invalid_request_error", "{request}"); + } + } + + #[test] + fn response_becomes_a_valid_anthropic_message() { + let message = openai_to_anthropic( + &json!({ + "id": "chatcmpl-abc123", + "model": MODEL, + "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 11, "completion_tokens": 2}, + }), + "fallback", + ) + .unwrap(); + assert_eq!( + message, + json!({ + "id": "msg_abc123", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 11, "output_tokens": 2}, + }) + ); + } + + #[test] + fn response_keeps_the_text_when_a_length_stop_truncates_tool_arguments() { + let reply = |finish: &str| { + json!({"choices": [{"message": {"content": "Checking.", "tool_calls": [ + {"id": "call_1", "function": {"name": "weather", "arguments": "{\"city\":\"Os"}} + ]}, "finish_reason": finish}]}) + }; + let message = openai_to_anthropic(&reply("length"), MODEL).unwrap(); + assert_eq!(message["stop_reason"], "max_tokens"); + assert_eq!( + message["content"], + json!([{"type": "text", "text": "Checking."}]) + ); + // Without the token limit the same arguments are still a bad gateway. + assert_eq!( + openai_to_anthropic(&reply("tool_calls"), MODEL) + .unwrap_err() + .status, + 502 + ); + } + + #[test] + fn response_maps_a_length_stop_and_falls_back_to_the_served_model() { + let message = openai_to_anthropic( + &json!({"choices": [{"message": {"content": null}, "finish_reason": "length"}]}), + MODEL, + ) + .unwrap(); + assert_eq!(message["stop_reason"], "max_tokens"); + assert_eq!(message["model"], MODEL); + assert_eq!(message["content"][0]["text"], ""); + assert_eq!(message["usage"]["input_tokens"], 0); + } + + #[test] + fn response_without_choices_is_a_bad_gateway() { + let error = openai_to_anthropic(&json!({"choices": []}), MODEL).unwrap_err(); + assert_eq!(error.status, 502); + } + + #[test] + fn upstream_errors_keep_their_status_and_message_with_anthropic_types() { + let error = + ProxyError::from_upstream(404, r#"{"error": {"message": "model `x` does not exist"}}"#); + assert_eq!( + (error.status, error.kind, error.message.as_str()), + (404, "not_found_error", "model `x` does not exist") + ); + let error = ProxyError::from_upstream(401, "not json"); + assert_eq!(error.kind, "authentication_error"); + assert_eq!(error.message, "the engine answered HTTP 401"); + assert_eq!(ProxyError::from_upstream(500, "").kind, "api_error"); + } + + #[test] + fn client_key_is_read_from_either_header() { + let mut headers = HeaderMap::new(); + assert_eq!(client_api_key(&headers), None); + headers.insert("authorization", "Bearer from-bearer".parse().unwrap()); + assert_eq!(client_api_key(&headers).as_deref(), Some("from-bearer")); + headers.insert("x-api-key", "from-x-api-key".parse().unwrap()); + assert_eq!(client_api_key(&headers).as_deref(), Some("from-x-api-key")); + } + + #[test] + fn an_empty_x_api_key_falls_back_to_the_authorization_header() { + let mut headers = HeaderMap::new(); + headers.insert("x-api-key", " ".parse().unwrap()); + headers.insert("authorization", "bearer from-bearer".parse().unwrap()); + assert_eq!(client_api_key(&headers).as_deref(), Some("from-bearer")); + headers.insert("authorization", "Basic dXNlcjpwdw==".parse().unwrap()); + assert_eq!(client_api_key(&headers), None); + } + + #[test] + fn engine_errors_shaped_as_detail_keep_their_message() { + let error = ProxyError::from_upstream(404, r#"{"detail": "no such route"}"#); + assert_eq!(error.message, "no such route"); + } + + #[test] + fn a_timed_out_engine_is_a_gateway_timeout_not_unreachable() { + let timeout = anyhow::Error::new(std::io::Error::from(std::io::ErrorKind::TimedOut)) + .context("failed to read HTTP POST /v1/chat/completions"); + assert_eq!(ProxyError::unreachable(&timeout).status, 504); + let refused = + anyhow::Error::new(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)); + assert_eq!(ProxyError::unreachable(&refused).status, 502); + } + + #[test] + fn requests_that_would_silently_lose_meaning_are_refused() { + let base = json!({"max_tokens": 4, "messages": [{"role": "user", "content": "x"}]}); + let mut request = base.clone(); + request["thinking"] = json!({"type": "enabled", "budget_tokens": 1024}); + assert_eq!(translate(&request).unwrap_err().status, 400); + let mut disabled = base; + disabled["thinking"] = json!({"type": "disabled"}); + assert!(translate(&disabled).is_ok()); + let missing_text = json!({ + "max_tokens": 4, + "messages": [{"role": "user", "content": [{"type": "text"}]}], + }); + let error = translate(&missing_text).unwrap_err(); + assert_eq!(error.status, 400); + assert!(error.message.contains("string `text`"), "{}", error.message); + } + + fn weather_tool() -> Value { + json!({ + "name": "get_weather", + "description": "Current weather for a city.", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }) + } + + #[test] + fn tools_and_tool_choice_become_function_tools() { + let mut request = json!({ + "max_tokens": 8, + "messages": [{"role": "user", "content": "weather?"}], + "tools": [weather_tool()], + }); + let out = translate(&request).unwrap(); + assert_eq!( + out["tools"], + json!([{"type": "function", "function": { + "name": "get_weather", + "description": "Current weather for a city.", + "parameters": weather_tool()["input_schema"], + }}]) + ); + assert!(out.get("tool_choice").is_none()); + for (choice, expected) in [ + (json!({"type": "auto"}), json!("auto")), + (json!({"type": "any"}), json!("required")), + (json!({"type": "none"}), json!("none")), + ( + json!({"type": "tool", "name": "get_weather"}), + json!({"type": "function", "function": {"name": "get_weather"}}), + ), + ] { + request["tool_choice"] = choice; + let out = translate(&request).unwrap(); + assert_eq!(out["tool_choice"], expected); + assert!(out.get("parallel_tool_calls").is_none()); + } + request["tool_choice"] = json!({"type": "auto", "disable_parallel_tool_use": true}); + assert_eq!(translate(&request).unwrap()["parallel_tool_calls"], false); + } + + #[test] + fn malformed_tool_definitions_and_choices_are_invalid_requests() { + let base = json!({"max_tokens": 8, "messages": [{"role": "user", "content": "x"}]}); + for (tools, choice) in [ + (json!([{"input_schema": {}}]), json!(null)), + (json!("get_weather"), json!(null)), + (json!([weather_tool()]), json!({"type": "sometimes"})), + (json!([weather_tool()]), json!({"type": "tool"})), + // A choice with nothing to choose from. + (json!([]), json!({"type": "any"})), + ] { + let mut request = base.clone(); + request["tools"] = tools; + request["tool_choice"] = choice; + assert_eq!( + translate(&request).unwrap_err().kind, + "invalid_request_error" + ); + } + } + + #[test] + fn tool_use_and_tool_result_blocks_become_tool_calls_and_tool_messages() { + let out = translate(&json!({ + "max_tokens": 8, + "tools": [weather_tool()], + "messages": [ + {"role": "user", "content": "weather in Oslo?"}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Checking."}, + {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ]}, + {"role": "user", "content": [ + {"type": "text", "text": "thanks"}, + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [{"type": "text", "text": "3C"}]}, + ]}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_2", "name": "get_weather", "input": {}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_2", "content": "no data", "is_error": true}, + ]}, + ], + })) + .unwrap(); + assert_eq!( + out["messages"], + json!([ + {"role": "user", "content": "weather in Oslo?"}, + {"role": "assistant", "content": "Checking.", "tool_calls": [{ + "id": "toolu_1", "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\":\"Oslo\"}"}, + }]}, + {"role": "tool", "tool_call_id": "toolu_1", "content": "3C"}, + {"role": "user", "content": "thanks"}, + {"role": "assistant", "content": null, "tool_calls": [{ + "id": "toolu_2", "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + }]}, + {"role": "tool", "tool_call_id": "toolu_2", "content": "Error: no data"}, + ]) + ); + } + + #[test] + fn tool_blocks_in_the_wrong_role_or_missing_fields_are_invalid_requests() { + for (role, block) in [ + ( + "user", + json!({"type": "tool_use", "id": "a", "name": "n", "input": {}}), + ), + ( + "assistant", + json!({"type": "tool_result", "tool_use_id": "a", "content": "x"}), + ), + ( + "assistant", + json!({"type": "tool_use", "name": "n", "input": {}}), + ), + ( + "assistant", + json!({"type": "tool_use", "id": "a", "input": {}}), + ), + ("user", json!({"type": "tool_result", "content": "x"})), + ] { + let error = translate(&json!({ + "max_tokens": 8, + "messages": [{"role": role, "content": [block.clone()]}], + })) + .unwrap_err(); + assert_eq!(error.kind, "invalid_request_error", "{block}"); + } + } + + #[test] + fn tool_calls_in_the_reply_become_tool_use_blocks_with_the_tool_use_stop() { + let message = openai_to_anthropic( + &json!({ + "id": "chatcmpl-t1", + "choices": [{ + "message": {"content": "Let me check.", "tool_calls": [ + {"id": "call_1", "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\":\"Oslo\"}"}}, + {"type": "function", "function": {"name": "get_time", "arguments": ""}}, + ]}, + // Engines that report `stop` beside tool calls are still tool use. + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 5, "completion_tokens": 7}, + }), + MODEL, + ) + .unwrap(); + assert_eq!(message["stop_reason"], "tool_use"); + assert_eq!( + message["content"], + json!([ + {"type": "text", "text": "Let me check."}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + {"type": "tool_use", "id": "toolu_rocm_1", "name": "get_time", "input": {}}, + ]) + ); + } + + #[test] + fn a_reply_of_only_tool_calls_has_no_empty_text_block() { + let message = openai_to_anthropic( + &json!({"choices": [{ + "message": {"content": null, "tool_calls": [ + {"id": "c", "function": {"name": "n", "arguments": "{}"}}, + ]}, + "finish_reason": "tool_calls", + }]}), + MODEL, + ) + .unwrap(); + assert_eq!(message["content"].as_array().unwrap().len(), 1); + assert_eq!(message["content"][0]["type"], "tool_use"); + } + + #[test] + fn unusable_tool_calls_from_the_engine_are_a_bad_gateway() { + for call in [ + json!({"id": "c", "function": {"name": "n", "arguments": "{not json"}}), + json!({"id": "c", "function": {"name": "n", "arguments": "[1]"}}), + json!({"id": "c", "function": {"arguments": "{}"}}), + ] { + let error = openai_to_anthropic( + &json!({"choices": [{"message": {"tool_calls": [call]}, "finish_reason": "tool_calls"}]}), + MODEL, + ) + .unwrap_err(); + assert_eq!(error.status, 502); + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn tool_use_round_trips_through_the_engine() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let (status, message) = tokio::task::spawn_blocking(move || { + post( + &base, + "/v1/messages", + None, + &json!({ + "model": "claude-sonnet-4", + "max_tokens": 64, + "tools": [weather_tool()], + "tool_choice": {"type": "tool", "name": "get_weather"}, + "messages": [{"role": "user", "content": "weather in Oslo?"}], + }), + ) + }) + .await + .unwrap(); + assert_eq!(status, 200, "{message}"); + assert_eq!(message["stop_reason"], "tool_use"); + assert_eq!( + message["content"], + json!([{"type": "tool_use", "id": "call_9", "name": "get_weather", "input": {"city": "Oslo"}}]) + ); + let seen = engine.seen.lock().unwrap(); + let forwarded: Value = serde_json::from_str(&seen[0].1).unwrap(); + assert_eq!(forwarded["tools"][0]["function"]["name"], "get_weather"); + assert_eq!(forwarded["tool_choice"]["function"]["name"], "get_weather"); + } + + #[test] + fn wildcard_listen_hosts_are_reached_over_loopback() { + let config = |host: &str| AnthropicProxyConfig { + host: host.to_owned(), + engine_port: 8000, + served_model: MODEL.to_owned(), + state_path: PathBuf::new(), + probe_api_key: None, + }; + assert_eq!(config("0.0.0.0").upstream_url(), "http://127.0.0.1:8000"); + assert_eq!(config("::").upstream_url(), "http://[::1]:8000"); + assert_eq!(config("127.0.0.1").upstream_url(), "http://127.0.0.1:8000"); + } + + fn chunk(delta: &Value, finish: Option<&str>) -> String { + json!({ + "id": "chatcmpl-t1", + "model": MODEL, + "choices": [{"delta": delta, "finish_reason": finish}], + }) + .to_string() + } + + fn pump(body: &str) -> Vec { + let (sender, mut receiver) = tokio::sync::mpsc::channel(64); + pump_stream( + Box::new(std::io::Cursor::new(body.to_owned())), + MODEL, + &sender, + ); + drop(sender); + let mut out = Vec::new(); + while let Ok(bytes) = receiver.try_recv() { + out.push(String::from_utf8(bytes.to_vec()).unwrap()); + } + out + } + + #[test] + fn a_plain_json_error_body_on_a_200_reply_reaches_the_client() { + let events = pump(r#"{"error": {"message": "model is loading"}}"#); + assert_eq!(names(&events), ["error"]); + assert!(events[0].contains("model is loading"), "{}", events[0]); + } + + fn names(events: &[String]) -> Vec<&str> { + events + .iter() + .map(|event| { + event + .strip_prefix("event: ") + .and_then(|rest| rest.split('\n').next()) + .unwrap() + }) + .collect() + } + + #[test] + fn the_stream_opens_once_and_each_text_fragment_is_a_delta() { + let mut translator = StreamTranslator::new("fallback"); + let first = translator.feed(&chunk(&json!({"role": "assistant", "content": ""}), None)); + assert_eq!(names(&first), ["message_start"]); + assert!(first[0].contains(r#""id":"msg_t1""#), "{}", first[0]); + assert!(first[0].contains(MODEL), "{}", first[0]); + let second = translator.feed(&chunk(&json!({"content": "Hel"}), None)); + assert_eq!( + names(&second), + ["content_block_start", "content_block_delta"] + ); + let third = translator.feed(&chunk(&json!({"content": "lo"}), None)); + assert_eq!(names(&third), ["content_block_delta"]); + assert!(third[0].contains(r#""text":"lo""#), "{}", third[0]); + assert!(!translator.is_finished()); + } + + #[test] + fn usage_after_the_finish_chunk_still_reaches_message_delta() { + let mut translator = StreamTranslator::new(MODEL); + translator.feed(&chunk(&json!({"content": "x"}), None)); + assert!( + translator + .feed(&chunk(&json!({}), Some("length"))) + .is_empty() + ); + translator + .feed(r#"{"choices": [], "usage": {"prompt_tokens": 7, "completion_tokens": 9}}"#); + let closing = translator.feed("[DONE]"); + assert_eq!( + names(&closing), + ["content_block_stop", "message_delta", "message_stop"] + ); + assert!( + closing[1].contains(r#""stop_reason":"max_tokens""#), + "{}", + closing[1] + ); + assert!(closing[1].contains(r#""input_tokens":7"#), "{}", closing[1]); + assert!( + closing[1].contains(r#""output_tokens":9"#), + "{}", + closing[1] + ); + assert!(translator.is_finished()); + assert!(translator.feed("[DONE]").is_empty()); + } + + #[test] + fn a_reply_without_text_is_an_empty_text_block() { + let mut translator = StreamTranslator::new(MODEL); + let mut events = translator.feed(&chunk(&json!({}), Some("stop"))); + events.extend(translator.feed("[DONE]")); + assert_eq!( + names(&events), + [ + "message_start", + "content_block_start", + "content_block_stop", + "message_delta", + "message_stop", + ] + ); + assert!(events[0].contains(MODEL), "{}", events[0]); + } + + #[test] + fn done_without_a_stop_reason_is_an_error() { + let mut translator = StreamTranslator::new(MODEL); + let events = translator.feed("[DONE]"); + assert_eq!(names(&events), ["error"]); + assert!(events[0].contains("ended before"), "{}", events[0]); + } + + #[test] + fn a_stream_that_ends_early_is_an_error_not_a_complete_reply() { + let mut translator = StreamTranslator::new(MODEL); + translator.feed(&chunk(&json!({"content": "x"}), None)); + let events = translator.end_of_stream(); + assert_eq!(names(&events), ["error"]); + assert!(events[0].contains("ended before"), "{}", events[0]); + assert!(translator.is_finished()); + // Ending after the finish chunk, without `[DONE]`, is still complete. + let mut translator = StreamTranslator::new(MODEL); + translator.feed(&chunk(&json!({"content": "x"}), Some("stop"))); + assert_eq!( + names(&translator.end_of_stream()), + ["content_block_stop", "message_delta", "message_stop"] + ); + } + + #[test] + fn an_engine_error_mid_stream_becomes_an_error_event_and_ends_the_stream() { + let mut translator = StreamTranslator::new(MODEL); + translator.feed(&chunk(&json!({"content": "x"}), None)); + let events = translator.feed(r#"{"error": {"message": "out of memory"}}"#); + assert_eq!(names(&events), ["error"]); + assert!(events[0].contains("out of memory"), "{}", events[0]); + assert!( + translator + .feed(&chunk(&json!({"content": "y"}), None)) + .is_empty() + ); + let events = StreamTranslator::new(MODEL).feed("not json"); + assert_eq!(names(&events), ["error"]); + } + + /// A stand-in OpenAI engine: lists its model, answers one chat completion, + /// and records the `Authorization` header and body of what it was sent. + struct FakeEngine { + port: u16, + seen: Arc>>, + } + + fn fake_engine() -> FakeEngine { + fake_engine_with(false) + } + + /// `native` makes it answer `/v1/messages` the way an engine that serves one + /// does (refusing an empty body); otherwise that route is a 404. + fn fake_engine_with(native: bool) -> FakeEngine { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let seen: Arc>> = Arc::default(); + let log = Arc::clone(&seen); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let mut raw = Vec::new(); + let mut chunk = [0_u8; 4096]; + loop { + let read = stream.read(&mut chunk).unwrap_or(0); + raw.extend_from_slice(&chunk[..read]); + let text = String::from_utf8_lossy(&raw).into_owned(); + let Some((head, body)) = text.split_once("\r\n\r\n") else { + if read == 0 { + break; + } + continue; + }; + let length = head + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(|v| v.trim().parse::().unwrap_or(0)) + }) + .unwrap_or(0); + if body.len() >= length || read == 0 { + break; + } + } + let text = String::from_utf8_lossy(&raw).into_owned(); + let (head, body) = text.split_once("\r\n\r\n").unwrap_or((&text, "")); + let request_line = head.lines().next().unwrap_or_default().to_owned(); + let auth = head + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .starts_with("authorization:") + .then(|| line.split_once(':').map(|(_, v)| v.trim().to_owned())) + .flatten() + }) + .unwrap_or_default(); + let streaming = body.contains("\"stream\":true"); + let (status, reply) = if request_line.contains("/v1/messages") { + if native { + ("400 Bad Request", json!({"error": "invalid request"})) + } else { + ("404 Not Found", json!({"detail": "Not Found"})) + } + } else if auth == "Bearer wrong" { + ("401 Unauthorized", json!({"error": {"message": "bad key"}})) + } else if request_line.contains("/v1/models") { + ("200 OK", json!({"data": [{"id": MODEL}]})) + } else if streaming { + log.lock().unwrap().push((auth, body.to_owned())); + let chunk = |delta: Value, finish: Value| { + format!( + "data: {}\n\n", + json!({ + "id": "chatcmpl-s1", + "model": MODEL, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + }) + ) + }; + let usage = json!({ + "id": "chatcmpl-s1", + "model": MODEL, + "choices": [], + "usage": {"prompt_tokens": 3, "completion_tokens": 2}, + }); + let events = format!( + "{}{}{}{}data: {usage}\n\ndata: [DONE]\n\n", + chunk(json!({"role": "assistant", "content": ""}), Value::Null), + chunk(json!({"content": "po"}), Value::Null), + chunk(json!({"content": "ng"}), Value::Null), + chunk(json!({}), json!("stop")), + ); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n{events}" + ); + continue; + } else if body.contains("\"tools\"") { + log.lock().unwrap().push((auth, body.to_owned())); + ( + "200 OK", + json!({ + "id": "chatcmpl-2", + "model": MODEL, + "choices": [{ + "message": {"content": null, "tool_calls": [{ + "id": "call_9", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\":\"Oslo\"}"}, + }]}, + "finish_reason": "tool_calls", + }], + "usage": {"prompt_tokens": 20, "completion_tokens": 9}, + }), + ) + } else { + log.lock().unwrap().push((auth, body.to_owned())); + ( + "200 OK", + json!({ + "id": "chatcmpl-1", + "model": MODEL, + "choices": [{"message": {"content": "pong"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}, + }), + ) + }; + let reply = reply.to_string(); + let _ = write!( + stream, + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", + reply.len() + ); + } + }); + FakeEngine { port, seen } + } + + /// Start the proxy router over a fake engine and return its base URL. + async fn proxy_over(engine: &FakeEngine) -> String { + let state = ProxyState { + upstream: format!("http://127.0.0.1:{}", engine.port), + served_model: MODEL.to_owned(), + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { axum::serve(listener, router(state)).await }); + format!("http://127.0.0.1:{port}") + } + + /// POST `body` to the proxy and return `(status, json body)`. + fn post(base: &str, path: &str, key: Option<&str>, body: &Value) -> (u16, Value) { + let mut request = ureq::post(&format!("{base}{path}")); + if let Some(key) = key { + request = request.set("x-api-key", key); + } + let parse = |response: ureq::Response| { + let status = response.status(); + ( + status, + serde_json::from_str(&response.into_string().unwrap()).unwrap(), + ) + }; + match request + .set("content-type", "application/json") + .send_string(&body.to_string()) + { + Ok(response) => parse(response), + Err(ureq::Error::Status(_, response)) => parse(response), + Err(error) => panic!("{error}"), + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn messages_round_trips_through_the_engine_with_the_clients_key() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let (status, message) = tokio::task::spawn_blocking(move || { + post( + &base, + "/v1/messages", + Some("client-key"), + &json!({ + "model": "claude-sonnet-4", + "max_tokens": 16, + "messages": [{"role": "user", "content": "ping"}], + }), + ) + }) + .await + .unwrap(); + assert_eq!(status, 200); + assert_eq!(message["type"], "message"); + assert_eq!(message["model"], MODEL); + assert_eq!(message["content"][0]["text"], "pong"); + assert_eq!(message["stop_reason"], "end_turn"); + let seen = engine.seen.lock().unwrap(); + assert_eq!(seen.len(), 1); + assert_eq!(seen[0].0, "Bearer client-key"); + let forwarded: Value = serde_json::from_str(&seen[0].1).unwrap(); + assert_eq!(forwarded["model"], MODEL); + assert_eq!(forwarded["messages"][0]["content"], "ping"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn engine_refusals_reach_the_client_as_anthropic_errors() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let (status, error) = tokio::task::spawn_blocking(move || { + post( + &base, + "/v1/messages", + Some("wrong"), + &json!({"max_tokens": 4, "messages": [{"role": "user", "content": "x"}]}), + ) + }) + .await + .unwrap(); + assert_eq!(status, 401); + assert_eq!( + error, + json!({"type": "error", "error": {"type": "authentication_error", "message": "bad key"}}) + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn untranslatable_requests_are_refused_before_reaching_the_engine() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let (status, error) = tokio::task::spawn_blocking(move || { + post( + &base, + "/v1/messages", + None, + &json!({"max_tokens": 4, "stream": true, "tools": [{"name": "t"}], "messages": [{"role": "user", "content": "x"}]}), + ) + }) + .await + .unwrap(); + assert_eq!(status, 400); + assert_eq!(error["error"]["type"], "invalid_request_error"); + assert!(engine.seen.lock().unwrap().is_empty()); + } + + /// POST a streaming request and return `(status, content-type, raw body)`. + fn post_streaming(base: &str, key: Option<&str>) -> (u16, String, String) { + let mut request = ureq::post(&format!("{base}/v1/messages")); + if let Some(key) = key { + request = request.set("x-api-key", key); + } + let body = json!({ + "model": "claude-sonnet-4", + "max_tokens": 16, + "stream": true, + "messages": [{"role": "user", "content": "ping"}], + }); + let read = |response: ureq::Response| { + ( + response.status(), + response.content_type().to_owned(), + response.into_string().unwrap(), + ) + }; + match request.send_string(&body.to_string()) { + Ok(response) => read(response), + Err(ureq::Error::Status(_, response)) => read(response), + Err(error) => panic!("{error}"), + } + } + + /// The `(event name, data)` pairs of a server-sent-event body. + fn sse_events(body: &str) -> Vec<(String, Value)> { + body.split("\n\n") + .filter(|frame| !frame.trim().is_empty()) + .map(|frame| { + let mut lines = frame.lines(); + let event = lines.next().unwrap().strip_prefix("event: ").unwrap(); + let data = lines.next().unwrap().strip_prefix("data: ").unwrap(); + assert!(lines.next().is_none(), "{frame}"); + (event.to_owned(), serde_json::from_str(data).unwrap()) + }) + .collect() + } + + #[tokio::test(flavor = "multi_thread")] + async fn a_streaming_request_is_answered_with_anthropic_server_sent_events() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let (status, content_type, body) = + tokio::task::spawn_blocking(move || post_streaming(&base, Some("client-key"))) + .await + .unwrap(); + assert_eq!((status, content_type.as_str()), (200, "text/event-stream")); + let events = sse_events(&body); + let names: Vec<_> = events.iter().map(|(name, _)| name.as_str()).collect(); + assert_eq!( + names, + [ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + ); + assert_eq!(events[0].1["message"]["id"], "msg_s1"); + assert_eq!(events[0].1["message"]["model"], MODEL); + let text: String = events + .iter() + .filter(|(name, _)| name == "content_block_delta") + .map(|(_, data)| data["delta"]["text"].as_str().unwrap()) + .collect(); + assert_eq!(text, "pong"); + assert_eq!(events[5].1["delta"]["stop_reason"], "end_turn"); + assert_eq!(events[5].1["usage"]["output_tokens"], 2); + let seen = engine.seen.lock().unwrap(); + assert_eq!(seen[0].0, "Bearer client-key"); + let forwarded: Value = serde_json::from_str(&seen[0].1).unwrap(); + assert_eq!(forwarded["stream"], true); + assert_eq!(forwarded["model"], MODEL); + } + + #[tokio::test(flavor = "multi_thread")] + async fn a_streaming_request_the_engine_refuses_gets_a_plain_anthropic_error() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let (status, content_type, body) = + tokio::task::spawn_blocking(move || post_streaming(&base, Some("wrong"))) + .await + .unwrap(); + assert_eq!((status, content_type.as_str()), (401, "application/json")); + let error: Value = serde_json::from_str(&body).unwrap(); + assert_eq!(error["error"]["type"], "authentication_error"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn a_models_auth_failure_is_relayed_not_reported_as_unreachable() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let (status, error) = tokio::task::spawn_blocking(move || { + match ureq::get(&format!("{base}/v1/models")) + .set("x-api-key", "wrong") + .call() + { + Err(ureq::Error::Status(status, response)) => ( + status, + serde_json::from_str::(&response.into_string().unwrap()).unwrap(), + ), + other => panic!("expected an HTTP error, got {other:?}"), + } + }) + .await + .unwrap(); + assert_eq!(status, 401); + assert_eq!(error["error"]["type"], "authentication_error"); + assert_eq!(error["error"]["message"], "bad key"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn models_are_forwarded_so_readiness_covers_the_proxy() { + let engine = fake_engine(); + let base = proxy_over(&engine).await; + let ready = tokio::task::spawn_blocking(move || { + rocm_core::anthropic_endpoint_ready(&base, Some(MODEL), None, Duration::from_secs(5)) + }) + .await + .unwrap(); + assert!(ready); + } + + #[test] + fn proxy_state_prefers_the_served_name_the_engine_reported() { + let dir = unique_temp_dir("served"); + std::fs::create_dir_all(&dir).unwrap(); + let mut config = AnthropicProxyConfig { + host: "127.0.0.1".to_owned(), + engine_port: 1, + served_model: "org/image:1".to_owned(), + state_path: dir.join("state.json"), + probe_api_key: None, + }; + assert_eq!(ProxyState::from_config(&config).served_model, "org/image:1"); + std::fs::write(&config.state_path, r#"{"served_model_id": "Org/Served"}"#).unwrap(); + assert_eq!(ProxyState::from_config(&config).served_model, "Org/Served"); + config.state_path = dir.join("missing.json"); + assert_eq!(ProxyState::from_config(&config).served_model, "org/image:1"); + let _ = std::fs::remove_dir_all(dir); + } + + fn published_endpoint_for(engine: &FakeEngine) -> (String, String) { + let dir = unique_temp_dir("run"); + std::fs::create_dir_all(&dir).unwrap(); + let state_path = dir.join("state.json"); + spawn_alongside_engine(AnthropicProxyConfig { + host: "127.0.0.1".to_owned(), + engine_port: engine.port, + served_model: MODEL.to_owned(), + state_path: state_path.clone(), + probe_api_key: None, + }); + let url = + rocm_core::wait_for_published_anthropic_endpoint(&state_path, Duration::from_secs(30)) + .expect("no Anthropic endpoint was published"); + let state: Value = serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + let _ = std::fs::remove_dir_all(dir); + ( + url, + state["anthropic_endpoint_mode"] + .as_str() + .unwrap() + .to_owned(), + ) + } + + #[test] + fn an_engine_without_the_route_is_fronted_by_a_proxy_on_its_own_port() { + let engine = fake_engine_with(false); + let (url, mode) = published_endpoint_for(&engine); + assert_eq!(mode, "proxy"); + assert_ne!(url, format!("http://127.0.0.1:{}", engine.port)); + let (status, message) = post( + &url, + "/v1/messages", + None, + &json!({"max_tokens": 8, "messages": [{"role": "user", "content": "ping"}]}), + ); + assert_eq!( + (status, message["content"][0]["text"].as_str()), + (200, Some("pong")) + ); + } + + #[test] + fn an_engine_that_serves_the_route_is_still_fronted_by_the_proxy() { + // vLLM and Lemonade answer `/v1/messages` themselves but reject any model + // name other than the one they serve; the proxy answers under any name. + let engine = fake_engine_with(true); + let (url, mode) = published_endpoint_for(&engine); + assert_eq!(mode, "proxy"); + assert_ne!(url, format!("http://127.0.0.1:{}", engine.port)); + let (status, message) = post( + &url, + "/v1/messages", + None, + &json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 8, + "messages": [{"role": "user", "content": "ping"}], + }), + ); + assert_eq!( + (status, message["content"][0]["text"].as_str()), + (200, Some("pong")) + ); + } + + #[test] + fn publish_records_the_endpoint_without_a_v1_suffix() { + let dir = unique_temp_dir("publish"); + let config = AnthropicProxyConfig { + host: "127.0.0.1".to_owned(), + engine_port: 1, + served_model: MODEL.to_owned(), + state_path: dir.join("state.json"), + probe_api_key: None, + }; + publish( + &config, + "http://127.0.0.1:8000/v1", + AnthropicEndpointMode::Proxy, + ) + .unwrap(); + let state: Value = + serde_json::from_slice(&std::fs::read(&config.state_path).unwrap()).unwrap(); + assert_eq!(state["anthropic_endpoint_url"], "http://127.0.0.1:8000"); + assert_eq!(state["anthropic_endpoint_mode"], "proxy"); + let _ = std::fs::remove_dir_all(dir); + } +} diff --git a/apps/rocmd/src/artifact_prefetch.rs b/apps/rocmd/src/artifact_prefetch.rs new file mode 100644 index 0000000000..d24530516a --- /dev/null +++ b/apps/rocmd/src/artifact_prefetch.rs @@ -0,0 +1,1001 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `prefetch_artifact` sandbox tool's download policy and driver: whether a +//! recipe artifact may be fetched under the approved source policy, the +//! host-scoped Hugging Face URL checks that gate a token, and the streamed, +//! sha256-verified download that records the cache marker. [`crate::sandbox`] +//! resolves the artifact reference and calls in; nothing here calls back. + +use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; +use anyhow::{Context, Result}; +use rocm_core::atomic_write::write_file_atomically; +use rocm_core::{ + AppPaths, ModelRecipeArtifactRecord, model_artifact_cache_status, unix_time_millis, +}; +use serde_json::Value; +use serde_json::json; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +const ARTIFACT_PREFETCH_TIMEOUT: Duration = Duration::from_mins(10); + +pub(crate) fn prefetch_artifact_value_with_policy( + paths: &AppPaths, + artifact_ref: &str, + model: &str, + artifact: ModelRecipeArtifactRecord, + policy: SandboxToolPolicy, +) -> Result { + let cache = model_artifact_cache_status(paths, model, &artifact); + if !policy.allow_artifact_download { + return Ok(json!({ + "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), + "artifact_ref": artifact_ref, + "model": model, + "artifact": artifact, + "cache": cache, + "status": "source_policy_required", + "mutating": false, + "network_used": false, + "message": "artifact prefetch requires an approved source policy before network access; no artifact bytes were downloaded", + })); + } + + if cache.marker_path.is_file() { + return Ok(json!({ + "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), + "artifact_ref": artifact_ref, + "model": model, + "artifact": artifact, + "cache": cache, + "status": "cached", + "mutating": false, + "network_used": false, + "message": "artifact cache marker already exists; no network request was made", + })); + } + + if let Some(message) = declared_source_policy_block_message(&artifact) { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + &message, + )); + } + let source_policy_requires_huggingface_auth = + artifact_declares_huggingface_authenticated_policy(&artifact); + let artifact_kind = artifact.kind.to_ascii_lowercase(); + let huggingface_artifact = artifact_is_huggingface(&artifact); + if artifact.gated.unwrap_or(false) && !huggingface_artifact { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "gated artifacts require a source-specific authentication policy before download", + )); + } + if !matches!(artifact_kind.as_str(), "url" | "http" | "https" | "direct") + && !huggingface_artifact + { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "approved live prefetch currently supports direct HTTP(S) artifacts only", + )); + } + if !artifact.uri.starts_with("https://") && !artifact.uri.starts_with("http://") { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + if huggingface_artifact { + "authenticated Hugging Face prefetch requires an HTTP(S) artifact URI in the signed recipe metadata" + } else { + "approved live prefetch requires an HTTP(S) artifact URI" + }, + )); + } + let mut request_headers = Vec::<(&str, String)>::new(); + let source_policy = if huggingface_artifact + && (artifact.gated.unwrap_or(false) + || policy.allow_huggingface_download + || source_policy_requires_huggingface_auth) + { + if !policy.allow_huggingface_download { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "Hugging Face artifacts require --allow-huggingface-download before rocm-cli may use an authentication token", + )); + } + if !artifact.uri.starts_with("https://") { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "rocm-cli will not send a Hugging Face token over plain HTTP; use an HTTPS Hugging Face artifact URI", + )); + } + if !is_huggingface_url(&artifact.uri) { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "rocm-cli will not send a Hugging Face token to a non-Hugging Face URL", + )); + } + let Some(token) = policy + .huggingface_token + .as_deref() + .map(str::trim) + .filter(|token| !token.is_empty()) + else { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "Hugging Face artifact prefetch needs ROCM_CLI_HUGGINGFACE_TOKEN, HF_TOKEN, or HUGGING_FACE_HUB_TOKEN", + )); + }; + request_headers.push(("Authorization", format!("Bearer {token}"))); + "huggingface_authenticated" + } else { + "explicit_allow_artifact_download" + }; + let Some(expected_sha256) = artifact + .sha256 + .as_deref() + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| value.len() == 64 && value.chars().all(|ch| ch.is_ascii_hexdigit())) + else { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "approved live prefetch requires a valid sha256 in the recipe artifact metadata", + )); + }; + let Some(size_bytes) = artifact.size_bytes else { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "approved live prefetch requires size_bytes in the recipe artifact metadata", + )); + }; + let max_bytes = policy.artifact_max_bytes.unwrap_or(size_bytes); + if size_bytes > max_bytes { + return Ok(prefetch_blocked_value( + artifact_ref, + model, + artifact, + cache, + "artifact size exceeds the approved download byte limit", + )); + } + + // Streamed straight to its final path: model artifacts run to gigabytes, so + // buffering one to hash it would hold the whole file in memory. The size and + // digest from the recipe are enforced inside the download, which also gives + // this path a free-space preflight and resume on a dropped connection. + let artifact_path = artifact_bytes_path_for_marker(&cache.marker_path); + let mut headers: Vec<(&str, &str)> = vec![("User-Agent", "rocm-cli")]; + headers.extend( + request_headers + .iter() + .map(|(name, value)| (*name, value.as_str())), + ); + let outcome = rocm_core::download_file_streaming(&rocm_core::DownloadRequest { + url: &artifact.uri, + destination: &artifact_path, + timeout: ARTIFACT_PREFETCH_TIMEOUT, + headers: &headers, + max_bytes: Some(max_bytes), + expected_len: Some(size_bytes), + expected_sha256: Some(&expected_sha256), + }) + .with_context(|| format!("failed to prefetch artifact `{artifact_ref}`"))?; + let actual_sha256 = outcome.sha256; + write_file_atomically( + &cache.marker_path, + &serde_json::to_vec_pretty(&json!({ + "artifact_ref": artifact_ref, + "model": model, + "artifact": artifact, + "bytes_path": artifact_path, + "size_bytes": size_bytes, + "sha256": actual_sha256, + "prefetched_at_unix_ms": unix_time_millis(), + "source_policy": source_policy, + })) + .context("failed to serialize artifact cache marker")?, + )?; + let cache = model_artifact_cache_status(paths, model, &artifact); + Ok(json!({ + "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), + "artifact_ref": artifact_ref, + "model": model, + "artifact": artifact, + "cache": cache, + "status": "prefetched", + "mutating": true, + "network_used": true, + "bytes_path": artifact_path, + "size_bytes": size_bytes, + "sha256": actual_sha256, + "source_policy": source_policy, + "message": "artifact downloaded and verified with recipe sha256", + })) +} + +fn declared_source_policy_block_message(artifact: &ModelRecipeArtifactRecord) -> Option { + let source_policy = artifact.source_policy.as_ref()?; + if !source_policy.required_hosts.is_empty() { + let Some(host) = http_url_host(&artifact.uri) else { + return Some( + "recipe source policy declares required hosts but the artifact URI is not HTTP(S)" + .to_owned(), + ); + }; + if !source_policy + .required_hosts + .iter() + .any(|required| required.eq_ignore_ascii_case(&host)) + { + return Some(format!( + "artifact host `{host}` is not allowed by the recipe source policy" + )); + } + } + + match source_policy.policy.as_str() { + "direct_https_sha256" => { + if artifact.uri.starts_with("https://") { + None + } else { + Some( + "recipe source policy direct_https_sha256 requires an HTTPS artifact URI" + .to_owned(), + ) + } + } + "huggingface_public" => { + if artifact.gated.unwrap_or(false) { + Some( + "recipe source policy marks this as public Hugging Face, but the artifact is gated" + .to_owned(), + ) + } else { + huggingface_policy_uri_block_message(artifact, "huggingface_public") + } + } + "huggingface_authenticated" => { + huggingface_policy_uri_block_message(artifact, "huggingface_authenticated") + } + "manual_only" => Some( + "recipe source policy marks this artifact as manual-only; rocm-cli will not download it" + .to_owned(), + ), + other => Some(format!( + "recipe source policy `{other}` is not supported by this rocm-cli build" + )), + } +} + +fn huggingface_policy_uri_block_message( + artifact: &ModelRecipeArtifactRecord, + policy: &str, +) -> Option { + if !artifact.uri.starts_with("https://") { + return Some(format!( + "recipe source policy {policy} requires an HTTPS Hugging Face artifact URI" + )); + } + if !is_huggingface_url(&artifact.uri) { + return Some(format!( + "recipe source policy {policy} requires a Hugging Face artifact URI" + )); + } + None +} + +fn artifact_declares_huggingface_authenticated_policy( + artifact: &ModelRecipeArtifactRecord, +) -> bool { + artifact + .source_policy + .as_ref() + .is_some_and(|policy| policy.policy == "huggingface_authenticated") +} + +fn artifact_is_huggingface(artifact: &ModelRecipeArtifactRecord) -> bool { + let kind = artifact.kind.to_ascii_lowercase(); + matches!(kind.as_str(), "huggingface" | "hf" | "hugging_face") + || is_huggingface_url(&artifact.uri) +} + +fn is_huggingface_url(url: &str) -> bool { + http_url_host(url).is_some_and(|host| { + host == "huggingface.co" + || host.ends_with(".huggingface.co") + || host == "hf.co" + || host.ends_with(".hf.co") + }) +} + +fn http_url_host(url: &str) -> Option { + let rest = url + .strip_prefix("https://") + .or_else(|| url.strip_prefix("http://"))?; + let authority = rest + .split(['/', '?', '#']) + .next() + .unwrap_or_default() + .trim(); + if authority.is_empty() || authority.contains('@') { + return None; + } + let host = authority + .strip_prefix('[') + .and_then(|value| value.split_once(']').map(|(host, _)| host)) + .unwrap_or_else(|| authority.split(':').next().unwrap_or_default()) + .trim() + .trim_end_matches('.') + .to_ascii_lowercase(); + (!host.is_empty()).then_some(host) +} + +fn prefetch_blocked_value( + artifact_ref: &str, + model: &str, + artifact: ModelRecipeArtifactRecord, + cache: rocm_core::ModelArtifactCacheStatus, + reason: &str, +) -> Value { + json!({ + "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), + "artifact_ref": artifact_ref, + "model": model, + "artifact": artifact, + "cache": cache, + "status": "blocked", + "mutating": false, + "network_used": false, + "message": reason, + }) +} + +fn artifact_bytes_path_for_marker(marker_path: &Path) -> PathBuf { + marker_path.with_extension("bin") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use sha2::Digest; + use std::fs; + use std::io::{Read, Write}; + use std::thread; + + /// Hex digest of a buffer. Production hashing happens incrementally inside the + /// streaming download; this exists only so tests can state an expected digest. + fn sha256_hex(bytes: &[u8]) -> String { + format!("{:x}", sha2::Sha256::digest(bytes)) + } + + fn serve_one_http_response(bytes: Vec) -> Result { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0))?; + let addr = listener.local_addr()?; + thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request); + let header = format!( + "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + bytes.len() + ); + let _ = stream.write_all(header.as_bytes()); + let _ = stream.write_all(&bytes); + } + }); + Ok(format!("http://{addr}/artifact.bin")) + } + + #[test] + fn sandbox_prefetch_reports_policy_required_without_network() { + let (root, paths) = temp_app_paths("sandbox-prefetch-policy"); + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#hf-main", + "Qwen/Test-1B", + ModelRecipeArtifactRecord { + artifact_id: "hf-main".to_owned(), + kind: "huggingface".to_owned(), + uri: "Qwen/Test-1B".to_owned(), + revision: Some("main".to_owned()), + sha256: Some("a".repeat(64)), + size_bytes: Some(1024), + license: Some("apache-2.0".to_owned()), + gated: Some(false), + quantization: Some("bf16".to_owned()), + engines: vec!["vllm".to_owned()], + source_policy: None, + }, + SandboxToolPolicy::default(), + ) + .expect("policy value should render"); + fs::remove_dir_all(root).ok(); + + assert_eq!( + value.get("status").and_then(Value::as_str), + Some("source_policy_required") + ); + assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(false)); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("cache") + .and_then(|cache| cache.get("status")) + .and_then(Value::as_str) + .is_some_and(|status| status == "missing") + ); + } + + #[test] + fn sandbox_prefetch_cached_marker_skips_network() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-cached"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "direct-bin".to_owned(), + kind: "url".to_owned(), + uri: "https://example.invalid/should-not-be-fetched.bin".to_owned(), + revision: None, + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(false), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + let cache = model_artifact_cache_status(&paths, "Qwen/Test-1B", &artifact); + write_file_atomically(&cache.marker_path, br#"{"cached":true}"#)?; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#direct-bin", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("cached")); + assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(false)); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_blocks_artifact_larger_than_approved_limit() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-byte-limit"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "direct-bin".to_owned(), + kind: "url".to_owned(), + uri: "https://example.invalid/too-large.bin".to_owned(), + revision: None, + sha256: Some("a".repeat(64)), + size_bytes: Some(2048), + license: Some("test-only".to_owned()), + gated: Some(false), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#direct-bin", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("byte limit")) + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_blocks_non_direct_non_huggingface_source() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-non-direct"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "torrent-bin".to_owned(), + kind: "torrent".to_owned(), + uri: "https://example.invalid/artifact.torrent".to_owned(), + revision: None, + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(false), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#torrent-bin", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("direct HTTP(S)")) + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_blocks_gated_huggingface_without_source_policy() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-hf-policy"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "hf-main".to_owned(), + kind: "huggingface".to_owned(), + uri: "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors".to_owned(), + revision: Some("main".to_owned()), + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(true), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#hf-main", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("--allow-huggingface-download")) + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_blocks_huggingface_source_policy_without_token() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-hf-token"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "hf-main".to_owned(), + kind: "huggingface".to_owned(), + uri: "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors".to_owned(), + revision: Some("main".to_owned()), + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(true), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#hf-main", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + allow_huggingface_download: true, + huggingface_token: None, + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("ROCM_CLI_HUGGINGFACE_TOKEN")) + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_respects_manual_only_source_policy() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-manual-policy"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "manual-bin".to_owned(), + kind: "url".to_owned(), + uri: "https://example.invalid/manual.bin".to_owned(), + revision: None, + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(false), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: Some(rocm_core::ModelRecipeArtifactSourcePolicyRecord { + policy: "manual_only".to_owned(), + required_hosts: Vec::new(), + notes: vec!["requires manual license review".to_owned()], + }), + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#manual-bin", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("manual-only")) + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_respects_declared_huggingface_auth_policy() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-hf-declared-auth"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "hf-main".to_owned(), + kind: "huggingface".to_owned(), + uri: "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors".to_owned(), + revision: Some("main".to_owned()), + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(false), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: Some(rocm_core::ModelRecipeArtifactSourcePolicyRecord { + policy: "huggingface_authenticated".to_owned(), + required_hosts: vec!["huggingface.co".to_owned()], + notes: Vec::new(), + }), + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#hf-main", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("--allow-huggingface-download")) + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_never_sends_huggingface_token_to_non_huggingface_url() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-hf-host"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "hf-main".to_owned(), + kind: "huggingface".to_owned(), + uri: "https://example.invalid/Qwen/Test-1B/model.safetensors".to_owned(), + revision: Some("main".to_owned()), + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(true), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#hf-main", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + allow_huggingface_download: true, + huggingface_token: Some("hf_secret_should_not_leak".to_owned()), + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("non-Hugging Face URL")) + ); + assert!( + !serde_json::to_string(&value)?.contains("hf_secret_should_not_leak"), + "prefetch report must not include authentication tokens" + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_never_sends_huggingface_token_over_plain_http() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-hf-https"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "hf-main".to_owned(), + kind: "huggingface".to_owned(), + uri: "http://huggingface.co/Qwen/Test-1B/model.safetensors".to_owned(), + revision: Some("main".to_owned()), + sha256: Some("a".repeat(64)), + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(true), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#hf-main", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + allow_huggingface_download: true, + huggingface_token: Some("hf_secret_should_not_leak".to_owned()), + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("plain HTTP")) + ); + assert!( + !serde_json::to_string(&value)?.contains("hf_secret_should_not_leak"), + "prefetch report must not include authentication tokens" + ); + Ok(()) + } + + #[test] + fn sandbox_prefetch_downloads_direct_artifact_when_approved() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-download"); + let bytes = b"tiny verified artifact".to_vec(); + let uri = serve_one_http_response(bytes.clone())?; + let sha256 = sha256_hex(&bytes); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "direct-bin".to_owned(), + kind: "url".to_owned(), + uri, + revision: None, + sha256: Some(sha256.clone()), + size_bytes: Some(bytes.len() as u64), + license: Some("test-only".to_owned()), + gated: Some(false), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#direct-bin", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + let bytes_path = value + .get("bytes_path") + .and_then(Value::as_str) + .map(PathBuf::from) + .expect("bytes_path should be present"); + let marker_path = value + .get("cache") + .and_then(|cache| cache.get("marker_path")) + .and_then(Value::as_str) + .map(PathBuf::from) + .expect("marker path should be present"); + + assert_eq!( + value.get("status").and_then(Value::as_str), + Some("prefetched") + ); + assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(true)); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + value.get("sha256").and_then(Value::as_str), + Some(sha256.as_str()) + ); + assert_eq!( + value.get("source_policy").and_then(Value::as_str), + Some("explicit_allow_artifact_download") + ); + assert_eq!(fs::read(&bytes_path)?, bytes); + assert!(marker_path.is_file()); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[test] + fn sandbox_prefetch_blocks_approved_artifact_without_sha256() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-prefetch-no-sha"); + let artifact = ModelRecipeArtifactRecord { + artifact_id: "direct-bin".to_owned(), + kind: "url".to_owned(), + uri: "https://example.invalid/artifact.bin".to_owned(), + revision: None, + sha256: None, + size_bytes: Some(12), + license: Some("test-only".to_owned()), + gated: Some(false), + quantization: None, + engines: vec!["vllm".to_owned()], + source_policy: None, + }; + + let value = prefetch_artifact_value_with_policy( + &paths, + "qwen#direct-bin", + "Qwen/Test-1B", + artifact, + SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + }, + )?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); + assert_eq!( + value.get("network_used").and_then(Value::as_bool), + Some(false) + ); + assert!( + value + .get("message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("sha256")) + ); + Ok(()) + } + + #[test] + fn huggingface_url_detection_is_host_scoped() { + assert!(is_huggingface_url( + "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors" + )); + assert!(is_huggingface_url( + "https://cdn-lfs.huggingface.co/repos/example" + )); + assert!(is_huggingface_url("https://hf.co/Qwen/Test-1B")); + assert!(!is_huggingface_url( + "https://huggingface.co.evil.example/Qwen/Test-1B" + )); + assert!(!is_huggingface_url( + "https://huggingface.co@evil.example/Qwen/Test-1B" + )); + } +} diff --git a/apps/rocmd/src/cli.rs b/apps/rocmd/src/cli.rs index dac75e120c..63d5fa0b66 100644 --- a/apps/rocmd/src/cli.rs +++ b/apps/rocmd/src/cli.rs @@ -166,6 +166,9 @@ pub fn parse_args(args: Vec) -> std::result::Result<(), clap::Error> { } pub fn run_from_args(args: Vec) -> Result<()> { + // `rocm daemon` is the one caller; see `daemon_host` for why the daemon + // has to know it is not running as the `rocmd` binary. + crate::daemon_host::mark_embedded_in_rocm(); let cli = Cli::try_parse_from(args)?; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 417491a716..c95ea8d24a 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -27,22 +27,7 @@ pub(crate) fn run_rocm_capture_for_paths( args: &[&str], timeout: Duration, ) -> Result { - rocm_core::run_rocm_capture_with_env( - args, - &[ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ], - timeout, - ) -} - -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } + rocm_core::run_rocm_capture_with_env(args, &paths.env_vars(), timeout) } pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { diff --git a/apps/rocmd/src/daemon_host.rs b/apps/rocmd/src/daemon_host.rs new file mode 100644 index 0000000000..9513e6a562 --- /dev/null +++ b/apps/rocmd/src/daemon_host.rs @@ -0,0 +1,423 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Which binary a daemon spawn runs. +//! +//! The daemon runs in one of two hosts: the `rocmd` binary (`rocmd run`), or +//! embedded in `rocm` (`rocm daemon`, which enters through +//! [`crate::run_from_args`]). The two binaries do not answer the same commands: +//! `supervise` belongs to `rocmd` alone, and the hidden engine commands +//! (`__engine-serve-http`, `__engine-stdio`) belong to `rocm` alone. A spawn +//! that re-executes the current executable is right only in the host that has +//! the command, so a spawn asks this module for the binary instead of reading +//! `current_exe` itself. + +use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +const ROCMD_BINARY: &str = "rocmd"; +const ROCM_BINARY: &str = "rocm"; + +/// Set once by the embedded entry point. An explicit signal rather than a guess +/// from the executable's name, which a renamed or upper-cased binary defeats. +static EMBEDDED_IN_ROCM: AtomicBool = AtomicBool::new(false); + +/// Record that this process is `rocm` running the daemon, for the rest of its life. +pub(crate) fn mark_embedded_in_rocm() { + EMBEDDED_IN_ROCM.store(true, Ordering::Relaxed); +} + +fn is_embedded_in_rocm() -> bool { + #[cfg(test)] + { + if let Some(flag) = EMBEDDED_OVERRIDE.with(std::cell::Cell::get) { + return flag; + } + } + EMBEDDED_IN_ROCM.load(Ordering::Relaxed) +} + +// Unit tests only: the answer `is_embedded_in_rocm` gives on this thread. The +// real flag is process-wide, so a test that set it would change the answer for +// every other test; a thread-local lets a test see both answers and no other +// test see either. +#[cfg(test)] +thread_local! { + static EMBEDDED_OVERRIDE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// The host the daemon is running in, with the one executable path that every +/// decision and every sibling lookup is made from, so they cannot disagree. +pub(crate) struct DaemonHost { + current_exe: PathBuf, + embedded_in_rocm: bool, + /// Unit tests only: the test binary has every command, so a spawn runs it + /// and no sibling is ever looked for. + #[cfg(test)] + test_binary_has_every_command: bool, + /// Unit tests only: a host that stands in for one binary looks for no + /// sibling, so asking it for the other is an error and not a lookup that + /// could find, and run, a build of the other binary beside the test binary. + #[cfg(test)] + test_looks_for_no_sibling: bool, +} + +impl DaemonHost { + /// The host this process is running in. + /// + /// Under test, the process is the test binary, which stands in for both + /// `rocm` and `rocmd` and looks for neither: code that reaches this runs + /// the test binary and nothing else, not even a `rocm` that happens to sit + /// in the build directory. A test that needs another binary passes the code + /// it exercises a host of its own. + pub(crate) fn detect() -> Result { + #[cfg(test)] + let current_exe = std::env::current_exe().context("the test binary has a path")?; + #[cfg(not(test))] + let current_exe = rocm_core::current_executable_path() + .context("failed to resolve the current executable path")?; + // The same composition in both builds, so a test reaches the line that + // takes the host's embedded flag from the process. + let host = Self::new(current_exe, is_embedded_in_rocm()); + #[cfg(test)] + let host = Self { + test_binary_has_every_command: true, + ..host + }; + Ok(host) + } + + pub(crate) const fn new(current_exe: PathBuf, embedded_in_rocm: bool) -> Self { + Self { + current_exe, + embedded_in_rocm, + #[cfg(test)] + test_binary_has_every_command: false, + #[cfg(test)] + test_looks_for_no_sibling: false, + } + } + + /// The binary that has the `supervise` command: this one when the daemon is + /// `rocmd`, the `rocmd` installed beside `rocm` when it is embedded there. + pub(crate) fn rocmd_binary(&self) -> Result { + self.rocmd_binary_via(rocm_core::sibling_binary_path_from) + } + + /// [`Self::rocmd_binary`] with the sibling lookup passed in, given the + /// executable and the binary's name. + fn rocmd_binary_via( + &self, + sibling: impl FnOnce(&Path, &str) -> Result, + ) -> Result { + #[cfg(test)] + { + if self.test_binary_has_every_command { + return Ok(self.current_exe.clone()); + } + } + if !self.embedded_in_rocm { + return Ok(self.current_exe.clone()); + } + #[cfg(test)] + { + self.refuse_sibling_lookup(ROCMD_BINARY)?; + } + sibling(&self.current_exe, ROCMD_BINARY).context( + "the daemon runs inside `rocm`, which has no `supervise` command, \ + and recovery restarts a service through the `rocmd` binary installed next to it", + ) + } + + /// The binary that has the hidden engine commands: this one when the daemon + /// is embedded in `rocm`, the `rocm` installed beside `rocmd` when it is not. + pub(crate) fn rocm_binary(&self) -> Result { + self.rocm_binary_via(rocm_core::sibling_binary_path_from) + } + + /// [`Self::rocm_binary`] with the sibling lookup passed in. + fn rocm_binary_via( + &self, + sibling: impl FnOnce(&Path, &str) -> Result, + ) -> Result { + #[cfg(test)] + { + if self.test_binary_has_every_command { + return Ok(self.current_exe.clone()); + } + } + if self.embedded_in_rocm { + return Ok(self.current_exe.clone()); + } + #[cfg(test)] + { + self.refuse_sibling_lookup(ROCM_BINARY)?; + } + sibling(&self.current_exe, ROCM_BINARY).context( + "the `rocmd` binary has no engine commands, and a managed service's engine \ + runs through the `rocm` binary installed next to it", + ) + } + + /// The error a test stand-in answers with when asked for the binary it does + /// not stand in for. + #[cfg(test)] + fn refuse_sibling_lookup(&self, binary: &str) -> Result<()> { + if self.test_looks_for_no_sibling { + anyhow::bail!( + "this test host stands in for one binary and looks for no sibling, \ + so it cannot name `{binary}`" + ); + } + Ok(()) + } + + /// A host whose own executable is taken to have every command, so a test + /// that reaches a spawn runs the test binary, as each one did before the + /// binary was chosen here. + #[cfg(test)] + pub(crate) fn test_binary_standing_in_for_rocm() -> Self { + let mut host = Self::new( + std::env::current_exe().expect("the test binary has a path"), + true, + ); + host.test_looks_for_no_sibling = true; + host + } + + /// The same for a test that restarts a service: the test binary is the + /// `rocmd` that is re-executed. + #[cfg(test)] + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub(crate) fn test_binary_standing_in_for_rocmd() -> Self { + let mut host = Self::new( + std::env::current_exe().expect("the test binary has a path"), + false, + ); + host.test_looks_for_no_sibling = true; + host + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + /// The names a binary can carry on disk. The choice is made from the + /// embedded signal, so none of them may change it. + const ROCM_NAMES: [&str; 4] = ["/opt/rocm-cli/bin/rocm", "rocm.exe", "ROCM.EXE", "rocm-0.5"]; + const ROCMD_NAMES: [&str; 4] = [ + "/opt/rocm-cli/bin/rocmd", + "rocmd.exe", + "ROCMD.EXE", + "/repo/target/debug/deps/rocmd-3f9a41c2d0e7b655", + ]; + + #[test] + fn embedded_in_rocm_runs_supervise_on_the_sibling_rocmd_whatever_rocm_is_called() { + for exe in ROCM_NAMES { + let asked = RefCell::new(None); + let chosen = DaemonHost::new(PathBuf::from(exe), true) + .rocmd_binary_via(|from, name| { + *asked.borrow_mut() = Some((from.to_path_buf(), name.to_owned())); + Ok(PathBuf::from("/opt/rocm-cli/bin/rocmd")) + }) + .unwrap(); + assert_eq!(chosen, PathBuf::from("/opt/rocm-cli/bin/rocmd"), "{exe}"); + // The lookup starts from the same executable the choice was made + // from, and asks for `rocmd`. + assert_eq!( + asked.into_inner(), + Some((PathBuf::from(exe), "rocmd".to_owned())), + "{exe}" + ); + } + } + + #[test] + fn the_rocmd_daemon_keeps_its_own_executable_whatever_it_is_called() { + for exe in ROCMD_NAMES.into_iter().chain(["rocm"]) { + let chosen = DaemonHost::new(PathBuf::from(exe), false) + .rocmd_binary_via(|_, _| { + panic!("{exe}: not embedded in rocm, so it has `supervise` itself") + }) + .unwrap(); + assert_eq!(chosen, PathBuf::from(exe)); + } + } + + #[test] + fn a_missing_rocmd_beside_rocm_is_named() { + let error = DaemonHost::new(PathBuf::from("/opt/rocm-cli/bin/rocm"), true) + .rocmd_binary_via(|_, _| { + anyhow::bail!("unable to locate sibling binary rocmd; checked /opt/rocm-cli/bin") + }) + .unwrap_err(); + let text = format!("{error:#}"); + assert!(text.contains("no `supervise` command"), "{text}"); + assert!( + text.contains("unable to locate sibling binary rocmd"), + "{text}" + ); + } + + /// The hidden engine commands are `rocm`'s: a daemon that is `rocmd`, under + /// any name, runs them on the `rocm` beside it, and says what it looked for. + #[test] + fn the_rocmd_daemon_runs_engine_commands_on_the_sibling_rocm() { + for exe in ROCMD_NAMES.into_iter().chain(["rocm"]) { + let asked = RefCell::new(None); + let chosen = DaemonHost::new(PathBuf::from(exe), false) + .rocm_binary_via(|from, name| { + *asked.borrow_mut() = Some((from.to_path_buf(), name.to_owned())); + Ok(PathBuf::from("/opt/rocm-cli/bin/rocm")) + }) + .unwrap(); + assert_eq!(chosen, PathBuf::from("/opt/rocm-cli/bin/rocm"), "{exe}"); + assert_eq!( + asked.into_inner(), + Some((PathBuf::from(exe), "rocm".to_owned())), + "{exe}" + ); + } + } + + #[test] + fn embedded_in_rocm_keeps_its_own_executable_for_engine_commands() { + for exe in ROCM_NAMES { + let chosen = DaemonHost::new(PathBuf::from(exe), true) + .rocm_binary_via(|_, _| { + panic!("{exe}: embedded in rocm, so it has the engine commands itself") + }) + .unwrap(); + assert_eq!(chosen, PathBuf::from(exe)); + } + } + + #[test] + fn a_missing_rocm_beside_rocmd_is_named() { + let error = DaemonHost::new(PathBuf::from("/opt/rocm-cli/bin/rocmd"), false) + .rocm_binary_via(|_, _| { + anyhow::bail!("unable to locate sibling binary rocm; checked /opt/rocm-cli/bin") + }) + .unwrap_err(); + let text = format!("{error:#}"); + assert!(text.contains("no engine commands"), "{text}"); + assert!( + text.contains("unable to locate sibling binary rocm"), + "{text}" + ); + } + + /// The real lookup, from the host's own executable: the `rocmd` next to a + /// `rocm` that is not the test binary. + #[test] + fn the_real_sibling_lookup_starts_from_the_hosts_executable() { + let dir = crate::test_support::unique_test_root("daemon-host-sibling-lookup"); + let rocm = dir.join(rocm_core::platform_binary_name("rocm")); + let rocmd = dir.join(rocm_core::platform_binary_name("rocmd")); + std::fs::write(&rocmd, b"").expect("write rocmd"); + + let chosen = DaemonHost::new(rocm, true).rocmd_binary(); + std::fs::remove_dir_all(&dir).ok(); + assert_eq!(chosen.unwrap(), rocmd); + } + + /// A stand-in host names only the binary it stands in for. Asked for the + /// other it fails, and it never starts the real sibling lookup, which would + /// find the other binary in the build directory and let a test run it. + #[test] + fn a_stand_in_host_looks_for_no_sibling() { + let this = std::env::current_exe().unwrap(); + let rocm = DaemonHost::test_binary_standing_in_for_rocm(); + assert_eq!(rocm.rocm_binary().unwrap(), this); + let error = rocm + .rocmd_binary_via(|_, _| panic!("a stand-in host must not look for a sibling")) + .unwrap_err(); + assert!( + format!("{error:#}").contains("looks for no sibling"), + "{error:#}" + ); + assert!(rocm.rocmd_binary().is_err()); + + let rocmd = DaemonHost::test_binary_standing_in_for_rocmd(); + assert_eq!(rocmd.rocmd_binary().unwrap(), this); + let error = rocmd + .rocm_binary_via(|_, _| panic!("a stand-in host must not look for a sibling")) + .unwrap_err(); + assert!( + format!("{error:#}").contains("looks for no sibling"), + "{error:#}" + ); + assert!(rocmd.rocm_binary().is_err()); + } + + /// `detect` takes the embedded flag from the process, and not its negation + /// or a constant: with the flag set on this thread the host it returns is + /// the embedded one, and with it clear it is not. The two answers are the + /// whole premise of the fix, since they decide which binary each spawn runs. + /// What a unit test still cannot reach is the production arm that resolves + /// the executable (`rocm_core::current_executable_path`). + #[test] + fn detect_takes_the_embedded_flag_from_the_process() { + let this = std::env::current_exe().unwrap(); + for flag in [false, true] { + EMBEDDED_OVERRIDE.with(|cell| cell.set(Some(flag))); + let host = DaemonHost::detect(); + EMBEDDED_OVERRIDE.with(|cell| cell.set(None)); + let host = host.unwrap(); + assert_eq!(host.embedded_in_rocm, flag, "embedded flag {flag}"); + // The stand-in still answers with the test binary for both + // commands whatever the flag says. + assert_eq!(host.rocmd_binary().unwrap(), this, "embedded flag {flag}"); + assert_eq!(host.rocm_binary().unwrap(), this, "embedded flag {flag}"); + } + assert!(!is_embedded_in_rocm(), "the override is cleared again"); + } + + /// A process that did not enter through `run_from_args` is not embedded. + #[test] + fn a_plain_process_is_not_embedded_in_rocm() { + assert!(!is_embedded_in_rocm()); + } + + /// The real host is not reachable from a unit test: under test, `detect` is + /// the test binary for both commands, and nothing beside it is looked for. + #[test] + fn under_test_detect_is_the_test_binary_and_looks_for_no_sibling() { + let host = DaemonHost::detect().unwrap(); + let this = std::env::current_exe().unwrap(); + assert_eq!(host.rocmd_binary().unwrap(), this); + assert_eq!(host.rocm_binary().unwrap(), this); + } + + /// `rocm daemon` enters through `run_from_args`, and that is what marks + /// the process. The flag is process-wide, so it is checked in a child run of + /// this test binary instead of being set here, where it would change the + /// answer for every other test. + #[test] + fn run_from_args_marks_the_process_as_embedded_in_rocm() { + const CHILD: &str = "ROCMD_TEST_RUN_FROM_ARGS_CHILD"; + let test_name = "daemon_host::tests::run_from_args_marks_the_process_as_embedded_in_rocm"; + if std::env::var_os(CHILD).is_some() { + // `--help` stops clap before anything runs; the marking comes first. + let _ = crate::run_from_args(vec!["rocmd".into(), "--help".into()]); + assert!(is_embedded_in_rocm()); + return; + } + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", test_name, "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output() + .expect("run the child test"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains(&format!("test {test_name} ... ok")), + "child run failed:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + } +} diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index 32f1834547..eda8243ac2 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -2,23 +2,36 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, bail}; +use crate::daemon_host::DaemonHost; +use anyhow::{Result, bail}; use rocm_core::AppPaths; use rocm_engine_protocol::{ - EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, - HealthcheckResponse, + EngineMethod, EngineStdioOptions, HealthcheckRequest, HealthcheckResponse, + decode_engine_response, engine_request_envelope, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use std::io::Write; -use std::process::{Command as ProcessCommand, Stdio}; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; pub(crate) fn engine_healthcheck_response( paths: &AppPaths, engine: &str, service_id: &str, ) -> Result { - engine_request::<_, HealthcheckResponse>( + engine_healthcheck_response_on(&DaemonHost::detect()?, paths, engine, service_id) +} + +/// [`engine_healthcheck_response`] on a given host. The supervisor polls +/// through the host it started its engine on, so a test can supply both. +pub(crate) fn engine_healthcheck_response_on( + daemon_host: &DaemonHost, + paths: &AppPaths, + engine: &str, + service_id: &str, +) -> Result { + engine_request_on::<_, HealthcheckResponse>( + daemon_host, paths, engine, service_id, @@ -60,15 +73,19 @@ pub(crate) fn apply_endpoint_key_env( paths: &AppPaths, service_id: &str, ) -> bool { - if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - { + if let Some(key_file) = usable_endpoint_key_file(paths, service_id) { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); return true; } false } +/// The service's endpoint key file, when it exists and holds a usable key. +fn usable_endpoint_key_file(paths: &AppPaths, service_id: &str) -> Option { + rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) +} + /// Refuse to respawn a recorded service on a public host without its endpoint /// API key, so daemon recovery cannot reopen a protected endpoint anonymously. /// @@ -103,6 +120,12 @@ pub(crate) fn ensure_public_service_has_endpoint_key( Ok(()) } +/// Ask an engine a question about a managed service, through the stdio client +/// `rocmd` shares with `rocm`. +/// +/// The engine is reached by running the `rocm` executable as +/// `__engine-stdio `, the verb `rocm` serves for its built-in engines +/// ([`DaemonHost::rocm_binary`] says which executable that is). pub(crate) fn engine_request( paths: &AppPaths, engine: &str, @@ -114,19 +137,48 @@ where T: Serialize, R: DeserializeOwned, { - let envelope = EngineRequestEnvelope { + engine_request_on( + &DaemonHost::detect()?, + paths, + engine, + service_id, method, - payload: serde_json::to_value(request).context("failed to serialize engine request")?, - }; - let engine_binary = - std::env::current_exe().context("failed to resolve current rocm executable path")?; - let mut command = ProcessCommand::new(&engine_binary); - command - .arg("__engine-stdio") - .arg(engine) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + request, + ) +} + +/// [`engine_request`] on a given host, so a test can supply the binary that +/// answers. The stdio command is `rocm`'s alone, so the daemon runs it on the +/// `rocm` binary, not on whatever it is itself. +fn engine_request_on( + daemon_host: &DaemonHost, + paths: &AppPaths, + engine: &str, + service_id: &str, + method: EngineMethod, + request: &T, +) -> Result +where + T: Serialize, + R: DeserializeOwned, +{ + let envelope = engine_request_envelope(method, request)?; + let engine_binary = daemon_host.rocm_binary()?; + engine_request_via(&engine_binary, paths, engine, service_id, &envelope) +} + +/// [`engine_request`] with the executable that serves `__engine-stdio` named by +/// the caller, so a test can point it at a stand-in. +fn engine_request_via( + engine_binary: &Path, + paths: &AppPaths, + engine: &str, + service_id: &str, + envelope: &rocm_engine_protocol::EngineRequestEnvelope, +) -> Result +where + R: DeserializeOwned, +{ // A protected public endpoint enforces its key on every request, including // this stdio-transported healthcheck — without the carrier the engine // adapter's HTTP probe is unauthenticated and the endpoint looks @@ -135,50 +187,38 @@ where // // No host to check here: this is a stdio plugin call, not a listener, so a // missing key only weakens this probe rather than opening a port. - let _ = apply_endpoint_key_env(&mut command, paths, service_id); - let mut child = command.spawn().with_context(|| { - format!( - "failed to spawn engine stdio process {}", - engine_binary.display() - ) - })?; - - { - let stdin = child - .stdin - .as_mut() - .context("engine stdio child did not expose stdin")?; - serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; - stdin.write_all(b"\n")?; - } + let key_file = usable_endpoint_key_file(paths, service_id); + let options = stdio_options(paths, key_file.as_deref()); + decode_engine_response(run_engine_stdio( + engine_binary, + ["__engine-stdio", engine], + &options, + envelope, + )?) +} - let output = child - .wait_with_output() - .context("failed waiting for engine stdio response")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {}", output.status); - } - bail!( - "engine stdio process exited with status {}: {}", - output.status, - stderr - ); - } - let envelope: EngineResponseEnvelope = - serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; - if !envelope.ok { - let detail = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |error| format!("{}: {}", error.code, error.message), - ); - bail!(detail); +/// What `rocmd` asks of the shared stdio client; see [`engine_request`]. +/// +/// The child runs on the daemon's own app paths and carries the service's +/// endpoint key file when there is a usable one. Its stderr is kept for the +/// error, not forwarded: nobody watches a daemon's terminal, and a healthcheck +/// asks for no progress. +const fn stdio_options<'a>( + paths: &'a AppPaths, + endpoint_key_file: Option<&'a Path>, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: Some(paths), + env_root: None, + endpoint_key_file, + // `None` here means "this loopback service needs no key", not "this + // child may not see one" — these are the daemon's own children, and + // clearing would change what they inherit. + withhold_inherited_key_file: false, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: true, } - let data = envelope - .data - .context("engine response did not include data")?; - serde_json::from_value(data).context("failed to deserialize engine response data") } #[cfg(test)] @@ -341,4 +381,305 @@ mod tests { }; assert!(!healthcheck_response_ready(&listing_only)); } + + // ---- What `rocmd` asks of the shared stdio client. ---- + // + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocmd`'s entry point against a stand-in engine child and pin what + // `rocmd` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocm/src/engine_ipc.rs`. + + #[test] + fn rocmd_runs_the_child_on_its_own_paths_keeps_stderr_and_asks_for_no_progress() { + let (_root, paths) = temp_app_paths("stdio-options"); + let key_file = Path::new("/keys/svc.endpoint-key"); + + let options = stdio_options(&paths, Some(key_file)); + assert_eq!( + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) + ); + assert_eq!(options.endpoint_key_file, Some(key_file)); + // A healthcheck has no install to report progress on, and nobody watches + // a daemon's terminal. + assert!(!options.request_progress); + assert!(!options.forward_stderr); + // A daemon's stderr is its log, so a failing exit after an answer leaves a note there. + assert!(options.note_failing_exit_after_answer); + assert_eq!(options.env_root, None); + + assert_eq!(stdio_options(&paths, None).endpoint_key_file, None); + } + + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::EngineRequestEnvelope; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const SERVICE_ID: &str = "svc-stdio-child"; + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn healthcheck_envelope() -> EngineRequestEnvelope { + EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({"service_id": SERVICE_ID}), + } + } + + fn request(engine: &FakeEngine, paths: &AppPaths) -> Result { + engine_request_via::( + engine.path(), + paths, + "vllm", + SERVICE_ID, + &healthcheck_envelope(), + ) + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_is_asked_to_serve_the_engine_over_stdio_with_one_line_of_request() { + let (_root, paths) = temp_app_paths("stdio-framing"); + let engine = FakeEngine::answering("rocmd-framing", STOPPED, 0); + let answer = request(&engine, &paths).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["__engine-stdio", "vllm"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"healthcheck\",\"payload\":{\"service_id\":\"svc-stdio-child\"}}\n" + ); + } + + #[test] + fn the_endpoint_key_file_reaches_the_child_only_when_it_holds_a_usable_key() { + let (_root, paths) = temp_app_paths("stdio-endpoint-key"); + let key_env = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + + let keyless = FakeEngine::answering("rocmd-keyless", STOPPED, 0); + request(&keyless, &paths).unwrap(); + assert_eq!(keyless.env(key_env), ambient(key_env)); + + let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, SERVICE_ID); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write(&key_path, " \n").unwrap(); + let unusable = FakeEngine::answering("rocmd-unusable-key", STOPPED, 0); + request(&unusable, &paths).unwrap(); + assert_eq!(unusable.env(key_env), ambient(key_env)); + + fs::write(&key_path, "secret-key").unwrap(); + let keyed = FakeEngine::answering("rocmd-keyed", STOPPED, 0); + request(&keyed, &paths).unwrap(); + assert_eq!(keyed.env(key_env).as_deref(), key_path.to_str()); + } + + #[test] + fn the_child_runs_on_the_daemons_app_paths() { + let (_root, paths) = temp_app_paths("stdio-app-paths"); + let engine = FakeEngine::answering("rocmd-app-paths", STOPPED, 0); + request(&engine, &paths).unwrap(); + assert_eq!( + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() + ); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let (_root, paths) = temp_app_paths("stdio-nonzero"); + let answered = FakeEngine::answering("rocmd-answers-then-fails", STOPPED, 2); + assert_eq!( + request(&answered, &paths).unwrap(), + json!({"stopped": true}) + ); + + let failed = FakeEngine::answering( + "rocmd-error-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + assert_eq!( + request(&failed, &paths).unwrap_err().to_string(), + "stop_failed: it broke" + ); + } + + #[test] + fn a_crash_with_no_answer_reports_the_status_and_stderr() { + let (_root, paths) = temp_app_paths("stdio-crash"); + let crash = FakeEngine::new("rocmd-crash", true, "echo boom >&2\nexit 3"); + assert_eq!( + request(&crash, &paths).unwrap_err().to_string(), + "engine stdio process exited with code 3: boom" + ); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let (_root, paths) = temp_app_paths("stdio-garbage"); + let garbage = + FakeEngine::new("rocmd-garbage", true, "printf 'not json'\necho oops >&2"); + assert_eq!( + request(&garbage, &paths).unwrap_err().to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + } + + /// A request to a child that has already exited is an ordinary error + /// wherever SIGPIPE stands: ignored (Rust's startup default, which the + /// test harness keeps) or at its default action, which is how `rocm` + /// runs and so how `rocm daemon` runs this code. Re-executes this + /// test in a fresh process for each case, since changing the disposition + /// process-wide would race sibling tests. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition() { + if let Some(arm) = std::env::var_os("ROCMD_TEST_ENGINE_SIGPIPE_ARM") { + if arm == "default-action" { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + } + let (_root, paths) = temp_app_paths("stdio-sigpipe"); + let engine = FakeEngine::new("rocmd-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let envelope = EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({ "blob": big }), + }; + let result = engine_request_via::( + engine.path(), + &paths, + "vllm", + SERVICE_ID, + &envelope, + ); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let run_arm = |arm: &str| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition", + "--exact", + "--nocapture", + ]) + .env("ROCMD_TEST_ENGINE_SIGPIPE_ARM", arm) + .status() + .unwrap() + }; + + let ignored = run_arm("ignored"); + assert!(ignored.success(), "{ignored:?}"); + + let default_action = run_arm("default-action"); + assert_eq!( + default_action.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {default_action:?}" + ); + assert!(default_action.success(), "{default_action:?}"); + } + } + + /// The stdio command is `rocm`'s alone: a daemon run as `rocmd` asks the + /// engine through the `rocm` beside it, which `rocmd` answered with an + /// unrecognised-subcommand error, so its readiness checks and its engine + /// stops could never succeed. + /// + /// The stand-in `rocm` records its arguments, reads the request line and + /// answers with an `ok` envelope. + #[cfg(target_os = "linux")] + #[test] + fn engine_request_runs_the_stdio_command_on_the_rocm_beside_rocmd() -> Result<()> { + use crate::test_support::{lock_script_tests, script_tools, write_executable_script}; + + let _serial = lock_script_tests(); + let tools = script_tools(); + let (root, paths) = temp_app_paths("engine-request-on-rocm"); + let seen = root.join("rocm-args.txt"); + write_executable_script( + &root.join("rocm"), + &tools, + &format!( + "printf '%s\\n' \"$@\" > '{}'\nread -r _request\n{}\n", + seen.display(), + r#"printf '%s\n' '{"ok":true,"data":{"status":"ready"}}'"# + ), + )?; + let daemon_host = DaemonHost::new(root.join("rocmd"), false); + + // The first exec of a new file can lose a race with an unrelated fork + // (ETXTBSY); the spawn then fails at once, and one more try is enough. + let mut outcome = Err(anyhow::anyhow!("not attempted")); + for _ in 0..10 { + outcome = engine_request_on::<_, serde_json::Value>( + &daemon_host, + &paths, + "vllm", + "svc-engine-request", + EngineMethod::Healthcheck, + &serde_json::json!({ "service_id": "svc-engine-request" }), + ); + match &outcome { + Err(error) if format!("{error:#}").contains("Text file busy") => { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + _ => break, + } + } + let args = fs::read_to_string(&seen).unwrap_or_default(); + fs::remove_dir_all(&root).ok(); + + assert_eq!( + args.lines().collect::>(), + ["__engine-stdio", "vllm"], + "the engine must be asked through `rocm __engine-stdio `" + ); + assert_eq!(outcome?, serde_json::json!({ "status": "ready" })); + Ok(()) + } + + /// With no `rocm` beside `rocmd` the request fails, and says what it looked + /// for, instead of running a command `rocmd` does not have. + #[test] + fn engine_request_without_a_rocm_beside_rocmd_names_it() { + let (root, paths) = temp_app_paths("engine-request-no-rocm"); + let error = engine_request_on::<_, serde_json::Value>( + &DaemonHost::new(root.join("rocmd"), false), + &paths, + "vllm", + "svc-engine-request", + EngineMethod::Healthcheck, + &serde_json::json!({ "service_id": "svc-engine-request" }), + ) + .expect_err("no rocm to ask"); + fs::remove_dir_all(root).ok(); + + let text = format!("{error:#}"); + assert!(text.contains("no engine commands"), "{text}"); + assert!(text.contains("unable to locate sibling binary"), "{text}"); + } } diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 5b7d3d2d88..37ed771631 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -4,9 +4,12 @@ #![allow(clippy::items_after_test_module)] +pub mod anthropic_proxy; +mod artifact_prefetch; mod cli; mod common; mod daemon; +mod daemon_host; mod engine_ipc; mod host_snapshot; mod mcp; diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index 2b8bca1bed..4bab67d0aa 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -18,8 +18,6 @@ use rocm_core::{ }; use serde_json::Value; use serde_json::json; -use std::collections::VecDeque; -use std::fs; use std::io::{self, BufRead, Write}; use std::time::Duration; @@ -631,16 +629,15 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result Result { - let content = - fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut lines = VecDeque::with_capacity(limit); - for line in content.lines() { - if lines.len() == limit { - lines.pop_front(); - } - lines.push_back(line.to_owned()); - } - Ok(lines.into_iter().collect::>().join("\n")) +/// The last `limit` lines of a service log, read through the shared bounded +/// tail routine, with what the byte budget left unread. +fn read_log_tail(path: &std::path::Path, limit: usize) -> Result { + rocm_core::tail_log_decoded( + path, + limit, + rocm_core::LOG_VIEW_MAX_TAIL_READ, + rocm_core::LogDecode::PLAIN, + ) + .with_context(|| format!("failed to read {}", path.display())) } #[cfg(test)] @@ -869,6 +865,12 @@ mod tests { use std::fs; use std::path::PathBuf; + /// [`read_log_tail`], joined with newlines, for the tests that only look + /// at the text. + fn read_tail_lines(path: &std::path::Path, limit: usize) -> Result { + Ok(read_log_tail(path, limit)?.lines.join("\n")) + } + fn unique_test_path(label: &str) -> PathBuf { let root = workspace_test_artifact_dir(); fs::create_dir_all(&root).expect("create workspace-local test dir"); @@ -937,6 +939,152 @@ mod tests { .expect("explicitly approved direct MCP mutation should pass the helper guard"); } + /// Call the `service_logs` tool for a service whose log holds `log` (no log + /// file at all for `None`), and return the tool's reply with the log's size + /// on disk, so a claim the reply makes can be checked against the file. + fn service_logs_reply(label: &str, log: Option<&[u8]>) -> Result<(Value, Option)> { + service_logs_reply_for(label, log, None) + } + + /// [`service_logs_reply`], asking for `lines` lines when given. + fn service_logs_reply_for( + label: &str, + log: Option<&[u8]>, + lines: Option, + ) -> Result<(Value, Option)> { + let (root, paths) = temp_app_paths(label); + paths.ensure()?; + let service_id = format!("svc-{label}"); + let record = rocm_core::ManagedServiceRecord::new( + &paths, + &service_id, + "llamacpp", + "a-model", + "a-model", + "127.0.0.1", + 18080, + "managed", + std::process::id(), + None, + None, + None, + ); + record.write()?; + let size = match log { + Some(bytes) => { + fs::write(&record.log_path, bytes)?; + Some(fs::metadata(&record.log_path)?.len()) + } + None => None, + }; + let mut arguments = json!({ "service_id": service_id }); + if let Some(lines) = lines { + arguments["lines"] = json!(lines); + } + let reply = handle_mcp_tool_call( + &paths, + &json!({"name": "service_logs", "arguments": arguments}), + ); + let _ = fs::remove_dir_all(root); + Ok((reply?, size)) + } + + #[test] + fn service_logs_tell_an_empty_log_from_one_whose_newest_line_exceeds_the_read_bound() + -> Result<()> { + let bound = usize::try_from(rocm_core::LOG_VIEW_MAX_TAIL_READ)?; + let notice = "the newest line is longer than the 4 MiB a log view reads"; + assert_eq!( + rocm_core::LOG_VIEW_MAX_TAIL_READ, + 4 * 1024 * 1024, + "the notice asserted below names this many MiB" + ); + + // An empty log: nothing was written, and the reply says so. + let (reply, size) = service_logs_reply("service-logs-empty", Some(b""))?; + assert_eq!(size, Some(0)); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert_eq!( + text, "Service `svc-service-logs-empty` has no log output yet.", + "{reply}" + ); + assert_eq!(reply["structuredContent"]["tail"], ""); + assert_eq!( + reply["structuredContent"]["newest_line_exceeds_read_bound"], + false + ); + + // One line longer than the bound, with and without a final newline: the + // file has content, so "no output" or "Read the last 0 line(s)" would be + // false, and the structured reply carries the same distinction. + let one_line = vec![b'x'; bound + 1024]; + let mut with_newline = one_line.clone(); + with_newline.push(b'\n'); + for (name, bytes) in [ + ("service-logs-long", one_line), + ("service-logs-long-nl", with_newline), + ] { + let (reply, size) = service_logs_reply(name, Some(&bytes))?; + assert!( + size.is_some_and(|size| size > rocm_core::LOG_VIEW_MAX_TAIL_READ), + "{name}: {size:?}" + ); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert_eq!( + text, + format!("Service `svc-{name}` has log output, but none is shown: {notice}."), + "{name}: {reply}" + ); + assert_eq!(reply["structuredContent"]["tail"], "", "{name}"); + assert_eq!( + reply["structuredContent"]["newest_line_exceeds_read_bound"], true, + "{name}" + ); + assert_eq!(reply["isError"], false, "{name}"); + } + + // Short lines are read, and the count is the number returned. + let (reply, _) = service_logs_reply("service-logs-two", Some(b"a\nb\n"))?; + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert_eq!( + text, + "Read the last 2 line(s) from service `svc-service-logs-two`." + ); + assert_eq!(reply["structuredContent"]["tail"], "a\nb"); + assert_eq!( + reply["structuredContent"]["newest_line_exceeds_read_bound"], + false + ); + + // No log file is an error, not an empty tail. + let error = service_logs_reply("service-logs-absent", None) + .expect_err("a missing log must not read as empty"); + assert!(format!("{error:#}").contains("failed to read"), "{error:#}"); + Ok(()) + } + + #[test] + fn service_logs_report_the_lines_returned_and_the_lines_asked_for_apart() -> Result<()> { + // Asking for 50 lines of a four-line log returns four, and says so in + // the structured reply as well as in the summary. + let (reply, _) = + service_logs_reply_for("service-logs-count", Some(b"a\nb\nc\nd\n"), Some(50))?; + let content = &reply["structuredContent"]; + assert_eq!(content["lines"], 4, "{reply}"); + assert_eq!(content["lines_requested"], 50, "{reply}"); + assert_eq!(content["tail"], "a\nb\nc\nd", "{reply}"); + // A log with more lines than asked for returns exactly what was asked. + let (reply, _) = + service_logs_reply_for("service-logs-count-cut", Some(b"a\nb\nc\nd\n"), Some(2))?; + assert_eq!(reply["structuredContent"]["lines"], 2, "{reply}"); + assert_eq!(reply["structuredContent"]["lines_requested"], 2, "{reply}"); + // An empty log returned none, whatever was asked for. + let (reply, _) = service_logs_reply_for("service-logs-count-empty", Some(b""), Some(50))?; + assert_eq!(reply["structuredContent"]["lines"], 0, "{reply}"); + assert_eq!(reply["structuredContent"]["lines_requested"], 50, "{reply}"); + Ok(()) + } + #[test] fn read_tail_lines_returns_last_lines_only() -> Result<()> { let path = unique_test_path(&format!( @@ -951,6 +1099,151 @@ mod tests { Ok(()) } + fn tail_fixture(label: &str, bytes: &[u8]) -> Result { + let path = unique_test_path(&format!( + "rocmd-tail-{label}-{}.log", + crate::test_support::unique_suffix() + )); + fs::write(&path, bytes)?; + Ok(path) + } + + /// 5 MiB: a marked first line, 5120 lines of 1 KiB, a marked last line. + fn large_log_with_head_marker() -> Vec { + let mut bytes = b"HEAD-MARKER\n".to_vec(); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..5120 { + bytes.extend_from_slice(filler.as_bytes()); + } + bytes.extend_from_slice(b"TAIL-MARKER\n"); + bytes + } + + #[test] + fn read_tail_lines_a_large_file_is_read_through_a_bounded_window() -> Result<()> { + // A line far outside the tail window is not returned however large the + // limit: the log is not read whole into one string. + let path = tail_fixture("large", &large_log_with_head_marker())?; + + let tail = read_tail_lines(&path, 10_000)?; + fs::remove_file(&path)?; + + assert!(!tail.lines().any(|line| line == "HEAD-MARKER")); + assert!(tail.ends_with("\nTAIL-MARKER")); + assert!(tail.lines().count() < 5122); + assert!(tail.lines().count() > 3500, "the window holds about 4 MiB"); + Ok(()) + } + + #[test] + fn read_tail_lines_shows_a_sixteen_bit_log_as_lossy_text_instead_of_failing() -> Result<()> { + // The tool reads the child's own bytes as UTF-8; a UTF-16 log is not + // detected here (only the ComfyUI log view does that), but it no + // longer fails the whole tool call. + let mut bytes = vec![0xff, 0xfe]; + for unit in "[INFO] one\r\n".encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + let path = tail_fixture("utf16", &bytes)?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert!( + tail.starts_with("\u{FFFD}\u{FFFD}["), + "the byte order mark is replaced, the rest kept: {tail:?}" + ); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_ansi_colour_codes() -> Result<()> { + let path = tail_fixture("ansi", b"\x1b[32mgreen\x1b[0m done\n")?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "\u{1b}[32mgreen\u{1b}[0m done"); + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_invalid_eight_bit_text_lossily() -> Result<()> { + let path = tail_fixture("invalid", b"ok\nbad\xffbyte\nlast\n")?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "ok\nbad\u{FFFD}byte\nlast"); + Ok(()) + } + + #[test] + fn read_tail_lines_missing_file_is_a_labelled_read_error() { + let path = unique_test_path(&format!( + "rocmd-tail-missing-{}.log", + crate::test_support::unique_suffix() + )); + + let error = read_tail_lines(&path, 5).expect_err("a missing log must not read"); + + assert_eq!( + error.to_string(), + format!("failed to read {}", path.display()) + ); + assert!( + format!("{error:#}").contains(&format!("failed to open {}", path.display())), + "the cause names the failing step: {error:#}" + ); + } + + #[test] + fn read_tail_lines_limit_zero_returns_nothing() -> Result<()> { + // With limit 0 the old reader never evicted and returned the whole + // file. The tool handler clamps the limit to 1..=500, so only a + // direct call could see it. + let path = tail_fixture("zero", b"a\nb\nc\n")?; + + let tail = read_tail_lines(&path, 0)?; + fs::remove_file(&path)?; + + assert_eq!(tail, ""); + Ok(()) + } + + #[test] + fn read_tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let path = tail_fixture("few", b"a\nb\n")?; + + let tail = read_tail_lines(&path, 50)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "a\nb"); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let path = tail_fixture("no-newline", b"a\nb\nc")?; + + let tail = read_tail_lines(&path, 2)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "b\nc"); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_crlf_line_endings() -> Result<()> { + let path = tail_fixture("crlf", b"a\r\nb\r\n")?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "a\nb"); + Ok(()) + } + fn mcp_stop_server(paths: &AppPaths, service_id: &str) -> Result { handle_mcp_tool_call( paths, diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 01b6d0ab71..1b9bcdfdf5 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use crate::artifact_prefetch::prefetch_artifact_value_with_policy; use crate::common; use crate::persistence::load_managed_services; use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; @@ -9,28 +10,17 @@ use crate::service::{stop_managed_service, stop_report_confirmed}; use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; use rocm_core::{ - AppPaths, AuditEventRecord, CommandCapture, ExamineSummary, ModelRecipeArtifactRecord, - append_audit_event, model_artifact_cache_status, resolve_model_recipe_artifact, - unix_time_millis, + AppPaths, AuditEventRecord, CommandCapture, ExamineSummary, append_audit_event, + resolve_model_recipe_artifact, unix_time_millis, }; use serde_json::Value; use serde_json::json; -#[cfg(test)] -use sha2::Digest; -use std::ffi::{OsStr, OsString}; +#[cfg(any(test, target_os = "linux"))] use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; #[cfg(target_os = "linux")] use std::process::{Command as ProcessCommand, Stdio}; -// Only the tests still park a thread here; the sandbox spawn now goes through -// `rocm_core::process::run_with_timeout`, which owns its own reader threads. -#[cfg(test)] -use std::thread; use std::time::Duration; -const ARTIFACT_PREFETCH_TIMEOUT: Duration = Duration::from_mins(10); - pub(crate) fn run_sandbox_runner( paths: &AppPaths, tool: SandboxToolArg, @@ -297,530 +287,6 @@ pub(crate) fn record_notification_audit( ) } -fn prefetch_artifact_value_with_policy( - paths: &AppPaths, - artifact_ref: &str, - model: &str, - artifact: ModelRecipeArtifactRecord, - policy: SandboxToolPolicy, -) -> Result { - let cache = model_artifact_cache_status(paths, model, &artifact); - if !policy.allow_artifact_download { - return Ok(json!({ - "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), - "artifact_ref": artifact_ref, - "model": model, - "artifact": artifact, - "cache": cache, - "status": "source_policy_required", - "mutating": false, - "network_used": false, - "message": "artifact prefetch requires an approved source policy before network access; no artifact bytes were downloaded", - })); - } - - if cache.marker_path.is_file() { - return Ok(json!({ - "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), - "artifact_ref": artifact_ref, - "model": model, - "artifact": artifact, - "cache": cache, - "status": "cached", - "mutating": false, - "network_used": false, - "message": "artifact cache marker already exists; no network request was made", - })); - } - - if let Some(message) = declared_source_policy_block_message(&artifact) { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - &message, - )); - } - let source_policy_requires_huggingface_auth = - artifact_declares_huggingface_authenticated_policy(&artifact); - let artifact_kind = artifact.kind.to_ascii_lowercase(); - let huggingface_artifact = artifact_is_huggingface(&artifact); - if artifact.gated.unwrap_or(false) && !huggingface_artifact { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "gated artifacts require a source-specific authentication policy before download", - )); - } - if !matches!(artifact_kind.as_str(), "url" | "http" | "https" | "direct") - && !huggingface_artifact - { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "approved live prefetch currently supports direct HTTP(S) artifacts only", - )); - } - if !artifact.uri.starts_with("https://") && !artifact.uri.starts_with("http://") { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - if huggingface_artifact { - "authenticated Hugging Face prefetch requires an HTTP(S) artifact URI in the signed recipe metadata" - } else { - "approved live prefetch requires an HTTP(S) artifact URI" - }, - )); - } - let mut request_headers = Vec::<(&str, String)>::new(); - let source_policy = if huggingface_artifact - && (artifact.gated.unwrap_or(false) - || policy.allow_huggingface_download - || source_policy_requires_huggingface_auth) - { - if !policy.allow_huggingface_download { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "Hugging Face artifacts require --allow-huggingface-download before rocm-cli may use an authentication token", - )); - } - if !artifact.uri.starts_with("https://") { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "rocm-cli will not send a Hugging Face token over plain HTTP; use an HTTPS Hugging Face artifact URI", - )); - } - if !is_huggingface_url(&artifact.uri) { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "rocm-cli will not send a Hugging Face token to a non-Hugging Face URL", - )); - } - let Some(token) = policy - .huggingface_token - .as_deref() - .map(str::trim) - .filter(|token| !token.is_empty()) - else { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "Hugging Face artifact prefetch needs ROCM_CLI_HUGGINGFACE_TOKEN, HF_TOKEN, or HUGGING_FACE_HUB_TOKEN", - )); - }; - request_headers.push(("Authorization", format!("Bearer {token}"))); - "huggingface_authenticated" - } else { - "explicit_allow_artifact_download" - }; - let Some(expected_sha256) = artifact - .sha256 - .as_deref() - .map(|value| value.trim().to_ascii_lowercase()) - .filter(|value| value.len() == 64 && value.chars().all(|ch| ch.is_ascii_hexdigit())) - else { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "approved live prefetch requires a valid sha256 in the recipe artifact metadata", - )); - }; - let Some(size_bytes) = artifact.size_bytes else { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "approved live prefetch requires size_bytes in the recipe artifact metadata", - )); - }; - let max_bytes = policy.artifact_max_bytes.unwrap_or(size_bytes); - if size_bytes > max_bytes { - return Ok(prefetch_blocked_value( - artifact_ref, - model, - artifact, - cache, - "artifact size exceeds the approved download byte limit", - )); - } - - // Streamed straight to its final path: model artifacts run to gigabytes, so - // buffering one to hash it would hold the whole file in memory. The size and - // digest from the recipe are enforced inside the download, which also gives - // this path a free-space preflight and resume on a dropped connection. - let artifact_path = artifact_bytes_path_for_marker(&cache.marker_path); - let mut headers: Vec<(&str, &str)> = vec![("User-Agent", "rocm-cli")]; - headers.extend( - request_headers - .iter() - .map(|(name, value)| (*name, value.as_str())), - ); - let outcome = rocm_core::download_file_streaming(&rocm_core::DownloadRequest { - url: &artifact.uri, - destination: &artifact_path, - timeout: ARTIFACT_PREFETCH_TIMEOUT, - headers: &headers, - max_bytes: Some(max_bytes), - expected_len: Some(size_bytes), - expected_sha256: Some(&expected_sha256), - }) - .with_context(|| format!("failed to prefetch artifact `{artifact_ref}`"))?; - let actual_sha256 = outcome.sha256; - write_file_atomically( - &cache.marker_path, - &serde_json::to_vec_pretty(&json!({ - "artifact_ref": artifact_ref, - "model": model, - "artifact": artifact, - "bytes_path": artifact_path, - "size_bytes": size_bytes, - "sha256": actual_sha256, - "prefetched_at_unix_ms": unix_time_millis(), - "source_policy": source_policy, - })) - .context("failed to serialize artifact cache marker")?, - )?; - let cache = model_artifact_cache_status(paths, model, &artifact); - Ok(json!({ - "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), - "artifact_ref": artifact_ref, - "model": model, - "artifact": artifact, - "cache": cache, - "status": "prefetched", - "mutating": true, - "network_used": true, - "bytes_path": artifact_path, - "size_bytes": size_bytes, - "sha256": actual_sha256, - "source_policy": source_policy, - "message": "artifact downloaded and verified with recipe sha256", - })) -} - -fn declared_source_policy_block_message(artifact: &ModelRecipeArtifactRecord) -> Option { - let source_policy = artifact.source_policy.as_ref()?; - if !source_policy.required_hosts.is_empty() { - let Some(host) = http_url_host(&artifact.uri) else { - return Some( - "recipe source policy declares required hosts but the artifact URI is not HTTP(S)" - .to_owned(), - ); - }; - if !source_policy - .required_hosts - .iter() - .any(|required| required.eq_ignore_ascii_case(&host)) - { - return Some(format!( - "artifact host `{host}` is not allowed by the recipe source policy" - )); - } - } - - match source_policy.policy.as_str() { - "direct_https_sha256" => { - if artifact.uri.starts_with("https://") { - None - } else { - Some( - "recipe source policy direct_https_sha256 requires an HTTPS artifact URI" - .to_owned(), - ) - } - } - "huggingface_public" => { - if artifact.gated.unwrap_or(false) { - Some( - "recipe source policy marks this as public Hugging Face, but the artifact is gated" - .to_owned(), - ) - } else { - huggingface_policy_uri_block_message(artifact, "huggingface_public") - } - } - "huggingface_authenticated" => { - huggingface_policy_uri_block_message(artifact, "huggingface_authenticated") - } - "manual_only" => Some( - "recipe source policy marks this artifact as manual-only; rocm-cli will not download it" - .to_owned(), - ), - other => Some(format!( - "recipe source policy `{other}` is not supported by this rocm-cli build" - )), - } -} - -fn huggingface_policy_uri_block_message( - artifact: &ModelRecipeArtifactRecord, - policy: &str, -) -> Option { - if !artifact.uri.starts_with("https://") { - return Some(format!( - "recipe source policy {policy} requires an HTTPS Hugging Face artifact URI" - )); - } - if !is_huggingface_url(&artifact.uri) { - return Some(format!( - "recipe source policy {policy} requires a Hugging Face artifact URI" - )); - } - None -} - -fn artifact_declares_huggingface_authenticated_policy( - artifact: &ModelRecipeArtifactRecord, -) -> bool { - artifact - .source_policy - .as_ref() - .is_some_and(|policy| policy.policy == "huggingface_authenticated") -} - -fn artifact_is_huggingface(artifact: &ModelRecipeArtifactRecord) -> bool { - let kind = artifact.kind.to_ascii_lowercase(); - matches!(kind.as_str(), "huggingface" | "hf" | "hugging_face") - || is_huggingface_url(&artifact.uri) -} - -fn is_huggingface_url(url: &str) -> bool { - http_url_host(url).is_some_and(|host| { - host == "huggingface.co" - || host.ends_with(".huggingface.co") - || host == "hf.co" - || host.ends_with(".hf.co") - }) -} - -fn http_url_host(url: &str) -> Option { - let rest = url - .strip_prefix("https://") - .or_else(|| url.strip_prefix("http://"))?; - let authority = rest - .split(['/', '?', '#']) - .next() - .unwrap_or_default() - .trim(); - if authority.is_empty() || authority.contains('@') { - return None; - } - let host = authority - .strip_prefix('[') - .and_then(|value| value.split_once(']').map(|(host, _)| host)) - .unwrap_or_else(|| authority.split(':').next().unwrap_or_default()) - .trim() - .trim_end_matches('.') - .to_ascii_lowercase(); - (!host.is_empty()).then_some(host) -} - -fn prefetch_blocked_value( - artifact_ref: &str, - model: &str, - artifact: ModelRecipeArtifactRecord, - cache: rocm_core::ModelArtifactCacheStatus, - reason: &str, -) -> Value { - json!({ - "tool": SandboxToolArg::PrefetchArtifact.as_cli_value(), - "artifact_ref": artifact_ref, - "model": model, - "artifact": artifact, - "cache": cache, - "status": "blocked", - "mutating": false, - "network_used": false, - "message": reason, - }) -} - -fn artifact_bytes_path_for_marker(marker_path: &Path) -> PathBuf { - marker_path.with_extension("bin") -} - -/// Hex digest of a buffer. Production hashing happens incrementally inside the -/// streaming download; this exists only so tests can state an expected digest. -#[cfg(test)] -fn sha256_hex(bytes: &[u8]) -> String { - format!("{:x}", sha2::Sha256::digest(bytes)) -} - -const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; - -fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { - let parent = path.parent().context("file path has no parent directory")?; - let mut file_name = path - .file_name() - .context("file path has no file name")? - .to_os_string(); - file_name.push(".tmp-"); - file_name.push(suffix); - Ok(parent.join(file_name)) -} - -fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - write_file_atomically_with( - path, - bytes, - |attempt| OsString::from(format!("{temp_id}-{attempt}")), - || {}, - ) -} - -fn write_file_atomically_with( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), -{ - write_file_atomically_with_publish( - path, - bytes, - suffix_for_attempt, - before_publish, - publish_temp_file, - ) -} - -fn write_file_atomically_with_publish( - path: &Path, - bytes: &[u8], - mut suffix_for_attempt: S, - before_publish: P, - publish: F, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - let parent = path.parent().context("file path has no parent directory")?; - fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; - - let mut reserved = None; - for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { - let suffix = suffix_for_attempt(attempt); - let tmp = temp_sibling_path(path, &suffix)?; - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - { - Ok(file) => { - reserved = Some((tmp, file)); - break; - } - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} - Err(error) => { - return Err(error).with_context(|| format!("failed to create {}", tmp.display())); - } - } - } - let Some((tmp, mut file)) = reserved else { - bail!( - "failed to reserve a temporary file next to {} after {} attempts", - path.display(), - ATOMIC_WRITE_TEMP_ATTEMPTS - ); - }; - - if let Err(error) = file.write_all(bytes) { - drop(file); - let _ = fs::remove_file(&tmp); - return Err(error).with_context(|| format!("failed to write {}", tmp.display())); - } - drop(file); - before_publish(); - - publish(&tmp, path).inspect_err(|_| { - let _ = fs::remove_file(&tmp); - })?; - Ok(()) -} - -#[cfg(not(windows))] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } -} - pub(crate) fn sandbox_check_updates_value(output: CommandCapture) -> Value { let status = update_check_status(&output); let update_available = @@ -993,230 +459,21 @@ mod tests { use super::*; use crate::persistence::load_service_record; use crate::test_support::{ - assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, - temp_app_paths, + RecipeIndexEnv, assert_unconfirmed_stop_kept_the_service, seed_keyed_service, + stop_with_outcome, temp_app_paths, }; use rocm_core::ManagedServiceRecord; - use std::io::Read; - - fn serve_one_http_response(bytes: Vec) -> Result { - let listener = std::net::TcpListener::bind(("127.0.0.1", 0))?; - let addr = listener.local_addr()?; - thread::spawn(move || { - if let Ok((mut stream, _)) = listener.accept() { - let mut request = [0_u8; 1024]; - let _ = stream.read(&mut request); - let header = format!( - "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", - bytes.len() - ); - let _ = stream.write_all(header.as_bytes()); - let _ = stream.write_all(&bytes); - } - }); - Ok(format!("http://{addr}/artifact.bin")) - } - - /// Regression: a failed write must not leave a `.tmp-*` scratch file - /// behind. The name is unique per attempt, so before this an orphan - /// accumulated per retry — and when the failure is a full disk, those - /// orphans are exactly what keeps it full. - /// - /// Provokes the failure by pointing the destination at a non-empty - /// directory: the temp file is written, then neither the rename nor the - /// replace fallback can succeed. Portable, unlike an out-of-space test. - #[test] - fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("manifest.json"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - write_file_atomically(&occupied, b"payload") - .expect_err("renaming onto a non-empty directory should fail"); - - let leftovers: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!( - leftovers.is_empty(), - "failed write left temp files behind: {leftovers:?}" - ); - } - - /// The temp name keeps every extension, so a cleanup sweep over a cache - /// directory can still tell what a leftover was going to be. - #[test] - fn write_file_atomically_temp_name_preserves_multi_dot_file_names() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-name-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("artifact.tar.gz"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - // Fails after the temp file exists, so the observed name is the real one. - write_file_atomically(&occupied, b"payload").expect_err("rename should fail"); - - let names: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!( - names, - vec!["artifact.tar.gz".to_owned()], - "only the occupied destination should remain" - ); - } - - #[test] - fn concurrent_atomic_writes_do_not_remove_a_published_destination() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("manifest.json"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .enumerate() - .map(|(writer, bytes)| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - write_file_atomically_with( - &destination, - bytes, - |attempt| { - if attempt == 0 { - OsString::from("same-millisecond") - } else { - OsString::from(format!("same-millisecond-{writer}-{attempt}")) - } - }, - || { - before_publish.wait(); - }, - ) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a writer must remain published"); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - } - - #[test] - fn failed_atomic_replace_preserves_destination_and_cleans_temp() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-replace-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("manifest.json"); - fs::write(&destination, b"published").unwrap(); - - write_file_atomically_with_publish( - &destination, - b"replacement", - |attempt| OsString::from(format!("replace-failure-{attempt}")), - || {}, - |tmp, path| { - assert_eq!(fs::read(tmp).unwrap(), b"replacement"); - assert_eq!(path, destination); - Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "simulated atomic replacement failure", - )) - }, - ) - .expect_err("simulated replacement failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!(leftovers, vec![OsString::from("manifest.json")]); - } - - #[cfg(unix)] - #[test] - fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - let file_name = OsString::from_vec(b"manifest-\xff.json".to_vec()); - let destination = Path::new("/tmp").join(&file_name); - let temp = temp_sibling_path(&destination, std::ffi::OsStr::new("collision")).unwrap(); - - let mut expected = file_name.into_vec(); - expected.extend_from_slice(b".tmp-collision"); - assert_eq!(temp.file_name().unwrap().as_bytes(), expected); - } - - /// Mirrors the `/dev/shm` reproduction from the original report: a genuine - /// ENOSPC, not a rename failure standing in for one. - /// - /// Ignored by default because it fills `/dev/shm`, which is shared with - /// anything else on the host, so it is not safe to run concurrently. - #[test] - #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] - fn write_file_atomically_cleans_up_temp_on_write_failure() { - let shm = Path::new("/dev/shm"); - if !shm.is_dir() { - eprintln!("skipping: /dev/shm unavailable"); - return; - } - let dir = shm.join(format!("rocmd-enospc-{}", std::process::id())); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - let dest = dir.join("manifest.json"); - // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. - let payload = vec![b'x'; 256 * 1024 * 1024]; - - write_file_atomically(&dest, &payload) - .expect_err("writing past the end of the filesystem should fail"); - let leftovers: Vec = fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - let destination_exists = dest.exists(); - let _ = fs::remove_dir_all(&dir); - - assert!( - leftovers.is_empty(), - "failed write left files behind: {leftovers:?}" - ); - assert!(!destination_exists); - } + #[cfg(target_os = "linux")] + use std::path::Path; #[test] fn sandbox_check_updates_value_is_read_only_and_preserves_output() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release status=up_to_date\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release status=up_to_date\n".to_owned(), + String::new(), + )); assert_eq!( value.get("tool").and_then(Value::as_str), @@ -1240,12 +497,12 @@ mod tests { #[test] fn sandbox_check_updates_value_marks_runtime_update_available() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), + String::new(), + )); assert_eq!( value.get("tool").and_then(Value::as_str), @@ -1279,12 +536,12 @@ mod tests { #[test] fn sandbox_check_updates_value_marks_runtime_repair_available() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release-wheel-multi-arch-7-14-0 status=repair_available installed=7.14.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release-wheel-multi-arch-7-14-0 status=repair_available installed=7.14.0 latest=7.14.0\n".to_owned(), + String::new(), + )); assert_eq!( value.get("status").and_then(Value::as_str), @@ -1307,12 +564,12 @@ mod tests { #[test] fn a_newer_version_outranks_a_composition_repair_in_the_watcher_report() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime old status=repair_available installed=7.14.0 latest=7.14.0\n runtime stale status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime old status=repair_available installed=7.14.0 latest=7.14.0\n runtime stale status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), + String::new(), + )); assert_eq!( value.get("status").and_then(Value::as_str), @@ -1322,18 +579,18 @@ mod tests { #[test] fn sandbox_driver_plan_value_is_read_only_and_preserves_output() { - let value = sandbox_driver_plan_value(CommandCapture { - argv: vec![ + let value = sandbox_driver_plan_value(CommandCapture::from_text( + vec![ "rocm".to_owned(), "install".to_owned(), "driver".to_owned(), "--dkms".to_owned(), "--dry-run".to_owned(), ], - exit_status: 0, - stdout: "driver install plan\n supported: true\n".to_owned(), - stderr: String::new(), - }); + 0, + "driver install plan\n supported: true\n".to_owned(), + String::new(), + )); assert_eq!( value.get("tool").and_then(Value::as_str), @@ -1673,52 +930,12 @@ mod tests { ); } - #[test] - fn sandbox_prefetch_reports_policy_required_without_network() { - let (root, paths) = temp_app_paths("sandbox-prefetch-policy"); - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#hf-main", - "Qwen/Test-1B", - ModelRecipeArtifactRecord { - artifact_id: "hf-main".to_owned(), - kind: "huggingface".to_owned(), - uri: "Qwen/Test-1B".to_owned(), - revision: Some("main".to_owned()), - sha256: Some("a".repeat(64)), - size_bytes: Some(1024), - license: Some("apache-2.0".to_owned()), - gated: Some(false), - quantization: Some("bf16".to_owned()), - engines: vec!["vllm".to_owned()], - source_policy: None, - }, - SandboxToolPolicy::default(), - ) - .expect("policy value should render"); - fs::remove_dir_all(root).ok(); - - assert_eq!( - value.get("status").and_then(Value::as_str), - Some("source_policy_required") - ); - assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(false)); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("cache") - .and_then(|cache| cache.get("status")) - .and_then(Value::as_str) - .is_some_and(|status| status == "missing") - ); - } - #[test] fn sandbox_prefetch_unknown_artifact_ref_errors() { let (root, paths) = temp_app_paths("sandbox-prefetch-unknown-ref"); + // The registry is read from the process environment, which the dispatch + // test below sets, so resolve against the built-in catalog under the guard. + let _env = RecipeIndexEnv::builtin_catalog(); let error = run_sandbox_tool( &paths, SandboxToolArg::PrefetchArtifact, @@ -1733,545 +950,111 @@ mod tests { assert!(error.to_string().contains("was not found"), "{error:#}"); } + /// Drives `prefetch_artifact` through `run_sandbox_tool` with an artifact + /// reference that the recipe registry resolves (the built-in catalog lists no + /// artifacts, so a signed recipe index supplies one). The callee's own tests + /// in `artifact_prefetch.rs` call it directly, and the two tests above stop + /// before it, so this is the one test that covers the dispatch seam: the + /// reference resolves to its recipe's canonical model id, and the caller's + /// policy reaches the policy decision. #[test] - fn sandbox_prefetch_cached_marker_skips_network() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-cached"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "direct-bin".to_owned(), - kind: "url".to_owned(), - uri: "https://example.invalid/should-not-be-fetched.bin".to_owned(), - revision: None, - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(false), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - let cache = model_artifact_cache_status(&paths, "Qwen/Test-1B", &artifact); - write_file_atomically(&cache.marker_path, br#"{"cached":true}"#)?; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#direct-bin", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("cached")); - assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(false)); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_blocks_artifact_larger_than_approved_limit() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-byte-limit"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "direct-bin".to_owned(), - kind: "url".to_owned(), - uri: "https://example.invalid/too-large.bin".to_owned(), - revision: None, - sha256: Some("a".repeat(64)), - size_bytes: Some(2048), - license: Some("test-only".to_owned()), - gated: Some(false), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#direct-bin", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("byte limit")) - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_blocks_non_direct_non_huggingface_source() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-non-direct"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "torrent-bin".to_owned(), - kind: "torrent".to_owned(), - uri: "https://example.invalid/artifact.torrent".to_owned(), - revision: None, - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(false), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#torrent-bin", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("direct HTTP(S)")) - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_blocks_gated_huggingface_without_source_policy() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-hf-policy"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "hf-main".to_owned(), - kind: "huggingface".to_owned(), - uri: "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors".to_owned(), - revision: Some("main".to_owned()), - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(true), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#hf-main", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("--allow-huggingface-download")) - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_blocks_huggingface_source_policy_without_token() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-hf-token"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "hf-main".to_owned(), - kind: "huggingface".to_owned(), - uri: "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors".to_owned(), - revision: Some("main".to_owned()), - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(true), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#hf-main", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - allow_huggingface_download: true, - huggingface_token: None, - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("ROCM_CLI_HUGGINGFACE_TOKEN")) - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_respects_manual_only_source_policy() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-manual-policy"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "manual-bin".to_owned(), - kind: "url".to_owned(), - uri: "https://example.invalid/manual.bin".to_owned(), - revision: None, - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(false), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: Some(rocm_core::ModelRecipeArtifactSourcePolicyRecord { - policy: "manual_only".to_owned(), - required_hosts: Vec::new(), - notes: vec!["requires manual license review".to_owned()], - }), - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#manual-bin", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("manual-only")) - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_respects_declared_huggingface_auth_policy() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-hf-declared-auth"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "hf-main".to_owned(), - kind: "huggingface".to_owned(), - uri: "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors".to_owned(), - revision: Some("main".to_owned()), - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(false), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: Some(rocm_core::ModelRecipeArtifactSourcePolicyRecord { - policy: "huggingface_authenticated".to_owned(), - required_hosts: vec!["huggingface.co".to_owned()], - notes: Vec::new(), - }), - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#hf-main", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, + fn sandbox_prefetch_resolves_artifact_and_applies_policy_through_run_sandbox_tool() -> Result<()> + { + let (root, paths) = temp_app_paths("sandbox-prefetch-dispatch"); + paths.ensure()?; + let index = root.join("recipes.json"); + let signature = root.join("recipes.json.sig"); + let public_key = root.join("recipes-public.pem"); + let document = json!({ + "schema_version": 1, + "source": "sandbox-prefetch-dispatch-test", + "recipes": [{ + "canonical_model_id": "Dispatch/Test-1B", + "aliases": [], + "task": "chat", + "source": "signed_recipe_index", + "revision": "main", + "loader": "transformers", + "trust_remote_code": false, + "dtype": "float16", + "device_policy": "gpu_required", + "artifacts": [{ + "artifact_id": "direct-bin", + "kind": "url", + "uri": "https://example.invalid/should-not-be-fetched.bin", + "revision": null, + "sha256": "a".repeat(64), + "size_bytes": 12, + "license": "test-only", + "gated": false, + "quantization": null, + "engines": ["vllm"] + }], + "engine_recipes": [], + "manual_alternatives": [], + "featured": false, + "chat_template_mode": "auto", + "preferred_engines": ["vllm"], + "warnings": [] + }] + }); + let payload = serde_json::to_vec_pretty(&document)?; + let (private_pem, public_pem) = rocm_core::generate_rsa_signing_keypair()?; + fs::write(&index, &payload)?; + fs::write( + &signature, + rocm_core::sign_rsa_pkcs1_sha256_signature(&private_pem, &payload)?, )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("--allow-huggingface-download")) - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_never_sends_huggingface_token_to_non_huggingface_url() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-hf-host"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "hf-main".to_owned(), - kind: "huggingface".to_owned(), - uri: "https://example.invalid/Qwen/Test-1B/model.safetensors".to_owned(), - revision: Some("main".to_owned()), - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(true), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, + fs::write(&public_key, public_pem)?; + + let run = |policy: SandboxToolPolicy| { + let _env = RecipeIndexEnv::signed_index(&index, &signature, &public_key); + run_sandbox_tool( + &paths, + SandboxToolArg::PrefetchArtifact, + None, + Some("Dispatch/Test-1B#direct-bin".to_owned()), + None, + policy, + ) }; - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#hf-main", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - allow_huggingface_download: true, - huggingface_token: Some("hf_secret_should_not_leak".to_owned()), - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); + // No download approval: the reference resolves, the answer names the + // recipe's canonical model, and nothing is fetched or cached. + let blocked = run(SandboxToolPolicy::default())?; + let marker_path = std::path::PathBuf::from( + blocked["cache"]["marker_path"] + .as_str() + .expect("the report names the cache marker path"), + ); + assert_eq!(blocked["status"], "source_policy_required", "{blocked}"); + assert_eq!(blocked["model"], "Dispatch/Test-1B", "{blocked}"); + assert_eq!(blocked["artifact_ref"], "Dispatch/Test-1B#direct-bin"); + assert_eq!(blocked["artifact"]["artifact_id"], "direct-bin"); + assert_eq!(blocked["network_used"], false); + assert_eq!(blocked["mutating"], false); assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("non-Hugging Face URL")) - ); - assert!( - !serde_json::to_string(&value)?.contains("hf_secret_should_not_leak"), - "prefetch report must not include authentication tokens" - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_never_sends_huggingface_token_over_plain_http() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-hf-https"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "hf-main".to_owned(), - kind: "huggingface".to_owned(), - uri: "http://huggingface.co/Qwen/Test-1B/model.safetensors".to_owned(), - revision: Some("main".to_owned()), - sha256: Some("a".repeat(64)), - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(true), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#hf-main", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - allow_huggingface_download: true, - huggingface_token: Some("hf_secret_should_not_leak".to_owned()), - }, - )?; + !marker_path.exists(), + "a blocked prefetch must not write the cache marker" + ); + + // Download approved and a marker already present: the policy the caller + // passed is the one that is applied, so the answer is `cached`, still + // without a network request. + rocm_core::atomic_write::write_file_atomically(&marker_path, br#"{"cached":true}"#)?; + let cached = run(SandboxToolPolicy { + allow_artifact_download: true, + artifact_max_bytes: Some(1024), + ..SandboxToolPolicy::default() + })?; fs::remove_dir_all(root).ok(); - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("plain HTTP")) - ); - assert!( - !serde_json::to_string(&value)?.contains("hf_secret_should_not_leak"), - "prefetch report must not include authentication tokens" - ); - Ok(()) - } - - #[test] - fn sandbox_prefetch_downloads_direct_artifact_when_approved() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-download"); - let bytes = b"tiny verified artifact".to_vec(); - let uri = serve_one_http_response(bytes.clone())?; - let sha256 = sha256_hex(&bytes); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "direct-bin".to_owned(), - kind: "url".to_owned(), - uri, - revision: None, - sha256: Some(sha256.clone()), - size_bytes: Some(bytes.len() as u64), - license: Some("test-only".to_owned()), - gated: Some(false), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#direct-bin", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, - )?; - let bytes_path = value - .get("bytes_path") - .and_then(Value::as_str) - .map(PathBuf::from) - .expect("bytes_path should be present"); - let marker_path = value - .get("cache") - .and_then(|cache| cache.get("marker_path")) - .and_then(Value::as_str) - .map(PathBuf::from) - .expect("marker path should be present"); - - assert_eq!( - value.get("status").and_then(Value::as_str), - Some("prefetched") - ); - assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(true)); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(true) - ); - assert_eq!( - value.get("sha256").and_then(Value::as_str), - Some(sha256.as_str()) - ); - assert_eq!( - value.get("source_policy").and_then(Value::as_str), - Some("explicit_allow_artifact_download") - ); - assert_eq!(fs::read(&bytes_path)?, bytes); - assert!(marker_path.is_file()); - fs::remove_dir_all(root).ok(); + assert_eq!(cached["status"], "cached", "{cached}"); + assert_eq!(cached["model"], "Dispatch/Test-1B", "{cached}"); + assert_eq!(cached["network_used"], false); Ok(()) } - #[test] - fn sandbox_prefetch_blocks_approved_artifact_without_sha256() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-prefetch-no-sha"); - let artifact = ModelRecipeArtifactRecord { - artifact_id: "direct-bin".to_owned(), - kind: "url".to_owned(), - uri: "https://example.invalid/artifact.bin".to_owned(), - revision: None, - sha256: None, - size_bytes: Some(12), - license: Some("test-only".to_owned()), - gated: Some(false), - quantization: None, - engines: vec!["vllm".to_owned()], - source_policy: None, - }; - - let value = prefetch_artifact_value_with_policy( - &paths, - "qwen#direct-bin", - "Qwen/Test-1B", - artifact, - SandboxToolPolicy { - allow_artifact_download: true, - artifact_max_bytes: Some(1024), - ..SandboxToolPolicy::default() - }, - )?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("blocked")); - assert_eq!( - value.get("network_used").and_then(Value::as_bool), - Some(false) - ); - assert!( - value - .get("message") - .and_then(Value::as_str) - .is_some_and(|message| message.contains("sha256")) - ); - Ok(()) - } - - #[test] - fn huggingface_url_detection_is_host_scoped() { - assert!(is_huggingface_url( - "https://huggingface.co/Qwen/Test-1B/resolve/main/model.safetensors" - )); - assert!(is_huggingface_url( - "https://cdn-lfs.huggingface.co/repos/example" - )); - assert!(is_huggingface_url("https://hf.co/Qwen/Test-1B")); - assert!(!is_huggingface_url( - "https://huggingface.co.evil.example/Qwen/Test-1B" - )); - assert!(!is_huggingface_url( - "https://huggingface.co@evil.example/Qwen/Test-1B" - )); - } - #[test] fn sandbox_tool_stop_server_updates_manifest_and_skips_current_pid() -> Result<()> { let (root, paths) = temp_app_paths("sandbox-stop-current-pid"); diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 41e7fcf844..c0579a668c 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -7,12 +7,9 @@ use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use serde_json::Value; use serde_json::json; -#[cfg(windows)] -use std::process::{Command as ProcessCommand, Stdio}; -use std::time::Duration; pub(crate) fn stop_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_managed_service_with(paths, service_id, terminate_recorded_pid) + stop_managed_service_with(paths, service_id, rocm_core::terminate_recorded_pid) } fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result { @@ -23,8 +20,8 @@ fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result Result { let mut stopping = find_managed_service(paths, service_id)?; // Record the request before the stop starts, not only after it ends. A stop - // can take up to `MANAGED_STOP_GRACE` twice per recorded PID, and for that + // can take up to `rocm_core::MANAGED_STOP_GRACE` twice per recorded PID, and for that // whole window the service still reads `ready`/`running` while its endpoint // stops answering — exactly what the daemon's `server-recover` watcher // restarts. The marker is what that watcher skips (`stop_requested`), so it @@ -73,32 +70,27 @@ pub(crate) fn stop_managed_service_via( // container-backed engine (`aim`) this is the only thing that stops the // container, since its recorded PIDs are only the supervisor that follows it. let engine_stop = (stopping.engine != "lemonade").then(|| engine_stop(paths, &stopping)); - let stops = terminate_recorded_service_pids(&stopping, terminate); - let pids_where = |keep: fn(&RecordedPidStop) -> bool| -> Vec { + let stops = rocm_core::terminate_recorded_service_pids(&stopping, terminate); + let pids_where = |keep: fn(rocm_core::RecordedPidStop) -> bool| -> Vec { stops .iter() - .filter(|stop| keep(stop)) - .map(|s| s.pid) + .copied() + .filter(|stop| keep(*stop)) + .map(|stop| stop.pid) .collect() }; - let signaled_pids = pids_where(|stop| stop.signaled); - let force_signaled_pids = pids_where(|stop| stop.forced); + let signaled_pids = pids_where(rocm_core::RecordedPidStop::signaled); + let force_signaled_pids = pids_where(rocm_core::RecordedPidStop::forced); // Everything that was deliberately not signalled, which is no longer one // kind of thing: the stopping process itself; a PID that was already gone; // a PID that is live but provably belongs to somebody else now; and a live // PID whose identity could not be read at all, which may still be the engine // and may still hold the device. Only `pid_outcomes` separates them, and // only `stopped` reports that the last of those is not a completed stop. - let skipped_pids = pids_where(|stop| !stop.signaled); + let skipped_pids = pids_where(|stop| !stop.signaled()); let pid_outcomes = stops .iter() - .map(|stop| { - json!({ - "pid": stop.pid, - "role": stop.role, - "outcome": stop.outcome, - }) - }) + .map(|stop| stop.report_entry()) .collect::>(); // Re-read the record rather than write back the copy loaded before the // stop: `write` replaces the whole file, and other writers — a supervisor @@ -113,8 +105,8 @@ pub(crate) fn stop_managed_service_via( // would each be false. let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.as_ref().is_some_and(Result::is_ok); - let all_stopped = stops.iter().all(|stop| stop.stopped) - && stopping.names_same_processes(&record) + let all_stopped = stops.iter().all(|stop| stop.stopped()) + && record.is_covered_by_stop_of(&stopping) && engine_stop_confirms; // A PID whose identity cannot be read at all — `/proc//stat` // unreadable while `kill(pid, 0)` still answers: another uid, `hidepid`, an @@ -135,11 +127,9 @@ pub(crate) fn stop_managed_service_via( record.stop_requested_unix_ms = None; // Recorded PIDs go stale the instant their processes exit, and a stale // PID is exactly what the identity check exists to catch — so do not - // leave one behind for the next stop to find. - record.supervisor_pid = 0; - record.supervisor_start_ticks = None; - record.engine_pid = None; - record.engine_start_ticks = None; + // leave one behind for the next stop to find. The same call + // `rocm services stop` makes on its confirmed path. + record.clear_recorded_processes(); } else { // The recorded PIDs stay: an unconfirmed survivor is still the // service's, and the record is what names it. @@ -208,179 +198,6 @@ pub(crate) fn stop_report_confirmed(report: &Value) -> bool { report.get("stopped").and_then(Value::as_bool) == Some(true) } -/// How long a stop waits for a recorded process tree to exit on its own before -/// escalating to a forced kill. Matches `rocm services stop`, which terminates -/// the same records. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// What a stop did about one PID recorded in a service manifest. -struct RecordedPidStop { - pid: u32, - /// Which recorded PID this is: `supervisor`, `engine`, or - /// `supervisor_and_engine` when the record holds the same PID for both. - role: &'static str, - /// Stable label for what happened, from - /// [`rocm_core::TerminationOutcome::as_str`], plus `self` for the stopping - /// process's own PID. - outcome: &'static str, - signaled: bool, - forced: bool, - /// Whether the recorded process is confirmed to be no longer running. - stopped: bool, -} - -/// Terminate the processes a service manifest records, verifying identity first. -/// -/// PIDs are recycled, so a persisted PID is not by itself evidence that the -/// recorded process is still the one holding it. Each PID is therefore paired -/// with **its own** start-time token — `supervisor_start_ticks` with -/// `supervisor_pid`, `engine_start_ticks` with `engine_pid` — and routed through -/// [`rocm_core::terminate_verified`], which signals nothing when the recorded -/// identity is refuted. `rocm services stop` reads the same records through the -/// same primitive, so neither command can decide differently about one. -/// -/// Descendants are reached from the *verified* root rather than expanded out of -/// a live process listing keyed on a PID nobody checked — which, given a stale -/// root, enumerated a stranger's children and signalled those too. Where the -/// kernel start-time is readable, `terminate_verified` additionally binds each -/// child to its own identity before any forced kill; see -/// [`terminate_recorded_pid`] for what Windows can and cannot do here. -/// -/// The stopping process's own PID is reported but never signalled, and does not -/// count towards "the recorded service is gone": `rocmd` records itself as the -/// supervisor of the services it launches, and it is the launcher, not the -/// service. -/// -/// `terminate` performs the verified termination of one recorded PID; -/// production passes [`terminate_recorded_pid`]. -fn terminate_recorded_service_pids( - record: &ManagedServiceRecord, - terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, -) -> Vec { - // Build the (pid, role, own-token) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. The two PIDs can - // coincide — a launcher that is also the server — and a token must never be - // read across from the PID it does not belong to. - let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); - for (pid, role, ticks) in [ - ( - Some(record.supervisor_pid), - "supervisor", - record.supervisor_start_ticks, - ), - (record.engine_pid, "engine", record.engine_start_ticks), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - // Only the engine entry can land here, aliasing the supervisor: the - // one PID is both, and is reported as both. - existing.1 = "supervisor_and_engine"; - if existing.2.is_none() { - existing.2 = ticks; - } - } else { - entries.push((pid, role, ticks)); - } - } - - entries - .into_iter() - .map(|(pid, role, ticks)| { - if pid == std::process::id() { - return RecordedPidStop { - pid, - role, - outcome: "self", - signaled: false, - forced: false, - stopped: true, - }; - } - let outcome = terminate(&rocm_core::ProcessIdentity::new(pid, ticks)); - RecordedPidStop { - pid, - role, - outcome: outcome.as_str(), - // TimedOut belongs with the signalled: the signal went out, the - // exit was just never observed. `stopped` carries that truth. - signaled: matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ), - // "The stop escalated to a forced kill", not "the forced kill - // worked": a forced stop only reaches `TimedOut` by way of that - // escalation, and `stopped` is what says whether the process - // actually went away. - forced: matches!( - outcome, - rocm_core::TerminationOutcome::Forced | rocm_core::TerminationOutcome::TimedOut - ), - stopped: outcome.stopped(), - } - }) - .collect() -} - -/// Terminate one recorded process after verifying its identity. -/// -/// The stop is forced: its contract is definitive, so the command must not -/// report success while an engine worker still holds the GPU. Only a process -/// whose recorded identity is confirmed is ever reached by it. -fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome { - // Windows needs the descendants taken separately. `rocm_core` has no way to - // walk a process tree there — `process_tree_pids` returns just the root — and - // the engine runs one level below the recorded PID, because `rocmd` launches - // it through an `__engine-serve-http` process. A root-only kill would leave - // the engine running and still holding the device. - // - // Identity verification does not apply on Windows. `process_start_ticks` - // has no `/proc` to read there and returns `None`, so no recorded identity - // can be refuted and the gate below only ever turns away a PID that is not - // running at all. A recorded PID that has been recycled by an unrelated - // process still passes it, and `taskkill /T /F` then takes that process and - // its whole subtree: the stale-PID hazard is unchanged on Windows, exactly - // as it was before this gate existed. Closing it needs `rocm_core` to read a - // Windows process creation time. - #[cfg(windows)] - if matches!( - rocm_core::identity_state(identity), - rocm_core::IdentityState::Matches - ) { - // Taken while the root is still alive: `taskkill /T` resolves children - // through the live parent, so it cannot reach them once the root is gone. - let _ = ProcessCommand::new("taskkill") - .args(["/PID", &identity.pid.to_string(), "/T", "/F"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - // `terminate_verified` still owns the bounded wait and the verdict, but - // a root the tree kill already took now reads back as `AlreadyGone` — - // which would deny a stop this command did perform. - let outcome = rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ); - return if matches!(outcome, rocm_core::TerminationOutcome::AlreadyGone) { - rocm_core::TerminationOutcome::Forced - } else { - outcome - }; - } - rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) -} - #[cfg(test)] mod tests { use super::*; @@ -510,10 +327,11 @@ mod tests { /// a different process: the recorded service already exited and its PID was /// recycled. The stop must refuse it rather than signal a stranger. /// - /// Linux-only: a refutable identity needs a readable start-time, and - /// `rocm_core::process_start_ticks` reads it from `/proc`. On Windows it is - /// always `None`, so no identity can be refuted there and the state this - /// asserts on cannot be constructed. + /// Linux-only: it stages the bystander with `sleep` and offsets its token + /// read from `/proc`. The same refusal on Windows, where the token is the + /// process creation time, is asserted in `rocm-core`'s `proc_lifecycle` + /// tests (`terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` + /// and its Windows-only neighbour) and by scenario `service-stop-08`. #[cfg(target_os = "linux")] #[test] fn stop_managed_service_refuses_a_pid_whose_recorded_identity_no_longer_matches() -> Result<()> @@ -1037,6 +855,137 @@ mod tests { Ok(()) } + /// A `rocm` listing that sees every process gone completes the pending stop + /// and persists it (marker and key cleared, recorded processes forgotten, + /// `stopped`), possibly before this stop reads the record again. That record + /// names no process, which is not a restart: the stop is still confirmed and + /// the marker is not put back. The write below is exactly the one the + /// listing's cleanup makes (it lives in the `rocm` binary, which this crate + /// cannot call; `rocm` has the test that runs the real refresh). + #[test] + fn a_stop_completed_by_a_concurrent_listing_is_still_confirmed() -> Result<()> { + let (root, paths) = temp_app_paths("stop-listing-completed-it"); + paths.ensure()?; + let service_id = "svc-stop-listing-completed-it"; + let key_path = seed_keyed_service(&paths, service_id, 11455)?; + + let result = stop_managed_service_with(&paths, service_id, |_| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + on_disk.clear_recorded_processes(); + on_disk.stop_requested_unix_ms = None; + on_disk.status = "stopped".to_owned(); + on_disk.write().expect("concurrent listing write"); + rocm_core::TerminationOutcome::Graceful + }); + let key_removed = !key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(true), + "{value}" + ); + let reloaded = reloaded?; + assert_eq!(reloaded.status, "stopped"); + assert_eq!( + reloaded.stop_requested_unix_ms, None, + "a confirmed stop does not put the marker back" + ); + assert!(key_removed); + Ok(()) + } + + /// The listing's cleanup of an empty record (see above) leaves a `starting` or + /// `failed` record at its status with the marker gone and the marker's stamp + /// kept: that is the stop's own marker consumed, not a relaunch, so the stop + /// is confirmed. A record naming no process has no PID to terminate, so the + /// per-PID callback never runs; the engine `Stop` does (for every engine but + /// lemonade), and that is where the listing's write is made, with an assertion + /// that it happened. `rocm` has the test that runs the real refresh. + #[test] + fn a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed() -> Result<()> + { + for status in ["starting", "failed"] { + let (root, paths) = temp_app_paths(&format!("stop-empty-listing-{status}")); + paths.ensure()?; + let service_id = "svc-stop-empty-listing"; + let key_path = seed_keyed_service(&paths, service_id, 11457)?; + let mut empty = load_service_record(&paths, service_id)?; + empty.clear_recorded_processes(); + status.clone_into(&mut empty.status); + empty.write()?; + let interleaved = std::cell::Cell::new(false); + + let result = stop_managed_service_via( + &paths, + service_id, + |_| unreachable!("a record naming no process has no PID to terminate"), + |_, _| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + assert!(on_disk.stop_requested_unix_ms.is_some(), "{status}"); + on_disk.settled_stop_unix_ms = on_disk.stop_requested_unix_ms.take(); + on_disk.write().expect("concurrent listing write"); + interleaved.set(true); + Ok(StopResponse { + stopped: true, + graceful: true, + }) + }, + ); + let key_removed = !key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!(interleaved.get(), "{status}: premise: the listing ran"); + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(true), + "{status}: {value}" + ); + let reloaded = reloaded?; + assert_eq!(reloaded.status, "stopped", "{status}"); + assert_eq!(reloaded.stop_requested_unix_ms, None, "{status}"); + assert!(key_removed, "{status}"); + } + Ok(()) + } + + /// A restart that has begun launching but recorded no process yet reads + /// `starting` with nothing named: not the stop completing, so the stop does + /// not claim it or drop the key. + #[test] + fn a_stop_does_not_claim_a_restart_that_has_recorded_no_process_yet() -> Result<()> { + let (root, paths) = temp_app_paths("stop-restart-in-flight"); + paths.ensure()?; + let service_id = "svc-stop-restart-in-flight"; + let key_path = seed_keyed_service(&paths, service_id, 11456)?; + + let result = stop_managed_service_with(&paths, service_id, |_| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + on_disk.clear_recorded_processes(); + on_disk.stop_requested_unix_ms = None; + on_disk.status = "starting".to_owned(); + on_disk.write().expect("concurrent restart write"); + rocm_core::TerminationOutcome::Graceful + }); + let key_kept = key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(false), + "{value}" + ); + assert_eq!(reloaded?.status, "starting"); + assert!(key_kept, "the restarting service's key must be kept"); + Ok(()) + } + /// A launcher that is also the server is recorded under both roles. The /// stop handles that PID once, and must report it under both. #[test] @@ -1080,4 +1029,289 @@ mod tests { ); Ok(()) } + + // ---- This module's stop, through the routine it shares with `rocm`. ---- + // + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. The termination is a stand-in throughout, so + // no process is signalled and the tests behave the same on every host. + + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + /// Plant a record naming `supervisor` and `engine`, with the given start-time + /// tokens, and return the paths it lives under. + fn plant_record_naming( + name: &str, + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> Result<(std::path::PathBuf, AppPaths, &'static str)> { + let (root, paths) = temp_app_paths(name); + paths.ensure()?; + let service_id = "svc-stop-through-the-shared-routine"; + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record.status = "ready".to_owned(); + record.write()?; + Ok((root, paths, service_id)) + } + + fn pids_listed(report: &Value, key: &str) -> Vec { + report[key] + .as_array() + .unwrap_or_else(|| panic!("no `{key}` in {report}")) + .iter() + .filter_map(Value::as_u64) + .collect() + } + + #[test] + fn a_stop_terminates_the_supervisor_then_the_engine_each_with_its_own_start_time() -> Result<()> + { + let (root, paths, service_id) = plant_record_naming( + "stop-order", + SUPERVISOR_PID, + Some(11), + Some(ENGINE_PID), + Some(22), + )?; + let seen = std::cell::RefCell::new(Vec::new()); + + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Graceful + }); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!( + seen.into_inner(), + vec![ + rocm_core::ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + rocm_core::ProcessIdentity::new(ENGINE_PID, Some(22)), + ] + ); + assert_eq!( + report["pid_outcomes"], + json!([ + {"pid": SUPERVISOR_PID, "role": "supervisor", "outcome": "graceful"}, + {"pid": ENGINE_PID, "role": "engine", "outcome": "graceful"}, + ]) + ); + assert_eq!( + pids_listed(&report, "signaled_pids"), + vec![u64::from(SUPERVISOR_PID), u64::from(ENGINE_PID)] + ); + Ok(()) + } + + /// The stopping process's own PID is reported as `self`, never handed to the + /// termination, and does not stand in the way of a confirmed stop. + #[test] + fn a_stop_reports_its_own_pid_as_self_and_skips_it() -> Result<()> { + let own = std::process::id(); + let (root, paths, service_id) = + plant_record_naming("stop-own-pid", own, None, Some(ENGINE_PID), None)?; + let seen = std::cell::RefCell::new(Vec::new()); + + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(identity.pid); + rocm_core::TerminationOutcome::Graceful + }); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!(seen.into_inner(), vec![ENGINE_PID]); + assert_eq!( + report["pid_outcomes"][0], + json!({"pid": own, "role": "supervisor", "outcome": "self"}) + ); + assert_eq!(pids_listed(&report, "skipped_pids"), vec![u64::from(own)]); + assert_eq!( + pids_listed(&report, "signaled_pids"), + vec![u64::from(ENGINE_PID)] + ); + assert_eq!(report["stopped"], true); + Ok(()) + } + + #[test] + fn a_stop_sorts_each_outcome_into_the_signaled_forced_and_skipped_lists() -> Result<()> { + use rocm_core::TerminationOutcome::{ + AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, + }; + // (outcome, signaled, forced, skipped, stopped) + let expected = [ + (AlreadyGone, false, false, true, true), + (IdentityMismatch, false, false, true, true), + (Unverified, false, false, true, false), + (Graceful, true, false, false, true), + (Forced, true, true, false, true), + (TimedOut, true, true, false, false), + ]; + for (outcome, signaled, forced, skipped, stopped) in expected { + let (root, paths, service_id) = + plant_record_naming("stop-classes", SUPERVISOR_PID, None, None, None)?; + + let report = stop_managed_service_with(&paths, service_id, |_| outcome); + fs::remove_dir_all(root).ok(); + + let report = report?; + let listed = |key| !pids_listed(&report, key).is_empty(); + assert_eq!( + ( + listed("signaled_pids"), + listed("force_signaled_pids"), + listed("skipped_pids"), + report["stopped"].as_bool() + ), + (signaled, forced, skipped, Some(stopped)), + "{outcome:?}: {report}" + ); + } + Ok(()) + } + + #[test] + fn the_production_stop_reports_an_absent_pid_as_already_gone() -> Result<()> { + let (root, paths, service_id) = + plant_record_naming("stop-production-absent", SUPERVISOR_PID, None, None, None)?; + + let report = stop_managed_service(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!(report["pid_outcomes"][0]["outcome"], "already_gone"); + assert_eq!(report["stopped"], true); + Ok(()) + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } + } + + /// A root that exited by itself before its tree could be taken down confirms + /// the stop only through the record's other roles, as this caller folds + /// them: the supervisor's own row is not enough, and the record, its PIDs, + /// its stop marker and its key are kept unless the engine role is + /// confirmed gone too. + #[test] + fn a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does() -> Result<()> { + use rocm_core::TerminationOutcome::{ExitedBeforeTreeKill, Forced, TimedOut}; + + for (engine, confirmed) in [(Forced, true), (TimedOut, false)] { + let (root, paths, service_id) = plant_record_naming( + "stop-root-exited-alone", + SUPERVISOR_PID, + None, + Some(ENGINE_PID), + None, + )?; + let key = rocm_engine_protocol::endpoint_key_file_path(&paths, service_id); + fs::create_dir_all(paths.services_dir())?; + fs::write(&key, "secret-key")?; + + let report = stop_managed_service_with(&paths, service_id, |identity| { + if identity.pid == SUPERVISOR_PID { + ExitedBeforeTreeKill + } else { + engine + } + })?; + let record = find_managed_service(&paths, service_id)?; + let key_kept = key.exists(); + fs::remove_dir_all(root).ok(); + + assert_eq!( + stop_report_confirmed(&report), + confirmed, + "{engine:?}: {report}" + ); + assert_eq!( + report["pid_outcomes"][0]["outcome"], + if confirmed { + "already_gone" + } else { + "exited_before_tree_kill" + }, + "{engine:?}: {report}" + ); + if confirmed { + assert_eq!(record.status, "stopped"); + assert_eq!( + record.supervisor_pid, 0, + "the confirmed stop clears the PIDs" + ); + assert!(!key_kept, "and the key"); + } else { + assert_eq!(record.status, "ready", "{engine:?}"); + assert_eq!(record.supervisor_pid, SUPERVISOR_PID, "the PIDs are kept"); + assert!( + record.stop_requested_unix_ms.is_some(), + "the marker is kept" + ); + assert!(key_kept, "the key is kept"); + } + } + Ok(()) + } + + /// A root that exited by itself with no other role recorded cannot be + /// vouched for, so this caller does not clear the record on its say-so. + #[test] + fn a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed() -> Result<()> { + let (root, paths, service_id) = plant_record_naming( + "stop-lone-root-exited-alone", + SUPERVISOR_PID, + None, + None, + None, + )?; + + let report = stop_managed_service_with(&paths, service_id, |_| { + rocm_core::TerminationOutcome::ExitedBeforeTreeKill + })?; + let record = find_managed_service(&paths, service_id)?; + fs::remove_dir_all(root).ok(); + + assert!(!stop_report_confirmed(&report), "{report}"); + assert_eq!(record.supervisor_pid, SUPERVISOR_PID); + assert!(record.stop_requested_unix_ms.is_some()); + Ok(()) + } } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index eb594e2167..bfb5092ada 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use crate::daemon_host::DaemonHost; use crate::persistence::load_service_record; use crate::watchers::stop_requested; use anyhow::{Context, Result}; @@ -36,6 +37,41 @@ pub(crate) fn supervise_service( device_policy: String, gpu: Option, engine_recipe_json: Option, +) -> Result<()> { + supervise_service_on( + &DaemonHost::detect()?, + paths, + service_id, + engine, + model_ref, + canonical_model_id, + runtime_id, + env_id, + host, + port, + device_policy, + gpu, + engine_recipe_json, + ) +} + +/// [`supervise_service`] on a given host, so a test can supply the binary that +/// runs the engine. +#[allow(clippy::too_many_arguments)] +fn supervise_service_on( + daemon_host: &DaemonHost, + paths: &AppPaths, + service_id: String, + engine: String, + model_ref: String, + canonical_model_id: String, + runtime_id: Option, + env_id: Option, + host: String, + port: u16, + device_policy: String, + gpu: Option, + engine_recipe_json: Option, ) -> Result<()> { paths.ensure()?; fs::create_dir_all(paths.engine_logs_dir(&engine))?; @@ -108,6 +144,10 @@ pub(crate) fn supervise_service( .is_some(), record.requires_api_key, )?; + // The engine runs through the hidden commands of the `rocm` binary, which + // `rocmd` does not have. Found before the first write, like the guard above, + // so a supervisor that cannot find it leaves the record as it was. + let rocm_binary = daemon_host.rocm_binary()?; record.write()?; let log_file = fs::File::create(&record.log_path) @@ -116,8 +156,6 @@ pub(crate) fn supervise_service( .try_clone() .context("failed to clone service log file handle")?; - let rocm_binary = - std::env::current_exe().context("failed to resolve current rocm executable path")?; let mut command = ProcessCommand::new(rocm_binary); command .args(engine_serve_http_args( @@ -169,6 +207,7 @@ pub(crate) fn supervise_service( })?; let became_ready = wait_for_service_ready( + daemon_host, paths, &record.engine, &record.service_id, @@ -277,9 +316,12 @@ fn engine_serve_http_args( if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); - args.extend(crate::common::optional_arg("--env-id", env_id)); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( + "--runtime-id", + runtime_id, + )); + args.extend(rocm_engine_protocol::optional_arg("--env-id", env_id)); + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", engine_recipe_json, )); @@ -354,9 +396,14 @@ fn read_new_log_phase(log_path: &Path, pos: &mut u64) -> Option<&'static str> { phase } -fn engine_healthcheck_ready(paths: &AppPaths, engine: &str, service_id: &str) -> Result { +fn engine_healthcheck_ready( + daemon_host: &DaemonHost, + paths: &AppPaths, + engine: &str, + service_id: &str, +) -> Result { Ok(crate::engine_ipc::healthcheck_response_ready( - &crate::engine_ipc::engine_healthcheck_response(paths, engine, service_id)?, + &crate::engine_ipc::engine_healthcheck_response_on(daemon_host, paths, engine, service_id)?, )) } @@ -364,6 +411,7 @@ fn engine_healthcheck_ready(paths: &AppPaths, engine: &str, service_id: &str) -> /// timeout elapses), tailing its log file meanwhile and reporting each coarse /// startup phase transition via `on_phase`. fn wait_for_service_ready( + daemon_host: &DaemonHost, paths: &AppPaths, engine: &str, service_id: &str, @@ -381,7 +429,7 @@ fn wait_for_service_ready( last_phase = Some(phase); on_phase(phase); } - if engine_healthcheck_ready(paths, engine, service_id).unwrap_or(false) { + if engine_healthcheck_ready(daemon_host, paths, engine, service_id).unwrap_or(false) { return true; } thread::sleep(Duration::from_millis(200)); @@ -436,6 +484,44 @@ fn restart_managed_service_bounded( spawn_override: Option<&SpawnOverride<'_>>, lock_timeout: Duration, ) -> Result<()> { + restart_managed_service_via( + &DaemonHost::detect()?, + paths, + record, + spawn_override, + lock_timeout, + ) +} + +/// [`restart_managed_service`] on a given host, so a test can be `rocm`. +#[cfg(test)] +fn restart_managed_service_on( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + host: &DaemonHost, +) -> Result<()> { + restart_managed_service_via( + host, + paths, + record, + None, + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + ) +} + +/// The restart itself, on a given host. +/// +/// The binary that runs `supervise` is chosen first, before the launch lock is +/// taken and before anything is read or written: a restart that cannot find it +/// leaves every record as it was. +fn restart_managed_service_via( + host: &DaemonHost, + paths: &AppPaths, + record: &mut ManagedServiceRecord, + spawn_override: Option<&SpawnOverride<'_>>, + lock_timeout: Duration, +) -> Result<()> { + let rocmd_binary = host.rocmd_binary()?; let _launch_lock = rocm_core::FileLock::acquire_bounded(paths.managed_launch_lock_path(), lock_timeout)?; @@ -519,8 +605,6 @@ fn restart_managed_service_bounded( None }; - let rocmd_binary = - std::env::current_exe().context("failed to resolve current rocmd executable path")?; let log_file = fs::OpenOptions::new() .create(true) .append(true) @@ -805,18 +889,18 @@ fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { .unwrap_or("gpu_required") .to_owned(), ]; - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--runtime-id", record.runtime_id.as_deref(), )); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--env-id", record.env_id.as_deref(), )); if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", record.engine_recipe_json.as_deref(), )); @@ -838,9 +922,14 @@ fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { #[cfg(test)] mod tests { use super::*; + use crate::daemon_host::DaemonHost; use crate::persistence::load_service_record; #[cfg(target_os = "linux")] use crate::test_support::UNCONFIRMED_STOP_PID; + #[cfg(target_os = "linux")] + use crate::test_support::{ + find_on_path, lock_script_tests, script_tools, write_executable_script, + }; use crate::test_support::{ free_loopback_port, identity_probe_record, temp_app_paths, unique_test_root, }; @@ -870,7 +959,8 @@ mod tests { let service_id = service_id.to_owned(); let (sender, receiver) = std::sync::mpsc::channel(); std::thread::spawn(move || { - let outcome = supervise_service( + let outcome = supervise_service_on( + &DaemonHost::test_binary_standing_in_for_rocm(), &paths, service_id, "llamacpp".to_owned(), @@ -937,7 +1027,8 @@ mod tests { let owned_id = service_id.to_owned(); let (sender, receiver) = std::sync::mpsc::channel(); std::thread::spawn(move || { - let outcome = supervise_service( + let outcome = supervise_service_on( + &DaemonHost::test_binary_standing_in_for_rocm(), &owned_paths, owned_id, "llamacpp".to_owned(), @@ -1224,8 +1315,9 @@ mod tests { /// calls `std::process::exit`, which inside a test binary would take the /// whole suite with it. /// - /// Linux-only: `rocm_core::process_start_ticks` reads `/proc` and returns - /// `None` everywhere else, so on other platforms there is no token to write. + /// Linux-only here: `rocm_core::process_start_ticks` gives a token on Linux + /// and on Windows (the creation time) and `None` elsewhere, and this test has + /// not been taken to Windows. #[cfg(target_os = "linux")] #[test] fn supervise_service_persists_the_supervisor_identity_token() -> Result<()> { @@ -1238,7 +1330,8 @@ mod tests { // A directory where the supervisor expects to create a file. fs::create_dir_all(&log_path)?; - let outcome = supervise_service( + let outcome = supervise_service_on( + &DaemonHost::test_binary_standing_in_for_rocm(), &paths, service_id.to_owned(), "llamacpp".to_owned(), @@ -1358,7 +1451,8 @@ mod tests { // `supervise_service_persists_the_supervisor_identity_token`. fs::create_dir_all(&seeded.log_path)?; - let outcome = supervise_service( + let outcome = supervise_service_on( + &DaemonHost::test_binary_standing_in_for_rocm(), &paths, service_id.to_owned(), "llamacpp".to_owned(), @@ -1436,7 +1530,11 @@ mod tests { "precondition: the seeded record carries no token" ); - let _ = restart_managed_service(&paths, &mut record); + let _ = restart_managed_service_on( + &paths, + &mut record, + &DaemonHost::test_binary_standing_in_for_rocmd(), + ); let persisted = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); @@ -1471,7 +1569,11 @@ mod tests { record.status = "failed".to_owned(); record.write()?; - let _ = restart_managed_service(&paths, &mut record); + let _ = restart_managed_service_on( + &paths, + &mut record, + &DaemonHost::test_binary_standing_in_for_rocmd(), + ); let persisted = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); @@ -1530,7 +1632,11 @@ mod tests { Ok(false) }) }; - let outcome = restart_managed_service(&paths, &mut record); + let outcome = restart_managed_service_on( + &paths, + &mut record, + &DaemonHost::test_binary_standing_in_for_rocmd(), + ); let stop_landed = stop.join().expect("stop thread"); let persisted = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); @@ -1550,6 +1656,262 @@ mod tests { Ok(()) } + /// The restart, when the daemon is embedded in `rocm`, launches `supervise` + /// on the `rocmd` it is given. Before the choice was made, it re-executed + /// `rocm` itself and the new supervisor died on an unrecognised subcommand. + /// + /// The stand-in is a script that records its arguments, so what the restart + /// actually ran can be read back. Its shell and `sleep` are found once and + /// asserted by name, and so is `setsid`, which the restart itself runs. + #[cfg(target_os = "linux")] + #[test] + fn restart_runs_supervise_on_the_rocmd_it_is_given() -> Result<()> { + let _serial = lock_script_tests(); + let tools = script_tools(); + assert!( + find_on_path("setsid").is_some(), + "this test needs `setsid` on PATH: the restart runs it" + ); + let (root, paths) = temp_app_paths("restart-runs-given-rocmd"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let seen = root.join("rocmd-args.txt"); + // The daemon is `rocm`, which does not exist here; the `rocmd` beside it + // is the stand-in. + let host = DaemonHost::new(root.join("rocm"), true); + let stand_in = root.join("rocmd"); + write_executable_script( + &stand_in, + &tools, + &format!( + "printf '%s\\n' \"$@\" > '{}'\nexec '{}' 3\n", + seen.display(), + tools.sleep.display() + ), + )?; + + let service_id = "svc-restart-given-rocmd"; + let mut record = identity_probe_record(&paths, service_id, 11470); + record.status = "failed".to_owned(); + record.write()?; + + // The first exec of a new file can still lose a race with an unrelated + // fork (ETXTBSY): setsid then exits at once and nothing is recorded. + let mut outcome = Err(anyhow::anyhow!("not attempted")); + for _ in 0..10 { + outcome = restart_managed_service_on(&paths, &mut record, &host); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !seen.exists() && std::time::Instant::now() < deadline { + thread::sleep(Duration::from_millis(20)); + } + if seen.exists() { + break; + } + } + let args = fs::read_to_string(&seen).unwrap_or_default(); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(&root).ok(); + + outcome?; + let args: Vec<&str> = args.lines().collect(); + assert_eq!( + args.get(..2), + Some(&["supervise", service_id][..]), + "the rocmd it is given must be started as `rocmd supervise `: {args:?}" + ); + let persisted = persisted?; + assert_eq!(persisted.status, "recovering"); + assert_ne!( + persisted.supervisor_pid, 0, + "the spawned supervisor's PID is recorded" + ); + Ok(()) + } + + /// With no `rocmd` to run, the restart reports it, and the record is exactly + /// as the restart found it: it is not marked `recovering` for a supervisor + /// that was never started. + #[test] + fn restart_without_a_rocmd_to_run_changes_nothing() -> Result<()> { + let (root, paths) = temp_app_paths("restart-no-rocmd"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-no-rocmd"; + let mut record = identity_probe_record(&paths, service_id, 11471); + record.status = "failed".to_owned(); + record.write()?; + let before = load_service_record(&paths, service_id)?; + + let host = DaemonHost::new(root.join("rocm"), true); + let outcome = restart_managed_service_on(&paths, &mut record, &host); + let after = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = format!( + "{:#}", + outcome.expect_err("no rocmd to run: the restart must fail") + ); + assert!(error.contains("no `supervise` command"), "{error}"); + assert!(error.contains("unable to locate sibling binary"), "{error}"); + let after = after?; + assert_eq!(after.status, "failed"); + assert_eq!(after.supervisor_pid, before.supervisor_pid); + assert_eq!(after.restart_count, before.restart_count); + Ok(()) + } + + /// A supervisor run by the `rocmd` binary starts its engine on the `rocm` + /// beside it: `__engine-serve-http` is a hidden command of `rocm` alone, and + /// `rocmd` answered it with an unrecognised-subcommand error. It polls the + /// engine's readiness through the same `rocm`. + /// + /// The stand-in `rocm` is the only binary involved: it records the engine + /// start, answers the readiness check, and lets the engine exit, so the + /// supervisor finishes by itself and nothing is left running. The test host + /// is the only host the supervisor has, so no other `rocm` can be reached. + #[cfg(target_os = "linux")] + #[test] + fn supervise_runs_the_engine_on_the_rocm_beside_rocmd() -> Result<()> { + let _serial = lock_script_tests(); + let tools = script_tools(); + let (root, paths) = temp_app_paths("supervise-engine-on-rocm"); + paths.ensure()?; + let served = root.join("rocm-serve-args.txt"); + let asked = root.join("rocm-stdio-args.txt"); + let ready = serde_json::to_string(&rocm_engine_protocol::EngineResponseEnvelope::success( + rocm_engine_protocol::HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cpu".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }, + ))?; + write_executable_script( + &root.join("rocm"), + &tools, + &format!( + "case \"$1\" in\n\ + __engine-serve-http)\n printf '%s\\n' \"$@\" > '{served}'\n exec '{sleep}' 2\n ;;\n\ + __engine-stdio)\n printf '%s\\n' \"$@\" > '{asked}'\n read -r _request\n printf '%s\\n' '{ready}'\n ;;\n\ + *) exit 2 ;;\nesac\n", + served = served.display(), + asked = asked.display(), + sleep = tools.sleep.display(), + ), + )?; + + let service_id = "svc-supervise-engine-on-rocm"; + let mut outcome = Err("not attempted".to_owned()); + for _ in 0..10 { + let (sender, receiver) = std::sync::mpsc::channel(); + // The daemon is `rocmd`, which does not exist here; the `rocm` beside + // it is the stand-in. + let (paths, daemon_host) = (paths.clone(), DaemonHost::new(root.join("rocmd"), false)); + thread::spawn(move || { + let outcome = supervise_service_on( + &daemon_host, + &paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11472, + "gpu_required".to_owned(), + None, + None, + ); + let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); + }); + outcome = receiver + .recv_timeout(Duration::from_secs(30)) + .unwrap_or_else(|_| { + panic!( + "the supervisor did not finish: it was not polling the stand-in `rocm`, \ + which answers its readiness check and lets the engine exit" + ) + }); + // The first exec of a new file can lose a race with an unrelated + // fork (ETXTBSY): the spawn fails at once, and the next attempt + // starts again. + if !matches!(&outcome, Err(error) if error.contains("Text file busy")) { + break; + } + } + let serve_args = fs::read_to_string(&served).unwrap_or_default(); + let stdio_args = fs::read_to_string(&asked).unwrap_or_default(); + fs::remove_dir_all(&root).ok(); + + assert_eq!(outcome, Ok(()), "the supervisor must finish cleanly"); + let serve_args: Vec<&str> = serve_args.lines().collect(); + assert_eq!( + serve_args.get(..3), + Some(&["__engine-serve-http", "llamacpp", service_id][..]), + "the engine must be started as `rocm __engine-serve-http `: {serve_args:?}" + ); + assert_eq!( + stdio_args.lines().collect::>(), + ["__engine-stdio", "llamacpp"], + "readiness must be polled through the same `rocm`" + ); + Ok(()) + } + + /// With no `rocm` beside `rocmd` the supervisor stops before it writes + /// anything, and says what it looked for: the service's record is as it was, + /// and no log was created for an engine that never started. + #[test] + fn supervise_without_a_rocm_beside_rocmd_changes_nothing() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-no-rocm"); + let service_id = "svc-supervise-no-rocm"; + // A failed service whose supervisor is gone: a write by a supervisor + // that then stops would change both fields. + let mut seeded = identity_probe_record(&paths, service_id, 11434); + seeded.status = "failed".to_owned(); + seeded.supervisor_pid = 0; + seeded.write()?; + let before = load_service_record(&paths, service_id)?; + fs::remove_file(&before.log_path).ok(); + + let outcome = supervise_service_on( + &DaemonHost::new(root.join("rocmd"), false), + &paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11434, + "gpu_required".to_owned(), + None, + None, + ); + let after = load_service_record(&paths, service_id); + let log_created = before.log_path.exists(); + fs::remove_dir_all(root).ok(); + + let error = format!("{:#}", outcome.expect_err("no rocm to run the engine on")); + assert!(error.contains("no engine commands"), "{error}"); + assert!(error.contains("unable to locate sibling binary"), "{error}"); + let after = after?; + assert_eq!(after.status, before.status); + assert_eq!(after.supervisor_pid, before.supervisor_pid); + assert_eq!(after.restart_count, before.restart_count); + assert!( + !log_created, + "no engine log for an engine that never started" + ); + Ok(()) + } + /// Between a restart's first write, which clears the recorded PIDs, and the /// one recording the supervisor it spawned, a stop can persist its marker. /// Writing the restart's own copy back would erase it while the new @@ -1852,7 +2214,7 @@ mod tests { } /// Mirrors `launch_releases_the_leased_port_before_spawning_the_child` in - /// `apps/rocm/src/service_launch.rs`: the probe lease on the recorded port + /// `apps/rocm/src/service_launch/spawn.rs`: the probe lease on the recorded port /// must already be released by the time the recovery supervisor is /// spawned, or the supervisor's own bind of that port would fail. #[test] @@ -1899,6 +2261,86 @@ mod tests { Ok(()) } + /// Both halves of a restart from the `rocm` binary, at the instant the + /// supervisor is spawned: the recorded port, which recovery leased to claim + /// it, is already released, and the program being started is the `rocmd` + /// beside `rocm`, not `rocm` itself. + #[cfg(target_os = "linux")] + #[test] + fn restart_from_the_rocm_binary_releases_the_port_and_runs_the_sibling_rocmd() -> Result<()> { + use std::cell::{Cell, RefCell}; + use std::net::TcpListener; + + let _serial = lock_script_tests(); + let tools = script_tools(); + let (root, paths) = temp_app_paths("restart-rocm-releases-lease"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let stand_in = root.join("rocmd"); + write_executable_script( + &stand_in, + &tools, + &format!("exec '{}' 1\n", tools.sleep.display()), + )?; + // The daemon is `rocm`, which does not exist here; the `rocmd` beside it + // is the stand-in. + let host = DaemonHost::new(root.join("rocm"), true); + + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + let service_id = "svc-restart-rocm-releases-lease"; + let mut record = identity_probe_record(&paths, service_id, port); + record.status = "failed".to_owned(); + record.write()?; + + let bind_succeeded = Cell::new(false); + let spawned_args = RefCell::new(Vec::::new()); + let spawn_override = + |command: &mut ProcessCommand| -> std::io::Result { + // Retry briefly: a sibling test's concurrent fork can hold a + // copy of the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(TcpListener::bind(("127.0.0.1", port)).is_ok()); + } + *spawned_args.borrow_mut() = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect(); + command.spawn() + }; + + // Whether the stand-in then stays up does not matter here: what is read + // is what the restart was about to run. + let _ = restart_managed_service_via( + &host, + &paths, + &mut record, + Some(&spawn_override), + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + ); + fs::remove_dir_all(&root).ok(); + + assert!( + bind_succeeded.get(), + "the leased port must already be released when the supervisor is spawned" + ); + let spawned_args = spawned_args.into_inner(); + assert_eq!( + spawned_args.get(..3), + Some( + &[ + stand_in.display().to_string(), + "supervise".to_owned(), + service_id.to_owned() + ][..] + ), + "the supervisor must be the sibling rocmd, started as `rocmd supervise `: {spawned_args:?}" + ); + Ok(()) + } + /// Recovery re-claims the concrete port already written in the record, so /// it must serialize on the same cross-process launch lock a fresh /// `rocm serve` launch takes, and must refuse a recorded port something @@ -2015,12 +2457,12 @@ mod tests { /// spawned yet: the rollback at that first write must undo the in-memory /// transition and leave the on-disk record exactly as it was, not a /// half-applied `recovering` row. The manifest write goes through - /// [`crate::write_atomic`][rocm_core_write_atomic]-style write-then-rename, + /// [`write_file_atomically`][rocm_core_atomic_write]-style write-then-rename, /// so a read-only *file* no longer blocks it (rename ignores the /// destination's own permission bits) — a read-only *directory* still does, /// since it can't hold the new temp file the rename needs. /// - /// [rocm_core_write_atomic]: rocm_core + /// [rocm_core_atomic_write]: rocm_core::atomic_write::write_file_atomically /// /// Unix only and skipped as root, where a read-only mode is not enforced /// and the write would simply succeed. diff --git a/apps/rocmd/src/test_support.rs b/apps/rocmd/src/test_support.rs index 759b892033..254706a973 100644 --- a/apps/rocmd/src/test_support.rs +++ b/apps/rocmd/src/test_support.rs @@ -15,8 +15,9 @@ use rocm_core::{ AppPaths, AutomationRuntimeState, ManagedServiceRecord, WatcherMode, WatcherRuntimeSnapshot, unix_time_millis, }; +use std::ffi::OsString; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; pub(crate) fn temp_app_paths(name: &str) -> (PathBuf, AppPaths) { @@ -251,3 +252,147 @@ mod tests { assert_each_call_gets_its_own_root(|| temp_app_paths("same-label").0); } } + +/// Tests that run a script they just wrote take this lock. Writing an +/// executable while another thread forks can leave the new file "text file +/// busy" at its first exec; the lock keeps these tests from doing it to each +/// other, and each retries for the forks it cannot see. +#[cfg(unix)] +static SCRIPT_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +#[cfg(unix)] +pub(crate) fn lock_script_tests() -> std::sync::MutexGuard<'static, ()> { + SCRIPT_TESTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// The tools a stand-in script needs, each found once on `PATH` as an absolute +/// path. A missing one fails the test by name, rather than being skipped or +/// surfacing as a confusing exec error. +#[cfg(unix)] +pub(crate) struct ScriptTools { + pub(crate) sh: PathBuf, + pub(crate) sleep: PathBuf, +} + +#[cfg(unix)] +pub(crate) fn find_on_path(name: &str) -> Option { + let path = std::env::var_os("PATH").unwrap_or_default(); + std::env::split_paths(&path) + .map(|dir| dir.join(name)) + .find(|candidate| candidate.is_file()) +} + +#[cfg(unix)] +pub(crate) fn script_tools() -> ScriptTools { + let find = |name: &str| { + find_on_path(name) + .unwrap_or_else(|| panic!("this test needs `{name}` on PATH to stand in for a binary")) + }; + ScriptTools { + sh: find("sh"), + sleep: find("sleep"), + } +} + +/// Write `body` as a script for `tools.sh` at `path` and make it executable. +/// Every writer is closed before this returns, so the first exec cannot find +/// the file open for writing. The script may use the shell's own commands and +/// the tools in `tools`. +#[cfg(unix)] +pub(crate) fn write_executable_script( + path: &std::path::Path, + tools: &ScriptTools, + body: &str, +) -> Result<()> { + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + + let mut file = fs::File::create(path)?; + write!(file, "#!{}\n{body}", tools.sh.display())?; + file.set_permissions(fs::Permissions::from_mode(0o755))?; + file.sync_all()?; + Ok(()) +} + +const RECIPE_INDEX_PATH_ENV: &str = "ROCM_CLI_MODEL_RECIPE_INDEX_PATH"; +const RECIPE_INDEX_SIGNATURE_ENV: &str = "ROCM_CLI_MODEL_RECIPE_INDEX_SIGNATURE_PATH"; +const RECIPE_INDEX_PUBLIC_KEY_ENV: &str = "ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH"; +const REQUIRE_SIGNED_RECIPE_INDEX_ENV: &str = "ROCM_CLI_REQUIRE_MODEL_RECIPE_SIGNATURE"; + +static RECIPE_INDEX_ENV: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Holds the process-wide model recipe registry configuration for one test. +/// +/// `rocm_core::resolve_model_recipe_artifact` (reached through +/// `run_sandbox_tool`'s `prefetch_artifact` arm) reads the recipe-index +/// environment variables on every call, and the environment is shared by every +/// test thread. A test that resolves a recipe reference, or sets those +/// variables, therefore takes this guard first: it serialises them against each +/// other, so a resolver never sees the index path set without its public key, +/// or the variables mid-restore. The previous values come back on drop. +pub(crate) struct RecipeIndexEnv { + previous: Vec<(&'static str, Option)>, + _serialised: std::sync::MutexGuard<'static, ()>, +} + +impl RecipeIndexEnv { + /// The built-in catalog: no index configured and no signature required, whatever + /// the developer's shell exports. + pub(crate) fn builtin_catalog() -> Self { + Self::apply(&[ + (RECIPE_INDEX_PATH_ENV, None), + (RECIPE_INDEX_SIGNATURE_ENV, None), + (RECIPE_INDEX_PUBLIC_KEY_ENV, None), + (REQUIRE_SIGNED_RECIPE_INDEX_ENV, None), + ]) + } + + /// A signed recipe index, set as the registry for as long as the guard lives. + pub(crate) fn signed_index(index: &Path, signature: &Path, public_key: &Path) -> Self { + Self::apply(&[ + (RECIPE_INDEX_PATH_ENV, Some(index)), + (RECIPE_INDEX_SIGNATURE_ENV, Some(signature)), + (RECIPE_INDEX_PUBLIC_KEY_ENV, Some(public_key)), + (REQUIRE_SIGNED_RECIPE_INDEX_ENV, None), + ]) + } + + #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 + fn apply(vars: &[(&'static str, Option<&Path>)]) -> Self { + let serialised = RECIPE_INDEX_ENV + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = vars + .iter() + .map(|(key, _)| (*key, std::env::var_os(key))) + .collect(); + for (key, value) in vars { + unsafe { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + } + Self { + previous, + _serialised: serialised, + } + } +} + +impl Drop for RecipeIndexEnv { + #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 + fn drop(&mut self) { + for (key, value) in &self.previous { + unsafe { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + } + } +} diff --git a/apps/rocmd/src/watchers/driver_upgrade.rs b/apps/rocmd/src/watchers/driver_upgrade.rs index 84967dd6d2..55bc78856d 100644 --- a/apps/rocmd/src/watchers/driver_upgrade.rs +++ b/apps/rocmd/src/watchers/driver_upgrade.rs @@ -231,18 +231,18 @@ mod tests { &event, |_paths| { Ok(crate::sandbox::sandbox_driver_plan_value( - rocm_core::CommandCapture { - argv: vec![ + rocm_core::CommandCapture::from_text( + vec![ "rocm".to_owned(), "install".to_owned(), "driver".to_owned(), "--dkms".to_owned(), "--dry-run".to_owned(), ], - exit_status: 0, - stdout: "driver install plan\n supported: true\n".to_owned(), - stderr: String::new(), - }, + 0, + "driver install plan\n supported: true\n".to_owned(), + String::new(), + ), )) }, )?; diff --git a/apps/rocmd/src/watchers/server_recover.rs b/apps/rocmd/src/watchers/server_recover.rs index f43402a4e1..aac0a9e9d5 100644 --- a/apps/rocmd/src/watchers/server_recover.rs +++ b/apps/rocmd/src/watchers/server_recover.rs @@ -798,8 +798,8 @@ mod tests { /// The record `rocm serve` keeps when it abandons a launch whose child /// already ran (`abandon_failed_managed_launch` in - /// `apps/rocm/src/service_launch.rs`) must not be respawned: the CLI has just - /// reported that launch as failed. The producing side is + /// `apps/rocm/src/service_launch/launch_transaction.rs`) must not be respawned: + /// the CLI has just reported that launch as failed. The producing side is /// `abandoned_launch_terminates_the_spawned_child` in that file; both go /// through `ManagedServiceRecord::mark_launch_abandoned`. #[test] diff --git a/apps/rocmd/src/watchers/therock_update.rs b/apps/rocmd/src/watchers/therock_update.rs index e364c923d7..91f0304bee 100644 --- a/apps/rocmd/src/watchers/therock_update.rs +++ b/apps/rocmd/src/watchers/therock_update.rs @@ -254,12 +254,12 @@ mod tests { &event, |_paths| { Ok(crate::sandbox::sandbox_check_updates_value( - rocm_core::CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n managed runtimes: none\n".to_owned(), - stderr: String::new(), - }, + rocm_core::CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n managed runtimes: none\n".to_owned(), + String::new(), + ), )) }, )?; @@ -319,12 +319,12 @@ mod tests { &mut state, &event, |_paths| { - Ok(crate::sandbox::sandbox_check_updates_value(rocm_core::CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - })) + Ok(crate::sandbox::sandbox_check_updates_value(rocm_core::CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), + String::new(), + ))) }, )?; diff --git a/coverage-floors.toml b/coverage-floors.toml index aa2ece4edd..3cc2e5514e 100644 --- a/coverage-floors.toml +++ b/coverage-floors.toml @@ -23,5 +23,5 @@ "rocm-engine-lemonade" = 63.82 # 5554 lines "rocm-engine-protocol" = 98.37 # 491 lines "rocm-engine-vllm" = 66.76 # 3030 lines -"rocmd" = 75.36 # 7714 lines +"rocmd" = 75.02 # 7549 lines "xtask" = 76.18 # 5447 lines diff --git a/crates/e2e-report/src/consolidated/html.rs b/crates/e2e-report/src/consolidated/html.rs index a58151b786..6c4717152f 100644 --- a/crates/e2e-report/src/consolidated/html.rs +++ b/crates/e2e-report/src/consolidated/html.rs @@ -341,6 +341,17 @@ mod tests {
  • Every other platform — real self-hosted GPU hardware; \ non-blocking while proven out.
  • "; assert!(html.contains(expected), "legend text changed:\n{html}"); + // The pin above would still pass if the legend hardcoded the words while + // a shared constant's value moved on; these tie the rendered text to the + // constants themselves. + assert!( + html.contains(MOCK_GATES_PR), + "legend must render MOCK_GATES_PR ({MOCK_GATES_PR:?}):\n{html}" + ); + assert!( + html.contains(OTHER_LANES_NON_BLOCKING), + "legend must render OTHER_LANES_NON_BLOCKING ({OTHER_LANES_NON_BLOCKING:?}):\n{html}" + ); } #[test] diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs index d89ee11822..d2d041373e 100644 --- a/crates/e2e-report/src/consolidated/markdown.rs +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -60,6 +60,12 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { s if s.is_empty() => plat_base, s => format!("{plat_base}
    {s}"), }; + // Exact target and package source on their own line, so a reader can see + // what the run installed from, not only which versions it ended up with. + let plat_cell = match r.versions.provenance() { + p if p.is_empty() => plat_cell, + p => format!("{plat_cell}
    {p}"), + }; let os_cell = match r.versions.os.as_deref() { Some(v) if v != r.desc.os => format!("{}
    {}", r.desc.os, v), _ => r.desc.os.clone(), @@ -393,6 +399,17 @@ mod tests { md.contains(expected), "summary explanation text changed:\n{md}" ); + // The pin above would still pass if the paragraph hardcoded the words + // while a shared constant's value moved on; these tie the rendered text + // to the constants themselves. + assert!( + md.contains(MOCK_GATES_PR), + "summary must render MOCK_GATES_PR ({MOCK_GATES_PR:?}):\n{md}" + ); + assert!( + md.contains(OTHER_LANES_NON_BLOCKING), + "summary must render OTHER_LANES_NON_BLOCKING ({OTHER_LANES_NON_BLOCKING:?}):\n{md}" + ); } #[test] @@ -779,6 +796,44 @@ mod tests { } } + #[test] + fn matrix_cell_names_the_exact_target_and_package_source() { + // platform.json carries the runtime's exact GFX target and package source; + // the matrix Platform cell shows both so a reader can tell what the run + // installed from. A platform.json without them shows neither. + let report = feature_json(&[(&["id:serve-x"], &["passed"])]); + let with = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "versions": {"rocm":"7.14.0a20260601","channel":"nightly","gfx_target":"gfx942", + "source":"https://rocm.nightlies.amd.com/whl-multi-arch (multi-arch-v2)"}, + "expectations": [{"id":"serve-x","effective_engine":"vllm","expected":"pass"}] + }"#; + let (_d, path) = write_platform(&report, with); + let md = + consolidated_summary_markdown(&load_platform_reports(&[("mi300x".to_string(), path)])); + assert!( + md.contains( + "GFX gfx942 · source https://rocm.nightlies.amd.com/whl-multi-arch (multi-arch-v2)" + ), + "target and source missing from the Platform cell:\n{md}" + ); + + let without = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "versions": {"rocm":"7.14.0"}, + "expectations": [{"id":"serve-x","effective_engine":"vllm","expected":"pass"}] + }"#; + let (_d, path) = write_platform(&report, without); + let md = + consolidated_summary_markdown(&load_platform_reports(&[("mi300x".to_string(), path)])); + assert!( + !md.contains("GFX gfx") && !md.contains("source http"), + "{md}" + ); + } + #[test] fn grid_heading_omits_absent_versions() { // A platform.json with only OS known renders just the distro (in the matrix diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 0ef063a7ff..911385029d 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -1764,12 +1764,14 @@ mod tests { fn renderers_take_the_pr_gating_phrases_from_the_shared_constants() { // What this proves, and what it does not. The rendered text is pinned // verbatim by `summary_explanation_paragraphs_are_unchanged` (markdown) - // and `legend_mock_and_other_platform_items_are_unchanged` (HTML); those - // catch any edit to the wording, but a renderer that re-typed the literal - // with identical words would still pass them and would silently stop - // following `MOCK_GATES_PR` / `OTHER_LANES_NON_BLOCKING`. This test - // closes that gap by reading each renderer's non-test source: it must - // name both constants and must not spell either phrase out itself. + // and `legend_mock_and_other_platform_items_are_unchanged` (HTML), and + // each of those also asserts that the rendered output contains + // `MOCK_GATES_PR` and `OTHER_LANES_NON_BLOCKING`; together they catch a + // renderer that hardcodes the words while a constant's value moves on. + // What they cannot catch is a renderer that re-types the CURRENT words + // with identical text, which renders the same bytes. This test closes + // that gap by reading each renderer's non-test source: it must not + // spell either phrase out itself. for (name, source) in [ ("markdown.rs", include_str!("markdown.rs")), ("html.rs", include_str!("html.rs")), @@ -1782,10 +1784,6 @@ mod tests { ("MOCK_GATES_PR", MOCK_GATES_PR), ("OTHER_LANES_NON_BLOCKING", OTHER_LANES_NON_BLOCKING), ] { - assert!( - production.contains(constant), - "{name} must build its gating sentence from `{constant}`" - ); assert!( !production.contains(phrase), "{name} re-types {phrase:?} instead of using `{constant}`" diff --git a/crates/e2e-report/src/consolidated/sidecars.rs b/crates/e2e-report/src/consolidated/sidecars.rs index b64b2065a7..b1ff323864 100644 --- a/crates/e2e-report/src/consolidated/sidecars.rs +++ b/crates/e2e-report/src/consolidated/sidecars.rs @@ -81,37 +81,51 @@ pub(super) struct PlatformVersions { /// managed runtime was active — those render as a single unlabelled column. #[serde(default)] pub(super) channel: Option, + /// Exact GFX target the active runtime was installed for. `None` on + /// artifacts predating the field or for a runtime that records none. + #[serde(default)] + pub(super) gfx_target: Option, + /// Package source the active runtime was installed from: index URL plus + /// source layout generation, as the runtime's own record has it. + #[serde(default)] + pub(super) source: Option, +} + +/// Join the present parts with " · ", skipping absent ones. +fn join_parts(parts: [Option; N]) -> String { + parts.into_iter().flatten().collect::>().join(" · ") } impl PlatformVersions { + /// "GFX gfx1100 · source ()" for the summary-matrix + /// Platform cell, skipping what is unknown. Empty when neither is known. + pub(super) fn provenance(&self) -> String { + join_parts([ + self.gfx_target.as_deref().map(|v| format!("GFX {v}")), + self.source.as_deref().map(|v| format!("source {v}")), + ]) + } + /// Rendered "os X · ROCm Y · vLLM Z · lemonade W" line for a column heading, /// skipping absent components. Empty when nothing is known. pub(super) fn summary(&self) -> String { - [ + join_parts([ self.os.as_deref().map(|v| format!("OS {v}")), self.rocm.as_deref().map(|v| format!("ROCm {v}")), self.vllm.as_deref().map(|v| format!("vLLM {v}")), self.lemonade.as_deref().map(|v| format!("lemonade {v}")), - ] - .into_iter() - .flatten() - .collect::>() - .join(" · ") + ]) } /// The software-stack versions (ROCm/vLLM/lemonade, WITHOUT the OS — that goes /// in its own cell), for the summary-matrix Platform cell. Empty when none are /// known (e.g. mock, which has no installed runtime). pub(super) fn platform_stack(&self) -> String { - [ + join_parts([ self.rocm.as_deref().map(|v| format!("ROCm {v}")), self.vllm.as_deref().map(|v| format!("vLLM {v}")), self.lemonade.as_deref().map(|v| format!("lemonade {v}")), - ] - .into_iter() - .flatten() - .collect::>() - .join(" · ") + ]) } } diff --git a/crates/rocm-core/Cargo.toml b/crates/rocm-core/Cargo.toml index 48afe6fee4..aaaa52c643 100644 --- a/crates/rocm-core/Cargo.toml +++ b/crates/rocm-core/Cargo.toml @@ -14,6 +14,10 @@ workspace = true # Lets the E2E suite re-root the hardware probes under a simulated host; see # `src/hardware_root.rs`. Never enabled in a release build. e2e-test-hooks = [] +# Compiles the `foreign_temp_probe` test probe so a dependent crate's tests can +# use it. Enabled only from a `[dev-dependencies]` entry; never in a shipped +# build. +test-support = [] [build-dependencies] cc = "1" @@ -45,7 +49,10 @@ ureq = { version = "2.12", features = ["native-certs"] } # # Also used by `disk_space`, for the same reason: its unit formatting, margin # arithmetic and mount selection are small total functions whose contracts are -# easy to state over ALL inputs and awkward to cover with examples. +# easy to state over ALL inputs and awkward to cover with examples. And by the +# integration suites under `tests/` (download digest gate, signature +# verification, model selection and readiness), whose generators aim at near +# misses no example list enumerates. # Test-only, so it adds nothing to any shipped binary. proptest = { version = "1", default-features = false, features = ["std"] } # Archive-extraction property tests (`src/archive_props.rs`) build hostile @@ -55,7 +62,7 @@ flate2 = "1" # its text, so the spelling and layout of a literal do not matter to it (its doc # comment lists what it still cannot see). Test-only; already in the lockfile # through the derive macros. -syn = { version = "2", features = ["full", "visit"] } +syn = { version = "3", features = ["full", "visit"] } tar = "0.4" tempfile = "3" # `report_trend` writes its own civil calendar so the shipped crate needs no date diff --git a/crates/rocm-core/src/anthropic_endpoint.rs b/crates/rocm-core/src/anthropic_endpoint.rs new file mode 100644 index 0000000000..744ae7d023 --- /dev/null +++ b/crates/rocm-core/src/anthropic_endpoint.rs @@ -0,0 +1,123 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The Anthropic-compatible `/v1/messages` surface of a managed service. +//! +//! An engine speaks the OpenAI API. A service additionally gets an Anthropic +//! Messages endpoint because `rocm` fronts the engine with a translation proxy +//! ([`AnthropicEndpointMode::Proxy`]), even when the engine has a `/v1/messages` +//! route of its own: an engine's route only accepts the model name it serves, +//! while an Anthropic client names a Claude model. The process that runs the +//! engine publishes the proxy's address under the state-file keys below, and [`crate::ManagedServiceRecord::refresh_from_engine_state`] adopts +//! it, so every reader (`rocm serve`, `rocm services list`, the readiness probe) +//! sees the same endpoint. + +use std::path::Path; +use std::time::{Duration, Instant}; + +use crate::openai_models_endpoint_has_model; + +/// Engine state-file key naming the Anthropic-compatible endpoint's base URL. +pub const ANTHROPIC_ENDPOINT_URL_STATE_KEY: &str = "anthropic_endpoint_url"; + +/// Engine state-file key recording how the Anthropic endpoint is provided. +pub const ANTHROPIC_ENDPOINT_MODE_STATE_KEY: &str = "anthropic_endpoint_mode"; + +/// How a service provides its Anthropic-compatible endpoint. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum AnthropicEndpointMode { + /// A `rocm`-hosted proxy translates Anthropic requests to the engine's + /// OpenAI API, on its own port. + Proxy, +} + +impl AnthropicEndpointMode { + /// The value stored under [`ANTHROPIC_ENDPOINT_MODE_STATE_KEY`]. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Proxy => "proxy", + } + } +} + +/// Whether the Anthropic endpoint at `anthropic_url` is up and fronts `model`. +/// +/// The proxy answers `GET /v1/models` by forwarding to the engine. So the same cheap listing check that gates +/// the OpenAI endpoint also covers this one, without queuing a generation. +pub fn anthropic_endpoint_ready( + anthropic_url: &str, + model: Option<&str>, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> bool { + openai_models_endpoint_has_model(anthropic_url, model, endpoint_api_key, timeout) + .unwrap_or(false) +} + +/// The Anthropic endpoint the serving process has published in `state_path`, +/// waiting up to `wait` for it to appear. +/// +/// The endpoint is published a moment after the engine first lists its model, so +/// a caller that has just seen the service become ready can still find nothing +/// there. `Duration::ZERO` reads once. +pub fn wait_for_published_anthropic_endpoint(state_path: &Path, wait: Duration) -> Option { + let started = Instant::now(); + loop { + let published = std::fs::read(state_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(|state| { + state + .get(ANTHROPIC_ENDPOINT_URL_STATE_KEY) + .and_then(serde_json::Value::as_str) + .filter(|url| !url.trim().is_empty()) + .map(str::to_owned) + }); + if published.is_some() || started.elapsed() >= wait { + return published; + } + std::thread::sleep(Duration::from_millis(200)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn published_endpoint_is_read_from_state_and_absent_when_unpublished() { + let dir = std::env::temp_dir().join(format!( + "rocm-core-anthropic-published-{}-{}", + std::process::id(), + crate::unix_time_millis() + )); + std::fs::create_dir_all(&dir).unwrap(); + let state = dir.join("state.json"); + assert_eq!( + wait_for_published_anthropic_endpoint(&state, Duration::ZERO), + None + ); + std::fs::write(&state, r#"{"status": "ready"}"#).unwrap(); + assert_eq!( + wait_for_published_anthropic_endpoint(&state, Duration::ZERO), + None + ); + std::fs::write( + &state, + r#"{"anthropic_endpoint_url": "http://127.0.0.1:41000"}"#, + ) + .unwrap(); + assert_eq!( + wait_for_published_anthropic_endpoint(&state, Duration::ZERO).as_deref(), + Some("http://127.0.0.1:41000") + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn mode_strings_are_the_values_published_in_state() { + assert_eq!(AnthropicEndpointMode::Proxy.as_str(), "proxy"); + } +} diff --git a/crates/rocm-core/src/archive_props.rs b/crates/rocm-core/src/archive_props.rs index ec0d1f9202..8a6efd92b5 100644 --- a/crates/rocm-core/src/archive_props.rs +++ b/crates/rocm-core/src/archive_props.rs @@ -1094,9 +1094,10 @@ pub fn run_property( // `PROPTEST_CASES` (proptest's own knob, which an explicit case count // would otherwise override) scales every property up for a longer hunt. // Never down: the reach floors below are percentages, which a handful of - // cases cannot meet. The same rule as rocm-core's `proptest_cases`, kept - // inline because this file is also `#[path]`-included into `rocm` and the - // Lemonade engine, where `crate::test_support` is a different module. + // cases cannot meet. The same rule as `tests/common/proptest_cases.rs`, + // kept inline because this file is also `#[path]`-included into `rocm` and + // the Lemonade engine, and a `#[path]` inside it would resolve relative to + // each includer. let requested = std::env::var("PROPTEST_CASES").ok(); let cases = match requested.as_deref().map(str::parse::) { Some(Ok(requested)) if requested >= cases => requested, diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs new file mode 100644 index 0000000000..e63f3e70b8 --- /dev/null +++ b/crates/rocm-core/src/atomic_write.rs @@ -0,0 +1,1565 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Atomic file replacement, shared by the `rocm` and `rocmd` apps, `rocm-core`'s +//! own config, automation and service-record writers, and the Lemonade engine. +//! +//! [`write_file_atomically`] stages the bytes in a fresh sibling temp file and +//! then publishes it over the destination in one step, so a reader sees either +//! the previous complete file or the new complete file, never a partial one, +//! and the destination is not removed before the new file takes its place. +//! +//! The temp file is named `{file name}.tmp-{pid}-{millis}-{attempt}` and +//! reserved with `create_new`, so two writers — threads of one process or +//! separate processes — can never write through the same temp file, even when +//! their pid and millisecond collide; a collision just moves on to the next +//! attempt. A failed write or publish removes its temp file, with the one +//! exception described below. +//! +//! On Windows the publish tries `rename` first. `std::fs::rename` replaces an +//! existing file in one step, so a lock-free reader (`RocmCliConfig::load`, the +//! automation stores) finds the old or the new file and never an absent one. +//! Only when that rename fails and the destination exists, which is the case of +//! another process holding it open without delete sharing, does the publish +//! fall back to `ReplaceFileW`. The concurrency tests below run on every +//! platform and publish by `rename`; two `ReplaceFileW` publishes of one file in +//! flight at once are not exercised by any test. +//! +//! A publish by `rename` leaves the destination with the staged file's +//! attributes and the ACL its directory hands down, where `ReplaceFileW` carries +//! over those of the file it replaces. No caller of this module sets a hidden or +//! read-only flag or a custom ACL on a file it writes, so nothing is lost today. +//! A file that must keep such settings across a rewrite should not rely on this +//! writer to carry them over. +//! +//! The fallback is the one path that can leave the destination absent: +//! Microsoft documents that `ReplaceFileW`, called without a backup name, can +//! fail after it has already moved the old file off the destination name +//! (`ERROR_UNABLE_TO_MOVE_REPLACEMENT` and `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2`). +//! The publish then retries once as a plain rename. If that fails too, the +//! temp file is kept rather than removed, since it holds the only copy of the +//! new contents, and the error is a [`KeptReplacement`] naming it. The +//! decision rests on the error code the replace returned, not on whether the +//! destination existed beforehand, so a second writer that created the +//! destination in between cannot turn it into a deletion. +//! +//! Durable as well as atomic: the temp file is flushed to disk before it is +//! published, so the new name never points at data still in the cache. On +//! Unix the directory is flushed after the rename too, so a power loss leaves +//! the old file or the new one rather than an empty one. Windows has no +//! directory flush here; there the rename's durability is the filesystem's. +//! +//! Replacing a file this way must not quietly change what the user set up: +//! +//! - on Unix the new file keeps the old one's permission bits, applied to the +//! temp file before any byte is written into it (on Windows nothing is +//! copied; see the note on attributes above); +//! - a destination that is a symbolic link — a dotfiles checkout linking +//! `config.json`, say — has its *target* replaced through +//! [`write_file_atomically`], so the link survives. A link loop is an error. +//! +//! This makes each file individually atomic. It does not serialize a +//! read-modify-write across processes: two writers that each read, change and +//! write the same file are still last-writer-wins. + +use crate::disk_space; +use crate::unix_time_millis; +use anyhow::{Context, Result, bail}; +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; + +/// How many symbolic links [`write_file_atomically`] follows before giving up, +/// matching the usual kernel limit (`MAXSYMLINKS` on Linux). +const MAX_SYMLINK_HOPS: usize = 40; + +/// The raw values of Windows' `ERROR_UNABLE_TO_MOVE_REPLACEMENT` (1176) and +/// `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2` (1177), the two codes with which +/// `ReplaceFileW` reports that it moved the old file off the destination name +/// and then failed. Plain numbers so the decision built on them can be tested +/// on every platform; a Windows-only test ties them to `windows-sys`. +#[cfg(any(windows, test))] +const PARTIAL_REPLACE_ERROR_CODES: [i32; 2] = [1176, 1177]; + +/// Whether a failed `ReplaceFileW` left the destination name empty. +#[cfg(any(windows, test))] +fn is_partial_replace(error: &io::Error) -> bool { + error + .raw_os_error() + .is_some_and(|code| PARTIAL_REPLACE_ERROR_CODES.contains(&code)) +} + +/// A publish that took the old destination away and could not put the new +/// file in its place: the replace failed part-way, and so did the rename +/// retried after it. +#[derive(Debug)] +struct DestinationRemoved { + replace: io::Error, + retry: io::Error, +} + +impl fmt::Display for DestinationRemoved { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}; retrying as a plain rename also failed: {}", + self.replace, self.retry + ) + } +} + +impl std::error::Error for DestinationRemoved {} + +/// Test seam: the error a publish returns when it took the old destination +/// away and could not put the new file in its place, which is what Windows' +/// partial `ReplaceFileW` followed by a failed rename produces. +/// +/// `pub` only so tests in other crates can drive that path through +/// [`write_file_atomically_with_publish`] on any platform; production code +/// never builds one. +#[doc(hidden)] +pub fn destination_removed_error(replace: io::Error, retry: io::Error) -> io::Error { + io::Error::new(retry.kind(), DestinationRemoved { replace, retry }) +} + +fn removed_the_destination(error: &io::Error) -> bool { + error + .get_ref() + .is_some_and(::is::) +} + +/// The error of a publish that took the old file away and kept the new +/// contents in the temp file, which then holds the only copy. +/// +/// A caller that knows better than to want those contents back — one that has +/// already put the previous file back, say — can downcast to this to find the +/// kept file and remove it. +#[derive(Debug)] +pub struct KeptReplacement { + destination: PathBuf, + kept: PathBuf, + cause: io::Error, +} + +impl KeptReplacement { + /// The path the publish was meant to replace, which is now missing. + pub fn destination(&self) -> &Path { + &self.destination + } + + /// The temp file holding the new contents. + pub fn kept(&self) -> &Path { + &self.kept + } + + /// Why the publish failed. + pub const fn cause(&self) -> &io::Error { + &self.cause + } +} + +impl fmt::Display for KeptReplacement { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "failed to publish {}, and the replace took the old file away; the new contents \ + are kept in {} - rename that file to {} to restore it", + self.destination.display(), + self.kept.display(), + self.destination.display() + ) + } +} + +impl std::error::Error for KeptReplacement { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.cause) + } +} + +/// What to do after `ReplaceFileW` failed with `error`: give up when the +/// destination is untouched, and otherwise put the replacement in place with +/// `rename`, which is a plain rename in production and injected by tests. +#[cfg(any(windows, test))] +fn recover_partial_replace( + error: io::Error, + replacement: &Path, + path: &Path, + rename: R, +) -> io::Result<()> +where + R: FnOnce(&Path, &Path) -> io::Result<()>, +{ + if !is_partial_replace(&error) { + return Err(error); + } + rename(replacement, path).map_err(|retry| destination_removed_error(error, retry)) +} + +/// A temp path next to `path`, preserving the full file name so a +/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes +/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). +fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { + let parent = path.parent().context("file path has no parent directory")?; + let mut file_name = path + .file_name() + .context("file path has no file name")? + .to_os_string(); + file_name.push(".tmp-"); + file_name.push(suffix); + Ok(parent.join(file_name)) +} + +fn process_temp_id() -> String { + format!("{}-{}", std::process::id(), unix_time_millis()) +} + +/// Replace `path` with `bytes` atomically, creating missing parent directories. +pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { + let temp_id = process_temp_id(); + write_file_atomically_with( + path, + bytes, + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + || {}, + ) +} + +/// [`write_file_atomically`] with test seams for the temp name and publish point. +/// +/// `suffix_for_attempt` names the temp file for each attempt, and +/// `before_publish` runs after the temp file is fully written and before it is +/// published. Both exist so tests can force a temp-name collision and hold +/// writers at the publish point; production code calls +/// [`write_file_atomically`]. +fn write_file_atomically_with( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, + before_publish: P, +) -> Result<()> +where + S: FnMut(u32) -> OsString, + P: FnOnce(), +{ + write_file_atomically_with_publish( + path, + bytes, + suffix_for_attempt, + before_publish, + publish_temp_file, + ) +} + +/// Test seam: [`write_file_atomically`] with the temp name, a hook before the +/// publish, and the publish step itself injected. +/// +/// Tests use it to force a temp-name collision, to hold writers at the publish +/// point, to observe the staged file and the destination at the moment of +/// publication, or to make the publication fail. Pass [`publish_temp_file`] +/// for the real publish. Production code calls [`write_file_atomically`]. +/// +/// `pub` only because `apps/rocm`'s metadata-cache tests drive it across the +/// crate boundary; hidden from the docs because it is not an API to build on. +#[doc(hidden)] +pub fn write_file_atomically_with_publish( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, + before_publish: P, + publish: F, +) -> Result<()> +where + S: FnMut(u32) -> OsString, + P: FnOnce(), + F: FnOnce(&Path, &Path) -> io::Result<()>, +{ + let path = resolve_symlink_target(path)?; + let tmp = stage_file_for_atomic_publish_with(&path, bytes, suffix_for_attempt)?; + before_publish(); + publish_staged_file_with(&tmp, &path, publish) +} + +/// The file a write to `path` should replace: `path` itself, or — when it is +/// a symbolic link — the file the link chain ends at, so the link survives. +/// A dangling link resolves to where its target would be, which is created. +fn resolve_symlink_target(path: &Path) -> Result { + let mut current = path.to_path_buf(); + for _ in 0..MAX_SYMLINK_HOPS { + match fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + let target = fs::read_link(¤t) + .with_context(|| format!("failed to read link {}", current.display()))?; + current = if target.is_absolute() { + target + } else { + current + .parent() + .context("link path has no parent directory")? + .join(target) + }; + } + _ => return Ok(current), + } + } + bail!( + "too many levels of symbolic links resolving {}", + path.display() + ) +} + +/// Write `bytes` to a fresh, uniquely named sibling of `path` and return the +/// staged file's path without publishing it. +/// +/// The caller owns the staged file and must remove it once done with it; on a +/// staging failure nothing is left behind. To *replace* `path` with the bytes, +/// call [`write_file_atomically`] instead, which also removes the staged file +/// when the publish fails. +/// +/// Unlike [`write_file_atomically`], `path` is taken as given: a symbolic link +/// there is not resolved, so the temp file sits next to the link, and +/// publishing it over that `path` replaces the link with a regular file. +pub fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { + let temp_id = process_temp_id(); + stage_file_for_atomic_publish_with(path, bytes, |attempt| { + OsString::from(format!("{temp_id}-{attempt}")) + }) +} + +fn stage_file_for_atomic_publish_with( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, +) -> Result +where + S: FnMut(u32) -> OsString, +{ + // Flushed before it can be published: a rename that reaches the disk + // before the data would leave an empty file under the real name after a + // crash. A full disk can surface in the flush rather than in `write_all`. + stage_file_for_atomic_publish_with_writer(path, bytes, suffix_for_attempt, |file, bytes| { + write_then_flush(file, bytes, fs::File::sync_all) + }) +} + +/// [`stage_file_for_atomic_publish_with`] with the write into the reserved +/// temp file injected, so a test can make it fail the way a full disk does +/// without filling one. +fn stage_file_for_atomic_publish_with_writer( + path: &Path, + bytes: &[u8], + mut suffix_for_attempt: S, + write: W, +) -> Result +where + S: FnMut(u32) -> OsString, + W: FnOnce(&mut fs::File, &[u8]) -> io::Result<()>, +{ + let parent = path.parent().context("file path has no parent directory")?; + fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; + + let mut reserved = None; + for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { + let suffix = suffix_for_attempt(attempt); + let tmp = temp_sibling_path(path, &suffix)?; + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(file) => { + reserved = Some((tmp, file)); + break; + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| format!("failed to create {}", tmp.display())); + } + } + } + let Some((tmp, mut file)) = reserved else { + bail!( + "failed to reserve a temporary file next to {} after {} attempts", + path.display(), + ATOMIC_WRITE_TEMP_ATTEMPTS + ); + }; + + // Before any byte is written, so a private file's new contents are never + // readable under the wider default mode, even for a moment. + if let Err(error) = keep_existing_permissions(path, &tmp) { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(error); + } + if let Err(error) = write(&mut file, bytes) { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(disk_space::map_write_error(error, path)); + } + drop(file); + Ok(tmp) +} + +/// Write `bytes` into the staged file, then flush it to disk with `flush`, so +/// the data reaches the disk before the rename can. A flush error fails the +/// write like any other, so the caller removes the temp file. `flush` is +/// injected so a test can see that it runs, and after the bytes; production +/// passes `File::sync_all`. +fn write_then_flush(file: &mut fs::File, bytes: &[u8], flush: F) -> io::Result<()> +where + F: FnOnce(&fs::File) -> io::Result<()>, +{ + file.write_all(bytes)?; + flush(file) +} + +/// Give the staged file the permission bits of the file it will replace, so a +/// `0600` file is not republished as `0644`. Nothing to keep for a new file. +#[cfg(unix)] +fn keep_existing_permissions(path: &Path, tmp: &Path) -> Result<()> { + match fs::metadata(path) { + Ok(existing) => fs::set_permissions(tmp, existing.permissions()) + .with_context(|| format!("failed to copy the permissions of {}", path.display())), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error) + .with_context(|| format!("failed to read the permissions of {}", path.display())), + } +} + +/// Nothing is copied on Windows: a publish by rename leaves the new file with +/// the staged file's attributes and the ACL its directory hands down (see the +/// module doc), and no caller sets a mode-like attribute it needs kept. +#[cfg(not(unix))] +#[allow(clippy::unnecessary_wraps)] +// Platform stub. The `cfg` sibling calls non-const code, so making only this +// arm `const` would give the two platforms different signatures and push +// `missing_const_for_fn` onto every caller in turn. +#[allow(clippy::missing_const_for_fn)] +fn keep_existing_permissions(_path: &Path, _tmp: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { + publish_staged_file_with(tmp, path, publish_temp_file) +} + +/// Publish a staged file over `path`, removing the temp file if publishing +/// fails — unless the failed publish took the existing file away. +/// +/// A rename either replaces the destination or leaves it alone, but Windows' +/// `ReplaceFileW` can fail after moving the old file off the destination name +/// (see `recover_partial_replace`). Removing the temp file then would leave no +/// copy of the contents under any name the user can find, so it is kept and +/// the error is a [`KeptReplacement`] that says where. The publish step itself +/// reports that case, so it is told apart by the error rather than by a look +/// at the destination taken before the publish, which another writer can make +/// stale. +fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> +where + F: FnOnce(&Path, &Path) -> io::Result<()>, +{ + publish_staged_file_with_dir_sync(tmp, path, publish, sync_parent_dir) +} + +/// [`publish_staged_file_with`] with the directory flush injected, so a test +/// can see that a successful publish flushes the destination's directory and +/// a failed one does not; production passes `sync_parent_dir`. +fn publish_staged_file_with_dir_sync( + tmp: &Path, + path: &Path, + publish: F, + sync_dir: D, +) -> Result<()> +where + F: FnOnce(&Path, &Path) -> io::Result<()>, + D: FnOnce(&Path), +{ + let Err(error) = publish(tmp, path) else { + sync_dir(path); + return Ok(()); + }; + if removed_the_destination(&error) && tmp.exists() { + return Err(anyhow::Error::new(KeptReplacement { + destination: path.to_path_buf(), + kept: tmp.to_path_buf(), + cause: error, + })); + } + let _ = fs::remove_file(tmp); + Err(error).with_context(|| format!("failed to publish {}", path.display())) +} + +/// Flush the directory entry the rename just changed, so the new name — not +/// the old file, or no file — is what a crash leaves behind. +/// +/// Best-effort: the file is already published, and some filesystems refuse to +/// sync a directory. Reporting failure here would claim the write failed when +/// it did not. +#[cfg(unix)] +fn sync_parent_dir(path: &Path) { + // A bare relative name has the empty path as its parent, which cannot be + // opened; its folder is the current one. + let parent = match path.parent() { + Some(parent) if parent.as_os_str().is_empty() => Path::new("."), + Some(parent) => parent, + None => return, + }; + if let Ok(dir) = fs::File::open(parent) { + let _ = dir.sync_all(); + } +} + +/// Not done on Windows: the rename's durability is left to the filesystem. +#[cfg(not(unix))] +// Platform stub. The `cfg` sibling calls non-const code, so making only this +// arm `const` would give the two platforms different signatures and push +// `missing_const_for_fn` onto every caller in turn. +#[allow(clippy::missing_const_for_fn)] +fn sync_parent_dir(_path: &Path) {} + +/// Test seam: move the staged file `tmp` over `path` in one step, replacing +/// any existing file without first removing it. +/// +/// The real publish step to pass to [`write_file_atomically_with_publish`]. +/// `pub` only for that cross-crate test use; it does not remove `tmp` on +/// failure, so production code calls [`write_file_atomically`] instead. +#[cfg(not(windows))] +#[doc(hidden)] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { + fs::rename(tmp, path) +} + +/// Test seam: move the staged file `tmp` over `path` in one step, replacing +/// any existing file without first removing it. +/// +/// The real publish step to pass to [`write_file_atomically_with_publish`]. +/// `pub` only for that cross-crate test use; it does not remove `tmp` on +/// failure, so production code calls [`write_file_atomically`] instead. +#[cfg(windows)] +#[doc(hidden)] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { + publish_rename_first(tmp, path, rename_file, replace_file_windows) +} + +/// The Windows publish order, with both steps passed in so a test can check it +/// on any host: `rename` first, because it replaces in one step and never +/// leaves the destination absent; `replace` only when the rename failed and the +/// destination exists. A failed rename onto a destination that does not exist is +/// returned as it is, since `replace` has nothing to replace. +#[cfg(any(windows, test))] +fn publish_rename_first(tmp: &Path, path: &Path, rename: R, replace: P) -> io::Result<()> +where + R: FnOnce(&Path, &Path) -> io::Result<()>, + P: FnOnce(&Path, &Path) -> io::Result<()>, +{ + match rename(tmp, path) { + Ok(()) => Ok(()), + Err(rename_error) => { + if path.try_exists()? { + replace(path, tmp) + } else { + Err(rename_error) + } + } + } +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; + + let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let replacement_wide: Vec = replacement + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect(); + + // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and + // remain alive for the duration of the synchronous Windows API call. The + // optional backup, exclude, and reserved pointers are intentionally null. + let replaced = unsafe { + ReplaceFileW( + path_wide.as_ptr(), + replacement_wide.as_ptr(), + std::ptr::null(), + 0, + std::ptr::null(), + std::ptr::null(), + ) + }; + if replaced != 0 { + return Ok(()); + } + // Microsoft documents two partial outcomes for a call without a backup + // name, and in both the replacement is still under its own name: + // `ERROR_UNABLE_TO_MOVE_REPLACEMENT` ("the replaced file no longer exists + // and the replacement file exists under its original name") and + // `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2` ("the file to be replaced still + // exists with a different name"). Either way nothing is at `path` any + // more. One plain rename puts the new contents there; if that fails too, + // `publish_staged_file_with` keeps the replacement file. + recover_partial_replace(io::Error::last_os_error(), replacement, path, rename_file) +} + +/// `fs::rename` with both paths borrowed, the shape `recover_partial_replace` +/// and `publish_rename_first` take. `fs::rename` itself is generic over +/// `AsRef`, so passed directly it is pinned to one lifetime per argument +/// and does not satisfy their `FnOnce(&Path, &Path)` bounds. +#[cfg(windows)] +fn rename_file(from: &Path, to: &Path) -> io::Result<()> { + fs::rename(from, to) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch_dir(label: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "rocm-core-{label}-{}-{}", + std::process::id(), + unix_time_millis() + )) + } + + /// A write that runs out of space while staging says so in words, and + /// leaves no temp file behind to keep the disk full. Drives the real + /// staging path — the one both [`write_file_atomically`] and + /// [`stage_file_for_atomic_publish`] go through — with only the byte write + /// into the reserved temp file swapped for one that fails like a full + /// disk, so the mapping is checked where it is applied rather than on its + /// own. + #[test] + fn staging_into_a_full_disk_reports_it_clearly_and_removes_the_temp_file() { + let root = scratch_dir("atomic-full-disk"); + let destination = root.join("rocm.tar.gz"); + + let error = stage_file_for_atomic_publish_with_writer( + &destination, + b"payload", + |attempt| OsString::from(format!("full-disk-{attempt}")), + |_, _| Err(io::Error::from(io::ErrorKind::StorageFull)), + ) + .expect_err("a write that hits a full disk must fail"); + + let leftovers: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + let _ = fs::remove_dir_all(&root); + let text = format!("{error:#}"); + assert!(text.contains("ran out of disk space"), "{text}"); + // The user knows the file they asked for, not the temp file that was + // already removed, so the message names the destination. + assert!( + text.contains(&destination.display().to_string()) && !text.contains(".tmp-"), + "{text}" + ); + assert!( + leftovers.is_empty(), + "the failed staging left files behind: {leftovers:?}" + ); + } + + /// When every temp name the writer tries is taken, it gives up after a + /// bounded number of tries with an error that says so, rather than + /// looping or writing through a file another writer reserved. + #[test] + fn staging_gives_up_when_every_temp_name_is_taken() { + let root = scratch_dir("atomic-names-exhausted"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("active.json"); + let taken = temp_sibling_path(&destination, OsStr::new("taken")).unwrap(); + fs::write(&taken, b"another writer's staged file").unwrap(); + let mut tries = 0; + + let error = stage_file_for_atomic_publish_with(&destination, b"payload", |_| { + tries += 1; + OsString::from("taken") + }) + .expect_err("a writer that finds every name taken must fail"); + + let foreign = fs::read(&taken); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + let _ = fs::remove_dir_all(&root); + let text = format!("{error:#}"); + assert_eq!(tries, ATOMIC_WRITE_TEMP_ATTEMPTS); + assert!( + text.contains(&format!("after {ATOMIC_WRITE_TEMP_ATTEMPTS} attempts")), + "{text}" + ); + assert_eq!(foreign.unwrap(), b"another writer's staged file"); + assert_eq!(leftovers, vec![OsString::from("active.json.tmp-taken")]); + } + + /// The temp name keeps every extension, so a cleanup sweep over a cache + /// directory can still tell what a leftover was going to be. + #[test] + fn temp_sibling_path_preserves_multi_dot_file_names() { + let temp = + temp_sibling_path(Path::new("/tmp/cache/sdk.tar.gz"), OsStr::new("test")).unwrap(); + let name = temp.file_name().unwrap().to_string_lossy().into_owned(); + assert_eq!(name, "sdk.tar.gz.tmp-test"); + assert_eq!(temp.parent().unwrap(), Path::new("/tmp/cache")); + } + + /// Same property end to end: the temp file a write stages, observed at the + /// moment it is published, keeps the destination's full name, and the + /// publish leaves only the destination behind. + #[test] + fn write_file_atomically_temp_name_preserves_multi_dot_file_names() { + let root = scratch_dir("atomic-name"); + let destination = root.join("artifact.tar.gz"); + let temp_id = process_temp_id(); + let mut staged_name = None; + + write_file_atomically_with_publish( + &destination, + b"payload", + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + || {}, + |tmp, path| { + staged_name = tmp.file_name().map(OsStr::to_os_string); + publish_temp_file(tmp, path) + }, + ) + .unwrap(); + + let names: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + let published = fs::read(&destination); + let _ = fs::remove_dir_all(&root); + assert_eq!( + staged_name, + Some(OsString::from(format!("artifact.tar.gz.tmp-{temp_id}-0"))) + ); + assert_eq!(published.unwrap(), b"payload"); + assert_eq!(names, vec!["artifact.tar.gz".to_owned()]); + } + + /// The Windows publish order: `rename` is tried first so the destination is + /// never absent, and `ReplaceFileW` is reached only after a failed rename + /// onto a destination that exists. Runs on every host because the order is + /// a plain function of the two injected steps. + #[test] + fn windows_publish_renames_first_and_skips_the_replace_when_it_succeeds() { + let root = scratch_dir("publish-rename-first"); + fs::create_dir_all(&root).unwrap(); + // The destination must exist: with it absent, no ordering of the two + // steps would reach the replace, so the test could not tell them apart. + let existing = root.join("existing"); + fs::write(&existing, b"old").unwrap(); + let replaced = std::cell::Cell::new(false); + let renamed = std::cell::Cell::new(false); + publish_rename_first( + Path::new("tmp"), + &existing, + |_, _| { + renamed.set(true); + Ok(()) + }, + |_, _| { + replaced.set(true); + Ok(()) + }, + ) + .expect("a successful rename is the whole publish"); + let _ = fs::remove_dir_all(&root); + assert!(renamed.get(), "the rename must be tried first"); + assert!( + !replaced.get(), + "ReplaceFileW must not run after a rename that worked" + ); + } + + #[test] + fn windows_publish_falls_back_to_the_replace_only_for_an_existing_destination() { + let root = scratch_dir("publish-fallback"); + fs::create_dir_all(&root).unwrap(); + let existing = root.join("existing"); + fs::write(&existing, b"old").unwrap(); + let sharing = || Err(io::Error::from(io::ErrorKind::PermissionDenied)); + + let replaced = std::cell::Cell::new(None); + publish_rename_first( + Path::new("tmp"), + &existing, + |_, _| sharing(), + |path, tmp| { + replaced.set(Some((path.to_path_buf(), tmp.to_path_buf()))); + Ok(()) + }, + ) + .expect("the fallback result is the publish result"); + assert_eq!( + replaced.take(), + Some((existing, PathBuf::from("tmp"))), + "the fallback replaces the destination with the staged file" + ); + + let missing = root.join("missing"); + let called = std::cell::Cell::new(false); + let error = publish_rename_first( + Path::new("tmp"), + &missing, + |_, _| sharing(), + |_, _| { + called.set(true); + Ok(()) + }, + ) + .expect_err("a failed rename onto a missing destination is an error"); + let _ = fs::remove_dir_all(&root); + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert!( + !called.get(), + "there is nothing for ReplaceFileW to replace" + ); + } + + /// A reader that polls the destination while a writer replaces it over and + /// over must never find it missing: that is what a lock-free reader of the + /// config or an automation store would read as an empty store. A transient + /// permission error while a replace is in flight is tolerated; an absent + /// destination is not. + /// + /// This is a Windows-lane check. On Linux `rename` replaces atomically, so + /// the test passes with either publish order and would not notice the order + /// being reversed; it only discriminates on Windows, where a publish that + /// removes the destination before putting the new file in place leaves a + /// window the reader can fall into. What it still does not cover is the + /// `ReplaceFileW` fallback, which runs only when the rename fails and is the + /// one path that can leave the destination absent: it is reached by no test + /// that races a reader, and its order is pinned only by the injected-step + /// tests above. + #[test] + fn a_reader_never_finds_the_destination_absent_while_it_is_replaced() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + /// Tells the reader to stop when dropped, so a writer that panics + /// through `unwrap` still ends the polling thread instead of leaving it + /// to spin until the process exits. + struct StopReader(Arc); + impl Drop for StopReader { + fn drop(&mut self) { + self.0.store(true, Ordering::Relaxed); + } + } + + let root = scratch_dir("atomic-reader-gap"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("store.json"); + write_file_atomically(&destination, b"0").unwrap(); + + let done = Arc::new(AtomicBool::new(false)); + let (first_read, reader_started) = std::sync::mpsc::channel::<()>(); + let reader = { + let destination = destination.clone(); + let done = Arc::clone(&done); + std::thread::spawn(move || { + let mut reads = 0u32; + while !done.load(Ordering::Relaxed) { + match fs::read(&destination) { + Ok(_) => { + reads += 1; + if reads == 1 { + let _ = first_read.send(()); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Err(error); + } + Err(_) => {} + } + } + Ok(reads) + }) + }; + // Start writing only once the reader is polling, so a late-scheduled + // reader cannot miss every replacement and fail on timing alone. A + // reader that exits before its first read (the destination was absent) + // disconnects the channel; that is reported from its outcome below. + let stop_reader = StopReader(Arc::clone(&done)); + let started = reader_started.recv_timeout(std::time::Duration::from_secs(30)); + if started.is_ok() { + for round in 1..=300u32 { + write_file_atomically(&destination, round.to_string().as_bytes()).unwrap(); + } + } + drop(stop_reader); + let outcome = reader.join().unwrap(); + let _ = fs::remove_dir_all(&root); + let reads = outcome.expect("the destination was absent while it was being replaced"); + assert!( + started.is_ok() && reads > 0, + "the reader never read the destination at all: {started:?}" + ); + } + + #[test] + fn concurrent_atomic_writes_do_not_remove_a_published_destination() { + let root = scratch_dir("atomic-collision"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("sdk.tar.gz"); + let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] + .into_iter() + .enumerate() + .map(|(writer, bytes)| { + let destination = destination.clone(); + let before_publish = std::sync::Arc::clone(&before_publish); + std::thread::spawn(move || { + write_file_atomically_with( + &destination, + bytes, + |attempt| { + if attempt == 0 { + OsString::from("same-millisecond") + } else { + OsString::from(format!("same-millisecond-{writer}-{attempt}")) + } + }, + || { + before_publish.wait(); + }, + ) + }) + }) + .collect(); + + for writer in writers { + writer.join().unwrap().unwrap(); + } + let published = fs::read(&destination).expect("a writer must remain published"); + let _ = fs::remove_dir_all(&root); + assert!(published == b"first" || published == b"second"); + } + + #[test] + fn concurrent_cached_publications_use_distinct_staging_files() { + let root = scratch_dir("cache-publish-collision"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("index.body"); + let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] + .into_iter() + .enumerate() + .map(|(writer, bytes)| { + let destination = destination.clone(); + let before_publish = std::sync::Arc::clone(&before_publish); + std::thread::spawn(move || { + // Both threads ask for the same first name, as two writers + // sharing a pid and a millisecond would, so the collision + // happens every run rather than when the clock lines up. + let staged = + stage_file_for_atomic_publish_with(&destination, bytes, |attempt| { + if attempt == 0 { + OsString::from("same-millisecond") + } else { + OsString::from(format!("same-millisecond-{writer}-{attempt}")) + } + })?; + before_publish.wait(); + publish_staged_file(&staged, &destination) + }) + }) + .collect(); + + for writer in writers { + writer.join().unwrap().unwrap(); + } + let published = fs::read(&destination).expect("a cache writer must remain published"); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| name.to_string_lossy().contains(".tmp-")) + .collect(); + let _ = fs::remove_dir_all(&root); + assert!(published == b"first" || published == b"second"); + assert!( + leftovers.is_empty(), + "staged cache files leaked: {leftovers:?}" + ); + } + + #[test] + fn failed_cached_publication_preserves_destination_and_cleans_staging_file() { + let root = scratch_dir("cache-publish-failure"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("index.body"); + fs::write(&destination, b"published").unwrap(); + let staged = stage_file_for_atomic_publish(&destination, b"replacement").unwrap(); + + publish_staged_file_with(&staged, &destination, |_, _| { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "simulated cache publication failure", + )) + }) + .expect_err("simulated cache publication failure must be returned"); + + assert_eq!(fs::read(&destination).unwrap(), b"published"); + assert!( + !staged.exists(), + "failed publication leaked its staging file" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn failed_atomic_replace_preserves_destination_and_cleans_temp() { + let root = scratch_dir("atomic-replace-failure"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("sdk.tar.gz"); + fs::write(&destination, b"published").unwrap(); + + write_file_atomically_with_publish( + &destination, + b"replacement", + |attempt| OsString::from(format!("replace-failure-{attempt}")), + || {}, + |tmp, path| { + assert_eq!(fs::read(tmp).unwrap(), b"replacement"); + assert_eq!(path, destination); + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "simulated atomic replacement failure", + )) + }, + ) + .expect_err("simulated replacement failure must be returned"); + + assert_eq!(fs::read(&destination).unwrap(), b"published"); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + let _ = fs::remove_dir_all(&root); + assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); + } + + fn simulated_partial_replace() -> io::Error { + destination_removed_error( + io::Error::from_raw_os_error(PARTIAL_REPLACE_ERROR_CODES[0]), + io::Error::from(io::ErrorKind::PermissionDenied), + ) + } + + /// A publish that fails after taking the old file away — what Windows' + /// `ReplaceFileW` documents for `ERROR_UNABLE_TO_MOVE_REPLACEMENT` without + /// a backup name, when the rename retried after it fails too — must not + /// also delete the temp file, which then holds the only copy. The error + /// names it so the user can put it back, and a caller can find it. + #[test] + fn a_publish_that_removes_the_destination_keeps_the_new_contents() { + let root = scratch_dir("replace-partial"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + fs::write(&destination, b"old").unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"new", + |attempt| OsString::from(format!("replace-partial-{attempt}")), + || {}, + |_tmp, path| { + fs::remove_file(path).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + let message = format!("{error:#}"); + let reported = error + .downcast_ref::() + .map(|kept| (kept.kept().to_path_buf(), kept.destination().to_path_buf())); + + let kept = temp_sibling_path(&destination, OsStr::new("replace-partial-0")).unwrap(); + let kept_bytes = fs::read(&kept); + let destination_exists = destination.exists(); + let _ = fs::remove_dir_all(&root); + assert!(!destination_exists); + assert_eq!(kept_bytes.unwrap(), b"new", "the new contents were deleted"); + assert!( + message.contains(&kept.display().to_string()), + "the error does not say where the contents are: {message}" + ); + assert_eq!(reported, Some((kept, destination))); + } + + /// The race a pre-publish look at the destination gets wrong: no + /// destination when this writer starts, another writer publishes one, and + /// then this writer's replace takes it away and fails. The temp file still + /// holds the only copy, so it is still kept. + #[test] + fn a_publish_that_removes_a_destination_another_writer_created_keeps_the_new_contents() { + let root = scratch_dir("replace-partial-race"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"second", + |attempt| OsString::from(format!("replace-partial-race-{attempt}")), + || {}, + |_tmp, path| { + fs::write(path, b"first").unwrap(); + fs::remove_file(path).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + + let kept = temp_sibling_path(&destination, OsStr::new("replace-partial-race-0")).unwrap(); + let kept_bytes = fs::read(&kept); + let _ = fs::remove_dir_all(&root); + assert_eq!( + kept_bytes.unwrap(), + b"second", + "the new contents were deleted" + ); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + } + + /// When the temp file is gone too there is nothing to keep, so the error + /// must not point the user at it. + #[test] + fn a_partial_replace_that_lost_the_temp_file_does_not_claim_to_keep_it() { + let root = scratch_dir("replace-partial-no-temp"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + fs::write(&destination, b"old").unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"new", + |attempt| OsString::from(format!("replace-partial-no-temp-{attempt}")), + || {}, + |tmp, path| { + fs::remove_file(path).unwrap(); + fs::remove_file(tmp).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + let _ = fs::remove_dir_all(&root); + let message = format!("{error:#}"); + assert!( + error.downcast_ref::().is_none(), + "{message}" + ); + assert!(!message.contains("kept in"), "{message}"); + } + + /// Any other failed replace leaves the destination alone, so nothing is + /// retried and the error comes back as it was. + #[test] + fn a_replace_that_left_the_destination_alone_is_not_retried() { + let mut renamed = false; + let error = recover_partial_replace( + io::Error::from_raw_os_error(5), + Path::new("staged"), + Path::new("destination"), + |_, _| { + renamed = true; + Ok(()) + }, + ) + .expect_err("an ordinary failure is returned"); + assert!(!renamed); + assert_eq!(error.raw_os_error(), Some(5)); + assert!(!removed_the_destination(&error)); + } + + /// Both documented partial codes retry once as a rename of the staged + /// file onto the destination, and a successful retry is a success. + #[test] + fn a_partial_replace_is_retried_as_a_rename() { + for code in PARTIAL_REPLACE_ERROR_CODES { + let mut renamed = None; + recover_partial_replace( + io::Error::from_raw_os_error(code), + Path::new("staged"), + Path::new("destination"), + |from, to| { + renamed = Some((from.to_path_buf(), to.to_path_buf())); + Ok(()) + }, + ) + .unwrap_or_else(|error| panic!("code {code}: {error}")); + assert_eq!( + renamed, + Some((PathBuf::from("staged"), PathBuf::from("destination"))), + "code {code}" + ); + } + } + + /// A retry that fails too is reported as the case that keeps the temp file. + #[test] + fn a_partial_replace_whose_retry_fails_reports_the_destination_removed() { + for code in PARTIAL_REPLACE_ERROR_CODES { + let error = recover_partial_replace( + io::Error::from_raw_os_error(code), + Path::new("staged"), + Path::new("destination"), + |_, _| Err(io::Error::from(io::ErrorKind::PermissionDenied)), + ) + .expect_err("a failed retry is returned"); + assert!(removed_the_destination(&error), "code {code}: {error}"); + assert!(error.to_string().contains("retrying as a plain rename")); + } + } + + /// The plain numbers the partial-replace decision uses are the codes + /// `ReplaceFileW` actually returns. + #[cfg(windows)] + #[test] + fn partial_replace_codes_match_the_windows_constants() { + use windows_sys::Win32::Foundation::{ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, + }; + assert_eq!( + PARTIAL_REPLACE_ERROR_CODES.map(|code| u32::try_from(code).unwrap()), + [ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, + ERROR_UNABLE_TO_MOVE_REPLACEMENT_2 + ] + ); + } + + #[cfg(unix)] + #[test] + fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let file_name = OsString::from_vec(b"sdk-\xff.tar.gz".to_vec()); + let destination = Path::new("/tmp").join(&file_name); + let temp = temp_sibling_path(&destination, OsStr::new("collision")).unwrap(); + + let mut expected = file_name.into_vec(); + expected.extend_from_slice(b".tmp-collision"); + assert_eq!(temp.file_name().unwrap().as_bytes(), expected); + } + + /// Regression: a failed write must not leave a `.tmp-*` scratch file + /// behind. The name is unique per attempt, so before this an orphan + /// accumulated per retry — and when the failure is a full disk, those + /// orphans are exactly what keeps it full. + /// + /// Provokes the failure by pointing the destination at a non-empty + /// directory: the temp file is written, then neither the rename nor the + /// replace fallback can succeed. Portable, unlike an out-of-space test. + #[test] + fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { + let root = scratch_dir("atomic"); + let occupied = root.join("sdk.tar.gz"); + fs::create_dir_all(occupied.join("nested")).unwrap(); + fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); + + write_file_atomically(&occupied, b"payload") + .expect_err("renaming onto a non-empty directory should fail"); + + let leftovers: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains(".tmp-")) + .collect(); + let _ = fs::remove_dir_all(&root); + assert!( + leftovers.is_empty(), + "failed write left temp files behind: {leftovers:?}" + ); + } + + /// Mirrors the `/dev/shm` reproduction from the original report: a genuine + /// ENOSPC, not a rename failure standing in for one. + /// + /// Ignored by default because it fills `/dev/shm`, which is shared with + /// anything else on the host, so it is not safe to run concurrently. Run + /// with `cargo test -p rocm-core -- --ignored write_file_atomically_cleans_up`. + #[test] + #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] + fn write_file_atomically_cleans_up_temp_on_write_failure() { + let shm = Path::new("/dev/shm"); + if !shm.is_dir() { + eprintln!("skipping: /dev/shm unavailable"); + return; + } + let dir = shm.join(format!("rocm-core-enospc-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let dest = dir.join("artifact.tar.gz"); + // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. + let payload = vec![0u8; 256 * 1024 * 1024]; + + let mut failures = Vec::new(); + for _ in 0..2 { + write_file_atomically(&dest, &payload) + .expect_err("writing past the end of the filesystem should fail"); + failures.push( + fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(), + ); + } + let destination_exists = dest.exists(); + let _ = fs::remove_dir_all(&dir); + + for leftovers in &failures { + assert!( + leftovers.is_empty(), + "failed write left files behind: {leftovers:?}" + ); + } + assert!( + !destination_exists, + "destination must not exist after failure" + ); + } + + /// The names in `dir`, sorted. + #[cfg(unix)] + fn names_in(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + + /// Replacing a file keeps its permission bits: a `0600` file must not be + /// republished as `0644`, and an unusual mode survives as set. + #[cfg(unix)] + #[test] + fn write_file_atomically_keeps_the_existing_permissions() { + use std::os::unix::fs::PermissionsExt; + + let root = scratch_dir("permissions"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("config.json"); + for mode in [0o600, 0o640, 0o604] { + fs::write(&destination, b"old").unwrap(); + fs::set_permissions(&destination, fs::Permissions::from_mode(mode)).unwrap(); + + write_file_atomically(&destination, b"new").unwrap(); + + let after = fs::metadata(&destination).unwrap().permissions().mode() & 0o777; + assert_eq!(after, mode, "mode {mode:o} was not kept"); + assert_eq!(fs::read(&destination).unwrap(), b"new"); + } + let _ = fs::remove_dir_all(&root); + } + + /// The kept mode is on the temp file before the first byte reaches it, so + /// a private file's new contents are never readable under the default mode. + #[cfg(unix)] + #[test] + fn staging_applies_the_kept_mode_before_writing_any_byte() { + use std::os::unix::fs::PermissionsExt; + + // 0o604 is a mode no usual umask gives a new file (022 gives 0o644, + // 077 gives 0o600), so seeing it on the temp file means it was set. + for mode in [0o600, 0o604] { + let root = scratch_dir("permissions-before-write"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("config.json"); + fs::write(&destination, b"old").unwrap(); + fs::set_permissions(&destination, fs::Permissions::from_mode(mode)).unwrap(); + + let mut mode_while_writing = None; + stage_file_for_atomic_publish_with_writer( + &destination, + b"secret", + |attempt| OsString::from(format!("mode-{attempt}")), + |file, bytes| { + mode_while_writing = Some(file.metadata()?.permissions().mode() & 0o777); + file.write_all(bytes) + }, + ) + .unwrap(); + let _ = fs::remove_dir_all(&root); + + assert_eq!(mode_while_writing, Some(mode), "mode {mode:o}"); + } + } + + /// When the existing file's permissions cannot be read, staging fails + /// with that error and removes its temp file. A symlink loop at the + /// destination makes the read fail on Unix without root or chmod; + /// `stage_file_for_atomic_publish` takes the path as given, so the loop + /// reaches the permission read rather than the link resolution. + #[cfg(unix)] + #[test] + fn staging_that_cannot_read_the_existing_permissions_removes_its_temp_file() { + use std::os::unix::fs::symlink; + + let root = scratch_dir("permissions-unreadable"); + fs::create_dir_all(&root).unwrap(); + let a = root.join("a.json"); + let b = root.join("b.json"); + symlink(&b, &a).unwrap(); + symlink(&a, &b).unwrap(); + + let error = stage_file_for_atomic_publish(&a, b"x").expect_err("the read must fail"); + let names = names_in(&root); + let _ = fs::remove_dir_all(&root); + assert!( + format!("{error:#}").contains("failed to read the permissions of"), + "{error:#}" + ); + assert_eq!(names, ["a.json", "b.json"], "the temp file was left behind"); + } + + /// The staged bytes are flushed, after they are written, and a flush + /// error fails the write. Whether the flush reaches the disk cannot be + /// observed from a test; that it is asked for can. + #[test] + fn staged_bytes_are_flushed_after_they_are_written() { + let root = scratch_dir("flush-order"); + fs::create_dir_all(&root).unwrap(); + let path = root.join("staged"); + let mut file = fs::File::create(&path).unwrap(); + + let mut seen_at_flush = None; + write_then_flush(&mut file, b"payload", |_| { + seen_at_flush = Some(fs::read(&path).unwrap()); + Ok(()) + }) + .unwrap(); + let failed = write_then_flush(&mut file, b"more", |_| { + Err(io::Error::other("flush refused")) + }); + drop(file); + let _ = fs::remove_dir_all(&root); + + assert_eq!(seen_at_flush.as_deref(), Some(&b"payload"[..])); + assert_eq!(failed.unwrap_err().to_string(), "flush refused"); + } + + /// A publish that succeeds flushes the destination's directory once; one + /// that fails does not. + #[test] + fn a_successful_publish_flushes_the_directory_and_a_failed_one_does_not() { + let root = scratch_dir("dir-flush"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("target.json"); + let tmp = root.join("target.json.tmp-1"); + fs::write(&tmp, b"new").unwrap(); + + let mut flushed = Vec::new(); + publish_staged_file_with_dir_sync(&tmp, &destination, publish_temp_file, |path| { + flushed.push(path.to_path_buf()); + }) + .unwrap(); + fs::write(&tmp, b"newer").unwrap(); + let mut flushed_after_failure = 0; + publish_staged_file_with_dir_sync( + &tmp, + &destination, + |_, _| Err(io::Error::other("publish refused")), + |_| flushed_after_failure += 1, + ) + .expect_err("the publish was refused"); + let _ = fs::remove_dir_all(&root); + + assert_eq!(flushed, [destination]); + assert_eq!(flushed_after_failure, 0); + } + + /// A destination that is a symbolic link keeps being one: the file it + /// points at is replaced, through a chain of links and a relative link + /// alike, and nothing is left next to either end. + #[cfg(unix)] + #[test] + fn write_file_atomically_replaces_a_symlinks_target_and_keeps_the_link() { + use std::os::unix::fs::{PermissionsExt, symlink}; + + let root = scratch_dir("symlink"); + let dotfiles = root.join("dotfiles"); + let config_dir = root.join("config"); + fs::create_dir_all(&dotfiles).unwrap(); + fs::create_dir_all(&config_dir).unwrap(); + let target = dotfiles.join("rocm-config.json"); + fs::write(&target, b"old").unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o600)).unwrap(); + // config/config.json -> links/deeper/hop.json + // -> ../../dotfiles/rocm-config.json, a relative hop in a directory + // two levels deep, so resolving it against the entry link's directory + // instead of the hop's own would miss the target. + let hop_dir = root.join("links").join("deeper"); + fs::create_dir_all(&hop_dir).unwrap(); + let hop = hop_dir.join("hop.json"); + symlink(Path::new("../../dotfiles/rocm-config.json"), &hop).unwrap(); + let link = config_dir.join("config.json"); + symlink(&hop, &link).unwrap(); + + write_file_atomically(&link, b"new").unwrap(); + + let link_kept = fs::symlink_metadata(&link) + .unwrap() + .file_type() + .is_symlink(); + let hop_kept = fs::symlink_metadata(&hop).unwrap().file_type().is_symlink(); + let through_link = fs::read(&link).unwrap(); + let target_contents = fs::read(&target).unwrap(); + let target_mode = fs::metadata(&target).unwrap().permissions().mode() & 0o777; + let config_names = names_in(&config_dir); + let hop_names = names_in(&hop_dir); + let dotfiles_names = names_in(&dotfiles); + let _ = fs::remove_dir_all(&root); + + assert!(link_kept && hop_kept, "a link was replaced by a file"); + assert_eq!(through_link, b"new"); + assert_eq!(target_contents, b"new"); + assert_eq!(target_mode, 0o600); + assert_eq!(config_names, ["config.json"]); + assert_eq!(hop_names, ["hop.json"]); + assert_eq!(dotfiles_names, ["rocm-config.json"]); + } + + /// A link loop is an error, not a hang, and writes nothing. + #[cfg(unix)] + #[test] + fn write_file_atomically_refuses_a_symlink_loop() { + use std::os::unix::fs::symlink; + + let root = scratch_dir("symlink-loop"); + fs::create_dir_all(&root).unwrap(); + let a = root.join("a.json"); + let b = root.join("b.json"); + symlink(&b, &a).unwrap(); + symlink(&a, &b).unwrap(); + + let error = write_file_atomically(&a, b"x").expect_err("a link loop must fail"); + let names = names_in(&root); + let _ = fs::remove_dir_all(&root); + assert!( + format!("{error:#}").contains("too many levels of symbolic links"), + "{error:#}" + ); + assert_eq!(names, ["a.json", "b.json"]); + } +} diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index 6cedd857a6..9bcf417207 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -402,7 +402,7 @@ fn write_automation_proposals( ); text.push('\n'); } - crate::write_atomic(&path, text.as_bytes(), "jsonl") + crate::atomic_write::write_file_atomically(&path, text.as_bytes()) } fn normalize_proposal_identity(proposal: &mut AutomationProposalRecord, index: usize) { @@ -451,9 +451,11 @@ pub fn load_recent_automation_events( #[cfg(test)] mod tests { use super::*; + use crate::foreign_temp_probe::assert_write_spares_foreign_temp_files; use crate::jsonl_store::jsonl_generation_path; use crate::test_support::{ - audit_line, audit_messages, audit_record, seed_audit_store, temp_app_paths, + audit_line, audit_messages, audit_record, retired_temp_path, seed_audit_store, + temp_app_paths, }; use std::path::Path; @@ -559,6 +561,62 @@ mod tests { Ok(()) } + /// A status update never writes through a file another writer already + /// staged next to the proposals store. The occupied names are the ones the + /// retired clock-and-pid temp naming would have picked, so a rewrite that + /// still used one would truncate a foreign file and rename it into place. + /// `proposal_status_update_rewrites_record` would pass for either writer; + /// the foreign files are what tell them apart. + #[test] + fn proposal_status_update_never_reuses_another_writers_temp_file() -> Result<()> { + let (root, paths) = temp_app_paths("proposal-status-foreign-temp"); + append_automation_proposal(&paths, &proposal_record("proposal-1"))?; + let store = paths.automation_proposals_path(); + + assert_write_spares_foreign_temp_files( + |millis| retired_temp_path(&store, millis), + || update_automation_proposal_status(&paths, "proposal-1", "rejected").map(|_| ()), + )?; + + let loaded = find_automation_proposal(&paths, "proposal-1")?; + let _ = fs::remove_dir_all(&root); + assert_eq!(loaded.status, "rejected"); + Ok(()) + } + + /// The torn-tail repair, the other place a store is replaced wholesale, + /// spares foreign staged files for the same reason. Each attempt seeds the + /// torn tail again, since a repair only happens once. + #[test] + fn torn_tail_repair_never_reuses_another_writers_temp_file() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-foreign-temp"); + // Creates the store directory the foreign files go in. + seed_torn_tail_in_every_store(&paths)?; + let store = paths.automation_events_path(); + + assert_write_spares_foreign_temp_files( + |millis| retired_temp_path(&store, millis), + || { + seed_torn_tail_in_every_store(&paths)?; + append_automation_event(&paths, &event_record("after-repair")) + }, + )?; + + // Only the events store was appended to; the other two still hold the + // torn tails the seeding left in them. + let tags = fs::read_to_string(&store)? + .lines() + .map(|line| serde_json::from_str::(line).map(|r| r.message)) + .collect::, _>>()?; + let _ = fs::remove_dir_all(&root); + assert_eq!( + tags, + vec!["seed".to_owned(), "after-repair".to_owned()], + "the repair must keep the clean prefix, drop the torn tail and add the new record" + ); + Ok(()) + } + #[test] fn load_recent_audit_events_returns_tail() -> Result<()> { let (root, paths) = temp_app_paths("audit-tail"); diff --git a/crates/rocm-core/src/binary_paths.rs b/crates/rocm-core/src/binary_paths.rs index 3a4a64c752..48ded88aa4 100644 --- a/crates/rocm-core/src/binary_paths.rs +++ b/crates/rocm-core/src/binary_paths.rs @@ -12,9 +12,17 @@ use crate::require_nonempty; use crate::{current_executable_path, platform_binary_name}; pub fn sibling_binary_path(binary_name: &str) -> Result { + sibling_binary_path_from(¤t_executable_path()?, binary_name) +} + +/// [`sibling_binary_path`] for a caller that already holds the executable path. +/// +/// A caller that decides something from the current executable and then looks +/// for a sibling passes the one value to both, so the two cannot disagree: the +/// lookup falls back to `argv[0]` where `std::env::current_exe` fails. +pub fn sibling_binary_path_from(current_exe: &Path, binary_name: &str) -> Result { require_nonempty(binary_name, "binary_name")?; - let current_exe = current_executable_path()?; - let candidates = sibling_binary_candidates(¤t_exe, binary_name)?; + let candidates = sibling_binary_candidates(current_exe, binary_name)?; for candidate in &candidates { if candidate.is_file() { return Ok(candidate.clone()); @@ -76,3 +84,34 @@ pub fn daemon_binary_path() -> Result { } Ok(current_exe) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The lookup starts from the executable it is given, not from the + /// process's own: a caller that chose something from one path must find the + /// sibling beside that same path. + #[test] + fn sibling_binary_path_from_looks_beside_the_given_executable() { + let dir = tempfile::tempdir().expect("tempdir"); + let exe = dir.path().join(platform_binary_name("rocm")); + let sibling = dir.path().join(platform_binary_name("rocmd")); + std::fs::write(&sibling, b"").expect("write sibling"); + + assert_eq!(sibling_binary_path_from(&exe, "rocmd").unwrap(), sibling); + + let error = sibling_binary_path_from(&exe, "rocm-engine-none").unwrap_err(); + let text = error.to_string(); + assert!(text.contains("unable to locate sibling binary"), "{text}"); + assert!( + text.contains(&dir.path().display().to_string()), + "the error names where it looked: {text}" + ); + } + + #[test] + fn sibling_binary_path_from_rejects_an_empty_name() { + assert!(sibling_binary_path_from(Path::new("/opt/bin/rocm"), "").is_err()); + } +} diff --git a/crates/rocm-core/src/command_capture.rs b/crates/rocm-core/src/command_capture.rs index 1b8c6bac37..5fb78a0169 100644 --- a/crates/rocm-core/src/command_capture.rs +++ b/crates/rocm-core/src/command_capture.rs @@ -2,11 +2,16 @@ // // SPDX-License-Identifier: MIT -//! Running the `rocm` CLI as a child process and capturing what it printed. +//! The result of running a command, and running the `rocm` CLI as a child +//! process and capturing what it printed. //! -//! Both apps do this — `rocmd`'s sandbox and MCP clusters, and `rocm`'s chat -//! tool bridge when it replays an approved mutating command — so both use the -//! one [`CommandCapture`], spawn wrapper and text formatter here. +//! [`CommandCapture`] is the one carrier of a command's result: the raw bytes +//! of both streams, a typed [`CommandOutcome`] and the status, for the host +//! probes the bounded executor ([`crate::process`]) runs as well as for the +//! `rocm` captures below. Both apps run `rocm` as a child — `rocmd`'s sandbox +//! and MCP clusters, and `rocm`'s chat tool bridge when it replays an approved +//! mutating command — so both use the one [`CommandCapture`], spawn wrapper and +//! text formatter here. //! //! The bounded-execution half — draining both pipes, the deadline, quoting a //! timed-out child's output — is [`run_with_timeout`]'s, not this module's. @@ -19,24 +24,169 @@ use crate::binary_paths::daemon_binary_path; use crate::process::run_with_timeout; use anyhow::{Context, Result}; +use std::io::ErrorKind; use std::path::Path; -use std::process::Command; +use std::process::{Command, ExitStatus, Output}; use std::time::Duration; -/// What a `rocm` command printed, and how it exited. +/// How a command ended, as far as the bounded executor +/// ([`crate::process::run_with_timeout`]'s core) can tell. /// -/// Usually filled in by [`run_rocm_capture_with_env`] from a real child. The -/// chat tool bridge also builds one by hand for read-only commands it answers -/// in-process, with no child at all, so the shape is "a command's result", not -/// proof that anything was spawned. +/// The variants are the ways a run can end that a caller handles differently: +/// a probe treats "not installed" quietly, "timed out" as a finding, and "exited +/// with a code" by reading the code. Telling a child killed by a signal from one +/// that exited with `1` is what the old integer `exit_status` could not do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CommandOutcome { + /// The program does not exist ([`ErrorKind::NotFound`] from the spawn). + NotFound, + /// The program could not be started for another reason, for example + /// [`ErrorKind::PermissionDenied`]. + SpawnFailed(ErrorKind), + /// The deadline passed and the child was killed. + TimedOut, + /// The executor could not poll or reap the child, killed it and gave up. + WaitFailed, + /// The child exited with this code. + Exited(i32), + /// The child ended without an exit code: killed by a signal. The number is + /// the signal where the platform reports one (Unix), else `None`. + Signaled(Option), +} + +impl CommandOutcome { + /// The outcome of a spawn that failed with `kind`. + #[must_use] + pub fn from_spawn_error(kind: ErrorKind) -> Self { + if kind == ErrorKind::NotFound { + Self::NotFound + } else { + Self::SpawnFailed(kind) + } + } + + /// The outcome of a child that was waited for and not timed out: an exit + /// code when there is one, else a signal. Takes the two parts rather than an + /// [`ExitStatus`] so every host can check it with the values a Windows or a + /// Unix status would give. + #[must_use] + pub const fn from_exit(code: Option, signal: Option) -> Self { + match code { + Some(code) => Self::Exited(code), + None => Self::Signaled(signal), + } + } + + /// The exit code when the child exited with one. + #[must_use] + pub const fn exit_code(self) -> Option { + match self { + Self::Exited(code) => Some(code), + _ => None, + } + } +} + +/// The outcome of a child that was waited for to the end of its life. +pub(crate) fn outcome_of_status(status: ExitStatus) -> CommandOutcome { + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + CommandOutcome::from_exit(status.code(), signal) +} + +/// What a command printed, and how it ended. +/// +/// Filled in by [`run_rocm_capture_with_env`] from a real child, and by the +/// bounded executor for the host probes. The chat tool bridge also builds one by +/// hand ([`Self::from_text`]) for read-only commands it answers in-process, with +/// no child at all (the daemon's tests build them the same way), so the shape is +/// "a command's result", not proof that anything was spawned. +/// +/// The text fields are the lossy form of the raw ones, kept because most callers +/// read text; a caller that must decode the bytes itself (UTF-16LE from +/// `wsl.exe`, or a strict UTF-8 check) reads `stdout_raw` and `stderr_raw`. #[derive(Debug)] pub struct CommandCapture { /// The program followed by its arguments. pub argv: Vec, - /// The exit code; `1` when a child had none (killed by a signal). + /// The exit code; `1` when there was none (a signal, a timeout, a program + /// that could not be started). Read [`Self::outcome`] to tell those apart. pub exit_status: i32, + /// Standard output, lossily decoded from [`Self::stdout_raw`]. pub stdout: String, + /// Standard error, lossily decoded from [`Self::stderr_raw`]. pub stderr: String, + /// Standard output as the child wrote it. On a timeout, whatever arrived + /// before the deadline and the drain grace. + pub stdout_raw: Vec, + /// Standard error as the child wrote it. + pub stderr_raw: Vec, + /// How the command ended. + pub outcome: CommandOutcome, + /// The status the operating system reported for a child that was reaped, for + /// a caller that prints it. `None` for a program that was never started or a + /// capture built by hand. + pub status: Option, + /// What the executor saw go wrong, in words: the spawn or wait error, or the + /// read error that cut a pipe short (the outcome is then still what the + /// child did, and the bytes are what arrived before the error). + pub detail: Option, +} + +impl CommandCapture { + /// A capture of a command that was answered without a child. + #[must_use] + pub fn from_text(argv: Vec, exit_status: i32, stdout: String, stderr: String) -> Self { + Self { + argv, + exit_status, + stdout_raw: stdout.clone().into_bytes(), + stderr_raw: stderr.clone().into_bytes(), + stdout, + stderr, + outcome: CommandOutcome::Exited(exit_status), + status: None, + detail: None, + } + } + + /// A capture of a finished child's [`Output`]. + fn from_output(argv: Vec, output: Output) -> Self { + let outcome = outcome_of_status(output.status); + Self::from_run( + argv, + outcome, + Some(output.status), + output.stdout, + output.stderr, + None, + ) + } + + /// The capture the bounded executor builds from what a run left behind. + #[must_use] + pub(crate) fn from_run( + argv: Vec, + outcome: CommandOutcome, + status: Option, + stdout_raw: Vec, + stderr_raw: Vec, + detail: Option, + ) -> Self { + Self { + argv, + exit_status: outcome.exit_code().unwrap_or(1), + stdout: String::from_utf8_lossy(&stdout_raw).into_owned(), + stderr: String::from_utf8_lossy(&stderr_raw).into_owned(), + stdout_raw, + stderr_raw, + outcome, + status, + detail, + } + } } /// Render a capture as the text shown to a tool caller: stdout alone when @@ -89,14 +239,12 @@ fn capture_program( } let output = run_with_timeout(command, timeout, "process") .with_context(|| format!("failed to run {}", program.display()))?; - Ok(CommandCapture { - argv: std::iter::once(program.display().to_string()) + Ok(CommandCapture::from_output( + std::iter::once(program.display().to_string()) .chain(args.iter().map(|value| (*value).to_owned())) .collect(), - exit_status: output.status.code().unwrap_or(1), - stdout: String::from_utf8_lossy(&output.stdout).into_owned(), - stderr: String::from_utf8_lossy(&output.stderr).into_owned(), - }) + output, + )) } #[cfg(test)] @@ -105,34 +253,34 @@ mod tests { #[test] fn command_capture_text_prefers_stdout_only_when_stderr_is_blank() { - let output = CommandCapture { - argv: vec!["rocm".to_owned()], - exit_status: 0, - stdout: " all good \n".to_owned(), - stderr: " \n".to_owned(), - }; + let output = CommandCapture::from_text( + vec!["rocm".to_owned()], + 0, + " all good \n".to_owned(), + " \n".to_owned(), + ); assert_eq!(command_capture_text(&output), "all good"); } #[test] fn command_capture_text_falls_back_to_stderr_when_stdout_is_blank() { - let output = CommandCapture { - argv: vec!["rocm".to_owned()], - exit_status: 1, - stdout: " ".to_owned(), - stderr: "boom".to_owned(), - }; + let output = CommandCapture::from_text( + vec!["rocm".to_owned()], + 1, + " ".to_owned(), + "boom".to_owned(), + ); assert_eq!(command_capture_text(&output), "stderr:\nboom"); } #[test] fn command_capture_text_joins_both_streams_when_present() { - let output = CommandCapture { - argv: vec!["rocm".to_owned()], - exit_status: 1, - stdout: "out".to_owned(), - stderr: "err".to_owned(), - }; + let output = CommandCapture::from_text( + vec!["rocm".to_owned()], + 1, + "out".to_owned(), + "err".to_owned(), + ); assert_eq!( command_capture_text(&output), "stdout:\nout\n\nstderr:\nerr" @@ -182,4 +330,164 @@ mod tests { assert_eq!(output.stdout.trim(), "marker-value"); assert_eq!(output.stderr.trim(), "oops"); } + + #[test] + fn an_exit_code_outranks_a_signal_and_no_code_is_a_signal() { + assert_eq!( + CommandOutcome::from_exit(Some(3), Some(9)), + CommandOutcome::Exited(3) + ); + assert_eq!( + CommandOutcome::from_exit(Some(0), None), + CommandOutcome::Exited(0) + ); + assert_eq!( + CommandOutcome::from_exit(None, Some(9)), + CommandOutcome::Signaled(Some(9)) + ); + assert_eq!( + CommandOutcome::from_exit(None, None), + CommandOutcome::Signaled(None), + "a status with neither (no signal number on Windows) is still not an exit" + ); + assert_eq!(CommandOutcome::Exited(3).exit_code(), Some(3)); + for outcome in [ + CommandOutcome::NotFound, + CommandOutcome::TimedOut, + CommandOutcome::WaitFailed, + CommandOutcome::Signaled(Some(9)), + CommandOutcome::SpawnFailed(ErrorKind::PermissionDenied), + ] { + assert_eq!(outcome.exit_code(), None, "{outcome:?}"); + } + } + + #[test] + fn only_a_missing_program_is_not_found() { + assert_eq!( + CommandOutcome::from_spawn_error(ErrorKind::NotFound), + CommandOutcome::NotFound + ); + assert_eq!( + CommandOutcome::from_spawn_error(ErrorKind::PermissionDenied), + CommandOutcome::SpawnFailed(ErrorKind::PermissionDenied) + ); + } + + #[test] + fn a_status_is_an_exit_code_or_a_signal() { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + // The raw wait status: the exit code in the second byte, or the + // signal number in the low bits. + assert_eq!( + outcome_of_status(ExitStatus::from_raw(3 << 8)), + CommandOutcome::Exited(3) + ); + assert_eq!( + outcome_of_status(ExitStatus::from_raw(9)), + CommandOutcome::Signaled(Some(9)) + ); + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt as _; + assert_eq!( + outcome_of_status(ExitStatus::from_raw(3)), + CommandOutcome::Exited(3) + ); + } + } + + #[test] + fn a_hand_built_capture_is_an_exit_whose_bytes_are_its_text() { + let capture = CommandCapture::from_text( + vec!["rocm".to_owned()], + 2, + "out".to_owned(), + "err".to_owned(), + ); + + assert_eq!(capture.outcome, CommandOutcome::Exited(2)); + assert_eq!(capture.exit_status, 2); + assert_eq!(capture.stdout_raw, b"out"); + assert_eq!(capture.stderr_raw, b"err"); + assert!(capture.status.is_none() && capture.detail.is_none()); + } + + #[test] + fn a_run_keeps_its_raw_bytes_and_reports_text_from_them_lossily() { + let capture = CommandCapture::from_run( + vec!["tool".to_owned()], + CommandOutcome::TimedOut, + None, + b"ok\xffdone".to_vec(), + Vec::new(), + None, + ); + + assert_eq!(capture.stdout_raw, b"ok\xffdone"); + assert_eq!(capture.stdout, "ok\u{FFFD}done"); + assert_eq!(capture.exit_status, 1, "no exit code to report"); + + let exited = CommandCapture::from_run( + Vec::new(), + CommandOutcome::Exited(7), + None, + Vec::new(), + Vec::new(), + None, + ); + assert_eq!(exited.exit_status, 7); + } + + /// Subprocess-runners pair of `docs/host-probing.md`: a child killed by a + /// signal has no exit code, so the legacy `exit_status` reads `1`, which is + /// also what a child that exits with `1` reads; the typed `outcome` is what + /// tells them apart. + #[cfg(unix)] + #[test] + fn a_child_killed_by_a_signal_reads_as_exit_status_1() { + let killed = capture_program( + Path::new("sh"), + &["-c", "echo partial; kill -9 $$"], + &[], + Duration::from_secs(10), + ) + .expect("a signal death is captured, not an error"); + let exited = capture_program( + Path::new("sh"), + &["-c", "echo partial; exit 1"], + &[], + Duration::from_secs(10), + ) + .expect("a non-zero exit is captured, not an error"); + + assert_eq!(killed.exit_status, 1); + assert_eq!(killed.stdout.trim(), "partial"); + assert_eq!( + killed.exit_status, exited.exit_status, + "the two cannot be told apart from the exit status" + ); + assert_eq!(killed.outcome, CommandOutcome::Signaled(Some(9))); + assert_eq!(exited.outcome, CommandOutcome::Exited(1)); + } + + /// A program that cannot be started is an error carrying the program, not a + /// capture, so a caller cannot tell "not installed" from a timeout except by + /// the text of the message. + #[test] + fn a_program_that_cannot_be_started_is_an_error_naming_it() { + let missing = Path::new("/nonexistent-dir-for-the-runner-pair/tool"); + let error = capture_program(missing, &[], &[], Duration::from_secs(10)) + .expect_err("a spawn failure is an error here"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains(&format!("failed to run {}", missing.display())), + "{rendered}" + ); + assert!(rendered.contains("failed to spawn process"), "{rendered}"); + } } diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index 58e59acfe4..514fe34a9b 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -10,7 +10,7 @@ //! whose entries carry a default `WatcherMode`), and the rocm-dash dashboard //! sub-config (daemon + TUI), including its one-shot legacy-TOML migration. -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fs; @@ -353,6 +353,18 @@ const fn default_instance_tick_secs() -> f64 { 2.0 } +fn default_dashboard_lemonade_host() -> String { + "127.0.0.1".to_owned() +} + +const fn default_dashboard_lemonade_port() -> u16 { + 13305 +} + +const fn default_dashboard_enable_lemonade() -> bool { + true +} + /// Telemetry daemon operational spec. Tick cadences are stored as f64 seconds in /// the unified JSON config; use the `*_tick()` accessors for `Duration`s. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -369,6 +381,14 @@ pub struct DashboardDaemonConfig { pub discovery_tick_secs: f64, #[serde(default = "default_instance_tick_secs")] pub instance_tick_secs: f64, + /// Discover loaded models on the configured external Lemonade endpoint. + /// Read-only; does not adopt, launch, or stop the external service. + #[serde(default = "default_dashboard_enable_lemonade")] + pub enable_lemonade: bool, + #[serde(default = "default_dashboard_lemonade_host")] + pub lemonade_host: String, + #[serde(default = "default_dashboard_lemonade_port")] + pub lemonade_port: u16, /// Watch this file for new normalized benchmark CSV rows. When unset, the /// daemon derives `/bench/results.csv` from the current `AppPaths` /// at startup so machine-specific paths are never persisted in config. @@ -384,6 +404,9 @@ impl Default for DashboardDaemonConfig { gpu_tick_secs: default_gpu_tick_secs(), discovery_tick_secs: default_discovery_tick_secs(), instance_tick_secs: default_instance_tick_secs(), + enable_lemonade: default_dashboard_enable_lemonade(), + lemonade_host: default_dashboard_lemonade_host(), + lemonade_port: default_dashboard_lemonade_port(), bench_results_dir: None, } } @@ -591,34 +614,72 @@ struct LegacyDashTui { } impl RocmCliConfig { + /// Read `config.json`, or the defaults when there is none yet. + /// + /// Only a file that is not there means "no settings yet". Anything else + /// that stops the read — a directory where the file should be, a + /// permission problem, a symlink loop — is an error, just as unparseable + /// contents are: answering with defaults would let the next save replace + /// the user's settings with them. Data-dir discovery draws the same line. + /// + /// The error says how to recover, so every command that loads the + /// settings strictly gives the same advice: repair the file, or rename the + /// exact file it names ([`AppPaths::config_aside_advice`]). pub fn load(paths: &AppPaths) -> Result { let path = paths.config_path(); - if !path.is_file() { - return Ok(Self::default()); - } - - let bytes = - fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; - serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display())) + let unreadable = |what: &str, error: &dyn std::fmt::Display| { + anyhow::anyhow!( + "failed to {what} {}: {error}. Repair that file, then run the command again. {}", + path.display(), + paths.config_aside_advice() + ) + }; + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Self::default()); + } + Err(error) => return Err(unreadable("read", &error)), + }; + serde_json::from_slice(&bytes).map_err(|error| unreadable("parse", &error)) } - /// Persist the config by replacing it via the shared `write_atomic` - /// helper (write a sibling temp file, then rename it over `config.json`; - /// see that helper's own doc for the write/rename/cleanup guarantees). + /// Write the config so that [`Self::load`] can always read it back. /// - /// The destination is never removed first, because `load` reads a - /// missing `config.json` as "no config" and answers with defaults — a - /// removed-then-not-yet-replaced window would lose every setting without - /// so much as an error. + /// - The bytes are parsed back before anything is written. A value that + /// serializes but does not deserialize, such as a non-finite `f64` + /// (JSON has no NaN, so it is written as `null`), fails the save and + /// leaves the existing file as it was. + /// - The file is replaced in one step through the shared writer in + /// [`crate::atomic_write`]: a reader sees the old file or the new one, + /// never a truncated one, and the existing file is never removed first + /// (`load` would read a missing `config.json` as "no settings"). The new + /// contents are flushed before the replace, the replaced file's Unix + /// mode is kept, and a symlinked `config.json` keeps its link while its + /// target is replaced. + /// + /// Not guaranteed, on Windows: + /// - Replacing a `config.json` that another process holds open without + /// delete sharing. The replace then fails with a sharing violation, the + /// save returns that error, and the old config stays in place. + /// - A replace that fails part-way. `ReplaceFileW` can move the old file + /// off the `config.json` name and then fail to move the new one on; the + /// writer retries once with a plain rename, and if that fails too the new + /// contents stay in the `config.json.tmp-*` file the error names. Until + /// it is renamed back there is no `config.json`, which `load` reads as + /// "no settings". pub fn save(&self, paths: &AppPaths) -> Result<()> { let path = paths.config_path(); - fs::create_dir_all(&paths.config_dir) - .with_context(|| format!("failed to create {}", paths.config_dir.display()))?; let bytes = serde_json::to_vec_pretty(self).context("failed to serialize rocm-cli config")?; - crate::write_atomic(&path, &bytes, "json") - .with_context(|| format!("failed to save rocm-cli config to {}", path.display())) + serde_json::from_slice::(&bytes).with_context(|| { + format!( + "refusing to write {}: the new contents would not load back", + path.display() + ) + })?; + crate::atomic_write::write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write {}", path.display())) } pub fn engine_config(&self, engine: &str) -> Option<&EngineUserConfig> { @@ -685,13 +746,20 @@ impl RocmCliConfig { } /// One-shot migration of a legacy rocm-dash `config.toml` into the unified - /// JSON config. If no `config.json` exists yet **and** a legacy TOML is - /// present, its knobs are mapped into `dashboard` (and the canonical - /// `default_engine`/`engines`), `config.json` is written once, and the - /// migrated legacy path is returned so the caller can print a notice. The - /// TOML is left untouched. Returns `Ok(None)` when there is nothing to do - /// (already on the unified config, or no legacy file) — never clobbers an - /// existing `config.json`. + /// JSON config. If nothing is at the `config.json` path yet **and** a + /// legacy TOML is present, its knobs are mapped into `dashboard` (and the + /// canonical `default_engine`/`engines`), `config.json` is written once, + /// and the migrated legacy path is returned so the caller can print a + /// notice. The TOML is left untouched. + /// + /// Returns `Ok(None)` when there is nothing to do: no legacy file, or + /// anything at all already at the `config.json` path — a settings file, + /// but also a directory or a symbolic link, dangling or not. Only an empty + /// path is "no settings yet" here, so the migration never writes over, + /// beside or through something the user put there; `load` reports a + /// directory itself. Returns an error, writing nothing and leaving the TOML + /// alone, when a daemon tick is not a valid `Duration` (rocm-dash's own + /// reader rejects such a file) or the TOML cannot be read or parsed. pub fn migrate_legacy_dashboard_toml(paths: &AppPaths) -> Result> { let Some(legacy) = Self::legacy_dashboard_toml_path() else { return Ok(None); @@ -705,9 +773,20 @@ impl RocmCliConfig { paths: &AppPaths, legacy: &Path, ) -> Result> { - if paths.config_path().is_file() || !legacy.is_file() { + if !legacy.is_file() { return Ok(None); } + // `symlink_metadata`, not `exists`/`is_file`: a directory, or a link + // whose target is missing, is something the user put there, not an + // empty slot to migrate into. + match fs::symlink_metadata(paths.config_path()) { + Ok(_) => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .with_context(|| format!("failed to check {}", paths.config_path().display())); + } + } let raw = fs::read_to_string(legacy) .with_context(|| format!("failed to read {}", legacy.display()))?; @@ -718,6 +797,29 @@ impl RocmCliConfig { // Dashboard-specific knobs map into the new sub-config. let d = &parsed.daemon; + // rocm-dash's own reader (`rocm_dash_core::config`) rejects a tick that + // is not a valid `Duration` — NaN, infinite, negative, or too large — + // so it already treats this file as unreadable. Migrating it anyway + // would also write a non-finite value as JSON `null`, which `load` + // rejects. Refuse on the same rule, leaving the TOML untouched. + for (key, value) in [ + ("gpu_tick", d.gpu_tick), + ("discovery_tick", d.discovery_tick), + ("instance_tick", d.instance_tick), + ] { + if let Some(secs) = value + && let Err(error) = Duration::try_from_secs_f64(secs) + { + bail!( + "legacy config {} sets daemon.{key} = {secs}, which is not a valid tick \ + ({error}); set it to a finite, non-negative number of seconds to migrate the file. \ + Migration only runs while {} does not exist, so fix it before any rocm command \ + saves settings, or re-enter the dashboard settings by hand afterwards", + legacy.display(), + paths.config_path().display() + ); + } + } if let Some(v) = &d.listen { config.dashboard.daemon.listen = v.clone(); } @@ -754,10 +856,17 @@ impl RocmCliConfig { } } +// Property tests of save, load and the legacy migration, a child module so they +// can use the crate's `proptest_cases`. +#[cfg(test)] +#[path = "config_proptests.rs"] +mod props; + #[cfg(test)] mod tests { use super::*; - use crate::test_support::temp_app_paths; + use crate::foreign_temp_probe::assert_write_spares_foreign_temp_files; + use crate::test_support::{retired_temp_path, temp_app_paths}; #[test] fn builtin_watchers_include_read_only_gpu_metrics() { @@ -915,6 +1024,84 @@ mod tests { Ok(()) } + /// Every strict load of an unreadable file says how to recover, naming the + /// exact file to rename and the name to give it, and leaves the file + /// alone; renaming it as told makes the next load start from defaults. + #[test] + fn load_of_an_unreadable_config_says_which_file_to_rename() -> Result<()> { + // The advice reads `ROCM_CLI_DATA_DIR`, which other tests set. + let _guard = crate::test_env::DATA_DIR_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (root, paths) = temp_app_paths("config-load-advice"); + fs::create_dir_all(&paths.config_dir)?; + let damaged = br#"{"active_runtime_key": "kept", "onboarding_dismissed": "yes"}"#; + fs::write(paths.config_path(), damaged)?; + + let error = RocmCliConfig::load(&paths).expect_err("a retyped field must not load"); + let message = format!("{error:#}"); + let after = fs::read(paths.config_path())?; + let rename = format!( + "rename \"{config}\" to \"{config}.broken\"", + config = paths.config_path().display() + ); + let renamed = paths.config_dir.join("config.json.broken"); + let followed = message + .contains(&rename) + .then(|| fs::rename(paths.config_path(), &renamed)); + let reloaded = RocmCliConfig::load(&paths); + let kept = fs::read(&renamed); + fs::remove_dir_all(root).ok(); + + assert!( + message.contains("failed to parse") + && message.contains("Repair that file") + && message.contains(&paths.config_aside_advice()), + "unexpected error: {message}" + ); + assert_eq!(after, damaged, "load changed config.json"); + assert!( + followed.is_some(), + "the error does not say {rename:?}: {message}" + ); + assert_eq!(reloaded?.active_runtime_key, None); + assert_eq!(kept?, damaged, "the renamed file lost its contents"); + Ok(()) + } + + /// A value that serializes but cannot be read back (a non-finite tick is + /// written as JSON `null`) must fail the save and leave the existing file + /// exactly as it was, rather than replace a good config with one `load` + /// rejects. + #[test] + fn config_save_refuses_contents_load_would_reject_and_keeps_the_old_file() -> Result<()> { + let (root, paths) = temp_app_paths("config-save-refuses"); + let good = RocmCliConfig { + active_runtime_key: Some("kept-runtime".to_owned()), + ..RocmCliConfig::default() + }; + good.save(&paths)?; + let before = fs::read(paths.config_path())?; + + let mut bad = good; + bad.dashboard.daemon.gpu_tick_secs = f64::NAN; + let error = bad + .save(&paths) + .expect_err("a config load would reject must not be saved"); + let after = fs::read(paths.config_path())?; + let reloaded = RocmCliConfig::load(&paths)?; + fs::remove_dir_all(root).ok(); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains("refusing to write") && rendered.contains("config.json"), + "unexpected error: {rendered}" + ); + assert_eq!(after, before, "the existing config.json was changed"); + assert_eq!(reloaded.active_runtime_key.as_deref(), Some("kept-runtime")); + Ok(()) + } + #[test] fn legacy_config_without_telemetry_uses_default_policy() -> Result<()> { let config = serde_json::from_value::(serde_json::json!({ @@ -1037,6 +1224,31 @@ mod tests { } } + #[test] + fn dashboard_daemon_defaults_enable_local_lemonade_discovery() { + let old_config: DashboardDaemonConfig = serde_json::from_str("{}").unwrap(); + let json = serde_json::to_value(old_config).unwrap(); + assert_eq!(json["enable_lemonade"], true); + assert_eq!(json["lemonade_host"], "127.0.0.1"); + assert_eq!(json["lemonade_port"], 13305); + } + + #[test] + fn dashboard_daemon_lemonade_config_overrides_round_trip() { + let config: DashboardDaemonConfig = serde_json::from_str( + r#"{ + "enable_lemonade": false, "lemonade_host": "localhost", "lemonade_port": 14567 + }"#, + ) + .unwrap(); + assert!(!config.enable_lemonade); + assert_eq!(config.lemonade_host, "localhost"); + assert_eq!(config.lemonade_port, 14567); + let restored: DashboardDaemonConfig = + serde_json::from_value(serde_json::to_value(&config).unwrap()).unwrap(); + assert_eq!(restored, config); + } + #[test] fn dashboard_daemon_tick_accessors_map_secs_to_duration() { let d = DashboardDaemonConfig { @@ -1131,6 +1343,150 @@ last_installed_runtime_id = "therock-release" Ok(()) } + /// A tick that is not a valid duration is refused, naming the key, and the + /// refusal writes nothing: no `config.json` appears and the TOML is + /// unchanged, so the startup notice "skipped legacy dashboard config + /// migration" is true and fixing the value lets the next run migrate — + /// provided nothing has created `config.json` meanwhile, which the refusal + /// itself warns about. + #[test] + fn migrate_legacy_dashboard_toml_refuses_an_invalid_tick_and_writes_nothing() -> Result<()> { + // Every tick key, and every way a value can fail to be a Duration: + // not a number, infinite, negative, and finite but too large. + for key in ["gpu_tick", "discovery_tick", "instance_tick"] { + for (value, rendered) in [ + ("nan", "NaN"), + ("inf", "inf"), + ("-inf", "-inf"), + ("-1.0", "-1"), + // Above `Duration::MAX` (about 1.8e19 seconds). + ("1e20", "100000000000000000000"), + ] { + let (root, paths) = temp_app_paths("migrate-dash-bad-tick"); + fs::create_dir_all(&root)?; + let legacy = root.join("legacy-config.toml"); + let text = format!( + "default_engine = \"vllm\"\n[daemon]\n{key} = {value}\n[tui]\ntheme = \"nord\"\n" + ); + fs::write(&legacy, &text)?; + + let error = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy) + .expect_err("an invalid tick must not migrate"); + let config_written = paths.config_path().exists(); + let legacy_after = fs::read_to_string(&legacy)?; + let _ = fs::remove_dir_all(&root); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!("daemon.{key} = {rendered}")), + "{key} = {value}: unexpected error: {message}" + ); + assert!(!config_written, "{key} = {value}: config.json was written"); + assert_eq!( + legacy_after, text, + "{key} = {value}: the legacy TOML was modified" + ); + } + } + + // The refusal warns that the window closes once config.json exists — + // and it does: after any save, the still-broken legacy file is skipped + // without a word, and fixing it later no longer migrates it either. + let (root, paths) = temp_app_paths("migrate-dash-window-closes"); + fs::create_dir_all(&root)?; + let legacy = root.join("legacy-config.toml"); + fs::write( + &legacy, + "[daemon]\ngpu_tick = nan\n[tui]\ntheme = \"nord\"\n", + )?; + let error = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy) + .expect_err("an invalid tick must not migrate"); + let message = format!("{error:#}"); + RocmCliConfig::default().save(&paths)?; + let while_broken = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy)?; + fs::write( + &legacy, + "[daemon]\ngpu_tick = 1.0\n[tui]\ntheme = \"nord\"\n", + )?; + let once_fixed = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy)?; + let theme = RocmCliConfig::load(&paths)?.dashboard.tui.theme; + let _ = fs::remove_dir_all(&root); + assert!( + message.contains(&format!( + "Migration only runs while {} does not exist", + paths.config_path().display() + )) && message.contains("re-enter the dashboard settings by hand"), + "unexpected error: {message}" + ); + assert_eq!(while_broken, None); + assert_eq!(once_fixed, None); + assert_ne!(theme, "nord", "the legacy theme was migrated after all"); + + // Zero is a valid duration for rocm-dash too, so it still migrates. + let (root, paths) = temp_app_paths("migrate-dash-zero-tick"); + fs::create_dir_all(&root)?; + let legacy = root.join("legacy-config.toml"); + fs::write(&legacy, "[daemon]\ngpu_tick = 0\n")?; + let migrated = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy)?; + let loaded = RocmCliConfig::load(&paths); + let _ = fs::remove_dir_all(&root); + assert!(migrated.is_some()); + assert!(loaded.is_ok()); + Ok(()) + } + + /// Only an empty `config.json` path is "no settings yet". A directory + /// there (which `load` rejects) and a link whose target is missing are the + /// user's, so the migration writes nothing: not into the directory, not + /// through the link, and the TOML stays as it was. + #[test] + fn migrate_legacy_dashboard_toml_leaves_a_directory_or_link_at_config_json_alone() -> Result<()> + { + let legacy_text = "default_engine = \"vllm\"\n[tui]\ntheme = \"nord\"\n"; + + let (root, paths) = temp_app_paths("migrate-dash-dir-at-config"); + fs::create_dir_all(paths.config_path())?; + let legacy = root.join("legacy-config.toml"); + fs::write(&legacy, legacy_text)?; + let migrated = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy); + let entries = fs::read_dir(paths.config_path())?.count(); + let still_a_dir = paths.config_path().is_dir(); + let legacy_after = fs::read_to_string(&legacy)?; + let leftovers: Vec = fs::read_dir(&paths.config_dir)? + .filter_map(std::result::Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name != "config.json") + .collect(); + fs::remove_dir_all(&root).ok(); + assert_eq!(migrated?, None, "migrated over a directory at config.json"); + assert!(still_a_dir && entries == 0, "the directory was changed"); + assert!( + leftovers.is_empty(), + "files were left beside it: {leftovers:?}" + ); + assert_eq!(legacy_after, legacy_text); + + #[cfg(unix)] + { + let (root, paths) = temp_app_paths("migrate-dash-dangling-link"); + fs::create_dir_all(&paths.config_dir)?; + let target = root.join("dotfiles").join("rocm-config.json"); + std::os::unix::fs::symlink(&target, paths.config_path())?; + let legacy = root.join("legacy-config.toml"); + fs::write(&legacy, legacy_text)?; + let migrated = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy); + let target_written = target.exists(); + let still_a_link = fs::symlink_metadata(paths.config_path())? + .file_type() + .is_symlink(); + fs::remove_dir_all(&root).ok(); + assert_eq!(migrated?, None, "migrated through a dangling link"); + assert!(!target_written, "the link's target was created"); + assert!(still_a_link, "the link was replaced"); + } + Ok(()) + } + #[test] fn migrate_legacy_dashboard_toml_without_legacy_is_noop() -> Result<()> { let (root, paths) = temp_app_paths("migrate-dash-absent"); @@ -1143,15 +1499,16 @@ last_installed_runtime_id = "therock-release" Ok(()) } - /// The `inspect_err` cleanup on the rename, which the happy-path test above + /// The temp-file cleanup when the publish step fails (in + /// `atomic_write::publish_staged_file_with`), which the happy-path test above /// cannot reach: it only ever asserts that a SUCCESSFUL save leaves nothing /// behind, so deleting the cleanup would go unnoticed until a user with a - /// failing disk accumulated one `config.json.tmp--` per attempt. + /// failing disk accumulated one `config.json.tmp---` per attempt. /// - /// A directory at the destination is what makes the rename fail on both + /// A directory at the destination is what makes the publish fail on both /// supported platforms without needing permissions a root CI lane ignores — /// the same device `write_file_atomically_cleans_up_temp_when_the_rename_fails` - /// uses in `apps/rocm`. + /// uses in `rocm_core::atomic_write`. #[test] fn a_config_save_that_cannot_publish_cleans_up_its_temp_file() -> Result<()> { let (root, paths) = temp_app_paths("config-save-rename-fails"); @@ -1208,6 +1565,35 @@ last_installed_runtime_id = "therock-release" Ok(()) } + /// A save never writes through a file another writer already staged next + /// to `config.json`. The names occupied are the ones the retired + /// clock-and-pid temp naming would have picked, so a save that still wrote + /// to one would truncate a foreign file and rename it into place, taking + /// the other writer's bytes with it. The save-and-load assertions below + /// would pass for either writer; the foreign files are what tell them apart. + #[test] + fn saving_the_config_never_reuses_another_writers_temp_file() -> Result<()> { + let (root, paths) = temp_app_paths("config-save-foreign-temp"); + fs::create_dir_all(&paths.config_dir)?; + let config_path = paths.config_path(); + let mut config = RocmCliConfig { + default_engine: Some("vllm".to_owned()), + ..RocmCliConfig::default() + }; + config.save(&paths)?; + config.default_engine = Some("lemonade".to_owned()); + + assert_write_spares_foreign_temp_files( + |millis| retired_temp_path(&config_path, millis), + || config.save(&paths), + )?; + + let loaded = RocmCliConfig::load(&paths)?; + let _ = fs::remove_dir_all(&root); + assert_eq!(loaded.default_engine.as_deref(), Some("lemonade")); + Ok(()) + } + #[cfg(unix)] #[test] fn saving_the_config_replaces_the_file_rather_than_rewriting_it_in_place() -> Result<()> { @@ -1234,11 +1620,17 @@ last_installed_runtime_id = "therock-release" }; config.save(&paths)?; let before = fs::metadata(paths.config_path())?.ino(); + let before_bytes = fs::read(paths.config_path())?; + // A second name for the old file: if the save rewrote it in place, the + // new contents would show through this link too. + let witness = root.join("config-before-save.json"); + fs::hard_link(paths.config_path(), &witness)?; config.default_engine = Some("lemonade".to_owned()); config.save(&paths)?; let after = fs::metadata(paths.config_path())?.ino(); + let witnessed = fs::read(&witness)?; let loaded = RocmCliConfig::load(&paths)?; let _ = fs::remove_dir_all(&root); @@ -1247,6 +1639,10 @@ last_installed_runtime_id = "therock-release" "an overwrite that keeps the same inode rewrote the live config in \ place, so an interrupted save can leave it truncated" ); + assert_eq!( + witnessed, before_bytes, + "save rewrote the existing config.json in place" + ); assert_eq!(loaded.default_engine.as_deref(), Some("lemonade")); Ok(()) } diff --git a/crates/rocm-core/src/config_proptests.rs b/crates/rocm-core/src/config_proptests.rs new file mode 100644 index 0000000000..0a164d6b57 --- /dev/null +++ b/crates/rocm-core/src/config_proptests.rs @@ -0,0 +1,439 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the persisted `config.json` (`RocmCliConfig`) and its +//! one-shot migration from the legacy rocm-dash `config.toml`. +//! +//! Each property runs through an explicit `TestRunner` rather than the +//! `proptest!` macro so it can count how often the generator actually reached +//! the interesting classes (absent optionals, empty and non-ASCII strings, +//! non-finite floats), print that, and fail a run that never reached them +//! rather than pass vacuously. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use crate::test_support::proptest_cases; +use crate::{AppPaths, EngineUserConfig, RocmCliConfig}; +use proptest::prelude::*; +use proptest::test_runner::{Config as RunnerConfig, TestCaseError, TestError, TestRunner}; + +static NEXT_DIR: AtomicUsize = AtomicUsize::new(0); + +/// A fresh, empty `AppPaths` under the system temp dir. No environment is read +/// or written. +fn fresh_paths(tag: &str) -> (PathBuf, AppPaths) { + let root = std::env::temp_dir().join(format!( + "rocm-core-props-{tag}-{}-{}", + std::process::id(), + NEXT_DIR.fetch_add(1, Ordering::Relaxed) + )); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + (root, paths) +} + +/// Strings biased toward the cases uniform generators miss. +fn text() -> impl Strategy { + prop_oneof![ + 2 => Just(String::new()), + 2 => "[a-z0-9:._-]{1,12}", + 1 => Just("gfx1151-é-日本-\u{1F600}".to_owned()), + 1 => Just(" padded ".to_owned()), + 2 => any::(), + ] +} + +fn opt_text() -> impl Strategy> { + prop_oneof![3 => Just(None), 2 => text().prop_map(Some)] +} + +fn path_text() -> impl Strategy { + prop_oneof![ + Just(PathBuf::from("")), + Just(PathBuf::from("/opt/rocm/../rocm-7/")), + Just(PathBuf::from(r"D:\ROCm\therock_venvs")), + Just(PathBuf::from("/mnt/d/ROCm venvs/é")), + "[a-z/. ]{1,20}".prop_map(PathBuf::from), + ] +} + +fn tick() -> BoxedStrategy { + prop_oneof![ + 4 => 0.001_f64..120.0, + 1 => Just(f64::NAN), + 1 => Just(f64::INFINITY), + 1 => Just(f64::NEG_INFINITY), + 1 => Just(-0.0), + 1 => Just(-1.0), + 1 => Just(f64::MAX), + 1 => Just(f64::from_bits(1)), + ] + .boxed() +} + +fn engine_cfg() -> impl Strategy { + (opt_text(), opt_text(), opt_text(), opt_text()).prop_map(|(a, b, c, d)| EngineUserConfig { + preferred_runtime_id: a, + preferred_env_id: b, + last_installed_runtime_id: c, + last_installed_env_id: d, + }) +} + +#[derive(Debug, Clone)] +struct ConfigSeed { + default_engine: Option, + default_runtime_id: Option, + active_runtime_key: Option, + previous_runtime_key: Option, + onboarding_dismissed: bool, + telemetry_mode: String, + therock_venv: Option, + engines: BTreeMap, + ticks: (f64, f64, f64), + chat_temperature: Option, + chat_top_p: Option, + chat_max_tokens: Option, +} + +fn config_seed(ticks: BoxedStrategy) -> impl Strategy { + ( + (opt_text(), opt_text(), opt_text(), opt_text()), + any::(), + text(), + prop_oneof![2 => Just(None), 1 => path_text().prop_map(Some)], + prop::collection::btree_map(text(), engine_cfg(), 0..3), + (ticks.clone(), ticks.clone(), ticks), + ( + prop_oneof![Just(None), (0.0_f32..2.0).prop_map(Some)], + prop_oneof![Just(None), (0.0_f32..=1.0).prop_map(Some)], + prop_oneof![Just(None), (1_u32..=u32::MAX).prop_map(Some)], + ), + ) + .prop_map( + |((de, dr, ak, pk), od, tm, tv, engines, ticks, (temp, top_p, max_tokens))| { + ConfigSeed { + default_engine: de, + default_runtime_id: dr, + active_runtime_key: ak, + previous_runtime_key: pk, + onboarding_dismissed: od, + telemetry_mode: tm, + therock_venv: tv, + engines, + ticks, + chat_temperature: temp, + chat_top_p: top_p, + chat_max_tokens: max_tokens, + } + }, + ) +} + +fn build_config(seed: &ConfigSeed) -> RocmCliConfig { + let mut config = RocmCliConfig { + default_engine: seed.default_engine.clone(), + default_runtime_id: seed.default_runtime_id.clone(), + active_runtime_key: seed.active_runtime_key.clone(), + previous_runtime_key: seed.previous_runtime_key.clone(), + onboarding_dismissed: seed.onboarding_dismissed, + engines: seed.engines.clone(), + ..RocmCliConfig::default() + }; + config.telemetry.mode.clone_from(&seed.telemetry_mode); + config.setup.therock_venv.clone_from(&seed.therock_venv); + config.dashboard.daemon.gpu_tick_secs = seed.ticks.0; + config.dashboard.daemon.discovery_tick_secs = seed.ticks.1; + config.dashboard.daemon.instance_tick_secs = seed.ticks.2; + config.dashboard.tui.chat_temperature = seed.chat_temperature; + config.dashboard.tui.chat_top_p = seed.chat_top_p; + config.dashboard.tui.chat_max_tokens = seed.chat_max_tokens; + config +} + +/// Compare configs by their JSON text re-read through the same parser, not by +/// `serde_json::to_value`: `to_value` widens an `f32` to `f64` (`0.17120193` +/// becomes `0.1712019294500351`), and serde_json's default float parser is not +/// correctly rounded (`59.913248351550116` reads back as `59.91324835155011`). +/// Both are serde_json properties, harmless for these fields, and must not mask +/// real loss. +fn as_value(config: &RocmCliConfig) -> serde_json::Value { + let text = serde_json::to_string(config).expect("config serializes"); + serde_json::from_str(&text).expect("serialized config parses as JSON") +} + +fn report(name: &str, result: Result<(), TestError>) { + match result { + Ok(()) => {} + Err(TestError::Fail(reason, value)) => { + panic!("{name}: minimal failing input {value:#?}\nreason: {reason}") + } + Err(TestError::Abort(reason)) => panic!("{name}: aborted: {reason}"), + } +} + +/// `load(save(x)) == x` for every config whose ticks are finite. This is the +/// half of the round-trip that holds; it stays enabled as a regression guard. +#[test] +fn config_round_trips_through_save_and_load_for_finite_values() { + let (root, paths) = fresh_paths("roundtrip-finite"); + let total = AtomicUsize::new(0); + let none_active = AtomicUsize::new(0); + let empty_strings = AtomicUsize::new(0); + let non_ascii = AtomicUsize::new(0); + let finite_ticks = (0.001_f64..120.0).boxed(); + let finite = prop_oneof![ + 3 => finite_ticks, + 1 => Just(-0.0), + 1 => Just(f64::MAX), + 1 => Just(f64::from_bits(1)), + 1 => Just(-1.0), + ] + .boxed(); + let mut runner = TestRunner::new(RunnerConfig { + cases: proptest_cases(512), + ..RunnerConfig::default() + }); + let result = runner.run(&config_seed(finite), |seed| { + total.fetch_add(1, Ordering::Relaxed); + let config = build_config(&seed); + let json = serde_json::to_string(&config).unwrap_or_default(); + if seed.active_runtime_key.is_none() { + none_active.fetch_add(1, Ordering::Relaxed); + } + if json.contains("\"\"") { + empty_strings.fetch_add(1, Ordering::Relaxed); + } + if !json.is_ascii() { + non_ascii.fetch_add(1, Ordering::Relaxed); + } + config + .save(&paths) + .map_err(|e| TestCaseError::fail(format!("{e:#}")))?; + let loaded = + RocmCliConfig::load(&paths).map_err(|e| TestCaseError::fail(format!("{e:#}")))?; + prop_assert_eq!(as_value(&loaded), as_value(&config)); + Ok(()) + }); + eprintln!( + "reach: of {} cases, active_runtime_key=None {}, empty strings {}, non-ASCII {}", + total.load(Ordering::Relaxed), + none_active.load(Ordering::Relaxed), + empty_strings.load(Ordering::Relaxed), + non_ascii.load(Ordering::Relaxed) + ); + let _ = std::fs::remove_dir_all(root); + report("finite round-trip", result); + // A property that never generated the inputs it is about passed without + // testing them. + assert!( + none_active.load(Ordering::Relaxed) > 0 + && empty_strings.load(Ordering::Relaxed) > 0 + && non_ascii.load(Ordering::Relaxed) > 0, + "the round-trip never generated an unset key, an empty string and non-ASCII text" + ); +} + +/// `save` must never write a file that `load` then refuses. A non-finite tick +/// serializes as JSON `null`, which the `f64` field then rejects, so the saved +/// config cannot be read back at all. +#[test] +fn config_save_never_writes_a_file_load_rejects() { + let (root, paths) = fresh_paths("roundtrip-any"); + let non_finite = AtomicUsize::new(0); + let mut runner = TestRunner::new(RunnerConfig { + cases: proptest_cases(256), + ..RunnerConfig::default() + }); + let result = runner.run(&config_seed(tick()), |seed| { + if ![seed.ticks.0, seed.ticks.1, seed.ticks.2] + .iter() + .all(|t| t.is_finite()) + { + non_finite.fetch_add(1, Ordering::Relaxed); + } + let config = build_config(&seed); + if config.save(&paths).is_err() { + // Refusing to save is acceptable; writing an unreadable file is not. + return Ok(()); + } + RocmCliConfig::load(&paths) + .map(|_| ()) + .map_err(|e| TestCaseError::fail(format!("{e:#}"))) + }); + eprintln!( + "reach: non-finite tick in {} cases", + non_finite.load(Ordering::Relaxed) + ); + let _ = std::fs::remove_dir_all(root); + report("save/load agreement", result); + assert!( + non_finite.load(Ordering::Relaxed) > 0, + "no case had a non-finite tick" + ); +} + +fn toml_float(value: f64) -> String { + if value.is_nan() { + "nan".to_owned() + } else if value == f64::INFINITY { + "inf".to_owned() + } else if value == f64::NEG_INFINITY { + "-inf".to_owned() + } else { + format!("{value:?}") + } +} + +/// Run the legacy-TOML migration property over the given tick strategy. +/// +/// The migration writes `config.json` once and never again, so it has to write +/// something the unified loader accepts, carrying the user's values. Generated +/// as TOML *text* so the real `toml` 1.x parser is in the loop. +/// +/// With `must_migrate`, every case must migrate, and carry its ticks over +/// unchanged: the strategy then generates only ticks rocm-dash accepts, so a +/// refusal is a defect rather than the safe outcome it is for any tick. +fn legacy_migration_property(ticks: &BoxedStrategy, label: &str, must_migrate: bool) { + use std::fmt::Write as _; + let total = AtomicUsize::new(0); + let non_finite = AtomicUsize::new(0); + let non_ascii_engine = AtomicUsize::new(0); + let mut runner = TestRunner::new(RunnerConfig { + cases: proptest_cases(256), + ..RunnerConfig::default() + }); + let strategy = (ticks.clone(), ticks.clone(), ticks.clone(), opt_text()); + let result = runner.run(&strategy, |(g, d, i, engine)| { + total.fetch_add(1, Ordering::Relaxed); + if ![g, d, i].iter().all(|t| t.is_finite()) { + non_finite.fetch_add(1, Ordering::Relaxed); + } + if engine.as_deref().is_some_and(|e| !e.is_ascii()) { + non_ascii_engine.fetch_add(1, Ordering::Relaxed); + } + let (root, paths) = fresh_paths("legacy"); + std::fs::create_dir_all(&root).unwrap(); + let legacy = root.join("config.toml"); + let mut toml_text = String::new(); + if let Some(engine) = &engine { + let _ = writeln!( + toml_text, + "default_engine = {}", + toml_string_literal(engine) + ); + } + let _ = write!( + toml_text, + "[daemon]\ngpu_tick = {}\ndiscovery_tick = {}\ninstance_tick = {}\n", + toml_float(g), + toml_float(d), + toml_float(i) + ); + std::fs::write(&legacy, &toml_text).unwrap(); + let migrated = RocmCliConfig::migrate_legacy_dashboard_toml_from(&paths, &legacy); + let loaded = RocmCliConfig::load(&paths); + let _ = std::fs::remove_dir_all(root); + match migrated { + Err(error) if must_migrate => Err(TestCaseError::fail(format!( + "refused to migrate ticks rocm-dash accepts: {error:#}\nlegacy toml:\n{toml_text}" + ))), + Ok(None) if must_migrate => Err(TestCaseError::fail(format!( + "did not migrate\nlegacy toml:\n{toml_text}" + ))), + // Refusing to migrate is fine otherwise: the legacy file is left + // untouched. + Err(_) | Ok(None) => Ok(()), + Ok(Some(_)) => { + let loaded = loaded.map_err(|e| { + TestCaseError::fail(format!( + "migrated config.json is unreadable: {e:#}\nlegacy toml:\n{toml_text}" + )) + })?; + prop_assert_eq!(loaded.default_engine, engine); + let daemon = &loaded.dashboard.daemon; + // Within one unit in the last place, and with the same sign, so + // `-0.0` stays `-0.0`. Not bit-exact: serde_json is built + // without its `float_roundtrip` feature, so reading + // `config.json` back can land one ULP off (`97.95315202189215` + // does), whatever the migration wrote. + let migrated = [ + daemon.gpu_tick_secs, + daemon.discovery_tick_secs, + daemon.instance_tick_secs, + ]; + prop_assert!( + migrated.iter().zip([g, d, i]).all(|(got, want)| { + got.is_sign_negative() == want.is_sign_negative() + && got.to_bits().abs_diff(want.to_bits()) <= 1 + }), + "migrated ticks {:?} differ from {:?}\nlegacy toml:\n{}", + migrated, + [g, d, i], + toml_text + ); + Ok(()) + } + } + }); + eprintln!( + "reach ({label}): of {} cases, non-finite tick {}, non-ASCII default_engine {}", + total.load(Ordering::Relaxed), + non_finite.load(Ordering::Relaxed), + non_ascii_engine.load(Ordering::Relaxed) + ); + report(label, result); + assert!( + non_ascii_engine.load(Ordering::Relaxed) > 0, + "{label}: no case had a non-ASCII default_engine" + ); + assert!( + must_migrate || non_finite.load(Ordering::Relaxed) > 0, + "{label}: no case had a non-finite tick" + ); +} + +/// Ticks rocm-dash accepts and arbitrary strings migrate, and load back with +/// the same values. Every generated tick passes `Duration::try_from_secs_f64`, +/// which is the rule the migration applies, so each case must migrate: a +/// refusal fails it. Also the `toml` 0.8 -> 1.x string guard. +#[test] +fn legacy_toml_migration_round_trips_accepted_ticks() { + let accepted = prop_oneof![ + 3 => 0.001_f64..120.0, + 1 => Just(0.0), + 1 => Just(-0.0), + ] + .prop_filter("a tick rocm-dash accepts", |secs| { + std::time::Duration::try_from_secs_f64(*secs).is_ok() + }) + .boxed(); + legacy_migration_property(&accepted, "legacy migration (accepted ticks)", true); +} + +#[test] +fn legacy_toml_migration_always_produces_a_loadable_config() { + legacy_migration_property(&tick(), "legacy migration (any)", false); +} + +fn toml_string_literal(value: &str) -> String { + use std::fmt::Write as _; + // A basic string with every char escaped as \u/\U keeps any input valid TOML. + let mut out = String::from("\""); + for ch in value.chars() { + let code = u32::from(ch); + if code <= 0xFFFF { + let _ = write!(out, "\\u{code:04X}"); + } else { + let _ = write!(out, "\\U{code:08X}"); + } + } + out.push('"'); + out +} diff --git a/crates/rocm-core/src/confirm.rs b/crates/rocm-core/src/confirm.rs index 20ee2587cc..e5fa9f9a03 100644 --- a/crates/rocm-core/src/confirm.rs +++ b/crates/rocm-core/src/confirm.rs @@ -23,6 +23,13 @@ //! without a controlled terminal. Real callers pass `std::io::stdout()` and //! `std::io::stdin().lock()`. //! +//! [`read_answer`] also tells the user when their answer was not understood, and +//! ends the prompt line when the input ended without a newline. Both go to the +//! `out` stream the caller passes, so every caller gets them and none can forget. +//! `y`/`yes` approve; `n`/`no`/an empty line decline silently (an empty line is +//! the documented default); anything else declines with a line saying so, rather +//! than being mistaken for a deliberate "no". +//! //! [`interactive_terminal`] answers the question callers ask before prompting at //! all: whether this process has a terminal on both streams. //! [`require_yes_or_interactive`] is the guard built on it for a mutating @@ -32,9 +39,24 @@ use anyhow::{Result, bail}; use std::io::{BufRead, IsTerminal, Write, stdin, stdout}; -/// Parse a user's typed response to a `[y/N]` prompt. -pub fn is_affirmative_answer(line: &str) -> bool { - matches!(line.trim().to_lowercase().as_str(), "y" | "yes") +/// How a typed response to a `[y/N]` prompt reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Answer { + /// `y` or `yes`, in any case. + Yes, + /// `n` or `no`, or an empty line: the prompt's default. + No, + /// Anything else. Declined like [`Answer::No`], but worth saying so. + Unrecognized, +} + +/// Classify a user's typed response to a `[y/N]` prompt. +pub fn parse_answer(line: &str) -> Answer { + match line.trim().to_lowercase().as_str() { + "y" | "yes" => Answer::Yes, + "" | "n" | "no" => Answer::No, + _ => Answer::Unrecognized, + } } /// Write `prompt` to `out` and flush it, so it's visible before a blocking @@ -54,10 +76,48 @@ pub fn print_prompt(out: &mut impl Write, prompt: &str) -> std::io::Result<()> { /// Read one line from `input` and parse it as yes/no. Only errors if the /// line can't be read; a declined answer — including an empty read at EOF — /// is `Ok(false)`, not an error. -pub fn read_answer(input: &mut impl BufRead) -> std::io::Result { +/// +/// Two things are written to `out` so the caller's own message lands where the +/// user can read it: +/// - when the line did not end in a newline (EOF, as from Ctrl-D, or piped input +/// with no final newline such as `printf no |`), a newline, ending the prompt +/// line: either way nothing echoed a line end, so the cursor is still on the +/// prompt line and the caller's next line would run into it; +/// - when the answer is [`Answer::Unrecognized`], a line saying it is being +/// treated as no, with the answer shown back quoted and escaped and cut to +/// [`MAX_ECHOED_ANSWER_CHARS`] characters, so a long pasted line does not fill +/// the screen. +/// +/// Both writes are best effort: a failure to explain a decline must not turn +/// into an error that replaces the decline, which `fix`'s `confirm()` relies on. +pub fn read_answer(input: &mut impl BufRead, out: &mut impl Write) -> std::io::Result { let mut line = String::new(); input.read_line(&mut line)?; - Ok(is_affirmative_answer(&line)) + if !line.ends_with('\n') { + let _ = writeln!(out); + } + let answer = parse_answer(&line); + if answer == Answer::Unrecognized { + let _ = writeln!( + out, + "{} is not yes or no; treating it as no.", + echoed_answer(line.trim()) + ); + } + Ok(answer == Answer::Yes) +} + +/// The most characters of a typed answer that the "not understood" line shows. +const MAX_ECHOED_ANSWER_CHARS: usize = 40; + +/// `answer` as shown back to the user: quoted with control characters escaped, and +/// cut to [`MAX_ECHOED_ANSWER_CHARS`] characters (never in the middle of one), with +/// `...` after the closing quote when there was more. +fn echoed_answer(answer: &str) -> String { + let mut chars = answer.chars(); + let shown: String = chars.by_ref().take(MAX_ECHOED_ANSWER_CHARS).collect(); + let more = if chars.next().is_some() { "..." } else { "" }; + format!("{shown:?}{more}") } /// Whether this process can hold an interactive exchange with a user. @@ -112,21 +172,34 @@ mod tests { use std::io::{Error, Result}; #[test] - fn is_affirmative_answer_accepts_only_y_and_yes() { + fn parse_answer_accepts_only_y_and_yes() { for accepted in ["y", "Y", "yes", "YES", "Yes", " y ", " yes\n"] { - assert!( - is_affirmative_answer(accepted), + assert_eq!( + parse_answer(accepted), + Answer::Yes, "expected {accepted:?} to be treated as a yes" ); } } #[test] - fn is_affirmative_answer_rejects_everything_else() { - for declined in ["n", "no", "", "\n", "yep", "ye"] { - assert!( - !is_affirmative_answer(declined), - "expected {declined:?} to be treated as a decline" + fn parse_answer_reads_n_no_and_an_empty_line_as_the_default_no() { + for declined in ["n", "N", "no", "NO", " no ", "", "\n", " \n"] { + assert_eq!( + parse_answer(declined), + Answer::No, + "expected {declined:?} to be the default no" + ); + } + } + + #[test] + fn parse_answer_flags_everything_else_as_unrecognized() { + for odd in ["maybe", "yep", "ye", "yes please", "nope", "1", "y n"] { + assert_eq!( + parse_answer(odd), + Answer::Unrecognized, + "expected {odd:?} to be unrecognized" ); } } @@ -189,19 +262,170 @@ mod tests { ); } + /// Run `read_answer` over `typed` and return the verdict and everything it + /// wrote to the output stream. + fn answer_and_output(typed: &[u8]) -> (bool, String) { + let mut input = typed; + let mut out = Vec::new(); + let verdict = + read_answer(&mut input, &mut out).expect("read from a byte slice cannot fail"); + ( + verdict, + String::from_utf8(out).expect("the helper writes UTF-8"), + ) + } + + #[test] + fn read_answer_approves_y_and_yes_without_writing_anything() { + for typed in [&b"y\n"[..], b"yes\n", b"YES\n"] { + let (verdict, written) = answer_and_output(typed); + assert!(verdict, "{typed:?} must approve"); + assert_eq!(written, "", "an approval needs no comment"); + } + } + + #[test] + fn read_answer_declines_n_no_and_enter_without_writing_anything() { + for typed in [&b"n\n"[..], b"no\n", b"\n"] { + let (verdict, written) = answer_and_output(typed); + assert!(!verdict, "{typed:?} must decline"); + assert_eq!( + written, "", + "a deliberate no (or the default) must not be commented on" + ); + } + } + + #[test] + fn read_answer_declines_an_unrecognized_answer_and_says_so() { + let (verdict, written) = answer_and_output(b"maybe\n"); + assert!(!verdict, "an unrecognized answer must decline"); + assert_eq!(written, "\"maybe\" is not yes or no; treating it as no.\n"); + } + + #[test] + fn read_answer_at_eof_declines_and_ends_the_prompt_line() { + let (verdict, written) = answer_and_output(b""); + assert!(!verdict, "EOF is Ok(false), not an error"); + assert_eq!(written, "\n", "EOF leaves the cursor on the prompt line"); + } + + #[test] + fn read_answer_for_a_partial_line_ends_the_prompt_line_then_says_so() { + // Typed text and then EOF, with no newline: the terminal echoed the text + // but no newline, so the line must be ended before the feedback line. + let (verdict, written) = answer_and_output(b"maybe"); + assert!(!verdict); + assert_eq!( + written, + "\n\"maybe\" is not yes or no; treating it as no.\n" + ); + } + + #[test] + fn read_answer_for_a_yes_without_a_newline_still_approves() { + let (verdict, written) = answer_and_output(b"yes"); + assert!(verdict, "a yes is a yes with or without the final newline"); + assert_eq!(written, "\n", "the unfinished prompt line is still ended"); + } + + #[test] + fn read_answer_cuts_a_long_answer_short_and_marks_the_cut() { + let typed = format!("{}\n", "a".repeat(100)); + let (verdict, written) = answer_and_output(typed.as_bytes()); + assert!(!verdict); + assert_eq!( + written, + format!( + "\"{}\"... is not yes or no; treating it as no.\n", + "a".repeat(MAX_ECHOED_ANSWER_CHARS) + ) + ); + } + + #[test] + fn read_answer_shows_an_answer_of_exactly_the_limit_whole() { + let answer = "b".repeat(MAX_ECHOED_ANSWER_CHARS); + let (_, written) = answer_and_output(format!("{answer}\n").as_bytes()); + assert_eq!( + written, + format!("\"{answer}\" is not yes or no; treating it as no.\n"), + "an answer that fits is not marked as cut" + ); + let (_, written) = answer_and_output(format!("{answer}c\n").as_bytes()); + assert_eq!( + written, + format!("\"{answer}\"... is not yes or no; treating it as no.\n"), + "one character over the limit is cut" + ); + } + + #[test] + fn read_answer_cuts_between_characters_not_inside_one() { + // 2-byte and 4-byte characters: a cut by bytes would split one and panic + // or write invalid UTF-8; the limit counts characters. + for ch in ['é', '🦀'] { + let typed = format!("{}\n", ch.to_string().repeat(MAX_ECHOED_ANSWER_CHARS + 5)); + let (_, written) = answer_and_output(typed.as_bytes()); + assert_eq!( + written, + format!( + "\"{}\"... is not yes or no; treating it as no.\n", + ch.to_string().repeat(MAX_ECHOED_ANSWER_CHARS) + ), + "answer of {ch:?}" + ); + } + } + + #[test] + fn read_answer_bounds_the_line_even_when_every_character_is_escaped() { + // An escaped control character is longer than itself, so the limit is on + // the typed characters and the line stays bounded by it. + let typed = format!("{}\n", "\x1b".repeat(100)); + let (_, written) = answer_and_output(typed.as_bytes()); + assert_eq!( + written, + format!( + "\"{}\"... is not yes or no; treating it as no.\n", + "\\u{1b}".repeat(MAX_ECHOED_ANSWER_CHARS) + ) + ); + assert!( + !written.contains('\x1b'), + "no raw escape reaches the terminal" + ); + } + #[test] - fn read_answer_parses_a_typed_line() { - let mut input: &[u8] = b"yes\n"; - assert!(read_answer(&mut input).expect("read from a byte slice cannot fail")); + fn read_answer_for_a_no_without_a_newline_declines_and_only_ends_the_prompt_line() { + // Piped input such as `printf no |` has no final newline either. The + // verdict is the decline `no\n` gives and nothing explains it, but the + // prompt line is still ended: no newline was echoed after the prompt. + let (verdict, written) = answer_and_output(b"no"); + assert!(!verdict, "a no is a no with or without the final newline"); + assert_eq!(written, "\n", "only the unfinished prompt line is ended"); + } - let mut input: &[u8] = b"n\n"; - assert!(!read_answer(&mut input).expect("read from a byte slice cannot fail")); + #[test] + fn read_answer_quotes_an_answer_that_holds_control_characters() { + // The answer is echoed back, so it is escaped rather than written raw. + let (_, written) = answer_and_output(b"\x1b[31mred\n"); + assert_eq!( + written, + "\"\\u{1b}[31mred\" is not yes or no; treating it as no.\n" + ); } #[test] - fn read_answer_at_eof_declines_without_erroring() { - let mut input: &[u8] = b""; - assert!(!read_answer(&mut input).expect("EOF is Ok(false), not an error")); + fn read_answer_keeps_the_verdict_when_the_feedback_cannot_be_written() { + // The explanation is best effort: a closed stdout must not turn a + // decline into an error, which `fix`'s `confirm()` relies on. + let mut input: &[u8] = b"maybe\n"; + let mut out = FailingWriter; + let verdict = read_answer(&mut input, &mut out) + .expect("a failed feedback write must not replace the verdict"); + assert!(!verdict); } /// A reader whose `read_line` always fails, so a real I/O failure on the @@ -224,7 +448,9 @@ mod tests { #[test] fn read_answer_surfaces_a_read_failure_as_an_error() { let mut input = FailingReader; - read_answer(&mut input).expect_err("a failing reader must return Err"); + let mut out = Vec::new(); + read_answer(&mut input, &mut out).expect_err("a failing reader must return Err"); + assert!(out.is_empty(), "nothing is written for a failed read"); } #[test] diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 8ff099d9fa..83f5533ad9 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -42,8 +42,8 @@ mod windows; mod wsl; pub use engine::{VLLM_IMPORT_CANONICAL_SYMPTOM, VLLM_OOM_CANONICAL_SYMPTOM}; -pub use linux::dkms_driver_plan_is_supported; -pub use report::{render_report_text, upstream_route}; +pub use linux::{AMDGPU_MISSING_EVIDENCE, dkms_driver_plan_is_supported}; +pub use report::{SYMPTOM_DID_NOT_CONTRIBUTE_NOTE, render_report_text, upstream_route}; /// At/above this score a diagnosis is treated as a real match. pub const MIN_SCORE_FOR_MATCH: i32 = 50; @@ -91,6 +91,25 @@ pub struct Diagnosis { pub score: i32, pub evidence: Vec, pub fix: Option, + /// Whether the symptom text the user supplied contributed to this entry. + /// + /// `true` when, without the symptom, this entry would have a different score + /// or different evidence, or would not be listed at all; `false` when it + /// comes from the examined host's state alone. A host-state finding can + /// outrank the entry the symptom added to, so a reader asking "did the text + /// I pasted contribute to this entry?" reads this rather than the ranking. + /// + /// It records a contribution, not a verdict that the entry explains the + /// error: any keyword hit counts, however weak. Read `score` and `evidence` + /// for how much the text added. + /// + /// Always serialized, and `false` when no symptom was given: nothing was + /// pasted, so nothing in it contributed. Omitting it in that case would make + /// "no symptom" indistinguishable from "a CLI too old to report this field". + /// `#[serde(default)]` so a payload written before the field existed still + /// deserializes. Set by [`diagnose`], never by a checker. + #[serde(default)] + pub symptom_matched: bool, } /// Where to send a report when no catalog entry matches. @@ -200,6 +219,43 @@ pub fn vllm_import_symptom_is_diagnosable(symptom: &str) -> bool { scores_on_every_engine_host(symptom, "fix-17-torch-dlpack", run_all_checks) } +/// Set [`Diagnosis::symptom_matched`] on every entry in `matched`. +/// +/// Decided by running the same catalog again on the same machine with no +/// symptom and comparing, rather than by each checker reporting what it read. +/// Checkers fold the symptom in several ways (keyword tables, single patterns, +/// a keyword whose own weight is replaced by a fixed one, a symptom that lifts a +/// negative host score above zero), and a flag each of them had to set by hand +/// is one a new checker can forget. The comparison asks the question itself: +/// would this entry look the same without the text? Evidence is compared as +/// well as the score because a score is capped at 100, so a keyword hit on an +/// entry already at the cap changes its evidence and not its number. +/// +/// The checkers are pure functions of the examination and the symptom, so the +/// second run reads no more of the machine than the first. +fn mark_symptom_contribution(matched: &mut [Diagnosis], e: &Examination, symptom: &str) { + if symptom.is_empty() { + return; + } + let host_only = run_all_checks(e, ""); + for d in matched { + d.symptom_matched = contributed(d, &host_only); + } +} + +/// Whether the symptom contributed to `d`, given `host_only`, the same catalog +/// run on the same host with no symptom. +/// +/// It did when `d` is absent from that run, or present with a different score +/// or different evidence. Either difference alone is enough: a keyword hit on +/// an entry already at the 100 cap changes only its evidence. +fn contributed(d: &Diagnosis, host_only: &[Diagnosis]) -> bool { + host_only + .iter() + .find(|h| h.id == d.id) + .is_none_or(|h| h.score != d.score || h.evidence != d.evidence) +} + /// Diagnose an examination against the closed catalog. #[must_use] pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { @@ -221,6 +277,7 @@ pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { } else { run_all_checks(e, symptom) }; + mark_symptom_contribution(&mut matched, e, symptom); take_applicability_from_the_catalog(&mut matched, platform_family(e)); DiagnoseReport { has_match: any_cleared_threshold(&matched), @@ -507,4 +564,167 @@ mod tests { // And a bare framework mention the engine would reject is not diagnosable. assert!(!vllm_oom_symptom_is_diagnosable("vllm: out of memory")); } + + /// The entry with `id` in `report`, or a panic naming what was listed. + fn entry<'a>(report: &'a DiagnoseReport, id: &str) -> &'a Diagnosis { + report + .matched + .iter() + .find(|d| d.id == id) + .unwrap_or_else(|| { + panic!( + "{id} was not listed: {:?}", + report + .matched + .iter() + .map(|d| (&d.id, d.score)) + .collect::>() + ) + }) + } + + fn with_gpu_override() -> Examination { + let mut e = linux_base(); + e.env + .insert("HSA_OVERRIDE_GFX_VERSION".to_owned(), "11.0.0".to_owned()); + e + } + + const ISA_PASTE: &str = "hipErrorNoBinaryForGpu: HSA_STATUS_ERROR_INVALID_ISA"; + + /// The case the field exists for: a finding about the machine sits beside + /// the entry the paste scores, and the paste contributed to only one of them. + #[test] + fn a_host_state_finding_is_not_credited_to_a_paste_about_something_else() { + let report = diagnose(&with_gpu_override(), ISA_PASTE); + + let host = entry(&report, "fix-2-unset-override"); + assert!( + !host.symptom_matched, + "the override finding comes from the environment alone: {host:?}" + ); + let pasted = entry(&report, "fix-1-arch"); + assert!( + pasted.symptom_matched, + "the ISA entry is scored from the paste: {pasted:?}" + ); + + // And both answers reach the document a consumer reads. + let json = serde_json::to_value(&report).unwrap(); + let field = |id: &str| { + json["matched"] + .as_array() + .unwrap() + .iter() + .find(|d| d["id"] == id) + .map(|d| d["symptom_matched"].clone()) + }; + assert_eq!(field("fix-2-unset-override"), Some(false.into())); + assert_eq!(field("fix-1-arch"), Some(true.into())); + } + + #[test] + fn a_paste_that_raises_a_host_finding_is_credited() { + // The override alone scores 30; a page fault in the paste adds to it. + let report = diagnose(&with_gpu_override(), "memory access page fault"); + let raised = entry(&report, "fix-2-unset-override"); + assert!( + raised.score > 30, + "the paste must have raised it: {raised:?}" + ); + assert!(raised.symptom_matched, "{raised:?}"); + } + + #[test] + fn a_paste_that_lifts_an_entry_the_host_alone_ruled_out_is_credited() { + // The framework already carries this GPU's target, which pulls the + // arch entry below zero on its own, so it is not listed without the + // paste at all. With it, the entry is listed, and only because of it. + let mut e = linux_base(); + e.gpus = vec![crate::examine::Gpu { + gfx_target: "gfx1100".to_owned(), + is_amd: true, + ..crate::examine::Gpu::default() + }]; + e.framework_arch_list = vec!["gfx1100".to_owned()]; + assert!( + diagnose(&e, "") + .matched + .iter() + .all(|d| d.id != "fix-1-arch"), + "the premise: without the paste this entry is not listed" + ); + let report = diagnose(&e, ISA_PASTE); + assert!(entry(&report, "fix-1-arch").symptom_matched); + } + + #[test] + fn with_no_symptom_every_entry_says_so_and_the_field_is_still_written() { + for symptom in ["", " "] { + let report = diagnose(&with_gpu_override(), symptom); + assert!( + !report.matched.is_empty(), + "this test is pointless unless something was listed" + ); + assert!( + report.matched.iter().all(|d| !d.symptom_matched), + "nothing was pasted for {symptom:?}, so nothing in it contributed: {:?}", + report.matched + ); + let json = serde_json::to_value(&report).unwrap(); + for d in json["matched"].as_array().unwrap() { + assert_eq!( + d.get("symptom_matched"), + Some(&serde_json::Value::Bool(false)), + "the field is written, not omitted, when no symptom was given: {d}" + ); + } + } + } + + #[test] + fn a_payload_written_before_the_field_existed_still_reads() { + let old = + r#"{"id":"fix-2-unset-override","title":"t","score":30,"evidence":[],"fix":null}"#; + let d: Diagnosis = serde_json::from_str(old).expect("an older payload deserializes"); + assert!(!d.symptom_matched); + } + + fn finding(id: &str, score: i32, evidence: &[&str]) -> Diagnosis { + Diagnosis { + id: id.to_owned(), + score, + evidence: evidence.iter().map(|&e| e.to_owned()).collect(), + ..Diagnosis::default() + } + } + + /// The three ways the symptom can show in an entry, one at a time, so each + /// half of the comparison has a case that only it decides. + #[test] + fn contribution_is_any_difference_from_the_run_without_the_symptom() { + let host_only = [finding("fix-a", 30, &["host fact"])]; + + // Same entry, same evidence, higher score: the score alone differs. + assert!(contributed( + &finding("fix-a", 55, &["host fact"]), + &host_only + )); + // Same score, extra evidence: what a keyword hit does to an entry + // already at the 100 cap. The evidence alone differs. + assert!(contributed( + &finding("fix-a", 30, &["host fact", "error mentions X"]), + &host_only + )); + // Not listed at all without the symptom. + assert!(contributed( + &finding("fix-b", 50, &["error mentions Y"]), + &host_only + )); + // Identical to the run without it: nothing contributed. + assert!(!contributed( + &finding("fix-a", 30, &["host fact"]), + &host_only + )); + } } diff --git a/crates/rocm-core/src/diagnose/host.rs b/crates/rocm-core/src/diagnose/host.rs index 4e81ddf823..32d43db2b7 100644 --- a/crates/rocm-core/src/diagnose/host.rs +++ b/crates/rocm-core/src/diagnose/host.rs @@ -60,8 +60,8 @@ pub(super) fn check_1_arch_not_in_wheel(e: &Examination, symptom: &str) -> Diagn summary: "Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.".to_owned(), commands: vec![ "# Recommended: a PyTorch ROCm nightly, which often carries kernels a".to_owned(), - "# release has not shipped yet. Pick the nightly for the ROCm major you".to_owned(), - "# have, not an older one.".to_owned(), + "# release has not shipped yet. Pick the nightly for the ROCm X.Y release".to_owned(), + "# you have, not an older one.".to_owned(), "pip uninstall -y torch torchvision torchaudio".to_owned(), "pip install --pre torch torchvision torchaudio \\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14".to_owned(), "# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.".to_owned(), @@ -299,14 +299,14 @@ pub(super) fn check_8_wheel_rocm_mismatch(e: &Examination, symptom: &str) -> Dia &e.rocm_version }; - let fw_major = major_version(fw_rocm); - let sys_major = major_version(sys_rocm); + let fw_release = release_line(fw_rocm); + let sys_release = release_line(sys_rocm); // Only meaningful when the framework resolves its HIP from the system. A // managed runtime's torch loads it from a sibling `_rocm_sdk_core` package - // inside the runtime, so its HIP major is free to differ from the system's + // inside the runtime, so its HIP release is free to differ from the system's // on a completely healthy host — and "reinstall torch" would be wrong there. let framework_uses_system_rocm = e.framework_source != "managed-runtime"; - if let (Some(fw), Some(sys)) = (&fw_major, &sys_major) + if let (Some(fw), Some(sys)) = (&fw_release, &sys_release) && fw != sys && framework_uses_system_rocm { @@ -326,12 +326,12 @@ pub(super) fn check_8_wheel_rocm_mismatch(e: &Examination, symptom: &str) -> Dia } let fix = if windows { Fix { - summary: "Reinstall the framework against the HIP SDK major you have installed (or install the HIP SDK major the wheel needs).".to_owned(), + summary: "Reinstall the framework against the HIP SDK X.Y release you have installed (or install the HIP SDK X.Y release the wheel needs).".to_owned(), commands: vec![ "pip uninstall -y torch torchvision torchaudio".to_owned(), "# TheRock publishes Windows ROCm wheels per HIP SDK release:".to_owned(), "# https://github.com/ROCm/TheRock".to_owned(), - "# Match the wheel index to the HIP SDK major you have on disk.".to_owned(), + "# Match the wheel index to the HIP SDK X.Y release you have on disk.".to_owned(), "python -c \"import torch; print(torch.__version__, torch.version.hip)\"".to_owned(), ], fix_id: "fix-8-wheel-rocm".to_owned(), @@ -340,10 +340,10 @@ pub(super) fn check_8_wheel_rocm_mismatch(e: &Examination, symptom: &str) -> Dia } } else { Fix { - summary: "Reinstall the framework from the wheel index that matches the system ROCm major (or upgrade the system ROCm to match the wheel).".to_owned(), + summary: "Reinstall the framework from the wheel index that matches the system ROCm X.Y release (or upgrade the system ROCm to match the wheel).".to_owned(), commands: vec![ "pip uninstall -y torch torchvision torchaudio".to_owned(), - "# Install the index for the ROCm major `rocm examine` reports:".to_owned(), + "# Install the index for the ROCm X.Y release `rocm examine` reports:".to_owned(), "pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14".to_owned(), "# ROCm 10 has no released PyTorch index yet; it is on the nightly channel:".to_owned(), "pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0".to_owned(), @@ -357,15 +357,30 @@ pub(super) fn check_8_wheel_rocm_mismatch(e: &Examination, symptom: &str) -> Dia }; finalize( "fix-8-wheel-rocm", - "Framework wheel built for a different ROCm major than the system", + "Framework wheel built for a different ROCm X.Y release than the system", score, evidence, fix, ) } -/// Extract `X.Y` from a version-ish string. -fn major_version(s: &str) -> Option { +/// Extract the `X.Y` release line from a version-ish string: `6.4` from +/// `hip=6.4.43482` or `6.4.1`. +/// +/// `check_8` compares this, not the bare major, on purpose: ROCm PyTorch wheels +/// are built and indexed per `X.Y` (`whl/rocm6.4`), so a torch built for 6.3 +/// beside a system 6.4 is a mismatch even though both are ROCm 6. The patch +/// level is dropped because the wheel index does not carry it. +/// +/// Returns `None` when the string has no minor, e.g. a framework that reports +/// only `hip=7`. `check_8` then skips the version comparison and scores on the +/// symptom alone, as it did before this function was renamed from +/// `major_version`; the regex is unchanged. +/// +/// Unlike `report.rs`'s `major_minor`, which reads only a leading all-digit +/// `X[.Y]` and keeps a bare major, this takes the first `X.Y` anywhere in the +/// string and returns `None` when there is no minor. +fn release_line(s: &str) -> Option { let re = Regex::new(r"(\d+)\.(\d+)").ok()?; let caps = re.captures(s)?; Some(format!("{}.{}", &caps[1], &caps[2])) @@ -546,6 +561,8 @@ pub(super) fn check_20_gpu_unsupported_by_rocm(e: &Examination, _symptom: &str) mod tests { use super::*; use crate::diagnose::test_support::{igpu_dgpu_host, linux_base, wsl_base}; + #[cfg(unix)] + use crate::diagnose::test_support::{run_in_sh, stub_bin_dir}; use crate::diagnose::{ DiagnoseReport, HIGH_CONFIDENCE, MIN_SCORE_FOR_MATCH, Route, diagnose, render_report_text, }; @@ -553,9 +570,9 @@ mod tests { use crate::fix::{OLDEST_ROCM_MAJOR_THE_CLI_INSTALLS, torch_rocm_indexes_named_in}; use crate::rocm_install::discover_rocm_installs_in; - /// A host whose framework HIP major differs from its system ROCm: torch on - /// HIP 7, a system ROCm 6 beside it. - fn hip_major_differs_from_system_rocm(framework_source: &str) -> Examination { + /// A host whose framework HIP release differs from its system ROCm: torch on + /// HIP 7.14, a system ROCm 6.4 beside it. + fn hip_release_differs_from_system_rocm(framework_source: &str) -> Examination { Examination { framework: "pytorch".to_owned(), framework_rocm_version: "hip=7.14.60850".to_owned(), @@ -567,13 +584,14 @@ mod tests { /// A remedy has to be able to resolve the mismatch that produced it. /// - /// `check_8` fires when the framework's HIP major differs from the system - /// ROCm major, and tells the user to pick the wheel index matching their + /// `check_8` fires when the framework's HIP `X.Y` release differs from the + /// system ROCm's, and tells the user to pick the wheel index matching their /// system. If every index it names is older than any ROCm this CLI installs, /// there is nothing to pick: following the instruction installs a torch for - /// a major the machine does not have, and this same check fires again on the - /// result. A remedy that re-creates its own precondition is worse than none, - /// because the user has spent a reinstall to arrive back where they started. + /// a ROCm release older than any this CLI installs, so its `X.Y` cannot match + /// the system's, and this same check fires again on the result. A remedy that + /// re-creates its own precondition is worse than none, because the user has + /// spent a reinstall to arrive back where they started. #[test] fn the_wheel_mismatch_remedy_can_resolve_the_mismatch_it_reports() { let machine = Examination { @@ -619,12 +637,12 @@ mod tests { fn a_managed_runtimes_hip_is_not_measured_against_the_system_rocm() { // A managed runtime's torch loads HIP from a sibling `_rocm_sdk_core` // package inside the runtime, never from the system install, so the two - // majors are free to differ on a perfectly healthy host. Before + // releases are free to differ on a perfectly healthy host. Before // `examine` probed the runtime this could not fire, because the field it // reads was always empty; now that it is populated, the comparison has // to be told when it is meaningless -- or fixing the probe would hand // every such host a spurious "reinstall torch". - let managed = diagnose(&hip_major_differs_from_system_rocm("managed-runtime"), ""); + let managed = diagnose(&hip_release_differs_from_system_rocm("managed-runtime"), ""); assert!( !managed.matched.iter().any(|d| d.id == "fix-8-wheel-rocm"), "a managed runtime must not be told to reinstall torch: {:?}", @@ -638,12 +656,12 @@ mod tests { // The same host, same versions, with torch coming from the ambient // interpreter: there the comparison is exactly right, and the checker // must keep its full strength. - let ambient = diagnose(&hip_major_differs_from_system_rocm("path"), ""); + let ambient = diagnose(&hip_release_differs_from_system_rocm("path"), ""); let finding = ambient .matched .iter() .find(|d| d.id == "fix-8-wheel-rocm") - .expect("an ambient torch built against a different ROCm major is a real mismatch"); + .expect("an ambient torch built against a different ROCm release is a real mismatch"); assert!( finding.score >= MIN_SCORE_FOR_MATCH, "the version evidence alone has to establish it: {}", @@ -651,6 +669,79 @@ mod tests { ); } + /// `check_8` compares the `X.Y` release line, because that is how ROCm + /// PyTorch wheels are indexed (`whl/rocm6.4`). Its title once said "ROCm + /// major" over this same comparison, so this pins both the comparison and + /// that neither title names the major again. + #[test] + fn the_wheel_mismatch_compares_the_x_y_release_and_says_so() { + let host = |framework_rocm_version: &str| Examination { + framework: "pytorch".to_owned(), + framework_rocm_version: framework_rocm_version.to_owned(), + framework_source: "path".to_owned(), + rocm_version: "6.4.1".to_owned(), + ..linux_base() + }; + + // Same major, different minor: a mismatch. + let minor = check_8_wheel_rocm_mismatch(&host("hip=6.3.42131"), ""); + assert_eq!( + minor.evidence, + vec!["Framework links HIP 6.3 but system ROCm is 6.4".to_owned()], + "a 6.3 torch beside a 6.4 system is built for another wheel index" + ); + assert!(minor.score >= MIN_SCORE_FOR_MATCH, "{}", minor.score); + + // Same X.Y, different patch: the same wheel index, so no mismatch. + let patch = check_8_wheel_rocm_mismatch(&host("hip=6.4.43482"), ""); + assert_eq!(patch.score, 0, "{:?}", patch.evidence); + + // No minor to compare: the comparison is skipped, not read as a mismatch. + let no_minor = check_8_wheel_rocm_mismatch(&host("hip=7"), ""); + assert_eq!(no_minor.score, 0, "{:?}", no_minor.evidence); + + let catalog_title = crate::fix::catalog_manifest() + .entries + .into_iter() + .find(|entry| entry.id == "fix-8-wheel-rocm") + .map(|entry| entry.title) + .expect("fix-8-wheel-rocm is in the recipe catalog"); + for title in [minor.title.as_str(), catalog_title.as_str()] { + assert!( + title.contains("X.Y") && !title.contains("major"), + "the title must name what is compared: {title}" + ); + } + } + + /// The symptom diagnose-67 uses raises fix-8 from its keywords alone, on + /// every platform family the check is registered for, whatever the host's + /// own ROCm: `libamdhip64.so` scores 50 and the loader failure 25, and the + /// version comparison can only add to that. This is what lets the scenario + /// run untagged on every lane. + #[test] + fn a_library_load_error_alone_raises_the_wheel_mismatch_on_every_family() { + let symptom = "libamdhip64.so.6: cannot open shared object file"; + let windows = Examination { + os_family: "windows".to_owned(), + ..Examination::default() + }; + for host in [linux_base(), windows, wsl_base()] { + let report = diagnose(&host, symptom); + let finding = report + .matched + .iter() + .find(|d| d.id == "fix-8-wheel-rocm") + .unwrap_or_else(|| panic!("no fix-8 on {}: {report:?}", host.os_family)); + assert!( + finding.score >= MIN_SCORE_FOR_MATCH, + "{}: {}", + host.os_family, + finding.score + ); + } + } + #[test] fn check_20_is_silent_without_a_confirmed_unsupported_gpu() { assert_eq!( @@ -1328,43 +1419,6 @@ mod tests { ); } - /// Writes each `(name, script)` as an executable in a new directory, to be - /// the whole `PATH` of a shell that runs a Linux verify line. - #[cfg(unix)] - fn stub_bin_dir(stubs: &[(&str, &str)]) -> tempfile::TempDir { - let dir = tempfile::tempdir().expect("the stub directory is created"); - for (name, script) in stubs { - let path = dir.path().join(name); - crate::fake_executable::write_fake_executable(&path, format!("#!/bin/sh\n{script}\n")) - .expect("a stub is planted"); - } - dir - } - - /// Runs one printed line in `/bin/sh` whose `PATH` is only `bin` and whose - /// environment is only `PATH` and `env`. Returns whether it exited 0, and - /// its stdout and stderr. - #[cfg(unix)] - fn run_in_sh( - line: &str, - bin: &std::path::Path, - env: &[(&str, &str)], - ) -> (bool, String, String) { - let out = std::process::Command::new("/bin/sh") - .arg("-c") - .arg(line) - .env_clear() - .env("PATH", bin) - .envs(env.iter().copied()) - .output() - .expect("/bin/sh runs"); - ( - out.status.success(), - String::from_utf8_lossy(&out.stdout).into_owned(), - String::from_utf8_lossy(&out.stderr).into_owned(), - ) - } - /// The fix-6 line `rocm diagnose` prints on Linux, run on a runtime-only /// install: `rocminfo` is on PATH once the fix has worked, and there is no /// `hipcc`. The line must succeed and show the GPU. The old line, with diff --git a/crates/rocm-core/src/diagnose/linux.rs b/crates/rocm-core/src/diagnose/linux.rs index 8f9d98fe38..9aa0d346da 100644 --- a/crates/rocm-core/src/diagnose/linux.rs +++ b/crates/rocm-core/src/diagnose/linux.rs @@ -13,6 +13,27 @@ use super::scoring::{ use super::{Diagnosis, Fix}; use crate::examine::Examination; +/// The phrase `check_3_rocm_kernel_unsupported` puts in its evidence when +/// `amdgpu_loaded` is `Some(false)` and ROCm is installed. +const AMDGPU_KERNEL_MODULE_NOT_LOADED: &str = "the amdgpu kernel module is not loaded"; +/// The evidence line `check_5_amdgpu_blacklisted` adds when `amdgpu_loaded` is +/// `Some(false)`. +const AMDGPU_MODULE_NOT_LOADED: &str = "amdgpu module is not loaded"; +/// The start of the evidence line `check_5_amdgpu_blacklisted` adds when Secure +/// Boot is enabled and `amdgpu_loaded` is `Some(false)`. +const SECURE_BOOT_BLOCKED_AMDGPU: &str = "Secure Boot is enabled and amdgpu didn't load"; + +/// One phrase per evidence line that only `amdgpu_loaded == Some(false)` produces. +/// +/// The checks build their lines from these same constants, so a test that asserts +/// a diagnosis cites none of them (the e2e scenario for a built-in `amdgpu`) +/// cannot drift from the wording and pass vacuously. +pub const AMDGPU_MISSING_EVIDENCE: [&str; 3] = [ + AMDGPU_KERNEL_MODULE_NOT_LOADED, + AMDGPU_MODULE_NOT_LOADED, + SECURE_BOOT_BLOCKED_AMDGPU, +]; + pub(super) fn check_3_rocm_kernel_unsupported(e: &Examination, symptom: &str) -> Diagnosis { let mut score = 0; let mut evidence = Vec::new(); @@ -24,7 +45,7 @@ pub(super) fn check_3_rocm_kernel_unsupported(e: &Examination, symptom: &str) -> if !rocm_version.is_empty() && e.amdgpu_loaded == Some(false) { score += 30; evidence.push(format!( - "ROCm {rocm_version} is installed but the amdgpu kernel module is not loaded; this is typical when DKMS failed against an unsupported kernel." + "ROCm {rocm_version} is installed but {AMDGPU_KERNEL_MODULE_NOT_LOADED}; this is typical when DKMS failed against an unsupported kernel." )); } @@ -115,7 +136,7 @@ pub(super) fn check_4_render_group(e: &Examination, symptom: &str) -> Diagnosis needs_sudo: true, needs_relogin: true, fix_id: "fix-4-render-group".to_owned(), - verify: "groups | tr ' ' '\\n' | grep -E '^(render|video)$' && ls -l /dev/kfd && rocminfo | head -n 5".to_owned(), + verify: crate::fix::RENDER_GROUP_VERIFY.to_owned(), notes: vec![ "Group membership only takes effect after a full re-login (or reboot). `newgrp render` will give the current shell access but not other terminals or services.".to_owned(), ], @@ -140,7 +161,7 @@ pub(super) fn check_5_amdgpu_blacklisted(e: &Examination, symptom: &str) -> Diag } if e.amdgpu_loaded == Some(false) { score += 35; - evidence.push("amdgpu module is not loaded".to_owned()); + evidence.push(AMDGPU_MODULE_NOT_LOADED.to_owned()); } if e.rocminfo_status == "not-loaded" { score += 25; @@ -148,7 +169,7 @@ pub(super) fn check_5_amdgpu_blacklisted(e: &Examination, symptom: &str) -> Diag } if e.secure_boot == "enabled" && e.amdgpu_loaded == Some(false) { score += 10; - evidence.push("Secure Boot is enabled and amdgpu didn't load -- DKMS modules are often blocked until you sign them or disable Secure Boot.".to_owned()); + evidence.push(format!("{SECURE_BOOT_BLOCKED_AMDGPU} -- DKMS modules are often blocked until you sign them or disable Secure Boot.")); } let (kw_score, kw_ev) = keyword_score(symptom, KEYWORDS_MODULE_NOT_LOADED); score += kw_score; @@ -646,9 +667,51 @@ pub(super) fn check_12_repo_native_broken(e: &Examination, symptom: &str) -> Dia mod tests { use super::*; use crate::diagnose::test_support::linux_base; + #[cfg(unix)] + use crate::diagnose::test_support::{run_in_sh, stub_bin_dir}; use crate::diagnose::{HIGH_CONFIDENCE, MIN_SCORE_FOR_MATCH, diagnose, render_report_text}; use crate::examine::{Device, Gpu}; + #[test] + fn every_amdgpu_missing_phrase_is_in_the_evidence_the_checks_produce() { + let mut e = linux_base(); + e.rocm_version = "7.1.0".to_owned(); + e.amdgpu_loaded = Some(false); + e.secure_boot = "enabled".to_owned(); + let lines: Vec = [ + check_3_rocm_kernel_unsupported(&e, ""), + check_5_amdgpu_blacklisted(&e, ""), + ] + .into_iter() + .flat_map(|d| d.evidence) + .collect(); + for phrase in AMDGPU_MISSING_EVIDENCE { + assert!( + lines.iter().any(|l| l.contains(phrase)), + "no evidence line contains {phrase:?}: {lines:#?}" + ); + } + } + + #[test] + fn built_in_amdgpu_produces_none_of_the_amdgpu_missing_phrases() { + let mut e = linux_base(); + e.rocm_version = "7.1.0".to_owned(); + e.amdgpu_loaded = Some(true); + e.secure_boot = "enabled".to_owned(); + for d in [ + check_3_rocm_kernel_unsupported(&e, ""), + check_5_amdgpu_blacklisted(&e, ""), + ] { + for line in &d.evidence { + assert!( + !AMDGPU_MISSING_EVIDENCE.iter().any(|p| line.contains(p)), + "{line}" + ); + } + } + } + #[test] fn render_group_missing_is_diagnosed() { let mut e = linux_base(); @@ -690,6 +753,93 @@ mod tests { assert_eq!(top.score, 60); // 35 render + 25 kfd } + /// fix-4's verify line is printed by `rocm diagnose` (from this check's + /// `Fix`) and by `rocm fix fix-4-render-group` (from the recipe). The two + /// copies drifted once, with only this one listing `/dev/kfd`. Both now read + /// `crate::fix::RENDER_GROUP_VERIFY`, and this keeps either side from going + /// back to a literal of its own. The check builds its verify line without + /// looking at the host, so one host that fires it is enough. + #[test] + fn the_render_group_fix_verify_matches_the_catalog_copy() { + let (catalog_verify, _, _) = + crate::fix::recipe_verify_notes_and_run_commands("fix-4-render-group") + .expect("fix-4-render-group is in the recipe catalog"); + let not_in_render = Examination { + in_render_group: Some(false), + ..linux_base() + }; + let fix = check_4_render_group(¬_in_render, "") + .fix + .expect("check 4 builds a Fix when the user is not in render"); + assert_eq!( + fix.verify, catalog_verify, + "the verify line for fix-4-render-group diverged between `diagnose` and `rocm fix`" + ); + // The owning group is in the line, as it is in the command the check + // tells the user to run; `ls -l` prints it for the user to read. + // `the_render_group_verify_accepts_the_group_that_owns_kfd` runs it. + assert!( + catalog_verify.contains("stat -c %G /dev/kfd") + && catalog_verify.contains("ls -l /dev/kfd"), + "{catalog_verify}" + ); + } + + /// fix-4's verify line, run in `/bin/sh` with stand-ins for `groups`, + /// `stat`, `ls` and `rocminfo`. The check tells the user to join the group + /// that owns `/dev/kfd`, which need not be `render`, so a member of that + /// group alone must pass, and a user in none of the groups must fail, + /// including when `stat` fails or names no group (`UNKNOWN`). + #[cfg(unix)] + #[test] + fn the_render_group_verify_accepts_the_group_that_owns_kfd() { + fn real(tool: &str) -> String { + ["/usr/bin", "/bin"] + .iter() + .map(|dir| format!("{dir}/{tool}")) + .find(|path| std::path::Path::new(path).exists()) + .unwrap_or_else(|| panic!("{tool} is installed")) + } + let passes = |groups: &str, owner: Option<&str>| { + let groups = format!("echo '{groups}'"); + let stat = owner.map_or_else( + || "echo 'stat: cannot stat' >&2; exit 1".to_owned(), + |owner| format!("echo '{owner}'"), + ); + let tr = format!("exec {} \"$@\"", real("tr")); + let grep = format!("exec {} \"$@\"", real("grep")); + let head = format!("exec {} \"$@\"", real("head")); + let bin = stub_bin_dir(&[ + ("groups", groups.as_str()), + ("stat", stat.as_str()), + ("ls", "echo 'crw-rw---- 1 root kfd 235, 0 /dev/kfd'"), + ("rocminfo", "echo agent"), + ("tr", tr.as_str()), + ("grep", grep.as_str()), + ("head", head.as_str()), + ]); + let (ok, _, _) = run_in_sh(crate::fix::RENDER_GROUP_VERIFY, bin.path(), &[]); + ok + }; + assert!( + passes("user kfd", Some("kfd")), + "a member of the group that owns /dev/kfd must pass" + ); + assert!(passes("user render", Some("kfd")), "render still counts"); + assert!( + !passes("user", Some("kfd")), + "a user in none of them must fail" + ); + assert!( + !passes("user", Some("UNKNOWN")), + "an owner with no name must not let a user in none of them pass" + ); + assert!( + !passes("user", None), + "a failing stat leaves an empty alternative that must match nothing" + ); + } + #[test] fn unknown_kfd_group_falls_back_to_render() { // `stat -c %G /dev/kfd` prints "UNKNOWN" when the GID has no group diff --git a/crates/rocm-core/src/diagnose/report.rs b/crates/rocm-core/src/diagnose/report.rs index 9e0f355305..0ea6614080 100644 --- a/crates/rocm-core/src/diagnose/report.rs +++ b/crates/rocm-core/src/diagnose/report.rs @@ -70,6 +70,17 @@ fn is_advice_only(fix_id: &str) -> bool { fix_id == crate::windows_driver::DISPLAY_DRIVER_OLD_FIX_ID } +/// The note under a cause the symptom text did not contribute to. +/// +/// Printed when the symptom contributed to some other cause in the same +/// report: the text form of [`super::Diagnosis::symptom_matched`] being +/// `false`. +/// +/// "The examined host", not "this machine": `rocm remote doctor` renders the +/// same report for a machine it reached over the network. +pub const SYMPTOM_DID_NOT_CONTRIBUTE_NOTE: &str = + "source: the examined host's state alone; the symptom text did not contribute to it"; + /// Render the human-facing diagnosis view (mirrors `diagnose.py`'s text output). #[must_use] pub fn render_report_text(report: &DiagnoseReport, top: usize) -> String { @@ -100,6 +111,7 @@ pub fn render_report_text(report: &DiagnoseReport, top: usize) -> String { "\nAdvice about this host follows. It does not explain the symptom, and nothing in it is applied automatically.\n\n", ); } + let symptom_matched_any = report.matched.iter().any(|d| d.symptom_matched); for (i, d) in report.matched.iter().take(top).enumerate() { let tier = if d.score >= HIGH_CONFIDENCE { "HIGH" @@ -124,6 +136,19 @@ pub fn render_report_text(report: &DiagnoseReport, top: usize) -> String { for ev in &d.evidence { let _ = writeln!(out, " - {ev}"); } + // Said only once the symptom contributed to something in this report: + // a finding about the host can outrank the one the symptom added to, + // and the ranking alone reads as if the symptom lay behind it. With no + // symptom, or one that contributed to nothing, every row would carry it + // and it would say nothing. + // + // `symptom_matched_any` reads the whole report, not only the rows + // `--top` shows. When the only entry the symptom contributed to is cut + // off, every row shown carries the note, which is still true of each, + // and the "Showing N of M" line below says more were found. + if symptom_matched_any && !d.symptom_matched { + let _ = writeln!(out, " {SYMPTOM_DID_NOT_CONTRIBUTE_NOTE}"); + } if let Some(fix) = &d.fix { let _ = writeln!(out, " plan: {}", fix.summary); for c in &fix.commands { @@ -216,6 +241,7 @@ mod tests { class: crate::fix::FixClass::DiagnoseOnly, ..Fix::default() }), + ..Diagnosis::default() }, Diagnosis { id: "fix-9-igpu-dgpu".to_owned(), @@ -227,6 +253,7 @@ mod tests { summary: "Pin HIP_VISIBLE_DEVICES.".to_owned(), ..Fix::default() }), + ..Diagnosis::default() }, ], has_match: true, @@ -388,6 +415,97 @@ mod tests { ); } + /// The cause block for `id` in rendered `out`: its header line and every + /// line under it up to the blank line that ends it. + fn block(out: &str, id: &str) -> String { + let marker = format!("({id})"); + out.lines() + .skip_while(|l| !l.contains(&marker)) + .take_while(|l| !l.trim().is_empty()) + .collect::>() + .join("\n") + } + + /// The note and the field it renders are asserted together: the text + /// claims the symptom did not contribute, and the report's own field is + /// what says so. + #[test] + fn a_cause_the_symptom_did_not_contribute_to_says_so_in_the_text() { + let mut e = linux_base(); + e.env + .insert("HSA_OVERRIDE_GFX_VERSION".to_owned(), "11.0.0".to_owned()); + let report = diagnose(&e, "hipErrorNoBinaryForGpu: HSA_STATUS_ERROR_INVALID_ISA"); + let out = render_report_text(&report, 20); + for d in &report.matched { + let shown = block(&out, &d.id); + assert!(!shown.is_empty(), "{} was not rendered:\n{out}", d.id); + assert_eq!( + shown.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + !d.symptom_matched, + "the note must appear exactly where the symptom did not contribute ({}):\n{out}", + d.id + ); + } + let host = report + .matched + .iter() + .find(|d| d.id == "fix-2-unset-override") + .expect("the override is listed"); + assert!(!host.symptom_matched); + assert!( + block(&out, &host.id).contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "{out}" + ); + } + + #[test] + fn without_a_symptom_no_cause_carries_the_note() { + let mut e = linux_base(); + e.env + .insert("HSA_OVERRIDE_GFX_VERSION".to_owned(), "11.0.0".to_owned()); + let report = diagnose(&e, ""); + assert!(!report.matched.is_empty(), "something must be listed"); + let out = render_report_text(&report, 20); + assert!( + !out.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "with nothing pasted the note would sit on every cause and say nothing:\n{out}" + ); + } + + /// The rule for `--top`: the note is decided per row from the whole + /// report, so when the only entry the symptom contributed to is cut off, + /// the row shown still carries it, which is true of that row, and the + /// truncation line says there is more. + #[test] + fn a_contributed_entry_past_top_still_marks_the_rows_shown() { + let row = |id: &str, score: i32, symptom_matched: bool| Diagnosis { + id: id.to_owned(), + title: id.to_owned(), + score, + symptom_matched, + ..Diagnosis::default() + }; + let report = DiagnoseReport { + matched: vec![ + row("fix-host-only", 60, false), + row("fix-from-the-paste", 40, true), + ], + has_match: true, + min_score_for_match: MIN_SCORE_FOR_MATCH, + high_confidence_threshold: HIGH_CONFIDENCE, + route_when_no_match: upstream_route("rocm-core"), + out_of_scope: None, + model: None, + }; + let out = render_report_text(&report, 1); + assert!( + block(&out, "fix-host-only").contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "{out}" + ); + assert!(!out.contains("(fix-from-the-paste)"), "{out}"); + assert!(out.contains("Showing 1 of 2"), "{out}"); + } + #[test] fn no_match_routes_upstream() { let mut e = linux_base(); diff --git a/crates/rocm-core/src/diagnose/scoring.rs b/crates/rocm-core/src/diagnose/scoring.rs index 19871c7a5e..2473403416 100644 --- a/crates/rocm-core/src/diagnose/scoring.rs +++ b/crates/rocm-core/src/diagnose/scoring.rs @@ -465,6 +465,9 @@ pub(super) fn finalize( score: score.min(100), evidence, fix: Some(fix), + // Not a checker's call: `diagnose` sets it by comparing against a run + // with no symptom. + symptom_matched: false, } } diff --git a/crates/rocm-core/src/diagnose/test_support.rs b/crates/rocm-core/src/diagnose/test_support.rs index 0525cfced5..cf13f8a33a 100644 --- a/crates/rocm-core/src/diagnose/test_support.rs +++ b/crates/rocm-core/src/diagnose/test_support.rs @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: MIT -//! `Examination` builders shared by the tests of more than one `diagnose` module. +//! `Examination` builders and shell stubs shared by the tests of more than one +//! `diagnose` module. use crate::examine::{Examination, Gpu, WslFacts}; @@ -77,3 +78,40 @@ pub(super) fn windows_driver_exam(version: &str, date: &str) -> Examination { ..Examination::default() } } + +/// Writes each `(name, script)` as an executable in a new directory, to be +/// the whole `PATH` of a shell that runs a Linux verify line. +#[cfg(unix)] +pub(super) fn stub_bin_dir(stubs: &[(&str, &str)]) -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("the stub directory is created"); + for (name, script) in stubs { + let path = dir.path().join(name); + crate::fake_executable::write_fake_executable(&path, format!("#!/bin/sh\n{script}\n")) + .expect("a stub is planted"); + } + dir +} + +/// Runs one printed line in `/bin/sh` whose `PATH` is only `bin` and whose +/// environment is only `PATH` and `env`. Returns whether it exited 0, and +/// its stdout and stderr. +#[cfg(unix)] +pub(super) fn run_in_sh( + line: &str, + bin: &std::path::Path, + env: &[(&str, &str)], +) -> (bool, String, String) { + let out = std::process::Command::new("/bin/sh") + .arg("-c") + .arg(line) + .env_clear() + .env("PATH", bin) + .envs(env.iter().copied()) + .output() + .expect("/bin/sh runs"); + ( + out.status.success(), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} diff --git a/crates/rocm-core/src/diagnose/windows.rs b/crates/rocm-core/src/diagnose/windows.rs index 63535a40ad..1ebec480b9 100644 --- a/crates/rocm-core/src/diagnose/windows.rs +++ b/crates/rocm-core/src/diagnose/windows.rs @@ -126,7 +126,7 @@ pub(super) fn check_13_hip_sdk_missing(e: &Examination, symptom: &str) -> Diagno let fix = Fix { summary: "Install the AMD HIP SDK for Windows; the HIP runtime DLLs and hipInfo.exe come from there.".to_owned(), commands: vec![ - "# Download and install the HIP SDK (matched to your framework's HIP major):".to_owned(), + "# Download and install the HIP SDK (matched to your framework's HIP X.Y release):".to_owned(), "# https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html".to_owned(), "# After install, reopen the shell so HIP_PATH and PATH pick up the new install.".to_owned(), ], @@ -1200,6 +1200,7 @@ mod tests { score, evidence: vec!["synthetic".to_owned()], fix: None, + ..Diagnosis::default() }, ); assert!(score < HIGH_CONFIDENCE); diff --git a/crates/rocm-core/src/download.rs b/crates/rocm-core/src/download.rs index 81f2c7d84d..bd9aad7ed5 100644 --- a/crates/rocm-core/src/download.rs +++ b/crates/rocm-core/src/download.rs @@ -26,9 +26,13 @@ use crate::disk_space; /// retry, so this is one initial try plus two retries. pub const DOWNLOAD_MAX_ATTEMPTS: u32 = 3; -/// Chunk size for the streaming copy. Fixed, so peak memory is independent of -/// the artifact size — the whole point of streaming a multi-gigabyte tarball. -const DOWNLOAD_CHUNK_BYTES: usize = 64 * 1024; +/// Chunk size for the streaming copy. +/// +/// Fixed, so peak memory is independent of the artifact size — the whole point +/// of streaming a multi-gigabyte tarball. Public so the digest gate's tests can +/// size a body that crosses a chunk boundary from this value rather than from a +/// copy of it. +pub const DOWNLOAD_CHUNK_BYTES: usize = 64 * 1024; /// Exponential backoff between download attempts. /// @@ -442,7 +446,7 @@ fn header_u64(response: &ureq::Response, name: &str) -> Option { } /// First byte offset from a `Content-Range: bytes -/` header. -fn content_range_start(response: &ureq::Response) -> Option { +pub(crate) fn content_range_start(response: &ureq::Response) -> Option { let value = response.header("Content-Range")?; let range = value.trim().strip_prefix("bytes")?.trim_start(); let start = range.split('-').next()?.trim(); diff --git a/crates/rocm-core/src/examine.rs b/crates/rocm-core/src/examine.rs index 5ebd630c16..98ad844b8d 100644 --- a/crates/rocm-core/src/examine.rs +++ b/crates/rocm-core/src/examine.rs @@ -37,6 +37,8 @@ use host_env::{ probe_container, probe_cpu_linux, probe_cpu_windows, probe_dmesg_amdgpu, probe_env, probe_os, probe_shared_memory, }; +#[cfg(test)] +pub(crate) use host_env::{probe_cpu_windows_with, record_cpu}; pub(crate) use library_copies::comgr_matches_runtime; use library_copies::probe_comgr; use linux_driver::{probe_devices, probe_modules, probe_secure_boot, probe_user}; @@ -269,8 +271,22 @@ pub struct Examination { pub has_discrete_amd: bool, // driver / runtime (Linux) + /// amdgpu is present: listed as a module, or registered as a PCI driver + /// (`/sys/bus/pci/drivers/amdgpu`), which it is whether amdgpu is loaded + /// as a module or built into the kernel (where it never appears in + /// `lsmod`), or kfd is present on the host (kfd is linked into amdgpu). + /// + /// This is the driver being present, not a GPU being bound to it: a + /// built-in driver registers at boot with or without a matching GPU, as a + /// loaded module does when `modprobe`d on any machine. Whether there is an + /// AMD GPU is `has_amd_gpu`. `diagnose` reads only `Some(false)`, as a + /// module that failed to load (DKMS, a blacklist, Secure Boot), none of + /// which can stop a driver that is present. pub amdgpu_loaded: Option, pub amdgpu_blacklisted_in: Vec, + /// kfd is available: as a separate `amdkfd` module on kernels before 4.20 + /// (where kfd was merged into `amdgpu.ko`), or through `/sys/class/kfd/kfd` + /// or `/dev/kfd` on current ones. The name predates the merge. pub amdkfd_loaded: Option, pub secure_boot: String, pub iommu_kernel_param: String, @@ -531,6 +547,29 @@ impl Examination { Self::probe_with_interpreter(framework, None) } + /// Probe this host and gather its summary from the examination, returning + /// both: what `rocm examine --json` prints as one document. + /// + /// The pairing is made here so no caller can hand the summary an + /// examination of another host. The summary takes the examination's kernel + /// release (a failed read included) instead of running `uname -r` again, so + /// one inspection reads it once: `summary.kernel` and `kernel_release` + /// cannot disagree on a transient failure, and a hung `uname` costs one + /// timeout rather than two. Probing is as [`Self::probe_with_interpreter`]. + /// + /// # Errors + /// + /// Fails when the host summary cannot be gathered (the application + /// directories cannot be resolved); the examination itself never fails. + pub fn probe_with_summary( + framework: FrameworkProbe, + interpreter: Option<&FrameworkInterpreter>, + ) -> anyhow::Result<(Self, crate::ExamineSummary)> { + let examination = Self::probe_with_interpreter(framework, interpreter); + let summary = crate::ExamineSummary::gather_after(&examination)?; + Ok((examination, summary)) + } + /// As [`Self::probe`], but running the framework probe under `interpreter` /// when one is given. /// @@ -633,7 +672,12 @@ impl Examination { /// Coarse machine-readable verdict reported via the `status` field. The /// process exit code does NOT encode this — `rocm examine` always exits 0 /// when it ran; callers branch on `status` instead. - fn compute_status(&self) -> String { + /// + /// Public so that a tool which edits a recorded examination (the skill-eval + /// recordings do) can recompute the verdict from the same rule the probe uses, + /// instead of typing a `status` that could disagree with the other fields. + #[must_use] + pub fn compute_status(&self) -> String { let value = if self.is_wsl { "wsl" } else if !matches!(self.os_family.as_str(), "linux" | "windows") { diff --git a/crates/rocm-core/src/examine/framework.rs b/crates/rocm-core/src/examine/framework.rs index 420f7b0755..6817a16a9f 100644 --- a/crates/rocm-core/src/examine/framework.rs +++ b/crates/rocm-core/src/examine/framework.rs @@ -125,6 +125,9 @@ fn runtime_library_path_env( } fn probe_pytorch_on_path(e: &mut Examination) { + // On Windows `which` also accepts any `PATHEXT` spelling (`python.PY`, `.PS1`), + // while the probe starts the bare name through `Command`, which only starts an + // `.exe`: the same caveat as in `probe_llama_cpp`. let py = if which("python") { "python" } else if which("python3") { @@ -222,6 +225,10 @@ fn record_pytorch_probe(e: &mut Examination, rc: i32, out: &str, err: &str) { } fn probe_llama_cpp(e: &mut Examination) { + // `which` follows `PATHEXT` on Windows, so a name can read as found through an + // extension this probe cannot run (`llama-cli.PS1`): the probe starts the bare + // name through `Command`, which only starts an `.exe`. The report can then + // name a binary it cannot start; the later run fails and says so. let binary = ["llama-cli", "llama-server", "main"] .into_iter() .find(|name| which(name)); diff --git a/crates/rocm-core/src/examine/host_env.rs b/crates/rocm-core/src/examine/host_env.rs index da192981da..625ed221d5 100644 --- a/crates/rocm-core/src/examine/host_env.rs +++ b/crates/rocm-core/src/examine/host_env.rs @@ -7,6 +7,9 @@ use super::Examination; use super::library_copies::libraries_on_ld_path; +use crate::host_gpu::{ + CpuFacts, read_linux_cpu_facts, run_windows_system_query, windows_cpu_facts_with, +}; use crate::probe_exec::{MEDIUM, SHORT, read_host_text, run, truncate_to_chars}; use crate::{runtime_is_linux, runtime_is_windows}; @@ -54,7 +57,10 @@ pub(super) fn probe_os(e: &mut Examination) { }; if runtime_is_linux() { e.os_family = "linux".to_owned(); - e.kernel_release = run("uname", &["-r"], SHORT).1.trim().to_owned(); + // The summary's read, so `kernel:` and `kernel_release` come from one + // `uname -r`. A failed read is empty here, with no `probe_failures` + // entry, as before: a host without a usable `uname` is not a degraded one. + e.kernel_release = crate::host_gpu::detect_kernel_version().unwrap_or_default(); e.kernel_cmdline = read_host_text("/proc/cmdline").trim().to_owned(); record_distro( e, @@ -114,64 +120,53 @@ pub(super) fn parse_iommu_param(cmdline: &str) -> Option { }) } +/// Fill the CPU fields from the shared reader's answer. A CPU that could not be +/// read leaves the model empty and the vendor `unknown`, which is how the JSON +/// has always said "not collected". +pub(crate) fn record_cpu(e: &mut Examination, facts: CpuFacts) { + e.cpu_model = facts.model.unwrap_or_default(); + e.cpu_vendor = facts.vendor; +} + +/// The CPU on Linux, from `/proc/cpuinfo` through the reader the host summary +/// uses. A cpuinfo with no CPU in it is not a probe failure. pub(super) fn probe_cpu_linux(e: &mut Examination) { - let txt = read_host_text("/proc/cpuinfo"); - for line in txt.lines() { - if (e.cpu_vendor == "unknown") - && line.starts_with("vendor_id") - && let Some((_, value)) = line.split_once(':') - { - let value = value.trim(); - e.cpu_vendor = if value.contains("AMD") { - "amd".to_owned() - } else if value.contains("Intel") { - "intel".to_owned() - } else { - value.to_lowercase() - }; - } - if e.cpu_model.is_empty() - && line.starts_with("model name") - && let Some((_, value)) = line.split_once(':') - { - e.cpu_model = value.trim().to_owned(); - } - if e.cpu_vendor != "unknown" && !e.cpu_model.is_empty() { - break; - } - } + record_cpu(e, read_linux_cpu_facts()); } +/// The CPU on Windows, from the CPU row of the system inventory through the +/// reader the host summary uses. pub(super) fn probe_cpu_windows(e: &mut Examination) { - let (rc, out, _) = run( - "powershell", - &[ - "-NoProfile", - "-Command", - "(Get-CimInstance Win32_Processor | Select-Object -First 1).Name", - ], - MEDIUM, - ); - if rc == 0 && !out.trim().is_empty() { - e.cpu_model = out - .trim() - .lines() - .next() - .unwrap_or_default() - .trim() - .to_owned(); - let lname = e.cpu_model.to_lowercase(); - e.cpu_vendor = if lname.contains("amd") { - "amd".to_owned() - } else if lname.contains("intel") { - "intel".to_owned() - } else { - "unknown".to_owned() - }; - } else { - e.probe_failures - .push("Get-CimInstance Win32_Processor failed; cannot identify CPU.".to_owned()); + probe_cpu_windows_with(e, run_windows_system_query); +} + +/// [`probe_cpu_windows`] with the query injected, so a test on any host drives +/// the examination's whole Windows read, the failure entry included. `query` is +/// handed the inventory's script and time limit; see `windows_cpu_facts_with`. +/// [`probe_cpu_windows`] passes the real PowerShell runner, which is the one +/// argument no test on a Linux host can reach. +pub(crate) fn probe_cpu_windows_with( + e: &mut Examination, + query: impl FnOnce(&str, std::time::Duration) -> Option, +) { + record_cpu_windows(e, windows_cpu_facts_with(query)); +} + +/// The `probe_failures` entry for a Windows read that found no CPU. It names the +/// query that ran, the system inventory's CPU and RAM script, and fires whether +/// that query failed, timed out, printed nothing or printed no CPU row. +pub(crate) const WINDOWS_CPU_NOT_FOUND: &str = + "The Windows system inventory query (CPU and RAM) reported no CPU; cannot identify CPU."; + +/// [`record_cpu`], and a failure entry when no CPU could be read: on Windows the +/// CPU comes from a query that can fail or time out, and an empty model alone +/// would not say so. Crate-visible so a test on any host can feed it the answer +/// of an injected query. +pub(crate) fn record_cpu_windows(e: &mut Examination, facts: CpuFacts) { + if facts.model.is_none() { + e.probe_failures.push(WINDOWS_CPU_NOT_FOUND.to_owned()); } + record_cpu(e, facts); } pub(super) fn probe_env(e: &mut Examination) { @@ -618,4 +613,359 @@ mod tests { ); assert_eq!(name.as_deref(), Some("Fallback 1.2")); } + + /// Plant `files` (path under the root, body) in a fresh directory and run + /// `read` with it as the simulated host root. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + fn with_planted_host_root(files: &[(&str, &str)], read: impl FnOnce() -> T) -> T { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-cpu-root-{}", + crate::test_support::unique_suffix() + )); + for (relative, body) in files { + let path = root.join(relative.trim_start_matches('/')); + std::fs::create_dir_all(path.parent().expect("planted path has a parent")) + .expect("create planted directory"); + std::fs::write(path, body).expect("write planted file"); + } + let result = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + read() + }; + std::fs::remove_dir_all(&root).ok(); + result + } + + /// CPU-model pair of docs/host-probing.md, native Linux. Both reads take + /// `/proc/cpuinfo` from the simulated host root, so a planted CPU no runner + /// has is what each reports, and both report the model with its whitespace + /// collapsed (the examination used to keep the runs of spaces). + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn cpu_reads_follow_the_simulated_host_root() { + let (summary, examined) = with_planted_host_root( + &[( + "proc/cpuinfo", + "processor\t: 0\nvendor_id\t: AuthenticAMD\nmodel name\t: Planted CPU 9000 Processor\n", + )], + || { + let mut examined = Examination::default(); + probe_cpu_linux(&mut examined); + (crate::host_gpu::detect_cpu_model(), examined) + }, + ); + assert_eq!(summary.as_deref(), Some("Planted CPU 9000 Processor")); + assert_eq!(examined.cpu_model, "Planted CPU 9000 Processor"); + assert_eq!(examined.cpu_vendor, "amd"); + } + + /// CPU-model pair, WSL: the summary reads the CPU and the examination does + /// not. `probe_with_interpreter` returns from its WSL branch before + /// `probe_cpu_linux`, so on every WSL host `cpu_model` is empty and + /// `cpu_vendor` is `unknown` while the summary shows the CPU. Pinned as an + /// existing difference of the CPU-model pair, which leaves it as it is. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn on_wsl_the_summary_reads_the_cpu_and_the_examination_does_not() { + let (summary, examined) = with_planted_host_root( + &[ + ("dev/dxg", ""), + ( + "proc/cpuinfo", + "processor\t: 0\nvendor_id\t: AuthenticAMD\nmodel name\t: Planted WSL CPU\n", + ), + ], + || { + ( + crate::host_gpu::detect_cpu_model(), + Examination::probe(crate::examine::FrameworkProbe::Skip), + ) + }, + ); + assert!(examined.is_wsl, "the planted /dev/dxg reads as WSL"); + assert_eq!(summary.as_deref(), Some("Planted WSL CPU")); + assert_eq!(examined.cpu_model, ""); + assert_eq!(examined.cpu_vendor, "unknown"); + } + + /// `probe_cpu_windows` runs the system inventory's query and nothing else: + /// off Windows there is no PowerShell for the real runner, so the read finds + /// no CPU and the entry says which query that was, and the injected-query + /// form given a CPU row fills the fields the same way. Replacing the runner + /// `probe_cpu_windows` passes, or the failure entry, fails this test. On + /// Windows the runner is live, so it is skipped there. + #[cfg(not(windows))] + #[test] + fn probe_cpu_windows_runs_the_inventory_query_and_reports_no_cpu_without_one() { + let mut examined = Examination::default(); + probe_cpu_windows(&mut examined); + assert_eq!( + (examined.cpu_model.as_str(), examined.cpu_vendor.as_str()), + ("", "unknown") + ); + assert_eq!(examined.probe_failures, [WINDOWS_CPU_NOT_FOUND]); + + let mut with_cpu = Examination::default(); + probe_cpu_windows_with(&mut with_cpu, |_, _| { + Some("CPU\tIntel(R) Core(TM) i5-12400\nRAM\t17179869184\n".to_owned()) + }); + assert_eq!( + (with_cpu.cpu_model.as_str(), with_cpu.cpu_vendor.as_str()), + ("Intel(R) Core(TM) i5-12400", "intel") + ); + assert!(with_cpu.probe_failures.is_empty()); + } + + // The kernel-release read, the second pair of `docs/host-probing.md` + // ("Kernel version"). It used to be two readers: `detect_kernel_version` in + // the summary stack (1.5 s, strict UTF-8, `None` on any failure or non-zero + // exit) and `probe_os` here (5 s, lossy, exit code ignored, empty on + // failure). `probe_os` now calls `detect_kernel_version`, so these tests pin + // the one reader's behaviour through both entry points: a release is read + // as printed, a non-zero exit, output that is not UTF-8, a silent `uname` + // and one that never answers read as nothing, an answer within the 5 s + // limit is read, and a failed read leaves no `probe_failures` entry. Each + // case plants a fake `uname` first on `PATH` and asks both entry points in + // a child copy of this test binary, because the lookup of `uname` follows + // the process's own `PATH`, which a test thread cannot change without + // disturbing every other test. The child's entry point, + // [`kernel_pair_child_reports_both_answers`], does nothing in a normal run. + // Linux only, because `probe_os` reads `uname -r` only when + // `runtime_is_linux()`, and the fake is a shell script; other Unix hosts are + // not supported platforms. + + #[cfg(target_os = "linux")] + mod kernel_pair { + use super::*; + use std::process::Command; + use std::time::Instant; + + const CHILD_ENV: &str = "ROCM_CORE_TEST_KERNEL_PAIR"; + const ANSWER_PREFIX: &str = "kernel-pair-answers:"; + + /// What each stack read for `uname -r`. + #[derive(Debug, PartialEq, Eq)] + struct Pair { + /// `detect_kernel_version`, the summary's `kernel`. + summary: Option, + /// `probe_os`'s `kernel_release`. + examination: String, + } + + fn pair(summary: Option<&str>, examination: &str) -> Pair { + Pair { + summary: summary.map(str::to_owned), + examination: examination.to_owned(), + } + } + + /// The child's side of [`ask`]: print both answers as JSON. + #[test] + fn kernel_pair_child_reports_both_answers() { + let Some(mode) = std::env::var_os(CHILD_ENV) else { + return; + }; + if mode == "inspect" { + // One inspection through the entry point `rocm examine --json` uses. + let (e, summary) = + Examination::probe_with_summary(crate::FrameworkProbe::Skip, None) + .expect("probe and gather"); + let answers = serde_json::json!({ + "summary": summary.kernel, + "examination": e.kernel_release, + "probe_failures": e.probe_failures, + }); + println!("{ANSWER_PREFIX}{answers}"); + return; + } + let mut e = Examination::default(); + probe_os(&mut e); + let answers = serde_json::json!({ + "summary": crate::host_gpu::detect_kernel_version(), + "examination": e.kernel_release, + "probe_failures": e.probe_failures, + }); + println!("{ANSWER_PREFIX}{answers}"); + } + + /// Plant a `uname` whose `-r` output is `body` (a shell fragment) and ask + /// both stacks. `uname -v` answers a fixed string, so `probe_os` has + /// nothing else to wait for. + fn ask(body: &str) -> Pair { + let (json, _release_reads) = run_child(body, "pair"); + // A failed read is empty, not a failure: `probe_os` records no + // `probe_failures` entry at all in any case (whatever its wording + // would be), so a host with a missing or broken `uname` does not read + // as degraded. The fake host's os-release reads cleanly, so the list + // is empty rather than compared with a baseline. + assert_eq!(json["probe_failures"], serde_json::json!([]), "{json}"); + Pair { + summary: json["summary"].as_str().map(str::to_owned), + examination: json["examination"] + .as_str() + .expect("the examination answer is a string") + .to_owned(), + } + } + + /// Run the child in `mode` against a fake `uname` whose `-r` output is + /// `body`, and return its answers and how many times `-r` was asked. + /// `uname -v` answers a fixed string, so `probe_os` has nothing else to + /// wait for. + fn run_child(body: &str, mode: &str) -> (serde_json::Value, usize) { + let scratch = tempfile::tempdir().expect("scratch directory"); + let bin = scratch.path().join("bin"); + std::fs::create_dir_all(&bin).expect("create the fake bin directory"); + let calls = scratch.path().join("release-reads"); + crate::fake_executable::write_fake_executable( + &bin.join("uname"), + format!( + "#!/bin/sh\nif [ \"$1\" = -v ]; then echo '#1 SMP fake'; exit 0; fi\necho x >> '{}'\n{body}\n", + calls.display() + ), + ) + .expect("plant the fake uname"); + let this_test = module_path!() + .split_once("::") + .expect("module path has a crate prefix") + .1; + let output = Command::new(std::env::current_exe().expect("path of the test binary")) + .args([ + "--exact", + &format!("{this_test}::kernel_pair_child_reports_both_answers"), + "--nocapture", + "--test-threads=1", + ]) + .env(CHILD_ENV, mode) + // The fake first; the system directories so `sleep` and `printf` + // resolve inside it. + .env("PATH", format!("{}:/usr/bin:/bin", bin.display())) + .output() + .expect("start the test binary again"); + let stdout = String::from_utf8_lossy(&output.stdout); + let line = stdout + .lines() + .find_map(|line| line.split_once(ANSWER_PREFIX).map(|(_, json)| json)) + .unwrap_or_else(|| { + panic!( + "the child printed no answers (status {}):\n{stdout}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ) + }); + let json: serde_json::Value = serde_json::from_str(line).expect("answers are JSON"); + let reads = std::fs::read_to_string(&calls).map_or(0, |log| log.lines().count()); + (json, reads) + } + + /// One inspection reads the release once, for the examination and the + /// summary together, so the two fields agree and a hung `uname` costs + /// one timeout. The reads are counted by the fake `uname` itself, so a + /// second read shows even when both return the same string. A failed + /// read is shared too: both are empty and it is still one read. + #[test] + fn one_inspection_reads_the_kernel_release_once() { + let (json, reads) = run_child("echo 6.8.0-fake", "inspect"); + assert_eq!(json["summary"], "6.8.0-fake", "{json}"); + assert_eq!(json["examination"], "6.8.0-fake", "{json}"); + assert_eq!(reads, 1, "{json}"); + + let (json, reads) = run_child("echo 6.8.0-partial; exit 3", "inspect"); + assert_eq!(json["summary"], serde_json::Value::Null, "{json}"); + assert_eq!(json["examination"], "", "{json}"); + assert_eq!(reads, 1, "{json}"); + } + + #[test] + fn both_read_the_release_a_working_uname_prints() { + assert_eq!( + ask("echo 6.8.0-fake"), + pair(Some("6.8.0-fake"), "6.8.0-fake") + ); + } + + /// A WSL 1 and a WSL 2 style release are read as printed by both, so + /// `is_wsl1_kernel` (which reads `kernel_release`) sees what the summary + /// shows. The consumer's answer is pinned in `wsl.rs`. + #[test] + fn both_read_wsl_style_releases_as_printed() { + for release in [ + "4.4.0-19041-Microsoft", + "5.15.167.4-microsoft-standard-WSL2", + ] { + assert_eq!( + ask(&format!("echo {release}")), + pair(Some(release), release), + "{release}" + ); + } + } + + /// A `uname` that prints a release and exits non-zero is dropped by + /// both. The examination used to keep the release. + #[test] + fn a_non_zero_exit_with_output_is_dropped_by_both() { + assert_eq!(ask("echo 6.8.0-partial; exit 3"), pair(None, "")); + } + + /// Both read a successful but silent `uname` as nothing: `None` in the + /// summary (shown as ``), empty in the examination, with no + /// `probe_failures` entry in either case. + #[test] + fn a_silent_uname_reads_as_nothing_in_both() { + assert_eq!(ask("exit 0"), pair(None, "")); + } + + /// Output that is not UTF-8 is dropped by both. The examination used to + /// keep it with the bad byte replaced. + #[test] + fn output_that_is_not_utf8_is_dropped_by_both() { + assert_eq!(ask("printf '6.8.0-\\377\\n'"), pair(None, "")); + } + + /// A `uname` that answers after 2 s is read by both: the one timeout is + /// the longer of the two old ones, 5 s. The summary used to give up at + /// 1.5 s, so 2 s is past that old limit and leaves 3 s of margin to the + /// new one on a loaded runner. + #[test] + fn an_answer_after_two_seconds_is_read_by_both() { + let started = Instant::now(); + assert_eq!( + ask("sleep 2; echo 6.8.0-slow"), + pair(Some("6.8.0-slow"), "6.8.0-slow") + ); + assert!(started.elapsed() >= std::time::Duration::from_secs(2)); + } + + /// A `uname` that never answers is read as nothing by both, each after + /// the one timeout. `exec` leaves no grandchild holding the pipe. + /// + /// The stand-in would exit successfully with nothing after 30 s, which is + /// also what a read with no timeout would turn into `None`, so the + /// answers alone cannot tell a timeout from no timeout. The elapsed time + /// can: the child reads twice in a row, each bounded by the 5 s limit, so + /// it takes about 10 s, and at least 9 s shows the stand-in really hung. + /// With the limit raised past the stand-in's 30 s it takes 60 s and the + /// upper bound fails. + #[test] + fn a_uname_that_never_answers_reads_as_nothing_in_both_within_the_timeout() { + let started = Instant::now(); + assert_eq!(ask("exec sleep 30"), pair(None, "")); + let elapsed = started.elapsed(); + assert!( + elapsed >= std::time::Duration::from_secs(9), + "two reads of a hung uname returned after {elapsed:?}" + ); + assert!( + elapsed < std::time::Duration::from_secs(25), + "the reads outlived the timeout: {elapsed:?}" + ); + } + } } diff --git a/crates/rocm-core/src/examine/library_copies.rs b/crates/rocm-core/src/examine/library_copies.rs index 090c71dfc5..ccbf0ec8b5 100644 --- a/crates/rocm-core/src/examine/library_copies.rs +++ b/crates/rocm-core/src/examine/library_copies.rs @@ -1265,10 +1265,7 @@ mod tests { assert!(rocm_install_siblings(&missing).is_empty()); } - /// Serializes the tests that point `ROCM_CLI_DATA_DIR` at a scratch - /// registry. Named `*_TEST_LOCK` so the env-mutation contract guard - /// recognises the discipline by suffix. - static DATA_DIR_ENV_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + use crate::test_env::DATA_DIR_ENV_TEST_LOCK; /// A wheel-format managed runtime's recorded library directories reach the /// search through the real chain, even though none of them sits under the diff --git a/crates/rocm-core/src/examine/linux_driver.rs b/crates/rocm-core/src/examine/linux_driver.rs index 06458b1ebf..8ae63ef834 100644 --- a/crates/rocm-core/src/examine/linux_driver.rs +++ b/crates/rocm-core/src/examine/linux_driver.rs @@ -9,22 +9,93 @@ use super::{Device, Examination}; use crate::probe_exec::{SHORT, read_host_text, read_text, run, which}; use std::time::Duration; -pub(super) fn probe_modules(e: &mut Examination) { +/// Whether the kernel's kfd is present on this host, however it got there. +/// +/// On current kernels kfd is not a module of its own: `HSA_AMD` is a `bool` +/// Kconfig option and amdkfd is linked into `amdgpu`, so no `amdkfd` line +/// appears in the module list, built in or not. (Before 4.20 it was a separate +/// `amdkfd` module, which the module-list check still covers.) The kernel's own +/// interfaces are the signal: the `/sys/class/kfd/kfd` class device, or the +/// `/dev/kfd` node. +fn kfd_present_on_host() -> bool { + crate::host_path("/sys/class/kfd/kfd").exists() || crate::host_path("/dev/kfd").exists() +} + +/// Whether the amdgpu driver is present on this host, loaded or built in. +/// +/// A built-in driver is absent from the module list. `/sys/module/amdgpu` is +/// no answer: the kernel creates it for a built-in driver at an early initcall, +/// before `amdgpu_init` can bail out (`nomodeset`, `amdgpu.modeset=0`). The PCI +/// driver directory exists only once `amdgpu_init` registered the driver. +fn amdgpu_present_on_host() -> bool { + crate::host_path("/sys/bus/pci/drivers/amdgpu").exists() +} + +/// What the host says about `amdgpu_loaded`: the amdgpu PCI driver is +/// registered, or kfd is present. kfd is linked into amdgpu, so kfd on the +/// host proves amdgpu too: a container that passes `/dev/kfd` through without +/// `/sys/bus/pci` must not report amdkfd up and amdgpu down. +/// +/// Neither signal means a GPU is bound. `amdgpu_init` creates kfd's device +/// before it calls `pci_register_driver`, and the PCI driver directory exists +/// once registration succeeds, before any device is probed. Both prove the +/// driver is present, which is what `amdgpu_loaded` reports. +fn amdgpu_implied_by_host() -> bool { + amdgpu_present_on_host() || kfd_present_on_host() +} + +/// Whether `name` is loaded or built in: listed in the module list, or present +/// on the host. +/// +/// `modules` is `None` when the module list could not be read; the host +/// answering `true` still answers `true` then, and nothing else is known. The +/// host probe runs only when the module list has not already answered. +fn module_or_host_state( + modules: Option<&[&str]>, + name: &str, + on_host: impl FnOnce() -> bool, +) -> Option { + match modules { + Some(modules) => Some(modules.contains(&name) || on_host()), + None => on_host().then_some(true), + } +} + +/// `amdgpu_loaded` and `amdkfd_loaded`, in that order, from the module list +/// and the host. kfd on the host answers both; the amdgpu PCI driver answers +/// only `amdgpu_loaded`. +fn driver_states(modules: Option<&[&str]>) -> (Option, Option) { + ( + module_or_host_state(modules, "amdgpu", amdgpu_implied_by_host), + module_or_host_state(modules, "amdkfd", kfd_present_on_host), + ) +} + +/// The kernel's module list, one module per line with its name first: `lsmod` +/// without its header, else `/proc/modules`. `None` when neither can be read. +fn read_module_list() -> Option { let (rc, out, _) = run("lsmod", &[], SHORT); - let module_text = if rc == 0 { + if rc == 0 { Some(out.lines().skip(1).collect::>().join("\n")) } else { let txt = read_host_text("/proc/modules"); if txt.is_empty() { None } else { Some(txt) } - }; - if let Some(text) = module_text { - let modules: Vec<&str> = text - .lines() - .filter_map(|line| line.split_whitespace().next()) - .collect(); - e.amdgpu_loaded = Some(modules.contains(&"amdgpu")); - e.amdkfd_loaded = Some(modules.contains(&"amdkfd")); } +} + +pub(super) fn probe_modules(e: &mut Examination) { + probe_modules_from(e, read_module_list().as_deref()); +} + +/// `probe_modules` given the module list, so a test can hand one in instead +/// of reading the machine's own. +fn probe_modules_from(e: &mut Examination, module_text: Option<&str>) { + let modules: Option> = module_text.map(|text| { + text.lines() + .filter_map(|line| line.split_whitespace().next()) + .collect() + }); + (e.amdgpu_loaded, e.amdkfd_loaded) = driver_states(modules.as_deref()); for dir in ["/etc/modprobe.d", "/usr/lib/modprobe.d", "/run/modprobe.d"] { let Ok(entries) = std::fs::read_dir(crate::host_path(dir)) else { @@ -183,4 +254,351 @@ mod tests { let (r, w) = mode_access("crw-rw----", "root", "render", "alice", &[]); assert_eq!((r, w), (Some(false), Some(false))); } + + #[test] + fn a_module_is_present_when_listed() { + assert_eq!( + module_or_host_state(Some(&["amdgpu", "amdkfd"]), "amdkfd", || false), + Some(true) + ); + assert_eq!( + module_or_host_state(Some(&["amdgpu", "drm"]), "amdgpu", || false), + Some(true) + ); + } + + /// Built into the kernel, a driver never appears in the module list. + #[test] + fn a_module_is_present_when_built_in_and_absent_from_the_module_list() { + assert_eq!( + module_or_host_state(Some(&["drm"]), "amdkfd", || true), + Some(true) + ); + assert_eq!( + module_or_host_state(Some(&[]), "amdgpu", || true), + Some(true) + ); + } + + #[test] + fn a_module_is_absent_with_neither_a_listing_nor_the_host() { + assert_eq!( + module_or_host_state(Some(&["drm"]), "amdkfd", || false), + Some(false) + ); + assert_eq!( + module_or_host_state(Some(&[]), "amdgpu", || false), + Some(false) + ); + } + + /// With no module list, the host still answers; without it the field + /// stays unknown rather than claiming `false`. + #[test] + fn an_unreadable_module_list_follows_the_host() { + assert_eq!(module_or_host_state(None, "amdgpu", || true), Some(true)); + assert_eq!(module_or_host_state(None, "amdgpu", || false), None); + } + + /// The host probe is skipped once the module list has answered. + #[test] + fn the_host_probe_is_skipped_when_the_module_list_already_answers() { + assert_eq!( + module_or_host_state(Some(&["amdgpu"]), "amdgpu", || unreachable!()), + Some(true) + ); + } + + /// The host side of the check, against a planted root. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn amdgpu_present_on_host_reads_the_pci_driver_dir() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-amdgpu-present-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, module_dir_only, planted) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = amdgpu_present_on_host(); + // The module directory a built-in driver keeps even when it never + // bound: not evidence. + std::fs::create_dir_all(root.join("sys/module/amdgpu")).expect("plant sysfs"); + let module_dir_only = amdgpu_present_on_host(); + std::fs::create_dir_all(root.join("sys/bus/pci/drivers/amdgpu")).expect("plant sysfs"); + (neither, module_dir_only, amdgpu_present_on_host()) + }; + std::fs::remove_dir_all(&root).ok(); + assert!(!neither, "an empty root has no amdgpu"); + assert!( + !module_dir_only, + "/sys/module/amdgpu alone is not a bound driver" + ); + assert!(planted, "/sys/bus/pci/drivers/amdgpu counts"); + } + + /// A module list, as `read_module_list` returns it, naming neither + /// `amdgpu` nor `amdkfd`. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + const NO_AMD_MODULES: &str = "snd_hda_intel 61440 3 - Live 0x0000000000000000\n\ + nvme 61440 2 - Live 0x0000000000000000"; + + /// The module list alone, against an empty host root: on a machine where + /// the drivers really are modules, each field must come from its own name + /// in the list. Both named answers both fields; `amdgpu` alone answers + /// only `amdgpu_loaded`, and `amdkfd` alone only `amdkfd_loaded`, so a + /// misspelt or swapped name in `driver_states` fails here. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn probe_modules_reports_each_driver_its_module_list_names() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = tempfile::tempdir().expect("create host root"); + let probe = |module_text: &str| { + let mut e = Examination::default(); + probe_modules_from(&mut e, Some(module_text)); + (e.amdgpu_loaded, e.amdkfd_loaded) + }; + let (both, amdgpu_only, amdkfd_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, root.path()); + ( + probe( + "amdgpu 19378176 12 - Live 0x0000000000000000\n\ + amdkfd 380928 1 amdgpu, Live 0x0000000000000000\n\ + nvme 61440 2 - Live 0x0000000000000000", + ), + probe( + "snd_hda_intel 61440 3 - Live 0x0000000000000000\n\ + amdgpu 19378176 12 - Live 0x0000000000000000", + ), + probe("amdkfd 380928 0 - Live 0x0000000000000000"), + ) + }; + assert_eq!(both, (Some(true), Some(true)), "both drivers listed"); + assert_eq!(amdgpu_only, (Some(true), Some(false)), "only amdgpu listed"); + assert_eq!(amdkfd_only, (Some(false), Some(true)), "only amdkfd listed"); + } + + /// The wiring, not just the helpers: `probe_modules_from` must consult the + /// host root. The module list it is handed names neither driver, so + /// `amdgpu_loaded` can only come from the planted + /// `/sys/bus/pci/drivers/amdgpu`, and that directory must not make + /// `amdkfd_loaded` true. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn probe_modules_reports_amdgpu_when_it_is_on_the_host() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-probe-amdgpu-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(root.join("sys/bus/pci/drivers/amdgpu")).expect("plant sysfs"); + let (amdgpu_loaded, amdkfd_loaded) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let mut e = Examination::default(); + probe_modules_from(&mut e, Some(NO_AMD_MODULES)); + (e.amdgpu_loaded, e.amdkfd_loaded) + }; + std::fs::remove_dir_all(&root).ok(); + assert_eq!(amdgpu_loaded, Some(true)); + assert_eq!( + amdkfd_loaded, + Some(false), + "the amdgpu PCI driver does not imply kfd" + ); + } + + /// The host side of the check, against a planted root: the sysfs class + /// device alone, the device node alone, and neither. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn kfd_present_on_host_reads_sysfs_and_the_device_node() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-kfd-present-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, sysfs_only, dev_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = kfd_present_on_host(); + std::fs::create_dir_all(root.join("sys/class/kfd/kfd")).expect("plant sysfs"); + let sysfs_only = kfd_present_on_host(); + std::fs::remove_dir_all(root.join("sys")).expect("unplant sysfs"); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + (neither, sysfs_only, kfd_present_on_host()) + }; + std::fs::remove_dir_all(&root).ok(); + assert!(!neither, "an empty root has no kfd"); + assert!(sysfs_only, "/sys/class/kfd/kfd alone counts"); + assert!(dev_only, "/dev/kfd alone counts"); + } + + /// kfd on the host alone answers `amdgpu_loaded`: no amdgpu PCI driver + /// directory, only a kfd signal, either one. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn kfd_alone_on_the_host_implies_amdgpu() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-kfd-implies-amdgpu-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, sysfs_only, dev_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = amdgpu_implied_by_host(); + std::fs::create_dir_all(root.join("sys/class/kfd/kfd")).expect("plant sysfs"); + let sysfs_only = amdgpu_implied_by_host(); + std::fs::remove_dir_all(root.join("sys")).expect("unplant sysfs"); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + (neither, sysfs_only, amdgpu_implied_by_host()) + }; + std::fs::remove_dir_all(&root).ok(); + assert!(!neither, "an empty root implies no amdgpu"); + assert!(sysfs_only, "/sys/class/kfd/kfd alone implies amdgpu"); + assert!(dev_only, "/dev/kfd alone implies amdgpu"); + } + + /// Which host signal answers which field. Each case plants one signal into + /// an otherwise empty root and hands over the module list directly, so the + /// answer cannot come from the machine the test runs on: kfd alone answers + /// both fields, the amdgpu PCI driver alone answers only `amdgpu_loaded`, + /// and with the module list unreadable the host still answers. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn driver_states_answer_each_field_from_its_own_host_signal() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-driver-states-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, unread_neither, kfd_only, unread_kfd_only, pci_only, unread_pci_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = driver_states(Some(&[])); + let unread_neither = driver_states(None); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + let kfd_only = driver_states(Some(&[])); + let unread_kfd_only = driver_states(None); + std::fs::remove_dir_all(root.join("dev")).expect("unplant dev"); + std::fs::create_dir_all(root.join("sys/bus/pci/drivers/amdgpu")).expect("plant sysfs"); + ( + neither, + unread_neither, + kfd_only, + unread_kfd_only, + driver_states(Some(&[])), + driver_states(None), + ) + }; + std::fs::remove_dir_all(&root).ok(); + assert_eq!( + neither, + (Some(false), Some(false)), + "nothing listed, nothing on the host" + ); + assert_eq!( + unread_neither, + (None, None), + "nothing read, nothing on the host" + ); + assert_eq!(kfd_only, (Some(true), Some(true)), "kfd implies amdgpu"); + assert_eq!( + unread_kfd_only, + (Some(true), Some(true)), + "kfd answers without a module list" + ); + assert_eq!( + pci_only, + (Some(true), Some(false)), + "amdgpu does not imply kfd" + ); + assert_eq!( + unread_pci_only, + (Some(true), None), + "amdgpu does not imply kfd" + ); + } + + /// The wiring, not just the helpers: `probe_modules_from` must consult the + /// host root. The module list it is handed names neither driver, so + /// `Some(true)` can only come from the planted `/dev/kfd`, which answers + /// `amdgpu_loaded` too. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn probe_modules_reports_amdkfd_when_kfd_is_on_the_host() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-probe-kfd-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + let (amdgpu_loaded, amdkfd_loaded) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let mut e = Examination::default(); + probe_modules_from(&mut e, Some(NO_AMD_MODULES)); + (e.amdgpu_loaded, e.amdkfd_loaded) + }; + std::fs::remove_dir_all(&root).ok(); + assert_eq!(amdkfd_loaded, Some(true)); + assert_eq!(amdgpu_loaded, Some(true), "kfd implies amdgpu"); + } } diff --git a/crates/rocm-core/src/examine/rocm_install_probe.rs b/crates/rocm-core/src/examine/rocm_install_probe.rs index 03bdda9535..13e2fca205 100644 --- a/crates/rocm-core/src/examine/rocm_install_probe.rs +++ b/crates/rocm-core/src/examine/rocm_install_probe.rs @@ -198,6 +198,11 @@ mod tests { .expect("write quarantined repo"); std::fs::write(dir.path().join("unrelated.list"), "deb ...\n").expect("write unrelated"); + // The scan reads through `host_path`; hold the lock so another test's + // host root cannot re-root it. + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let mut seen = live_rocm_repo_paths_in_dir(&dir.path().to_string_lossy()); seen.sort(); @@ -235,6 +240,11 @@ mod tests { yum_dir.path().to_string_lossy().into_owned(), zypp_dir.path().to_string_lossy().into_owned(), ]; + // The scan reads through `host_path`; hold the lock so another test's + // host root cannot re-root it. + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let dir_refs: Vec<&str> = dirs.iter().map(String::as_str).collect(); merge_live_repo_paths(&mut e, &dir_refs); @@ -311,6 +321,11 @@ mod tests { .to_string_lossy() .into_owned(), ); + // The scan reads through `host_path`; hold the lock so another test's + // host root cannot re-root it. + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); merge_live_repo_paths(&mut e, &[dir_str.as_str()]); assert_eq!( diff --git a/crates/rocm-core/src/examine/wsl.rs b/crates/rocm-core/src/examine/wsl.rs index 28981417ea..75a52ab4a1 100644 --- a/crates/rocm-core/src/examine/wsl.rs +++ b/crates/rocm-core/src/examine/wsl.rs @@ -515,6 +515,32 @@ mod tests { ); } + /// The consumer of `kernel_release` for the WSL facts. The kernel read is a + /// pair of the host-probing note ("Kernel version"); this pins what + /// `probe_wsl` makes of each string the two reads can hand it, including + /// the empty string a failed read leaves: a WSL 1 style release is version + /// 1, a WSL 2 style one and an empty one are version 2. + #[test] + fn probe_wsl_reads_the_wsl_version_from_the_kernel_release() { + for (release, version) in [ + ("4.4.0-19041-Microsoft", 1), + ("5.15.167.4-microsoft-standard-WSL2", 2), + ("4.19.104-microsoft-standard", 2), + ("", 2), + ] { + let mut e = Examination { + kernel_release: release.to_owned(), + ..Examination::default() + }; + probe_wsl(&mut e); + assert_eq!( + e.wsl.as_ref().map(|wsl| wsl.version), + Some(version), + "{release:?}" + ); + } + } + #[test] fn the_distro_list_decodes_as_utf16_whatever_script_it_is_in() { fn utf16le(text: &str, bom: bool) -> Vec { diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index daa59a991a..3a6977626d 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -39,7 +39,9 @@ use render::{ }; use std::io::Write; -pub(crate) use catalog::{COMGR_CONFLICT_NEITHER_OPTION_RECOMMENDED, is_catalog_id}; +pub(crate) use catalog::{ + COMGR_CONFLICT_NEITHER_OPTION_RECOMMENDED, RENDER_GROUP_VERIFY, is_catalog_id, +}; pub use catalog::{class_on, needs_on}; pub use manifest::{ CATALOG_CONTRACT_VERSION, CatalogManifest, ManifestEntry, ManifestPlatform, catalog_manifest, @@ -64,7 +66,7 @@ pub(crate) use catalog::{ShellRun, run_typed_in_cmd_and_powershell}; #[must_use] pub fn apply(fix_id: &str, opts: &FixOptions) -> i32 { let Some(recipe) = find_recipe(fix_id) else { - fail!("Unknown fix-id: {fix_id}"); + fail!("{} {fix_id}", exit::UNKNOWN_ID_PREFIX); if looks_like_a_diagnosis_position(fix_id) { // `rocm diagnose` ranks findings `#1`, `#2`, and users reach for that // number here. It is a position in one report, not a name -- and it @@ -281,6 +283,25 @@ mod tests { ); } + /// The published `internal` description: a recipe the catalog says the CLI + /// carries out here, with no runner to carry it out, is the CLI's own fault + /// and exits with the code published as `internal`, not `failed`. + /// `a_recipe_that_acts_has_a_runner` keeps the real catalog off this path; + /// this drives the path itself with a synthetic recipe. + #[test] + fn a_recipe_the_catalog_says_is_applied_with_no_runner_exits_internal() { + // `Auto` is the class that says the CLI applies the fix itself + // (`FixClass::applies_itself`); a runner-less `NeedsArgument` entry only + // prints steps, so it is not this path. + let recipe = unfixable_recipe(FixClass::Auto); + assert_eq!(plan_of(&recipe), Plan::MissingRunner); + assert_eq!( + act_on(&recipe, &FixOptions::default()), + exit::INTERNAL, + "an AUTO entry with no runner must exit with the code published as internal" + ); + } + /// A Linux verify line is only worth having on a recipe that applies on /// Linux or WSL; on any other it could never be run. #[test] diff --git a/crates/rocm-core/src/fix/catalog.rs b/crates/rocm-core/src/fix/catalog.rs index 9d392f907b..086a5e58a2 100644 --- a/crates/rocm-core/src/fix/catalog.rs +++ b/crates/rocm-core/src/fix/catalog.rs @@ -23,6 +23,23 @@ use super::runners; /// apart in wording before they shared this constant. pub(crate) const COMGR_CONFLICT_NEITHER_OPTION_RECOMMENDED: &str = "Neither option is recommended over the other: which is right depends on which stack you mean to keep, and removing the wrong one breaks a working environment."; +/// fix-4's verify line, shared verbatim by this recipe and the `Fix` that +/// `diagnose/linux.rs`'s `check_4_render_group` builds, so `rocm fix` and +/// `rocm diagnose` cannot tell the user to check two different things. The two +/// copies had drifted: only the diagnosis listed `/dev/kfd`. +/// +/// What its exit status checks: that the user is in the group that owns +/// `/dev/kfd` (as `stat -c %G` names it), or in `render` or `video`, and that +/// `/dev/kfd` exists. The owning group is in the pattern because the diagnosis +/// tells the user to join that group, which need not be `render`. When `stat` +/// fails or `/dev/kfd` is missing, that alternative is empty and matches no +/// line `groups` prints, and `ls` then fails. An owner shown as `UNKNOWN` (a +/// GID with no name) adds only a literal `UNKNOWN`. It checks membership, not +/// the device's mode, and it does not check `rocminfo`: `rocminfo | head` +/// reports `head`'s status. `ls -l /dev/kfd` prints the owning group and mode +/// for the user to read. The `\$` keeps the anchor literal in sh, bash and fish. +pub(crate) const RENDER_GROUP_VERIFY: &str = r#"groups | tr ' ' '\n' | grep -E "^($(stat -c %G /dev/kfd 2>/dev/null)|render|video)\$" && ls -l /dev/kfd && rocminfo | head -n 5"#; + /// One operating system a recipe applies on, and what it does there. /// /// Scope and class are held together rather than in two parallel lists because @@ -162,7 +179,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ rationale: "Your GPU's gfx target is not in the framework wheel's compiled kernel list. Re-install the framework from an index that includes this gfx, OR rebuild llama.cpp with AMDGPU_TARGETS=.", commands: &[ "# PyTorch (Linux): a nightly often carries kernels a release has not shipped yet.", - "# Pick the nightly for the ROCm major you have, not an older one.", + "# Pick the nightly for the ROCm X.Y release you have, not an older one.", "pip uninstall -y torch torchvision torchaudio", "pip install --pre torch torchvision torchaudio \\", " --index-url https://download.pytorch.org/whl/nightly/rocm7.14", @@ -248,7 +265,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ needs_sudo: true, needs_reboot: false, needs_relogin: true, - verify: "groups | tr ' ' '\\n' | grep -E '^(render|video)$' && rocminfo | head -n 5", + verify: RENDER_GROUP_VERIFY, verify_linux: None, notes: &[], doc_url: None, @@ -342,15 +359,15 @@ pub(super) const RECIPES: &[FixRecipe] = &[ }, FixRecipe { fix_id: "fix-8-wheel-rocm", - title: "Reinstall the framework against the system ROCm/HIP major", + title: "Reinstall the framework against the system ROCm/HIP X.Y release", rationale: "The framework's bundled HIP version doesn't match the system ROCm (Linux) or HIP SDK (Windows). libamdhip64.so.X / amdhip64_X.dll load failures are the usual signal.", commands: &[ "pip uninstall -y torch torchvision torchaudio", - "# Linux: install the index for the ROCm major `rocm examine` reports:", + "# Linux: install the index for the ROCm X.Y release `rocm examine` reports:", "pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14", "# ROCm 10 has no released PyTorch index yet; it is on the nightly channel:", "pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0", - "# Windows: use TheRock's wheels matching your HIP SDK major:", + "# Windows: use TheRock's wheels matching your HIP SDK X.Y release:", "# https://github.com/ROCm/TheRock", ], needs_sudo: false, @@ -483,7 +500,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ title: "Install the AMD HIP SDK for Windows", rationale: "Your framework links against HIP but the HIP SDK isn't installed on this host. The runtime DLLs (amdhip64_X.dll, hipblas.dll, hsa-runtime64.dll) and hipInfo.exe ship inside the SDK installer.", commands: &[ - "# Download and install the HIP SDK (matched to your framework's HIP major):", + "# Download and install the HIP SDK (matched to your framework's HIP X.Y release):", "# https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html", "# After install, reopen the shell so HIP_PATH and PATH pick up the new install.", ], diff --git a/crates/rocm-core/src/fix/consent.rs b/crates/rocm-core/src/fix/consent.rs index ee3b0afbc3..95b9d8931f 100644 --- a/crates/rocm-core/src/fix/consent.rs +++ b/crates/rocm-core/src/fix/consent.rs @@ -75,7 +75,9 @@ pub(super) fn confirm(prompt: &str, assume_yes: bool) -> bool { return verdict; } let _ = crate::confirm::print_prompt(&mut std::io::stdout(), &format!("{prompt} [y/N]: ")); - let confirmed = crate::confirm::read_answer(&mut std::io::stdin().lock()).unwrap_or(false); + let confirmed = + crate::confirm::read_answer(&mut std::io::stdin().lock(), &mut std::io::stdout()) + .unwrap_or(false); if !confirmed { fail!("Not confirmed; refusing to apply."); } diff --git a/crates/rocm-core/src/fix/manifest.rs b/crates/rocm-core/src/fix/manifest.rs index 6bc3f5bc07..c4266199db 100644 --- a/crates/rocm-core/src/fix/manifest.rs +++ b/crates/rocm-core/src/fix/manifest.rs @@ -55,8 +55,17 @@ pub struct ManifestEntry { pub struct CatalogManifest { pub contract_version: u32, pub entries: Vec, - /// What each exit code from `rocm fix` means. + /// The number behind each exit-code name from `rocm fix`. pub exit_codes: std::collections::BTreeMap, + /// What each exit-code name in [`Self::exit_codes`] means, keyed by the same + /// names. A sibling map rather than a richer value in `exit_codes`, so a + /// reader written against name -> number keeps working: an added field, + /// which does not raise the contract version. + /// + /// `unknown_id` (2) says that a malformed command line exits 2 as well, and + /// how to tell the two refusals apart. + #[serde(default)] + pub exit_code_descriptions: std::collections::BTreeMap, } /// Describe the catalog for a machine to read. @@ -88,17 +97,14 @@ pub fn catalog_manifest() -> CatalogManifest { doc_url: r.doc_url.map(ToOwned::to_owned), }) .collect(), - exit_codes: [ - ("ok", exit::OK), - ("internal", exit::INTERNAL), - ("unknown_id", exit::UNKNOWN_ID), - ("not_applicable", exit::NOT_APPLICABLE), - ("failed", exit::FAILED), - ("declined", exit::DECLINED), - ] - .into_iter() - .map(|(name, code)| (name.to_owned(), code)) - .collect(), + exit_codes: exit::MEANINGS + .iter() + .map(|m| (m.name.to_owned(), m.code)) + .collect(), + exit_code_descriptions: exit::MEANINGS + .iter() + .map(|m| (m.name.to_owned(), m.description.to_owned())) + .collect(), } } @@ -156,6 +162,135 @@ mod tests { } } + /// Every published code carries a description, under the same name, and + /// the table they come from names each code once. A name with a number and + /// no meaning is the gap ROCMAI-955 reported; two names for one number would + /// leave a reader unable to say which it got. + #[test] + fn every_published_exit_code_is_described_once() { + let manifest = catalog_manifest(); + let named: Vec<&String> = manifest.exit_codes.keys().collect(); + let described: Vec<&String> = manifest.exit_code_descriptions.keys().collect(); + assert_eq!(named, described, "every exit code needs a description"); + for (name, text) in &manifest.exit_code_descriptions { + assert!(!text.trim().is_empty(), "`{name}` is described by nothing"); + } + let mut codes: Vec = exit::MEANINGS.iter().map(|m| m.code).collect(); + codes.sort_unstable(); + codes.dedup(); + assert_eq!( + codes.len(), + exit::MEANINGS.len(), + "two names publish the same exit code" + ); + assert_eq!( + manifest.exit_codes.len(), + exit::MEANINGS.len(), + "two entries share a name" + ); + } + + /// Exit 2 means two things, and the description has to say both and how to + /// tell them apart, or a caller that reads it still cannot. The message it + /// names is the one `apply` prints, held to the same constant; that the + /// argument parser really exits 2 is checked in `apps/rocm`, which owns it + /// (`a_usage_error_exits_with_the_code_published_for_an_unknown_fix_id`). + #[test] + fn the_unknown_id_description_names_the_usage_error_it_shares_a_code_with() { + let manifest = catalog_manifest(); + assert_eq!(manifest.exit_codes["unknown_id"], exit::UNKNOWN_ID); + let text = &manifest.exit_code_descriptions["unknown_id"]; + for needed in [ + exit::UNKNOWN_ID_PREFIX, + "argument parser", + "malformed command line", + "`error:`", + // A one-time `rocm:` notice can precede either refusal, so the + // promise is about a line, not about where stderr starts. + "stderr line", + ] { + assert!( + text.contains(needed), + "the unknown_id description must mention `{needed}`: {text}" + ); + } + } + + /// `MEANINGS` claims to list every code `rocm fix` returns. The runners + /// return the `exit::` constants (held by + /// `fix_runners_return_named_exit_constants_not_integers`), so the set of + /// constants is the set of codes, and this holds the table to it: a new + /// constant fails here until it is described and published. + #[test] + fn every_exit_constant_is_published_and_nothing_else_is() { + let file = syn::parse_file(include_str!("options.rs")).expect("options.rs parses"); + let exit_mod = file + .items + .iter() + .find_map(|item| match item { + syn::Item::Mod(m) if m.ident == "exit" => m.content.as_ref(), + _ => None, + }) + .expect("options.rs has an inline `mod exit`"); + let mut declared: Vec<(String, i32)> = exit_mod + .1 + .iter() + .filter_map(|item| match item { + syn::Item::Const(c) if matches!(&*c.ty, syn::Type::Path(p) if p.path.is_ident("i32")) => { + let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Int(value), + .. + }) = &*c.expr + else { + panic!("exit::{} is not an integer literal", c.ident); + }; + Some(( + c.ident.to_string(), + value.base10_parse().expect("an i32 literal"), + )) + } + _ => None, + }) + .collect(); + declared.sort_by_key(|(_, code)| *code); + let mut published: Vec = exit::MEANINGS.iter().map(|m| m.code).collect(); + published.sort_unstable(); + let declared_codes: Vec = declared.iter().map(|(_, code)| *code).collect(); + assert_eq!( + declared_codes, published, + "every exit:: constant must have exactly one MEANINGS entry, and no entry may \ + publish a code no constant names; constants: {declared:?}" + ); + // Name by name too: the published name is the constant's, lower-cased, + // so a reader of the manifest and a reader of the code use one word. + for (constant, code) in &declared { + let meaning = exit::MEANINGS + .iter() + .find(|m| m.code == *code) + .expect("checked above"); + assert_eq!( + meaning.name, + constant.to_lowercase(), + "exit::{constant} is published under another name" + ); + } + } + + /// A manifest written before the descriptions existed still reads: the + /// field was added, so the contract version did not move, and that is only + /// true if an older document parses. + #[test] + fn a_manifest_without_descriptions_still_parses() { + let mut value = serde_json::to_value(catalog_manifest()).expect("serializes"); + value + .as_object_mut() + .expect("an object") + .remove("exit_code_descriptions"); + let older: CatalogManifest = serde_json::from_value(value).expect("an older manifest"); + assert!(older.exit_code_descriptions.is_empty()); + assert_eq!(older.contract_version, CATALOG_CONTRACT_VERSION); + } + #[test] fn a_published_doc_url_is_a_url() { // Most entries carry none, which is the honest answer until someone diff --git a/crates/rocm-core/src/fix/options.rs b/crates/rocm-core/src/fix/options.rs index adf32a997f..a761f19214 100644 --- a/crates/rocm-core/src/fix/options.rs +++ b/crates/rocm-core/src/fix/options.rs @@ -20,20 +20,158 @@ use serde::{Deserialize, Serialize}; /// have to come from one place -- a manifest that restated them by hand would be /// the same defect this whole contract exists to remove, one level up. pub mod exit { - /// Applied, previewed with `--dry-run`, printed, or listed. Nothing failed. + // What each code means is published in [`MEANINGS`] and said only there; + // a second wording here would be a copy that can drift from the one a + // caller reads. + + /// `ok` in [`MEANINGS`]. pub const OK: i32 = 0; - /// The CLI is inconsistent with itself -- a recipe it says it can carry out, - /// with nothing to carry it out with. Never a fault of the machine. + /// `internal` in [`MEANINGS`]. pub const INTERNAL: i32 = 1; - /// No such fix-id. + /// `unknown_id` in [`MEANINGS`]. pub const UNKNOWN_ID: i32 = 2; - /// Real, but not for this machine: wrong platform, or a precondition the - /// host does not meet. Distinct from [`FAILED`], which means it was tried. + /// `not_applicable` in [`MEANINGS`]. pub const NOT_APPLICABLE: i32 = 3; - /// Attempted, and a command it ran failed. + /// `failed` in [`MEANINGS`]. pub const FAILED: i32 = 4; - /// The user was asked and said no. + /// `declined` in [`MEANINGS`]. pub const DECLINED: i32 = 5; + + /// How the stderr line refusing an unknown fix-id starts. + /// + /// Published inside the [`UNKNOWN_ID`] description, because it is the only + /// way a caller can tell that refusal from a usage error, which exits with + /// the same code (see [`MEANINGS`]). + pub const UNKNOWN_ID_PREFIX: &str = "Unknown fix-id:"; + + /// One exit code as the manifest publishes it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct Meaning { + /// The key a reader looks the code up by. + pub name: &'static str, + pub code: i32, + /// What a caller should conclude on receiving the code. + pub description: &'static str, + } + + /// Every code `rocm fix` returns, with its name and what it means. + /// + /// The single table both published maps (`exit_codes`, + /// `exit_code_descriptions`) are built from, so a code cannot gain a name + /// without a meaning or move without its description moving with it. + /// `every_exit_constant_is_published_and_nothing_else_is` (in + /// `manifest.rs`) holds this table to the constants above, so a new + /// constant fails until it is described here. + /// + /// Each description is a claim about what the CLI did, so the comment above + /// each entry names the tests that prove it (AGENTS.md section 3), and the + /// branches no test drives. + /// + /// Exit 2 is shared on purpose and documented rather than split: clap + /// returns 2 for a malformed command line, and giving a usage error a code + /// of its own would change what existing readers of exit 2 receive. + pub const MEANINGS: &[Meaning] = &[ + // Proved in `fix.rs` by + // `a_problem_with_no_reliable_fix_reports_success_and_does_not_look_portable` + // (diagnose-only) and `print_only_fix_returns_zero` (printed steps); in + // `fix/runners.rs` by `granting_consent_appends_the_export_and_returns_0` + // (applied), `an_rc_file_that_already_pins_a_device_is_left_alone` + // (nothing to change), `dry_run_returns_0_and_writes_nothing_even_with_consent` + // (preview) and `windows_path_read_failure_aborts_before_plan_or_write` + // (a preview that exits 4 instead); listing the catalog by diagnose-34 + // (`@id:diagnose-fix-catalog-is-machine-readable`). + Meaning { + name: "ok", + code: OK, + description: "Nothing failed: the fix was applied, found nothing to change \ + (already in place, or a setting it leaves for you to edit), was \ + previewed with `--dry-run`, or only printed what to do (steps to \ + run yourself, or the explanation of a diagnose-only entry); or \ + the catalog was listed. A `--dry-run` preview is not always 0: \ + it exits 3 or 4 when the fix cannot be planned.", + }, + // Proved in `fix.rs` by + // `a_recipe_the_catalog_says_is_applied_with_no_runner_exits_internal`, and + // for `--json` with a fix-id by diagnose-35 + // (`@id:diagnose-fix-catalog-json-rejects-a-fix-id`). + Meaning { + name: "internal", + code: INTERNAL, + description: "The CLI is inconsistent with itself (a recipe it says it can carry \ + out, with nothing to carry it out with); never a fault of the \ + machine. Exit 1 is also any other error the command reports on \ + stderr, such as `--json` combined with a fix-id.", + }, + // Proved by `the_unknown_id_description_names_the_usage_error_it_shares_a_code_with` + // (`manifest.rs`), `a_usage_error_exits_with_the_code_published_for_an_unknown_fix_id` + // (`apps/rocm/src/main.rs`) and diagnose-34, which runs the binary. + Meaning { + name: "unknown_id", + code: UNKNOWN_ID, + description: "No catalog entry has that fix-id; no fix step was run, and a \ + stderr line starts with `Unknown fix-id:`. Exit 2 is also what \ + the argument parser returns for a malformed command line (an \ + unknown option, a missing value, or a value it cannot read, such \ + as a non-number for `--device-index`, or a negative one written \ + after a space, `--device-index -1`, which it reads as an \ + unknown option), before any fix-id is looked up; that refusal \ + has a stderr line starting with `error:` and never says \ + `Unknown fix-id:`. Another line, such as a one-time `rocm:` \ + notice, can come first. The code alone cannot tell the two \ + apart; the message can.", + }, + // Proved in `fix.rs` by `the_class_gate_is_reached_only_after_the_platform_gate` + // and `the_published_exit_codes_are_the_ones_apply_returns` (another + // platform), and in `fix/runners.rs` by + // `a_negative_device_index_is_not_applicable_and_nothing_runs` (an argument + // it cannot use); that only `--device-index=-1` reaches the runner, and + // `--device-index -1` is a usage error, is pinned through the real parser + // by `a_negative_device_index_reaches_the_fix_only_when_written_with_equals` + // in `apps/rocm/src/main.rs`. The missing tool, install and home-directory + // branches are not driven by a test. A missing home directory returns 3 + // only from the fixes that write a file there (on Linux, fix-6, and fix-9 + // when given `--device-index`; without one fix-9 only prints the query and + // exits 0 before it looks); fix-2 on Linux only reports, and exits 0 + // without one. + Meaning { + name: "not_applicable", + code: NOT_APPLICABLE, + description: "The entry exists but cannot be used here: it is for another \ + platform, the host lacks something the fix cannot do without (a \ + tool it runs, or `sudo` when not root; an install it points at; \ + the current user's name; or a home directory to write a file \ + in), or an argument it was given is one it cannot \ + use, such as a negative device index written `--device-index=-1`. \ + Nothing was tried. Written with a space instead, `--device-index -1` \ + is a usage error and exits 2 (see `unknown_id`).", + }, + // Proved in `fix/runners.rs` by + // `a_failed_write_returns_4_rather_than_reporting_success` (a file write) + // and `windows_path_read_failure_aborts_before_plan_or_write` (a failed + // read, under `--dry-run`). A command that exits non-zero (`usermod`, + // `setx`, PowerShell) is not driven by a test: those runners start the + // real program. + Meaning { + name: "failed", + code: FAILED, + description: "A step of the fix failed: a command it ran, a file it wrote, or a \ + read of state it needed before planning (the current User PATH \ + on Windows, read even under `--dry-run`). The change that step \ + was to make was not made.", + }, + // Proved in `fix/runners.rs` by `declining_consent_returns_5_and_writes_nothing`, + // `windows_decline_on_user_scope_returns_5` and + // `windows_decline_with_machine_scope_also_set_still_returns_5`, and for a + // stdin that is not a terminal by + // `non_interactive_shell_refuses_instead_of_proceeding` in `fix/consent.rs`. + Meaning { + name: "declined", + code: DECLINED, + description: "The fix needed confirmation and did not get it: the user answered \ + no, or stdin is not a terminal and `--yes` was not given. Nothing \ + the confirmation guarded was changed.", + }, + ]; } /// Options controlling how a fix is applied. diff --git a/crates/rocm-core/src/fix/runners.rs b/crates/rocm-core/src/fix/runners.rs index 896091cc01..c6c84d2df1 100644 --- a/crates/rocm-core/src/fix/runners.rs +++ b/crates/rocm-core/src/fix/runners.rs @@ -995,6 +995,13 @@ mod tests { code, 5, "a failed read was never offered a prompt to decline" ); + // The published `failed` description names this case: a read of state + // the fix needed, failing even under `--dry-run`. + assert_eq!( + code, + exit::FAILED, + "a failed read must exit with the code published as `failed`" + ); } /// A genuinely empty User PATH (the read succeeded and returned `""`, @@ -1213,4 +1220,31 @@ mod tests { assert_eq!(code, 4, "a failed write must exit 4, not 0"); std::fs::remove_dir_all(&dir).ok(); } + + /// A negative `--device-index` written `--device-index=-1` parses (it is an + /// `i64`), so the argument parser does not reject it (written with a space, + /// `--device-index -1`, it does: that is a usage error, exit 2, pinned by + /// `a_negative_device_index_reaches_the_fix_only_when_written_with_equals` in + /// `apps/rocm/src/main.rs`), and the runner refuses it with the code + /// published as `not_applicable` before reading or changing anything. The + /// description says so; this is what holds it, on every host, since the + /// guard runs before the platform split. + /// + /// Driven under `--dry-run` without `--yes`, so that if the guard were ever + /// lost the runner would only preview, never write the user's environment + /// or rc file from a test. The guard sits before the dry-run check, so the + /// same code comes back with or without it. + #[test] + fn a_negative_device_index_is_not_applicable_and_nothing_runs() { + let opts = FixOptions { + yes: false, + dry_run: true, + device_index: Some(-1), + }; + assert_eq!( + run_hip_visible_devices(&opts), + exit::NOT_APPLICABLE, + "a negative device index must exit with the code published as not_applicable" + ); + } } diff --git a/crates/rocm-core/src/foreign_temp_probe.rs b/crates/rocm-core/src/foreign_temp_probe.rs new file mode 100644 index 0000000000..9d9fd4f520 --- /dev/null +++ b/crates/rocm-core/src/foreign_temp_probe.rs @@ -0,0 +1,290 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A test probe for the claim "this writer never touches another writer's +//! temp file", shared by this crate's tests and by the tests of the crates +//! that publish through [`crate::atomic_write`]. +//! +//! Compiled only for this crate's own tests and under the `test-support` +//! feature, which a dependent crate enables from its `[dev-dependencies]` +//! entry, so it is absent from every shipped build. + +use crate::unix_time_millis; +use std::path::PathBuf; + +const FOREIGN: &[u8] = b"another writer's staged bytes"; +const ATTEMPTS: usize = 3; +const INITIAL_WINDOW_MS: u128 = 400; +/// Ceiling for both the window and the lead before the probe gives up. +const MAX_MS: u128 = 16_000; + +/// What one run of the probe established. +#[derive(Debug, PartialEq, Eq)] +enum Outcome { + /// The write ran inside the window of occupied names and left every one + /// of them alone. + Spared, + /// The host was too slow for any window to cover the write, so a name may + /// not have been in its way and the run says nothing about the writer. + Inconclusive(String), +} + +/// Runs `write` while a foreign file occupies every name `foreign_name` +/// returns for each millisecond of a window, and asserts that none of them was +/// written through, truncated or moved. +/// +/// This is what tells a writer that reserves its own temp file +/// (`atomic_write`) from one that writes to a clock-and-pid name with +/// `fs::write`: the second opens the foreign file, truncates it and renames +/// it away, taking another writer's staged bytes with it. Every name in the +/// window is taken, so a writer that picks one of them hits a foreign file +/// whatever millisecond its write runs in. +/// +/// The window starts a "lead" after the names are first created, and `write` +/// only runs once it has begun, so creating the names can take as long as the +/// host needs (about a millisecond per file is common) without the clock +/// running past them. `write` runs once per attempt, so it must set up +/// whatever it needs itself, inside the window. When the creation outlasts +/// the lead or the write outlasts the window, the attempt is retried with a +/// longer lead or window. A host too slow for any of them fails the test with +/// an "inconclusive" panic, because a probe that quietly passes there is a +/// test that does not test. The one exception is Windows, where the cost of +/// creating files has not been measured: there the probe says so on stderr +/// and passes, so a run on a Windows host can pass without the probe having +/// proven anything. The directory the names live in must exist already. +pub fn assert_write_spares_foreign_temp_files( + foreign_name: impl Fn(u128) -> PathBuf, + write: impl FnMut() -> anyhow::Result<()>, +) -> anyhow::Result<()> { + let outcome = probe(foreign_name, write, INITIAL_WINDOW_MS, MAX_MS, ATTEMPTS)?; + require_conclusive(outcome, cfg!(windows)); + Ok(()) +} + +/// Fails the test for an inconclusive `outcome` unless `tolerate_inconclusive` +/// is set, in which case it only says so on stderr. A parameter rather than a +/// `cfg!` inside, so both branches run in a test on every platform. +fn require_conclusive(outcome: Outcome, tolerate_inconclusive: bool) { + if let Outcome::Inconclusive(why) = outcome { + if tolerate_inconclusive { + eprintln!("inconclusive (tolerated on Windows): {why}"); + } else { + panic!( + "the foreign-temp-file probe is inconclusive, so it cannot show the writer is \ + safe: {why}" + ); + } + } +} + +fn probe( + foreign_name: impl Fn(u128) -> PathBuf, + mut write: impl FnMut() -> anyhow::Result<()>, + initial_window_ms: u128, + max_ms: u128, + attempts: usize, +) -> anyhow::Result { + let mut window_ms = initial_window_ms; + let mut lead_ms = 2 * initial_window_ms; + for attempt in 1..=attempts { + let created_from = unix_time_millis(); + let window_start = created_from + lead_ms; + let window_end = window_start + window_ms; + let foreign: Vec = (window_start..window_end).map(&foreign_name).collect(); + for path in &foreign { + std::fs::write(path, FOREIGN)?; + } + let setup_ms = unix_time_millis() - created_from; + while unix_time_millis() < window_start { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + + let write_started = unix_time_millis(); + write()?; + let finished = unix_time_millis(); + + for path in &foreign { + assert_eq!( + std::fs::read(path).ok().as_deref(), + Some(FOREIGN), + "{} belongs to another writer but was written through or moved", + path.display() + ); + } + for path in &foreign { + std::fs::remove_file(path)?; + } + if finished < window_end { + return Ok(Outcome::Spared); + } + window_ms = window_ms.max(4 * (finished - write_started)); + lead_ms = lead_ms.max(2 * setup_ms); + if window_ms.max(lead_ms) > max_ms { + return Ok(Outcome::Inconclusive(format!( + "creating {} names took {setup_ms} ms and the write took {} ms, past the \ + window; this host is too slow for the test to say anything", + foreign.len(), + finished - write_started + ))); + } + eprintln!( + "attempt {attempt}/{attempts}: the write finished after the pre-created names ran \ + out; retrying with a {window_ms} ms window after a {lead_ms} ms lead" + ); + } + Ok(Outcome::Inconclusive( + "on every attempt the write ran after the window of pre-created names, so this host \ + is too slow for this test to say anything" + .to_owned(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + fn occupied_name(dir: &Path, millis: u128) -> PathBuf { + dir.join(format!("target.tmp-{millis}")) + } + + /// How many times a self-test below reruns the probe when the host was too + /// slow for it to be conclusive, before accepting that it cannot say. + const SELF_TEST_RUNS: usize = 3; + + /// Reruns `run` until it reaches a verdict about the writer, `None` when + /// every run was inconclusive. A panic from the probe propagates. + fn run_until_conclusive(mut run: impl FnMut() -> anyhow::Result) -> Option { + for _ in 0..SELF_TEST_RUNS { + match run().expect("probe ran") { + Outcome::Inconclusive(why) => eprintln!("self-test run inconclusive: {why}"), + Outcome::Spared => return Some(Outcome::Spared), + } + } + None + } + + fn panic_text(payload: &(dyn std::any::Any + Send)) -> String { + payload + .downcast_ref::() + .cloned() + .or_else(|| { + payload + .downcast_ref::<&str>() + .map(|text| (*text).to_owned()) + }) + .unwrap_or_default() + } + + #[test] + fn a_writer_that_reuses_a_retired_name_is_caught() { + let dir = tempfile::tempdir().expect("scratch directory"); + // A writer shaped like the retired `write_atomic`: a temp name fixed + // by the clock alone, opened with `fs::write`, which truncates + // whatever is already there. + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_until_conclusive(|| { + probe( + |millis| occupied_name(dir.path(), millis), + || { + std::fs::write( + occupied_name(dir.path(), unix_time_millis()), + b"clobbered", + )?; + Ok(()) + }, + INITIAL_WINDOW_MS, + MAX_MS, + ATTEMPTS, + ) + }) + })); + match result { + Err(payload) => { + let text = panic_text(payload.as_ref()); + assert!( + text.contains("belongs to another writer"), + "the probe failed for another reason: {text}" + ); + } + Ok(Some(verdict)) => { + panic!("a writer that truncates a foreign temp file was reported {verdict:?}") + } + // A host too slow for any window to cover the write says nothing + // about the probe; the run above already printed why. + Ok(None) => {} + } + } + + #[test] + fn a_writer_that_picks_its_own_name_is_spared() { + let dir = tempfile::tempdir().expect("scratch directory"); + let verdict = run_until_conclusive(|| { + probe( + |millis| occupied_name(dir.path(), millis), + || { + std::fs::write(dir.path().join("unrelated.tmp"), b"staged")?; + Ok(()) + }, + INITIAL_WINDOW_MS, + MAX_MS, + ATTEMPTS, + ) + }); + // `None` is a host too slow to say; a panic inside would be the probe + // wrongly blaming a writer that never touched a foreign name. + assert!( + matches!(verdict, None | Some(Outcome::Spared)), + "{verdict:?}" + ); + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .expect("list scratch directory") + .map(|entry| entry.expect("directory entry").file_name()) + .collect(); + assert_eq!( + leftovers, + [std::ffi::OsString::from("unrelated.tmp")], + "the probe must remove the names it occupied" + ); + } + + #[test] + fn a_write_slower_than_every_window_is_reported_inconclusive() { + let dir = tempfile::tempdir().expect("scratch directory"); + let outcome = probe( + |millis| occupied_name(dir.path(), millis), + || { + std::thread::sleep(std::time::Duration::from_millis(60)); + Ok(()) + }, + 5, + 20, + ATTEMPTS, + ) + .expect("probe ran"); + assert!( + matches!(outcome, Outcome::Inconclusive(_)), + "a write that outlasts the window proves nothing: {outcome:?}" + ); + } + + #[test] + #[should_panic(expected = "inconclusive")] + fn an_inconclusive_probe_fails_when_not_tolerated() { + require_conclusive(Outcome::Inconclusive("too slow".to_owned()), false); + } + + #[test] + fn an_inconclusive_probe_is_tolerated_when_allowed() { + // Returning without a panic is the whole claim. The public entry point + // passes `cfg!(windows)` for this flag. + require_conclusive(Outcome::Inconclusive("too slow".to_owned()), true); + } + + #[test] + fn a_conclusive_probe_passes_either_way() { + require_conclusive(Outcome::Spared, false); + require_conclusive(Outcome::Spared, true); + } +} diff --git a/crates/rocm-core/src/gguf.rs b/crates/rocm-core/src/gguf.rs new file mode 100644 index 0000000000..694d22a5ca --- /dev/null +++ b/crates/rocm-core/src/gguf.rs @@ -0,0 +1,1611 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Read the shape of a GGUF model from its header, without its weights. +//! +//! A GGUF file starts with a header -- key/value metadata, then one record per +//! tensor -- and only after that, at an aligned offset, the tensor data. The +//! header is what an estimate of memory needs: layer count, attention head +//! counts, head width, and the parameter count summed from the +//! tensor shapes. It is a few megabytes at most (most of it the tokenizer), where +//! the file it describes can be tens of gigabytes, so [`parse_gguf_header`] takes +//! any `Read + Seek` and a caller that can fetch ranges (see +//! [`crate::gguf_source`]) never has to download the weights. +//! +//! The parser is deliberately a reader of *what the estimate needs* and nothing +//! more: arrays are skipped without being stored (a tokenizer is hundreds of +//! thousands of strings), string values are kept only when short, and a layout it +//! does not understand is reported as missing rather than guessed. See +//! for the format. + +use std::collections::BTreeMap; +use std::fmt; +use std::io::{self, Read, Seek, SeekFrom}; + +const MAGIC: [u8; 4] = *b"GGUF"; +/// Version 1 used 32-bit counts and is long obsolete; 2 and 3 differ only in +/// that 3 added big-endian files, which are not read here. +const SUPPORTED_VERSIONS: [u32; 2] = [2, 3]; +const DEFAULT_ALIGNMENT: u64 = 32; +/// A header claiming more entries than this is not one -- real files carry a few +/// dozen metadata keys and some thousands of tensors -- and the bound keeps a +/// hostile or corrupt count from becoming a loop that runs until memory ends. +const MAX_ENTRIES: u64 = 1 << 20; +const MAX_KEY_LEN: u64 = 4096; + +/// The input-embedding tensor. llama.cpp keeps it in host memory when the file +/// has its own [`OUTPUT_NAME`]; with tied embeddings it is the output layer's +/// weight too, on the GPU. +const TOKEN_EMBD_NAME: &str = "token_embd.weight"; +const OUTPUT_NAME: &str = "output.weight"; +/// String values longer than this are skipped rather than stored; the only +/// strings the estimate reads (an architecture name) are far shorter. +const MAX_KEPT_STRING: u64 = 4096; +/// Integer arrays longer than this are skipped rather than stored. The arrays +/// the estimate reads are per-layer (one entry per block); tokenizer arrays are +/// orders of magnitude longer. +const MAX_KEPT_ARRAY: u64 = 4096; +const MAX_TENSOR_DIMS: u32 = 8; +const MAX_ARRAY_DEPTH: u8 = 2; +/// The most a header may span, in bytes, counting everything read or skipped. +/// Real headers are a few megabytes (mostly the tokenizer); a local file has no +/// other bound, and a hostile one could otherwise make the parse walk all of it. +/// The remote reader enforces the same figure on what it fetches. +pub(crate) const MAX_HEADER_BYTES: u64 = 64 * 1024 * 1024; +/// The most layers a model can plausibly have; the largest real ones have a few +/// hundred. Anything above is a corrupt or hostile count, and the estimate loops +/// once per layer. +pub(crate) const MAX_BLOCK_COUNT: u64 = 10_000; +/// How many metadata values of interest a header may carry. A real one has about +/// ten for its architecture; the bound keeps a hostile one from stacking up +/// arrays under thousands of distinct keys. +const MAX_KEPT_KEYS: usize = 64; +/// How many array elements are skipped between checks of the header's size. +const SIZE_CHECK_EVERY: u64 = 4096; +/// The metadata keys (after the architecture prefix) whose values the estimate +/// reads. Everything else is skipped without being stored. +const KEPT_KEY_SUFFIXES: [&str; 9] = [ + ".block_count", + ".embedding_length", + ".attention.head_count", + ".attention.head_count_kv", + ".attention.key_length", + ".attention.value_length", + ".feed_forward_length", + ".context_length", + ".attention.sliding_window", +]; + +fn is_kept_key(key: &str) -> bool { + KEPT_KEY_SUFFIXES.iter().any(|suffix| key.ends_with(suffix)) +} + +/// Why a GGUF header could not be read. +#[derive(Debug)] +pub enum GgufError { + /// The data does not start with the GGUF magic. + NotGguf, + /// A GGUF version this reader does not handle (this includes a big-endian + /// file, which reads as a byte-swapped version). + UnsupportedVersion(u32), + /// The data ended before the header did. + Truncated, + /// The header is structurally wrong. + Malformed(String), + /// The underlying reader failed. + Io(io::Error), +} + +impl fmt::Display for GgufError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotGguf => f.write_str("not a GGUF file (bad magic)"), + Self::UnsupportedVersion(version) => { + write!(f, "GGUF version {version} is not supported (2 and 3 are)") + } + Self::Truncated => f.write_str("the GGUF header ends before it is complete"), + Self::Malformed(why) => write!(f, "malformed GGUF header: {why}"), + Self::Io(error) => write!(f, "could not read the GGUF header: {error}"), + } + } +} + +impl std::error::Error for GgufError {} + +impl From for GgufError { + fn from(error: io::Error) -> Self { + if error.kind() == io::ErrorKind::UnexpectedEof { + Self::Truncated + } else { + Self::Io(error) + } + } +} + +/// A per-layer figure: one value for every layer, or one per layer. +/// +/// Models that mix layer kinds (attention interleaved with state-space or +/// sliding-window blocks) store the head counts as an array. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PerLayer { + Uniform(u64), + Each(Vec), +} + +impl PerLayer { + /// The value for `layer`, or `None` when a per-layer array is too short. + #[must_use] + pub fn at(&self, layer: usize) -> Option { + match self { + Self::Uniform(value) => Some(*value), + Self::Each(values) => values.get(layer).copied(), + } + } +} + +/// What the header says about the model, as far as a memory estimate needs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GgufHeader { + pub version: u32, + /// `general.architecture`, the prefix every other key of interest carries. + pub architecture: Option, + pub tensor_count: u64, + /// Sum of the elements of every tensor. + pub param_count: u64, + /// Byte offset at which the tensor data starts: the end of the header + /// rounded up to the file's alignment. + pub data_offset: u64, + pub block_count: Option, + pub embedding_length: Option, + pub head_count: Option, + pub head_count_kv: Option, + pub key_length: Option, + pub value_length: Option, + pub feed_forward_length: Option, + /// The context length the model was trained for. + pub context_length: Option, + /// `split.count`: set when the file is one shard of several. + pub split_count: Option, + /// A sliding-window attention key is present: the KV cache of the windowed + /// layers can be smaller than a full-context figure. + pub sliding_window: bool, + /// A state-space (`*.ssm.*`) key is present: some layers keep no KV cache. + pub ssm: bool, + /// A latent-attention (`kv_lora_rank`) key is present: the KV cache is not + /// laid out as heads times head width. + pub latent_attention: bool, + /// An expert count above one is present: a mixture-of-experts model, whose + /// feed-forward width is per expert. + pub moe: bool, + /// Where `token_embd.weight` sits in the tensor data, when the file has one. + pub token_embd: Option, + /// The file has its own `output.weight`. Without one the embeddings are + /// tied: llama.cpp uses `token_embd.weight` for the output layer too, and + /// places that use of it with the output layer, on the GPU. + pub has_output: bool, +} + +/// A tensor's place in the tensor data. +/// +/// That is enough to size it without knowing its quantization: it runs from +/// `offset` to the start of whichever tensor comes next in the data, or to the +/// end of the file when none does. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TensorSpan { + /// Offset from the start of the tensor data. + pub offset: u64, + /// The smallest offset of any other tensor beyond this one; `None` when this + /// is the last tensor, which then runs to the end of the file. + pub next_offset: Option, +} + +/// What a metadata value was, once read. +enum Value { + Uint(u64), + Str(String), + Ints(Vec), + /// Read past and not kept. + Skipped, +} + +struct Reader<'a, R> { + inner: &'a mut R, +} + +impl Reader<'_, R> { + fn bytes(&mut self) -> Result<[u8; N], GgufError> { + let mut buf = [0_u8; N]; + self.inner.read_exact(&mut buf)?; + Ok(buf) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.bytes()?)) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.bytes()?)) + } + + /// Move forward `count` bytes. Seeking past the end is allowed by the + /// reader, and the next read then fails as truncated, which is the right + /// error for a length that points outside the data. A single skip longer + /// than any header may be is refused at once, as malformed: on a file the + /// seek itself can fail with an I/O error for a huge offset, which would + /// blame the disk for a defect of the header. + fn skip(&mut self, count: u64) -> Result<(), GgufError> { + if count > MAX_HEADER_BYTES { + return Err(GgufError::Malformed(format!( + "a length of {count} bytes is larger than any header, which is at most {} MiB", + MAX_HEADER_BYTES / (1024 * 1024) + ))); + } + let count = i64::try_from(count) + .map_err(|_| GgufError::Malformed(format!("length {count} is out of range")))?; + self.inner.seek(SeekFrom::Current(count))?; + Ok(()) + } + + /// Refuse a header that has already spanned more than [`MAX_HEADER_BYTES`]. + fn check_size(&mut self) -> Result<(), GgufError> { + if self.inner.stream_position()? > MAX_HEADER_BYTES { + return Err(GgufError::Malformed(format!( + "the header is larger than {} MiB, which no real model's is", + MAX_HEADER_BYTES / (1024 * 1024) + ))); + } + Ok(()) + } + + /// A string: a 64-bit length, then that many bytes. `None` when it is + /// longer than `keep_up_to`, in which case it has been skipped. + fn string(&mut self, keep_up_to: u64) -> Result, GgufError> { + let len = self.u64()?; + if len > keep_up_to { + self.skip(len)?; + return Ok(None); + } + let len = usize::try_from(len) + .map_err(|_| GgufError::Malformed(format!("length {len} is out of range")))?; + let mut buf = vec![0_u8; len]; + self.inner.read_exact(&mut buf)?; + Ok(Some(String::from_utf8_lossy(&buf).into_owned())) + } + + fn value(&mut self, value_type: u32, depth: u8, keep: bool) -> Result { + if let Some((width, num)) = fixed_width(value_type) { + return self.scalar(width, num); + } + match value_type { + 8 => Ok(self + .string(MAX_KEPT_STRING)? + .map_or(Value::Skipped, Value::Str)), + 9 => self.array(depth, keep), + other => Err(GgufError::Malformed(format!( + "unknown metadata value type {other}" + ))), + } + } + + /// A fixed-width scalar. Integers (and booleans) are kept when they are not + /// negative: every figure the estimate reads is one, and the writers in common + /// use (gguf-py among them) store a Python integer as a signed 32-bit one, so + /// refusing signed types would leave most models unreadable. A negative or + /// floating value under a key of interest is not something to interpret, and + /// reads as missing. + fn scalar(&mut self, width: usize, num: Num) -> Result { + let mut buf = [0_u8; 8]; + self.inner.read_exact(&mut buf[..width])?; + Ok(decode_int(buf, width, num).map_or(Value::Skipped, Value::Uint)) + } + + fn array(&mut self, depth: u8, keep: bool) -> Result { + let element_type = self.u32()?; + let len = self.u64()?; + if let Some((width, num)) = fixed_width(element_type) { + if keep && num != Num::Other && len <= MAX_KEPT_ARRAY { + let mut values = Vec::with_capacity(usize::try_from(len).unwrap_or(0)); + let mut usable = true; + for _ in 0..len { + let mut buf = [0_u8; 8]; + self.inner.read_exact(&mut buf[..width])?; + match decode_int(buf, width, num) { + Some(value) => values.push(value), + None => usable = false, + } + } + // One negative entry makes the whole per-layer list meaningless; + // it reads as missing rather than as a list with a hole in it. + return Ok(if usable { + Value::Ints(values) + } else { + Value::Skipped + }); + } + let total = len.checked_mul(width as u64).ok_or_else(|| { + GgufError::Malformed(format!("array of {len} elements is out of range")) + })?; + self.skip(total)?; + return Ok(Value::Skipped); + } + match element_type { + 8 => { + for item in 0..len { + let item_len = self.u64()?; + self.skip(item_len)?; + if item % SIZE_CHECK_EVERY == SIZE_CHECK_EVERY - 1 { + self.check_size()?; + } + } + } + 9 => { + if depth >= MAX_ARRAY_DEPTH { + return Err(GgufError::Malformed("arrays nested too deeply".to_owned())); + } + for item in 0..len { + self.value(9, depth + 1, false)?; + if item % SIZE_CHECK_EVERY == SIZE_CHECK_EVERY - 1 { + self.check_size()?; + } + } + } + other => { + return Err(GgufError::Malformed(format!( + "unknown array element type {other}" + ))); + } + } + Ok(Value::Skipped) + } +} + +/// The width in bytes, and how to read it, of a fixed-width GGUF metadata type +/// (a scalar value, or an array element); `None` for a string, an array, or a +/// type id the format does not define. The one table of fixed-width type ids: a +/// scalar and an array element share it, so a type added here is read in both +/// places. +const fn fixed_width(type_id: u32) -> Option<(usize, Num)> { + // GGUF metadata value types, in the order the format numbers them; 8 is a + // string and 9 an array, neither of fixed width. + match type_id { + // uint8, and bool (one byte, read as its byte value). + 0 | 7 => Some((1, Num::Unsigned)), + 1 => Some((1, Num::Signed)), + 2 => Some((2, Num::Unsigned)), + 3 => Some((2, Num::Signed)), + 4 => Some((4, Num::Unsigned)), + 5 => Some((4, Num::Signed)), + 6 => Some((4, Num::Other)), + 10 => Some((8, Num::Unsigned)), + 11 => Some((8, Num::Signed)), + 12 => Some((8, Num::Other)), + _ => None, + } +} + +/// How the bytes of a fixed-width metadata value are read. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Num { + Unsigned, + Signed, + /// A float, or anything else not read as an integer. + Other, +} + +/// The non-negative integer in the first `width` bytes of `buf`, little-endian; +/// `None` for a negative signed value or a type that is not an integer. +fn decode_int(buf: [u8; 8], width: usize, num: Num) -> Option { + match num { + Num::Other => None, + Num::Unsigned => Some(u64::from_le_bytes(buf)), + Num::Signed => { + // Sign-extend from the value's own width. + let shift = 64 - 8 * u32::try_from(width).unwrap_or(8); + let signed = i64::from_le_bytes(buf) << shift >> shift; + u64::try_from(signed).ok() + } + } +} + +fn per_layer(value: Option<&Value>) -> Option { + match value? { + Value::Uint(value) => Some(PerLayer::Uniform(*value)), + Value::Ints(values) => Some(PerLayer::Each(values.clone())), + Value::Str(_) | Value::Skipped => None, + } +} + +fn uint(value: Option<&Value>) -> Option { + match value? { + Value::Uint(value) => Some(*value), + // A per-layer array where a single figure was wanted (a feed-forward + // width that varies by layer): the widest is the one a buffer is sized + // for. + Value::Ints(values) => values.iter().copied().max(), + Value::Str(_) | Value::Skipped => None, + } +} + +const fn align_up(value: u64, alignment: u64) -> u64 { + value.div_ceil(alignment).saturating_mul(alignment) +} + +/// Read the header of a GGUF file from `source`, positioned at its start. +/// +/// Reads the metadata and the tensor records and stops there: the tensor data is +/// never touched. +/// +/// # Errors +/// +/// [`GgufError`] when the data is not a GGUF this reader handles, ends early, or +/// is structurally wrong. +pub fn parse_gguf_header(source: &mut R) -> Result { + let mut reader = Reader { inner: source }; + let magic: [u8; 4] = reader.bytes()?; + if magic != MAGIC { + return Err(GgufError::NotGguf); + } + let version = reader.u32()?; + if !SUPPORTED_VERSIONS.contains(&version) { + return Err(GgufError::UnsupportedVersion(version)); + } + let tensor_count = reader.u64()?; + let metadata_count = reader.u64()?; + for (what, count) in [("tensor", tensor_count), ("metadata", metadata_count)] { + if count > MAX_ENTRIES { + return Err(GgufError::Malformed(format!( + "{count} {what} entries is not a plausible header" + ))); + } + } + + let mut architecture = None; + let mut alignment = DEFAULT_ALIGNMENT; + let mut split_count = None; + let mut sliding_window = false; + let mut ssm = false; + let mut latent_attention = false; + let mut moe = false; + let mut kept: BTreeMap = BTreeMap::new(); + + for _ in 0..metadata_count { + let key = reader + .string(MAX_KEY_LEN)? + .ok_or_else(|| GgufError::Malformed("a metadata key is implausibly long".to_owned()))?; + let value_type = reader.u32()?; + let keep = is_kept_key(&key); + let value = reader.value(value_type, 0, keep)?; + reader.check_size()?; + match (key.as_str(), value) { + ("general.architecture", Value::Str(name)) => architecture = Some(name), + ("general.alignment", Value::Uint(value)) => alignment = value, + ("split.count", Value::Uint(value)) => split_count = Some(value), + (_, value) => { + ssm |= key.contains(".ssm."); + latent_attention |= key.ends_with(".attention.kv_lora_rank"); + if key.ends_with(".expert_count") { + moe |= uint(Some(&value)).is_some_and(|experts| experts > 1); + } + if key.ends_with(".attention.sliding_window") { + sliding_window |= uint(Some(&value)).is_some_and(|window| window > 0); + } + // The sliding-window key is decoded as kept (so a per-layer array + // is read, not skipped) for the flag above; its value is not + // read again, so it does not take one of the kept slots. + let read_later = !key.ends_with(".attention.sliding_window"); + if keep && read_later && matches!(value, Value::Uint(_) | Value::Ints(_)) { + if kept.len() >= MAX_KEPT_KEYS && !kept.contains_key(&key) { + return Err(GgufError::Malformed(format!( + "more than {MAX_KEPT_KEYS} distinct model-shape keys is not a \ + plausible header" + ))); + } + kept.insert(key, value); + } + } + } + } + if alignment == 0 { + return Err(GgufError::Malformed("general.alignment is zero".to_owned())); + } + + let mut param_count = 0_u64; + let mut offsets: Vec = Vec::new(); + let mut token_embd_offset = None; + let mut has_output = false; + for _ in 0..tensor_count { + // Held to the same bound as a metadata key: a real tensor name is a few + // dozen bytes. + let name = reader + .string(MAX_KEY_LEN)? + .ok_or_else(|| GgufError::Malformed("a tensor name is implausibly long".to_owned()))?; + let dims = reader.u32()?; + if dims > MAX_TENSOR_DIMS { + return Err(GgufError::Malformed(format!( + "a tensor claims {dims} dimensions" + ))); + } + let mut elements = 1_u64; + for _ in 0..dims { + elements = elements.saturating_mul(reader.u64()?); + } + let _ggml_type = reader.u32()?; + let offset = reader.u64()?; + param_count = param_count.saturating_add(elements); + offsets.push(offset); + reader.check_size()?; + if name == TOKEN_EMBD_NAME { + token_embd_offset = Some(offset); + } + has_output |= name == OUTPUT_NAME; + } + let token_embd = token_embd_offset.map(|offset| TensorSpan { + offset, + next_offset: offsets + .iter() + .copied() + .filter(|other| *other > offset) + .min(), + }); + let header_end = reader.inner.stream_position()?; + + let get = |suffix: &str| { + architecture + .as_ref() + .and_then(|arch| kept.get(&format!("{arch}.{suffix}"))) + }; + let block_count = uint(get("block_count")); + if let Some(blocks) = block_count.filter(|blocks| *blocks > MAX_BLOCK_COUNT) { + return Err(GgufError::Malformed(format!( + "{blocks} layers is not a plausible model" + ))); + } + Ok(GgufHeader { + version, + block_count, + embedding_length: uint(get("embedding_length")), + head_count: per_layer(get("attention.head_count")), + head_count_kv: per_layer(get("attention.head_count_kv")), + key_length: uint(get("attention.key_length")), + value_length: uint(get("attention.value_length")), + feed_forward_length: uint(get("feed_forward_length")), + context_length: uint(get("context_length")), + architecture, + tensor_count, + param_count, + data_offset: align_up(header_end, alignment), + split_count, + sliding_window, + ssm, + latent_attention, + moe, + token_embd, + has_output, + }) +} + +#[cfg(test)] +pub(crate) mod test_builder { + //! Byte builders for synthetic GGUF headers, shared with the estimator and + //! source tests so every layer is exercised against the same fixtures. + + pub const U8: u32 = 0; + pub const U32: u32 = 4; + pub const I32: u32 = 5; + pub const F32: u32 = 6; + pub const STRING: u32 = 8; + pub const ARRAY: u32 = 9; + + #[derive(Default)] + pub struct Builder { + version: u32, + metadata: Vec, + metadata_count: u64, + tensors: Vec, + tensor_count: u64, + alignment: u64, + } + + impl Builder { + pub fn new() -> Self { + Self { + version: 3, + alignment: 32, + ..Self::default() + } + } + + pub fn version(mut self, version: u32) -> Self { + self.version = version; + self + } + + fn push_string(buf: &mut Vec, value: &str) { + buf.extend_from_slice(&(value.len() as u64).to_le_bytes()); + buf.extend_from_slice(value.as_bytes()); + } + + fn key(&mut self, key: &str, value_type: u32) { + Self::push_string(&mut self.metadata, key); + self.metadata.extend_from_slice(&value_type.to_le_bytes()); + self.metadata_count += 1; + } + + pub fn string(mut self, key: &str, value: &str) -> Self { + self.key(key, STRING); + Self::push_string(&mut self.metadata, value); + self + } + + pub fn u32(mut self, key: &str, value: u32) -> Self { + self.key(key, U32); + self.metadata.extend_from_slice(&value.to_le_bytes()); + self + } + + /// A signed 32-bit integer, which is how gguf-py stores a Python integer. + pub fn i32(mut self, key: &str, value: i32) -> Self { + self.key(key, I32); + self.metadata.extend_from_slice(&value.to_le_bytes()); + self + } + + pub fn i32_array(mut self, key: &str, values: &[i32]) -> Self { + self.key(key, ARRAY); + self.metadata.extend_from_slice(&I32.to_le_bytes()); + self.metadata + .extend_from_slice(&(values.len() as u64).to_le_bytes()); + for value in values { + self.metadata.extend_from_slice(&value.to_le_bytes()); + } + self + } + + pub fn f32(mut self, key: &str, value: f32) -> Self { + self.key(key, F32); + self.metadata.extend_from_slice(&value.to_le_bytes()); + self + } + + pub fn u32_array(mut self, key: &str, values: &[u32]) -> Self { + self.key(key, ARRAY); + self.metadata.extend_from_slice(&U32.to_le_bytes()); + self.metadata + .extend_from_slice(&(values.len() as u64).to_le_bytes()); + for value in values { + self.metadata.extend_from_slice(&value.to_le_bytes()); + } + self + } + + /// A string array, the shape of a tokenizer's token list. + pub fn string_array(mut self, key: &str, count: usize) -> Self { + self.key(key, ARRAY); + self.metadata.extend_from_slice(&STRING.to_le_bytes()); + self.metadata + .extend_from_slice(&(count as u64).to_le_bytes()); + for index in 0..count { + Self::push_string(&mut self.metadata, &format!("tok{index}")); + } + self + } + + pub fn u8_array_of_len(mut self, key: &str, count: u64) -> Self { + self.key(key, ARRAY); + self.metadata.extend_from_slice(&U8.to_le_bytes()); + self.metadata.extend_from_slice(&count.to_le_bytes()); + self.metadata + .extend(std::iter::repeat_n(0_u8, count as usize)); + self + } + + /// A metadata entry with a value type and bytes of the caller's choosing, + /// for building headers no real writer would produce. + pub fn raw(mut self, key: &str, value_type: u32, value: &[u8]) -> Self { + self.key(key, value_type); + self.metadata.extend_from_slice(value); + self + } + + pub fn alignment(mut self, alignment: u32) -> Self { + self.alignment = u64::from(alignment); + self.u32("general.alignment", alignment) + } + + pub fn tensor(self, name: &str, dims: &[u64]) -> Self { + self.tensor_at(name, dims, 0) + } + + /// A tensor record with an explicit offset into the tensor data. + pub fn tensor_at(mut self, name: &str, dims: &[u64], offset: u64) -> Self { + Self::push_string(&mut self.tensors, name); + self.tensors + .extend_from_slice(&(dims.len() as u32).to_le_bytes()); + for dim in dims { + self.tensors.extend_from_slice(&dim.to_le_bytes()); + } + self.tensors.extend_from_slice(&0_u32.to_le_bytes()); + self.tensors.extend_from_slice(&offset.to_le_bytes()); + self.tensor_count += 1; + self + } + + /// The header bytes alone, ending where the tensor data would begin + /// before alignment padding. + pub fn header_bytes(&self) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(b"GGUF"); + out.extend_from_slice(&self.version.to_le_bytes()); + out.extend_from_slice(&self.tensor_count.to_le_bytes()); + out.extend_from_slice(&self.metadata_count.to_le_bytes()); + out.extend_from_slice(&self.metadata); + out.extend_from_slice(&self.tensors); + out + } + + /// A whole file: the header, padding to the alignment, then + /// `data_len` bytes standing in for the weights. + pub fn file(&self, data_len: usize) -> Vec { + let mut out = self.header_bytes(); + let aligned = (out.len() as u64).div_ceil(self.alignment) * self.alignment; + out.resize(aligned as usize, 0); + out.extend(std::iter::repeat_n(0_u8, data_len)); + out + } + } + + /// A llama-shaped model: 32 layers, 32 heads of which 8 are KV heads, head + /// width 128, so 4096 embedding. Vocabulary of `vocab` tokens. It has its + /// own `output.weight`, as Llama 3 does, so its embeddings are not tied. + pub fn llama_like(vocab: usize) -> Builder { + Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 32) + .u32("llama.embedding_length", 4096) + .u32("llama.feed_forward_length", 14336) + .u32("llama.attention.head_count", 32) + .u32("llama.attention.head_count_kv", 8) + .u32("llama.context_length", 131_072) + .string_array("tokenizer.ggml.tokens", vocab) + .tensor_at("token_embd.weight", &[4096, vocab as u64], 0) + .tensor_at("blk.0.attn_q.weight", &[4096, 4096], vocab as u64 * 4096) + .tensor_at( + "output.weight", + &[4096, vocab as u64], + vocab as u64 * 4096 + 4096 * 4096, + ) + } +} + +#[cfg(test)] +mod tests { + use super::test_builder::{Builder, llama_like}; + use super::*; + use std::io::Cursor; + + fn parse(bytes: &[u8]) -> Result { + parse_gguf_header(&mut Cursor::new(bytes)) + } + + #[test] + fn reads_the_figures_an_estimate_needs() { + let file = llama_like(10).file(1000); + let header = parse(&file).unwrap(); + assert_eq!(header.version, 3); + assert_eq!(header.architecture.as_deref(), Some("llama")); + assert_eq!(header.block_count, Some(32)); + assert_eq!(header.embedding_length, Some(4096)); + assert_eq!(header.feed_forward_length, Some(14336)); + assert_eq!(header.head_count, Some(PerLayer::Uniform(32))); + assert_eq!(header.head_count_kv, Some(PerLayer::Uniform(8))); + assert_eq!(header.context_length, Some(131_072)); + assert_eq!(header.tensor_count, 3); + assert_eq!(header.param_count, 2 * 4096 * 10 + 4096 * 4096); + assert!(!header.sliding_window && !header.ssm && !header.latent_attention); + } + + #[test] + fn the_input_embedding_runs_to_the_next_tensor_in_the_data() { + // Listed out of data order on purpose: the span ends at the next offset + // beyond it, not at the next record. + let file = Builder::new() + .tensor_at("blk.0.attn_q.weight", &[8, 8], 4096) + .tensor_at("token_embd.weight", &[8, 8], 1024) + .tensor_at("output.weight", &[8, 8], 2048) + .file(8192); + let header = parse(&file).unwrap(); + assert_eq!( + header.token_embd, + Some(TensorSpan { + offset: 1024, + next_offset: Some(2048) + }) + ); + } + + #[test] + fn the_input_embedding_is_the_last_tensor_when_nothing_follows_it() { + let file = Builder::new() + .tensor_at("blk.0.attn_q.weight", &[8, 8], 0) + .tensor_at("token_embd.weight", &[8, 8], 512) + .file(1024); + let header = parse(&file).unwrap(); + assert_eq!( + header.token_embd, + Some(TensorSpan { + offset: 512, + next_offset: None + }) + ); + } + + #[test] + fn an_output_tensor_is_noticed_and_its_absence_means_tied_embeddings() { + // Not the last tensor, so a later one cannot clear what it set. + let untied = Builder::new() + .tensor_at("token_embd.weight", &[8, 8], 0) + .tensor_at("output.weight", &[8, 8], 64) + .tensor_at("output_norm.weight", &[8], 128) + .file(160); + assert!(parse(&untied).unwrap().has_output); + let tied = Builder::new() + .tensor_at("token_embd.weight", &[8, 8], 0) + .tensor_at("output_norm.weight", &[8], 64) + .file(128); + assert!(!parse(&tied).unwrap().has_output); + } + + #[test] + fn a_file_without_an_input_embedding_has_no_span() { + let file = Builder::new() + .tensor_at("blk.0.attn_q.weight", &[8, 8], 0) + .file(64); + assert_eq!(parse(&file).unwrap().token_embd, None); + } + + #[test] + fn the_data_offset_is_the_header_end_rounded_up_to_the_alignment() { + let builder = llama_like(10); + let unpadded = builder.header_bytes().len() as u64; + let header = parse(&builder.file(64)).unwrap(); + assert_eq!(header.data_offset, unpadded.div_ceil(32) * 32); + // Something other than the default must be honoured: a parser that + // ignored `general.alignment` would land on the 32-byte boundary. + let wide = llama_like(10).alignment(4096); + let header = parse(&wide.file(64)).unwrap(); + assert_eq!(header.data_offset, 4096); + } + + #[test] + fn version_two_is_read_and_other_versions_are_refused() { + assert_eq!(parse(&llama_like(3).version(2).file(0)).unwrap().version, 2); + for version in [1, 4, 0x0300_0000] { + let error = parse(&llama_like(3).version(version).file(0)).unwrap_err(); + assert!( + matches!(error, GgufError::UnsupportedVersion(found) if found == version), + "{error:?}" + ); + } + } + + #[test] + fn a_file_without_the_magic_is_not_gguf() { + let mut bytes = llama_like(3).file(0); + bytes[0] = b'X'; + assert!(matches!(parse(&bytes), Err(GgufError::NotGguf))); + assert!(matches!(parse(b"GG"), Err(GgufError::Truncated))); + } + + #[test] + fn a_header_cut_short_is_truncated_wherever_it_is_cut() { + let header = llama_like(50).header_bytes(); + // Every proper prefix must fail, and fail as truncation: a parser that + // returned a header from half the data would report a model it never + // finished reading. + for cut in 0..header.len() { + let error = parse(&header[..cut]).unwrap_err(); + assert!( + matches!(error, GgufError::Truncated | GgufError::NotGguf), + "prefix of {cut} bytes gave {error:?}" + ); + } + parse(&header).unwrap(); + } + + #[test] + fn a_per_layer_head_count_array_is_kept_per_layer() { + let file = Builder::new() + .string("general.architecture", "hybrid") + .u32("hybrid.block_count", 4) + .u32_array("hybrid.attention.head_count_kv", &[0, 8, 0, 8]) + .file(0); + let header = parse(&file).unwrap(); + let heads = header.head_count_kv.unwrap(); + assert_eq!(heads, PerLayer::Each(vec![0, 8, 0, 8])); + assert_eq!(heads.at(1), Some(8)); + assert_eq!(heads.at(2), Some(0)); + assert_eq!(heads.at(4), None); + } + + #[test] + fn parsing_resumes_exactly_after_arrays_it_skips() { + // A kept key placed after two large arrays the parser skips: it is read + // with its value only if each skip moved exactly past its array. + let file = llama_like(1000) + .u8_array_of_len("tokenizer.ggml.token_type", 5000) + .u32_array("tokenizer.ggml.merges_like", &vec![7; 5000]) + .u32("llama.attention.key_length", 99) + .file(0); + let header = parse(&file).unwrap(); + assert_eq!(header.key_length, Some(99)); + assert_eq!(header.tensor_count, 3); + } + + /// A feed-forward width given per layer (an array) is read as the widest, + /// which is the one a compute buffer is sized for, not the first or the + /// narrowest. + #[test] + fn a_per_layer_feed_forward_width_is_read_as_the_widest() { + let file = Builder::new() + .string("general.architecture", "llama") + .u32_array("llama.feed_forward_length", &[11_008, 14_336, 8_192]) + .tensor("w", &[16]) + .file(0); + assert_eq!(parse(&file).unwrap().feed_forward_length, Some(14_336)); + } + + #[test] + fn layouts_the_estimate_cannot_model_are_flagged() { + let base = || { + Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 2) + }; + let header = parse(&base().u32("x.attention.sliding_window", 4096).file(0)).unwrap(); + assert!(header.sliding_window); + // A zero window means no window. + let header = parse(&base().u32("x.attention.sliding_window", 0).file(0)).unwrap(); + assert!(!header.sliding_window); + let header = parse(&base().u32("x.ssm.conv_kernel", 4).file(0)).unwrap(); + assert!(header.ssm); + let header = parse(&base().u32("x.attention.kv_lora_rank", 512).file(0)).unwrap(); + assert!(header.latent_attention); + let header = parse(&base().u32("split.count", 3).file(0)).unwrap(); + assert_eq!(header.split_count, Some(3)); + } + + #[test] + fn implausible_counts_are_refused_before_they_become_a_loop() { + // The tensor count (bytes 8..16) and the metadata count (16..24), each + // made absurd in turn. The message names the count, so the test fails if + // the guard is removed and the parser fails somewhere else instead. + for (range, what) in [(8..16, "tensor entries"), (16..24, "metadata entries")] { + let mut bytes = llama_like(3).header_bytes(); + bytes[range].copy_from_slice(&u64::MAX.to_le_bytes()); + let error = parse(&bytes).unwrap_err(); + assert!( + matches!(&error, GgufError::Malformed(message) if message.contains(what)), + "{what}: {error}" + ); + } + } + + /// A tensor's element count is a product of dimensions the file states; one + /// that overflows saturates rather than panicking. + #[test] + fn tensor_dimensions_that_overflow_saturate() { + let file = Builder::new() + .string("general.architecture", "x") + .tensor("huge", &[u64::MAX, u64::MAX]) + .file(0); + assert_eq!(parse(&file).unwrap().param_count, u64::MAX); + } + + /// A tensor name longer than any real one is refused, as a metadata key is. + #[test] + fn an_implausibly_long_tensor_name_is_refused() { + let file = Builder::new() + .string("general.architecture", "x") + .tensor(&"t".repeat(5000), &[8]) + .file(0); + let error = parse(&file).unwrap_err(); + assert!( + matches!(&error, GgufError::Malformed(message) if message.contains("tensor name")), + "{error}" + ); + } + + #[test] + fn a_zero_alignment_is_malformed_rather_than_a_division_by_zero() { + let file = Builder::new() + .string("general.architecture", "llama") + .u32("general.alignment", 0) + .file(0); + assert!(matches!(parse(&file), Err(GgufError::Malformed(_)))); + } + + #[test] + fn floats_and_other_values_under_keys_of_interest_are_ignored_not_misread() { + let file = Builder::new() + .string("general.architecture", "llama") + .f32("llama.block_count", 32.0) + .file(0); + assert_eq!(parse(&file).unwrap().block_count, None); + } + + // ----- untrusted-input guards --------------------------------------------- + // + // A header can come from a stranger's Hugging Face repository, so each bound + // below is one a hostile or corrupt file would otherwise walk past. Every test + // names the guard it exercises and fails when that guard is removed. + + fn malformed(result: Result) -> String { + match result { + Err(GgufError::Malformed(why)) => why, + other => panic!("expected a malformed header, got {other:?}"), + } + } + + /// The bytes of an array value (after its type marker) nested `depth` deep, + /// innermost an empty byte array. + fn nested_array(depth: usize) -> Vec { + let mut out = Vec::new(); + if depth <= 1 { + out.extend_from_slice(&test_builder::U8.to_le_bytes()); + out.extend_from_slice(&0_u64.to_le_bytes()); + } else { + out.extend_from_slice(&test_builder::ARRAY.to_le_bytes()); + out.extend_from_slice(&1_u64.to_le_bytes()); + out.extend(nested_array(depth - 1)); + } + out + } + + #[test] + fn a_metadata_key_beyond_the_length_bound_is_malformed() { + let long = "k".repeat(usize::try_from(MAX_KEY_LEN).unwrap() + 1); + let file = Builder::new().u32(&long, 1).file(0); + assert!(malformed(parse(&file)).contains("key")); + // Exactly at the bound is still a key. + let ok = "k".repeat(usize::try_from(MAX_KEY_LEN).unwrap()); + parse(&Builder::new().u32(&ok, 1).file(0)).unwrap(); + } + + #[test] + fn a_tensor_count_beyond_the_bound_is_malformed() { + let mut bytes = llama_like(3).header_bytes(); + // The tensor count is bytes 8..16, ahead of the metadata count. + bytes[8..16].copy_from_slice(&(MAX_ENTRIES + 1).to_le_bytes()); + assert!(malformed(parse(&bytes)).contains("tensor")); + } + + #[test] + fn a_tensor_with_too_many_dimensions_is_malformed() { + let file = Builder::new().tensor("t", &[1; 9]).file(0); + assert!(malformed(parse(&file)).contains("dimensions")); + parse(&Builder::new().tensor("t", &[1; 8]).file(0)).unwrap(); + } + + #[test] + fn arrays_nested_beyond_the_depth_bound_are_malformed() { + let at = |depth| { + let value = nested_array(depth); + parse(&Builder::new().raw("k", test_builder::ARRAY, &value).file(0)) + }; + at(MAX_ARRAY_DEPTH as usize + 1).unwrap(); + assert!(malformed(at(MAX_ARRAY_DEPTH as usize + 2)).contains("nested too deeply")); + } + + #[test] + fn an_array_whose_byte_length_overflows_is_malformed_not_truncated() { + // 2^63 four-byte elements: the product does not fit in 64 bits. + let mut value = Vec::new(); + value.extend_from_slice(&test_builder::F32.to_le_bytes()); + value.extend_from_slice(&(1_u64 << 63).to_le_bytes()); + let file = Builder::new().raw("k", test_builder::ARRAY, &value).file(0); + assert!(malformed(parse(&file)).contains("out of range")); + // A length that fits in 64 bits but not in the signed offset a seek takes: + // far past any header, so refused as that before a seek is tried. + let mut value = Vec::new(); + value.extend_from_slice(&test_builder::F32.to_le_bytes()); + value.extend_from_slice(&((u64::MAX / 4) - 1).to_le_bytes()); + let file = Builder::new().raw("k", test_builder::ARRAY, &value).file(0); + assert!(malformed(parse(&file)).contains("larger than any header")); + } + + /// A length far beyond any header is malformed whatever the reader does with + /// a seek that large: an in-memory reader would call it truncated, a file can + /// fail the seek with an I/O error. + #[test] + fn a_single_skip_longer_than_any_header_is_malformed() { + for length in [MAX_HEADER_BYTES + 1, 1_u64 << 44, 1_u64 << 50] { + let file = Builder::new() + .raw("k", test_builder::STRING, &length.to_le_bytes()) + .file(0); + let message = malformed(parse(&file)); + assert!(message.contains("larger than any header"), "{message}"); + } + // At the bound itself the skip is allowed, and the header it leaves + // behind is then past the size bound: refused there, by the size check. + let file = Builder::new() + .raw("k", test_builder::STRING, &MAX_HEADER_BYTES.to_le_bytes()) + .file(0); + let message = malformed(parse(&file)); + assert!(message.contains("header is larger than"), "{message}"); + } + + #[test] + fn an_unknown_value_or_element_type_is_malformed() { + let file = Builder::new().raw("k", 99, &[0; 8]).file(0); + assert!(malformed(parse(&file)).contains("unknown metadata value type 99")); + let mut value = Vec::new(); + value.extend_from_slice(&99_u32.to_le_bytes()); + value.extend_from_slice(&1_u64.to_le_bytes()); + let file = Builder::new().raw("k", test_builder::ARRAY, &value).file(0); + assert!(malformed(parse(&file)).contains("unknown array element type 99")); + } + + #[test] + fn a_layer_count_beyond_the_bound_is_malformed() { + let with = |layers| { + parse( + &Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", layers) + .file(0), + ) + }; + assert_eq!( + with(u32::try_from(MAX_BLOCK_COUNT).unwrap()) + .unwrap() + .block_count, + Some(MAX_BLOCK_COUNT) + ); + assert!(malformed(with(u32::MAX)).contains("not a plausible model")); + assert!( + malformed(with(u32::try_from(MAX_BLOCK_COUNT).unwrap() + 1)) + .contains("not a plausible model") + ); + } + + #[test] + fn too_many_distinct_shape_keys_are_malformed_and_other_keys_are_not_kept() { + // Arrays under keys the estimate never reads are skipped, however many. + let mut builder = Builder::new(); + for index in 0..200 { + builder = builder.u32_array(&format!("junk{index}.table"), &[1; 4000]); + } + parse(&builder.file(0)).unwrap(); + // Keys it does read are kept, so there is a bound on how many. + let mut builder = Builder::new(); + for index in 0..=MAX_KEPT_KEYS { + builder = builder.u32_array(&format!("a{index}.attention.head_count"), &[1; 8]); + } + assert!(malformed(parse(&builder.file(0))).contains("distinct")); + let mut builder = Builder::new(); + for index in 0..MAX_KEPT_KEYS { + builder = builder.u32_array(&format!("a{index}.attention.head_count"), &[1; 8]); + } + parse(&builder.file(0)).unwrap(); + } + + /// The sliding-window key is decoded as a kept key but never read again, so + /// it takes no one of the kept slots: a header at the cap of other shape + /// keys is still readable with it, and the flag is set. + #[test] + fn the_sliding_window_key_does_not_take_a_kept_slot() { + let mut builder = Builder::new(); + for index in 0..MAX_KEPT_KEYS { + builder = builder.u32(&format!("a{index}.attention.head_count"), 1); + } + let header = parse(&builder.u32("x.attention.sliding_window", 4096).file(0)).unwrap(); + assert!(header.sliding_window); + } + + /// A key repeated once the kept slots are full replaces its own value; it is + /// not a new distinct key, so the cap does not refuse it. + #[test] + fn a_repeated_shape_key_at_the_cap_is_not_a_new_distinct_key() { + let mut builder = Builder::new(); + for index in 0..MAX_KEPT_KEYS { + builder = builder.u32(&format!("a{index}.attention.head_count"), 1); + } + builder = builder.u32("a0.attention.head_count", 2); + parse(&builder.file(0)).unwrap(); + } + + #[test] + fn a_header_that_spans_more_than_the_size_bound_is_refused() { + // Two arrays, each under the bound on its own, that together carry the + // header past it: no single length is refused, so it is the running + // check on the header's size that has to catch it. + let half = MAX_HEADER_BYTES / 2 + 1; + let file = Builder::new() + .u8_array_of_len("blob.a", half) + .u8_array_of_len("blob.b", half) + .file(0); + let message = malformed(parse(&file)); + assert!(message.contains("the header is larger than"), "{message}"); + let under = usize::try_from(MAX_HEADER_BYTES).unwrap() / 2; + parse(&Builder::new().u8_array_of_len("blob", under as u64).file(0)).unwrap(); + } + + // ----- signed integers ----------------------------------------------------- + + #[test] + fn signed_integers_are_read_as_the_figures_they_are() { + // gguf-py stores a Python integer as a signed 32-bit value, so this is the + // common shape of a real file, not a corner. + let file = Builder::new() + .string("general.architecture", "x") + .i32("x.block_count", 4) + .i32("x.embedding_length", 256) + .i32_array("x.attention.head_count", &[8, 8, 8, 8]) + .i32_array("x.attention.head_count_kv", &[0, 2, 0, 2]) + .file(0); + let header = parse(&file).unwrap(); + assert_eq!(header.block_count, Some(4)); + assert_eq!(header.embedding_length, Some(256)); + assert_eq!(header.head_count, Some(PerLayer::Each(vec![8; 4]))); + assert_eq!(header.head_count_kv, Some(PerLayer::Each(vec![0, 2, 0, 2]))); + } + + #[test] + fn a_negative_signed_value_reads_as_missing_not_as_a_huge_one() { + let file = Builder::new() + .string("general.architecture", "x") + .i32("x.block_count", -1) + .i32_array("x.attention.head_count_kv", &[2, -2, 2]) + .file(0); + let header = parse(&file).unwrap(); + assert_eq!(header.block_count, None); + // One bad entry spoils the list: it is not a list with a hole in it. + assert_eq!(header.head_count_kv, None); + // The same value as an unsigned one is a large number, not a negative. + let file = Builder::new() + .string("general.architecture", "x") + .u32("x.embedding_length", u32::MAX) + .file(0); + assert_eq!( + parse(&file).unwrap().embedding_length, + Some(u64::from(u32::MAX)) + ); + } + + #[test] + fn signed_values_of_every_width_decode_by_their_own_sign_bit() { + let read = |value_type: u32, bytes: &[u8]| { + parse( + &Builder::new() + .string("general.architecture", "x") + .raw("x.embedding_length", value_type, bytes) + .file(0), + ) + .unwrap() + .embedding_length + }; + assert_eq!(read(1, &[0x7f]), Some(127)); + assert_eq!(read(1, &[0xff]), None); + assert_eq!(read(3, &[0xff, 0x7f]), Some(32767)); + assert_eq!(read(3, &[0x00, 0x80]), None); + assert_eq!(read(5, &[0xff, 0xff, 0xff, 0x7f]), Some(2_147_483_647)); + assert_eq!(read(5, &[0x00, 0x00, 0x00, 0x80]), None); + assert_eq!(read(11, &7_i64.to_le_bytes()), Some(7)); + assert_eq!(read(11, &(-7_i64).to_le_bytes()), None); + // The unsigned types of the same widths read the high bit as a value. + assert_eq!(read(0, &[0xff]), Some(255)); + assert_eq!(read(2, &[0xff, 0xff]), Some(65535)); + } + + /// Every fixed-width value type, as a scalar and as an array element, under a + /// key the parser skips and under one it keeps, each followed by a kept + /// sentinel. The widths are written here from the GGUF specification, not + /// from [`fixed_width`]: a wrong width on any row would desync every later + /// key, so the sentinel would be misread or the parse would fail. + #[test] + fn every_fixed_width_type_is_read_at_its_own_width_as_a_scalar_and_in_an_array() { + // (type id, width in bytes, an integer type the parser keeps a value of) + let types: [(u32, usize, bool); 11] = [ + (0, 1, true), // uint8 + (1, 1, true), // int8 + (2, 2, true), // uint16 + (3, 2, true), // int16 + (4, 4, true), // uint32 + (5, 4, true), // int32 + (6, 4, false), // float32 + (7, 1, true), // bool + (10, 8, true), // uint64 + (11, 8, true), // int64 + (12, 8, false), // float64 + ]; + let le = |value: u64, width: usize| value.to_le_bytes()[..width].to_vec(); + let array = |type_id: u32, width: usize, values: &[u64]| { + let mut bytes = type_id.to_le_bytes().to_vec(); + bytes.extend_from_slice(&(values.len() as u64).to_le_bytes()); + for value in values { + bytes.extend(le(*value, width)); + } + bytes + }; + let parsed = |entry: Builder| { + parse(&entry.u32("x.block_count", 42).file(0)).unwrap_or_else(|error| panic!("{error}")) + }; + let base = || Builder::new().string("general.architecture", "x"); + for (type_id, width, integer) in types { + let skipped_scalar = parsed(base().raw("x.unrelated", type_id, &le(1, width))); + assert_eq!( + skipped_scalar.block_count, + Some(42), + "scalar of type {type_id}" + ); + let skipped_array = parsed(base().raw( + "x.unrelated", + test_builder::ARRAY, + &array(type_id, width, &[1, 0, 1]), + )); + assert_eq!( + skipped_array.block_count, + Some(42), + "array of type {type_id}" + ); + + let kept_scalar = parsed(base().raw("x.feed_forward_length", type_id, &le(1, width))); + assert_eq!( + kept_scalar.block_count, + Some(42), + "kept scalar of type {type_id}" + ); + let kept_array = parsed(base().raw( + "x.feed_forward_length", + test_builder::ARRAY, + &array(type_id, width, &[0, 1, 0]), + )); + assert_eq!( + kept_array.block_count, + Some(42), + "kept array of type {type_id}" + ); + if integer { + assert_eq!(kept_scalar.feed_forward_length, Some(1), "type {type_id}"); + assert_eq!(kept_array.feed_forward_length, Some(1), "type {type_id}"); + } else { + // A float is never read as an integer from its bits. + assert_eq!(kept_scalar.feed_forward_length, None, "type {type_id}"); + assert_eq!(kept_array.feed_forward_length, None, "type {type_id}"); + } + } + } + + #[test] + fn an_expert_count_above_one_marks_a_mixture_of_experts_model() { + let base = |experts| { + parse( + &Builder::new() + .string("general.architecture", "x") + .u32("x.expert_count", experts) + .file(0), + ) + .unwrap() + .moe + }; + assert!(base(8)); + assert!(!base(1)); + assert!(!base(0)); + } + + // ----- guards inside arrays and on what is kept ----------------------------- + + /// A reader that yields `prefix` and then zeros up to `len` bytes, so a + /// header far larger than any test should allocate can be walked by a parser + /// that does not stop itself. + struct ZerosAfter { + prefix: Vec, + len: u64, + pos: u64, + } + + impl Read for ZerosAfter { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let mut written = 0; + while written < out.len() && self.pos < self.len { + out[written] = usize::try_from(self.pos) + .ok() + .and_then(|at| self.prefix.get(at).copied()) + .unwrap_or(0); + written += 1; + self.pos += 1; + } + Ok(written) + } + } + + impl Seek for ZerosAfter { + fn seek(&mut self, to: SeekFrom) -> io::Result { + let next = match to { + SeekFrom::Start(at) => i128::from(at), + SeekFrom::Current(by) => i128::from(self.pos) + i128::from(by), + SeekFrom::End(by) => i128::from(self.len) + i128::from(by), + }; + self.pos = u64::try_from(next) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "seek before start"))?; + Ok(self.pos) + } + } + + /// A header whose only metadata entry is an array of `count` elements of + /// `element_type`, followed by an endless run of zeros that stands for the + /// elements, `total_len` bytes in all. + fn endless_array(element_type: u32, count: u64, total_len: u64) -> ZerosAfter { + let mut prefix = Vec::new(); + prefix.extend_from_slice(b"GGUF"); + prefix.extend_from_slice(&3_u32.to_le_bytes()); + prefix.extend_from_slice(&0_u64.to_le_bytes()); + prefix.extend_from_slice(&1_u64.to_le_bytes()); + prefix.extend_from_slice(&1_u64.to_le_bytes()); + prefix.push(b'k'); + prefix.extend_from_slice(&test_builder::ARRAY.to_le_bytes()); + prefix.extend_from_slice(&element_type.to_le_bytes()); + prefix.extend_from_slice(&count.to_le_bytes()); + ZerosAfter { + prefix, + len: total_len, + pos: 0, + } + } + + #[test] + fn a_string_array_that_walks_past_the_size_bound_is_refused_midway() { + // 20 million empty strings are 160 MiB of eight-byte lengths: well past the + // bound, and a count a real tokenizer never approaches. The check inside + // the loop is what stops it; the one after the value would be too late + // for a file this long. + let mut source = endless_array(test_builder::STRING, 20_000_000, 400 << 20); + let error = parse_gguf_header(&mut source).unwrap_err(); + assert!( + matches!(&error, GgufError::Malformed(why) if why.contains("is larger than")), + "{error:?}" + ); + // The parser stopped near the bound rather than reading to the end. + assert!( + source.pos <= MAX_HEADER_BYTES + 1024 * 1024, + "{}", + source.pos + ); + } + + #[test] + fn an_array_of_arrays_that_walks_past_the_size_bound_is_refused_midway() { + // Each inner array is a 12-byte header of its own: eight million of them + // are 96 MiB. + let mut source = endless_array(test_builder::ARRAY, 8_000_000, 400 << 20); + let error = parse_gguf_header(&mut source).unwrap_err(); + assert!( + matches!(&error, GgufError::Malformed(why) if why.contains("is larger than")), + "{error:?}" + ); + assert!( + source.pos <= MAX_HEADER_BYTES + 1024 * 1024, + "{}", + source.pos + ); + } + + /// A reader that yields `prefix` and then `record` over and over, up to `len` + /// bytes: a tensor table far longer than any test should allocate. + struct RepeatAfter { + prefix: Vec, + record: Vec, + len: u64, + pos: u64, + } + + impl Read for RepeatAfter { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let mut written = 0; + while written < out.len() && self.pos < self.len { + let at = usize::try_from(self.pos).unwrap(); + out[written] = + self.prefix.get(at).copied().unwrap_or_else(|| { + self.record[(at - self.prefix.len()) % self.record.len()] + }); + written += 1; + self.pos += 1; + } + Ok(written) + } + } + + impl Seek for RepeatAfter { + fn seek(&mut self, to: SeekFrom) -> io::Result { + let next = match to { + SeekFrom::Start(at) => i128::from(at), + SeekFrom::Current(by) => i128::from(self.pos) + i128::from(by), + SeekFrom::End(by) => i128::from(self.len) + i128::from(by), + }; + self.pos = u64::try_from(next) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "seek before start"))?; + Ok(self.pos) + } + } + + #[test] + fn a_tensor_table_that_walks_past_the_size_bound_is_refused_midway() { + // The most tensors a header may declare (a count the entry bound allows), + // each a 152-byte record with a long name and eight dimensions: about + // 160 MiB, well past the bound. Only the check inside the tensor loop + // stops it; the metadata loop's checks never see these records. + let mut prefix = Vec::new(); + prefix.extend_from_slice(b"GGUF"); + prefix.extend_from_slice(&3_u32.to_le_bytes()); + prefix.extend_from_slice(&MAX_ENTRIES.to_le_bytes()); + prefix.extend_from_slice(&0_u64.to_le_bytes()); + let mut record = Vec::new(); + record.extend_from_slice(&64_u64.to_le_bytes()); + record.extend_from_slice(&[b'w'; 64]); + record.extend_from_slice(&8_u32.to_le_bytes()); + for _ in 0..8 { + record.extend_from_slice(&1_u64.to_le_bytes()); + } + record.extend_from_slice(&0_u32.to_le_bytes()); + record.extend_from_slice(&0_u64.to_le_bytes()); + assert_eq!(record.len(), 152); + let mut source = RepeatAfter { + prefix, + record, + len: 400 << 20, + pos: 0, + }; + let error = parse_gguf_header(&mut source).unwrap_err(); + assert!( + matches!(&error, GgufError::Malformed(why) if why.contains("is larger than")), + "{error:?}" + ); + assert!( + source.pos <= MAX_HEADER_BYTES + 1024 * 1024, + "{}", + source.pos + ); + } + + #[test] + fn an_integer_array_longer_than_the_kept_bound_is_skipped_not_stored() { + let at = |len: usize| { + parse( + &Builder::new() + .string("general.architecture", "x") + .u32_array("x.attention.head_count_kv", &vec![2; len]) + .file(0), + ) + .unwrap() + .head_count_kv + }; + let kept = usize::try_from(MAX_KEPT_ARRAY).unwrap(); + assert_eq!(at(kept), Some(PerLayer::Each(vec![2; kept]))); + // One more and it is not a per-layer list of any model: the parse goes on + // past it, with nothing stored. + assert_eq!(at(kept + 1), None); + } + + #[test] + fn a_string_longer_than_the_kept_bound_is_skipped_not_stored() { + let at = |len: usize| { + parse( + &Builder::new() + .string("general.architecture", &"a".repeat(len)) + .tensor("w", &[4]) + .file(0), + ) + .unwrap() + }; + let kept = usize::try_from(MAX_KEPT_STRING).unwrap(); + assert_eq!( + at(kept).architecture.as_deref(), + Some("a".repeat(kept).as_str()) + ); + let over = at(kept + 1); + assert_eq!(over.architecture, None); + // What follows the long string was still read: the skip landed on it. + assert_eq!(over.param_count, 4); + } +} diff --git a/crates/rocm-core/src/gguf_fit.rs b/crates/rocm-core/src/gguf_fit.rs new file mode 100644 index 0000000000..1a97dd0e64 --- /dev/null +++ b/crates/rocm-core/src/gguf_fit.rs @@ -0,0 +1,3739 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Will this GGUF model fit in GPU memory at this context size? +//! +//! Weights plus KV cache plus a compute buffer plus a runtime reserve, compared +//! against the dedicated VRAM the readiness check already reads. Pure +//! arithmetic over a [`GgufHeader`]: nothing here reads a file or the network. +//! +//! The KV-cache formula is exact for a plain full-attention model (it is how +//! llama.cpp lays the cache out), and the weights are the file's own size from +//! its tensor offsets, less the input embedding when the file has its own output +//! layer, which is where llama.cpp keeps it: in host memory. A model with tied +//! embeddings (no `output.weight`) uses the same tensor as its output layer's +//! weight, and that layer is on the GPU, so there the embedding is counted. That +//! case is **not measured**: both calibration models have their own output +//! layer, so it is the side that can only overstate. +//! Only two allowances are **calibrated against real loads on one GPU and one +//! engine build**: the compute buffer and the runtime overhead. The loads were on +//! an MI300X (gfx942) with llama.cpp b9752 on ROCm 7.2, flash attention on, two +//! dense full-attention models, each with its own output layer (a 32-layer +//! Llama-3.1-8B-shaped Q4_K_M and a 28-layer Qwen2.5-1.5B-shaped Q8_0), each at +//! several context sizes and KV-cache types (fifteen loads), and one more load of +//! the Llama-shaped model made with the card almost full. In fifteen of the +//! sixteen the total came out *above* what the GPU's used memory grew by: by +//! 0.05% to 2.9% with an f16 KV cache and up to 4.5% with a quantized one, where +//! the dequantization allowance is deliberately generous. In the sixteenth it was +//! 5.5% above, because the process overhead measured there was about 0.4 GiB +//! against 1.05-1.28 GiB in the others, for a reason that is not known. Across +//! all sixteen it was 0.05% to 5.5% above and never below. That is the same data +//! the allowances were fitted to, so it measures the fit, not a guarantee: +//! another engine build, another GPU or a different batch size can miss by more. +//! That is why the answer is labelled an *estimate* everywhere it is shown. Where +//! the figure can be too low -- a mixture-of-experts model, whose expert compute +//! buffers were not measured; a state-space model, whose recurrent state is not +//! counted; a header without the widths the compute buffer is sized from -- the +//! figures are given with no fits / does-not-fit verdict. Layouts the formula +//! does not describe (latent attention, split files, models with no attention +//! layers) get no figures either: all of these are reported as +//! [`EstimateOutcome::CannotEstimate`] rather than guessed at. +//! +//! This is the one place the figures above -- which loads, and how close the +//! estimate came to them -- are written down. The README, `docs/testing.md` and +//! `docs/manual-testing.md` point here for them. The GPU and engine build are +//! also named, without figures, in [`ESTIMATE_CAVEAT`], the README and the +//! calibration tests' comments, so a recalibration on other hardware changes +//! those names as well as this paragraph, the constants below and the tests +//! under "Calibration against measured loads". + +use crate::gguf::{GgufHeader, MAX_BLOCK_COUNT}; +use crate::gguf_source::ProjectorNamed; +use crate::model_readiness::{AcceleratorMemory, format_gib}; +use serde::{Deserialize, Serialize}; +use std::fmt; +#[cfg(test)] +use std::str::FromStr; + +const GIB: f64 = 1024.0 * 1024.0 * 1024.0; + +/// Memory the engine and the GPU runtime hold beyond the weights, the KV cache +/// and the compute buffer (context objects, loaded kernels, graph metadata). +/// Measured at 1050-1101 MiB for the Llama-shaped model and 1228-1277 MiB for the +/// Qwen2-shaped one on the build named in the module docs; 1.25 GiB covers both. +const RUNTIME_RESERVE_BYTES: f64 = 1280.0 * 1024.0 * 1024.0; + +/// llama.cpp's default physical batch size, which sizes its compute buffers. +const COMPUTE_BATCH_TOKENS: f64 = 512.0; + +/// Bytes of the attention mask per token of context and batch token: a 16-bit +/// float each. +const MASK_BYTES: f64 = 2.0; + +/// The largest context size an estimate is made for: 2^24 tokens. +/// +/// Far beyond any model's, and small enough that no figure derived from it can +/// overflow. The CLI refuses a larger request outright, so the report never shows +/// a number a context that size would produce. +pub const MAX_CONTEXT_SIZE: u64 = 1 << 24; + +/// No host has this many bytes of memory, so an estimate that comes to more is +/// not a model any host could load: a corrupt header's head counts or widths. +const MAX_PLAUSIBLE_BYTES: f64 = (1_u64 << 60) as f64; + +const MIB: u64 = 1024 * 1024; + +/// Reported context sizes are rounded down to a multiple of this, so a figure +/// is a number a user would type rather than the exact byte boundary. +const CONTEXT_STEP: u64 = 256; + +/// The range of bits per weight a real GGUF falls in: a 1-bit-ish quantization +/// at the bottom, 32-bit floats at the top. +const MIN_PLAUSIBLE_BITS_PER_WEIGHT: f64 = 1.0; +const MAX_PLAUSIBLE_BITS_PER_WEIGHT: f64 = 33.0; + +/// A smaller quantization is only suggested when it is meaningfully smaller than +/// the current one; two files within this factor of each other are the same +/// quantization for this purpose. +const QUANT_SMALLER_FACTOR: f64 = 0.95; + +/// Typical bits per weight of common llama.cpp quantizations, averaged over a +/// whole model. Approximate: the real figure depends on the model's tensor mix. +/// The K-quant and Q8_0 rows are llama.cpp's `quantize` table sizes for +/// LLaMA-v1-7B (in GiB) converted to bits over its 6 738 415 616 weights; a +/// test holds them to that. +const QUANT_BITS_PER_WEIGHT: [(&str, f64); 7] = [ + ("Q2_K", 3.35), + ("Q3_K_M", 3.91), + ("IQ4_XS", 4.25), + ("Q4_K_M", 4.85), + ("Q5_K_M", 5.69), + ("Q6_K", 6.56), + ("Q8_0", 8.5), +]; + +/// The element type of the KV cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum KvCacheType { + #[serde(rename = "f16")] + F16, + #[serde(rename = "q8_0")] + Q8_0, + #[serde(rename = "q4_0")] + Q4_0, +} + +impl KvCacheType { + /// Every type, widest first. + pub const ALL: [Self; 3] = [Self::F16, Self::Q8_0, Self::Q4_0]; + + /// Bytes per cached element. The quantized types store a 16-bit scale per + /// block of 32 values: 34 and 18 bytes per block. + #[must_use] + pub const fn bytes_per_element(self) -> f64 { + match self { + Self::F16 => 2.0, + Self::Q8_0 => 34.0 / 32.0, + Self::Q4_0 => 18.0 / 32.0, + } + } + + #[must_use] + pub const fn name(self) -> &'static str { + match self { + Self::F16 => "f16", + Self::Q8_0 => "q8_0", + Self::Q4_0 => "q4_0", + } + } +} + +impl fmt::Display for KvCacheType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +// The CLI parses the flag with clap (`KvCacheTypeArg`); this is for tests +// that read a type back out of the report text. +#[cfg(test)] +impl FromStr for KvCacheType { + type Err = String; + + fn from_str(value: &str) -> Result { + Self::ALL + .into_iter() + .find(|kind| kind.name() == value) + .ok_or_else(|| format!("`{value}` is not a KV cache type (f16, q8_0 or q4_0)")) + } +} + +/// What the estimate concluded. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EstimateOutcome { + /// The estimated total is within the GPU's dedicated memory. The total + /// compared is the sum of the parts as the report shows them, each in whole + /// MiB rounded up, so it can be a few MiB above `total_bytes`. + Fits, + /// The estimated total exceeds the GPU's dedicated memory. + DoesNotFit, + /// No comparison could be made. See [`ContextEstimate::cannot_estimate_reason`]. + CannotEstimate, +} + +/// Why no comparison could be made. +/// +/// The first group is about the file, the second about the host; the figures +/// the file allows are still reported in the second. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CannotEstimateReason { + /// The GGUF header could not be fetched or read. + HeaderUnreadable, + /// The file named is a multimodal projector (`mmproj`), not a language + /// model, so there is no model to size. + NotAModel, + /// The file is one shard of a split model; the metadata and the weights are + /// spread over several files. + SplitFile, + /// Latent attention (the KV cache is not heads times head width). + LatentAttention, + /// No layer keeps a KV cache. + NoAttentionLayers, + /// State-space (pure or hybrid) or mixture-of-experts layers: the figures + /// are given, but part of the memory is either not counted (a recurrent + /// state) or not measured (expert compute buffers), so the total can be too + /// low and no fits / does-not-fit verdict is given. + UnmeasuredLayout, + /// A figure the formula needs is not in the header, or is there but not + /// plausible (a layer count past the parser's bound, head counts or widths + /// whose total is past any real model). When only the + /// feed-forward or embedding width is missing, the figures are still given + /// (with that width counted as zero, so they can be too low) and no + /// fits / does-not-fit verdict is. + MissingMetadata, + /// No GPU is visible to ROCm. + NoGpu, + /// The GPU's memory could not be measured. + AcceleratorMemoryUnknown, + /// An APU: the pool the engine allocates from is not readable. + UnifiedMemoryUnreadable, +} + +/// What kind of change a [`FitSuggestion`] is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SuggestionKind { + SmallerContext, + KvCacheType, + SmallerQuant, + CpuOffload, +} + +/// One thing that would make a model that does not fit, fit. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FitSuggestion { + pub kind: SuggestionKind, + pub text: String, +} + +/// The estimate for one model at one context size. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ContextEstimate { + /// What the header was read from: a path, or `owner/repo` and file name. + pub source: String, + pub outcome: EstimateOutcome, + pub cannot_estimate_reason: Option, + /// The specific cause, in words, when `cannot_estimate_reason` is set. + pub detail: Option, + pub architecture: Option, + pub layers: Option, + /// The context size the figures below are for. + pub context_size: u64, + /// Set when the user did not ask for a context size and this is the default. + pub context_size_defaulted: bool, + pub kv_cache_type: KvCacheType, + pub weights_gib: Option, + pub kv_cache_gib: Option, + pub compute_buffer_gib: Option, + pub runtime_reserve_gib: Option, + pub total_gib: Option, + /// The same figures in bytes (rounded to the nearest byte), for comparing each + /// component against a measured load without unit conversion. The `*_gib` + /// fields above are these divided by 2^30. + pub weights_bytes: Option, + pub kv_cache_bytes: Option, + pub compute_buffer_bytes: Option, + pub runtime_reserve_bytes: Option, + pub total_bytes: Option, + /// KV cache bytes added by each token of context at `kv_cache_type`. + pub kv_cache_bytes_per_token: Option, + /// Dedicated VRAM, when it was measured. + pub available_gib: Option, + /// The largest context that fits at `kv_cache_type`: a multiple of 256, or + /// the context the model was trained for when that fits and is the limit. + /// `None` when not even one 256-token step fits, or when nothing was + /// compared (the GPU's memory was not measured). It can sit below + /// `context_size` for an outcome of `fits`: a request smaller than one step, + /// or beyond the trained context, fits without being a step the ceiling + /// counts. + pub max_context: Option, + /// The context length the model was trained for. + pub training_context: Option, + /// Set for a sliding-window model, whose KV cache is sized as if every layer + /// attended over the whole context: `max_context` is then a conservative + /// figure, not the exact boundary. + #[serde(default)] + pub conservative_ceiling: bool, + /// What would make it fit; empty unless the outcome is `does_not_fit`. + pub suggestions: Vec, + pub notes: Vec, +} + +/// What was asked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FitRequest { + pub context_size: u64, + pub context_size_defaulted: bool, + pub kv_cache_type: KvCacheType, +} + +impl ContextEstimate { + /// An estimate that could not be made because opening the file failed. + /// Carries no figure: nothing was learnt about the model. + /// + /// Usually the header could not be reached or read + /// ([`CannotEstimateReason::HeaderUnreadable`]). A name refused as a + /// multimodal projector ([`ProjectorNamed`]) is + /// [`CannotEstimateReason::NotAModel`] instead: nothing failed to be read, the + /// file is not a model. + #[must_use] + pub fn unreadable(source: &str, request: &FitRequest, error: &anyhow::Error) -> Self { + let (reason, detail) = match error.downcast_ref::() { + Some(refusal) => (CannotEstimateReason::NotAModel, refusal.to_string()), + None => ( + CannotEstimateReason::HeaderUnreadable, + format!("the GGUF header could not be read: {error:#}"), + ), + }; + Self { + source: source.to_owned(), + outcome: EstimateOutcome::CannotEstimate, + cannot_estimate_reason: Some(reason), + detail: Some(detail), + architecture: None, + layers: None, + context_size: request.context_size, + context_size_defaulted: request.context_size_defaulted, + kv_cache_type: request.kv_cache_type, + weights_gib: None, + kv_cache_gib: None, + compute_buffer_gib: None, + runtime_reserve_gib: None, + total_gib: None, + weights_bytes: None, + kv_cache_bytes: None, + compute_buffer_bytes: None, + runtime_reserve_bytes: None, + total_bytes: None, + kv_cache_bytes_per_token: None, + available_gib: None, + max_context: None, + training_context: None, + conservative_ceiling: false, + suggestions: Vec::new(), + notes: Vec::new(), + } + } +} + +/// The pieces of the total that do not depend on the context size, plus the +/// KV-cache cost per token. +struct Sizing { + /// Everything after the header: what a bits-per-weight figure is over. + data: f64, + /// The part of `data` that lands in VRAM. llama.cpp keeps the input + /// embedding (`token_embd.weight`) in host memory when the file has its own + /// `output.weight`, and then it is left out; with tied embeddings it is the + /// output layer's weight on the GPU, and `weights` is all of `data`. + weights: f64, + /// The compute buffer's size before anything that grows with the context. + compute_base: f64, + /// KV elements cached per token across all layers. + kv_elements_per_token: f64, + /// KV elements per token in the widest single layer. + widest_layer_kv_elements: f64, + layers: u64, +} + +impl Sizing { + fn kv_bytes(&self, context: u64, kv: KvCacheType) -> f64 { + self.kv_elements_per_token * context as f64 * kv.bytes_per_element() + } + + /// What one more token of context costs in the compute buffer. The attention + /// mask grows with it. A quantized KV cache is also dequantized to 16-bit + /// floats a layer at a time, which the measured loads showed adding up to + /// that layer's K and V width per token; where the fixed part is larger than + /// that it hides it, so this errs high for a small model. + fn compute_bytes_per_token(&self, kv: KvCacheType) -> f64 { + let mask = COMPUTE_BATCH_TOKENS * MASK_BYTES; + if kv == KvCacheType::F16 { + mask + } else { + self.widest_layer_kv_elements.mul_add(2.0, mask) + } + } + + fn compute_bytes(&self, context: u64, kv: KvCacheType) -> f64 { + self.compute_bytes_per_token(kv) + .mul_add(context as f64, self.compute_base) + } + + /// Bytes that do not depend on the context size. + fn fixed(&self) -> f64 { + self.weights + self.compute_base + RUNTIME_RESERVE_BYTES + } + + /// Bytes each token of context adds: its KV cache and its compute buffer. + fn bytes_per_token(&self, kv: KvCacheType) -> f64 { + self.kv_elements_per_token + .mul_add(kv.bytes_per_element(), self.compute_bytes_per_token(kv)) + } + + fn total(&self, context: u64, kv: KvCacheType) -> f64 { + self.bytes_per_token(kv) + .mul_add(context as f64, self.fixed()) + } + + /// The total the report shows for `weights` of weights at `context` and + /// `kv`: its four parts, each as printed (whole MiB, rounded up), summed. + /// + /// This is the one figure a fit is decided on. The verdict, the largest + /// context and every "which fits" suggestion go through [`Self::fits`], so + /// none of them can disagree about whether a given context fits. + fn shown_total_with(&self, weights: f64, context: u64, kv: KvCacheType) -> f64 { + shown_sum([ + weights, + self.kv_bytes(context, kv), + self.compute_bytes(context, kv), + RUNTIME_RESERVE_BYTES, + ]) + } + + fn shown_total(&self, context: u64, kv: KvCacheType) -> f64 { + self.shown_total_with(self.weights, context, kv) + } + + fn fits(&self, context: u64, kv: KvCacheType, available: f64) -> bool { + self.shown_total(context, kv) <= available + } + + /// Whether the GPU holds `on_gpu` of the layers at `context` and `kv`, by the + /// same measure as [`Self::fits`]: the layers' share of the weights and the + /// cache, plus the compute buffer and the reserve, as the report would show + /// them. At every layer this is `fits`. + fn layers_fit(&self, on_gpu: u64, context: u64, kv: KvCacheType, available: f64) -> bool { + let share = on_gpu as f64 / self.layers as f64; + shown_sum([ + self.weights * share, + self.kv_bytes(context, kv) * share, + self.compute_bytes(context, kv), + RUNTIME_RESERVE_BYTES, + ]) <= available + } + + /// The part of a `bits_per_weight` quantization of `params` weights that + /// lands in VRAM: the same share of the file as now, because whether the + /// input embedding stays in host memory (it does unless the embeddings are + /// tied) does not change with the quantization. + fn quant_weights(&self, params: f64, bits_per_weight: f64) -> f64 { + params * bits_per_weight / 8.0 * (self.weights / self.data) + } + + /// The largest context that fits in `available` bytes at `kv`, in steps of + /// [`CONTEXT_STEP`] and no larger than `training` (the trained context, as + /// [`trained_context`] gives it, so never zero): the trained context itself + /// where it fits, else the largest step for which [`Self::fits`] holds. + fn max_context(&self, available: f64, kv: KvCacheType, training: Option) -> Option { + let room = available - self.fixed(); + if room <= 0.0 { + return None; + } + let per_token = self.bytes_per_token(kv); + let raw = (room / per_token).floor(); + // `as` saturates; a context beyond u64 is a model with no KV cost, which + // `sizing` has already refused. + let cap = training; + // Where the estimate reaches the cap, the search starts at the cap + // itself, whole step or not: rounding first would start a cap of 1500 + // at 1280, and the climb below never passes the cap to reach it. + let mut context = cap + .filter(|cap| raw as u64 >= *cap) + .unwrap_or((raw as u64) / CONTEXT_STEP * CONTEXT_STEP); + // The estimate above is on the exact total; what is decided on is the + // shown one, a few MiB higher. Step down until it fits, then up while the + // next step still does, so the answer is exactly the predicate's. + // A cap that is not a whole number of steps (a trained context of 250, or + // of 600) is where this starts, so the first step down lands on a whole + // step and a start below one step goes to zero rather than below it. + while context > 0 && !self.fits(context, kv, available) { + context = if context.is_multiple_of(CONTEXT_STEP) { + context - CONTEXT_STEP + } else { + context / CONTEXT_STEP * CONTEXT_STEP + }; + } + while cap.is_none_or(|cap| context + CONTEXT_STEP <= cap) + && self.fits(context + CONTEXT_STEP, kv, available) + { + context += CONTEXT_STEP; + } + (context > 0).then_some(context) + } +} + +/// The context length the model was trained for, when the header says one. A +/// header that declares zero declares none: no model was trained for no tokens, +/// and a cap of zero would leave no context that fits and a note that every +/// request is "beyond the 0 tokens the model was trained for". +fn trained_context(header: &GgufHeader) -> Option { + header.context_length.filter(|trained| *trained > 0) +} + +fn cannot( + reason: CannotEstimateReason, + detail: impl Into, +) -> (CannotEstimateReason, String) { + (reason, detail.into()) +} + +/// Text from a stranger's header as it may be shown: every character that would +/// change how a terminal renders the text around it (a control, a +/// bidirectional override, a line separator) becomes `?`. The architecture name +/// is the one header string the report prints. +fn shown_text(text: &str) -> String { + text.chars() + .map(|c| { + if crate::terminal::is_control_or_format(c) + || crate::terminal::is_control_or_line_separator(c) + { + '?' + } else { + c + } + }) + .collect() +} + +/// Whether the header lacks the feed-forward or the embedding width, which size +/// the compute buffer's fixed part. A width of zero is no width: no real model +/// has one, and taking it at its word would understate the buffer the same way. +fn widths_missing(header: &GgufHeader) -> bool { + let known = |width: Option| width.is_some_and(|width| width > 0); + !known(header.feed_forward_length) || !known(header.embedding_length) +} + +/// Work out the sizing, or why it cannot be worked out. +fn sizing(header: &GgufHeader, file_size: u64) -> Result { + if header.split_count.is_some_and(|count| count > 1) { + return Err(cannot( + CannotEstimateReason::SplitFile, + "this file is one shard of a split model; its weights and metadata are spread over \ + several files, so one file's size is not the model's", + )); + } + if header.latent_attention { + return Err(cannot( + CannotEstimateReason::LatentAttention, + "this model uses latent attention, whose KV cache is not laid out as heads times \ + head width, so the estimate's formula does not describe it", + )); + } + let missing = |what: &str| { + cannot( + CannotEstimateReason::MissingMetadata, + format!("the GGUF header does not carry {what}"), + ) + }; + let layers = header + .block_count + .filter(|layers| *layers > 0) + .ok_or_else(|| missing("a layer count"))?; + // The sizing below loops once per layer. The parser refuses an implausible + // count; a header built some other way gets the same bound here, so no + // caller can turn a corrupt figure into a loop that runs until memory ends. + if layers > MAX_BLOCK_COUNT { + return Err(cannot( + CannotEstimateReason::MissingMetadata, + format!("the header's layer count ({layers}) is not a plausible model"), + )); + } + let data = file_size + .checked_sub(header.data_offset) + .filter(|w| *w > 0) + .ok_or_else(|| { + cannot( + CannotEstimateReason::MissingMetadata, + "the file is no larger than its header, so there are no weights to size", + ) + })?; + // The input embedding runs from its offset to the next tensor's, or to the + // end of the data when it is the last. Clamped so a header that disagrees + // with the file's size cannot make the weights negative. llama.cpp keeps it + // in host memory only when the file has its own output layer: with tied + // embeddings (no `output.weight`) the same tensor is the output layer's + // weight too, and is placed with that layer on the GPU, so it stays in. + let embedding = header + .token_embd + .filter(|_| header.has_output) + .map_or(0, |span| { + span.next_offset + .unwrap_or(data) + .min(data) + .saturating_sub(span.offset) + }); + let weights = data.saturating_sub(embedding); + + let mut kv_elements = 0_u64; + let mut widest_layer = 0_u64; + for layer in 0..usize::try_from(layers).unwrap_or(usize::MAX) { + let heads = header.head_count.as_ref().and_then(|p| p.at(layer)); + // llama.cpp reads an absent KV head count as "as many as query heads". + let kv_heads = match &header.head_count_kv { + Some(per_layer) => per_layer.at(layer), + None => heads, + } + .ok_or_else(|| missing("a KV head count for every layer"))?; + if kv_heads == 0 { + continue; + } + let width = |explicit: Option| -> Result { + if let Some(width) = explicit { + return Ok(width); + } + match (header.embedding_length, heads) { + (Some(embedding), Some(heads)) if heads > 0 => Ok(embedding / heads), + _ => Err(missing("the attention head width")), + } + }; + let per_head = width(header.key_length)?.saturating_add(width(header.value_length)?); + let layer_elements = kv_heads.saturating_mul(per_head); + widest_layer = widest_layer.max(layer_elements); + kv_elements = kv_elements.saturating_add(layer_elements); + } + if kv_elements == 0 { + return Err(cannot( + CannotEstimateReason::NoAttentionLayers, + "no layer of this model keeps a KV cache, so a context-size estimate has nothing to \ + size", + )); + } + + // The part of llama.cpp's compute buffer that does not grow with the + // context: three activations of the widest intermediate width and three of + // the embedding width per batch token, in 32-bit floats. Flash attention is + // assumed, so no attention-score matrix appears. This reproduced the measured + // buffer exactly for both calibration models. + let activations = 3.0_f64.mul_add( + header.feed_forward_length.unwrap_or(0) as f64, + 3.0 * header.embedding_length.unwrap_or(0) as f64, + ); + let compute_base = COMPUTE_BATCH_TOKENS * activations * 4.0; + Ok(Sizing { + data: data as f64, + weights: weights as f64, + compute_base, + kv_elements_per_token: kv_elements as f64, + widest_layer_kv_elements: widest_layer as f64, + layers, + }) +} + +/// A figure that has to cover a need, so it rounds up. +fn need(bytes: f64) -> String { + format!("{:.1} GiB", (bytes / GIB * 10.0).ceil() / 10.0) +} + +fn gib(bytes: f64) -> f64 { + bytes / GIB +} + +/// "1 token", "4096 tokens". +pub(crate) fn tokens(count: u64) -> String { + if count == 1 { + "1 token".to_owned() + } else { + format!("{count} tokens") + } +} + +/// A byte count in whole MiB, rounded up: a figure that has to cover a need. +const fn mib_up(bytes: u64) -> u64 { + bytes.div_ceil(MIB) +} + +/// The sum of four byte figures as the report shows them: each rounded to a +/// whole byte count as the byte fields are, then up to a whole MiB, and added. +fn shown_sum(parts: [f64; 4]) -> f64 { + let mib: u64 = parts.iter().map(|part| mib_up(part.round() as u64)).sum(); + (mib * MIB) as f64 +} + +/// The parts of an estimate as the report shows them -- each in whole MiB, +/// rounded up -- and their sum. The verdict is decided on that sum, so the +/// figures printed always add up and never contradict it. +fn shown_parts(estimate: &ContextEstimate) -> Option<([u64; 4], u64)> { + let parts = [ + mib_up(estimate.weights_bytes?), + mib_up(estimate.kv_cache_bytes?), + mib_up(estimate.compute_buffer_bytes?), + mib_up(estimate.runtime_reserve_bytes?), + ]; + Some((parts, parts.iter().sum())) +} + +/// Estimate whether the model described by `header` fits at `request`. +/// +/// `file_size` is the size of the whole file in bytes; the weights are what +/// follows the header. `memory` and `system_ram_gib` are the host's, as the +/// readiness check measured them. +#[must_use] +pub fn estimate_fit( + header: &GgufHeader, + file_size: u64, + source: &str, + request: &FitRequest, + memory: AcceleratorMemory, + system_ram_gib: Option, +) -> ContextEstimate { + let mut estimate = ContextEstimate { + source: source.to_owned(), + outcome: EstimateOutcome::CannotEstimate, + cannot_estimate_reason: None, + detail: None, + architecture: header.architecture.as_deref().map(shown_text), + layers: header.block_count, + context_size: request.context_size, + context_size_defaulted: request.context_size_defaulted, + kv_cache_type: request.kv_cache_type, + weights_gib: None, + kv_cache_gib: None, + compute_buffer_gib: None, + runtime_reserve_gib: None, + total_gib: None, + weights_bytes: None, + kv_cache_bytes: None, + compute_buffer_bytes: None, + runtime_reserve_bytes: None, + total_bytes: None, + kv_cache_bytes_per_token: None, + available_gib: memory.measured_gib(), + max_context: None, + training_context: trained_context(header), + conservative_ceiling: header.sliding_window, + suggestions: Vec::new(), + notes: Vec::new(), + }; + let sizing = match sizing(header, file_size) { + Ok(sizing) => sizing, + Err((reason, detail)) => { + estimate.cannot_estimate_reason = Some(reason); + estimate.detail = Some(detail); + return estimate; + } + }; + + let kv = request.kv_cache_type; + let total = sizing.total(request.context_size, kv); + if !total.is_finite() || total >= MAX_PLAUSIBLE_BYTES { + estimate.cannot_estimate_reason = Some(CannotEstimateReason::MissingMetadata); + estimate.detail = Some(format!( + "at {} the figures this header implies come to more than an exabyte, which no model \ + or host has: the header's head counts or widths look corrupt, or the context size \ + is not a real one (the most an estimate is made for is {MAX_CONTEXT_SIZE})", + tokens(request.context_size) + )); + return estimate; + } + estimate.weights_gib = Some(gib(sizing.weights)); + estimate.kv_cache_gib = Some(gib(sizing.kv_bytes(request.context_size, kv))); + let compute = sizing.compute_bytes(request.context_size, kv); + estimate.compute_buffer_gib = Some(gib(compute)); + estimate.runtime_reserve_gib = Some(gib(RUNTIME_RESERVE_BYTES)); + estimate.total_gib = Some(gib(total)); + let bytes = |value: f64| Some(value.round() as u64); + estimate.weights_bytes = bytes(sizing.weights); + estimate.kv_cache_bytes = bytes(sizing.kv_bytes(request.context_size, kv)); + estimate.compute_buffer_bytes = bytes(compute); + estimate.runtime_reserve_bytes = bytes(RUNTIME_RESERVE_BYTES); + estimate.total_bytes = bytes(total); + estimate.kv_cache_bytes_per_token = Some(sizing.kv_elements_per_token * kv.bytes_per_element()); + + if header.sliding_window { + estimate.notes.push( + "this model uses sliding-window attention; its KV cache is sized as if every layer \ + attended over the full context, so the real figure can be lower and the largest \ + context below is conservative" + .to_owned(), + ); + } + if header.ssm { + estimate.notes.push( + "this model has state-space layers: the recurrent state they keep is not counted, so \ + the real figure can be higher; its KV cache is sized for every layer unless the \ + header lists head counts per layer, so that part can be lower" + .to_owned(), + ); + } + if header.moe { + estimate.notes.push( + "this is a mixture-of-experts model: the weights are the file's exact size, but the \ + compute buffer is sized from one feed-forward width and was not measured for \ + experts, so the real figure can be higher" + .to_owned(), + ); + } + if widths_missing(header) { + estimate.notes.push( + "the header does not carry both the feed-forward and the embedding width, so the \ + compute buffer's fixed part is understated and the real figure can be higher" + .to_owned(), + ); + } + if let Some(trained) = trained_context(header) + && request.context_size > trained + { + estimate.notes.push(format!( + "{} is beyond the {} the model was trained for", + tokens(request.context_size), + tokens(trained) + )); + } + + let Some(available_gib) = memory.measured_gib() else { + let (reason, detail) = match memory { + AcceleratorMemory::None => ( + CannotEstimateReason::NoGpu, + "no GPU is visible to ROCm, so there is no memory to compare against", + ), + AcceleratorMemory::UnifiedMemoryUnreadable => ( + CannotEstimateReason::UnifiedMemoryUnreadable, + "this host has no dedicated VRAM and the CLI cannot yet read the memory pool its \ + engine allocates from, so there is nothing to compare the figures against", + ), + AcceleratorMemory::Unknown | AcceleratorMemory::Dedicated(_) => ( + CannotEstimateReason::AcceleratorMemoryUnknown, + "the GPU memory on this host could not be read, so there is nothing to compare \ + the figures against", + ), + }; + estimate.cannot_estimate_reason = Some(reason); + estimate.detail = Some(detail.to_owned()); + return estimate; + }; + // A layout whose figure can be too low gets the figures and no verdict: a + // confident "fits" that is wrong costs a user a failed load, and a margin + // with no measurement behind it would be a guess. The host's memory is + // still reported, so the reader can compare it themselves. + let widths_missing = widths_missing(header); + if header.ssm || header.moe || widths_missing { + let why = match (header.ssm, header.moe) { + (true, true) => { + "it has state-space layers, whose recurrent state is not counted, and \ + mixture-of-experts layers, whose compute buffer was not measured" + } + (true, false) => "it has state-space layers, whose recurrent state is not counted", + (false, true) => { + "it is a mixture-of-experts model, whose compute buffer was not measured" + } + (false, false) => { + "the header does not carry both the feed-forward and the embedding width, \ + which size the compute buffer's fixed part" + } + }; + // A layout the formula does not measure is the stronger statement; a + // missing width only matters on its own, and its note says so either way. + estimate.cannot_estimate_reason = Some(if header.ssm || header.moe { + CannotEstimateReason::UnmeasuredLayout + } else { + CannotEstimateReason::MissingMetadata + }); + estimate.detail = Some(format!( + "no fits / does-not-fit verdict for this model: {why}, so the total above can be too \ + low; compare it with this GPU's {} MiB yourself and leave headroom", + whole_mib_offered(available_gib) + )); + return estimate; + } + let available = available_gib * GIB; + estimate.max_context = sizing.max_context(available, kv, trained_context(header)); + + // Decided on the sum of the parts as the report shows them (each rounded up + // to a whole MiB), which is at most a few MiB above the exact total: what is + // printed then always adds up, and can never sit next to a contradicting + // verdict. `max_context` and the suggestions use the same predicate. + if sizing.fits(request.context_size, kv, available) { + estimate.outcome = EstimateOutcome::Fits; + return estimate; + } + estimate.outcome = EstimateOutcome::DoesNotFit; + estimate.suggestions = suggestions( + header, + &sizing, + request, + available, + estimate.max_context, + system_ram_gib, + ); + estimate +} + +fn suggestions( + header: &GgufHeader, + sizing: &Sizing, + request: &FitRequest, + available: f64, + max_context: Option, + system_ram_gib: Option, +) -> Vec { + let kv = request.kv_cache_type; + let context = request.context_size; + let mut out = Vec::new(); + + if let Some(max) = max_context { + let qualifier = if header.sliding_window { + " (a conservative figure for this model: its sliding-window layers are sized at the \ + full context)" + } else { + "" + }; + out.push(FitSuggestion { + kind: SuggestionKind::SmallerContext, + text: format!( + "use a context of {} or less (llama-server --ctx-size {max}); that is the largest \ + that fits with the KV cache at {kv}{qualifier}", + tokens(max) + ), + }); + } + + // A narrower cache is only worth naming when it lowers the need. A quantized + // cache is also dequantized a layer at a time, which adds to the compute + // buffer; in a model with one or two cached layers that costs more than the + // cache saves, and "a q8_0 cache would cut the need to ..." would be false. + let now = sizing.shown_total(context, kv); + for smaller in KvCacheType::ALL.into_iter().filter(|other| { + other.bytes_per_element() < kv.bytes_per_element() + && sizing.shown_total(context, *other) < now + }) { + let total_mib = sizing.shown_total(context, smaller) as u64 / MIB; + let text = if sizing.fits(context, smaller, available) { + let largest = sizing + .max_context(available, smaller, trained_context(header)) + .map_or(String::new(), |max| { + format!("; the largest context then is {}", tokens(max)) + }); + format!( + "use a {smaller} KV cache (llama-server --cache-type-k {smaller} --cache-type-v \ + {smaller}): about {total_mib} MiB at {}, which fits{largest}", + tokens(context) + ) + } else { + format!( + "a {smaller} KV cache would cut the need to about {total_mib} MiB, which is still \ + more than this GPU offers" + ) + }; + out.push(FitSuggestion { + kind: SuggestionKind::KvCacheType, + text, + }); + } + if out + .iter() + .any(|suggestion| suggestion.kind == SuggestionKind::KvCacheType) + { + out.push(FitSuggestion { + kind: SuggestionKind::KvCacheType, + text: "a quantized V cache needs flash attention enabled in llama.cpp".to_owned(), + }); + } + + let params = header.param_count as f64; + let current_bpw = if header.param_count > 0 { + sizing.data * 8.0 / params + } else { + 0.0 + }; + // Bits per weight outside what any real quantization reaches means the + // header's tensor shapes and the file's size do not describe the same + // model (a truncated or padded file, or a fixture); a "smaller quant" + // computed from that would be noise. + if (MIN_PLAUSIBLE_BITS_PER_WEIGHT..=MAX_PLAUSIBLE_BITS_PER_WEIGHT).contains(¤t_bpw) { + let candidates: Vec<(&str, f64)> = QUANT_BITS_PER_WEIGHT + .iter() + .rev() + .filter(|(_, bpw)| *bpw < current_bpw * QUANT_SMALLER_FACTOR) + .map(|(label, bpw)| (*label, sizing.quant_weights(params, *bpw))) + .collect(); + let fitting = candidates + .iter() + .find(|(_, weights)| sizing.shown_total_with(*weights, context, kv) <= available); + let text = match (fitting, candidates.last()) { + (Some((label, weights)), _) => Some(format!( + "a smaller quantization of the same model, about {label} size (roughly {} of \ + weights against {} now), would fit at {} if the repository \ + publishes one", + need(*weights), + need(sizing.weights), + tokens(context) + )), + (None, Some((label, _))) => Some(format!( + "no smaller quantization in the usual range, down to about {label}, fits at \ + {} with the KV cache at {kv}", + tokens(context) + )), + (None, None) => None, + }; + if let Some(text) = text { + out.push(FitSuggestion { + kind: SuggestionKind::SmallerQuant, + text, + }); + } + } + + out.extend(offload_suggestion( + sizing, + request, + available, + system_ram_gib, + )); + out +} + +/// Partial offload: keep as many layers as fit on the GPU and run the rest from +/// system RAM. Weights are spread evenly over the layers, which ignores the +/// embedding and output tensors; the figure is a rough guide. +fn offload_suggestion( + sizing: &Sizing, + request: &FitRequest, + available: f64, + system_ram_gib: Option, +) -> Option { + let layers = sizing.layers; + let context = request.context_size; + let kv = request.kv_cache_type; + let per_layer = (sizing.weights + sizing.kv_bytes(context, kv)) / layers as f64; + let compute = sizing.compute_bytes(context, kv); + let gpu_fits = |n: u64| sizing.layers_fit(n, context, kv, available); + let room = available - compute - RUNTIME_RESERVE_BYTES; + let mut on_gpu = if room > 0.0 { + ((room / per_layer).floor() as u64).min(layers) + } else { + 0 + }; + while on_gpu > 0 && !gpu_fits(on_gpu) { + on_gpu -= 1; + } + while on_gpu < layers && gpu_fits(on_gpu + 1) { + on_gpu += 1; + } + if on_gpu == 0 || on_gpu >= layers { + return None; + } + let host_bytes = per_layer * (layers - on_gpu) as f64; + if system_ram_gib.is_some_and(|ram| host_bytes > ram * GIB) { + return None; + } + let ram_note = if system_ram_gib.is_some() { + "" + } else { + " (system RAM could not be read, so that is unchecked)" + }; + Some(FitSuggestion { + kind: SuggestionKind::CpuOffload, + text: format!( + "keep about {on_gpu} of {layers} layers on the GPU (llama-server --n-gpu-layers \ + {on_gpu}); the other {} of weights and cache stay in system RAM{ram_note}, and \ + generation is slower", + need(host_bytes) + ), + }) +} + +/// A host's memory in whole MiB, rounded down: a host never reads as offering +/// more than it does, and an integer figure sits beside the integer figures in +/// the report. The `1e-6` keeps floating-point error in the GiB figure from +/// costing a whole MiB. Every line that shows the host's memory in MiB goes +/// through here, so two lines never show it differently. +const fn whole_mib_offered(gib: f64) -> f64 { + gib.mul_add(1024.0, 1e-6).floor() +} + +/// The evidence lines for an estimate, in the report's own voice. +#[must_use] +pub fn evidence_lines(estimate: &ContextEstimate) -> Vec { + let mut lines = Vec::new(); + // Where the figures came from is only worth saying when there was a header + // to read them from; for an unreadable one, or a file that is not a model, + // the detail says what happened. + if !matches!( + estimate.cannot_estimate_reason, + Some(CannotEstimateReason::HeaderUnreadable | CannotEstimateReason::NotAModel) + ) { + lines.push(format!( + "estimated from the GGUF header of {} ({} architecture, {} layers); the weights were \ + not downloaded", + estimate.source, + estimate.architecture.as_deref().unwrap_or("unknown"), + estimate + .layers + .map_or_else(|| "unknown".to_owned(), |layers| layers.to_string()), + )); + } + if let Some(detail) = &estimate.detail { + lines.push(detail.clone()); + } + if let Some(([weights, kv, compute, reserve], total)) = shown_parts(estimate) { + lines.push(format!( + "at {}{}: weights {weights} MiB + KV cache ({}) {kv} MiB + compute buffer {compute} \ + MiB + runtime reserve {reserve} MiB = about {total} MiB ({})", + tokens(estimate.context_size), + if estimate.context_size_defaulted { + " (the default; pass --context-size to change it)" + } else { + "" + }, + estimate.kv_cache_type, + need((total * MIB) as f64), + )); + } + if let Some(available) = estimate.available_gib { + let offered = whole_mib_offered(available); + lines.push(format!( + "the GPU with the most dedicated VRAM that ROCm can see has {offered} MiB ({}) in \ + total; memory other processes already hold is not subtracted", + format_gib(available) + )); + let conservative = if estimate.conservative_ceiling { + " (conservative for this model: its sliding-window layers are sized at the full \ + context)" + } else { + "" + }; + match (estimate.outcome, estimate.max_context) { + (EstimateOutcome::CannotEstimate, _) => {} + // A request that fits is never followed by a ceiling below it: that + // would read as the request not fitting. When the request is smaller + // than one step or beyond the trained context, the ceiling is left + // out and the trained-context note says what applies. + (EstimateOutcome::Fits, Some(max)) if max < estimate.context_size => {} + (EstimateOutcome::Fits, None) => {} + (_, Some(max)) => lines.push(format!( + "largest context that fits, with the KV cache at {}: {}{conservative}", + estimate.kv_cache_type, + tokens(max) + )), + (_, None) => lines.push(format!( + "not even a {CONTEXT_STEP}-token context fits with the KV cache at {}", + estimate.kv_cache_type + )), + } + } + lines +} + +/// The standing caveat that goes with every estimate that has figures. +/// +/// It names the GPU and engine build the allowances were calibrated on, because +/// that is what a reader of the output needs to judge whether their own setup is +/// like it, and printed output cannot link anywhere. How close the calibration +/// came is in the module documentation, and only there. +pub const ESTIMATE_CAVEAT: &str = "these are estimates from the file's metadata, with the compute buffer and runtime \ + overhead calibrated on one GPU and engine build (an MI300X running llama.cpp b9752) with \ + flash attention on, which is assumed here (without it llama.cpp also allocates an \ + attention-score matrix that is not counted, and the figure is too low); another build or GPU can differ, so leave some headroom \ + and check with a real run"; + +/// The standing note about `rocm serve` when there are suggestions. +/// +/// The options they name are the engine's, and `rocm serve` does not pass them +/// through today. `serve_takes_no_estimate_options_yet` in `apps/rocm/src/cli.rs` fails +/// when `rocm serve` gains one of them, so this text cannot go stale unnoticed. +pub const SERVE_OPTIONS_NOTE: &str = "`rocm serve` does not take a context size, KV cache type or \ + GPU layer count yet, and passes none to the engine, so a model it serves gets the engine's \ + own default context; the options named above are llama.cpp's own llama-server flags"; + +/// The same fact for an estimate with no suggestions: the context size it was +/// sized for is not one `rocm serve` would use. Pinned by the same test as +/// [`SERVE_OPTIONS_NOTE`]. +pub const SERVE_CONTEXT_NOTE: &str = "`rocm serve` does not take a context size yet and passes \ + none to the engine, so a model it serves gets the engine's own default context, which can \ + differ from the size estimated here"; + +#[cfg(test)] +mod tests { + use super::*; + use crate::gguf::test_builder::{Builder, llama_like}; + use crate::gguf::{GgufHeader, parse_gguf_header}; + use std::io::Cursor; + + const MIB: u64 = 1024 * 1024; + const GIB_U: u64 = 1024 * MIB; + + fn header_of(builder: &Builder, data: usize) -> (GgufHeader, u64) { + let file = builder.file(data); + let header = parse_gguf_header(&mut Cursor::new(&file)).unwrap(); + (header, file.len() as u64) + } + + /// 32 layers, 8 KV heads, key and value width 128 each: 32 * 8 * 256 = 65536 + /// elements per token, so 128 KiB per token at f16 and exactly 1 GiB at 8192. + /// + /// `data` is the size of the weights. The file is never built: the header is + /// parsed from a header-only file and the size is declared, so a test of a + /// multi-gigabyte model allocates nothing. + fn llama(data: usize) -> (GgufHeader, u64) { + let header = parse_gguf_header(&mut Cursor::new(llama_like(10).file(0))).unwrap(); + let size = header.data_offset + data as u64; + (header, size) + } + + fn request(context_size: u64, kv: KvCacheType) -> FitRequest { + FitRequest { + context_size, + context_size_defaulted: false, + kv_cache_type: kv, + } + } + + fn estimate( + header: &GgufHeader, + size: u64, + context: u64, + kv: KvCacheType, + memory: AcceleratorMemory, + ) -> ContextEstimate { + estimate_fit( + header, + size, + "test.gguf", + &request(context, kv), + memory, + Some(64.0), + ) + } + + #[test] + fn the_kv_cache_is_layers_times_kv_heads_times_head_width_times_context() { + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + let kv = result.kv_cache_gib.unwrap(); + assert!( + (kv - 1.0).abs() < 1e-9, + "32 layers x 8 kv heads x 256 x 8192 x 2 B = 1 GiB, got {kv}" + ); + // The quantized types are the block sizes, not a round factor. + let q8 = estimate( + &header, + size, + 8192, + KvCacheType::Q8_0, + AcceleratorMemory::Dedicated(64.0), + ); + assert!((q8.kv_cache_gib.unwrap() - 34.0 / 64.0).abs() < 1e-9); + let q4 = estimate( + &header, + size, + 8192, + KvCacheType::Q4_0, + AcceleratorMemory::Dedicated(64.0), + ); + assert!((q4.kv_cache_gib.unwrap() - 18.0 / 64.0).abs() < 1e-9); + } + + /// The vocabulary `llama_like(10)` declares puts its `token_embd.weight` at + /// the start of the data and the next tensor 10 * 4096 bytes later. + const EMBD_BYTES: u64 = 10 * 4096; + /// 512 batch tokens x (3 x ffn 14336 + 3 x embedding 4096) x 4 bytes: the + /// 108 MiB the Llama-shaped calibration model's compute buffer measured. + const LLAMA_COMPUTE_BASE: u64 = 512 * (3 * 14336 + 3 * 4096) * 4; + + #[test] + fn the_total_is_the_sum_of_its_parts() { + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + // The file has its own output layer, so the input embedding stays in + // host memory and is not VRAM. + let weights = (size - header.data_offset - EMBD_BYTES) as f64 / GIB; + // The attention mask adds 512 x 2 bytes per token of context. + let compute = (LLAMA_COMPUTE_BASE + 512 * 2 * 8192) as f64 / GIB; + assert!((result.weights_gib.unwrap() - weights).abs() < 1e-12); + assert!((result.compute_buffer_gib.unwrap() - compute).abs() < 1e-12); + assert!((result.runtime_reserve_gib.unwrap() - 1.25).abs() < 1e-12); + let expected = weights + 1.0 + compute + 1.25; + assert!((result.total_gib.unwrap() - expected).abs() < 1e-9); + } + + #[test] + fn the_byte_fields_are_exact_and_add_up_to_the_total() { + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + // 32 layers x 8 kv heads x (128 + 128) x 2 bytes = 131072 per token. + assert_eq!(result.kv_cache_bytes_per_token, Some(131_072.0)); + assert_eq!(result.kv_cache_bytes, Some(131_072 * 8192)); + assert_eq!( + result.weights_bytes, + Some(size - header.data_offset - EMBD_BYTES) + ); + assert_eq!(result.runtime_reserve_bytes, Some(1280 * MIB)); + assert_eq!( + result.compute_buffer_bytes, + Some(LLAMA_COMPUTE_BASE + 512 * 2 * 8192) + ); + let parts = result.weights_bytes.unwrap() + + result.kv_cache_bytes.unwrap() + + result.compute_buffer_bytes.unwrap() + + result.runtime_reserve_bytes.unwrap(); + assert_eq!(result.total_bytes, Some(parts)); + } + + #[test] + fn a_per_layer_head_array_skips_layers_without_a_cache() { + let builder = Builder::new() + .string("general.architecture", "hybrid") + .u32("hybrid.block_count", 4) + .u32("hybrid.embedding_length", 1024) + .u32("hybrid.attention.head_count", 8) + .u32_array("hybrid.attention.head_count_kv", &[0, 2, 0, 2]) + .tensor("t", &[1000]); + let (header, size) = header_of(&builder, 4096); + let result = estimate( + &header, + size, + 1000, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + // 2 layers x 2 kv heads x (128 + 128) x 1000 tokens x 2 bytes + let expected = 2.0 * 2.0 * 256.0 * 1000.0 * 2.0 / GIB; + assert!((result.kv_cache_gib.unwrap() - expected).abs() < 1e-12); + } + + #[test] + fn an_absent_kv_head_count_means_as_many_as_query_heads() { + let builder = Builder::new() + .string("general.architecture", "mha") + .u32("mha.block_count", 2) + .u32("mha.embedding_length", 512) + .u32("mha.attention.head_count", 4) + .tensor("t", &[10]); + let (header, size) = header_of(&builder, 64); + let result = estimate( + &header, + size, + 100, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + // 2 layers x 4 heads x (128 + 128) x 100 x 2 bytes + let expected = 2.0 * 4.0 * 256.0 * 100.0 * 2.0 / GIB; + assert!((result.kv_cache_gib.unwrap() - expected).abs() < 1e-12); + } + + #[test] + fn explicit_key_and_value_lengths_override_the_derived_head_width() { + let builder = Builder::new() + .string("general.architecture", "wide") + .u32("wide.block_count", 1) + .u32("wide.embedding_length", 512) + .u32("wide.attention.head_count", 4) + .u32("wide.attention.head_count_kv", 1) + .u32("wide.attention.key_length", 192) + .u32("wide.attention.value_length", 128) + .tensor("t", &[10]); + let (header, size) = header_of(&builder, 64); + let result = estimate( + &header, + size, + 100, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + let expected = (192.0 + 128.0) * 100.0 * 2.0 / GIB; + assert!((result.kv_cache_gib.unwrap() - expected).abs() < 1e-12); + } + + /// The verdict is decided on the total the report shows -- the sum of its + /// parts, each rounded up to a whole MiB -- so a GPU of exactly that many MiB + /// fits and one MiB less does not, and what is printed always agrees. + #[test] + fn fit_is_decided_at_the_total_the_report_shows() { + let (header, size) = llama(GIB_U as usize / 1024); + let reference = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + let (_, shown_mib) = shown_parts(&reference).unwrap(); + let mib_gib = |mib: u64| mib as f64 / 1024.0; + let exact = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(mib_gib(shown_mib)), + ); + assert_eq!(exact.outcome, EstimateOutcome::Fits); + let under = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(mib_gib(shown_mib - 1)), + ); + assert_eq!(under.outcome, EstimateOutcome::DoesNotFit); + // The exact total is never above what is shown, and never more than a few + // MiB below it (one rounding per part). + let exact_bytes = reference.total_bytes.unwrap(); + assert!(exact_bytes <= shown_mib * MIB, "{exact_bytes} {shown_mib}"); + assert!(shown_mib * MIB - exact_bytes < 4 * MIB); + // Memory that is above the exact total and below the shown one: the + // verdict follows the figure that is printed, never the unrounded one. + assert!( + shown_mib * MIB - exact_bytes >= 2, + "the total is a whole number of MiB" + ); + let between = (exact_bytes + shown_mib * MIB) as f64 / 2.0 / GIB; + let tight = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(between), + ); + assert_eq!(tight.outcome, EstimateOutcome::DoesNotFit); + } + + /// The figures printed are the figures decided on: whole MiB, summing to the + /// total, with the host's memory in the same unit -- so no verdict can sit + /// beside a figure that contradicts it. + #[test] + fn the_printed_parts_add_up_and_never_contradict_the_verdict() { + let (header, size) = llama(GIB_U as usize / 1024); + let figures = |lines: &str| -> Vec { + let line = lines.lines().find(|line| line.starts_with("at ")).unwrap(); + regex::Regex::new(r"(\d+) MiB") + .unwrap() + .captures_iter(line) + .map(|found| found[1].parse().unwrap()) + .collect() + }; + // Small and large, fitting and not, f16 and quantized. + for (context, kv, gpu) in [ + (128, KvCacheType::F16, 24.0), + (8192, KvCacheType::Q8_0, 24.0), + (131_072, KvCacheType::F16, 24.0), + (4096, KvCacheType::F16, 1.0), + ] { + let result = estimate( + &header, + size, + context, + kv, + AcceleratorMemory::Dedicated(gpu), + ); + let lines = evidence_lines(&result).join("\n"); + let numbers = figures(&lines); + assert_eq!(numbers.len(), 5, "{lines}"); + assert_eq!(numbers[..4].iter().sum::(), numbers[4], "{lines}"); + // The GiB figure beside the MiB total is that same total, rounded up. + let in_gib = need((numbers[4] * MIB) as f64); + assert!( + lines.contains(&format!("= about {} MiB ({in_gib})", numbers[4])), + "{lines}" + ); + let offered = (gpu * 1024.0) as u64; + assert!(lines.contains(&format!("has {offered} MiB")), "{lines}"); + // The verdict is the comparison of the two printed figures. + assert_eq!( + result.outcome == EstimateOutcome::Fits, + numbers[4] <= offered, + "{lines}" + ); + } + // No part of a small model reads as a rounded-away zero. + let tiny = Builder::new() + .string("general.architecture", "t") + .u32("t.block_count", 2) + .u32("t.embedding_length", 128) + .u32("t.feed_forward_length", 256) + .u32("t.attention.head_count", 2) + .u32("t.attention.head_count_kv", 1) + .tensor("w", &[1024]); + let (header, size) = header_of(&tiny, 4096); + let result = estimate( + &header, + size, + 256, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + let lines = evidence_lines(&result).join("\n"); + assert!( + !lines.contains("0.0 GiB") && lines.contains("weights 1 MiB"), + "{lines}" + ); + } + + /// A GPU of exactly `mib` MiB. + fn gpu_mib(mib: u64) -> AcceleratorMemory { + AcceleratorMemory::Dedicated(mib as f64 / 1024.0) + } + + /// The total the report shows for this model at `context`, in whole MiB. + fn shown_mib(header: &GgufHeader, size: u64, context: u64, kv: KvCacheType) -> u64 { + shown_parts(&estimate( + header, + size, + context, + kv, + AcceleratorMemory::Dedicated(1024.0), + )) + .unwrap() + .1 + } + + #[test] + fn the_largest_context_is_rounded_down_to_a_step_and_capped_at_training() { + let (header, size) = llama(GIB_U as usize / 1024); + let kv = KvCacheType::F16; + let shown = |context| shown_mib(&header, size, context, kv); + let max_with = |mib: u64| estimate(&header, size, 100_000, kv, gpu_mib(mib)).max_context; + // A card with exactly the room the report shows for 24576 tokens. + assert_eq!(max_with(shown(24576)), Some(24576)); + // A step is 256 tokens, 32 MiB here: room for 100 more tokens (12 MiB) + // rounds down to the same step. + assert_eq!(max_with(shown(24576) + 12), Some(24576)); + // One MiB short of the step drops to the one below. + assert_eq!(max_with(shown(24576) - 1), Some(24320)); + // A huge GPU is capped at the context the model was trained for. + let result = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(10_000.0), + ); + assert_eq!(result.max_context, Some(131_072)); + } + + #[test] + fn a_gpu_too_small_for_the_weights_has_no_context_that_fits() { + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(0.5), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + assert_eq!(result.max_context, None); + assert!( + !result + .suggestions + .iter() + .any(|s| s.kind == SuggestionKind::SmallerContext), + "a smaller context was suggested where none can fit: {:?}", + result.suggestions + ); + } + + #[test] + fn a_model_that_fits_has_no_suggestions_and_still_reports_its_ceiling() { + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(24.0), + ); + assert_eq!(result.outcome, EstimateOutcome::Fits); + assert!(result.suggestions.is_empty()); + assert_eq!(result.max_context, Some(131_072)); + } + + // The three memory states the readiness check distinguishes. In each the + // figures the file allows are still reported, and in none is anything + // compared. + #[test] + fn memory_that_cannot_be_compared_still_reports_what_the_model_needs() { + let (header, size) = llama(GIB_U as usize / 1024); + for (memory, reason) in [ + (AcceleratorMemory::None, CannotEstimateReason::NoGpu), + ( + AcceleratorMemory::Unknown, + CannotEstimateReason::AcceleratorMemoryUnknown, + ), + ( + AcceleratorMemory::UnifiedMemoryUnreadable, + CannotEstimateReason::UnifiedMemoryUnreadable, + ), + ] { + let result = estimate(&header, size, 4096, KvCacheType::F16, memory); + assert_eq!( + result.outcome, + EstimateOutcome::CannotEstimate, + "{memory:?}" + ); + assert_eq!(result.cannot_estimate_reason, Some(reason)); + assert!( + result.total_gib.is_some(), + "{memory:?}: the need was not reported" + ); + assert_eq!(result.available_gib, None); + assert_eq!(result.max_context, None); + assert!(result.suggestions.is_empty()); + } + } + + #[test] + fn suggestions_name_a_quantized_kv_cache_only_when_it_would_fit() { + let (header, size) = llama(GIB_U as usize / 1024); + let fixed = estimate( + &header, + size, + 0, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ) + .total_gib + .unwrap(); + // 8192 tokens: f16 needs 1 GiB, q8_0 0.53, q4_0 0.28. Room for 0.6. + let result = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(fixed + 0.6), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + let texts: Vec<&str> = result.suggestions.iter().map(|s| s.text.as_str()).collect(); + assert!( + texts + .iter() + .any(|t| t.contains("q8_0 KV cache") && t.contains("which fits")), + "{texts:?}" + ); + assert!( + texts + .iter() + .any(|t| t.contains("q4_0 KV cache") && t.contains("which fits")), + "{texts:?}" + ); + assert!( + texts.iter().any(|t| t.contains("flash attention")), + "{texts:?}" + ); + // Room for 0.4: q8_0 no longer fits and must not be offered as if it did. + let result = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(fixed + 0.4), + ); + let q8 = result + .suggestions + .iter() + .find(|s| s.text.contains("q8_0 KV cache")) + .unwrap(); + assert!(q8.text.contains("still more than"), "{}", q8.text); + let q4 = result + .suggestions + .iter() + .find(|s| s.text.contains("q4_0 KV cache")) + .unwrap(); + assert!(q4.text.contains("which fits"), "{}", q4.text); + } + + #[test] + fn an_already_quantized_cache_is_not_offered_itself() { + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 131_072, + KvCacheType::Q8_0, + AcceleratorMemory::Dedicated(4.0), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + // Only the KV-type suggestions are about the cache's type: the + // context-size one legitimately names the current type. + for suggestion in result + .suggestions + .iter() + .filter(|s| s.kind == SuggestionKind::KvCacheType) + { + assert!( + !suggestion.text.contains("q8_0 KV cache") + && !suggestion.text.contains("f16 KV cache"), + "{}", + suggestion.text + ); + } + let none_wider = estimate( + &header, + size, + 131_072, + KvCacheType::Q4_0, + AcceleratorMemory::Dedicated(4.0), + ); + assert!( + !none_wider + .suggestions + .iter() + .any(|s| s.kind == SuggestionKind::KvCacheType), + "the narrowest cache has no narrower one to suggest" + ); + } + + /// Weights sized like a 7B-parameter model at ~8.5 bits per weight: the + /// header declares one large tensor, and the file carries that many bytes. + fn eight_bit_seven_b() -> (GgufHeader, u64) { + let params: u64 = 7_000_000_000; + let builder = llama_like(10).tensor("big", &[params]); + // A real 7.4 GB buffer is not worth allocating: parse a header-only + // file and size the "file" from the declared data length. + let file = builder.file(0); + let header = parse_gguf_header(&mut Cursor::new(&file)).unwrap(); + let size = header.data_offset + (params as f64 * 8.5 / 8.0) as u64; + (header, size) + } + + #[test] + fn a_smaller_quant_is_suggested_only_when_the_estimate_says_it_fits() { + let (header, size) = eight_bit_seven_b(); + let quant_of = |gpu_gib: f64| { + let result = estimate( + &header, + size, + 2048, + KvCacheType::F16, + AcceleratorMemory::Dedicated(gpu_gib), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit, "{gpu_gib} GiB"); + result + .suggestions + .into_iter() + .find(|s| s.kind == SuggestionKind::SmallerQuant) + .unwrap() + .text + }; + // The 8.5 bpw weights are ~6.9 GiB. The suggestion is the LARGEST quant + // whose weights plus the other ~1.6 GiB (cache, compute buffer, reserve) + // fit, not the smallest that exists: Q5_K_M (~4.6 GiB) is what 6.5 GiB + // leaves room for, Q4_K_M (~3.9 GiB) what 6, and Q3_K_M (~3.2 GiB) what 5. + assert!(quant_of(6.5).contains("Q5_K_M size"), "{}", quant_of(6.5)); + assert!(quant_of(6.0).contains("Q4_K_M size"), "{}", quant_of(6.0)); + assert!(quant_of(5.0).contains("Q3_K_M size"), "{}", quant_of(5.0)); + // A GPU too small for any of them says so instead. + let text = quant_of(1.0); + assert!(text.contains("no smaller quantization"), "{text}"); + } + + #[test] + fn no_quant_is_suggested_when_the_tensor_shapes_and_the_file_size_disagree() { + // 2 GiB of "weights" for ~17 million declared parameters is about a + // thousand bits per weight: not a quantization, a mismatched file. + let (header, size) = llama(2 * GIB_U as usize); + let result = estimate( + &header, + size, + 2048, + KvCacheType::F16, + AcceleratorMemory::Dedicated(1.0), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + assert!( + !result + .suggestions + .iter() + .any(|s| s.kind == SuggestionKind::SmallerQuant), + "{:?}", + result.suggestions + ); + } + + #[test] + fn no_smaller_quant_is_suggested_for_a_model_already_at_the_bottom_of_the_table() { + let params: u64 = 7_000_000_000; + let builder = llama_like(10).tensor("big", &[params]); + let file = builder.file(0); + let header = parse_gguf_header(&mut Cursor::new(&file)).unwrap(); + let size = header.data_offset + (params as f64 * 2.5 / 8.0) as u64; + let result = estimate( + &header, + size, + 2048, + KvCacheType::F16, + AcceleratorMemory::Dedicated(1.0), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + assert!( + !result + .suggestions + .iter() + .any(|s| s.kind == SuggestionKind::SmallerQuant) + ); + } + + #[test] + fn partial_offload_counts_the_layers_that_fit_and_checks_system_ram() { + let (header, size) = llama(GIB_U as usize); + // ~1 GiB weights over 32 layers + 1 GiB KV at 8192: ~64 MiB per layer. + let fixed = estimate( + &header, + size, + 0, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ) + .total_gib + .unwrap(); + // Fixed allowances plus room for half the layers (1 GiB per ... 2 GiB total / 2). + let weights = (size - header.data_offset) as f64 / GIB; + let per_layer = (weights + 1.0) / 32.0; + let memory = AcceleratorMemory::Dedicated(fixed - weights + per_layer * 16.5); + let result = estimate(&header, size, 8192, KvCacheType::F16, memory); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + let offload = result + .suggestions + .iter() + .find(|s| s.kind == SuggestionKind::CpuOffload) + .unwrap(); + assert!(offload.text.contains("16 of 32 layers"), "{}", offload.text); + assert!( + offload.text.contains("--n-gpu-layers 16"), + "{}", + offload.text + ); + // With almost no system RAM the offloaded half has nowhere to go. + let starved = estimate_fit( + &header, + size, + "t", + &request(8192, KvCacheType::F16), + memory, + Some(0.01), + ); + assert!( + !starved + .suggestions + .iter() + .any(|s| s.kind == SuggestionKind::CpuOffload) + ); + // Unknown RAM: still offered, and the report says it is unchecked. + let unknown = estimate_fit( + &header, + size, + "t", + &request(8192, KvCacheType::F16), + memory, + None, + ); + let offload = unknown + .suggestions + .iter() + .find(|s| s.kind == SuggestionKind::CpuOffload) + .unwrap(); + assert!(offload.text.contains("unchecked"), "{}", offload.text); + } + + #[test] + fn no_offload_is_suggested_when_not_even_one_layer_fits() { + let (header, size) = llama(GIB_U as usize); + let result = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(0.55), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + assert!( + !result + .suggestions + .iter() + .any(|s| s.kind == SuggestionKind::CpuOffload) + ); + } + + fn cannot_estimate(builder: &Builder) -> ContextEstimate { + let (header, size) = header_of(builder, 4096); + estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ) + } + + #[test] + fn layouts_the_formula_does_not_describe_are_not_guessed_at() { + let base = || { + Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 4) + .u32("x.embedding_length", 512) + .u32("x.attention.head_count", 8) + .tensor("t", &[10]) + }; + let split = cannot_estimate(&base().u32("split.count", 3)); + assert_eq!( + split.cannot_estimate_reason, + Some(CannotEstimateReason::SplitFile) + ); + let mla = cannot_estimate(&base().u32("x.attention.kv_lora_rank", 512)); + assert_eq!( + mla.cannot_estimate_reason, + Some(CannotEstimateReason::LatentAttention) + ); + let ssm = cannot_estimate( + &Builder::new() + .string("general.architecture", "mamba") + .u32("mamba.block_count", 4) + .u32("mamba.ssm.state_size", 16) + .u32_array("mamba.attention.head_count_kv", &[0, 0, 0, 0]) + .tensor("t", &[10]), + ); + assert_eq!( + ssm.cannot_estimate_reason, + Some(CannotEstimateReason::NoAttentionLayers) + ); + let no_layers = cannot_estimate( + &Builder::new() + .string("general.architecture", "x") + .tensor("t", &[10]), + ); + assert_eq!( + no_layers.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + let no_heads = cannot_estimate( + &Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 4) + .tensor("t", &[10]), + ); + assert_eq!( + no_heads.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + let short_array = cannot_estimate(&base().u32_array("x.attention.head_count_kv", &[1, 1])); + assert_eq!( + short_array.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + for result in [&split, &mla, &ssm, &no_layers, &no_heads, &short_array] { + assert_eq!(result.outcome, EstimateOutcome::CannotEstimate); + assert_eq!( + result.total_gib, None, + "a figure was invented for an unsupported layout" + ); + assert!(result.detail.is_some()); + } + } + + #[test] + fn a_file_with_no_weights_after_the_header_is_not_sized() { + let (header, _) = llama(0); + let result = estimate( + &header, + header.data_offset, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + assert_eq!( + result.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + } + + #[test] + fn mixed_attention_and_an_oversized_context_are_noted_not_hidden() { + let builder = llama_like(10).u32("llama.attention.sliding_window", 4096); + let (header, size) = header_of(&builder, 4096); + let result = estimate( + &header, + size, + 200_000, + KvCacheType::F16, + AcceleratorMemory::Dedicated(1000.0), + ); + assert!( + result.notes.iter().any(|n| n.contains("full context")), + "{:?}", + result.notes + ); + assert!( + result.notes.iter().any(|n| n.contains("beyond the 131072")), + "{:?}", + result.notes + ); + } + + /// The notes say which way each attention kind can move the figure: a + /// sliding-window cache is over-counted, a state-space model's recurrent state + /// is not counted at all, and neither is hidden behind the other's wording. + #[test] + fn each_layout_the_formula_only_approximates_says_which_way_it_can_be_off() { + let notes_of = |builder: &Builder| { + let (header, size) = header_of(builder, 4096); + estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(1000.0), + ) + .notes + .join("\n") + }; + let sliding = notes_of(&llama_like(10).u32("llama.attention.sliding_window", 4096)); + assert!(sliding.contains("can be lower"), "{sliding}"); + assert!(!sliding.contains("can be higher"), "{sliding}"); + let ssm = notes_of(&llama_like(10).u32("llama.ssm.state_size", 16)); + assert!(ssm.contains("recurrent state"), "{ssm}"); + assert!(ssm.contains("can be higher"), "{ssm}"); + let moe = notes_of(&llama_like(10).u32("llama.expert_count", 8)); + assert!( + moe.contains("mixture-of-experts") && moe.contains("can be higher"), + "{moe}" + ); + // A dense model with every width present carries none of them. + let plain = notes_of(&llama_like(10)); + assert!(plain.is_empty(), "{plain}"); + } + + /// A layout whose figure can be too low gets its figures and no verdict, on + /// a card where a dense model of the same size fits and on one where it does + /// not: part of its memory is not counted or not measured, so neither "fits" + /// nor "does not fit" can be relied on. A host that could not compare keeps + /// its own reason. + fn assert_figures_and_no_verdict(layout: &str, builder: &Builder, why: &str) { + let (header, size) = header_of(builder, 4096); + for memory in [ + AcceleratorMemory::Dedicated(1000.0), + AcceleratorMemory::Dedicated(0.5), + ] { + let result = estimate(&header, size, 4096, KvCacheType::F16, memory); + assert_eq!( + result.outcome, + EstimateOutcome::CannotEstimate, + "{layout} on {memory:?}" + ); + assert_eq!( + result.cannot_estimate_reason, + Some(CannotEstimateReason::UnmeasuredLayout), + "{layout}" + ); + assert!( + result.total_gib.is_some(), + "{layout}: the figures are given" + ); + assert!( + result.available_gib.is_some(), + "{layout}: the card is named" + ); + assert_eq!(result.max_context, None, "{layout}: a ceiling is a verdict"); + assert!( + result.suggestions.is_empty(), + "{layout}: {:?}", + result.suggestions + ); + let detail = result.detail.clone().unwrap_or_default(); + assert!( + detail.contains(why) && detail.contains("can be too low"), + "{layout}: {detail}" + ); + let lines = evidence_lines(&result).join("\n"); + assert!(!lines.contains("largest context"), "{layout}: {lines}"); + } + // A host that could not compare keeps its own reason: there was no + // verdict to withhold. + let unknown = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Unknown, + ); + assert_eq!( + unknown.cannot_estimate_reason, + Some(CannotEstimateReason::AcceleratorMemoryUnknown), + "{layout}" + ); + // And with no GPU visible at all, the answer is that, with the figures. + let none = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::None, + ); + assert_eq!( + none.cannot_estimate_reason, + Some(CannotEstimateReason::NoGpu), + "{layout}" + ); + assert!(none.total_gib.is_some(), "{layout}: the figures are given"); + } + + #[test] + fn a_state_space_model_gets_figures_and_no_verdict() { + assert_figures_and_no_verdict( + "state-space", + &llama_like(10).u32("llama.ssm.state_size", 16), + "recurrent state is not counted", + ); + } + + /// Attention in some layers (a KV head count per layer, zero for the + /// state-space ones) and state-space keys: a hybrid. + #[test] + fn a_hybrid_model_gets_figures_and_no_verdict() { + let hybrid = Builder::new() + .string("general.architecture", "h") + .u32("h.block_count", 4) + .u32("h.embedding_length", 512) + .u32("h.feed_forward_length", 1024) + .u32("h.attention.head_count", 8) + .u32_array("h.attention.head_count_kv", &[0, 8, 0, 8]) + .u32("h.context_length", 8192) + .u32("h.ssm.state_size", 16) + .tensor("w", &[1024]); + assert_figures_and_no_verdict("hybrid", &hybrid, "recurrent state is not counted"); + } + + #[test] + fn a_mixture_of_experts_model_gets_figures_and_no_verdict() { + assert_figures_and_no_verdict( + "mixture-of-experts", + &llama_like(10).u32("llama.expert_count", 8), + "compute buffer was not measured", + ); + } + + /// The no-verdict detail and the evidence line show the same MiB figure, + /// even for a GiB figure a hair under a whole MiB, which a plain floor + /// would show one MiB lower. + #[test] + fn the_no_verdict_detail_shows_the_same_mib_as_the_evidence_line() { + let (header, size) = header_of(&llama_like(10).u32("llama.expert_count", 8), 4096); + let gib = (8192.0 - 1e-9) / 1024.0; + let result = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(gib), + ); + let detail = result.detail.clone().unwrap_or_default(); + assert!(detail.contains("this GPU's 8192 MiB"), "{detail}"); + let lines = evidence_lines(&result).join("\n"); + assert!(lines.contains("has 8192 MiB"), "{lines}"); + } + + /// The control: a dense model on the same card still gets its verdict. + #[test] + fn a_dense_model_still_gets_its_verdict() { + let (header, size) = header_of(&llama_like(10), 4096); + let dense = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(1000.0), + ); + assert_eq!(dense.outcome, EstimateOutcome::Fits); + } + + /// The architecture name comes from the file and is printed in the report, + /// so a character that would reorder or break the line is replaced. + #[test] + fn an_architecture_name_cannot_steer_the_terminal() { + let builder = llama_like(10).string("general.architecture", "llama\u{202E}evil\n"); + let (header, size) = header_of(&builder, 4096); + let result = estimate( + &header, + size, + 1024, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.architecture.as_deref(), Some("llama?evil?")); + assert!(evidence_lines(&result).iter().all(|line| line.is_ascii())); + } + + /// U+2028 and U+2029 are line separators, not controls: a terminal and a + /// log viewer may break the line on them, so they are replaced as well. + #[test] + fn an_architecture_name_cannot_break_the_line_with_a_unicode_separator() { + let builder = llama_like(10).string("general.architecture", "a\u{2028}b\u{2029}c"); + let (header, size) = header_of(&builder, 4096); + let result = estimate( + &header, + size, + 1024, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.architecture.as_deref(), Some("a?b?c")); + } + + /// A header may declare zero query heads while keeping a KV head count and + /// no explicit head width; the width would be `embedding / 0`. + #[test] + fn a_zero_head_count_is_a_missing_width_not_a_division_by_zero() { + let result = cannot_estimate( + &Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 4) + .u32("x.embedding_length", 512) + .u32("x.attention.head_count", 0) + .u32("x.attention.head_count_kv", 2) + .tensor("t", &[10]), + ); + assert_eq!(result.outcome, EstimateOutcome::CannotEstimate); + assert_eq!( + result.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + assert_eq!(result.total_gib, None); + } + + #[test] + fn a_header_without_the_widths_the_compute_buffer_needs_gets_figures_and_no_verdict() { + let builder = Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 2) + .u32("x.attention.head_count", 4) + .u32("x.attention.key_length", 64) + .u32("x.attention.value_length", 64) + .tensor("w", &[1024]); + let (header, size) = header_of(&builder, 4096); + let result = estimate( + &header, + size, + 1024, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.outcome, EstimateOutcome::CannotEstimate); + assert_eq!( + result.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + assert!(result.total_bytes.is_some(), "the figures are still given"); + assert!(result.max_context.is_none() && result.suggestions.is_empty()); + assert!( + result.notes.iter().any(|note| note.contains("understated")), + "{:?}", + result.notes + ); + } + + #[test] + fn a_sliding_window_model_is_told_its_largest_context_is_conservative() { + let builder = llama_like(10).u32("llama.attention.sliding_window", 4096); + let (header, size) = header_of(&builder, 4096); + let result = estimate( + &header, + size, + 131_072, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + assert!(result.conservative_ceiling); + let lines = evidence_lines(&result).join("\n"); + assert!( + lines.contains("largest context that fits") + && lines.contains("conservative for this model"), + "{lines}" + ); + assert!( + result.suggestions.iter().any(|suggestion| suggestion + .text + .contains("conservative figure for this model")), + "{:?}", + result.suggestions + ); + // A dense model's ceiling is exact, and says nothing of the kind. + let (header, size) = llama(GIB_U as usize); + let dense = estimate( + &header, + size, + 131_072, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert!(!dense.conservative_ceiling); + assert!(!evidence_lines(&dense).join("\n").contains("conservative")); + } + + #[test] + fn a_context_size_too_large_to_be_real_is_refused_not_printed_as_an_absurd_figure() { + let (header, size) = llama(GIB_U as usize / 1024); + for context in [u64::MAX, MAX_CONTEXT_SIZE * 1_000_000_000_000] { + let result = estimate( + &header, + size, + context, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + assert_eq!(result.outcome, EstimateOutcome::CannotEstimate, "{context}"); + assert_eq!( + result.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + // No figure at all, in any unit: text and JSON cannot disagree about a + // number that is not there. + assert_eq!(result.total_bytes, None); + assert_eq!(result.total_gib, None); + assert_eq!(result.kv_cache_bytes, None); + assert!(result.detail.unwrap().contains("more than an exabyte")); + } + // The cap itself is a real request. + let at_cap = estimate( + &header, + size, + MAX_CONTEXT_SIZE, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + assert!(at_cap.total_bytes.is_some()); + // Head counts that make the cache absurd are refused as well, whatever the + // context size: no figure is derived from a corrupt shape. + let builder = llama_like(10).u32("llama.attention.head_count_kv", u32::MAX); + let (header, size) = header_of(&builder, 4096); + let hostile = estimate( + &header, + size, + MAX_CONTEXT_SIZE, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + assert_eq!(hostile.outcome, EstimateOutcome::CannotEstimate); + assert_eq!(hostile.total_bytes, None); + } + + #[test] + fn signed_integer_headers_are_sized_like_unsigned_ones() { + // gguf-py writes Python integers as signed 32-bit values, so this is the + // shape of most real files: every figure the estimate reads, as type 5. + let signed = Builder::new() + .string("general.architecture", "x") + .i32("x.block_count", 4) + .i32("x.embedding_length", 512) + .i32("x.feed_forward_length", 1024) + .i32_array("x.attention.head_count", &[8, 8, 8, 8]) + .i32_array("x.attention.head_count_kv", &[0, 2, 0, 2]) + .i32("x.context_length", 8192) + .tensor("w", &[4096]); + let unsigned = Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 4) + .u32("x.embedding_length", 512) + .u32("x.feed_forward_length", 1024) + .u32_array("x.attention.head_count", &[8, 8, 8, 8]) + .u32_array("x.attention.head_count_kv", &[0, 2, 0, 2]) + .u32("x.context_length", 8192) + .tensor("w", &[4096]); + let size_up = |builder: &Builder| { + let (header, size) = header_of(builder, 8192); + estimate( + &header, + size, + 2048, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ) + }; + let (a, b) = (size_up(&signed), size_up(&unsigned)); + assert_eq!(a.outcome, EstimateOutcome::Fits); + assert_eq!(a, b); + // Two layers keep a cache of 2 heads x (64 + 64): 256 elements each. + assert_eq!(a.kv_cache_bytes_per_token, Some(2.0 * 256.0 * 2.0)); + } + + #[test] + fn a_single_token_is_not_a_plural() { + assert_eq!(tokens(1), "1 token"); + assert_eq!(tokens(0), "0 tokens"); + assert_eq!(tokens(4096), "4096 tokens"); + let (header, size) = llama(GIB_U as usize / 1024); + let one = estimate( + &header, + size, + 1, + KvCacheType::F16, + AcceleratorMemory::Dedicated(24.0), + ); + let lines = evidence_lines(&one).join("\n"); + assert!(lines.contains("at 1 token:"), "{lines}"); + assert!(!lines.contains("1 tokens"), "{lines}"); + } + + #[test] + fn the_kv_cache_type_round_trips_through_its_name_and_serde() { + for kind in KvCacheType::ALL { + assert_eq!(kind.name().parse::(), Ok(kind)); + assert_eq!( + serde_json::to_string(&kind).unwrap(), + format!("\"{}\"", kind.name()) + ); + } + assert!("q5_0".parse::().is_err()); + } + + #[test] + fn the_evidence_states_where_the_figures_came_from_and_how_big_they_are() { + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 8192, + KvCacheType::F16, + AcceleratorMemory::Dedicated(24.0), + ); + let lines = evidence_lines(&result).join("\n"); + assert!(lines.contains("GGUF header of test.gguf"), "{lines}"); + assert!(lines.contains("llama architecture, 32 layers"), "{lines}"); + assert!(lines.contains("KV cache (f16) 1024 MiB"), "{lines}"); + assert!( + lines.contains( + "has 24576 MiB (24 GiB) in total; memory other processes already hold is not \ + subtracted" + ), + "{lines}" + ); + assert!( + lines.contains("largest context that fits, with the KV cache at f16: 131072 tokens"), + "{lines}" + ); + } + + #[test] + fn a_request_that_fits_is_never_followed_by_a_ceiling_that_says_otherwise() { + // Room for a 128-token request and not for a whole 256-token step: it + // fits, and the report must not go on to say that not even 256 tokens do. + let (header, size) = llama(GIB_U as usize / 1024); + let at_128 = estimate( + &header, + size, + 128, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + let available = (at_128.total_bytes.unwrap() + 4 * MIB) as f64 / GIB; + let tight = estimate( + &header, + size, + 128, + KvCacheType::F16, + AcceleratorMemory::Dedicated(available), + ); + assert_eq!(tight.outcome, EstimateOutcome::Fits); + assert_eq!(tight.max_context, None, "less than one step of room"); + let lines = evidence_lines(&tight).join("\n"); + assert!( + !lines.contains("not even") && !lines.contains("largest context"), + "{lines}" + ); + // And the same request on a card with no room at all is the opposite. + let none = estimate( + &header, + size, + 128, + KvCacheType::F16, + AcceleratorMemory::Dedicated(1.0), + ); + assert_eq!(none.outcome, EstimateOutcome::DoesNotFit); + assert!( + evidence_lines(&none) + .join("\n") + .contains("not even a 256-token context fits"), + ); + } + + #[test] + fn a_request_beyond_the_trained_context_that_fits_is_not_told_a_lower_largest_context() { + // 262144 tokens fits in 192 GiB; the model was trained for 131072, which + // caps the ceiling. Printing "largest context that fits: 131072" right + // after showing 262144 fits would contradict itself. + let (header, size) = llama(GIB_U as usize / 1024); + let result = estimate( + &header, + size, + 262_144, + KvCacheType::F16, + AcceleratorMemory::Dedicated(192.0), + ); + assert_eq!(result.outcome, EstimateOutcome::Fits); + assert_eq!(result.max_context, Some(131_072)); + let lines = evidence_lines(&result).join("\n"); + assert!(!lines.contains("largest context that fits"), "{lines}"); + assert!( + result.notes.iter().any(|note| note + .contains("262144 tokens is beyond the 131072 tokens the model was trained for")), + "{:?}", + result.notes + ); + } + + #[test] + fn a_layer_count_beyond_the_bound_is_refused_not_looped_over() { + let (mut header, size) = llama(GIB_U as usize / 1024); + header.block_count = Some(crate::gguf::MAX_BLOCK_COUNT + 1); + let result = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + assert_eq!(result.outcome, EstimateOutcome::CannotEstimate); + assert_eq!( + result.cannot_estimate_reason, + Some(CannotEstimateReason::MissingMetadata) + ); + assert!( + result + .detail + .as_deref() + .unwrap_or("") + .contains("not a plausible model"), + "{:?}", + result.detail + ); + assert_eq!(result.total_gib, None); + } + + // ---- Calibration against measured loads ---------------------------------- + // + // The figures below were measured on an MI300X (gfx942) with llama.cpp b9752 + // on ROCm 7.2: `llama-server -fa on -np 1`, the default physical batch of + // 512, every layer on the GPU. The buffer sizes are what llama.cpp logged; + // the "used" figure is how much the GPU's used VRAM grew across the load. + // The headers are synthetic but carry the real shapes, tensor offsets and + // file sizes of the two models. + + /// Llama-3.1-8B Q4_K_M: 32 layers, 8 KV heads, head width 128, embedding 4096, + /// feed-forward 14336. 4_912_898_304 bytes of tensor data, of which the + /// 128256 x 4096 Q4_K `token_embd.weight` is 295_501_824 (llama.cpp logged + /// 281.81 MiB of it host-mapped and 4403.49 MiB on the GPU). + fn measured_llama() -> (GgufHeader, u64) { + let builder = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 32) + .u32("llama.embedding_length", 4096) + .u32("llama.feed_forward_length", 14336) + .u32("llama.attention.head_count", 32) + .u32("llama.attention.head_count_kv", 8) + .u32("llama.context_length", 131_072) + .tensor_at("token_embd.weight", &[4096, 128_256], 0) + .tensor_at("blk.0.attn_q.weight", &[4096, 4096], 295_501_824) + .tensor_at("output.weight", &[4096, 128_256], 4_617_396_480); + let (header, size) = header_of(&builder, 0); + (header, size + 4_912_898_304) + } + + /// Qwen2.5-1.5B Q8_0: 28 layers, 2 KV heads, head width 128, embedding 1536, + /// feed-forward 8960. The model's weights are tied, but the GGUF file has its + /// own `output.weight` (a second 247_959_552-byte copy, last in the data), so + /// it is not a tied-embedding file. 1_888_581_632 bytes of tensor data, of + /// which the 151936 x 1536 Q8_0 `token_embd.weight` is 247_959_552 (236.47 MiB + /// logged host-mapped, 1564.63 MiB on the GPU). + fn measured_qwen() -> (GgufHeader, u64) { + let builder = Builder::new() + .string("general.architecture", "qwen2") + .u32("qwen2.block_count", 28) + .u32("qwen2.embedding_length", 1536) + .u32("qwen2.feed_forward_length", 8960) + .u32("qwen2.attention.head_count", 12) + .u32("qwen2.attention.head_count_kv", 2) + .u32("qwen2.context_length", 32_768) + .tensor_at("token_embd.weight", &[1536, 151_936], 0) + .tensor_at("blk.0.attn_q.weight", &[1536, 1536], 247_959_552) + .tensor_at("output.weight", &[1536, 151_936], 1_640_622_080); + let (header, size) = header_of(&builder, 0); + (header, size + 1_888_581_632) + } + + struct Load { + context: u64, + kv: KvCacheType, + /// llama.cpp's logged GPU KV buffer, MiB. + kv_mib: f64, + /// llama.cpp's logged GPU compute buffer, MiB. + compute_mib: f64, + /// Growth of the GPU's used VRAM across the load, MiB. + used_mib: f64, + } + + const fn load( + context: u64, + kv: KvCacheType, + kv_mib: f64, + compute_mib: f64, + used_mib: f64, + ) -> Load { + Load { + context, + kv, + kv_mib, + compute_mib, + used_mib, + } + } + + const LLAMA_LOADS: [Load; 11] = [ + load(4096, KvCacheType::F16, 512.0, 112.01, 6128.0), + load(32768, KvCacheType::F16, 4096.0, 140.01, 9734.0), + load(65536, KvCacheType::F16, 8192.0, 172.01, 13869.0), + load(131_072, KvCacheType::F16, 16384.0, 236.01, 22125.0), + load(4096, KvCacheType::Q8_0, 272.0, 112.09, 5839.0), + load(32768, KvCacheType::Q8_0, 2176.0, 192.09, 7819.0), + load(65536, KvCacheType::Q8_0, 4352.0, 352.09, 10152.0), + load(131_072, KvCacheType::Q8_0, 8704.0, 672.09, 14830.0), + load(4096, KvCacheType::Q4_0, 144.0, 112.09, 5711.0), + load(32768, KvCacheType::Q4_0, 1152.0, 192.09, 6797.0), + load(131_072, KvCacheType::Q4_0, 4608.0, 672.09, 10734.0), + ]; + + const QWEN_LOADS: [Load; 4] = [ + load(4096, KvCacheType::F16, 112.0, 65.51, 3019.0), + load(32768, KvCacheType::F16, 896.0, 93.51, 3829.0), + load(32768, KvCacheType::Q8_0, 476.02, 93.59, 3362.0), + load(131_072, KvCacheType::F16, 3584.0, 189.51, 6615.0), + ]; + + fn mib(bytes: u64) -> f64 { + bytes as f64 / MIB as f64 + } + + /// `above` is how far above the measured use the total may come, as a factor. + fn check_against_measured(header: &GgufHeader, size: u64, loads: &[Load], above: f64) { + for measured in loads { + let result = estimate( + header, + size, + measured.context, + measured.kv, + AcceleratorMemory::Dedicated(192.0), + ); + let label = format!("{} tokens, {}", measured.context, measured.kv); + // The KV cache is exact in every case, quantized ones included. + let kv = mib(result.kv_cache_bytes.unwrap()); + assert!((kv - measured.kv_mib).abs() < 0.03, "{label}: KV {kv}"); + let compute = mib(result.compute_buffer_bytes.unwrap()); + if measured.kv == KvCacheType::F16 { + // The fixed part plus the attention mask reproduces the buffer. + assert!( + (compute - measured.compute_mib).abs() < 0.02, + "{label}: compute {compute} vs {}", + measured.compute_mib + ); + } else { + // The dequantization allowance is an upper bound, never below. + assert!( + compute >= measured.compute_mib, + "{label}: compute {compute}" + ); + assert!( + compute <= measured.compute_mib + 80.0, + "{label}: compute {compute}" + ); + } + // The total is above what the load really used, and by less than + // `above`. + let total = mib(result.total_bytes.unwrap()); + assert!(total >= measured.used_mib, "{label}: total {total}"); + assert!( + total <= measured.used_mib * above, + "{label}: total {total} vs {}", + measured.used_mib + ); + } + } + + #[test] + fn the_llama_shaped_model_reproduces_the_measured_loads() { + let (header, size) = measured_llama(); + check_against_measured(&header, size, &LLAMA_LOADS, 1.05); + } + + #[test] + fn the_qwen_shaped_model_reproduces_the_measured_loads() { + let (header, size) = measured_qwen(); + check_against_measured(&header, size, &QWEN_LOADS, 1.05); + } + + /// The sixteenth load, made with the card almost full: the process overhead + /// measured only about 0.4 GiB there against 1.05-1.28 GiB in the other + /// loads, so the estimate was further above what was used. It is still + /// above, by 5.5% (the module documentation gives the range over all + /// sixteen loads), so the 5% bound the model's other loads meet is not + /// asserted here. + #[test] + fn the_load_made_with_the_card_almost_full_is_estimated_above_by_about_five_and_a_half_percent() + { + let (header, size) = measured_llama(); + let result = estimate( + &header, + size, + 86_016, + KvCacheType::F16, + AcceleratorMemory::Dedicated(192.0), + ); + // llama.cpp logged a 10752 MiB cache and a 192.01 MiB compute buffer. + assert!((mib(result.kv_cache_bytes.unwrap()) - 10752.0).abs() < 0.03); + assert!((mib(result.compute_buffer_bytes.unwrap()) - 192.0).abs() < 0.02); + let total = mib(result.total_bytes.unwrap()); + let used = 15767.0; + assert!(total >= used, "total {total}"); + assert!(total > used * 1.05 && total < used * 1.06, "total {total}"); + } + + #[test] + fn the_components_of_two_measured_loads_are_pinned_to_the_byte() { + let (header, size) = measured_llama(); + let result = estimate( + &header, + size, + 32768, + KvCacheType::F16, + AcceleratorMemory::Dedicated(192.0), + ); + // 4403.49 MiB on the GPU, the embedding's 281.81 MiB left on the host. + assert_eq!(result.weights_bytes, Some(4_617_396_480)); + assert_eq!(result.kv_cache_bytes, Some(4096 * MIB)); + // 108 MiB fixed + 32 MiB of mask. + assert_eq!(result.compute_buffer_bytes, Some(140 * MIB)); + assert_eq!(result.runtime_reserve_bytes, Some(1280 * MIB)); + assert_eq!(result.total_bytes, Some(10_401_341_696)); + + let (header, size) = measured_qwen(); + let result = estimate( + &header, + size, + 32768, + KvCacheType::F16, + AcceleratorMemory::Dedicated(192.0), + ); + // 1564.63 MiB on the GPU, the embedding's 236.47 MiB left on the host. + assert_eq!(result.weights_bytes, Some(1_640_622_080)); + assert_eq!(result.kv_cache_bytes, Some(896 * MIB)); + assert_eq!(result.compute_buffer_bytes, Some(64_487_424 + 32 * MIB)); + assert_eq!(result.total_bytes, Some(4_020_365_312)); + } + + #[test] + fn a_quantized_cache_adds_the_dequantization_buffer_and_f16_does_not() { + let (header, size) = measured_llama(); + let compute = |kv| { + estimate( + &header, + size, + 131_072, + kv, + AcceleratorMemory::Dedicated(192.0), + ) + .compute_buffer_bytes + .unwrap() + }; + let base_and_mask = LLAMA_COMPUTE_BASE + 512 * 2 * 131_072; + assert_eq!(compute(KvCacheType::F16), base_and_mask); + // One layer's K and V as 16-bit floats per token: 8 x 256 x 2 bytes. + let extra = 131_072 * 8 * 256 * 2; + assert_eq!(compute(KvCacheType::Q8_0), base_and_mask + extra); + assert_eq!(compute(KvCacheType::Q4_0), base_and_mask + extra); + } + + #[test] + fn the_largest_context_accounts_for_what_each_token_adds_to_the_compute_buffer() { + let (header, size) = measured_llama(); + for kv in KvCacheType::ALL { + // A card with exactly what the report shows for 32768 tokens holds + // 32768 and not a step more; one MiB less holds a step fewer. + let card = shown_mib(&header, size, 32768, kv); + let result = estimate(&header, size, 1, kv, gpu_mib(card)); + assert_eq!(result.max_context, Some(32768), "{kv}"); + let result = estimate(&header, size, 1, kv, gpu_mib(card - 1)); + assert_eq!(result.max_context, Some(32512), "{kv}"); + } + } + + /// `count` model shapes with awkward sizes: parts that are not whole MiB, so + /// the shown (rounded-up) total differs from the exact one by up to a few MiB + /// -- the case a rounding mismatch needs. Deterministic. + fn odd_shapes(count: usize) -> Vec<(GgufHeader, u64)> { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = |bound: u64| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state % bound + }; + (0..count) + .map(|_| { + let layers = 4 + next(40) as u32; + let kv_heads = 1 + next(16) as u32; + let head = 32 + next(224) as u32; + let embd = 256 + next(7936) as u32; + let ffn = 512 + next(20000) as u32; + let builder = Builder::new() + .string("general.architecture", "odd") + .u32("odd.block_count", layers) + .u32("odd.embedding_length", embd) + .u32("odd.feed_forward_length", ffn) + .u32("odd.attention.head_count", kv_heads * 4) + .u32("odd.attention.head_count_kv", kv_heads) + .u32("odd.attention.key_length", head) + .u32("odd.attention.value_length", head) + .u32("odd.context_length", 8192 * (1 + next(16) as u32)) + .tensor_at("token_embd.weight", &[u64::from(embd), 1000], 0) + .tensor_at("blk.0.attn_q.weight", &[64, 64], 1_000_003); + let data = 50_000_000 + next(900_000_000); + // One tensor holds the rest of the parameters, at a bits-per-weight + // a real quantization has, so the smaller-quant suggestion is + // reachable (it is withheld when shape and size disagree). + let bits = 4.0 + next(50) as f64 / 10.0; + let params = (data as f64 * 8.0 / bits) as u64; + let builder = builder.tensor_at("blk.1.ffn_up.weight", &[params], 2_000_000); + let (header, size) = header_of(&builder, 0); + (header, size + data) + }) + .collect() + } + + /// The class behind the rounding defect: the verdict, the largest context + /// and every suggestion are one decision. Swept over memory sizes at and + /// within a few MiB of every boundary, for every cache type and several + /// model shapes. + #[test] + fn the_verdict_the_largest_context_and_the_suggestions_are_one_decision() { + let mut shapes = vec![measured_llama(), measured_qwen(), llama(GIB_U as usize / 2)]; + shapes.extend(odd_shapes(24)); + let quant_bits = |label: &str| { + QUANT_BITS_PER_WEIGHT + .iter() + .find(|(name, _)| *name == label) + .map(|(_, bits)| *bits) + .unwrap() + }; + let label_re = regex::Regex::new(r"about (\S+) size").unwrap(); + let layers_re = regex::Regex::new(r"keep about (\d+) of (\d+) layers").unwrap(); + let kv_re = regex::Regex::new(r"use a (q8_0|q4_0) KV cache .*about (\d+) MiB").unwrap(); + let mut checked = 0; + for (header, size) in &shapes { + let sizing = sizing(header, *size).unwrap(); + for kv in KvCacheType::ALL { + // The request every card is judged against, and the boundaries of + // every way to make room for it: each cache type's own total at a + // spread of contexts, and the GPU's share of the layers. + let request_context = 8192; + let mut bases: Vec = Vec::new(); + for other in KvCacheType::ALL { + for context in [256_u64, 4096, 8192, 32768, 65536] { + bases.push(shown_mib(header, *size, context, other)); + } + for on_gpu in [1, sizing.layers / 2, sizing.layers - 1] { + bases.push( + shown_sum([ + sizing.weights * on_gpu as f64 / sizing.layers as f64, + sizing.kv_bytes(request_context, other) * on_gpu as f64 + / sizing.layers as f64, + sizing.compute_bytes(request_context, other), + RUNTIME_RESERVE_BYTES, + ]) as u64 + / MIB, + ); + } + } + { + // Around each boundary, and well away from it. + let mut cards: Vec = Vec::new(); + for base in bases { + cards.extend((-3..=3).map(|d| base.saturating_add_signed(d))); + cards.extend([base / 2 + 1, base + 40]); + } + for card in cards { + let memory = gpu_mib(card); + let result = estimate(header, *size, request_context, kv, memory); + let available = (card * MIB) as f64; + // The verdict is the predicate. + assert_eq!( + result.outcome == EstimateOutcome::Fits, + sizing.fits(request_context, kv, available) + ); + // The largest context fits, and no larger step does. + if let Some(max) = result.max_context { + let at = estimate(header, *size, max, kv, memory); + assert_eq!(at.outcome, EstimateOutcome::Fits, "{kv} {card} MiB: {max}"); + let cap = header.context_length.unwrap_or(u64::MAX); + if max + CONTEXT_STEP <= cap { + let above = estimate(header, *size, max + CONTEXT_STEP, kv, memory); + assert_ne!( + above.outcome, + EstimateOutcome::Fits, + "{kv} {card} MiB: {max} + a step also fits" + ); + } + } else { + let at = estimate(header, *size, CONTEXT_STEP, kv, memory); + assert_ne!(at.outcome, EstimateOutcome::Fits, "{kv} {card} MiB"); + } + if result.outcome == EstimateOutcome::Fits { + assert!(result.suggestions.is_empty()); + continue; + } + for suggestion in &result.suggestions { + checked += 1; + match suggestion.kind { + SuggestionKind::SmallerContext => { + let max = result.max_context.unwrap(); + assert!( + suggestion.text.contains(&format!("--ctx-size {max}")), + "{}", + suggestion.text + ); + } + SuggestionKind::KvCacheType => { + if let Some(found) = kv_re.captures(&suggestion.text) { + let smaller: KvCacheType = found[1].parse().unwrap(); + assert!(sizing.fits(request_context, smaller, available)); + assert_eq!( + found[2].parse::().unwrap(), + shown_mib(header, *size, request_context, smaller) + ); + } else if suggestion.text.contains("is still more than") { + let smaller = KvCacheType::ALL + .into_iter() + .filter(|o| { + o.bytes_per_element() < kv.bytes_per_element() + }) + .any(|o| !sizing.fits(request_context, o, available)); + assert!(smaller, "{}", suggestion.text); + } + } + SuggestionKind::SmallerQuant => { + if suggestion.text.contains("would fit at") { + let label = + &label_re.captures(&suggestion.text).unwrap()[1]; + let weights = sizing.quant_weights( + header.param_count as f64, + quant_bits(label), + ); + assert!( + sizing.shown_total_with(weights, request_context, kv) + <= available, + "{}", + suggestion.text + ); + } + } + SuggestionKind::CpuOffload => { + let found = layers_re.captures(&suggestion.text).unwrap(); + let on_gpu: u64 = found[1].parse().unwrap(); + assert!(sizing.layers_fit( + on_gpu, + request_context, + kv, + available + )); + assert!( + !sizing.layers_fit( + on_gpu + 1, + request_context, + kv, + available + ), + "one more layer also fits: {}", + suggestion.text + ); + } + } + } + } + } + } + } + assert!( + checked > 50, + "the sweep reached too few suggestions: {checked}" + ); + } + + /// The shown total can sit a fraction of a byte *below* the exact one (each + /// part rounds to a byte, then up to a MiB), so the analytic estimate can + /// fall one step short of the largest context the verdict accepts. The + /// largest context is then the verdict's, not the analytic one. + #[test] + fn the_largest_context_is_the_largest_one_the_verdict_accepts() { + let sizing = Sizing { + data: 0.0, + // 0.4 of a byte over a whole MiB: it rounds to the MiB below. + weights: 1000.0_f64.mul_add(MIB as f64, 0.4), + compute_base: 0.0, + kv_elements_per_token: 0.0, + widest_layer_kv_elements: 0.0, + layers: 1, + }; + let kv = KvCacheType::F16; + // Only the attention mask grows: 512 x 2 bytes a token, 4 MiB at 4096. + let available = (2284 * MIB) as f64; + assert!( + sizing.total(4096, kv) > available, + "the exact total is above" + ); + assert!(sizing.fits(4096, kv, available), "the shown total is not"); + assert_eq!(sizing.max_context(available, kv, None), Some(4096)); + assert!(!sizing.fits(4352, kv, available)); + } + + /// The estimate assumes flash attention is on, and says so where the user reads + /// it: with it off the compute buffer grows with the context and the figure is + /// too low, which only the caveat can tell them. + #[test] + fn the_standing_caveat_says_flash_attention_is_assumed() { + assert!( + ESTIMATE_CAVEAT.contains("flash attention"), + "{ESTIMATE_CAVEAT}" + ); + assert!(ESTIMATE_CAVEAT.contains("assumed"), "{ESTIMATE_CAVEAT}"); + assert!(ESTIMATE_CAVEAT.contains("too low"), "{ESTIMATE_CAVEAT}"); + } + + /// A trained context below one step, on a card whose memory falls between the + /// exact total and the shown one, used to step below zero: plain subtraction + /// took 250 to `250 - CONTEXT_STEP`, a panic in a debug build and a wrap to a + /// huge context, then a loop, in a release one. The first step down now + /// rounds onto a whole step, and the answer is that no step fits. (The + /// `context > 0` guard at zero itself is covered by the next test.) + #[test] + fn a_trained_context_below_one_step_that_does_not_fit_gives_no_context() { + let sizing = Sizing { + data: 0.0, + weights: 1000.0_f64.mul_add(MIB as f64, 0.4), + compute_base: 0.0, + kv_elements_per_token: 0.0, + widest_layer_kv_elements: 0.0, + layers: 1, + }; + let kv = KvCacheType::F16; + // 250 tokens of mask is under a MiB and shows as one: 2281 MiB in all. + let available = 2280.5 * MIB as f64; + assert!(!sizing.fits(250, kv, available)); + assert_eq!(sizing.max_context(available, kv, Some(250)), None); + } + + /// Where the card's memory falls between the exact total at zero context and + /// the shown one, even zero context does not fit, yet the room left after the + /// exact fixed part is positive, so the search reaches zero (stepping down + /// from 256 with no cap, or rounding 250 down with one) and is stopped there + /// only by its `context > 0` guard. Without that guard it would step to + /// `0 - CONTEXT_STEP`: a panic in a debug build, a wrap in a release one. The + /// test above does not reach this, because there zero context fits. + #[test] + fn when_even_zero_context_does_not_fit_the_search_stops_at_zero() { + let sizing = Sizing { + data: 0.0, + // Half a MiB over 1000 MiB: shown as 1001, so the exact fixed part is + // 2280.5 MiB and the shown one 2281 MiB. + weights: 1000.5 * MIB as f64, + compute_base: 0.0, + kv_elements_per_token: 0.0, + widest_layer_kv_elements: 0.0, + layers: 1, + }; + let kv = KvCacheType::F16; + let available = 2280.75 * MIB as f64; + assert!(available > sizing.fixed(), "the early return must not fire"); + assert!(!sizing.fits(0, kv, available), "zero context must not fit"); + assert_eq!(sizing.max_context(available, kv, None), None); + assert_eq!(sizing.max_context(available, kv, Some(250)), None); + } + + /// A cap that is not a whole number of steps steps down onto whole steps: + /// 1500 does not fit, so the answer is 1024, not 1244 or 988. + #[test] + fn a_trained_context_that_is_not_a_whole_step_steps_down_to_a_whole_one() { + let sizing = Sizing { + data: 0.0, + weights: 1000.0_f64.mul_add(MIB as f64, 0.4), + compute_base: 0.0, + kv_elements_per_token: 0.0, + widest_layer_kv_elements: 0.0, + layers: 1, + }; + let kv = KvCacheType::F16; + // The mask at 1500 tokens is 1.46 MiB and shows as 2; at 1024 it is 1. + // The exact total fits 1500 tokens here, so the search starts at the cap + // itself rather than at a whole step below it. + let available = 2281.9 * MIB as f64; + assert!(sizing.total(1500, kv) <= available); + assert!(!sizing.fits(1500, kv, available)); + assert!(sizing.fits(1024, kv, available)); + assert_eq!(sizing.max_context(available, kv, Some(1500)), Some(1024)); + } + + /// A trained context that is not a whole step and fits is the answer, even + /// where the estimate does not reach the next whole step: rounding to a step + /// before capping gave 1280 here. + #[test] + fn a_trained_context_that_fits_is_the_largest_even_when_not_a_whole_step() { + let sizing = Sizing { + data: 0.0, + weights: 1000.0 * MIB as f64, + compute_base: 0.0, + // One MiB of f16 cache per token, so a MiB of rounding is a token. + kv_elements_per_token: 512.0 * 1024.0, + widest_layer_kv_elements: 512.0 * 1024.0, + layers: 1, + }; + let kv = KvCacheType::F16; + // 1500 tokens show as 1000 + 1500 + 2 + 1280 = 3782 MiB; the exact + // estimate reaches about 1502 tokens, short of the step at 1536. + let available = 3784.0 * MIB as f64; + assert!(sizing.fits(1500, kv, available)); + assert!(!sizing.fits(1536, kv, available)); + assert_eq!(sizing.max_context(available, kv, Some(1500)), Some(1500)); + // Without the cap the answer is a whole step. + assert_eq!(sizing.max_context(available, kv, None), Some(1280)); + } + + /// The same for partial offload: a layer count that the verdict's own + /// measure accepts is offered, even where the analytic share says none. + #[test] + fn partial_offload_offers_the_layers_the_verdict_accepts() { + let sizing = Sizing { + data: 0.0, + // A quarter of it, 1000 MiB and 0.4 of a byte, rounds to 1000 MiB. + weights: 4000.0_f64.mul_add(MIB as f64, 1.6), + compute_base: 0.0, + kv_elements_per_token: 0.0, + widest_layer_kv_elements: 0.0, + layers: 4, + }; + let request = request(4096, KvCacheType::F16); + // 1000 MiB of weights, 4 MiB of mask, 1280 MiB of reserve. + let available = (2284 * MIB) as f64; + assert!(sizing.layers_fit(1, 4096, KvCacheType::F16, available)); + assert!(!sizing.layers_fit(2, 4096, KvCacheType::F16, available)); + let suggestion = offload_suggestion(&sizing, &request, available, Some(64.0)) + .expect("one layer fits by the verdict's measure"); + assert!( + suggestion.text.contains("keep about 1 of 4 layers"), + "{}", + suggestion.text + ); + } + + #[test] + fn the_input_embedding_is_only_left_out_of_the_weights_when_the_header_has_one() { + // Last in the data: it runs to the end of the tensor data. + let builder = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 2) + .u32("llama.embedding_length", 64) + .u32("llama.attention.head_count", 4) + .tensor_at("blk.0.attn_q.weight", &[64, 64], 0) + .tensor_at("output.weight", &[64, 100], 500) + .tensor_at("token_embd.weight", &[64, 100], 1000); + let (header, size) = header_of(&builder, 0); + let size = size + 5000; + let result = estimate( + &header, + size, + 512, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.weights_bytes, Some(1000)); + // The same file without its own output layer has tied embeddings: the + // output layer uses the embedding on the GPU, so it stays in. + let tied = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 2) + .u32("llama.embedding_length", 64) + .u32("llama.attention.head_count", 4) + .tensor_at("blk.0.attn_q.weight", &[64, 64], 0) + .tensor_at("token_embd.weight", &[64, 100], 1000); + let (header, size) = header_of(&tied, 0); + let result = estimate( + &header, + size + 5000, + 512, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.weights_bytes, Some(5000)); + // No input embedding at all: every byte of the data is weights. + let builder = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 2) + .u32("llama.embedding_length", 64) + .u32("llama.attention.head_count", 4) + .tensor_at("blk.0.attn_q.weight", &[64, 64], 0); + let (header, size) = header_of(&builder, 0); + let result = estimate( + &header, + size + 5000, + 512, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.weights_bytes, Some(5000)); + // A header that puts the embedding past the end of the data cannot + // make the weights negative. + let builder = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 2) + .u32("llama.embedding_length", 64) + .u32("llama.attention.head_count", 4) + .tensor_at("output.weight", &[64, 100], 0) + .tensor_at("token_embd.weight", &[64, 100], 9_000); + let (header, size) = header_of(&builder, 0); + let result = estimate( + &header, + size + 5000, + 512, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.weights_bytes, Some(5000)); + } + + // The tests below pin one rule each, with figures a mutation of that rule + // changes: a case where the right and the wrong expression happen to agree + // proves nothing, so every one is asymmetric. + + /// A model that costs only its weights, the attention mask and the reserve. + fn weights_only(weights: u64, layers: u64) -> Sizing { + Sizing { + data: weights as f64, + weights: weights as f64, + compute_base: 0.0, + kv_elements_per_token: 0.0, + widest_layer_kv_elements: 0.0, + layers, + } + } + + #[test] + fn partial_offload_names_the_share_left_in_system_ram_not_the_share_on_the_gpu() { + // Eight layers of 1 GiB. The card holds two: 2048 MiB of them, the 4 MiB + // mask at 4096 tokens and the 1280 MiB reserve make 3332 MiB; a third + // layer would need 4356. + let sizing = weights_only(8 * GIB_U, 8); + let request = request(4096, KvCacheType::F16); + let available = f64::from(3332 + 100) * MIB as f64; + assert!(sizing.layers_fit(2, 4096, KvCacheType::F16, available)); + assert!(!sizing.layers_fit(3, 4096, KvCacheType::F16, available)); + let text = |ram: Option| { + offload_suggestion(&sizing, &request, available, ram).map(|suggestion| suggestion.text) + }; + let offered = text(Some(64.0)).unwrap(); + assert!(offered.contains("keep about 2 of 8 layers"), "{offered}"); + // The six layers that do not fit stay in RAM: 6 GiB, not the 2 GiB on the GPU. + assert!( + offered.contains("the other 6.0 GiB of weights and cache"), + "{offered}" + ); + // So 4 GiB of RAM cannot hold them (the 2 GiB on the GPU would fit in + // it), and 7 GiB can. + assert!(text(Some(4.0)).is_none()); + assert!(text(Some(7.0)).is_some()); + } + + #[test] + fn a_smaller_quantization_keeps_the_input_embedding_out_of_vram() { + let sizing = Sizing { + data: 1000.0, + weights: 400.0, + compute_base: 0.0, + kv_elements_per_token: 0.0, + widest_layer_kv_elements: 0.0, + layers: 1, + }; + // 1000 weights at 8 bits is 1000 bytes of file, of which 60% is the + // embedding that is not VRAM, leaving 400 bytes on the GPU. + assert!((sizing.quant_weights(1000.0, 8.0) - 400.0).abs() < 1e-9); + assert!((sizing.quant_weights(1000.0, 4.0) - 200.0).abs() < 1e-9); + } + + /// Each row is a bits-per-weight figure, not a file size: llama.cpp's + /// `quantize` table gives LLaMA-v1-7B sizes in GiB, and the row is that size + /// in bits over the model's weights. A size entered as bits (Q2_K's 2.63 GiB + /// read as 2.63 bits) prices the quantization about 20% light, and would + /// name it as fitting where it does not. + #[test] + fn the_bits_per_weight_table_is_the_published_sizes_in_bits() { + const LLAMA_V1_7B_WEIGHTS: f64 = 6_738_415_616.0; + let published_gib = [ + ("Q2_K", 2.63), + ("Q3_K_M", 3.07), + ("Q4_K_M", 3.80), + ("Q5_K_M", 4.45), + ("Q6_K", 5.15), + ("Q8_0", 6.70), + ]; + for (label, gib) in published_gib { + let bits = gib * GIB * 8.0 / LLAMA_V1_7B_WEIGHTS; + let row = QUANT_BITS_PER_WEIGHT + .iter() + .find(|(name, _)| *name == label) + .unwrap_or_else(|| panic!("{label} is not in the table")) + .1; + assert!((row - bits).abs() < 0.05, "{label}: {row} vs {bits:.3}"); + } + // The table runs from smallest to largest, which the suggestion relies on. + assert!( + QUANT_BITS_PER_WEIGHT + .windows(2) + .all(|pair| pair[0].1 < pair[1].1) + ); + } + + /// A table entry within [`QUANT_SMALLER_FACTOR`] of the current figure is the + /// same quantization for this purpose: at 4.4 bits per weight, IQ4_XS (4.25) + /// is not "smaller", so the suggestion skips it for Q3_K_M even though + /// IQ4_XS would fit. At a factor of 1.0 it would be named. + #[test] + fn a_quantization_within_the_smaller_factor_of_the_current_one_is_not_offered() { + let params: u64 = 7_000_000_000; + let builder = llama_like(10).tensor("big", &[params]); + let file = builder.file(0); + let header = parse_gguf_header(&mut Cursor::new(&file)).unwrap(); + let size = header.data_offset + (params as f64 * 4.4 / 8.0) as u64; + // 7e9 weights at 4.25 bits are about 3547 MiB, at 4.4 bits about 3672; + // with the cache (256 MiB), compute buffer (about 110) and the 1280 MiB + // reserve, IQ4_XS needs about 5193 MiB and the file as it is about 5318. + let result = estimate(&header, size, 2048, KvCacheType::F16, gpu_mib(5250)); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + let text = &result + .suggestions + .iter() + .find(|s| s.kind == SuggestionKind::SmallerQuant) + .unwrap() + .text; + assert!(text.contains("Q3_K_M size"), "{text}"); + assert!(!text.contains("IQ4_XS"), "{text}"); + } + + #[test] + fn a_large_embedding_changes_which_smaller_quantization_is_named() { + // 7e9 parameters at 8.5 bits, 40% of the data being the embedding. + let params: u64 = 7_000_000_000; + let data = (params as f64 * 8.5 / 8.0) as u64; + let embedding = data / 10 * 4; + let builder = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 32) + .u32("llama.embedding_length", 4096) + .u32("llama.feed_forward_length", 14336) + .u32("llama.attention.head_count", 32) + .u32("llama.attention.head_count_kv", 8) + .u32("llama.context_length", 131_072) + .tensor_at("token_embd.weight", &[4096, 10], 0) + .tensor_at("big", &[params], embedding) + .tensor_at("output.weight", &[4096, 10], data); + let (header, size) = header_of(&builder, 0); + let size = size + data; + // What Q5_K_M (5.69 bits per weight) needs in VRAM, worked out here: the + // 60% of it that is not the embedding, then the cache, compute buffer and + // reserve at 2048 tokens. + let weights_q5 = params as f64 * 5.69 / 8.0 * 0.6; + let other = (128.0_f64 * 1024.0) + .mul_add(2048.0, (LLAMA_COMPUTE_BASE + 512 * 2 * 2048) as f64) + + (1280 * MIB) as f64; + let card_mib = ((weights_q5 + other) / MIB as f64).ceil() as u64 + 8; + let result = estimate(&header, size, 2048, KvCacheType::F16, gpu_mib(card_mib)); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + let text = &result + .suggestions + .iter() + .find(|s| s.kind == SuggestionKind::SmallerQuant) + .unwrap() + .text; + // Counting the embedding as VRAM would price Q5_K_M at 4.6 GiB, past this + // card, and the suggestion would be a much smaller quantization. + assert!(text.contains("Q5_K_M size"), "{text}"); + } + + #[test] + fn the_largest_context_after_a_smaller_cache_is_the_one_for_that_cache() { + let (header, size) = llama(GIB_U as usize / 1024); + let fixed = estimate( + &header, + size, + 0, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ) + .total_gib + .unwrap(); + let memory = AcceleratorMemory::Dedicated(fixed + 0.6); + let result = estimate(&header, size, 8192, KvCacheType::F16, memory); + let for_cache = |kv| { + estimate(&header, size, 8192, kv, memory) + .max_context + .unwrap() + }; + assert_ne!( + for_cache(KvCacheType::F16), + for_cache(KvCacheType::Q8_0), + "the two caches must give different ceilings for this to mean anything" + ); + let q8 = result + .suggestions + .iter() + .find(|s| s.text.contains("q8_0 KV cache")) + .unwrap(); + assert!( + q8.text.contains(&format!( + "the largest context then is {}", + tokens(for_cache(KvCacheType::Q8_0)) + )), + "{}", + q8.text + ); + } + + /// A smaller cache's "largest context then" is capped at the trained context, + /// as the main ceiling is: a q4_0 cache with room for ~29k tokens on a model + /// trained for 8192 offers 8192, not a context the model was never trained + /// for. + #[test] + fn a_smaller_cache_suggestion_caps_its_largest_context_at_the_trained_one() { + let (header, size) = llama_trained_for(8192); + let fixed = estimate( + &header, + size, + 0, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ) + .total_gib + .unwrap(); + // 16384 tokens at f16 is 2 GiB of cache, at q4_0 about 0.56: room for 1. + let memory = AcceleratorMemory::Dedicated(fixed + 1.0); + let result = estimate(&header, size, 16384, KvCacheType::F16, memory); + assert_eq!(result.outcome, EstimateOutcome::DoesNotFit); + let uncapped = sizing(&header, size) + .unwrap() + .max_context((fixed + 1.0) * GIB, KvCacheType::Q4_0, None) + .unwrap(); + assert!( + uncapped > 8192, + "the cap must bind for this to mean anything" + ); + let q4 = result + .suggestions + .iter() + .find(|s| s.text.contains("q4_0 KV cache")) + .expect("a q4_0 suggestion"); + assert!( + q4.text + .contains(&format!("the largest context then is {}", tokens(8192))), + "{}", + q4.text + ); + } + + /// A request exactly at the ceiling fits, and the ceiling is still printed: + /// it is only a ceiling *below* a request that fits which is left out. + #[test] + fn a_request_at_exactly_the_ceiling_still_prints_the_ceiling() { + let (header, size) = llama(GIB_U as usize / 1024); + let kv = KvCacheType::F16; + let card = gpu_mib(shown_mib(&header, size, 24576, kv)); + let result = estimate(&header, size, 24576, kv, card); + assert_eq!(result.outcome, EstimateOutcome::Fits); + assert_eq!(result.max_context, Some(24576)); + let lines = evidence_lines(&result).join("\n"); + assert!( + lines.contains(&format!( + "largest context that fits, with the KV cache at f16: {}", + tokens(24576) + )), + "{lines}" + ); + } + + /// Either width missing understates the compute buffer, not only both, and + /// either one withholds the verdict. + #[test] + fn a_header_missing_either_width_says_the_compute_buffer_is_understated() { + let base = || { + Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 2) + .u32("x.attention.head_count", 4) + .u32("x.attention.key_length", 64) + .u32("x.attention.value_length", 64) + }; + for (which, builder) in [ + ( + "no feed-forward width", + base().u32("x.embedding_length", 256), + ), + ( + "no embedding width", + base().u32("x.feed_forward_length", 512), + ), + ( + "a feed-forward width of zero", + base() + .u32("x.embedding_length", 256) + .u32("x.feed_forward_length", 0), + ), + ( + "an embedding width of zero", + base() + .u32("x.embedding_length", 0) + .u32("x.feed_forward_length", 512), + ), + ] { + let (header, size) = header_of(&builder.tensor("w", &[1024]), 4096); + let result = estimate( + &header, + size, + 1024, + KvCacheType::F16, + AcceleratorMemory::Dedicated(8.0), + ); + assert!( + result.notes.iter().any(|note| note.contains("understated")), + "{which}: {:?}", + result.notes + ); + assert_eq!(result.outcome, EstimateOutcome::CannotEstimate, "{which}"); + } + } + + fn split_header(count: u32) -> Builder { + Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 4) + .u32("x.embedding_length", 512) + .u32("x.feed_forward_length", 1024) + .u32("x.attention.head_count", 8) + .u32("split.count", count) + .tensor("t", &[10]) + } + + #[test] + fn a_two_shard_model_is_refused_and_a_one_file_split_is_sized() { + assert_eq!( + cannot_estimate(&split_header(2)).cannot_estimate_reason, + Some(CannotEstimateReason::SplitFile), + "half the weights is not the model's" + ); + // `split.count` of 1 is a whole model that happens to carry the key. + let whole = cannot_estimate(&split_header(1)); + assert_ne!(whole.outcome, EstimateOutcome::CannotEstimate, "{whole:?}"); + assert!(whole.total_gib.is_some()); + } + + /// The measured Llama-shaped header, with `llama.context_length` set to `trained`. + fn llama_trained_for(trained: u32) -> (GgufHeader, u64) { + let builder = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", 32) + .u32("llama.embedding_length", 4096) + .u32("llama.feed_forward_length", 14336) + .u32("llama.attention.head_count", 32) + .u32("llama.attention.head_count_kv", 8) + .u32("llama.context_length", trained) + .tensor_at("token_embd.weight", &[4096, 10], 0) + .tensor_at("blk.0.attn_q.weight", &[64, 64], 40_960); + let (header, size) = header_of(&builder, 0); + (header, size + GIB_U) + } + + #[test] + fn a_trained_context_of_zero_is_no_trained_context() { + let (header, size) = llama_trained_for(0); + let result = estimate( + &header, + size, + 4096, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ); + // No cap: 64 GiB holds far more than the 131072 a real model would trap it at. + assert!(result.max_context.unwrap() > 131_072, "{result:?}"); + assert_eq!(result.training_context, None); + assert!( + !result.notes.iter().any(|note| note.contains("trained for")), + "a request was called beyond zero trained tokens: {:?}", + result.notes + ); + } + + #[test] + fn only_a_request_past_the_trained_context_is_told_so() { + let (header, size) = llama_trained_for(8192); + let notes = |context| { + estimate( + &header, + size, + context, + KvCacheType::F16, + AcceleratorMemory::Dedicated(64.0), + ) + .notes + }; + let beyond = |notes: &[String]| notes.iter().any(|n| n.contains("beyond the")); + assert!(!beyond(¬es(8192)), "the trained context itself is fine"); + assert!(beyond(¬es(8193))); + } + + #[test] + fn each_part_of_the_shown_total_rounds_to_a_byte_and_up_to_a_whole_mib() { + let mib = MIB as f64; + // A part 0.6 of a byte over a whole MiB is a byte over it: it rounds up + // to the next MiB. A truncating conversion would show the MiB below. + assert_eq!( + shown_sum([mib.mul_add(1000.0, 0.6), 0.0, 0.0, 0.0]) as u64, + 1001 * MIB + ); + // 0.4 of a byte over rounds down to the MiB itself. + assert_eq!( + shown_sum([mib.mul_add(1000.0, 0.4), 0.0, 0.0, 0.0]) as u64, + 1000 * MIB + ); + } + + #[test] + fn the_byte_fields_round_a_fractional_cache_size_to_the_nearest_byte() { + // One layer, one KV head, key and value 4 and 6 wide: 10 elements a + // token, 10.625 bytes at q8_0 for one token. + let builder = Builder::new() + .string("general.architecture", "x") + .u32("x.block_count", 1) + .u32("x.embedding_length", 64) + .u32("x.attention.head_count", 1) + .u32("x.attention.head_count_kv", 1) + .u32("x.attention.key_length", 4) + .u32("x.attention.value_length", 6) + .tensor("t", &[10]); + let (header, size) = header_of(&builder, 4096); + let result = estimate( + &header, + size, + 1, + KvCacheType::Q8_0, + AcceleratorMemory::Dedicated(8.0), + ); + assert_eq!(result.kv_cache_bytes, Some(11)); + let parts = result.weights_bytes.unwrap() + + 11 + + result.compute_buffer_bytes.unwrap() + + result.runtime_reserve_bytes.unwrap(); + assert_eq!(result.total_bytes, Some(parts)); + } + + /// A model with `layers` cached layers of 8 KV heads and 128-wide keys and values. + fn few_layered(layers: u32) -> (GgufHeader, u64) { + let builder = Builder::new() + .string("general.architecture", "llama") + .u32("llama.block_count", layers) + .u32("llama.embedding_length", 4096) + .u32("llama.feed_forward_length", 14336) + .u32("llama.attention.head_count", 32) + .u32("llama.attention.head_count_kv", 8) + .u32("llama.context_length", 131_072) + .tensor_at("token_embd.weight", &[4096, 10], 0) + .tensor_at("blk.0.attn_q.weight", &[64, 64], 40_960); + let (header, size) = header_of(&builder, 0); + (header, size + GIB_U) + } + + #[test] + fn a_quantized_cache_is_not_offered_where_it_would_need_more_than_the_wider_one() { + // With two cached layers a q8_0 cache (0.53 GiB at this context) plus the + // f16 copy of one layer it is dequantized through (0.5 GiB) comes to more + // than the f16 cache itself (1 GiB), mask aside: it would not cut the need. + let (header, size) = few_layered(2); + let suggestions = |memory_gib| { + estimate( + &header, + size, + 131_072, + KvCacheType::F16, + AcceleratorMemory::Dedicated(memory_gib), + ) + .suggestions + }; + let texts: Vec = suggestions(2.5).into_iter().map(|s| s.text).collect(); + assert!( + !texts.iter().any(|t| t.contains("q8_0 KV cache")), + "{texts:?}" + ); + assert!( + texts.iter().any(|t| t.contains("q4_0 KV cache")), + "{texts:?}" + ); + // With four layers the same cache does save, and is offered. + let (header, size) = few_layered(4); + let texts: Vec = estimate( + &header, + size, + 131_072, + KvCacheType::F16, + AcceleratorMemory::Dedicated(2.5), + ) + .suggestions + .into_iter() + .map(|s| s.text) + .collect(); + assert!( + texts.iter().any(|t| t.contains("q8_0 KV cache")), + "{texts:?}" + ); + } +} diff --git a/crates/rocm-core/src/gguf_source.rs b/crates/rocm-core/src/gguf_source.rs new file mode 100644 index 0000000000..74b0b12da0 --- /dev/null +++ b/crates/rocm-core/src/gguf_source.rs @@ -0,0 +1,2508 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Where a GGUF header comes from: a local file, or a Hugging Face repository +//! read by HTTP range requests. +//! +//! A remote file is found by one repository-listing request (plus a `HEAD` +//! request for a file the listing does not size), then read in windows. +//! +//! The point of the range reader is that a model's *shape* is in the first few +//! megabytes of a file that may be tens of gigabytes. [`RangedReader`] presents +//! the remote file as `Read + Seek` and fetches it in 4 MiB windows on demand, so +//! [`crate::gguf::parse_gguf_header`] stops reading where the header ends and +//! the weights are never requested. A server that ignores `Range` is refused +//! rather than streamed, because streaming it is exactly the download this +//! exists to avoid. + +use anyhow::{Context, Result, anyhow, bail}; +use serde_json::Value; +use std::fs::File; +use std::io::{self, BufReader, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +/// How much of the remote file one request asks for. +const WINDOW_BYTES: u64 = 4 * 1024 * 1024; +/// The most a header read may fetch in total. Real headers are a few megabytes +/// (mostly the tokenizer); a read that goes beyond this is not reading a header. +const MAX_FETCH_BYTES: u64 = crate::gguf::MAX_HEADER_BYTES; +/// A repository listing larger than this is not read. +const MAX_LISTING_BYTES: u64 = 16 * 1024 * 1024; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const REQUEST_TIMEOUT: Duration = Duration::from_mins(1); +/// How long listing a repository and reading a header may take in all. The +/// per-request timeout alone would let sixteen slow windows hold the command for +/// sixteen minutes. +const TOTAL_BUDGET: Duration = Duration::from_mins(2); +const DEFAULT_HUB: &str = "https://huggingface.co"; + +/// A model reference that names a GGUF file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GgufRef { + /// A path on this machine. + Local(PathBuf), + /// `owner/repo` on Hugging Face, with the quantization label or file name + /// after a colon. The variant is optional here so that a reference without + /// one can be refused with a message, not mistaken for something else. + Hub { + owner: String, + repo: String, + variant: Option, + }, +} + +/// Whether a string is a safe Hugging Face path component: the charset of +/// `is_safe_hf_component` in the lemonade engine, which is what `rocm serve` +/// applies to `owner/repo:variant`, so a reference serve would refuse is not sent +/// to the hub either. +fn is_safe_component(component: &str) -> bool { + !component.is_empty() + && component != "." + && component != ".." + && component + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) +} + +/// Whether a string is a valid `:variant`: a quantization label, or a file name +/// ending in `.gguf` that may carry a repository sub-directory +/// (`dir/model-Q4_K_M.gguf`), which is the only way to name one of two files that +/// share a name in different directories. `rocm serve` takes no `/` in a variant; +/// here it is safe because every segment is checked and the whole is sent to the +/// hub as percent-encoded path segments, never to the filesystem. +/// +/// A file name may also carry `+`, which hub file names do (`m+x-Q4_K_M.gguf`): +/// it reaches the hub percent-encoded, so it is as safe as the other characters, +/// and refusing it would leave such a file impossible to name exactly. +fn is_safe_variant(variant: &str) -> bool { + if is_safe_component(variant) { + return true; + } + let file_component = |component: &str| is_safe_component(&component.replace('+', "_")); + variant.to_ascii_lowercase().ends_with(".gguf") + && variant.split('/').count() <= 8 + && variant.split('/').all(file_component) +} + +/// Read a model reference as a GGUF file, if it is shaped like one. +/// +/// A file that exists on this machine is that file, whatever its name looks +/// like. Otherwise `owner/repo:variant` is a Hugging Face repository, a name +/// ending in `.gguf` is a local path (that does not exist, so it is reported as +/// not openable rather than sent to the hub), and a bare `owner/repo` is a +/// repository with no variant, recognised so the caller can say a quantization is +/// needed. Returns `None` for names, aliases and anything else, so a catalog model +/// never reaches this module. +#[must_use] +pub fn parse_gguf_ref(model_ref: &str) -> Option { + parse_gguf_ref_with(model_ref, &|path| path.is_file()) +} + +/// [`parse_gguf_ref`] with the "does this file exist" question answered by the +/// caller, so a test can say a path exists without a working directory to put it +/// in. +fn parse_gguf_ref_with(model_ref: &str, is_file: &dyn Fn(&Path) -> bool) -> Option { + let trimmed = model_ref.trim(); + if trimmed.is_empty() { + return None; + } + if is_file(Path::new(trimmed)) { + return Some(GgufRef::Local(PathBuf::from(trimmed))); + } + let is_gguf_name = |name: &str| name.to_ascii_lowercase().ends_with(".gguf"); + if let Some((repo_path, variant)) = trimmed.split_once(':') + && let Some((owner, repo)) = repo_path.split_once('/') + && is_safe_component(owner) + && is_safe_component(repo) + && is_safe_variant(variant) + { + return Some(GgufRef::Hub { + owner: owner.to_owned(), + repo: repo.to_owned(), + variant: Some(variant.to_owned()), + }); + } + if is_gguf_name(trimmed) { + return Some(GgufRef::Local(PathBuf::from(trimmed))); + } + let (owner, repo) = trimmed.split_once('/')?; + (is_safe_component(owner) && is_safe_component(repo)).then(|| GgufRef::Hub { + owner: owner.to_owned(), + repo: repo.to_owned(), + variant: None, + }) +} + +/// A file of a Hugging Face repository. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HubFile { + pub name: String, + /// The size the listing reported; `None` when it did not. + pub size: Option, +} + +fn is_boundary(c: Option, extra: &[char]) -> bool { + c.is_none_or(|c| matches!(c, '-' | '.' | '/') || extra.contains(&c)) +} + +/// Whether `label` appears in `name` as a whole quantization label: bounded by +/// separators, so `Q4_0` does not match `Q4_0_4_4`. +fn has_label(name: &str, label: &str) -> bool { + let name = name.to_ascii_lowercase(); + let label = label.to_ascii_lowercase(); + name.match_indices(&label).any(|(start, _)| { + let before = name[..start].chars().next_back(); + let after = name[start + label.len()..].chars().next(); + is_boundary(before, &['_']) && is_boundary(after, &[]) + }) +} + +/// A name that matches only multimodal projectors, which are not models. +/// +/// Raised by [`select_gguf_file`] as a typed error so a caller can tell it from +/// a file that could not be found or read, and offer one of `models` instead. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectorNamed { + /// What the user named, as they wrote it. + pub variant: String, + /// The projector file or files that name matched, by full name in the + /// repository: what the user actually pointed at. + pub projectors: Vec, + /// The repository's model files (every GGUF that is not a projector), by + /// full name. + pub models: Vec, +} + +impl std::fmt::Display for ProjectorNamed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Capped as `names_for_error` caps a listing; `models` itself stays + // complete. + const SHOWN: usize = 8; + let models = if self.models.is_empty() { + "none: every GGUF file in it is a projector".to_owned() + } else { + let mut shown: Vec<&str> = self.models.iter().take(SHOWN).map(String::as_str).collect(); + if self.models.len() > SHOWN { + shown.push("..."); + } + shown.join(", ") + }; + let mut named: Vec = self + .projectors + .iter() + .take(SHOWN) + .map(|name| format!("`{name}`")) + .collect(); + if self.projectors.len() > SHOWN { + named.push("...".to_owned()); + } + let named = named.join(", "); + let is = if self.projectors.len() == 1 { + "is a multimodal projector" + } else { + "are multimodal projectors" + }; + write!( + f, + "{named} {is} (mmproj), not a language model, so `{}` cannot be estimated; name \ + the model's own GGUF file or its quantization (the repository's models: {models})", + self.variant + ) + } +} + +impl std::error::Error for ProjectorNamed {} + +/// Whether a GGUF file name is a multimodal projector (`mmproj`, in any case), +/// which ships beside a vision model and is not a language model itself. +#[must_use] +pub fn names_projector(name: &str) -> bool { + name.to_ascii_lowercase().contains("mmproj") +} + +/// Pick the GGUF file a `variant` names out of a repository's files. +/// +/// An exact file name, or a quantization label as `rocm serve` takes it. Never +/// picks between two quantizations: a label that matches several files is an +/// error listing them, except that the shards of one split model resolve to +/// their first shard. A multimodal projector (`mmproj` in its name, in any +/// case) is not a model: it is never picked, and a file name that matches only +/// projectors is refused as naming one ([`ProjectorNamed`]) rather than +/// reported as missing. A quantization label that matches only a projector is +/// reported as matching nothing. +/// +/// # Errors +/// +/// When nothing matches, the match is ambiguous, or the file named is a +/// projector. +pub fn select_gguf_file<'a>(files: &'a [HubFile], variant: &str) -> Result<&'a HubFile> { + let is_projector = |file: &HubFile| names_projector(&file.name); + let all_ggufs: Vec<&HubFile> = files + .iter() + .filter(|file| file.name.to_ascii_lowercase().ends_with(".gguf")) + .collect(); + let ggufs: Vec<&HubFile> = all_ggufs + .iter() + .copied() + .filter(|file| !is_projector(file)) + .collect(); + let matches: Vec<&HubFile> = if variant.to_ascii_lowercase().ends_with(".gguf") { + // A name is unique in a repository, so one that matches in full is the + // file asked for, even where another file in a directory has the same + // last component (`m.gguf` at the root, `dir/m.gguf` below it). Searched + // over every GGUF, projectors included, so that naming one is answered + // for what it is. + let exact: Vec<&HubFile> = all_ggufs + .iter() + .copied() + .filter(|file| file.name == variant) + .collect(); + let named = if exact.is_empty() { + all_ggufs + .iter() + .copied() + .filter(|file| file.name.rsplit('/').next() == Some(variant)) + .collect() + } else { + exact + }; + // A projector named this way is not the model asked for. Where the name + // also matches a model (a bare name both share), the model is the + // answer; where it matches only projectors, the user is told what they + // named rather than that it is missing. + let models: Vec<&HubFile> = named + .iter() + .copied() + .filter(|file| !is_projector(file)) + .collect(); + if models.is_empty() && !named.is_empty() { + return Err(anyhow::Error::new(ProjectorNamed { + variant: variant.to_owned(), + projectors: named.iter().map(|file| file.name.clone()).collect(), + models: ggufs.iter().map(|file| file.name.clone()).collect(), + })); + } + models + } else { + ggufs + .iter() + .copied() + .filter(|file| has_label(&file.name, variant)) + .collect() + }; + match matches.as_slice() { + [only] => Ok(only), + // The repository's GGUF files are listed with the projectors among them, + // marked as such, rather than left out (up to eight, then `...`). + [] => bail!( + "no GGUF file matching `{variant}` in the repository (it has: {})", + names_for_error_marking_projectors(&all_ggufs) + ), + several => first_shard_of_one_set(several).ok_or_else(|| { + anyhow!( + "`{variant}` matches several files ({}); name one exactly, as \ + `owner/repo:file.gguf` (or `owner/repo:dir/file.gguf` when the name is not \ + unique)", + names_for_error(several) + ) + }), + } +} + +/// Split a shard's name `-00002-of-00004.gguf` into what the shards of one +/// model share (the stem and the total) and the shard's own index. +fn shard_parts(name: &str) -> Option<(String, u32)> { + let of = name.rfind("-of-")?; + let digits = |text: &str| -> Option { + (text.len() == 5 && text.bytes().all(|b| b.is_ascii_digit())) + .then(|| text.parse().ok()) + .flatten() + }; + let index_start = of.checked_sub(5)?; + let index = digits(name.get(index_start..of)?)?; + let stem = name.get(..index_start)?.strip_suffix('-')?; + let rest = name.get(of + 4..)?; + let total_end = rest.find('.').unwrap_or(rest.len()); + let total = digits(rest.get(..total_end)?)?; + Some((format!("{stem}|{total}|{}", &rest[total_end..]), index)) +} + +/// The first shard, when every candidate is a shard of the same split model. +/// Shards that belong to different models (two sets under one label) are +/// ambiguous, not one model: taking "the" first shard would size the wrong one. +fn first_shard_of_one_set<'a>(candidates: &[&'a HubFile]) -> Option<&'a HubFile> { + let parts: Vec<(String, u32)> = candidates + .iter() + .map(|file| shard_parts(&file.name)) + .collect::>()?; + let set = &parts.first()?.0; + if parts.iter().any(|(other, _)| other != set) { + return None; + } + candidates + .iter() + .zip(&parts) + .find(|(_, (_, index))| *index == 1) + .map(|(file, _)| *file) +} + +/// [`names_for_error`], with each multimodal projector marked as one. +fn names_for_error_marking_projectors(files: &[&HubFile]) -> String { + const SHOWN: usize = 8; + let mut names: Vec = files + .iter() + .take(SHOWN) + .map(|file| { + if names_projector(&file.name) { + format!("{} (multimodal projector)", file.name) + } else { + file.name.clone() + } + }) + .collect(); + if files.len() > SHOWN { + names.push("...".to_owned()); + } + if names.is_empty() { + "no GGUF files".to_owned() + } else { + names.join(", ") + } +} + +fn names_for_error(files: &[&HubFile]) -> String { + const SHOWN: usize = 8; + let mut names: Vec<&str> = files.iter().take(SHOWN).map(|f| f.name.as_str()).collect(); + if files.len() > SHOWN { + names.push("..."); + } + if names.is_empty() { + "no GGUF files".to_owned() + } else { + names.join(", ") + } +} + +/// `Read + Seek`, as one object-safe trait. +pub trait ReadSeek: Read + Seek {} +impl ReadSeek for T {} + +/// A GGUF that has been opened for header reading. +pub struct OpenedGguf { + pub reader: Box, + /// The size of the whole file, in bytes. + pub size: u64, + /// What to call the file in a report. + pub description: String, +} + +/// A buffered reader whose relative seeks stay inside its buffer where they can. +/// +/// The header parser skips every string it does not keep with a relative seek, +/// and a vocabulary is 128k to 262k of them. `BufReader::seek` drops the buffer +/// every time, so each skip became a syscall and the next read a refill: a +/// 300k-token header took seconds to read from disk. `seek_relative` keeps the +/// buffer when the target is inside it. The position is tracked here, because +/// a relative seek has to know where it lands to report it, and `BufReader`'s +/// own `stream_position` would ask the file, which is a seek too. +struct RelativeSeekBuf { + inner: BufReader, + /// Where the next read starts, from the start of the stream. + position: u64, +} + +impl RelativeSeekBuf { + /// Over a stream positioned at its start. + const fn new(inner: BufReader) -> Self { + Self { inner, position: 0 } + } +} + +impl Read for RelativeSeekBuf { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let read = self.inner.read(buf)?; + self.position += read as u64; + Ok(read) + } +} + +impl Seek for RelativeSeekBuf { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match pos { + SeekFrom::Current(offset) => { + let target = self.position.checked_add_signed(offset).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "seek before the start") + })?; + self.inner.seek_relative(offset)?; + self.position = target; + } + other => self.position = self.inner.seek(other)?, + } + Ok(self.position) + } + + fn stream_position(&mut self) -> io::Result { + Ok(self.position) + } +} + +/// Opens a GGUF named by a [`GgufRef`]. A trait so the estimate can be tested +/// without a network, and so the one real implementation is the only place that +/// reaches for one. +pub trait GgufSource { + /// # Errors + /// + /// When the file cannot be found or reached, or (for a hub file named by + /// its file name) is a multimodal projector: then the error is a + /// [`ProjectorNamed`], which a caller can downcast to. + fn open(&self, reference: &GgufRef) -> Result; +} + +/// The real source: the local filesystem and Hugging Face. +#[derive(Debug, Clone)] +pub struct LiveGgufSource { + hub: String, + /// How long listing a repository and reading a header may take in all. + budget: Duration, +} + +impl Default for LiveGgufSource { + fn default() -> Self { + Self::new() + } +} + +impl LiveGgufSource { + /// Uses `HF_ENDPOINT` as the Hugging Face base URL when it is set (the + /// variable the Hugging Face tooling reads), else `https://huggingface.co`. + #[must_use] + pub fn new() -> Self { + let hub = std::env::var("HF_ENDPOINT") + .ok() + .map(|value| value.trim().trim_end_matches('/').to_owned()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| DEFAULT_HUB.to_owned()); + Self { + hub, + budget: TOTAL_BUDGET, + } + } + + /// A source against an explicit hub base URL. + #[cfg(test)] + #[must_use] + pub fn with_hub(hub: &str) -> Self { + Self { + hub: hub.trim_end_matches('/').to_owned(), + budget: TOTAL_BUDGET, + } + } + + /// The same source with a shorter total budget, so a test can show the + /// reader is bound by the deadline `open` sets rather than by a fresh one. + #[cfg(test)] + #[must_use] + const fn with_budget(mut self, budget: Duration) -> Self { + self.budget = budget; + self + } + + fn agent() -> ureq::Agent { + // The same User-Agent the CLI's other downloads send. The agent follows + // no redirect itself: [`send`] does, so that it can refuse one that + // leaves https for plain http. + ureq::AgentBuilder::new() + .user_agent("rocm-cli") + .redirects(0) + .timeout_connect(CONNECT_TIMEOUT) + .timeout(REQUEST_TIMEOUT) + .build() + } + + fn list_files( + &self, + agent: &ureq::Agent, + owner: &str, + repo: &str, + deadline: Instant, + ) -> Result> { + let url = format!("{}/api/models/{owner}/{repo}?blobs=true", self.hub); + let response = send( + agent, + "GET", + &url, + deadline, + None, + &format!("`{owner}/{repo}`"), + )?; + let mut text = Vec::new(); + // One byte past the cap, so a listing that is exactly too long is told + // apart from one that fits instead of being cut short and misread as bad + // JSON. Read as bytes: the cut can fall inside a multi-byte character, + // and reading it as text would report that, not the size. + response + .into_reader() + .take(MAX_LISTING_BYTES + 1) + .read_to_end(&mut text) + .with_context(|| format!("could not read the file list of `{owner}/{repo}`"))?; + if text.len() as u64 > MAX_LISTING_BYTES { + bail!( + "the file list of `{owner}/{repo}` is larger than {} MiB, which is not read", + MAX_LISTING_BYTES / (1024 * 1024) + ); + } + let body: Value = serde_json::from_slice(&text) + .with_context(|| format!("the file list of `{owner}/{repo}` was not JSON"))?; + let siblings = body + .get("siblings") + .and_then(Value::as_array) + .ok_or_else(|| anyhow!("the file list of `{owner}/{repo}` has no `siblings`"))?; + // A name the hub lists is put into a request path, so a name that could + // climb out of the repository (`..`), or hide in a terminal (a control, + // bidirectional or zero-width character), is dropped here rather than + // offered, requested or printed. + // A real repository has none; a hostile mirror is what could. + Ok(siblings + .iter() + .filter_map(|sibling| { + let name = sibling.get("rfilename")?.as_str()?; + is_safe_listed_name(name).then(|| HubFile { + name: name.to_owned(), + size: sibling.get("size").and_then(Value::as_u64), + }) + }) + .collect()) + } +} + +/// Whether a file name from a repository listing may be requested and shown: no +/// empty, `.` or `..` path segment, and no character that changes how a terminal +/// shows the text around it: the predicates in `terminal` decide which, so this +/// and the other untrusted text the CLI prints are held to one set. +fn is_safe_listed_name(name: &str) -> bool { + !name.is_empty() + && !name.chars().any(|c| { + crate::terminal::is_control_or_format(c) + || crate::terminal::is_control_or_line_separator(c) + }) + && name + .split('/') + .all(|segment| !segment.is_empty() && segment != "." && segment != "..") +} + +fn describe_http_error(error: &ureq::Error, what: &str) -> anyhow::Error { + match error { + // The hub answers 401 for a repository that does not exist as well as + // for one that is gated or private, so the two cannot be told apart. No + // token is ever sent, so authentication is not something to offer. + ureq::Error::Status(401 | 403 | 404, _) => anyhow!( + "{what} was not found, or its repository is gated or private; no token is sent, so \ + only public repositories can be read" + ), + ureq::Error::Status(code, _) => anyhow!("the hub answered {code} for {what}"), + ureq::Error::Transport(transport) => { + anyhow!("could not reach the hub for {what}: {transport}") + } + } +} + +/// What to say when a path that does not exist is shaped like a file in a Hugging +/// Face repository (`owner/repo/dir/file.gguf`): the hub form of the same name. +/// Empty for anything else. +fn hub_hint(path: &Path) -> String { + let Some(text) = path.to_str() else { + return String::new(); + }; + let parts: Vec<&str> = text.split('/').collect(); + if path.is_absolute() + || parts.len() < 3 + || !text.to_ascii_lowercase().ends_with(".gguf") + || !parts.iter().all(|part| is_safe_component(part)) + { + return String::new(); + } + format!( + " (if you meant a file in a Hugging Face repository, write it `{}/{}:{}`, with a colon \ + after the repository)", + parts[0], + parts[1], + parts[2..].join("/") + ) +} + +impl GgufSource for LiveGgufSource { + fn open(&self, reference: &GgufRef) -> Result { + match reference { + GgufRef::Local(path) => { + // Checked before opening: opening a named pipe blocks until + // something writes to it, and a directory opens and then fails + // on its first read. + let metadata = std::fs::metadata(path).with_context(|| { + format!("could not open {}{}", path.display(), hub_hint(path)) + })?; + if !metadata.is_file() { + bail!("{} is not a regular file", path.display()); + } + let file = File::open(path) + .with_context(|| format!("could not open {}", path.display()))?; + let size = metadata.len(); + Ok(OpenedGguf { + reader: Box::new(RelativeSeekBuf::new(BufReader::new(file))), + size, + description: path.display().to_string(), + }) + } + GgufRef::Hub { + owner, + repo, + variant, + } => { + let Some(variant) = variant else { + bail!( + "`{owner}/{repo}` needs a quantization to read: use \ + `{owner}/{repo}:QUANT` (for example `{owner}/{repo}:Q4_K_M`) or a file \ + name ending in .gguf" + ); + }; + let agent = Self::agent(); + let deadline = Instant::now() + self.budget; + let files = self.list_files(&agent, owner, repo, deadline)?; + let file = select_gguf_file(&files, variant)?; + let path: Vec = file + .name + .split('/') + .map(crate::report_delivery::percent_encode) + .collect(); + let url = format!( + "{}/{owner}/{repo}/resolve/main/{}", + self.hub, + path.join("/") + ); + // The listing carries a size for a file the hub stores whole; if + // it left one out, the file's own headers say. + let size = match file.size { + Some(size) => size, + None => head_size(&agent, &url, deadline)?, + }; + Ok(OpenedGguf { + reader: Box::new(RangedReader::new(agent, url, size).with_deadline(deadline)), + size, + description: format!("{owner}/{repo}: {}", file.name), + }) + } + } + } +} + +/// How long is left before `deadline`, as the timeout for the next request. +fn time_left(deadline: Instant) -> io::Result { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(io::Error::other(format!( + "reading the header took longer than {} seconds", + TOTAL_BUDGET.as_secs() + ))); + } + Ok(left.min(REQUEST_TIMEOUT)) +} + +/// The most redirects one request follows: the hub sends `resolve/main` to its +/// CDN, and a chain longer than this is a loop or a mirror that is not one. +const MAX_REDIRECTS: usize = 5; + +/// Send one request and follow its redirects by hand, each hop under what is +/// left of `deadline`, keeping the `Range` header on every hop. +/// +/// By hand because the agent cannot say where a redirect is going until it has +/// gone there, and [`redirect_target`] must refuse one that leaves https for +/// plain http: `HF_ENDPOINT` is an environment variable, and a mirror that is +/// not trusted must not be able to send the request out in the clear. +fn send( + agent: &ureq::Agent, + method: &str, + url: &str, + deadline: Instant, + range: Option<&str>, + what: &str, +) -> Result { + let mut current = url.to_owned(); + for _ in 0..=MAX_REDIRECTS { + let mut request = agent + .request(method, ¤t) + .timeout(time_left(deadline).map_err(|error| anyhow!("{error}"))?); + if let Some(range) = range { + request = request.set("Range", range); + } + let response = request + .call() + .map_err(|error| describe_http_error(&error, what))?; + if !matches!(response.status(), 301 | 302 | 303 | 307 | 308) { + return Ok(response); + } + let location = response + .header("Location") + .ok_or_else(|| anyhow!("the hub redirected {what} without saying where"))?; + current = redirect_target(¤t, location, what)?; + } + bail!("the hub redirected {what} more than {MAX_REDIRECTS} times") +} + +/// Where a redirect from `current` to `location` goes, or why it is refused: it +/// must stay on http(s), and a request that began on https never ends on http. +fn redirect_target(current: &str, location: &str, what: &str) -> Result { + let location = location.trim(); + let (scheme, rest) = current + .split_once("://") + .ok_or_else(|| anyhow!("{what} is at `{current}`, which is not a URL"))?; + let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest); + let lower = location.to_ascii_lowercase(); + let target = if lower.starts_with("http://") || lower.starts_with("https://") { + location.to_owned() + } else if location.starts_with("//") { + format!("{scheme}:{location}") + } else if location.starts_with('/') { + format!("{scheme}://{authority}{location}") + } else if location.contains("://") { + bail!("the hub redirected {what} to something that is not http(s)"); + } else { + let path = rest[authority.len()..] + .split(['?', '#']) + .next() + .unwrap_or(""); + let directory = path.rsplit_once('/').map_or("", |(directory, _)| directory); + format!("{scheme}://{authority}{directory}/{location}") + }; + if scheme.eq_ignore_ascii_case("https") && target.to_ascii_lowercase().starts_with("http://") { + bail!( + "the hub redirected {what} from https to plain http, which is refused: no token is \ + sent, but the request must not leave an encrypted connection" + ); + } + Ok(target) +} + +/// The size of a remote file from its `Content-Length`, without fetching it. +fn head_size(agent: &ureq::Agent, url: &str, deadline: Instant) -> Result { + let response = send(agent, "HEAD", url, deadline, None, "the model file")?; + response + .header("Content-Length") + .and_then(|value| value.trim().parse().ok()) + .ok_or_else(|| anyhow!("the hub did not report how large the model file is")) +} + +/// A remote file as `Read + Seek`, fetched in windows by `Range` request. +pub struct RangedReader { + agent: ureq::Agent, + url: String, + len: u64, + position: u64, + window: Vec, + window_start: u64, + fetched: u64, + deadline: Instant, +} + +impl RangedReader { + #[must_use] + pub fn new(agent: ureq::Agent, url: String, len: u64) -> Self { + Self { + agent, + url, + len, + position: 0, + window: Vec::new(), + window_start: 0, + fetched: 0, + deadline: Instant::now() + TOTAL_BUDGET, + } + } + + /// Stop reading at `deadline`, however many windows are left. + #[must_use] + pub const fn with_deadline(mut self, deadline: Instant) -> Self { + self.deadline = deadline; + self + } + + /// Bytes requested from the server so far. + #[cfg(test)] + #[must_use] + pub const fn fetched(&self) -> u64 { + self.fetched + } + + fn fill(&mut self) -> io::Result<()> { + let end = self.position.saturating_add(WINDOW_BYTES).min(self.len); + let wanted = end - self.position; + if self.fetched.saturating_add(wanted) > MAX_FETCH_BYTES { + return Err(io::Error::other(format!( + "the header is larger than {} MiB; this is not a GGUF header read", + MAX_FETCH_BYTES / (1024 * 1024) + ))); + } + let response = send( + &self.agent, + "GET", + &self.url, + self.deadline, + Some(&format!("bytes={}-{}", self.position, end - 1)), + "the model file", + ) + .map_err(|error| io::Error::other(format!("{error:#}")))?; + // 200 means the server ignored the range and would send the whole file. + if response.status() != 206 { + return Err(io::Error::other(format!( + "the server answered {} to a range request; it does not support reading part of \ + a file, and the whole file is not downloaded to read its header", + response.status() + ))); + } + // A 206 says which bytes it carries. One that starts somewhere else would + // be parsed as if it began at `position` -- a header read from the wrong + // offset -- so it is refused, as the download path refuses it. + let start = crate::download::content_range_start(&response); + if start != Some(self.position) { + return Err(io::Error::other(format!( + "the server answered a range that starts at {} when {} was asked for", + start.map_or_else(|| "an unstated offset".to_owned(), |at| at.to_string()), + self.position + ))); + } + // `wanted` is at most WINDOW_BYTES already; the clamp states that bound + // where the allocation is made. A comparison rather than `min`, so a + // static analyser sees the bound as well as a reader does. + let capacity = if wanted <= WINDOW_BYTES { + wanted + } else { + WINDOW_BYTES + }; + let mut window = Vec::with_capacity(usize::try_from(capacity).unwrap_or(0)); + // A body that ends early surfaces as `UnexpectedEof`, which would read as + // "the header ends here". It is the connection that ended. + if let Err(error) = response.into_reader().take(wanted).read_to_end(&mut window) { + return Err(io::Error::other(format!( + "the connection ended after {} of the {wanted} bytes asked for: {error}", + window.len() + ))); + } + if window.len() as u64 != wanted { + // Not `UnexpectedEof`: that reads as "the header ends here", which + // would blame the file for a connection that dropped. + return Err(io::Error::other(format!( + "the connection ended after {} of the {wanted} bytes asked for", + window.len() + ))); + } + self.fetched += wanted; + self.window = window; + self.window_start = self.position; + Ok(()) + } +} + +impl Read for RangedReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.position >= self.len || buf.is_empty() { + return Ok(0); + } + let window_end = self.window_start + self.window.len() as u64; + if self.position < self.window_start || self.position >= window_end { + self.fill()?; + } + let offset = usize::try_from(self.position - self.window_start) + .map_err(|_| io::Error::other("offset out of range"))?; + let count = buf.len().min(self.window.len() - offset); + buf[..count].copy_from_slice(&self.window[offset..offset + count]); + self.position += count as u64; + Ok(count) + } +} + +impl Seek for RangedReader { + fn seek(&mut self, target: SeekFrom) -> io::Result { + let next = match target { + SeekFrom::Start(offset) => Some(offset), + SeekFrom::Current(delta) => self.position.checked_add_signed(delta), + SeekFrom::End(delta) => self.len.checked_add_signed(delta), + }; + let next = next.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "seek before the start of the file", + ) + })?; + self.position = next; + Ok(next) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::gguf::test_builder::llama_like; + use crate::gguf::{GgufError, parse_gguf_header}; + use std::io::{Cursor, Write as _}; + use std::net::TcpListener; + use std::sync::{Arc, Mutex}; + use std::thread; + + fn file(name: &str, size: u64) -> HubFile { + HubFile { + name: name.to_owned(), + size: Some(size), + } + } + + #[test] + fn a_reference_is_read_as_a_hub_repository_a_path_or_neither() { + assert_eq!( + parse_gguf_ref("owner/repo:Q4_K_M"), + Some(GgufRef::Hub { + owner: "owner".into(), + repo: "repo".into(), + variant: Some("Q4_K_M".into()) + }) + ); + assert_eq!( + parse_gguf_ref("owner/repo:model-Q4_K_M.gguf"), + Some(GgufRef::Hub { + owner: "owner".into(), + repo: "repo".into(), + variant: Some("model-Q4_K_M.gguf".into()) + }) + ); + // A bare repository is recognised so it can be refused with a message. + assert_eq!( + parse_gguf_ref("owner/repo"), + Some(GgufRef::Hub { + owner: "owner".into(), + repo: "repo".into(), + variant: None + }) + ); + // A path ending in .gguf is a file even though it looks like owner/repo. + assert_eq!( + parse_gguf_ref("models/thing.gguf"), + Some(GgufRef::Local(PathBuf::from("models/thing.gguf"))) + ); + assert_eq!( + parse_gguf_ref("/abs/path/Thing.GGUF"), + Some(GgufRef::Local(PathBuf::from("/abs/path/Thing.GGUF"))) + ); + for not_a_gguf in [ + "qwen", + "", + " ", + "a/b/c", + "../x/y:Q4", + "owner/..:Q4", + "o/r:a/b", + ] { + assert_eq!(parse_gguf_ref(not_a_gguf), None, "{not_a_gguf:?}"); + } + } + + /// `m.gguf` at the root and `dir/m.gguf` share a last component. Naming the + /// full path of either picks that one; the bare name picks the root file, which + /// matches in full, instead of reporting the two as ambiguous. + #[test] + fn a_full_path_match_beats_a_match_on_the_last_component() { + let files = [file("dir/m.gguf", 4), file("m.gguf", 4)]; + assert_eq!(select_gguf_file(&files, "m.gguf").unwrap().name, "m.gguf"); + assert_eq!( + select_gguf_file(&files, "dir/m.gguf").unwrap().name, + "dir/m.gguf" + ); + // With no root file the bare name finds the one nested file it names... + let nested = [file("dir/m.gguf", 4), file("dir/other.gguf", 4)]; + assert_eq!( + select_gguf_file(&nested, "m.gguf").unwrap().name, + "dir/m.gguf" + ); + // ...and is ambiguous when it is the last component of two. + let two = [file("a/m.gguf", 4), file("b/m.gguf", 4)]; + assert!(select_gguf_file(&two, "m.gguf").is_err()); + } + + #[test] + fn listed_names_that_could_climb_out_of_the_repository_or_hide_in_a_terminal_are_dropped() { + for name in [ + "../m-Q4_K_M.gguf", + "a/../m-Q4_K_M.gguf", + "./m-Q4_K_M.gguf", + "a//m-Q4_K_M.gguf", + "/m-Q4_K_M.gguf", + "m-Q4_K_M\n.gguf", + "m-Q4_K_M\u{1b}[2J.gguf", + "m-\u{202E}fuggQ4_K_M.gguf", + "m-Q4_K_M\u{200B}.gguf", + "m-Q4_K_M\u{2066}.gguf", + // Both ends of each range, and the byte order mark. + "m-Q4_K_M\u{200F}.gguf", + "m-Q4_K_M\u{202A}.gguf", + "m-Q4_K_M\u{2069}.gguf", + "m-Q4_K_M\u{FEFF}.gguf", + // What the shared terminal predicates add: line and paragraph + // separators, invisible operators, the soft hyphen, tag characters. + "m-Q4_K_M\u{2028}.gguf", + "m-Q4_K_M\u{2029}.gguf", + "m-Q4_K_M\u{2060}.gguf", + "m-Q4_K_M\u{2064}.gguf", + "m-Q4\u{00AD}_K_M.gguf", + "m-Q4_K_M\u{E0041}.gguf", + ] { + assert!(!is_safe_listed_name(name), "{name:?}"); + } + for name in ["m-Q4_K_M.gguf", "dir/m-Q4_K_M.gguf", "a.b/c_d-e.gguf"] { + assert!(is_safe_listed_name(name), "{name:?}"); + } + } + + /// The same through a listing a hub sent: the hostile names are not offered, + /// so the one safe file is the only match and nothing but it is requested. + #[test] + fn a_hostile_listing_is_not_followed_out_of_the_repository() { + let listing = r#"{"siblings":[ + {"rfilename":"../m-Q4_K_M.gguf","size":1048576}, + {"rfilename":"a/../m-Q4_K_M.gguf","size":1048576}, + {"rfilename":"m-Q4_K_M\u001b[2J.gguf","size":1048576}, + {"rfilename":"ok-Q4_K_M.gguf","size":1048576}]}"#; + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Honour, + listing.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + assert_eq!(opened.description, "o/r: ok-Q4_K_M.gguf"); + parse_gguf_header(&mut opened.reader).unwrap(); + let requests = hub.requests.lock().unwrap().clone(); + assert!( + !requests.is_empty() + && requests + .iter() + .all(|request| request.ends_with("/o/r/resolve/main/ok-Q4_K_M.gguf")), + "{requests:?}" + ); + } + + /// A projector is never picked as the model, but a name that matches only + /// projectors says what it named, rather than that the repository has no + /// such file (which is what filtering projectors out before the match used to + /// say, with the file missing from the list too). + #[test] + fn a_name_that_matches_only_projectors_is_refused_as_naming_a_projector() { + let files = [ + file("m-Q4_K_M.gguf", 5), + file("mmproj-m-f16.gguf", 1), + file("vision/MMPROJ-Q8_0.gguf", 1), + file("x/mmproj.gguf", 1), + file("y/mmproj.gguf", 1), + ]; + // What was typed, and the real file (or files) it points at, which the + // refusal names. + for (named, real) in [ + ( + "mmproj-m-f16.gguf", + "`mmproj-m-f16.gguf` is a multimodal projector", + ), + ( + "vision/MMPROJ-Q8_0.gguf", + "`vision/MMPROJ-Q8_0.gguf` is a multimodal projector", + ), + ( + "MMPROJ-Q8_0.gguf", + "`vision/MMPROJ-Q8_0.gguf` is a multimodal projector", + ), + ( + "mmproj.gguf", + "`x/mmproj.gguf`, `y/mmproj.gguf` are multimodal projectors", + ), + ] { + let error = select_gguf_file(&files, named).unwrap_err().to_string(); + assert!(error.starts_with(real), "{named}: {error}"); + assert!( + error.contains(&format!("so `{named}` cannot be estimated")), + "{named}: {error}" + ); + assert!(!error.contains("no GGUF file matching"), "{named}: {error}"); + assert!(!error.contains("matches several files"), "{named}: {error}"); + // The model is listed as the alternative, and no projector is. + let models = error.split("the repository's models:").nth(1).unwrap_or(""); + assert!(models.contains("m-Q4_K_M.gguf"), "{named}: {error}"); + assert!( + !models.to_ascii_lowercase().contains("mmproj"), + "{named}: {error}" + ); + } + // The model itself is still picked by name and by label. + assert_eq!( + select_gguf_file(&files, "m-Q4_K_M.gguf").unwrap().name, + "m-Q4_K_M.gguf" + ); + assert_eq!( + select_gguf_file(&files, "Q4_K_M").unwrap().name, + "m-Q4_K_M.gguf" + ); + // A name that matches nothing at all is still "no such file", and the + // listing shows every real file, the projectors marked as such. + for absent in ["other.gguf", "F16"] { + let missing = select_gguf_file(&files, absent).unwrap_err().to_string(); + assert!(missing.contains("no GGUF file matching"), "{missing}"); + assert!( + missing.contains("m-Q4_K_M.gguf, mmproj-m-f16.gguf (multimodal projector)"), + "{missing}" + ); + assert!( + missing.contains("vision/MMPROJ-Q8_0.gguf (multimodal projector)"), + "{missing}" + ); + } + } + + /// A bare name shared by a model and a projector picks the model, as it did + /// before projectors were searched by name at all. + #[test] + fn a_bare_name_shared_with_a_projector_picks_the_model() { + let files = [file("a/m.gguf", 5), file("mmproj/m.gguf", 1)]; + assert_eq!(select_gguf_file(&files, "m.gguf").unwrap().name, "a/m.gguf"); + } + + /// A repository with nothing but projectors says so, not that it has no GGUF. + #[test] + fn a_repository_of_only_projectors_says_it_has_no_model() { + let files = [file("mmproj-a.gguf", 1)]; + let error = select_gguf_file(&files, "mmproj-a.gguf") + .unwrap_err() + .to_string(); + assert!( + error.contains("every GGUF file in it is a projector"), + "{error}" + ); + // Asked for something else, it lists the real file rather than claiming + // the repository has no GGUF. + let other = select_gguf_file(&files, "Q4_K_M").unwrap_err().to_string(); + assert!( + other.contains("it has: mmproj-a.gguf (multimodal projector)"), + "{other}" + ); + } + + /// A letter before a label is not a boundary: `F16` does not match `BF16`, + /// in either order of listing. + #[test] + fn a_label_inside_a_longer_label_is_not_matched() { + for files in [ + [file("m-BF16.gguf", 16), file("m-F16.gguf", 16)], + [file("m-F16.gguf", 16), file("m-BF16.gguf", 16)], + ] { + assert_eq!(select_gguf_file(&files, "F16").unwrap().name, "m-F16.gguf"); + assert_eq!( + select_gguf_file(&files, "BF16").unwrap().name, + "m-BF16.gguf" + ); + } + let only_bf16 = [file("m-BF16.gguf", 16)]; + assert!(select_gguf_file(&only_bf16, "F16").is_err()); + } + + /// A shard's numbering is the last `-of-` in its name, so a model whose own + /// name contains `-of-` is still recognised as one split model. + #[test] + fn the_shards_of_a_model_with_of_in_its_name_are_one_model() { + let shards = [ + file("Chain-of-Thought-Q4_K_M-00002-of-00003.gguf", 1), + file("Chain-of-Thought-Q4_K_M-00001-of-00003.gguf", 1), + file("Chain-of-Thought-Q4_K_M-00003-of-00003.gguf", 1), + ]; + assert_eq!( + select_gguf_file(&shards, "Q4_K_M").unwrap().name, + "Chain-of-Thought-Q4_K_M-00001-of-00003.gguf" + ); + } + + /// `_` before a label is a boundary as `-` and `.` are: llama.cpp's own + /// conversions name files `model_Q4_K_M.gguf` as often as `model-Q4_K_M.gguf`. + #[test] + fn a_label_after_an_underscore_is_matched() { + let files = [file("model_Q4_K_M.gguf", 5), file("model_Q8_0.gguf", 8)]; + assert_eq!( + select_gguf_file(&files, "Q4_K_M").unwrap().name, + "model_Q4_K_M.gguf" + ); + } + + /// A listed name is sent percent-encoded, one segment at a time: a `+` sent + /// raw is read as a space by some servers, and a space breaks the request. + #[test] + fn a_listed_file_name_is_requested_percent_encoded() { + let listing = r#"{"siblings":[{"rfilename":"dir/m+x-Q4_K_M.gguf","size":1048576}]}"#; + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Honour, + listing.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + parse_gguf_header(&mut opened.reader).unwrap(); + let requests = hub.requests.lock().unwrap().clone(); + assert!( + !requests.is_empty() + && requests + .iter() + .all(|request| request.ends_with("/o/r/resolve/main/dir/m%2Bx-Q4_K_M.gguf")), + "{requests:?}" + ); + } + + /// A loopback server that accepts every connection, counts it, and answers + /// nothing for `stall`: the shape of a hub that has stopped responding. + fn stalling_server(stall: Duration) -> (String, Arc>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let connections = Arc::new(Mutex::new(0)); + let counted = Arc::clone(&connections); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { break }; + *counted.lock().unwrap() += 1; + thread::spawn(move || { + thread::sleep(stall); + drop(stream); + }); + } + }); + (base, connections) + } + + /// The listing request is bound by the read's deadline, not only by the + /// agent's one-minute timeout: a spent deadline makes no request at all, and + /// a hub that stops answering is cut off when the deadline passes. + #[test] + fn the_listing_request_is_bound_by_the_deadline() { + let (base, connections) = stalling_server(Duration::from_secs(3)); + let source = LiveGgufSource::with_hub(&base); + let agent = LiveGgufSource::agent(); + let error = source + .list_files(&agent, "o", "r", Instant::now()) + .unwrap_err() + .to_string(); + assert!(error.contains("took longer than"), "{error}"); + assert_eq!( + *connections.lock().unwrap(), + 0, + "a request after the deadline" + ); + let began = Instant::now(); + assert!( + source + .list_files( + &agent, + "o", + "r", + Instant::now() + Duration::from_millis(300) + ) + .is_err() + ); + assert!( + began.elapsed() < Duration::from_secs(2), + "waited {:?} for a hub that never answered", + began.elapsed() + ); + } + + /// The same bound on the `HEAD` request for a file the listing did not size. + #[test] + fn the_head_request_is_bound_by_the_deadline() { + let (base, connections) = stalling_server(Duration::from_secs(3)); + let url = format!("{base}/o/r/resolve/main/m.gguf"); + let agent = LiveGgufSource::agent(); + let error = head_size(&agent, &url, Instant::now()) + .unwrap_err() + .to_string(); + assert!(error.contains("took longer than"), "{error}"); + assert_eq!( + *connections.lock().unwrap(), + 0, + "a request after the deadline" + ); + let began = Instant::now(); + assert!(head_size(&agent, &url, Instant::now() + Duration::from_millis(300)).is_err()); + assert!( + began.elapsed() < Duration::from_secs(2), + "waited {:?} for a hub that never answered", + began.elapsed() + ); + } + + /// Counts the seeks that reach the file under the buffer. + struct CountingSeeks { + inner: R, + seeks: Arc>, + } + + impl Read for CountingSeeks { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } + } + + impl Seek for CountingSeeks { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + *self.seeks.lock().unwrap() += 1; + self.inner.seek(pos) + } + } + + /// Skipping a large vocabulary stays inside the read buffer: the file under + /// it is sought a handful of times, not once per token, which is what made a + /// real header take seconds to read from disk. + #[test] + fn a_local_header_is_skipped_through_without_seeking_the_file_per_token() { + let tokens = 50_000; + let file = llama_like(tokens).header_bytes(); + let seeks = Arc::new(Mutex::new(0)); + let mut reader = RelativeSeekBuf::new(BufReader::new(CountingSeeks { + inner: Cursor::new(file), + seeks: Arc::clone(&seeks), + })); + let header = parse_gguf_header(&mut reader).unwrap(); + assert_eq!(header.tensor_count, 3); + // The same header read straight from memory: the wrapper changes how + // the bytes are reached, never what is read. + let direct = + parse_gguf_header(&mut Cursor::new(llama_like(tokens).header_bytes())).unwrap(); + assert_eq!(header.data_offset, direct.data_offset); + assert_eq!(header.token_embd, direct.token_embd); + let seeks = *seeks.lock().unwrap(); + assert!( + seeks < tokens / 100, + "{seeks} seeks reached the file for {tokens} tokens" + ); + } + + /// The position reported is where the reads and skips have reached, and + /// asking for it never reaches the file under the buffer. + #[test] + fn the_buffered_reader_tracks_its_position_without_asking_the_file() { + let seeks = Arc::new(Mutex::new(0)); + let mut reader = RelativeSeekBuf::new(BufReader::new(CountingSeeks { + inner: Cursor::new(vec![7_u8; 100_000]), + seeks: Arc::clone(&seeks), + })); + let mut byte = [0_u8; 10]; + reader.read_exact(&mut byte).unwrap(); + assert_eq!(reader.seek(SeekFrom::Current(90)).unwrap(), 100); + reader.read_exact(&mut byte).unwrap(); + assert_eq!(reader.seek(SeekFrom::Current(-5)).unwrap(), 105); + let before = *seeks.lock().unwrap(); + for _ in 0..1000 { + assert_eq!(reader.stream_position().unwrap(), 105); + } + assert_eq!( + *seeks.lock().unwrap(), + before, + "asking the position sought the file" + ); + // A skip past the buffer still lands where it was asked to. + assert_eq!(reader.seek(SeekFrom::Current(50_000)).unwrap(), 50_105); + reader.read_exact(&mut byte).unwrap(); + assert_eq!(reader.stream_position().unwrap(), 50_115); + } + + /// A `+` is allowed in an exact file name, which is how a hub file such as + /// `m+x-Q4_K_M.gguf` is named; it is sent percent-encoded. + #[test] + fn an_exact_file_name_with_a_plus_can_be_named() { + assert!(matches!( + parse_gguf_ref_with("o/r:dir/m+x-Q4_K_M.gguf", &|_| false), + Some(GgufRef::Hub { variant: Some(ref v), .. }) if v == "dir/m+x-Q4_K_M.gguf" + )); + // Not in a bare quantization label, which `rocm serve` also reads. + assert!(parse_gguf_ref_with("o/r:Q4+K", &|_| false).is_none()); + } + + #[test] + fn a_quantization_label_picks_one_file_and_never_a_lookalike() { + let files = [ + file("m-Q4_0.gguf", 4), + file("m-Q4_0_4_4.gguf", 4), + file("m-Q4_K_M.gguf", 5), + file("m-Q8_0.gguf", 8), + file("README.md", 1), + file("mmproj-Q8_0.gguf", 1), + ]; + assert_eq!( + select_gguf_file(&files, "Q4_K_M").unwrap().name, + "m-Q4_K_M.gguf" + ); + assert_eq!( + select_gguf_file(&files, "q4_k_m").unwrap().name, + "m-Q4_K_M.gguf" + ); + assert_eq!( + select_gguf_file(&files, "Q4_0").unwrap().name, + "m-Q4_0.gguf" + ); + // The vision projector shares the label and is not the model. + assert_eq!( + select_gguf_file(&files, "Q8_0").unwrap().name, + "m-Q8_0.gguf" + ); + assert_eq!( + select_gguf_file(&files, "m-Q8_0.gguf").unwrap().name, + "m-Q8_0.gguf" + ); + let missing = select_gguf_file(&files, "IQ2_XS").unwrap_err().to_string(); + assert!( + missing.contains("no GGUF file matching `IQ2_XS`") && missing.contains("m-Q4_K_M.gguf"), + "{missing}" + ); + } + + #[test] + fn two_files_for_one_label_are_refused_but_the_shards_of_one_model_are_not() { + let twins = [file("a/m-Q4_K_M.gguf", 1), file("b/m-Q4_K_M.gguf", 1)]; + let error = select_gguf_file(&twins, "Q4_K_M").unwrap_err().to_string(); + assert!(error.contains("matches several files"), "{error}"); + // The exact name still resolves one of them. + assert_eq!( + select_gguf_file(&twins, "a/m-Q4_K_M.gguf").unwrap().name, + "a/m-Q4_K_M.gguf" + ); + let shards = [ + file("Q4_K_M/m-Q4_K_M-00002-of-00002.gguf", 1), + file("Q4_K_M/m-Q4_K_M-00001-of-00002.gguf", 1), + ]; + assert_eq!( + select_gguf_file(&shards, "Q4_K_M").unwrap().name, + "Q4_K_M/m-Q4_K_M-00001-of-00002.gguf" + ); + } + + #[test] + fn a_bare_repository_is_refused_without_touching_the_network() { + // No server is listening on this port: a refusal that came from a + // connection error rather than from the missing variant would show it. + let source = LiveGgufSource::with_hub("http://127.0.0.1:1"); + let error = source + .open(&GgufRef::Hub { + owner: "o".into(), + repo: "r".into(), + variant: None, + }) + .err() + .unwrap() + .to_string(); + assert!(error.contains("needs a quantization"), "{error}"); + } + + #[test] + fn a_local_file_is_opened_with_its_size() { + let dir = std::env::temp_dir().join(format!("rocm-gguf-local-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("m.gguf"); + let bytes = llama_like(5).file(100); + std::fs::write(&path, &bytes).unwrap(); + let mut opened = LiveGgufSource::with_hub("http://127.0.0.1:1") + .open(&GgufRef::Local(path)) + .unwrap(); + assert_eq!(opened.size, bytes.len() as u64); + assert_eq!( + parse_gguf_header(&mut opened.reader).unwrap().block_count, + Some(32) + ); + let missing = LiveGgufSource::new() + .open(&GgufRef::Local(dir.join("absent.gguf"))) + .err() + .unwrap() + .to_string(); + assert!(missing.contains("could not open"), "{missing}"); + // An ordinary missing file is not told it might be a hub reference... + assert!(!missing.contains("Hugging Face"), "{missing}"); + // ...but one shaped like a file in a repository is shown its hub form. + let hubby = LiveGgufSource::new() + .open(&GgufRef::Local(PathBuf::from("owner/repo/dir/file.gguf"))) + .err() + .unwrap() + .to_string(); + assert!(hubby.contains("`owner/repo:dir/file.gguf`"), "{hubby}"); + let two = LiveGgufSource::new() + .open(&GgufRef::Local(PathBuf::from("owner/file.gguf"))) + .err() + .unwrap() + .to_string(); + assert!(!two.contains("Hugging Face"), "{two}"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// How the fake hub answers a request for the model file. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Serve { + /// A correct `206` for the range asked. + Honour, + /// A `200` with the whole (here tiny) body, ignoring `Range`. + IgnoreRange, + /// A `206` whose `Content-Range` starts one byte after the asked offset. + WrongRange, + /// A `206` that promises the whole range and drops the connection partway. + ShortBody, + /// A `206` whose `Content-Range` names the whole range but whose body, with + /// a `Content-Length` that matches it, is only half of it: nothing breaks + /// at the HTTP level, the body is simply short. + ShortConsistent, + /// A `401`, as for a gated repository. + Unauthorized, + /// A `403`, as for a repository that refuses the request. + Forbidden, + /// A `206` that arrives after a pause. + Slow, + /// A `206` with no `Content-Range`, so which bytes it carries is unstated. + NoContentRange, + /// A `HEAD` answered with no `Content-Length`; ranges are honoured. + HeadWithoutLength, + /// A `206` whose `Content-Range` is right but whose body, with no + /// `Content-Length`, goes on past the range: the bytes asked for, then + /// 8 MiB of `0xFF`, then the connection closes. + Overlong, + } + + /// A hub on loopback. The "model" is a real header followed by zeros out to + /// `virtual_size`, served only as far as a range asks for it, so a client that + /// tried to read the weights would show in `ranges`. + struct FakeHub { + base: String, + ranges: Arc>>, + /// Every request line that reached the model file, with its method. + requests: Arc>>, + } + + fn fake_hub(header: Vec, virtual_size: u64, honour_range: bool) -> FakeHub { + let serve = if honour_range { + Serve::Honour + } else { + Serve::IgnoreRange + }; + let listing = format!( + r#"{{"siblings":[{{"rfilename":"m-Q4_K_M.gguf","size":{virtual_size}}},{{"rfilename":"README.md","size":10}}]}}"# + ); + fake_hub_with(header, virtual_size, serve, listing) + } + + fn fake_hub_with(header: Vec, virtual_size: u64, serve: Serve, listing: String) -> FakeHub { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let ranges = Arc::new(Mutex::new(Vec::new())); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = Arc::clone(&ranges); + let logged = Arc::clone(&requests); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).unwrap_or(0) == 0 { + break; + } + request.push(byte[0]); + } + let text = String::from_utf8_lossy(&request).into_owned(); + let method = text.split_whitespace().next().unwrap_or("").to_owned(); + let path = text.split_whitespace().nth(1).unwrap_or("").to_owned(); + let range = text + .lines() + .find_map(|line| { + line.strip_prefix("Range: bytes=") + .or_else(|| line.strip_prefix("range: bytes=")) + }) + .map(str::to_owned); + if path.starts_with("/api/models/o/r") { + let body = listing.clone(); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } else if path.starts_with("/o/r/resolve/main/") { + logged.lock().unwrap().push(format!("{method} {path}")); + if method == "HEAD" && serve == Serve::HeadWithoutLength { + let _ = write!(stream, "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"); + continue; + } + if method == "HEAD" { + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {virtual_size}\r\nConnection: close\r\n\r\n" + ); + continue; + } + seen.lock() + .unwrap() + .push(range.clone().unwrap_or_else(|| "none".into())); + match (range, serve) { + (_, Serve::Unauthorized) => { + let _ = write!( + stream, + "HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + } + (_, Serve::Forbidden) => { + let _ = write!( + stream, + "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + } + ( + Some(range), + Serve::Honour + | Serve::WrongRange + | Serve::ShortBody + | Serve::ShortConsistent + | Serve::Slow + | Serve::NoContentRange + | Serve::HeadWithoutLength + | Serve::Overlong, + ) => { + let (start, end) = range.split_once('-').unwrap(); + let (start, end): (u64, u64) = + (start.parse().unwrap(), end.parse().unwrap()); + let mut body: Vec = (start..=end) + .map(|at| header.get(at as usize).copied().unwrap_or(0)) + .collect(); + if serve == Serve::ShortConsistent { + body.truncate(body.len() / 2); + } + let claimed_start = if serve == Serve::WrongRange { + start + 1 + } else { + start + }; + if serve == Serve::Slow { + thread::sleep(Duration::from_millis(1500)); + } + let content_range = if serve == Serve::NoContentRange { + String::new() + } else { + format!( + "Content-Range: bytes {claimed_start}-{end}/{virtual_size}\r\n" + ) + }; + if serve == Serve::Overlong { + let _ = write!( + stream, + "HTTP/1.1 206 Partial Content\r\n{content_range}Connection: close\r\n\r\n" + ); + let _ = stream.write_all(&body); + let _ = stream.write_all(&vec![0xFF_u8; 8 << 20]); + continue; + } + let _ = write!( + stream, + "HTTP/1.1 206 Partial Content\r\n{content_range}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + if serve == Serve::ShortBody { + let _ = stream.write_all(&body[..body.len() / 2]); + } else { + let _ = stream.write_all(&body); + } + } + _ => { + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nGGUF" + ); + } + } + } else { + let _ = write!( + stream, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + } + } + }); + FakeHub { + base, + ranges, + requests, + } + } + + /// A loopback server that answers every request with a `302` to the same + /// path on `target`, as the hub's `resolve/main` answers with a CDN host. + /// Counts the requests it redirected. + fn redirecting_server(target: String) -> (String, Arc>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let redirected = Arc::new(Mutex::new(0)); + let counted = Arc::clone(&redirected); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).unwrap_or(0) == 0 { + break; + } + request.push(byte[0]); + } + let text = String::from_utf8_lossy(&request).into_owned(); + let path = text.split_whitespace().nth(1).unwrap_or("/").to_owned(); + *counted.lock().unwrap() += 1; + let _ = write!( + stream, + "HTTP/1.1 302 Found\r\nLocation: {target}{path}\r\nContent-Length: \ + 0\r\nConnection: close\r\n\r\n" + ); + } + }); + (base, redirected) + } + + /// The hub answers `resolve/main` with a redirect to its CDN. The listing, + /// the `HEAD` and every range request follow it to the other host, and the + /// range requests still carry their `Range` there. + #[test] + fn a_redirect_to_another_host_is_followed_with_the_range_kept() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Honour, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf"}]}"#.to_owned(), + ); + let (front, redirected) = redirecting_server(hub.base.clone()); + let mut opened = LiveGgufSource::with_hub(&front) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + assert_eq!( + opened.size, + 1 << 20, + "the HEAD's size, read after the redirect" + ); + let header = parse_gguf_header(&mut opened.reader).unwrap(); + assert_eq!(header.block_count, Some(32)); + let ranges = hub.ranges.lock().unwrap().clone(); + assert!( + !ranges.is_empty() && ranges.iter().all(|range| range != "none"), + "{ranges:?}" + ); + // The listing, the HEAD and each range request went through the front. + assert!(*redirected.lock().unwrap() >= 2 + ranges.len()); + } + + #[test] + fn only_the_header_is_requested_from_a_file_far_larger_than_it() { + // 40 000 tokens makes the header a few hundred KB: bigger than nothing, + // far smaller than the 8 GiB the server claims the file to be. + let header = llama_like(40_000).header_bytes(); + let virtual_size: u64 = 8 * 1024 * 1024 * 1024; + let hub = fake_hub(header.clone(), virtual_size, true); + let source = LiveGgufSource::with_hub(&hub.base); + let mut opened = source + .open(&GgufRef::Hub { + owner: "o".into(), + repo: "r".into(), + variant: Some("Q4_K_M".into()), + }) + .unwrap(); + assert_eq!(opened.size, virtual_size); + assert_eq!(opened.description, "o/r: m-Q4_K_M.gguf"); + let parsed = parse_gguf_header(&mut opened.reader).unwrap(); + assert_eq!(parsed.block_count, Some(32)); + let ranges = hub.ranges.lock().unwrap().clone(); + assert!( + !ranges.is_empty() && ranges.iter().all(|r| r != "none"), + "{ranges:?}" + ); + let fetched: u64 = ranges + .iter() + .map(|r| { + let (start, end) = r.split_once('-').unwrap(); + end.parse::().unwrap() - start.parse::().unwrap() + 1 + }) + .sum(); + assert!( + fetched <= 2 * WINDOW_BYTES && fetched < virtual_size / 1000, + "{fetched} bytes were requested for a header of {} bytes: {ranges:?}", + header.len() + ); + } + + #[test] + fn a_server_that_ignores_range_is_refused_rather_than_streamed() { + let hub = fake_hub(llama_like(10).header_bytes(), 1 << 30, false); + let source = LiveGgufSource::with_hub(&hub.base); + let mut opened = source + .open(&GgufRef::Hub { + owner: "o".into(), + repo: "r".into(), + variant: Some("Q4_K_M".into()), + }) + .unwrap(); + let error = parse_gguf_header(&mut opened.reader) + .unwrap_err() + .to_string(); + assert!( + error.contains("does not support reading part of a file"), + "{error}" + ); + } + + #[test] + fn a_missing_repository_is_reported_as_not_found() { + let hub = fake_hub(Vec::new(), 1, true); + let source = LiveGgufSource::with_hub(&hub.base); + let error = source + .open(&GgufRef::Hub { + owner: "nobody".into(), + repo: "nothing".into(), + variant: Some("Q4_K_M".into()), + }) + .err() + .unwrap() + .to_string(); + assert!( + error.contains("was not found, or its repository is gated or private"), + "{error}" + ); + } + + #[test] + fn the_reader_never_fetches_more_than_its_budget() { + let mut reader = RangedReader::new( + ureq::AgentBuilder::new().build(), + "http://127.0.0.1:1/unused".into(), + 1 << 40, + ); + reader.fetched = MAX_FETCH_BYTES; + let mut buf = [0_u8; 1]; + let error = reader.read(&mut buf).unwrap_err().to_string(); + assert!(error.contains("not a GGUF header read"), "{error}"); + } + + #[test] + fn seeking_within_the_window_costs_no_request_and_past_the_end_reads_nothing() { + let header = llama_like(10).header_bytes(); + let hub = fake_hub(header, 1 << 20, true); + let mut reader = RangedReader::new( + ureq::AgentBuilder::new().build(), + format!("{}/o/r/resolve/main/m-Q4_K_M.gguf", hub.base), + 1 << 20, + ); + let mut four = [0_u8; 4]; + reader.read_exact(&mut four).unwrap(); + assert_eq!(&four, b"GGUF"); + reader.seek(SeekFrom::Current(100)).unwrap(); + reader.read_exact(&mut four).unwrap(); + reader.seek(SeekFrom::Start(0)).unwrap(); + reader.read_exact(&mut four).unwrap(); + assert_eq!(hub.ranges.lock().unwrap().len(), 1); + assert_eq!(reader.fetched(), 1 << 20); + reader.seek(SeekFrom::Start(1 << 21)).unwrap(); + assert_eq!(reader.read(&mut four).unwrap(), 0); + assert!(reader.seek(SeekFrom::Current(-(1 << 30))).is_err()); + } + + #[test] + fn a_file_that_exists_wins_over_the_hub_reading_of_its_name() { + // `m/x:y.gguf` has the shape of `owner/repo:variant`. When such a file is + // there it is the file; when it is not, the same string is the hub's. + let exists = |path: &Path| path == Path::new("m/x:y.gguf"); + assert_eq!( + parse_gguf_ref_with("m/x:y.gguf", &exists), + Some(GgufRef::Local(PathBuf::from("m/x:y.gguf"))) + ); + assert_eq!( + parse_gguf_ref_with("m/x:y.gguf", &|_| false), + Some(GgufRef::Hub { + owner: "m".into(), + repo: "x".into(), + variant: Some("y.gguf".into()) + }) + ); + // A `.gguf` name that does not exist is a path that cannot be opened, not + // a repository to ask the hub about. + assert_eq!( + parse_gguf_ref_with("nosuch.gguf", &|_| false), + Some(GgufRef::Local(PathBuf::from("nosuch.gguf"))) + ); + } + + #[test] + fn a_variant_may_carry_a_directory_only_when_it_names_a_gguf_file() { + assert_eq!( + parse_gguf_ref("o/r:b/m-Q4_K_M.gguf"), + Some(GgufRef::Hub { + owner: "o".into(), + repo: "r".into(), + variant: Some("b/m-Q4_K_M.gguf".into()) + }) + ); + // None of these is a hub name: a directory without a file name, a + // traversal, an empty segment, a backslash, an escape. The ones ending in + // `.gguf` are read as local paths (which then cannot be opened), never + // sent to the hub. + for rejected in [ + "o/r:b/Q4_K_M", + "o/r:../m.gguf", + "o/r:b/../m.gguf", + "o/r:/m.gguf", + "o/r:b//m.gguf", + "o/r:b\\m.gguf", + "o/r:b/m%2F.gguf", + ] { + assert!( + !matches!(parse_gguf_ref(rejected), Some(GgufRef::Hub { .. })), + "{rejected:?}" + ); + } + } + + #[test] + fn shards_of_two_different_models_are_not_one_model() { + let two_sets = [ + file("a/m-Q4_K_M-00001-of-00002.gguf", 1), + file("a/m-Q4_K_M-00002-of-00002.gguf", 1), + file("b/m-Q4_K_M-00001-of-00003.gguf", 1), + file("b/m-Q4_K_M-00002-of-00003.gguf", 1), + file("b/m-Q4_K_M-00003-of-00003.gguf", 1), + ]; + let error = select_gguf_file(&two_sets, "Q4_K_M") + .unwrap_err() + .to_string(); + assert!(error.contains("matches several files"), "{error}"); + // Naming one shard exactly still selects it. + assert_eq!( + select_gguf_file(&two_sets, "b/m-Q4_K_M-00001-of-00003.gguf") + .unwrap() + .name, + "b/m-Q4_K_M-00001-of-00003.gguf" + ); + // The same stem with a different shard count is a different split. + let recut = [ + file("m-Q4_K_M-00001-of-00002.gguf", 1), + file("m-Q4_K_M-00001-of-00003.gguf", 1), + ]; + assert!(select_gguf_file(&recut, "Q4_K_M").is_err()); + // A match that is not shards at all is never "the first shard". + let mixed = [ + file("m-Q4_K_M-00001-of-00002.gguf", 1), + file("other-Q4_K_M.gguf", 1), + ]; + assert!(select_gguf_file(&mixed, "Q4_K_M").is_err()); + } + + fn hub_ref(variant: &str) -> GgufRef { + parse_gguf_ref(&format!("o/r:{variant}")).unwrap() + } + + #[test] + fn an_exact_file_in_a_directory_is_reachable_from_the_reference_string() { + // Two files share a name in different directories. The label is + // ambiguous, and the error says how to be exact; doing what it says, + // as the user would type it, reads that file and no other. + let listing = r#"{"siblings":[{"rfilename":"a/m-Q4_K_M.gguf","size":1048576},{"rfilename":"b/m-Q4_K_M.gguf","size":1048576}]}"#; + let header = llama_like(10).header_bytes(); + let hub = fake_hub_with(header, 1 << 20, Serve::Honour, listing.to_owned()); + let source = LiveGgufSource::with_hub(&hub.base); + let error = source.open(&hub_ref("Q4_K_M")).err().unwrap().to_string(); + assert!( + error.contains("matches several files") && error.contains("owner/repo:dir/file.gguf"), + "{error}" + ); + let mut opened = source.open(&hub_ref("b/m-Q4_K_M.gguf")).unwrap(); + assert_eq!(opened.description, "o/r: b/m-Q4_K_M.gguf"); + parse_gguf_header(&mut opened.reader).unwrap(); + let requests = hub.requests.lock().unwrap().clone(); + assert!( + !requests.is_empty() + && requests + .iter() + .all(|request| request.ends_with("/o/r/resolve/main/b/m-Q4_K_M.gguf")), + "{requests:?}" + ); + } + + #[test] + fn a_range_that_starts_elsewhere_is_refused_rather_than_read_from_the_wrong_offset() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::WrongRange, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let error = parse_gguf_header(&mut opened.reader).unwrap_err(); + assert!( + matches!(&error, GgufError::Io(_)) && error.to_string().contains("starts at"), + "{error}" + ); + } + + /// A `206` that does not say which bytes it carries is refused like one that + /// names the wrong ones: the reader cannot know it starts where it asked. + #[test] + fn a_partial_answer_with_no_content_range_is_refused() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::NoContentRange, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let error = parse_gguf_header(&mut opened.reader).unwrap_err(); + assert!( + matches!(&error, GgufError::Io(_)) && error.to_string().contains("an unstated offset"), + "{error}" + ); + } + + /// A file the listing gave no size for, whose `HEAD` gives none either, is + /// refused with a reason rather than read as zero bytes long. + #[test] + fn a_head_with_no_content_length_is_refused() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::HeadWithoutLength, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf"}]}"#.to_owned(), + ); + let error = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .err() + .expect("a file of unknown size is not opened") + .to_string(); + assert!(error.contains("did not report how large"), "{error}"); + } + + #[test] + fn a_connection_that_drops_is_not_reported_as_a_header_that_ends() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::ShortBody, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let error = parse_gguf_header(&mut opened.reader).unwrap_err(); + // `Truncated` is "the GGUF header ends before it is complete": a claim + // about the file that a dropped connection does not support. + assert!( + matches!(&error, GgufError::Io(_)) && error.to_string().contains("connection ended"), + "{error}" + ); + } + + #[test] + fn a_short_but_well_formed_body_is_reported_as_the_connection_ending() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::ShortConsistent, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let error = parse_gguf_header(&mut opened.reader).unwrap_err(); + // The body ended cleanly, so the HTTP layer reported nothing: it is the + // length check that says fewer bytes came than were asked for, and it must + // not say the header ends (`Truncated`), which blames the file. + assert!( + matches!(&error, GgufError::Io(_)) + && error.to_string().contains("connection ended after") + && !error.to_string().contains("bytes asked for:"), + "{error}" + ); + } + + #[test] + fn a_gated_file_is_said_to_be_unreadable_without_offering_a_token() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Unauthorized, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let error = parse_gguf_header(&mut opened.reader) + .unwrap_err() + .to_string(); + assert!( + error.contains("gated or private") && error.contains("no token is sent"), + "{error}" + ); + assert!( + !error.contains("authentication"), + "no token is sent, so authentication is not on offer: {error}" + ); + } + + /// The hub answers 403 as well as 401 for a repository it will not serve to + /// an anonymous client; both read as "gated or private", and neither as a + /// generic status. + #[test] + fn a_forbidden_file_is_said_to_be_gated_or_private_like_an_unauthorized_one() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Forbidden, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let error = parse_gguf_header(&mut opened.reader) + .unwrap_err() + .to_string(); + assert!( + error.contains("gated or private") && !error.contains("answered 403"), + "{error}" + ); + } + + /// `open` sets one deadline for the listing and the reads that follow; the + /// reader it returns is bound by it, not by a fresh budget of its own. + #[test] + fn the_reader_open_returns_is_bound_by_the_budget_open_was_given() { + let slow = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Slow, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&slow.base) + .with_budget(Duration::from_millis(500)) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let began = Instant::now(); + assert!(parse_gguf_header(&mut opened.reader).is_err()); + assert!( + began.elapsed() < Duration::from_millis(1400), + "waited for the slow reply" + ); + } + + #[test] + fn a_variant_directory_may_be_at_most_eight_segments_deep() { + let deep = |segments: usize| { + let dirs = vec!["d"; segments - 1].join("/"); + format!("o/r:{dirs}/m-Q4_K_M.gguf") + }; + assert!(matches!( + parse_gguf_ref(&deep(8)), + Some(GgufRef::Hub { .. }) + )); + assert!(!matches!( + parse_gguf_ref(&deep(9)), + Some(GgufRef::Hub { .. }) + )); + } + + /// The hub's redirect to its CDN may be relative, absolute or + /// scheme-relative; none may carry the request from https to plain http. + #[test] + fn a_redirect_may_not_leave_https_for_plain_http() { + let from = "https://hub.example/o/r/resolve/main/m.gguf?download=1"; + let refused = |location: &str| { + redirect_target(from, location, "the model file") + .unwrap_err() + .to_string() + }; + assert!( + refused("http://cdn.example/x").contains("from https to plain http"), + "an absolute downgrade" + ); + assert!( + refused("HTTP://cdn.example/x").contains("from https to plain http"), + "the scheme is not case-sensitive" + ); + assert!(refused("ftp://cdn.example/x").contains("not http(s)")); + let allowed = |current: &str, location: &str| { + redirect_target(current, location, "the model file").unwrap() + }; + assert_eq!( + allowed(from, "https://cdn.example/x"), + "https://cdn.example/x" + ); + assert_eq!(allowed(from, "//cdn.example/x"), "https://cdn.example/x"); + assert_eq!( + allowed(from, "/api/x?y=http://z"), + "https://hub.example/api/x?y=http://z" + ); + assert_eq!( + allowed(from, "other.gguf"), + "https://hub.example/o/r/resolve/main/other.gguf" + ); + // A plain-http mirror may stay on http, and may move up to https. + assert_eq!( + allowed("http://127.0.0.1:9/a/b", "http://127.0.0.1:8/c"), + "http://127.0.0.1:8/c" + ); + assert_eq!( + allowed("http://127.0.0.1:9/a/b", "https://cdn.example/c"), + "https://cdn.example/c" + ); + } + + /// A server that redirects to itself is followed `MAX_REDIRECTS` times and + /// then refused, relative `Location` and all. + #[test] + fn a_redirect_loop_is_stopped_after_the_limit() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let hits = Arc::new(Mutex::new(0_usize)); + let counted = Arc::clone(&hits); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).unwrap_or(0) == 0 { + break; + } + request.push(byte[0]); + } + *counted.lock().unwrap() += 1; + let _ = write!( + stream, + "HTTP/1.1 302 Found\r\nLocation: /again\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + } + }); + let error = send( + &LiveGgufSource::agent(), + "GET", + &format!("{base}/start"), + Instant::now() + Duration::from_secs(20), + None, + "the model file", + ) + .err() + .unwrap() + .to_string(); + assert!(error.contains("more than 5 times"), "{error}"); + assert_eq!(*hits.lock().unwrap(), MAX_REDIRECTS + 1); + } + + #[test] + fn a_listing_without_a_size_falls_back_to_the_files_own_length() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Honour, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf"}]}"#.to_owned(), + ); + let opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + assert_eq!(opened.size, 1 << 20); + let requests = hub.requests.lock().unwrap().clone(); + assert!( + requests.iter().any(|request| request.starts_with("HEAD ")), + "{requests:?}" + ); + } + + #[test] + fn a_file_the_listing_gave_no_size_for_is_still_named_in_a_refusal() { + // A sibling the listing gave no size for is still a file the repository + // has, so the "it has: ..." list names it. + let files = [ + HubFile { + name: "sized-Q8_0.gguf".into(), + size: Some(1), + }, + HubFile { + name: "unsized-Q6_K.gguf".into(), + size: None, + }, + ]; + let error = select_gguf_file(&files, "Q2_K").unwrap_err().to_string(); + assert!( + error.contains("sized-Q8_0.gguf") && error.contains("unsized-Q6_K.gguf"), + "{error}" + ); + } + + #[test] + fn a_listing_beyond_the_cap_is_refused_not_cut_short() { + let cap = usize::try_from(MAX_LISTING_BYTES).unwrap(); + // ASCII padding, and two-byte characters placed so that the cut falls + // inside one: the size is what is reported either way, not bad text. + for padding in ["x".repeat(cap), "é".repeat(cap / 2 + 1)] { + let listing = format!(r#"{{"siblings":[],"pad":"{padding}"}}"#); + let hub = fake_hub_with(Vec::new(), 1, Serve::Honour, listing); + let error = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .err() + .unwrap() + .to_string(); + assert!(error.contains("is larger than 16 MiB"), "{error}"); + } + } + + /// A range answer that runs on past the bytes asked for is read only as far + /// as asked. The header spans two windows, so a window that kept the extra + /// bytes would hand the parser `0xFF` where the second window should begin. + #[test] + fn a_range_answer_longer_than_asked_is_read_only_as_far_as_asked() { + let header = llama_like(400_000).header_bytes(); + assert!( + header.len() as u64 > WINDOW_BYTES, + "the header must span two windows" + ); + let hub = fake_hub_with( + header.clone(), + 1 << 30, + Serve::Overlong, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1073741824}]}"#.to_owned(), + ); + let mut opened = LiveGgufSource::with_hub(&hub.base) + .open(&hub_ref("Q4_K_M")) + .unwrap(); + let parsed = parse_gguf_header(&mut opened.reader).unwrap(); + let direct = parse_gguf_header(&mut Cursor::new(header)).unwrap(); + assert_eq!(parsed.data_offset, direct.data_offset); + assert!(hub.ranges.lock().unwrap().len() >= 2); + } + + /// A listing that never ends is cut at the cap and refused as too large, + /// promptly: the read stops one byte past the cap rather than waiting for a + /// body that does not end. + #[test] + fn an_endless_listing_is_cut_at_the_cap() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + thread::spawn(move || { + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).unwrap_or(0) == 0 { + return; + } + request.push(byte[0]); + } + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: \ + close\r\n\r\n{{\"siblings\":[],\"pad\":\"" + ); + // Twice the cap, then the connection is held open: a reader + // with no cap would wait here until its timeout. + let chunk = vec![b'x'; 1 << 20]; + for _ in 0..((2 * MAX_LISTING_BYTES) >> 20) { + if stream.write_all(&chunk).is_err() { + return; + } + } + thread::sleep(Duration::from_secs(90)); + }); + } + }); + let began = Instant::now(); + let error = LiveGgufSource::with_hub(&base) + .open(&hub_ref("Q4_K_M")) + .err() + .expect("an endless listing is refused") + .to_string(); + assert!(error.contains("is larger than 16 MiB"), "{error}"); + assert!( + began.elapsed() < Duration::from_secs(20), + "took {:?}", + began.elapsed() + ); + } + + #[test] + fn the_whole_read_has_a_deadline_not_only_each_request() { + let hub = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Honour, + r#"{"siblings":[{"rfilename":"m-Q4_K_M.gguf","size":1048576}]}"#.to_owned(), + ); + let url = format!("{}/o/r/resolve/main/m-Q4_K_M.gguf", hub.base); + let mut spent = RangedReader::new(ureq::AgentBuilder::new().build(), url, 1 << 20) + .with_deadline(Instant::now()); + let mut buf = [0_u8; 4]; + let error = spent.read(&mut buf).unwrap_err().to_string(); + assert!(error.contains("took longer than"), "{error}"); + assert!( + hub.requests.lock().unwrap().is_empty(), + "a request was made after the deadline" + ); + // A reply slower than what is left is cut off, not waited for. + let slow = fake_hub_with( + llama_like(10).header_bytes(), + 1 << 20, + Serve::Slow, + String::new(), + ); + let url = format!("{}/o/r/resolve/main/m-Q4_K_M.gguf", slow.base); + let mut reader = RangedReader::new(ureq::AgentBuilder::new().build(), url, 1 << 20) + .with_deadline(Instant::now() + Duration::from_millis(300)); + let began = Instant::now(); + assert!(reader.read(&mut buf).is_err()); + assert!( + began.elapsed() < Duration::from_millis(1400), + "waited for the slow reply" + ); + } + + #[test] + fn a_path_that_is_not_a_regular_file_is_refused_without_being_opened() { + let dir = std::env::temp_dir().join(format!("rocm-gguf-kind-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let source = LiveGgufSource::with_hub("http://127.0.0.1:1"); + let error = source + .open(&GgufRef::Local(dir.clone())) + .err() + .unwrap() + .to_string(); + assert!(error.contains("is not a regular file"), "{error}"); + // Opening a named pipe would block until something wrote to it. + #[cfg(unix)] + { + let fifo = dir.join("pipe.gguf"); + let made = std::process::Command::new("mkfifo").arg(&fifo).status(); + if made.is_ok_and(|status| status.success()) { + let error = source + .open(&GgufRef::Local(fifo)) + .err() + .unwrap() + .to_string(); + assert!(error.contains("is not a regular file"), "{error}"); + } + } + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn path_segments_are_percent_encoded() { + let encode = crate::report_delivery::percent_encode; + assert_eq!(encode("m Q4+K.gguf"), "m%20Q4%2BK.gguf"); + assert_eq!(encode("a-b_c.d~e"), "a-b_c.d~e"); + } +} diff --git a/crates/rocm-core/src/hardware_root.rs b/crates/rocm-core/src/hardware_root.rs index 14b04f19bc..cd74692430 100644 --- a/crates/rocm-core/src/hardware_root.rs +++ b/crates/rocm-core/src/hardware_root.rs @@ -9,8 +9,10 @@ //! directly. That is every one of these reads, and only these: //! //! - device nodes: `/dev/kfd`, the render nodes under `/dev/dri`, `/dev/dxg`; -//! - `sysfs`: the KFD topology under `/sys/class/kfd/kfd/topology/nodes`, the -//! DRM cards under `/sys/class/drm`, `/sys/module/amdgpu` and +//! - `sysfs`: the `/sys/class/kfd/kfd` class device and the KFD topology under +//! `/sys/class/kfd/kfd/topology/nodes`, the +//! DRM cards under `/sys/class/drm`, `/sys/bus/pci/drivers/amdgpu`, +//! `/sys/module/amdgpu` and //! `/sys/module/amdgpu/version`; //! - `procfs`: `/proc/version`, `/proc/cpuinfo`, `/proc/meminfo`, //! `/proc/cmdline`, `/proc/modules` (the fallback when `lsmod` cannot run) @@ -103,20 +105,34 @@ pub fn host_path_for_str(path: &str) -> PathBuf { host_path(path) } -/// [`read_os_release`]'s `resolve` parameter for a caller that must always -/// read the real machine. +/// [`read_os_release`]'s `resolve` parameter for the real machine: the path +/// as written. A bare `PathBuf::from` closure is rejected the same way a bare +/// `host_path` is (see [`host_path_for_str`]), hence a named function. /// -/// Never routed through [`host_path`], because it builds a package-manager -/// plan or install hint that runs against the real host, so a simulated -/// host root must not choose its distro (see each caller's own doc comment -/// for why). A bare `PathBuf::from` is rejected the same way a bare -/// `host_path` is (see [`host_path_for_str`]), so every such caller passes -/// this instead of `|p| PathBuf::from(p)`. -#[must_use] -pub fn identity_path_for_str(path: &str) -> PathBuf { +/// Private: the only reader that must always see the real machine is +/// [`real_os_release`], and having one place that names this resolver is what +/// keeps a second, diverging copy of that reader from appearing elsewhere. +fn identity_path_for_str(path: &str) -> PathBuf { PathBuf::from(path) } +/// The real host's `/etc/os-release`, falling back to `/usr/lib/os-release` +/// when the former is absent (per `os-release(5)`). +/// +/// The one reader for every caller that builds a package-manager plan or an +/// install hint (`apt`, `dnf`, `zypper`, ...): those commands run against the +/// real machine, so a simulated host root (see [`host_path`]) must not choose +/// the distro they are written for. Deliberately not routed through +/// [`host_path`] for that reason. The descriptive distro name and id that +/// `examine` reports come from [`read_os_release`] with [`host_path_for_str`] +/// instead, and do follow a simulated root. +/// +/// Reads the same files on every platform; off Linux they are simply absent. +#[must_use] +pub fn real_os_release() -> OsRelease { + read_os_release(identity_path_for_str) +} + #[cfg(feature = "e2e-test-hooks")] fn hardware_root() -> Option { std::env::var_os(TEST_HOST_ROOT_ENV) diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index fa08dd40c7..c2ea229b61 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -12,13 +12,13 @@ //! [`crate::confirm`] and is re-exported here too, so the //! `crate::host_gpu::interactive_terminal` path keeps resolving. //! -//! The probes live in child modules: `capture` (optional-command capture and the -//! tool-on-`PATH` lookup), `os_facts` (kernel, distro, CPU and RAM), `linux` -//! (sysfs, KFD and DRM), `gpu_visibility` (which GPUs this process may use), -//! `windows_inventory` (the Windows display-adapter inventory), `wsl` (WSL -//! detection and the Windows host driver as seen from the guest), `render` (the -//! `rocm examine` text of an `ExamineSummary`), `diagnostics` (the GPU -//! detection diagnostics text) and `engine_default` (the default serve engine). +//! The probes live in child modules: `capture` (optional-command capture), +//! `os_facts` (kernel, distro, CPU and RAM), `linux` (sysfs, KFD and DRM), +//! `gpu_visibility` (which GPUs this process may use), `windows_inventory` (the +//! Windows display-adapter inventory), `wsl` (WSL detection and the Windows host +//! driver as seen from the guest), `render` (the `rocm examine` text of an +//! `ExamineSummary`), `diagnostics` (the GPU detection diagnostics text) and +//! `engine_default` (the default serve engine). //! //! The `examine` child modules and `fix.rs` still reach several items here //! through `lib.rs`'s flat `crate::` re-export paths rather than @@ -37,12 +37,15 @@ use crate::{ host_path, runtime_is_linux, runtime_is_windows, runtime_os_name, }; use anyhow::Result; -pub(crate) use os_facts::detect_distro_name; pub use os_facts::detect_system_ram_gib; -use os_facts::{detect_cpu_model, detect_kernel_version}; +pub(crate) use os_facts::{ + CpuFacts, detect_cpu_model, detect_distro_name, detect_kernel_version, read_linux_cpu_facts, + windows_cpu_facts_with, +}; use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; +pub(crate) use windows_inventory::run_windows_system_query; mod capture; mod diagnostics; @@ -194,12 +197,45 @@ pub struct HostGpuSummary { pub therock_family: Option, } +/// The kernel the summary reports: the examination's, when the caller has one and +/// this is Linux (the examination reads it there, and an empty one is a failed +/// read); otherwise whatever `read` gives. `read` is not called in the first case. +fn summary_kernel( + examined: Option<&str>, + linux: bool, + read: impl FnOnce() -> Option, +) -> Option { + match examined { + Some(kernel) if linux => Some(kernel) + .filter(|kernel| !kernel.is_empty()) + .map(str::to_owned), + _ => read(), + } +} + impl ExamineSummary { /// The full host summary, including the Windows display driver version and /// date. Everything that prints the summary, or serializes it for a client /// (`rocm examine`, the MCP and bridge snapshots), wants this one. pub fn gather() -> Result { - Self::gather_scoped(true) + Self::gather_scoped(true, None) + } + + /// [`gather`](Self::gather) for the examination of this same host, which + /// [`Examination::probe_with_summary`](crate::Examination::probe_with_summary) + /// has just made (`rocm examine --json` builds both documents in one run). + /// Crate-private so that no caller can pass an examination of another host, + /// such as one deserialised from a report: its kernel release would be + /// reported as this host's. + /// + /// On Linux the kernel release is the examination's `kernel_release`, a failed + /// read included, instead of a second `uname -r`. One inspection then reads the + /// release once, so `summary.kernel` and `kernel_release` cannot disagree on a + /// transient failure, and a hung `uname` costs one timeout rather than two. + /// Everywhere else (the examination does not read the kernel on Windows) the + /// summary reads it itself, as [`gather`](Self::gather) does. + pub(crate) fn gather_after(examination: &crate::Examination) -> Result { + Self::gather_scoped(true, Some(&examination.kernel_release)) } /// The host summary without the Windows driver version and date, for a @@ -207,10 +243,10 @@ impl ExamineSummary { /// PowerShell video-controller query `gather` adds for them; the GPU /// target, CPU, RAM and every other field are the same. pub fn gather_without_driver_facts() -> Result { - Self::gather_scoped(false) + Self::gather_scoped(false, None) } - fn gather_scoped(driver_facts: bool) -> Result { + fn gather_scoped(driver_facts: bool, examined_kernel: Option<&str>) -> Result { let paths = AppPaths::discover()?; let windows_inventory = if driver_facts { detect_windows_examine_inventory_with_driver_facts() @@ -237,7 +273,7 @@ impl ExamineSummary { Ok(Self { os: runtime_os_name().to_owned(), arch: std::env::consts::ARCH.to_owned(), - kernel: detect_kernel_version(), + kernel: summary_kernel(examined_kernel, runtime_is_linux(), detect_kernel_version), distro: detect_distro_name(), cpu: detect_cpu_model_with_windows_inventory(windows_inventory.as_ref()), system_ram_gib: detect_system_ram_gib_with_windows_inventory( @@ -269,13 +305,23 @@ impl ExamineSummary { fn detect_cpu_model_with_windows_inventory( windows_inventory: Option<&WindowsExamineInventory>, ) -> Option { - if runtime_is_windows() - && let Some(inventory) = windows_inventory - { - return inventory.cpu_model.clone(); - } + summary_cpu(runtime_is_windows(), windows_inventory, detect_cpu_model) +} - detect_cpu_model() +/// The summary's CPU. On Windows it is the inventory's CPU row, which is the one +/// query the examination's reader runs too; with no inventory there is no CPU, +/// because the inventory run already asked for it and asking again is the same +/// query. Elsewhere it is `read`, the Linux reader. +fn summary_cpu( + windows: bool, + windows_inventory: Option<&WindowsExamineInventory>, + read: impl FnOnce() -> Option, +) -> Option { + if windows { + os_facts::cpu_facts_from_windows_inventory(windows_inventory).model + } else { + read() + } } fn detect_system_ram_gib_with_windows_inventory( @@ -490,6 +536,28 @@ mod tests { use std::path::PathBuf; use windows_inventory::WindowsInventorySources; + /// The summary's CPU on Windows is the inventory's CPU row and nothing else: + /// with an inventory that has no such row, or with no inventory at all, it + /// is no CPU, and the Linux reader is never asked. Before, an absent + /// inventory sent the summary to a CPU query of its own. Off Windows the + /// Linux reader answers and the inventory is not looked at. + #[test] + fn the_summary_cpu_comes_from_the_windows_inventory_or_not_at_all() { + let never = || -> Option { panic!("the Linux reader runs only off Windows") }; + let with_cpu = parse_windows_examine_inventory("CPU\tAMD EPYC 7763\nRAM\t17179869184\n"); + let without_cpu = parse_windows_examine_inventory("RAM\t17179869184\n"); + assert_eq!( + summary_cpu(true, Some(&with_cpu), never).as_deref(), + Some("AMD EPYC 7763") + ); + assert_eq!(summary_cpu(true, Some(&without_cpu), never), None); + assert_eq!(summary_cpu(true, None, never), None); + assert_eq!( + summary_cpu(false, Some(&with_cpu), || Some("from cpuinfo".to_owned())).as_deref(), + Some("from cpuinfo") + ); + } + #[test] fn windows_examine_gfx_detection_uses_the_display_inventory() { if !cfg!(windows) { @@ -735,4 +803,30 @@ mod tests { assert!(!rendered.contains(crate::windows_driver::DISPLAY_DRIVER_OLD_FIX_ID)); } } + + /// The summary takes the examination's kernel release on Linux without a + /// second read, and an empty one is a failed read, not a release. Off Linux, + /// or with no examination, it reads for itself. The reads are counted, so a + /// second `uname -r` shows up as a count of 1. + #[test] + fn the_summary_reads_the_kernel_only_when_the_examination_did_not() { + for (examined, linux, reads, expected) in [ + (Some("6.8.0-fake"), true, 0, Some("6.8.0-fake")), + (Some(""), true, 0, None), + (Some("6.8.0-fake"), false, 1, Some("read")), + (None, true, 1, Some("read")), + (None, false, 1, Some("read")), + ] { + let count = std::cell::Cell::new(0); + let kernel = summary_kernel(examined, linux, || { + count.set(count.get() + 1); + Some("read".to_owned()) + }); + assert_eq!( + (kernel.as_deref(), count.get()), + (expected, reads), + "{examined:?} on linux={linux}" + ); + } + } } diff --git a/crates/rocm-core/src/host_gpu/capture.rs b/crates/rocm-core/src/host_gpu/capture.rs index fa2c1c68fe..8fe6fbd91a 100644 --- a/crates/rocm-core/src/host_gpu/capture.rs +++ b/crates/rocm-core/src/host_gpu/capture.rs @@ -5,16 +5,17 @@ //! Optional-command capture for the host probes. //! //! Runs a tool that may not be installed under a short timeout and keeps its -//! stdout only on success, resolving the program the way the platform does -//! (`PATHEXT` and the fixed Windows system locations included). A missing, -//! slow or failing tool reads as `None`, never as an error. +//! stdout only on success, trying each spelling of the program that +//! [`crate::probe_exec::tool_path_candidates`] gives (`PATHEXT` and the fixed +//! Windows system locations included). A missing, slow or failing tool reads as +//! `None`, never as an error. -use crate::runtime_is_windows; -use std::io::Read; +use crate::command_capture::{CommandCapture, CommandOutcome}; +use crate::probe_exec::tool_path_candidates; +use crate::process::run_bounded; use std::path::Path; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; +use std::process::Command; +use std::time::Duration; pub(super) const OPTIONAL_COMMAND_TIMEOUT: Duration = Duration::from_millis(1_500); @@ -37,129 +38,557 @@ pub(super) fn capture_optional_command_with_timeout( None } +/// One spelling of the program, run through [`run_bounded`], the executor every +/// host probe shares; [`optional_stdout`] says what the run is worth. fn capture_optional_command_candidate_with_timeout( program: &Path, args: &[&str], timeout: Duration, ) -> Option { - let mut child = match Command::new(program) - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .spawn() - { - Ok(child) => child, - Err(error) => { - debug_command_capture_failure(program, "spawn", &error.to_string()); - return None; - } - }; - let mut stdout_reader = child.stdout.take().map(|mut stdout| { - thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stdout.read_to_end(&mut bytes); - bytes - }) - }); - - let start = Instant::now(); - loop { - match child.try_wait() { - Ok(Some(status)) => { - let bytes = stdout_reader - .take() - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - if status.success() { - return String::from_utf8(bytes).ok(); - } - debug_command_capture_failure(program, "exit", &format!("status {status}")); - return None; - } - Ok(None) if start.elapsed() < timeout => { - thread::sleep(Duration::from_millis(25)); - } - Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - if let Some(reader) = stdout_reader.take() { - let _ = reader.join(); - } - debug_command_capture_failure(program, "timeout", "timed out"); - return None; - } - Err(_) => { - let _ = child.kill(); - let _ = child.wait(); - if let Some(reader) = stdout_reader.take() { - let _ = reader.join(); - } - debug_command_capture_failure(program, "wait", "failed to wait"); - return None; - } + candidate_with_debug(program, args, timeout, &mut debug_command_capture) +} + +/// [`capture_optional_command_candidate_with_timeout`] with the place its debug +/// lines go as an argument, so a test can read them. The sink is handed every +/// line; whether it is printed is its own decision. +fn candidate_with_debug( + program: &Path, + args: &[&str], + timeout: Duration, + debug: &mut impl FnMut(String), +) -> Option { + let mut command = Command::new(program); + command.args(args); + let capture = run_bounded(command, timeout, "process"); + if let Some(note) = incomplete_output_note(&capture) { + debug(format!( + "rocm debug: command capture output from {} may be incomplete: {note}", + program.display() + )); + } + match optional_stdout(capture) { + Ok(stdout) => stdout, + Err((stage, detail)) => { + debug(format!( + "rocm debug: command capture {stage} failed for {}: {detail}", + program.display() + )); + None } } } -fn debug_command_capture_failure(program: &Path, stage: &str, detail: &str) { +/// Why the stdout of a run that exited 0 may be shorter than the program wrote: +/// a descendant still writing when the executor stopped waiting for it, or a +/// pipe that failed to read. [`optional_stdout`] gives the bytes that arrived +/// either way, so this is the only place the shortfall shows. +fn incomplete_output_note(capture: &CommandCapture) -> Option<&str> { + match capture.outcome { + CommandOutcome::Exited(0) => capture.detail.as_deref(), + _ => None, + } +} + +/// What an optional capture makes of a finished run: the stdout of a program +/// that exited 0 and wrote valid UTF-8, and nothing for any other run. +/// +/// `Ok(None)` is a successful run whose output is not UTF-8, which reads as no +/// answer and is not worth a debug line. `Err` is a run that did not succeed, +/// with the stage and wording for that line. A pipe that failed to read part way +/// does not change the answer: the bytes that arrived are used, as they were +/// before the capture moved onto the shared executor. +fn optional_stdout(capture: CommandCapture) -> Result, (&'static str, String)> { + match capture.outcome { + CommandOutcome::Exited(0) => Ok(String::from_utf8(capture.stdout_raw).ok()), + CommandOutcome::Exited(_) | CommandOutcome::Signaled(_) => Err(( + "exit", + capture.status.map_or_else( + || "no status".to_owned(), + |status| format!("status {status}"), + ), + )), + CommandOutcome::TimedOut => Err(("timeout", "timed out".to_owned())), + CommandOutcome::WaitFailed => Err(("wait", "failed to wait".to_owned())), + CommandOutcome::NotFound | CommandOutcome::SpawnFailed(_) => Err(( + "spawn", + capture + .detail + .unwrap_or_else(|| "failed to start".to_owned()), + )), + } +} + +/// Print one debug line, when `ROCM_CLI_DEBUG_COMMAND_CAPTURE` asks for them. +fn debug_command_capture(line: String) { if !crate::runtime::env_flag("ROCM_CLI_DEBUG_COMMAND_CAPTURE") { return; } - eprintln!( - "rocm debug: command capture {stage} failed for {}: {detail}", - program.display() - ); + eprintln!("{line}"); } -pub(super) fn tool_on_path(program: &str) -> bool { - std::env::var_os("PATH").is_some_and(|path| { - std::env::split_paths(&path).any(|dir| { - tool_path_candidates(program) - .into_iter() - .any(|name| dir.join(name).is_file()) - }) - }) -} +/// The subprocess-runners pair of `docs/host-probing.md`: the three runners the +/// host probes use, which are the summary stack's +/// `capture_optional_command_with_timeout` (this file), the examination stack's +/// `run` over `run_raw` ([`crate::probe_exec`]), and `run_with_timeout` +/// ([`crate::process`]), with the first two now thin adapters over the +/// executor `run_with_timeout` shares. +/// +/// Every case runs one stand-in program through all three and pins what each +/// returns. The cases that stayed as the characterisation left them are the +/// adapters keeping their old answers; the ones that moved are the differences +/// the note lists, each named for the answer it now gives. Programs are planted +/// with [`crate::fake_executable::write_fake_executable`] and started by +/// absolute path, so no `PATH` is involved. Each case that leaves a descendant +/// holding a pipe gives it a short, fixed life, so a runner that waits for it is +/// slow and cannot hang the suite. +#[cfg(all(test, unix))] +mod subprocess_runners_pair { + use super::{candidate_with_debug, capture_optional_command_with_timeout}; + use crate::fake_executable::write_fake_executable; + use crate::probe_exec::{run, run_utf16le}; + use crate::process::run_with_timeout; + use std::os::unix::process::ExitStatusExt as _; + use std::path::Path; + use std::process::{Command, Output}; + use std::time::{Duration, Instant}; + + /// How long a descendant that keeps a pipe open lives. An unbounded reader + /// join waits this long; a bounded one does not. + const DESCENDANT_LIFE: &str = "2"; + + /// Under this a call did not wait for a [`DESCENDANT_LIFE`] descendant, and + /// over it a call did, with the margin on both sides for a loaded host. + const WAITED: Duration = Duration::from_millis(1_500); + + struct Fixture { + _dir: tempfile::TempDir, + program: String, + } + + /// Plant `body` as a shell script and return where it is. + fn fixture(body: &str) -> Fixture { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("fake-tool"); + write_fake_executable(&path, format!("#!/bin/sh\n{body}\n")).expect("plant the fake"); + let program = path.to_str().expect("a UTF-8 temp path").to_owned(); + Fixture { _dir: dir, program } + } -pub(super) fn tool_path_candidates(program: &str) -> Vec { - let path = Path::new(program); - if path.extension().is_some() || !runtime_is_windows() { - return vec![program.to_owned()]; - } - let mut names = Vec::new(); - names.push(program.to_owned()); - let pathext = std::env::var("PATHEXT").unwrap_or_else(|_| ".COM;.EXE;.BAT;.CMD".to_owned()); - for ext in pathext - .split(';') - .map(str::trim) - .filter(|ext| !ext.is_empty()) + fn timed(call: impl FnOnce() -> T) -> (T, Duration) { + let started = Instant::now(); + let value = call(); + (value, started.elapsed()) + } + + fn bounded(program: &str, timeout: Duration) -> anyhow::Result { + run_with_timeout(Command::new(program), timeout, "process") + } + + const LONG: Duration = Duration::from_secs(20); + + #[test] + fn a_program_that_succeeds_is_read_alike_by_all_three() { + let tool = fixture("printf 'out\\n'; printf 'err\\n' >&2; exit 0"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "out\n".to_owned(), "err\n".to_owned()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + Some("out\n".to_owned()), + "the optional capture keeps stdout and discards stderr" + ); + let output = bounded(&tool.program, LONG).expect("a success is not an error"); + assert!(output.status.success()); + assert_eq!(output.stdout, b"out\n"); + assert_eq!(output.stderr, b"err\n"); + } + + #[test] + fn a_non_zero_exit_keeps_its_output_for_run_but_reads_as_nothing_to_the_optional_capture() { + let tool = fixture("printf 'partial\\n'; printf 'bad\\n' >&2; exit 3"); + + assert_eq!( + run(&tool.program, &[], LONG), + (3, "partial\n".to_owned(), "bad\n".to_owned()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + None, + "the optional capture keeps stdout only on success" + ); + let output = bounded(&tool.program, LONG).expect("a non-zero exit is not an error"); + assert_eq!(output.status.code(), Some(3)); + assert_eq!(output.stdout, b"partial\n"); + assert_eq!(output.stderr, b"bad\n"); + } + + #[test] + fn a_program_that_cannot_be_started_is_127_empty_none_or_an_error() { + let missing = "/nonexistent-dir-for-the-runner-pair/tool"; + + assert_eq!(run(missing, &[], LONG), (127, String::new(), String::new())); + assert_eq!( + capture_optional_command_with_timeout(missing, &[], LONG), + None + ); + let error = bounded(missing, LONG).expect_err("a spawn failure is an error here"); + assert!( + format!("{error:#}").contains("failed to spawn process"), + "{error:#}" + ); + } + + #[test] + fn a_timeout_keeps_partial_output_for_run_and_quotes_it_for_the_bounded_executor() { + // `exec` so killing the shell at the deadline leaves no descendant on the + // pipe: this case is about what a timeout returns, not about the join. + let tool = fixture("printf 'before\\n'; exec sleep 30"); + // Room for a loaded host to start the shell and run its `printf` before + // the deadline, since the assertion is on what was printed by then. + let timeout = Duration::from_secs(3); + + assert_eq!( + run(&tool.program, &[], timeout), + (124, "before\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], timeout), + None + ); + let error = bounded(&tool.program, timeout).expect_err("a timeout is an error here"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("exceeded 3s timeout"), "{rendered}"); + assert!(rendered.contains("before"), "{rendered}"); + } + + /// The difference the note asks to be pinned first: the child prints `early` + /// and exits 0, and a descendant prints `late` half a second later, past the + /// 200 ms the executor gives the readers once the child has gone. All three + /// now return `early` alone; `run` and the optional capture used to join + /// their readers and return both lines. + #[test] + fn output_written_more_than_the_drain_grace_after_the_child_exits_is_lost_to_all_three() { + let tool = fixture("printf 'early\\n'; (sleep 0.5; printf 'late\\n') &"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "early\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + Some("early\n".to_owned()) + ); + let output = bounded(&tool.program, LONG).expect("the child exited 0"); + assert_eq!(output.stdout, b"early\n"); + } + + /// The other side of the grace: a descendant that writes 0.1 s after the + /// child exits is inside the 200 ms the readers are given, so every runner + /// still returns both lines. This is what keeps a probe whose tool leaves a + /// short-lived helper behind from losing its tail. + #[test] + fn output_written_within_the_drain_grace_after_the_child_exits_is_kept_by_all_three() { + let tool = fixture("printf 'early\\n'; (sleep 0.1; printf 'late\\n') &"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "early\nlate\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + Some("early\nlate\n".to_owned()) + ); + let output = bounded(&tool.program, LONG).expect("the child exited 0"); + assert_eq!(output.stdout, b"early\nlate\n"); + } + + /// A descendant holds both pipes for [`DESCENDANT_LIFE`] seconds after the + /// child exits 0. `run` and the optional capture used to join their readers + /// with no limit, so each call lasted as long as the descendant; now none of + /// the three waits for it, and each still returns what the child wrote. + #[test] + fn a_descendant_holding_the_pipe_no_longer_extends_run_or_the_optional_capture() { + let tool = fixture(&format!("printf 'ready\\n'; sleep {DESCENDANT_LIFE} &")); + + let (answer, elapsed) = timed(|| run(&tool.program, &[], LONG)); + assert_eq!(answer, (0, "ready\n".to_owned(), String::new())); + assert!(elapsed < WAITED, "run returned after {elapsed:?}"); + + let (answer, elapsed) = + timed(|| capture_optional_command_with_timeout(&tool.program, &[], LONG)); + assert_eq!(answer, Some("ready\n".to_owned())); + assert!( + elapsed < WAITED, + "the optional capture returned after {elapsed:?}" + ); + + let (answer, elapsed) = timed(|| bounded(&tool.program, LONG)); + assert_eq!(answer.expect("the child exited 0").stdout, b"ready\n"); + assert!(elapsed < WAITED, "the executor returned after {elapsed:?}"); + } + + /// The same descendant behind a child that is killed at its deadline: the + /// timeout used to bound nothing for the two older runners (issue 623), and + /// now bounds all three. `run` keeps the partial output and its `124`. + #[test] + fn a_descendant_holding_the_pipe_no_longer_outlasts_the_timeout_of_run_or_the_optional_capture() { - names.push(format!("{program}{ext}")); - names.push(format!("{program}{}", ext.to_ascii_lowercase())); + let tool = fixture(&format!( + "printf 'start\\n'; sleep {DESCENDANT_LIFE} & wait" + )); + let timeout = Duration::from_millis(500); + + let (answer, elapsed) = timed(|| run(&tool.program, &[], timeout)); + // The rc and the time are the point here; what was printed before the + // deadline is `a_timeout_keeps_partial_output_for_run_...`'s, with room + // for a loaded host that this short deadline does not leave. + assert_eq!(answer.0, 124); + assert!(elapsed < WAITED, "run returned after {elapsed:?}"); + + let (answer, elapsed) = + timed(|| capture_optional_command_with_timeout(&tool.program, &[], timeout)); + assert_eq!(answer, None); + assert!( + elapsed < WAITED, + "the optional capture returned after {elapsed:?}" + ); + + let (answer, elapsed) = timed(|| bounded(&tool.program, timeout)); + assert!(answer.is_err(), "a timeout is an error here"); + assert!(elapsed < WAITED, "the executor returned after {elapsed:?}"); + } + + /// What the optional capture says on its debug channel for the same two + /// runs: a descendant that writes after the grace gives a short answer and a + /// line saying so, and a run that finished cleanly gives no line. The answer + /// and the line are read from one call, so the line cannot be claiming a + /// shortfall the answer does not have. + #[test] + fn a_short_answer_from_the_optional_capture_comes_with_a_debug_line_and_a_whole_one_does_not() { + let late = fixture("printf 'early\\n'; (sleep 0.5; printf 'late\\n') &"); + let mut lines = Vec::new(); + let answer = candidate_with_debug(Path::new(&late.program), &[], LONG, &mut |line| { + lines.push(line); + }); + assert_eq!(answer, Some("early\n".to_owned())); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!( + lines[0].starts_with("rocm debug: command capture output from ") + && lines[0].contains(&late.program) + && lines[0].contains("may be incomplete: output was still arriving"), + "{}", + lines[0] + ); + + let whole = fixture("printf 'early\\n'; (sleep 0.1; printf 'late\\n') &"); + let mut lines = Vec::new(); + let answer = candidate_with_debug(Path::new(&whole.program), &[], LONG, &mut |line| { + lines.push(line); + }); + assert_eq!(answer, Some("early\nlate\n".to_owned())); + assert!(lines.is_empty(), "{lines:?}"); + } + + /// A run that failed is described by the failure line alone, not also as an + /// incomplete answer: the second line would add nothing to the first. + #[test] + fn a_failed_run_gets_its_failure_line_and_no_incomplete_line() { + let tool = fixture("printf 'partial\\n'; (sleep 0.5; printf 'late\\n') & exit 3"); + let mut lines = Vec::new(); + let answer = candidate_with_debug(Path::new(&tool.program), &[], LONG, &mut |line| { + lines.push(line); + }); + assert_eq!(answer, None); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!( + lines[0].starts_with("rocm debug: command capture exit failed for "), + "{}", + lines[0] + ); + } + + /// A descendant that holds only stderr. The optional capture used to send + /// stderr to the null device, so such a descendant held nothing of its and + /// the call returned at once; it now pipes stderr like the others and gives + /// the descendant the drain grace. The answer is unchanged. The extra cost, + /// up to the 200 ms grace, is too small for a timing assertion to tell from + /// none, so the assertion is only that the call does not wait out the + /// descendant's life. `run` pipes stderr as it always did, and used to wait + /// the descendant out and return its late line. + #[test] + fn a_descendant_holding_only_stderr_costs_the_optional_capture_at_most_the_drain_grace() { + let tool = fixture(&format!( + "printf 'x\\n'; (exec >/dev/null; sleep {DESCENDANT_LIFE}; printf 'y\\n' >&2) &" + )); + + let (answer, elapsed) = + timed(|| capture_optional_command_with_timeout(&tool.program, &[], LONG)); + assert_eq!(answer, Some("x\n".to_owned())); + assert!(elapsed < WAITED, "returned after {elapsed:?}"); + + let (answer, elapsed) = timed(|| run(&tool.program, &[], LONG)); + assert_eq!(answer, (0, "x\n".to_owned(), String::new())); + assert!(elapsed < WAITED, "run returned after {elapsed:?}"); + } + + #[test] + fn output_that_is_not_utf8_is_lossy_for_run_none_for_the_optional_capture_and_raw_for_the_executor() + { + let tool = fixture("printf 'ok\\377done\\n'"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "ok\u{FFFD}done\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + None, + "strict UTF-8: a stray byte reads as no answer" + ); + let output = bounded(&tool.program, LONG).expect("the child exited 0"); + assert_eq!(output.stdout, b"ok\xffdone\n"); + } + + #[test] + fn utf16le_output_is_decoded_by_declaration_from_the_raw_bytes() { + // `h` `i` as UTF-16LE, preceded by the byte order mark. + let tool = fixture("printf '\\377\\376h\\000i\\000'"); + + assert_eq!(run_utf16le(&tool.program, &[], LONG), (0, "hi".to_owned())); + } + + #[test] + fn a_program_killed_by_a_signal_has_no_exit_code() { + let tool = fixture("printf 'partial\\n'; kill -9 $$"); + + assert_eq!( + run(&tool.program, &[], LONG), + (-1, "partial\n".to_owned(), String::new()), + "run reports -1 where the status has no code" + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + None + ); + let output = bounded(&tool.program, LONG).expect("a signal death is not an error"); + assert_eq!(output.status.code(), None); + assert_eq!(output.status.signal(), Some(9)); + assert_eq!(output.stdout, b"partial\n"); } - names.extend(windows_absolute_tool_candidates(program)); - names.sort(); - names.dedup(); - names } -fn windows_absolute_tool_candidates(program: &str) -> Vec { - if !runtime_is_windows() { - return Vec::new(); - } - let program = program.trim().to_ascii_lowercase(); - let system_root = std::env::var("SystemRoot") - .or_else(|_| std::env::var("WINDIR")) - .unwrap_or_else(|_| r"C:\Windows".to_owned()); - match program.as_str() { - "cmd" | "cmd.exe" => vec![format!(r"{system_root}\System32\cmd.exe")], - "pnputil" | "pnputil.exe" => vec![format!(r"{system_root}\System32\pnputil.exe")], - "powershell" | "powershell.exe" => vec![ - format!(r"{system_root}\System32\WindowsPowerShell\v1.0\powershell.exe"), - "powershell.exe".to_owned(), - ], - "pwsh" | "pwsh.exe" => vec!["pwsh.exe".to_owned()], - _ => Vec::new(), +/// What the optional capture makes of each way a run can end. Runs on every +/// host, over captures built by hand, so the Windows job checks the translation +/// the real-process cases above cannot drive there. +#[cfg(test)] +mod optional_stdout_tests { + use super::{CommandCapture, CommandOutcome, incomplete_output_note, optional_stdout}; + + fn capture(outcome: CommandOutcome, stdout: &[u8], detail: Option<&str>) -> CommandCapture { + CommandCapture::from_run( + vec!["tool".to_owned()], + outcome, + None, + stdout.to_vec(), + b"ignored".to_vec(), + detail.map(str::to_owned), + ) + } + + #[test] + fn a_program_that_exited_0_gives_its_stdout_and_nothing_else() { + assert_eq!( + optional_stdout(capture(CommandOutcome::Exited(0), b"out\n", None)), + Ok(Some("out\n".to_owned())) + ); + } + + #[test] + fn a_program_that_exited_0_with_output_that_is_not_utf8_gives_no_answer_and_no_debug_line() { + assert_eq!( + optional_stdout(capture(CommandOutcome::Exited(0), b"ok\xffdone", None)), + Ok(None) + ); + } + + #[test] + fn a_pipe_that_failed_to_read_does_not_change_the_answer() { + // Kept as it was: the bytes that arrived are used, and the error is not + // an answer of its own. + assert_eq!( + optional_stdout(capture( + CommandOutcome::Exited(0), + b"partial", + Some("failed to read the output: broken pipe") + )), + Ok(Some("partial".to_owned())) + ); + } + + #[test] + fn every_other_way_a_run_ends_is_no_answer_with_its_stage_named() { + let cases = [ + (CommandOutcome::Exited(3), "exit"), + (CommandOutcome::Signaled(Some(9)), "exit"), + (CommandOutcome::TimedOut, "timeout"), + (CommandOutcome::WaitFailed, "wait"), + (CommandOutcome::NotFound, "spawn"), + ( + CommandOutcome::SpawnFailed(std::io::ErrorKind::PermissionDenied), + "spawn", + ), + ]; + for (outcome, stage) in cases { + // Stdout the child wrote before ending badly is not an answer. + let result = optional_stdout(capture(outcome, b"partial", Some("the os said no"))); + assert_eq!( + result.map_err(|(stage, _)| stage), + Err(stage), + "{outcome:?}" + ); + } + } + + #[test] + fn only_a_run_that_exited_0_with_something_cutting_its_output_short_is_called_incomplete() { + let note = "output was still arriving when the drain grace ended"; + assert_eq!( + incomplete_output_note(&capture(CommandOutcome::Exited(0), b"x", Some(note))), + Some(note) + ); + assert_eq!( + incomplete_output_note(&capture(CommandOutcome::Exited(0), b"x", None)), + None, + "a whole capture has nothing to report" + ); + // Every other ending already has a failure line of its own. + for outcome in [ + CommandOutcome::Exited(3), + CommandOutcome::Signaled(Some(9)), + CommandOutcome::TimedOut, + CommandOutcome::WaitFailed, + CommandOutcome::NotFound, + ] { + assert_eq!( + incomplete_output_note(&capture(outcome, b"x", Some(note))), + None, + "{outcome:?}" + ); + } + } + + #[test] + fn a_spawn_failure_reports_the_error_the_os_gave() { + let (stage, detail) = optional_stdout(capture( + CommandOutcome::NotFound, + b"", + Some("No such file or directory (os error 2)"), + )) + .expect_err("a program that cannot be started is no answer"); + + assert_eq!(stage, "spawn"); + assert_eq!(detail, "No such file or directory (os error 2)"); } } diff --git a/crates/rocm-core/src/host_gpu/diagnostics.rs b/crates/rocm-core/src/host_gpu/diagnostics.rs index 1a456f361a..739eb34c5c 100644 --- a/crates/rocm-core/src/host_gpu/diagnostics.rs +++ b/crates/rocm-core/src/host_gpu/diagnostics.rs @@ -5,8 +5,6 @@ //! The text `rocm install sdk` prints when it cannot resolve a GPU: what the //! detection saw, and on Windows the raw output of each inventory query. -#[cfg(windows)] -use super::capture::tool_path_candidates; use super::detect_host_gpu_summary; use super::linux::{detect_linux_primary_gpu_name, detect_linux_sysfs_gfx_target}; #[cfg(windows)] @@ -16,22 +14,24 @@ use super::windows_inventory::{ parse_windows_examine_inventory, parse_windows_pnputil_display_inventory, }; use super::wsl::{detect_wsl_windows_display_probe_text, is_wsl_host}; +#[cfg(any(windows, test))] +use crate::command_capture::{CommandCapture, CommandOutcome}; #[cfg(windows)] use crate::gfx_target::{ amd_pci_device_id_from_pnp_id, gfx_target_from_amd_marketing_name, gfx_target_from_amd_pci_device_id, }; -use crate::{runtime_is_linux, runtime_is_windows, runtime_os_name}; #[cfg(windows)] -use std::io::Read; +use crate::probe_exec::tool_path_candidates; #[cfg(windows)] -use std::path::PathBuf; -#[cfg(windows)] -use std::process::{Command, Stdio}; +use crate::process::run_bounded; +use crate::{runtime_is_linux, runtime_is_windows, runtime_os_name}; +#[cfg(any(windows, test))] +use std::path::{Path, PathBuf}; #[cfg(windows)] -use std::thread; +use std::process::Command; #[cfg(windows)] -use std::time::{Duration, Instant}; +use std::time::Duration; pub fn detect_host_gpu_diagnostics() -> String { use std::fmt::Write as _; @@ -186,7 +186,7 @@ fn append_windows_probe_diagnostics( } #[derive(Debug)] -#[cfg(windows)] +#[cfg(any(windows, test))] struct DiagnosticCommandResult { program: Option, status: Option, @@ -196,96 +196,79 @@ struct DiagnosticCommandResult { timed_out: bool, } +/// Run `program` under every spelling [`tool_path_candidates`] gives, through the +/// executor the other host probes share ([`run_bounded`]), and describe the +/// first run that started. #[cfg(windows)] fn capture_diagnostic_command( program: &str, args: &[&str], timeout: Duration, ) -> DiagnosticCommandResult { - let candidates = tool_path_candidates(program); + capture_diagnostic_with(program, &tool_path_candidates(program), |path| { + let mut command = Command::new(path); + command.args(args); + run_bounded(command, timeout, "process") + }) +} + +/// [`capture_diagnostic_command`] with the candidates and the runner passed in, +/// so a test on any host can say how each run ends. +/// +/// The next spelling is tried only when a program could not be started. Any run +/// that started is final: an exit (non-zero included) reports its status and +/// both streams, a timeout reports the output that arrived and says so, and a +/// failure to wait reports that, with no output. +#[cfg(any(windows, test))] +fn capture_diagnostic_with( + program: &str, + candidates: &[String], + mut run: impl FnMut(&Path) -> CommandCapture, +) -> DiagnosticCommandResult { let mut last_error = None; for candidate in candidates { - let path = PathBuf::from(&candidate); - let mut child = match Command::new(&path) - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - { - Ok(child) => child, - Err(error) => { - last_error = Some(format!("failed to launch {}: {error}", path.display())); - continue; + let path = PathBuf::from(candidate); + let capture = run(&path); + match capture.outcome { + CommandOutcome::NotFound | CommandOutcome::SpawnFailed(_) => { + last_error = Some(format!( + "failed to launch {}: {}", + path.display(), + capture.detail.as_deref().unwrap_or("unknown error") + )); } - }; - let stdout_reader = child.stdout.take().map(|mut stdout| { - thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stdout.read_to_end(&mut bytes); - bytes - }) - }); - let stderr_reader = child.stderr.take().map(|mut stderr| { - thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stderr.read_to_end(&mut bytes); - bytes - }) - }); - - let start = Instant::now(); - loop { - match child.try_wait() { - Ok(Some(status)) => { - let stdout = stdout_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - let stderr = stderr_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - return DiagnosticCommandResult { - program: Some(path), - status: Some(status.to_string()), - stdout: String::from_utf8_lossy(&stdout).into_owned(), - stderr: String::from_utf8_lossy(&stderr).into_owned(), - error: None, - timed_out: false, - }; - } - Ok(None) if start.elapsed() < timeout => { - thread::sleep(Duration::from_millis(25)); - } - Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - let stdout = stdout_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - let stderr = stderr_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - return DiagnosticCommandResult { - program: Some(path), - status: None, - stdout: String::from_utf8_lossy(&stdout).into_owned(), - stderr: String::from_utf8_lossy(&stderr).into_owned(), - error: None, - timed_out: true, - }; - } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return DiagnosticCommandResult { - program: Some(path), - status: None, - stdout: String::new(), - stderr: String::new(), - error: Some(format!("failed to wait: {error}")), - timed_out: false, - }; - } + CommandOutcome::Exited(_) | CommandOutcome::Signaled(_) => { + return DiagnosticCommandResult { + program: Some(path), + status: capture.status.map(|status| status.to_string()), + stdout: capture.stdout, + stderr: capture.stderr, + error: None, + timed_out: false, + }; + } + CommandOutcome::TimedOut => { + return DiagnosticCommandResult { + program: Some(path), + status: None, + stdout: capture.stdout, + stderr: capture.stderr, + error: None, + timed_out: true, + }; + } + CommandOutcome::WaitFailed => { + return DiagnosticCommandResult { + program: Some(path), + status: None, + stdout: String::new(), + stderr: String::new(), + error: Some(format!( + "failed to wait: {}", + capture.detail.as_deref().unwrap_or("unknown error") + )), + timed_out: false, + }; } } } @@ -330,3 +313,187 @@ fn truncate_diagnostic_line(line: &str, max_chars: usize) -> String { truncated } } + +/// The Windows-only diagnostic capture's choice of candidate and its report of +/// each way a run can end, driven through an injected runner so it runs on every +/// host, the Windows job included. +#[cfg(test)] +mod capture_diagnostic_tests { + use super::{CommandCapture, CommandOutcome, capture_diagnostic_with}; + use std::path::Path; + use std::process::ExitStatus; + + fn status(code: i32) -> ExitStatus { + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt as _; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt as _; + #[cfg(unix)] + let raw = code << 8; + #[cfg(windows)] + let raw = code.unsigned_abs(); + ExitStatus::from_raw(raw) + } + + fn candidates(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_owned()).collect() + } + + fn finished( + outcome: CommandOutcome, + code: Option, + detail: Option<&str>, + ) -> CommandCapture { + CommandCapture::from_run( + vec!["tool".to_owned()], + outcome, + code.map(status), + b"out\n".to_vec(), + b"err\n".to_vec(), + detail.map(str::to_owned), + ) + } + + /// A runner that gives each candidate the next scripted capture, and records + /// which candidates it was asked to start. + fn scripted( + mut script: Vec, + asked: &mut Vec, + ) -> impl FnMut(&Path) -> CommandCapture + '_ { + script.reverse(); + move |path| { + asked.push(path.display().to_string()); + script + .pop() + .expect("more candidates were started than scripted") + } + } + + #[test] + fn the_next_spelling_is_tried_only_when_a_program_could_not_be_started() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b", "c"]), + scripted( + vec![ + finished(CommandOutcome::NotFound, None, Some("no a")), + finished( + CommandOutcome::SpawnFailed(std::io::ErrorKind::PermissionDenied), + None, + Some("no b"), + ), + finished(CommandOutcome::Exited(0), Some(0), None), + ], + &mut asked, + ), + ); + + assert_eq!(asked, ["a", "b", "c"]); + assert_eq!(result.program.as_deref(), Some(Path::new("c"))); + assert_eq!(result.status, Some(status(0).to_string())); + assert_eq!( + (result.stdout.as_str(), result.stderr.as_str()), + ("out\n", "err\n") + ); + assert!(result.error.is_none() && !result.timed_out); + } + + #[test] + fn a_program_that_never_starts_reports_the_last_launch_error() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![ + finished(CommandOutcome::NotFound, None, Some("no a")), + finished(CommandOutcome::NotFound, None, Some("no b")), + ], + &mut asked, + ), + ); + + assert!(result.program.is_none() && result.status.is_none()); + assert_eq!(result.error.as_deref(), Some("failed to launch b: no b")); + assert!(result.stdout.is_empty() && !result.timed_out); + + let none = capture_diagnostic_with("tool", &[], |_| unreachable!("no candidate")); + assert_eq!(none.error.as_deref(), Some("tool was not found")); + } + + #[test] + fn a_non_zero_exit_is_final_and_reports_its_status_and_both_streams() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![finished(CommandOutcome::Exited(3), Some(3), None)], + &mut asked, + ), + ); + + assert_eq!(asked, ["a"], "a run that started must not be retried"); + assert_eq!(result.program.as_deref(), Some(Path::new("a"))); + assert_eq!(result.status, Some(status(3).to_string())); + #[cfg(windows)] + assert_eq!(result.status.as_deref(), Some("exit code: 3")); + assert_eq!( + (result.stdout.as_str(), result.stderr.as_str()), + ("out\n", "err\n") + ); + assert!(result.error.is_none() && !result.timed_out); + } + + #[test] + fn a_timeout_is_final_keeps_the_output_that_arrived_and_says_so() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![finished(CommandOutcome::TimedOut, Some(1), None)], + &mut asked, + ), + ); + + assert_eq!(asked, ["a"], "a hung program must not be started again"); + assert!(result.timed_out); + assert!( + result.status.is_none(), + "a killed child has no status to report" + ); + assert_eq!( + (result.stdout.as_str(), result.stderr.as_str()), + ("out\n", "err\n") + ); + assert!(result.error.is_none()); + } + + #[test] + fn a_failure_to_wait_is_final_and_reports_no_output() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![finished( + CommandOutcome::WaitFailed, + None, + Some("the os said no"), + )], + &mut asked, + ), + ); + + assert_eq!(asked, ["a"]); + assert_eq!(result.program.as_deref(), Some(Path::new("a"))); + assert_eq!( + result.error.as_deref(), + Some("failed to wait: the os said no") + ); + assert!(result.stdout.is_empty() && result.stderr.is_empty()); + assert!(!result.timed_out && result.status.is_none()); + } +} diff --git a/crates/rocm-core/src/host_gpu/os_facts.rs b/crates/rocm-core/src/host_gpu/os_facts.rs index 0cfab952ff..c5cf1a4fd2 100644 --- a/crates/rocm-core/src/host_gpu/os_facts.rs +++ b/crates/rocm-core/src/host_gpu/os_facts.rs @@ -8,19 +8,52 @@ use super::capture::{ OPTIONAL_COMMAND_TIMEOUT, capture_optional_command, capture_optional_command_with_timeout, }; +use super::windows_inventory::{WindowsExamineInventory, windows_system_inventory_with}; use crate::{hardware_root, host_path, runtime_is_linux, runtime_is_windows}; use std::fs; +use std::time::Duration; -pub(super) fn detect_kernel_version() -> Option { - if runtime_is_windows() { +/// How long `uname -r` may take. +/// +/// 5 s, the longer of the two limits the summary (1.5 s) and the examination +/// (5 s) used to apply separately. The release is a few bytes read from the +/// kernel, so a `uname` that takes more than a moment is a stalled host, where +/// waiting longer costs little and a spurious "unknown" costs more: +/// `kernel_release` feeds the WSL version check and the kernel-dependent +/// diagnoses, which read an empty value as "nothing to report". +/// +/// Written out rather than borrowed from another module's constant, so a change +/// to that constant cannot move this limit unnoticed; +/// `the_kernel_release_timeout_is_five_seconds` pins it. +const KERNEL_RELEASE_TIMEOUT: Duration = Duration::from_secs(5); + +/// The running kernel, as the text of `rocm examine` and `summary.kernel` show it +/// and as `kernel_release` in the examination reports it. +/// +/// On Linux (native and WSL), the trimmed output of `uname -r`. That is the one +/// read of the kernel release for both stacks: `Examination`'s `kernel_release` +/// is this value, with `None` shown as empty, and `Examination::probe_with_summary` +/// hands the examination's value to the summary, so one `rocm examine --json` run +/// reads it once. The read is strict: +/// +/// - a non-zero exit gives `None`, whatever `uname` printed; +/// - output that is not UTF-8 gives `None` rather than a string with replacement +/// characters in it; +/// - an empty answer gives `None`; +/// - a `uname` that has not answered after [`KERNEL_RELEASE_TIMEOUT`] gives +/// `None`. +/// +/// On Windows, the trimmed output of `cmd /C ver`, under the shorter optional +/// command timeout; the examination does not read that into `kernel_release`. +pub(crate) fn detect_kernel_version() -> Option { + let output = if runtime_is_windows() { capture_optional_command("cmd", &["/C", "ver"]) - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) } else { - capture_optional_command("uname", &["-r"]) - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) - } + capture_optional_command_with_timeout("uname", &["-r"], KERNEL_RELEASE_TIMEOUT) + }; + output + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) } pub(crate) fn detect_distro_name() -> Option { @@ -64,40 +97,139 @@ fn distro_name_from_os_release(read: &hardware_root::OsRelease) -> String { } } -pub(super) fn detect_cpu_model() -> Option { - if runtime_is_windows() { - let script = - "Get-CimInstance Win32_Processor | Select-Object -First 1 -ExpandProperty Name"; - return capture_optional_command_with_timeout( - "powershell", - &["-NoProfile", "-Command", script], - OPTIONAL_COMMAND_TIMEOUT, - ) - .map(|value| normalize_cpu_model(&value)) - .filter(|value| !value.is_empty()); +/// The CPU's model and vendor, read together by the one reader both stacks use: +/// the summary's `cpu` through [`detect_cpu_model`] on Linux and the Windows +/// inventory's CPU row on Windows, +/// the examination's `cpu_model` and `cpu_vendor` through `probe_cpu_linux` and +/// `probe_cpu_windows`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CpuFacts { + /// The model with every run of whitespace collapsed to one space, or `None` + /// when no model could be read. `None` covers every way of not knowing (no + /// file, no line, a blank value, a query that failed or timed out), so a + /// caller cannot tell a failed query from an absent line; the Windows + /// examination records a failure for any `None`, and Linux for none. + pub(crate) model: Option, + /// `amd` or `intel`, any other vendor lower-cased as the file spells it, or + /// [`UNKNOWN_CPU_VENDOR`] when none could be read. + pub(crate) vendor: String, +} + +/// The vendor of a CPU no source named. +pub(crate) const UNKNOWN_CPU_VENDOR: &str = "unknown"; + +impl CpuFacts { + fn unknown() -> Self { + Self { + model: None, + vendor: UNKNOWN_CPU_VENDOR.to_owned(), + } } +} - if runtime_is_linux() - && let Some(model) = fs::read_to_string(host_path("/proc/cpuinfo")) - .ok() - .and_then(|text| { - text.lines().find_map(|line| { - let value = line - .strip_prefix("model name") - .and_then(|rest| rest.split_once(':').map(|(_, value)| value)) - .or_else(|| { - line.strip_prefix("Hardware") - .and_then(|rest| rest.split_once(':').map(|(_, value)| value)) - })?; - let value = normalize_cpu_model(value); - (!value.is_empty()).then_some(value) - }) - }) - { - return Some(model); +/// The CPU summary line on Linux: the model of [`read_linux_cpu_facts`]. +/// +/// Linux only, on purpose. On Windows the summary's CPU is the CPU row of the +/// inventory it already holds (`summary_cpu` in `host_gpu.rs`), and the +/// examination's is its own injected read of the same query, so a Windows arm +/// here would be a second PowerShell query for the same fact. It answers `None` +/// off Linux instead, which is what a future caller on Windows gets rather than +/// a duplicate query. +pub(crate) fn detect_cpu_model() -> Option { + linux_cpu_model(runtime_is_linux(), read_linux_cpu_facts) +} + +/// [`detect_cpu_model`] with the host check and the reader as arguments, so a +/// test on any host shows that nothing is read off Linux. +fn linux_cpu_model(linux: bool, read: impl FnOnce() -> CpuFacts) -> Option { + linux.then(read).and_then(|facts| facts.model) +} + +/// The CPU facts from `/proc/cpuinfo`, read through [`host_path`]. +pub(crate) fn read_linux_cpu_facts() -> CpuFacts { + cpu_facts_from_cpuinfo(&fs::read_to_string(host_path("/proc/cpuinfo")).unwrap_or_default()) +} + +/// The CPU facts from `/proc/cpuinfo` text. +/// +/// The model is the first `model name` or `Hardware` line that has a value, +/// whichever comes first: an Arm board that prints no `model name` is still +/// named. The vendor is the first `vendor_id` line that has a value, `amd` or +/// `intel` by substring (`AuthenticAMD`, `GenuineIntel`) and any other +/// lower-cased. A file with no `vendor_id` (Arm kernels print none) has an +/// unknown vendor beside a known model. +pub(super) fn cpu_facts_from_cpuinfo(text: &str) -> CpuFacts { + let mut facts = CpuFacts::unknown(); + for line in text.lines() { + if facts.model.is_none() + && let Some(value) = line + .strip_prefix("model name") + .or_else(|| line.strip_prefix("Hardware")) + .and_then(|rest| rest.split_once(':').map(|(_, value)| value)) + { + let value = normalize_cpu_model(value); + if !value.is_empty() { + facts.model = Some(value); + } + } + if facts.vendor == UNKNOWN_CPU_VENDOR + && let Some(value) = line + .strip_prefix("vendor_id") + .and_then(|rest| rest.split_once(':').map(|(_, value)| value.trim())) + .filter(|value| !value.is_empty()) + { + facts.vendor = if value.contains("AMD") { + "amd".to_owned() + } else if value.contains("Intel") { + "intel".to_owned() + } else { + value.to_lowercase() + }; + } + if facts.model.is_some() && facts.vendor != UNKNOWN_CPU_VENDOR { + break; + } } + facts +} - None +/// The CPU facts from the Windows system inventory's `CPU` row, with the query +/// injected, so a test on any host can drive it. The examination's +/// `probe_cpu_windows` is the one caller that passes the real PowerShell runner. +/// `query` is handed the script and the time the inventory allows it, +/// and answers with what the script printed, or `None` when it could not run, +/// failed or timed out. +pub(crate) fn windows_cpu_facts_with( + query: impl FnOnce(&str, std::time::Duration) -> Option, +) -> CpuFacts { + cpu_facts_from_windows_inventory(windows_system_inventory_with(query).as_ref()) +} + +/// The CPU facts of a Windows inventory: its CPU row, or unknown when there is +/// no inventory or the row is absent or blank. The summary reads the inventory +/// it already holds through this, so it starts no second query for the CPU and +/// reads the same row the examination's reader does. +/// +/// Windows reports no vendor field here, so it is derived from the model string +/// (`amd` or `intel`, else unknown). +pub(super) fn cpu_facts_from_windows_inventory( + inventory: Option<&WindowsExamineInventory>, +) -> CpuFacts { + let Some(model) = inventory.and_then(|inventory| inventory.cpu_model.clone()) else { + return CpuFacts::unknown(); + }; + let lowered = model.to_lowercase(); + let vendor = if lowered.contains("amd") { + "amd" + } else if lowered.contains("intel") { + "intel" + } else { + UNKNOWN_CPU_VENDOR + }; + CpuFacts { + model: Some(model), + vendor: vendor.to_owned(), + } } pub fn detect_system_ram_gib() -> Option { @@ -221,4 +353,358 @@ mod tests { "Linux (/usr/lib/os-release line 2 is not a plain assignment: \"unset ID\")" ); } + + // ── CPU model: the summary's read and the examination's read ────────── + // + // Tests of the CPU-model pair of docs/host-probing.md. `rocm examine --json` + // reads the CPU in two places: the summary (`summary.cpu`, the `cpu:` line of + // the text) and the examination (`cpu_model` and `cpu_vendor`). Both go + // through one reader now, so each case below hands both the same input and + // asserts one answer. The cases that used to differ say so in their + // comment, with what each read answered before. + + use crate::examine::{Examination, probe_cpu_windows_with, record_cpu}; + use crate::host_gpu::windows_inventory::{ + InventoryScope, WindowsInventorySources, assemble_windows_inventory, + }; + + /// One `/proc/cpuinfo` and the CPU both reads make of it. + struct CpuinfoCase { + name: &'static str, + text: &'static str, + /// The model: `summary.cpu`, and `cpu_model` where `Some`, empty where `None`. + model: Option<&'static str>, + /// `cpu_vendor`. + vendor: &'static str, + } + + #[test] + fn linux_cpuinfo_both_reads_report_one_cpu() { + let cases = [ + CpuinfoCase { + name: "Intel, model name and vendor_id", + text: "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel name\t: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\nprocessor\t: 1\nvendor_id\t: GenuineIntel\nmodel name\t: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\n", + model: Some("Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz"), + vendor: "intel", + }, + CpuinfoCase { + name: "AMD, model name and vendor_id", + text: "vendor_id\t: AuthenticAMD\nmodel name\t: AMD Ryzen 9 9950X 16-Core Processor\n", + model: Some("AMD Ryzen 9 9950X 16-Core Processor"), + vendor: "amd", + }, + CpuinfoCase { + // No `vendor_id` (Arm kernels print none): the model is read, the + // vendor stays "unknown". + name: "model name and no vendor_id", + text: "processor\t: 0\nmodel name\t: ARMv8 Processor rev 1 (v8l)\n", + model: Some("ARMv8 Processor rev 1 (v8l)"), + vendor: "unknown", + }, + CpuinfoCase { + name: "a blank model name line, then a real one", + text: "vendor_id\t: GenuineIntel\nmodel name\t: \nmodel name\t: Intel(R) Xeon(R) Gold 6248\n", + model: Some("Intel(R) Xeon(R) Gold 6248"), + vendor: "intel", + }, + CpuinfoCase { + // `Hardware` after the per-core blocks is what an Arm board + // prints, and it follows `model name`, so both read the latter. + name: "model name first, Hardware last", + text: "processor\t: 0\nmodel name\t: ARMv7 Processor rev 4 (v7l)\nHardware\t: BCM2835\n", + model: Some("ARMv7 Processor rev 4 (v7l)"), + vendor: "unknown", + }, + CpuinfoCase { + name: "nothing to read", + text: "processor\t: 0\nCPU implementer\t: 0x41\n", + model: None, + vendor: "unknown", + }, + CpuinfoCase { + name: "an empty file", + text: "", + model: None, + vendor: "unknown", + }, + CpuinfoCase { + // A vendor other than AMD and Intel is kept, lower-cased. + name: "a vendor_id that is neither AMD nor Intel", + text: "vendor_id\t: HygonGenuine\nmodel name\t: Hygon C86 7185 32-core Processor\n", + model: Some("Hygon C86 7185 32-core Processor"), + vendor: "hygongenuine", + }, + ]; + for case in cases { + assert_cpuinfo_case(&case); + } + } + + /// The inputs on which the two reads used to give different answers. Each + /// comment says what the summary and the examination answered before the + /// reads were merged; the answer asserted is the one both give now. + #[test] + fn linux_cpuinfo_the_inputs_the_two_reads_used_to_disagree_on() { + let cases = [ + CpuinfoCase { + // Before: summary collapsed whitespace runs, examination trimmed + // only (`Intel(R) Pentium(R) CPU N3700 @ 1.60GHz`). + name: "repeated spaces inside the model name", + text: "vendor_id\t: GenuineIntel\nmodel name\t: Intel(R) Pentium(R) CPU N3700 @ 1.60GHz \n", + model: Some("Intel(R) Pentium(R) CPU N3700 @ 1.60GHz"), + vendor: "intel", + }, + CpuinfoCase { + // Before: summary read `Hardware`, examination read `model name` + // only and reported no CPU (empty model, vendor unknown). + name: "Hardware and no model name", + text: "processor\t: 0\nBogoMIPS\t: 38.40\nHardware\t: BCM2835\nRevision\t: a02082\n", + model: Some("BCM2835"), + vendor: "unknown", + }, + CpuinfoCase { + // Before: summary took whichever of the two came first + // (`BCM2835`), examination only the `model name`. + name: "Hardware before model name", + text: "Hardware\t: BCM2835\nmodel name\t: ARMv7 Processor rev 4 (v7l)\n", + model: Some("BCM2835"), + vendor: "unknown", + }, + CpuinfoCase { + // Before: examination took an empty `vendor_id` as the vendor + // (`""`) and stopped looking. An empty value names no vendor, so + // the next `vendor_id` line is read. + name: "an empty vendor_id value", + text: "vendor_id\t:\nmodel name\t: Some CPU\nvendor_id\t: GenuineIntel\n", + model: Some("Some CPU"), + vendor: "intel", + }, + CpuinfoCase { + // The same input where no later line names a vendor. Before: + // examination `""`; now `unknown`, which is what the JSON says + // for a vendor that was not read. + name: "an empty vendor_id value and no other", + text: "vendor_id\t:\nmodel name\t: Some CPU\n", + model: Some("Some CPU"), + vendor: "unknown", + }, + ]; + for case in cases { + assert_cpuinfo_case(&case); + } + } + + fn assert_cpuinfo_case(case: &CpuinfoCase) { + // The summary's `cpu`: what `detect_cpu_model` makes of the file. + assert_eq!( + cpu_facts_from_cpuinfo(case.text).model.as_deref(), + case.model, + "summary, {}", + case.name + ); + // The examination's fields. + let mut examined = Examination::default(); + record_cpu(&mut examined, cpu_facts_from_cpuinfo(case.text)); + assert_eq!( + (examined.cpu_model.as_str(), examined.cpu_vendor.as_str()), + (case.model.unwrap_or(""), case.vendor), + "examination, {}", + case.name + ); + assert!( + examined.probe_failures.is_empty(), + "a cpuinfo with no CPU in it is not a probe failure, {}: {:?}", + case.name, + examined.probe_failures + ); + } + + /// `detect_cpu_model` reads the CPU on Linux and nothing anywhere else, so no + /// Windows query can start from it: with the host check false the reader is + /// never called. Calling the reader whatever the host turns this red. + #[test] + fn detect_cpu_model_reads_nothing_off_linux() { + let never = + || -> CpuFacts { panic!("the CPU is read through detect_cpu_model on Linux only") }; + assert_eq!(linux_cpu_model(false, never), None); + assert_eq!( + linux_cpu_model(true, || cpu_facts_from_cpuinfo("model name\t: A B\n")).as_deref(), + Some("A B") + ); + } + + /// What the examination makes of one answer from the system query. + fn examination_on_windows(output: Option<&str>) -> (Examination, Duration) { + let mut examined = Examination::default(); + let mut asked = None; + probe_cpu_windows_with(&mut examined, |_, timeout| { + asked = Some(timeout); + output.map(str::to_owned) + }); + (examined, asked.expect("the examination ran its query")) + } + + /// The summary's system inventory with only `system` wired, so a test sees + /// the query the CPU row comes from and counts how many run. + struct SystemQuery<'a> { + output: Option<&'a str>, + asked: Vec<(String, Duration)>, + } + + impl WindowsInventorySources for SystemQuery<'_> { + fn pnputil(&mut self) -> Option { + None + } + fn video_controller(&mut self) -> Option { + None + } + fn pnp_entity(&mut self) -> Option { + None + } + fn system(&mut self) -> Option { + windows_system_inventory_with(|script, timeout| { + self.asked.push((script.to_owned(), timeout)); + self.output.map(str::to_owned) + }) + } + } + + /// What the summary reports for the CPU on Windows when the system query + /// prints `output` (`None`: it could not run), and every query that ran. + fn summary_on_windows(output: Option<&str>) -> (Option, Vec<(String, Duration)>) { + let mut sources = SystemQuery { + output, + asked: Vec::new(), + }; + let inventory = assemble_windows_inventory(&mut sources, InventoryScope::SUMMARY); + ( + cpu_facts_from_windows_inventory(inventory.as_ref()).model, + sources.asked, + ) + } + + const WINDOWS_CPU_NAME: &str = "AMD Ryzen 9 9950X 16-Core Processor"; + + #[test] + fn windows_both_reads_report_the_name_the_query_returns() { + let output = format!("CPU\t{WINDOWS_CPU_NAME}\r\nRAM\t68719476736\r\n"); + let (summary, _) = summary_on_windows(Some(&output)); + assert_eq!(summary.as_deref(), Some(WINDOWS_CPU_NAME)); + let (examined, _) = examination_on_windows(Some(&output)); + assert_eq!(examined.cpu_model, WINDOWS_CPU_NAME); + assert_eq!(examined.cpu_vendor, "amd"); + assert!(examined.probe_failures.is_empty()); + } + + /// Before: the summary collapsed whitespace runs and the examination + /// trimmed and kept the first line only. Windows pads the name of some CPUs. + /// Both collapse now. + #[test] + fn windows_both_reads_collapse_whitespace_inside_the_name() { + let output = "CPU\t Intel(R) Pentium(R) CPU N3700 @ 1.60GHz \r\n"; + let (summary, _) = summary_on_windows(Some(output)); + assert_eq!( + summary.as_deref(), + Some("Intel(R) Pentium(R) CPU N3700 @ 1.60GHz") + ); + let (examined, _) = examination_on_windows(Some(output)); + assert_eq!( + examined.cpu_model, + "Intel(R) Pentium(R) CPU N3700 @ 1.60GHz" + ); + assert_eq!(examined.cpu_vendor, "intel"); + } + + #[test] + fn windows_the_vendor_is_derived_from_the_model_string() { + for (name, vendor) in [ + ("AMD Ryzen 7 5800X 8-Core Processor", "amd"), + ("amd epyc 7763", "amd"), + ("Intel(R) Core(TM) i9-13900K", "intel"), + ("Genuine Intel(R) CPU 0000", "intel"), + ("Qualcomm(R) Snapdragon(R) X Elite", "unknown"), + ] { + let (examined, _) = examination_on_windows(Some(&format!("CPU\t{name}\n"))); + assert_eq!(examined.cpu_vendor, vendor, "{name}"); + assert_eq!(examined.cpu_model, name); + } + } + + /// The query printed no CPU row (only the RAM row, or a blank one), or + /// nothing, or could not run: both reads report no CPU. The examination + /// records a failure and leaves the vendor `unknown`; the summary starts no + /// second query. Before: an empty answer or a query that could not run + /// sent the summary to a second, separate PowerShell query for the CPU. + #[test] + fn windows_an_answer_with_no_cpu_is_no_cpu_in_both_reads() { + for output in [ + Some("RAM\t17179869184\n"), + Some("CPU\t \nRAM\t17179869184\n"), + Some(""), + None, + ] { + let (summary, queries) = summary_on_windows(output); + assert_eq!(summary, None, "{output:?}"); + assert_eq!( + queries.len(), + 1, + "one system query, no second one: {output:?}" + ); + let (examined, _) = examination_on_windows(output); + assert_eq!(examined.cpu_model, "", "{output:?}"); + assert_eq!(examined.cpu_vendor, "unknown", "{output:?}"); + // The message and the state it describes are asserted together: the + // entry names the inventory query that ran (CPU and RAM), not a + // `Win32_Processor` query nothing runs any more, and it is there + // exactly when the model is empty and the vendor unknown. + assert_eq!( + examined.probe_failures, + [ + "The Windows system inventory query (CPU and RAM) reported no CPU; cannot identify CPU." + ], + "{output:?}" + ); + } + } + + /// One limit for both reads: 5 s, the limit of every inventory query. Before: + /// 8 s for the examination, 1.5 s for the summary's system query and for its + /// own CPU query. + #[test] + fn windows_both_reads_are_given_the_same_time() { + let (_, examination_limit) = examination_on_windows(Some("CPU\tAMD EPYC 7763\n")); + assert_eq!(examination_limit, Duration::from_secs(5)); + let (_, queries) = summary_on_windows(Some("CPU\tAMD EPYC 7763\n")); + let (script, summary_limit) = &queries[0]; + assert_eq!(*summary_limit, Duration::from_secs(5)); + assert!( + script.contains("Win32_Processor") && script.contains("Win32_ComputerSystem"), + "the CPU row comes from the system query, which also asks for RAM: {script}" + ); + } + + /// With no inventory at all the summary has no CPU, and reads the CPU row of + /// an inventory it holds without asking again. + #[test] + fn windows_the_inventory_is_the_summarys_only_source() { + assert_eq!(cpu_facts_from_windows_inventory(None), CpuFacts::unknown()); + let inventory = crate::host_gpu::windows_inventory::parse_windows_examine_inventory( + "CPU\tAMD EPYC 7763\n", + ); + assert_eq!( + cpu_facts_from_windows_inventory(Some(&inventory)), + CpuFacts { + model: Some("AMD EPYC 7763".to_owned()), + vendor: "amd".to_owned() + } + ); + } + + /// The decision recorded in `docs/host-probing.md`: one 5 s limit for the + /// kernel read. The timeout behaviour itself is exercised, with a stand-in + /// `uname`, by the `kernel_pair` tests in `examine/host_env.rs`. + #[test] + fn the_kernel_release_timeout_is_five_seconds() { + assert_eq!(KERNEL_RELEASE_TIMEOUT, Duration::from_secs(5)); + } } diff --git a/crates/rocm-core/src/host_gpu/windows_inventory.rs b/crates/rocm-core/src/host_gpu/windows_inventory.rs index 3a47d86747..ec6db3e7b5 100644 --- a/crates/rocm-core/src/host_gpu/windows_inventory.rs +++ b/crates/rocm-core/src/host_gpu/windows_inventory.rs @@ -6,9 +6,7 @@ //! their output, and the assembly of one inventory from several sources. #[cfg(windows)] -use super::capture::{ - OPTIONAL_COMMAND_TIMEOUT, capture_optional_command, capture_optional_command_with_timeout, -}; +use super::capture::{capture_optional_command, capture_optional_command_with_timeout}; use super::os_facts::{bytes_text_to_gib, normalize_cpu_model}; use crate::gfx_target::{ amd_pci_device_id_from_pnp_id, gfx_target_from_amd_marketing_name, @@ -20,12 +18,20 @@ use std::time::Duration; pub(super) const WINDOWS_INVENTORY_QUERY_TIMEOUT: Duration = Duration::from_secs(5); +/// What the system query (CPU model and RAM) is allowed: the same 5 s every +/// other inventory query gets, for the summary's inventory and the +/// examination's CPU read alike. It was the 1.5 s of an optional command in +/// the summary and 8 s in the examination's own CPU query. One limit sits +/// between the two: a CIM query on a PowerShell that has not started yet can +/// outlast 1.5 s, so the summary's read waits longer than it did, and the +/// examination's gives up 3 s sooner. +const WINDOWS_SYSTEM_INVENTORY_TIMEOUT: Duration = WINDOWS_INVENTORY_QUERY_TIMEOUT; + pub(super) const WINDOWS_VIDEO_CONTROLLER_INVENTORY_SCRIPT: &str = r#"$gpus = Get-CimInstance -ClassName Win32_VideoController -Property Name,DriverVersion,DriverDate,PNPDeviceID,AdapterCompatibility | Where-Object { $_.PNPDeviceID -match 'VEN_1002' -or $_.AdapterCompatibility -match 'AMD|Advanced Micro Devices' -or $_.Name -match 'AMD|Radeon|Instinct' }; foreach ($gpu in $gpus) { $date = ''; if ($gpu.DriverDate) { $date = $gpu.DriverDate.ToUniversalTime().ToString('yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) }; "GPU`t$($gpu.Name)`t$($gpu.DriverVersion)`t$($gpu.PNPDeviceID)`t$date" }"#; #[cfg(windows)] pub(super) const WINDOWS_PNP_ENTITY_INVENTORY_SCRIPT: &str = r#"$displayGuid = '{4d36e968-e325-11ce-bfc1-08002be10318}'; $gpus = Get-CimInstance -ClassName Win32_PnPEntity -Property Name,DeviceID,PNPClass,ClassGuid,Manufacturer | Where-Object { (($_.PNPClass -eq 'Display' -or $_.ClassGuid -eq $displayGuid) -and ($_.DeviceID -match 'VEN_1002' -or $_.Name -match 'AMD|Radeon|Instinct|Graphics' -or $_.Manufacturer -match 'AMD|Advanced Micro Devices')) -or ($_.DeviceID -match 'PCI\\VEN_1002' -and $_.Name -match 'Radeon|Instinct|Graphics') }; foreach ($gpu in $gpus) { "GPU`t$($gpu.Name)`t`t$($gpu.DeviceID)" }"#; -#[cfg(windows)] const WINDOWS_SYSTEM_INVENTORY_SCRIPT: &str = r#"$cpu = Get-CimInstance -ClassName Win32_Processor -Property Name | Select-Object -First 1 -ExpandProperty Name; if ($cpu) { "CPU`t$cpu" }; $ram = Get-CimInstance -ClassName Win32_ComputerSystem -Property TotalPhysicalMemory | Select-Object -First 1 -ExpandProperty TotalPhysicalMemory; if ($ram) { "RAM`t$ram" }"#; #[cfg(windows)] @@ -507,6 +513,13 @@ fn detect_windows_system_inventory_from_cim() -> Option if !runtime_is_windows() { return None; } + windows_system_inventory_with(run_windows_system_query) +} + +/// Run the system query's `script` under PowerShell for at most `timeout`, and +/// answer with its stdout, or `None` when it could not run, failed or timed out. +#[cfg(windows)] +pub(crate) fn run_windows_system_query(script: &str, timeout: Duration) -> Option { capture_optional_command_with_timeout( "powershell", &[ @@ -514,9 +527,29 @@ fn detect_windows_system_inventory_from_cim() -> Option "-NoProfile", "-NonInteractive", "-Command", - WINDOWS_SYSTEM_INVENTORY_SCRIPT, + script, ], - OPTIONAL_COMMAND_TIMEOUT, + timeout, + ) +} + +/// Off Windows there is no PowerShell to run the system query. +#[cfg(not(windows))] +pub(crate) const fn run_windows_system_query(_script: &str, _timeout: Duration) -> Option { + None +} + +/// The system inventory (CPU model and installed RAM) from one query, with the +/// query injected so a test on any host can drive it. `query` is handed the +/// script and the time [`WINDOWS_SYSTEM_INVENTORY_TIMEOUT`] allows it, and +/// answers with what the script printed, or `None` when it could not run, +/// failed or timed out. +pub(super) fn windows_system_inventory_with( + query: impl FnOnce(&str, Duration) -> Option, +) -> Option { + query( + WINDOWS_SYSTEM_INVENTORY_SCRIPT, + WINDOWS_SYSTEM_INVENTORY_TIMEOUT, ) .map(|output| parse_windows_examine_inventory(&output)) } diff --git a/crates/rocm-core/src/host_gpu/wsl.rs b/crates/rocm-core/src/host_gpu/wsl.rs index 098d99ef22..c5900a039a 100644 --- a/crates/rocm-core/src/host_gpu/wsl.rs +++ b/crates/rocm-core/src/host_gpu/wsl.rs @@ -5,9 +5,7 @@ //! WSL detection and the Windows host driver as seen from the guest: the WSL2 //! signals, the librocdxg chain, and the local and WSL host-driver probes. -use super::capture::{ - capture_optional_command, capture_optional_command_with_timeout, tool_on_path, -}; +use super::capture::{capture_optional_command, capture_optional_command_with_timeout}; use super::windows_inventory::{ WINDOWS_INVENTORY_QUERY_TIMEOUT, WINDOWS_VIDEO_CONTROLLER_INVENTORY_SCRIPT, WindowsExamineInventory, detect_windows_examine_inventory_with_driver_facts, @@ -15,6 +13,7 @@ use super::windows_inventory::{ }; use super::{DriverSummary, WslSummary}; use crate::gfx_target::parse_windows_display_gfx_target; +use crate::probe_exec::which; use crate::{discover_rocm_installs, host_path, runtime_is_linux, runtime_is_windows}; use std::fs; use std::path::{Path, PathBuf}; @@ -372,8 +371,8 @@ pub(crate) fn detect_wsl_summary() -> Option { // The roots are already re-rooted through `host_path`, so this existence check // takes the path as given. let librocdxg_beside_runtime = librocdxg_beside_runtime(&roots, Path::exists); - let rocminfo = tool_on_path("rocminfo"); - let cargo = tool_on_path("cargo"); + let rocminfo = which("rocminfo"); + let cargo = which("cargo"); let missing = missing_plumbing( dxg_device, dxcore, diff --git a/crates/rocm-core/src/jsonl_store.rs b/crates/rocm-core/src/jsonl_store.rs index 497664c844..abd0ac0a91 100644 --- a/crates/rocm-core/src/jsonl_store.rs +++ b/crates/rocm-core/src/jsonl_store.rs @@ -11,22 +11,22 @@ //! with the store. A torn tail — crash debris, or left by a write that failed //! partway — is cut by the next append rather than truncated in place: that //! append writes the store's clean prefix plus its own new record as a single -//! rewrite via `write_atomic`, so a lock-free reader never observes a file -//! shrunk mid-repair. That rewrite costs O(store) rather than O(record), the -//! same as a proposal status update, which rewrites the whole store via -//! `write_atomic` too. Only when that rewrite itself fails (a full disk, or on -//! Windows another program holding the store open without delete sharing) does -//! the append cut the debris in place instead, so a store cannot be wedged by -//! a failed repair; otherwise nothing in this module ever shrinks a store in -//! place. Readers take no lock, and an unterminated last line that is not -//! valid JSON — an append caught mid-write — is skipped rather than reported -//! as corruption. +//! rewrite via [`crate::atomic_write::write_file_atomically`], so a lock-free +//! reader never observes a file shrunk mid-repair. That rewrite costs O(store) +//! rather than O(record), the same as a proposal status update, which rewrites +//! the whole store the same way. Only when that rewrite itself fails (a full +//! disk, or on Windows another program holding the store open without delete +//! sharing) does the append cut the debris in place instead, so a store cannot +//! be wedged by a failed repair; otherwise nothing in this module ever shrinks +//! a store in place. Readers take no lock, and an unterminated last line that +//! is not valid JSON — an append caught mid-write — is skipped rather than +//! reported as corruption. //! //! A store can additionally be capped by size-based rotation (see //! [`JsonlRotation`]): the append that finds the live file at the cap renames //! it whole to `.1` first, under the same lock — before any torn tail //! in it would otherwise be repaired, so an at-cap store is never paid for -//! with a `write_atomic` rewrite only to be rotated away immediately after — +//! with an atomic rewrite only to be rotated away immediately after — //! and the recent-record readers span the rotated generations. //! //! The engine is generic over the record type. Which stores exist, their @@ -200,7 +200,7 @@ pub(crate) fn append_jsonl_record( file, tail_start, &line, - |path, content| crate::write_atomic(path, content, "jsonl"), + crate::atomic_write::write_file_atomically, truncate_jsonl_in_place, )?, TailState::ValidUnterminated(start) => { @@ -259,12 +259,11 @@ fn repair_torn_tail_and_append( content.extend_from_slice(line); // Done with the pre-repair file's handle before replacing it, rather than // leaving that to whatever happens to still have `file` open at function - // return. The rename already succeeds with this handle open regardless — - // std opens files with `FILE_SHARE_DELETE` by default on Windows, so a - // rename isn't blocked by it (see `write_atomic`'s doc for the case that - // *isn't* covered: another process holding the destination open without - // delete-sharing) — but this states the intent rather than leaning on that - // default. + // return. The replace already succeeds with this handle open regardless — + // std opens files with `FILE_SHARE_DELETE` by default on Windows, so it + // isn't blocked by it (see the `atomic_write` module doc for how the publish + // treats a destination another process holds open) — but this states the + // intent rather than leaning on that default. drop(file); let replace_error = match replace(path, &content) { Ok(()) => return Ok(()), @@ -432,7 +431,7 @@ fn inspect_tail(path: &Path, file: &fs::File) -> Result { /// /// Returns fewer bytes if the file now ends before `end`. Nothing this /// module's own writers do shrinks a store in place — an append only ever -/// grows it, and a torn-tail repair replaces it wholesale via `write_atomic` +/// grows it, and a torn-tail repair replaces it wholesale via the atomic writer /// (see [`inspect_tail`] and [`TailState::Torn`]) — except the repair's /// fallback, taken only when that replacement fails and only under the store's /// lock (see [`repair_torn_tail_and_append`]); otherwise a short read here @@ -841,10 +840,68 @@ mod test_hooks { #[cfg(test)] mod tests { use super::*; - use crate::test_support::{ - audit_line, audit_messages, audit_record, seed_audit_store, temp_app_paths, - }; - use crate::{AppPaths, AuditEventRecord, append_audit_event, load_recent_audit_events}; + use crate::test_support::temp_app_paths; + use serde::Deserialize; + + /// The record these tests store. The engine is generic over its record + /// type, so its tests carry their own rather than borrow one of its users': + /// `automation.rs` covers the stores it builds on this engine. + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] + struct Entry { + tag: String, + /// Padding, so a line is about as long as a real store's. The + /// rotation tests size their byte caps, and the torn-tail tests cut + /// their debris, against that length. + detail: String, + } + + fn entry(tag: &str) -> Entry { + Entry { + tag: tag.to_owned(), + detail: "d".repeat(110), + } + } + + /// One serialized record, without its newline. + fn entry_line(tag: &str) -> String { + serde_json::to_string(&entry(tag)).expect("serialize entry") + } + + fn entry_tags(entries: &[Entry]) -> Vec<&str> { + entries.iter().map(|entry| entry.tag.as_str()).collect() + } + + /// A throwaway root, and the path of a store inside it, whose directory + /// exists. + fn temp_store(name: &str) -> Result<(PathBuf, PathBuf)> { + let (root, _paths) = temp_app_paths(name); + let path = root.join("store").join("events.jsonl"); + fs::create_dir_all(path.parent().context("a store has a directory")?)?; + Ok((root, path)) + } + + /// Write `contents` as the store, as an out-of-band edit or a killed + /// writer would have left it. + fn seed_store(path: &Path, contents: &str) -> Result<()> { + fs::write(path, contents)?; + Ok(()) + } + + /// Append the record tagged `tag` to `path`, with no rotation. + fn append_entry(path: &Path, tag: &str) -> Result<()> { + append_jsonl_record(path, &entry(tag), None, "entry record") + } + + /// The most recent `limit` records of the store at `path`, reading up to + /// the generations a real store keeps. The 4 mirrors + /// `EVENT_LOG_ROTATED_GENERATIONS` in `automation.rs`, which the audit + /// reader these tests replaced used. It is a literal, not that constant, + /// so these tests keep no edge to `automation.rs`; if the production + /// value changes, change this one on purpose, since the tests reading + /// through it then no longer see the span a real store keeps. + fn load_recent_entries(path: &Path, limit: usize) -> Result> { + load_recent_jsonl_records(path, 4, limit, "entry") + } #[test] fn a_rotation_failure_is_reported_once_per_store() { @@ -919,18 +976,16 @@ mod tests { // it — sees the file end early; that is not an error. #[test] fn a_scan_that_finds_the_file_shorter_than_it_was_still_succeeds() -> Result<()> { - let (root, paths) = temp_app_paths("scan-shrunk"); - seed_audit_store( - &paths, - &format!("{}\n{}\n", audit_line("a"), audit_line("b")), + let (root, path) = temp_store("scan-shrunk")?; + seed_store( + &path, + &format!("{}\n{}\n", entry_line("a"), entry_line("b")), )?; - let path = paths.audit_events_path(); let file = fs::File::open(&path)?; let stale_len = file.metadata()?.len() + 40; - let recent = - scan_recent_jsonl_records::(&path, &file, stale_len, 5, "audit"); + let recent = scan_recent_jsonl_records::(&path, &file, stale_len, 5, "entry"); fs::remove_dir_all(root).ok(); - assert_eq!(audit_messages(&recent?), ["a", "b"]); + assert_eq!(entry_tags(&recent?), ["a", "b"]); Ok(()) } @@ -946,11 +1001,10 @@ mod tests { // `automation.rs`, just complete JSON rather than torn). #[test] fn load_recent_retries_past_a_shrink_mid_scan_and_succeeds() -> Result<()> { - let (root, paths) = temp_app_paths("scan-retry-after-shrink"); - let seed = format!("{}\n", audit_line("seed")); - let giant = audit_line(&"x".repeat(200_000)); - seed_audit_store(&paths, &format!("{seed}{giant}"))?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("scan-retry-after-shrink")?; + let seed = format!("{}\n", entry_line("seed")); + let giant = entry_line(&"x".repeat(200_000)); + seed_store(&path, &format!("{seed}{giant}"))?; let seed_len = seed.len() as u64; let hook_path = path.clone(); test_hooks::set_before_scan_read( @@ -964,10 +1018,10 @@ mod tests { .expect("truncate mid-scan"); }), ); - let recent = load_recent_audit_events(&paths, 5); + let recent = load_recent_entries(&path, 5); fs::remove_dir_all(root).ok(); assert_eq!( - audit_messages(&recent?), + entry_tags(&recent?), ["seed"], "a shrink strictly between two chunked reads must be retried, not failed" ); @@ -1044,21 +1098,20 @@ mod tests { )) } - /// Run the torn-tail repair on the seeded audit store with the replace and - /// truncate steps supplied by the test, appending the record `b`. - fn repair_audit_store_with( - paths: &AppPaths, + /// Run the torn-tail repair on the seeded store at `path` with the replace + /// and truncate steps supplied by the test, appending the record `b`. + fn repair_store_with( + path: &Path, replace: impl FnOnce(&Path, &[u8]) -> Result<()>, truncate: impl FnOnce(&Path, u64) -> Result<()>, ) -> Result<()> { - let path = paths.audit_events_path(); - let file = open_jsonl_for_append(&path)?; - let TailState::Torn(tail_start) = inspect_tail(&path, &file)? else { + let file = open_jsonl_for_append(path)?; + let TailState::Torn(tail_start) = inspect_tail(path, &file)? else { bail!("the seeded store has no torn tail"); }; - let mut line = audit_line("b").into_bytes(); + let mut line = entry_line("b").into_bytes(); line.push(b'\n'); - repair_torn_tail_and_append(&path, file, tail_start, &line, replace, truncate) + repair_torn_tail_and_append(path, file, tail_start, &line, replace, truncate) } // The fallback the whole-store rewrite lacked: when it cannot complete @@ -1069,15 +1122,14 @@ mod tests { // in `automation.rs` pins. #[test] fn a_torn_tail_is_cut_in_place_when_replacing_the_store_fails() -> Result<()> { - let (root, paths) = temp_app_paths("torn-tail-fallback-truncates"); - let torn = &audit_line("torn")[..20]; - seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("torn-tail-fallback-truncates")?; + let torn = &entry_line("torn")[..20]; + seed_store(&path, &format!("{}\n{torn}", entry_line("a")))?; let before = file_identity(&path)?; let mut attempted = Vec::new(); - repair_audit_store_with( - &paths, + repair_store_with( + &path, |_, content| { attempted = content.to_vec(); bail!("no space left on device") @@ -1087,9 +1139,9 @@ mod tests { let after = file_identity(&path)?; let text = fs::read_to_string(&path); - let recent = load_recent_audit_events(&paths, 5); + let recent = load_recent_entries(&path, 5); fs::remove_dir_all(root).ok(); - let expected = format!("{}\n{}\n", audit_line("a"), audit_line("b")); + let expected = format!("{}\n{}\n", entry_line("a"), entry_line("b")); assert_eq!( String::from_utf8(attempted)?, expected, @@ -1097,7 +1149,7 @@ mod tests { ); assert_eq!(text?, expected); assert_eq!(before, after, "the fallback must cut the debris in place"); - assert_eq!(audit_messages(&recent?), ["a", "b"]); + assert_eq!(entry_tags(&recent?), ["a", "b"]); Ok(()) } @@ -1106,39 +1158,37 @@ mod tests { // every repair. #[test] fn a_torn_tail_is_not_truncated_when_replacing_the_store_works() -> Result<()> { - let (root, paths) = temp_app_paths("torn-tail-fallback-unused"); - let torn = &audit_line("torn")[..20]; - seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("torn-tail-fallback-unused")?; + let torn = &entry_line("torn")[..20]; + seed_store(&path, &format!("{}\n{torn}", entry_line("a")))?; let before = file_identity(&path)?; - repair_audit_store_with( - &paths, - |path, content| crate::write_atomic(path, content, "jsonl"), - |_, _| bail!("truncate must not run when the replace succeeded"), - )?; + repair_store_with(&path, crate::atomic_write::write_file_atomically, |_, _| { + bail!("truncate must not run when the replace succeeded") + })?; let after = file_identity(&path)?; let text = fs::read_to_string(&path); fs::remove_dir_all(root).ok(); - assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(text?, format!("{}\n{}\n", entry_line("a"), entry_line("b"))); assert_ne!(before, after, "a working replace publishes a new file"); Ok(()) } #[test] fn a_torn_tail_that_can_be_neither_replaced_nor_cut_reports_both_causes() -> Result<()> { - let (root, paths) = temp_app_paths("torn-tail-fallback-both-fail"); - let torn = &audit_line("torn")[..20]; - let seeded = format!("{}\n{torn}", audit_line("a")); - seed_audit_store(&paths, &seeded)?; - let path = paths.audit_events_path(); - - // Layered like the real steps (`write_atomic` and - // `truncate_jsonl_in_place` both wrap the OS error in a context), so - // the assertion fails if either cause is formatted without its chain. - let result = repair_audit_store_with( - &paths, + let (root, path) = temp_store("torn-tail-fallback-both-fail")?; + let torn = &entry_line("torn")[..20]; + let seeded = format!("{}\n{torn}", entry_line("a")); + seed_store(&path, &seeded)?; + + // Layered like the real steps (`write_file_atomically` puts a context + // such as `failed to publish ` over the OS error, and + // `truncate_jsonl_in_place` puts `failed to truncate ` over its + // own), so the assertion fails if either cause is formatted without its + // chain. + let result = repair_store_with( + &path, |_, _| { Err(anyhow::anyhow!("no space left on device")) .context("failed to replace the store") @@ -1149,7 +1199,7 @@ mod tests { ); let text = fs::read_to_string(&path); - let recent = load_recent_audit_events(&paths, 5); + let recent = load_recent_entries(&path, 5); fs::remove_dir_all(root).ok(); assert_eq!( format!("{:#}", result.expect_err("both steps failed")), @@ -1164,7 +1214,7 @@ mod tests { text?, seeded, "a failed repair must leave the store as it was" ); - assert_eq!(audit_messages(&recent?), ["a"]); + assert_eq!(entry_tags(&recent?), ["a"]); Ok(()) } @@ -1173,13 +1223,12 @@ mod tests { // stub swaps the store for a directory, which makes reopening it fail. #[test] fn a_failed_append_after_the_cut_still_reports_why_the_replace_failed() -> Result<()> { - let (root, paths) = temp_app_paths("torn-tail-fallback-append-fails"); - let torn = &audit_line("torn")[..20]; - seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("torn-tail-fallback-append-fails")?; + let torn = &entry_line("torn")[..20]; + seed_store(&path, &format!("{}\n{torn}", entry_line("a")))?; - let result = repair_audit_store_with( - &paths, + let result = repair_store_with( + &path, |_, _| { Err(anyhow::anyhow!("no space left on device")) .context("failed to replace the store") @@ -1208,18 +1257,17 @@ mod tests { // one-line append: no second repair, no new file. #[test] fn appends_after_a_fallback_repair_are_plain_appends() -> Result<()> { - let (root, paths) = temp_app_paths("torn-tail-fallback-then-append"); - let torn = &audit_line("torn")[..20]; - seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; - let path = paths.audit_events_path(); - repair_audit_store_with( - &paths, + let (root, path) = temp_store("torn-tail-fallback-then-append")?; + let torn = &entry_line("torn")[..20]; + seed_store(&path, &format!("{}\n{torn}", entry_line("a")))?; + repair_store_with( + &path, |_, _| bail!("no space left on device"), truncate_jsonl_in_place, )?; let repaired = file_identity(&path)?; - append_audit_event(&paths, &audit_record("c"))?; + append_entry(&path, "c")?; let after = file_identity(&path)?; let text = fs::read_to_string(&path); @@ -1228,9 +1276,9 @@ mod tests { text?, format!( "{}\n{}\n{}\n", - audit_line("a"), - audit_line("b"), - audit_line("c") + entry_line("a"), + entry_line("b"), + entry_line("c") ) ); assert_eq!(repaired, after); @@ -1250,11 +1298,10 @@ mod tests { fn append_repairs_a_torn_tail_in_a_directory_that_takes_no_new_files() -> Result<()> { use std::os::unix::fs::PermissionsExt; - let (root, paths) = temp_app_paths("torn-tail-read-only-dir"); - let torn = &audit_line("torn")[..20]; - seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; - let path = paths.audit_events_path(); - let dir = path.parent().context("the audit store has a directory")?; + let (root, path) = temp_store("torn-tail-read-only-dir")?; + let torn = &entry_line("torn")[..20]; + seed_store(&path, &format!("{}\n{torn}", entry_line("a")))?; + let dir = path.parent().context("the store has a directory")?; // Create the lock file now: the append reuses an existing one, but // could not create it inside the read-only directory. drop(FileLock::acquire(jsonl_lock_path(&path))?); @@ -1262,7 +1309,7 @@ mod tests { fs::set_permissions(dir, fs::Permissions::from_mode(0o555))?; let read_only_holds = fs::File::create(dir.join("probe")).is_err(); - let result = read_only_holds.then(|| append_audit_event(&paths, &audit_record("b"))); + let result = read_only_holds.then(|| append_entry(&path, "b")); fs::set_permissions(dir, fs::Permissions::from_mode(0o755))?; let after = file_identity(&path)?; @@ -1274,7 +1321,7 @@ mod tests { return Ok(()); }; result?; - assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(text?, format!("{}\n{}\n", entry_line("a"), entry_line("b"))); assert_eq!(before, after); Ok(()) } @@ -1288,24 +1335,23 @@ mod tests { use std::os::windows::fs::OpenOptionsExt; const FILE_SHARE_READ_WRITE: u32 = 0x1 | 0x2; - let (root, paths) = temp_app_paths("torn-tail-share-violation"); - let torn = &audit_line("torn")[..20]; - seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("torn-tail-share-violation")?; + let torn = &entry_line("torn")[..20]; + seed_store(&path, &format!("{}\n{torn}", entry_line("a")))?; let before = file_identity(&path)?; let blocker = fs::OpenOptions::new() .read(true) .share_mode(FILE_SHARE_READ_WRITE) .open(&path)?; - let result = append_audit_event(&paths, &audit_record("b")); + let result = append_entry(&path, "b"); drop(blocker); let after = file_identity(&path)?; let text = fs::read_to_string(&path); fs::remove_dir_all(root).ok(); result?; - assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(text?, format!("{}\n{}\n", entry_line("a"), entry_line("b"))); assert_eq!(before, after); Ok(()) } @@ -1343,47 +1389,41 @@ mod tests { #[test] fn the_store_lock_is_a_sibling_of_the_store() { - let paths = AppPaths { - config_dir: PathBuf::from("c"), - data_dir: PathBuf::from("d"), - cache_dir: PathBuf::from("k"), - }; + let events = PathBuf::from("d").join("first").join("events.jsonl"); + let proposals = PathBuf::from("d").join("second").join("proposals.jsonl"); assert_eq!( - jsonl_lock_path(&paths.audit_events_path()), - paths.audit_dir().join("events.jsonl.lock") + jsonl_lock_path(&events), + PathBuf::from("d").join("first").join("events.jsonl.lock") ); assert_eq!( - jsonl_lock_path(&paths.automation_proposals_path()), - paths.automations_dir().join("proposals.jsonl.lock") + jsonl_lock_path(&proposals), + PathBuf::from("d") + .join("second") + .join("proposals.jsonl.lock") ); } #[test] fn rotated_generations_are_numbered_siblings_of_the_store() { - let path = PathBuf::from("d").join("audit").join("events.jsonl"); + let path = PathBuf::from("d").join("store").join("events.jsonl"); assert_eq!(jsonl_generation_path(&path, 0), path); assert_eq!( jsonl_generation_path(&path, 1), - PathBuf::from("d").join("audit").join("events.jsonl.1") + PathBuf::from("d").join("store").join("events.jsonl.1") ); assert_eq!( jsonl_generation_path(&path, 4), - PathBuf::from("d").join("audit").join("events.jsonl.4") + PathBuf::from("d").join("store").join("events.jsonl.4") ); } - /// Append an audit record tagged `tag` to `path` under `rotation`. + /// Append the record tagged `tag` to `path` under `rotation`. fn append_rotating(path: &Path, tag: &str, rotation: JsonlRotation) -> Result<()> { - append_jsonl_record( - path, - &audit_record(tag), - Some(rotation), - "audit event record", - ) + append_jsonl_record(path, &entry(tag), Some(rotation), "entry record") } - /// The tags in every generation of the audit-shaped store at `path`, - /// oldest first, read straight from the files. + /// The tags in every generation of the store at `path`, oldest first, read + /// straight from the files. fn tags_in_every_generation(path: &Path, generations: usize) -> Result> { let mut tags = Vec::new(); for generation in (0..=generations).rev() { @@ -1392,7 +1432,7 @@ mod tests { continue; } for line in fs::read_to_string(&generation_path)?.lines() { - tags.push(serde_json::from_str::(line)?.message); + tags.push(serde_json::from_str::(line)?.tag); } } Ok(tags) @@ -1400,18 +1440,16 @@ mod tests { fn recent_tags(path: &Path, generations: usize, limit: usize) -> Result> { Ok( - load_recent_jsonl_records::(path, generations, limit, "audit")? + load_recent_jsonl_records::(path, generations, limit, "entry")? .into_iter() - .map(|record| record.message) + .map(|record| record.tag) .collect(), ) } #[test] fn rotation_keeps_the_live_file_and_the_configured_generations() -> Result<()> { - let (root, paths) = temp_app_paths("rotation-generations"); - paths.ensure()?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("rotation-generations")?; // One record per file: every append after the first rotates. let rotation = JsonlRotation { max_bytes: 1, @@ -1439,10 +1477,8 @@ mod tests { // asks for; the rest must come from the rotated generations, in order. #[test] fn load_recent_spans_a_rollover() -> Result<()> { - let (root, paths) = temp_app_paths("rotation-spans"); - paths.ensure()?; - let path = paths.audit_events_path(); - let line_len = audit_line("00").len() as u64 + 1; + let (root, path) = temp_store("rotation-spans")?; + let line_len = entry_line("00").len() as u64 + 1; // Rotate once a file holds five records. let rotation = JsonlRotation { max_bytes: 5 * line_len, @@ -1473,11 +1509,9 @@ mod tests { // where `.1` was shifted to `.2` but the live file not yet renamed. #[test] fn load_recent_reads_a_store_caught_mid_rotation_in_order() -> Result<()> { - let (root, paths) = temp_app_paths("rotation-mid"); - paths.ensure()?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("rotation-mid")?; let write = |generation: usize, tags: &[&str]| -> Result<()> { - let text: String = tags.iter().map(|tag| audit_line(tag) + "\n").collect(); + let text: String = tags.iter().map(|tag| entry_line(tag) + "\n").collect(); fs::write(jsonl_generation_path(&path, generation), text)?; Ok(()) }; @@ -1499,11 +1533,9 @@ mod tests { // nor rename the live file over a `.1` that was never shifted away. #[test] fn a_rotation_that_fails_still_appends_the_record() -> Result<()> { - let (root, paths) = temp_app_paths("rotation-fails"); - paths.ensure()?; - let path = paths.audit_events_path(); - fs::write(&path, audit_line("a") + "\n")?; - fs::write(jsonl_generation_path(&path, 1), audit_line("old") + "\n")?; + let (root, path) = temp_store("rotation-fails")?; + fs::write(&path, entry_line("a") + "\n")?; + fs::write(jsonl_generation_path(&path, 1), entry_line("old") + "\n")?; // `.1` cannot be renamed over a non-empty directory. let blocker = jsonl_generation_path(&path, 2); fs::create_dir_all(&blocker)?; @@ -1518,12 +1550,47 @@ mod tests { let recent = recent_tags(&path, 2, 10); fs::remove_dir_all(root).ok(); appended?; - assert_eq!(live?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); - assert_eq!(first?, audit_line("old") + "\n"); + assert_eq!(live?, format!("{}\n{}\n", entry_line("a"), entry_line("b"))); + assert_eq!(first?, entry_line("old") + "\n"); assert_eq!(recent?, ["old", "a", "b"]); Ok(()) } + // A live file at the cap that also ends in torn debris is rotated aside + // whole, before the tail is inspected, so the repair never rewrites a copy + // that is about to be moved away. The torn-tail tests above append with no + // rotation policy and the rotation tests never seed debris, so this is the + // one test that has repair and rotation meet on the same store. + #[test] + fn a_torn_live_file_at_the_cap_is_rotated_whole_and_the_new_record_starts_a_clean_file() + -> Result<()> { + let (root, path) = temp_store("rotation-of-a-torn-file")?; + let torn = &entry_line("torn")[..20]; + let seeded = format!("{}\n{torn}", entry_line("a")); + seed_store(&path, &seeded)?; + let rotation = JsonlRotation { + max_bytes: seeded.len() as u64, + generations: 2, + }; + let appended = append_rotating(&path, "b", rotation); + let live = fs::read_to_string(&path); + let rotated = fs::read_to_string(jsonl_generation_path(&path, 1)); + let recent = recent_tags(&path, 2, 10); + fs::remove_dir_all(root).ok(); + appended?; + assert_eq!( + rotated?, seeded, + "the rotated copy keeps its debris untouched: it was not repaired first" + ); + assert_eq!( + live?, + entry_line("b") + "\n", + "the live file starts clean, with only the new record" + ); + assert_eq!(recent?, ["a", "b"], "readers skip the debris and span both"); + Ok(()) + } + // `rotation_failure_warning` is unit-tested on its own; this pins that // `append_jsonl_record` actually prints it, once for a store that keeps // failing to rotate. Run in a child process to read its stderr, which the @@ -1542,11 +1609,9 @@ mod tests { return Ok(()); } - let (root, paths) = temp_app_paths("rotation-warning"); - paths.ensure()?; - let path = paths.audit_events_path(); - fs::write(&path, audit_line("a") + "\n")?; - fs::write(jsonl_generation_path(&path, 1), audit_line("old") + "\n")?; + let (root, path) = temp_store("rotation-warning")?; + fs::write(&path, entry_line("a") + "\n")?; + fs::write(jsonl_generation_path(&path, 1), entry_line("old") + "\n")?; // `.1` cannot be renamed over a non-empty directory. let blocker = jsonl_generation_path(&path, 2); fs::create_dir_all(&blocker)?; @@ -1589,13 +1654,11 @@ mod tests { // live file open without delete sharing). #[test] fn a_rotation_after_one_left_part_way_drops_no_generation() -> Result<()> { - let (root, paths) = temp_app_paths("rotation-part-way"); - paths.ensure()?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("rotation-part-way")?; let write = |generation: usize, tag: &str| -> Result<()> { fs::write( jsonl_generation_path(&path, generation), - audit_line(tag) + "\n", + entry_line(tag) + "\n", )?; Ok(()) }; @@ -1656,9 +1719,7 @@ mod tests { fn rotation_under_concurrent_threads_loses_no_records() -> Result<()> { const WRITERS: usize = 8; const APPENDS_PER_WRITER: usize = 25; - let (root, paths) = temp_app_paths("rotation-threads"); - paths.ensure()?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("rotation-threads")?; let barrier = std::sync::Barrier::new(WRITERS); std::thread::scope(|scope| { let handles: Vec<_> = (0..WRITERS) @@ -1708,9 +1769,7 @@ mod tests { return Ok(()); } - let (root, paths) = temp_app_paths("rotation-processes"); - paths.ensure()?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("rotation-processes")?; let exe = std::env::current_exe().context("current test executable")?; let children = (0..WRITERS) .map(|writer| { @@ -1757,9 +1816,7 @@ mod tests { max_bytes: 2048, generations: 3, }; - let (root, paths) = temp_app_paths("rotation-read-race"); - paths.ensure()?; - let path = paths.audit_events_path(); + let (root, path) = temp_store("rotation-read-race")?; let done = std::sync::atomic::AtomicBool::new(false); let outcome = std::thread::scope(|scope| { let reader = scope.spawn(|| -> Result { diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 707c71332c..36950d0c36 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -2,14 +2,14 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, bail}; -use std::fs; -use std::path::Path; +use anyhow::{Result, bail}; use std::time::{SystemTime, UNIX_EPOCH}; pub mod amd_smi; +pub mod anthropic_endpoint; #[cfg(all(test, unix))] mod archive_props; +pub mod atomic_write; pub mod automation; pub mod binary_paths; pub mod browser; @@ -27,6 +27,9 @@ pub mod examine; pub mod fake_executable; pub mod fix; mod gfx_target; +pub mod gguf; +pub mod gguf_fit; +pub mod gguf_source; pub mod hardware_root; pub mod host_gpu; mod jsonl_store; @@ -52,9 +55,12 @@ pub mod rocm_install; pub mod runtime; pub mod service_identity; pub mod service_record; +mod sigpipe; pub mod terminal; pub mod user_path; +#[cfg(any(test, feature = "test-support"))] +pub mod foreign_temp_probe; #[cfg(test)] mod test_env; #[cfg(test)] @@ -63,6 +69,10 @@ pub mod uv; pub mod vllm_oom; pub mod windows_driver; pub use amd_smi::{amd_smi_json, device_readable_bounded, resolve_amd_smi_binary}; +pub use anthropic_endpoint::{ + ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AnthropicEndpointMode, + anthropic_endpoint_ready, wait_for_published_anthropic_endpoint, +}; pub use automation::{ AuditEventRecord, AutomationEventRecord, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, EVENT_LOG_ROTATE_BYTES, EVENT_LOG_ROTATED_GENERATIONS, @@ -72,9 +82,13 @@ pub use automation::{ load_recent_automation_events, load_recent_automation_proposals, update_automation_proposal_status, }; -pub use binary_paths::{daemon_binary_path, engine_binary_path, sibling_binary_path}; +pub use binary_paths::{ + daemon_binary_path, engine_binary_path, sibling_binary_path, sibling_binary_path_from, +}; pub use codex_bridge::{CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot}; -pub use command_capture::{CommandCapture, command_capture_text, run_rocm_capture_with_env}; +pub use command_capture::{ + CommandCapture, CommandOutcome, command_capture_text, run_rocm_capture_with_env, +}; pub use config::{ AutomationsConfig, BuiltinWatcherSpec, DashboardConfig, DashboardDaemonConfig, DashboardTuiConfig, EngineUserConfig, ManagedToolConfig, PERMISSIONS_MODE_ASK, @@ -99,8 +113,8 @@ pub use disk_space::{ mount_for_path, on_same_mount, warn_if_low_space, with_margin, }; pub use download::{ - Backoff, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, download_file_streaming, - download_file_streaming_with_progress, download_file_to_path, + Backoff, DOWNLOAD_CHUNK_BYTES, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, + download_file_streaming, download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, }; pub use endpoint_key::{endpoint_api_key_has_forbidden_chars, generate_endpoint_api_key}; @@ -138,7 +152,11 @@ use host_gpu::{ detect_wsl_summary, is_wsl1_kernel, ldconfig_cache, ldconfig_lists_librocdxg, }; pub use jsonl_store::jsonl_lock_path; -pub use log_tail::tail_lines; +pub use log_tail::{ + LOG_VIEW_MAX_TAIL_READ, LogDecode, LogTail, newest_line_exceeds_bound_notice, + service_log_tool_summary, strip_ansi_sequences, tail_lines, tail_lines_decoded, + tail_log_decoded, +}; pub use managed_runtime::{ FrameworkInterpreter, ManagedRuntimeEnvironment, active_managed_framework_interpreter, active_managed_therock_channel, active_managed_therock_environment, @@ -161,19 +179,23 @@ pub use model_catalog::{ pub use net::{ DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, HttpResponseParts, connect_tcp_stream, format_host_for_url, format_host_port, format_http_base_url, http_get_text, - http_get_text_with_auth, http_post_json_with_auth, parse_http_endpoint, + http_get_text_with_auth, http_get_with_auth, http_post_json_with_auth, parse_http_endpoint, read_http_response_bounded, write_all_tcp_stream, }; -pub use paths::{AppPaths, engine_plugin_dirs}; +pub use paths::{ + AppPaths, CACHE_DIR_ENV, CONFIG_DIR_ENV, DATA_DIR_ENV, ENGINE_ENVS_ROOT_ENV, engine_plugin_dirs, +}; pub use proc_lifecycle::{ - IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, - process_start_ticks, terminate_verified, + IdentityState, KillScope, MANAGED_STOP_GRACE, ProcessIdentity, RecordedPidOutcome, + RecordedPidStop, TerminationOutcome, identity_state, process_start_ticks, + terminate_recorded_pid, terminate_recorded_pid_within, terminate_recorded_service_pids, + terminate_verified, }; pub use process::{ - FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, + FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, ProcessProbe, detach_command_session, is_canonical_loopback_host, launch_port_argument, lease_loopback_port, - lease_loopback_port_or_bind_error, process_is_running, terminate_process, - terminate_process_tree, + lease_loopback_port_or_bind_error, probe_process, process_is_running, terminate_process, + terminate_process_tree, windows_system_tool, }; #[cfg(windows)] pub use process::{ @@ -197,19 +219,20 @@ use runtime::env_path_override; pub use runtime::{ RUNTIME_LIBRARY_PATH_ENV, RuntimeHost, RuntimePlatform, current_executable_path, default_cache_dir, default_config_dir, default_data_dir, default_interactive_shell_program, - managed_logs_dir, managed_pip_cache_dir, managed_runtime_cache_dir, managed_runtime_data_root, - managed_tools_dir, managed_uv_cache_dir, normalize_runtime_path_for_host, - normalize_runtime_path_for_storage, normalize_runtime_path_text_for_host, - normalize_runtime_path_text_for_platform, normalize_runtime_path_text_for_storage, - platform_binary_name, prepend_runtime_path, resolve_path_through_symlinks, - runtime_directory_label, runtime_drive_root_for_key, runtime_drive_roots, runtime_exe_suffix, - runtime_home_dir, runtime_install_root_is_protected, runtime_is_linux, runtime_is_windows, - runtime_os_name, runtime_path_for_child, runtime_path_for_windows_child, - runtime_path_is_same_or_inside, runtime_path_list_join, runtime_path_list_split, - runtime_path_sort_key, runtime_path_text_is_absolute_for_host, + install_root_has_payload, managed_logs_dir, managed_pip_cache_dir, managed_runtime_cache_dir, + managed_runtime_data_root, managed_tools_dir, managed_uv_cache_dir, + normalize_runtime_path_for_host, normalize_runtime_path_for_storage, + normalize_runtime_path_text_for_host, normalize_runtime_path_text_for_platform, + normalize_runtime_path_text_for_storage, platform_binary_name, prepend_runtime_path, + resolve_path_through_symlinks, runtime_directory_label, runtime_drive_root_for_key, + runtime_drive_roots, runtime_exe_suffix, runtime_home_dir, runtime_install_root_is_protected, + runtime_is_linux, runtime_is_windows, runtime_os_name, runtime_path_for_child, + runtime_path_for_windows_child, runtime_path_is_same_or_inside, runtime_path_list_join, + runtime_path_list_split, runtime_path_sort_key, runtime_path_text_is_absolute_for_host, runtime_path_text_is_absolute_for_platform, runtime_paths_equivalent, runtime_python_activation_hint, runtime_python_activation_script, runtime_python_bin_dir_name, - runtime_python_env_bin_dir, runtime_python_executable_in_env, runtime_python_executable_name, + runtime_python_env_bin_dir, runtime_python_env_root_for_executable, + runtime_python_executable_in_env, runtime_python_executable_name, runtime_rocm_library_filename, shell_command_for_host, user_runtime_dir, }; pub use service_identity::{ServiceId, generate_service_id}; @@ -218,12 +241,14 @@ pub use service_record::{ INFERENCE_VERIFIED_STATE_KEY, ManagedServiceNotFound, ManagedServiceRecord, engine_stop_decides_confirmation, }; +pub use sigpipe::with_sigpipe_ignored; pub use terminal::{quotable_in_single_quotes, strip_terminal_control_sequences}; pub use uv::{ DEFAULT_UV_TIMEOUT_SECS, DependencyViolation, UV_CACHE_DIR_ENV, UV_CACHE_DIR_OVERRIDE_ENV, - UvCacheSource, ViolationSubject, check_dependencies, ensure_uv_binary, split_local_version, - uv_binary_name, uv_cache_source, uv_command_env, uv_http_timeout_secs, uv_pip_check_args, - uv_pip_freeze_args, uv_pip_install_base, uv_venv_args, violation_subject, violations_requiring, + UvCacheSource, ViolationSubject, check_dependencies, ensure_uv_binary, find_uv_binary, + split_local_version, uv_binary_name, uv_cache_source, uv_command_env, uv_http_timeout_secs, + uv_pip_check_args, uv_pip_freeze_args, uv_pip_install_base, uv_venv_args, violation_subject, + violations_requiring, }; pub use vllm_oom::{ VLLM_OOM_DETECTED_PREFIX, VLLM_OOM_DIAGNOSE_COMMAND, vllm_oom_diagnose_symptom, @@ -252,6 +277,17 @@ pub fn torch_alignment_disabled() -> bool { std::env::var_os(TORCH_ALIGNMENT_DISABLED_ENV).is_some() } +/// The packages the SDK installs together with torch, in one transaction. +/// +/// Under [`torch_alignment_disabled`] these are all treated as the user's: the SDK +/// overwrites the lot, and vLLM pins an exact build of each, so sparing only torch +/// leaves the others as defects whose repair reinstalls the engine's torch over the +/// one the opt-out keeps. The CLI's dependency check and the engine's repair +/// assessment both read this list, so they agree on which packages are spared. They +/// still gate it differently: the engine spares them whenever the opt-out is set, the +/// CLI only when the install settled on a torch divergence. +pub const SDK_TORCH_STACK_PACKAGES: &[&str] = &["torch", "torchvision", "torchaudio", "triton"]; + pub fn require_nonempty(value: &str, field_name: &str) -> Result<()> { if value.trim().is_empty() { bail!("{field_name} must not be empty"); @@ -341,70 +377,10 @@ pub fn unix_time_millis() -> u128 { .as_millis() } -/// Replace `path`'s content with `bytes`, atomically. -/// -/// Writes a sibling temp file and renames it over the destination: an -/// interrupted write (full disk, crash, killed process) can never leave a -/// truncated file in place, and a reader sees -/// either the old content or the new content. The destination is NOT removed -/// first: `fs::rename` replaces an existing file on both supported platforms -/// (it is `MoveFileEx` with `MOVEFILE_REPLACE_EXISTING` on Windows), and -/// removing it would open a window with no file at all, which a concurrent -/// reader could see as "file doesn't exist" instead of the old or new -/// content. The temp file is removed if the write or the rename fails. -/// -/// `tmp_ext` names the temp file's extension (e.g. `"json"`, `"jsonl"`). -/// Both halves of the generated suffix matter: the timestamp keeps -/// successive writes to the same path apart, and the pid keeps two -/// processes writing in the same millisecond from clobbering each other's -/// temp file and losing one of the two updates. -/// -/// Not guaranteed: -/// - Serializing writers: this takes no lock, so two read-modify-write -/// cycles that overlap still lose one update (last rename wins). A caller -/// whose new content depends on the old must hold a lock across the whole -/// cycle, as the JSONL proposal store does (`jsonl_store::jsonl_lock_path`). -/// Two callers in the *same process* within the same millisecond are a -/// sharper case of this, not just a milder one: they generate the exact -/// same temp path below, so concurrent `fs::write`s truncating that one -/// path can publish mixed bytes, and the losing rename then fails outright -/// (its source file already gone) rather than merely losing an update — -/// this is the same-process collision -/// `automation::tests::concurrent_appends_from_many_threads_lose_no_records`'s -/// own comment describes ("two threads of one process even shared a -/// temp-file name") — fixed for its own callers by the per-store lock -/// `automation.rs` takes before ever calling into this function. -/// - Durability: the bytes are not fsynced before the rename, so a power -/// loss can still surface the new name with unflushed contents. -/// - Replacing a destination another process holds open without delete -/// sharing: on Windows the rename then fails with a sharing violation, -/// this returns that error, and the old destination stays in place. -pub fn write_atomic(path: &Path, bytes: &[u8], tmp_ext: &str) -> Result<()> { - let tmp_path = path.with_extension(format!( - "{tmp_ext}.tmp-{}-{}", - unix_time_millis(), - std::process::id() - )); - fs::write(&tmp_path, bytes) - .with_context(|| format!("failed to write {}", tmp_path.display())) - .inspect_err(|_| { - let _ = fs::remove_file(&tmp_path); - })?; - fs::rename(&tmp_path, path) - .with_context(|| format!("failed to replace {}", path.display())) - .inspect_err(|_| { - let _ = fs::remove_file(&tmp_path); - })?; - Ok(()) -} - #[cfg(test)] mod tests { use super::*; - use std::ffi::OsString; - use std::fs; - fn test_exit_status() -> std::process::ExitStatus { std::process::ExitStatus::default() } @@ -434,64 +410,4 @@ mod tests { let output = test_process_output(" \n", " "); assert_eq!(command_failure_detail(&output), "no output"); } - - #[test] - fn write_atomic_writes_bytes_and_leaves_no_temp_file() -> Result<()> { - let temp_root = - std::env::temp_dir().join(format!("rocm-cli-write-atomic-ok-{}", unix_time_millis())); - fs::create_dir_all(&temp_root)?; - let path = temp_root.join("store.jsonl"); - - write_atomic(&path, b"hello\n", "jsonl")?; - - let written = fs::read(&path)?; - let leftovers: Vec<_> = fs::read_dir(&temp_root)? - .filter_map(Result::ok) - .map(|entry| entry.file_name()) - .collect(); - let _ = fs::remove_dir_all(&temp_root); - - assert_eq!(written, b"hello\n"); - assert_eq!( - leftovers, - vec![OsString::from("store.jsonl")], - "a successful write must leave no `*.tmp-*` sibling behind" - ); - Ok(()) - } - - #[test] - fn write_atomic_rename_failure_leaves_destination_and_dir_untouched() -> Result<()> { - let temp_root = - std::env::temp_dir().join(format!("rocm-cli-write-atomic-fail-{}", unix_time_millis())); - fs::create_dir_all(&temp_root)?; - let path = temp_root.join("store.jsonl"); - // A file can't be renamed over an existing directory, so making the - // destination a directory forces `write_atomic`'s rename step to fail - // after the temp file has already been written. - fs::create_dir_all(&path)?; - - let result = write_atomic(&path, b"hello\n", "jsonl"); - let destination_is_still_a_dir = path.is_dir(); - let leftovers: Vec<_> = fs::read_dir(&temp_root)? - .filter_map(Result::ok) - .map(|entry| entry.file_name()) - .collect(); - let _ = fs::remove_dir_all(&temp_root); - - assert!( - result.is_err(), - "renaming a file onto an existing directory must fail" - ); - assert!( - destination_is_still_a_dir, - "a failed rename must leave the destination untouched" - ); - assert_eq!( - leftovers, - vec![OsString::from("store.jsonl")], - "the temp file must be removed when the rename fails" - ); - Ok(()) - } } diff --git a/crates/rocm-core/src/log_tail.rs b/crates/rocm-core/src/log_tail.rs index 9ab1fb93f9..a99ce0daa8 100644 --- a/crates/rocm-core/src/log_tail.rs +++ b/crates/rocm-core/src/log_tail.rs @@ -2,83 +2,453 @@ // // SPDX-License-Identifier: MIT +//! The one routine that reads the last lines of a log file. +//! +//! The engine adapters, the ComfyUI log view, the services text module and the +//! `rocmd` MCP `service_logs` tool all go through [`tail_lines_decoded`] +//! (`tail_lines` is the plain-text form of it), so the seek-boundary handling, +//! the lossy decoding and the "never read the whole file" bound each have one +//! implementation. A caller picks how the bytes become text with a +//! [`LogDecode`]; it does not pick how the file is read. + use anyhow::{Context, Result}; use std::collections::VecDeque; use std::fs; use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; use std::path::Path; +/// How many bytes from the end of a log the log views (ComfyUI logs, `rocm +/// services logs`, the MCP `service_logs` tool) read at most. The engine +/// adapters keep their own budgets. +pub const LOG_VIEW_MAX_TAIL_READ: u64 = 4 * 1024 * 1024; + +/// How the bytes of a log become lines of text. +/// +/// Whatever the choice, text that is not valid in the chosen encoding is +/// replaced with U+FFFD rather than failing the read: a log view must show +/// what it can. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LogDecode { + /// Decode the log as UTF-16 little-endian when it carries a byte order + /// mark, or (without one) when at least half of the odd-numbered bytes in + /// its first 512 bytes are zero (mostly-ASCII UTF-16 text, which has a zero + /// high byte for each character), as a Windows shell redirect writes it. + /// Otherwise the log is UTF-8. + pub utf16_le: bool, + /// Remove ANSI CSI escape sequences (colour and cursor codes) from each + /// returned line. + pub strip_ansi: bool, +} + +impl LogDecode { + /// UTF-8 only, escape sequences kept: what the engine adapters and the + /// service logs return. + pub const PLAIN: Self = Self { + utf16_le: false, + strip_ansi: false, + }; + /// Detect UTF-16 little-endian and strip ANSI sequences: for logs a + /// Windows shell may have redirected, shown to a person in a terminal. + pub const CONSOLE: Self = Self { + utf16_le: true, + strip_ansi: true, + }; +} + +/// The lines a bounded tail read returned, and what the bound left unread. +/// +/// A caller that only wants the lines uses [`tail_lines_decoded`]; one that +/// tells a person what it found uses [`tail_log_decoded`], so that "the log +/// has no lines" and "the log's newest line does not fit in the read bound" +/// are not the same empty list. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LogTail { + /// The last lines of the log, oldest first. + pub lines: Vec, + /// How many bytes at the start of the file the read did not look at + /// because of the byte budget; zero when the whole file was read. + pub skipped_bytes: u64, +} + +impl LogTail { + /// True when the read found no complete line even though the log has + /// content: its newest line begins before the byte budget's window, so the + /// window held only a fragment of it, which is dropped. An empty log, or a + /// limit of zero, is not this. + pub const fn newest_line_exceeds_the_read_bound(&self) -> bool { + self.lines.is_empty() && self.skipped_bytes > 0 + } +} + +/// The clause a log view prints when none of a log's content is shown. +/// +/// That is the case when the newest line does not fit the read budget +/// ([`LogTail::newest_line_exceeds_the_read_bound`]). Derived from +/// [`LOG_VIEW_MAX_TAIL_READ`], the budget the views read with, so the number +/// cannot drift from it. +pub fn newest_line_exceeds_bound_notice() -> String { + format!( + "the newest line is longer than the {} MiB a log view reads", + LOG_VIEW_MAX_TAIL_READ / (1024 * 1024) + ) +} + +/// What the `service_logs` tool says about a service's log tail. +/// +/// An empty log and a log whose newest line does not fit the read bound are +/// not both reported as "Read the last 0 line(s)". Shared by the daemon's MCP +/// server and the CLI's own tool bridge, which answer the same tool. +/// +/// The count is the number of lines returned, not the number asked for. +pub fn service_log_tool_summary(service_id: &str, tail: &LogTail) -> String { + if tail.newest_line_exceeds_the_read_bound() { + format!( + "Service `{service_id}` has log output, but none is shown: {}.", + newest_line_exceeds_bound_notice() + ) + } else if tail.lines.is_empty() { + format!("Service `{service_id}` has no log output yet.") + } else { + format!( + "Read the last {} line(s) from service `{service_id}`.", + tail.lines.len() + ) + } +} + +/// Reads the last `limit` lines from a file as UTF-8, seeking near the end +/// instead of reading the whole file for files larger than `max_tail_read`. +/// +/// Shorthand for [`tail_lines_decoded`] with [`LogDecode::PLAIN`]. +pub fn tail_lines(path: &Path, limit: usize, max_tail_read: u64) -> Result> { + tail_lines_decoded(path, limit, max_tail_read, LogDecode::PLAIN) +} + /// Reads the last `limit` lines from a file, seeking near the end instead of /// reading the whole file for files larger than `max_tail_read`. /// -/// Shared by the engine adapters so the seek-boundary classification has one -/// implementation: when the seek lands in the middle of a line, the first -/// line read back is a partial fragment and must be dropped; when it lands -/// exactly on a line boundary (the byte before the seek position is a -/// newline), that first line is complete and must be kept. `max_tail_read` -/// stays a parameter rather than a shared constant because each engine picks -/// its own tail-read budget. -pub fn tail_lines(path: &Path, limit: usize, max_tail_read: u64) -> Result> { +/// Shared by every reader of a log tail so the seek-boundary classification +/// has one implementation: when the seek lands in the middle of a line, the +/// first line read back is a partial fragment and must be dropped; when it +/// lands exactly on a line boundary (the character before the seek position +/// is a newline), that first line is complete and must be kept. +/// `max_tail_read` stays a parameter rather than a shared constant because +/// each engine picks its own tail-read budget (the log views share +/// [`LOG_VIEW_MAX_TAIL_READ`]). +/// +/// A limit of zero returns no lines without opening the file. Line endings +/// are `\n` or `\r\n`. See [`LogDecode`] for the text decoding. +pub fn tail_lines_decoded( + path: &Path, + limit: usize, + max_tail_read: u64, + decode: LogDecode, +) -> Result> { + Ok(tail_log_decoded(path, limit, max_tail_read, decode)?.lines) +} + +/// [`tail_lines_decoded`], also reporting how much of the start of the file +/// the byte budget left unread (see [`LogTail`]). The lines are the same. +pub fn tail_log_decoded( + path: &Path, + limit: usize, + max_tail_read: u64, + decode: LogDecode, +) -> Result { if limit == 0 { - return Ok(Vec::new()); + return Ok(LogTail::default()); } - let mut file = + let file = fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; let metadata = file .metadata() .with_context(|| format!("failed to stat {}", path.display()))?; - let file_size = metadata.len(); + tail_lines_from(file, metadata.len(), path, limit, max_tail_read, decode) +} + +/// How many times a read measures the source again after finding it shorter +/// than the size it was given, before it gives up on a log that will not hold +/// still. +const SHRINK_RETRIES: usize = 3; + +/// What one pass of the tail read found. +enum Pass { + /// The lines of the log. + Read(LogTail), + /// The source ended before the `size` it was given: the log was truncated + /// or recreated after the size was taken, as a restart does. + Shrunk, +} + +/// The tail read over any seekable source of `size` bytes; `path` only names +/// the source in error messages. Split out so a test can count the bytes the +/// read touches, or hand it a source that shrinks. +/// +/// A log can be recreated between the size being taken and the read (a service +/// restart creates its log afresh), so a read that runs into the end of the +/// source early is not an error: the source is measured again and the read +/// starts over from that size, which also covers a log that is now small enough +/// to be read from its start. A log that keeps shrinking through +/// [`SHRINK_RETRIES`] passes has nothing stable to show and reads as empty. +fn tail_lines_from( + mut source: R, + mut size: u64, + path: &Path, + limit: usize, + max_tail_read: u64, + decode: LogDecode, +) -> Result { + for _ in 0..SHRINK_RETRIES { + match tail_pass(&mut source, size, path, limit, max_tail_read, decode)? { + Pass::Read(tail) => return Ok(tail), + Pass::Shrunk => { + size = source + .seek(SeekFrom::End(0)) + .with_context(|| format!("failed to measure {}", path.display()))?; + } + } + } + Ok(LogTail::default()) +} + +/// One pass of [`tail_lines_from`] against a source believed to hold `size` +/// bytes. +fn tail_pass( + source: &mut R, + size: u64, + path: &Path, + limit: usize, + max_tail_read: u64, + decode: LogDecode, +) -> Result { + let utf16_bom = if decode.utf16_le { + sniff_utf16_le(source, size, path)? + } else { + None + }; + // The width of one character unit, and so of the newline probe below: a + // UTF-16 window must start on a unit boundary. + let unit: u64 = if utf16_bom.is_some() { 2 } else { 1 }; + let bom_len: u64 = if utf16_bom == Some(true) { 2 } else { 0 }; let mut first_line_is_partial = false; - if file_size > max_tail_read { - let seek_pos = file_size - max_tail_read; - // Probe the byte preceding `seek_pos` to classify the first line, then - // leave the cursor at `seek_pos` for the buffered read below. - file.seek(SeekFrom::Start(seek_pos - 1)) + let window_start = if size > max_tail_read { + let mut seek_pos = size - max_tail_read; + if unit == 2 && seek_pos % 2 == 1 { + seek_pos += 1; + } + if seek_pos >= size { + return Ok(Pass::Read(LogTail { + lines: Vec::new(), + skipped_bytes: size, + })); + } + // Probe the unit preceding `seek_pos` to classify the first line, then + // leave the cursor at `seek_pos` for the read below. + source + .seek(SeekFrom::Start(seek_pos - unit)) .with_context(|| format!("failed to seek in {}", path.display()))?; - let mut probe = [0u8; 1]; - file.read_exact(&mut probe) - .with_context(|| format!("failed to read from {}", path.display()))?; - first_line_is_partial = probe[0] != b'\n'; - } + let mut probe = [0u8; 2]; + let probe = &mut probe[..unit as usize]; + match source.read_exact(probe) { + Ok(()) => {} + // `seek_pos` is inside the size the source was measured at, so + // running out here means the source got shorter since. + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { + return Ok(Pass::Shrunk); + } + Err(error) => { + return Err(error) + .with_context(|| format!("failed to read from {}", path.display())); + } + } + // The unit before the first line of a file with a byte order mark is + // the mark itself, not the end of a previous line. + first_line_is_partial = seek_pos != bom_len && probe != [b'\n'] && probe != [b'\n', 0]; + seek_pos + } else { + // The whole file is the window; skip a byte order mark the way a + // reader of the file from its start would. + source + .seek(SeekFrom::Start(bom_len)) + .with_context(|| format!("failed to seek in {}", path.display()))?; + bom_len + }; - let mut reader = BufReader::new(file); // Capped, not `VecDeque::with_capacity(limit)`: callers may pass // `usize::MAX` to mean "no limit", which would try to allocate that // capacity upfront. Real callers ask for small limits (tens to a few // hundred lines), so capping still gives them a single allocation. let mut lines: VecDeque = VecDeque::with_capacity(limit.min(4096)); - // Read raw bytes rather than `BufRead::lines()`: when the seek above lands - // inside a multi-byte UTF-8 character, the leading fragment is invalid - // UTF-8 on its own. `lines()` would surface that as an `Err` and abort the - // whole read before `first_line_is_partial` gets a chance to drop exactly - // that fragment. Decoding lossily keeps the drop-the-fragment logic in - // charge of it instead. - let mut raw_line = Vec::new(); - loop { - raw_line.clear(); - let bytes_read = reader - .read_until(b'\n', &mut raw_line) + let mut keep = |text: String, terminated: bool| { + // The unterminated tail of a console log can be only the escape + // sequence a program writes after its last newline (a colour reset). + // It is not a line: counting it would push the last real line out + // when the limit is small. A terminated line that strips to nothing + // is a blank line the program wrote, and stays. + if decode.strip_ansi && !terminated && strip_ansi_sequences(&text).is_empty() { + return; + } + if lines.len() == limit { + lines.pop_front(); + } + lines.push_back(text); + }; + + if unit == 2 { + // The window is at most `max_tail_read` bytes (or the whole, smaller, + // file), so it is read in one go and split on the newline unit. + let mut window = Vec::new(); + source + .take(size - window_start) + .read_to_end(&mut window) .with_context(|| format!("failed to read lines from {}", path.display()))?; - if bytes_read == 0 { - break; + if (window.len() as u64) < size - window_start { + return Ok(Pass::Shrunk); + } + let units: Vec = window + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(); + let mut rest = units.as_slice(); + while !rest.is_empty() { + let (mut line, terminated) = + if let Some(end) = rest.iter().position(|&unit| unit == 0x000A) { + let line = &rest[..end]; + rest = &rest[end + 1..]; + (line, true) + } else { + let line = rest; + rest = &[]; + (line, false) + }; + if terminated && line.last() == Some(&0x000D) { + line = &line[..line.len() - 1]; + } + if first_line_is_partial { + first_line_is_partial = false; + continue; + } + keep(String::from_utf16_lossy(line), terminated); } - if raw_line.last() == Some(&b'\n') { - raw_line.pop(); - if raw_line.last() == Some(&b'\r') { + } else { + // Capped at the snapshot like the sixteen-bit window above: a log + // whose writer is still appending must not pull the read past the + // budget by however much it grew since `size` was taken. + let mut reader = BufReader::new(source.take(size - window_start)); + let mut window_len = 0u64; + // Read raw bytes rather than `BufRead::lines()`: when the seek above + // lands inside a multi-byte UTF-8 character, the leading fragment is + // invalid UTF-8 on its own. `lines()` would surface that as an `Err` + // and abort the whole read before `first_line_is_partial` gets a + // chance to drop exactly that fragment. Decoding lossily keeps the + // drop-the-fragment logic in charge of it instead. + let mut raw_line = Vec::new(); + loop { + raw_line.clear(); + let bytes_read = reader + .read_until(b'\n', &mut raw_line) + .with_context(|| format!("failed to read lines from {}", path.display()))?; + if bytes_read == 0 { + break; + } + window_len += bytes_read as u64; + let terminated = raw_line.last() == Some(&b'\n'); + if terminated { raw_line.pop(); + if raw_line.last() == Some(&b'\r') { + raw_line.pop(); + } + } + if first_line_is_partial { + first_line_is_partial = false; + continue; } + keep(String::from_utf8_lossy(&raw_line).into_owned(), terminated); + } + if window_len < size - window_start { + return Ok(Pass::Shrunk); } - if first_line_is_partial { - first_line_is_partial = false; + } + + let lines = lines + .into_iter() + .map(|line| { + if decode.strip_ansi { + strip_ansi_sequences(&line) + } else { + line + } + }) + .collect(); + Ok(Pass::Read(LogTail { + lines, + // Not `window_start`: a byte order mark skipped at the start of a + // whole-file read is not something the budget left unread. + skipped_bytes: if size > max_tail_read { + window_start + } else { + 0 + }, + })) +} + +/// `Some(has_byte_order_mark)` when the start of the source reads as UTF-16 +/// little-endian, `None` when it reads as UTF-8. Looks only at the first 512 +/// bytes, so it costs the same however large the log is. +fn sniff_utf16_le(source: &mut R, size: u64, path: &Path) -> Result> { + const SAMPLE: u64 = 512; + let mut head = Vec::new(); + source + .seek(SeekFrom::Start(0)) + .with_context(|| format!("failed to seek in {}", path.display()))?; + source + .take(size.min(SAMPLE)) + .read_to_end(&mut head) + .with_context(|| format!("failed to read from {}", path.display()))?; + if head.starts_with(&[0xff, 0xfe]) { + return Ok(Some(true)); + } + // Mostly-ASCII UTF-16 has a zero in every other byte. At least a quarter + // of the sampled bytes being zeros at odd positions is at least half of the + // odd positions: the rule the ComfyUI reader this replaced used. + if size >= 8 { + let zero_odd_bytes = head + .iter() + .enumerate() + .filter(|(index, byte)| index % 2 == 1 && **byte == 0) + .count(); + if zero_odd_bytes * 4 >= head.len() { + return Ok(Some(false)); + } + } + Ok(None) +} + +/// Removes ANSI CSI sequences from one line of text. +/// +/// A sequence is `ESC [` up to and including a final byte in `@`..=`~`. A bare +/// `ESC` not followed by `[` is dropped alone. The log tail applies this per +/// line, so an unterminated sequence ends with its line instead of swallowing +/// the lines after it. +pub fn strip_ansi_sequences(text: &str) -> String { + let mut cleaned = String::with_capacity(text.len()); + let mut chars = text.chars().peekable(); + while let Some(ch) = chars.next() { + if ch != '\x1b' { + cleaned.push(ch); continue; } - if lines.len() == limit { - lines.pop_front(); + if chars.peek() == Some(&'[') { + let _ = chars.next(); + for next in chars.by_ref() { + if ('@'..='~').contains(&next) { + break; + } + } } - lines.push_back(String::from_utf8_lossy(&raw_line).into_owned()); } - Ok(lines.into_iter().collect()) + cleaned } #[cfg(test)] @@ -219,6 +589,68 @@ mod tests { Ok(()) } + #[test] + fn tail_lines_on_a_large_file_returns_only_whole_lines_from_the_window() -> Result<()> { + // 1 MiB of 1 KiB lines under a 64 KiB budget, with a marked first and + // last line. The first line lies far outside the window, so a read + // that returned it was not bounded; the window starts mid-line (the + // budget is not a multiple of the line length), so a leading fragment + // that was kept would be shorter than a whole line. + let mut content = String::from("HEAD-MARKER\n"); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..1024 { + content.push_str(&filler); + } + content.push_str("TAIL-MARKER\n"); + let path = + std::env::temp_dir().join(format!("rocm-core-tail-large-{}.log", unique_suffix())); + fs::write(&path, &content)?; + + let lines = tail_lines(&path, usize::MAX, 64 * 1024 + 7)?; + fs::remove_file(&path).ok(); + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + assert!(lines.len() <= 65, "got {} lines", lines.len()); + for line in &lines[..lines.len() - 1] { + assert_eq!(line.len(), 1023, "a partial leading fragment was kept"); + } + Ok(()) + } + + #[test] + fn tail_lines_missing_file_is_an_open_error_naming_the_path() { + let path = Path::new("/does/not/exist/rocm-core-tail-missing.log"); + + let error = tail_lines(path, 5, TEST_MAX_TAIL_READ).expect_err("missing file"); + + assert_eq!( + error.to_string(), + format!("failed to open {}", path.display()) + ); + } + + #[test] + fn tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let path = std::env::temp_dir().join(format!("rocm-core-tail-few-{}.log", unique_suffix())); + fs::write(&path, "a\nb\n")?; + let lines = tail_lines(&path, 50, TEST_MAX_TAIL_READ)?; + fs::remove_file(&path).ok(); + assert_eq!(lines, vec!["a".to_owned(), "b".to_owned()]); + Ok(()) + } + + #[test] + fn tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let path = + std::env::temp_dir().join(format!("rocm-core-tail-no-nl-{}.log", unique_suffix())); + fs::write(&path, "a\nb\nc")?; + let lines = tail_lines(&path, 2, TEST_MAX_TAIL_READ)?; + fs::remove_file(&path).ok(); + assert_eq!(lines, vec!["b".to_owned(), "c".to_owned()]); + Ok(()) + } + #[test] fn tail_lines_limit_zero_returns_empty_without_opening_file() -> Result<()> { let path = Path::new("/does/not/exist/rocm-core-tail-limit-zero.log"); @@ -226,4 +658,819 @@ mod tests { assert!(lines.is_empty()); Ok(()) } + + // --- bounded reads ------------------------------------------------- + + /// Counts the bytes its inner reader hands out, so a test can show how + /// much of a source a tail read touched. + struct CountingReader { + inner: R, + bytes_read: std::rc::Rc>, + } + + impl Read for CountingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buf)?; + self.bytes_read.set(self.bytes_read.get() + read as u64); + Ok(read) + } + } + + impl Seek for CountingReader { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.inner.seek(position) + } + } + + /// Runs the tail read over `content` and returns the lines with the number + /// of bytes the read pulled from the source. + fn read_counting( + content: Vec, + limit: usize, + max_tail_read: u64, + decode: LogDecode, + ) -> Result<(Vec, u64)> { + let size = content.len() as u64; + let bytes_read = std::rc::Rc::new(std::cell::Cell::new(0)); + let source = CountingReader { + inner: std::io::Cursor::new(content), + bytes_read: bytes_read.clone(), + }; + let lines = tail_lines_from( + source, + size, + Path::new("counting-source"), + limit, + max_tail_read, + decode, + )? + .lines; + Ok((lines, bytes_read.get())) + } + + fn utf16_le_bytes(text: &str, bom: bool) -> Vec { + let mut bytes = if bom { vec![0xff, 0xfe] } else { Vec::new() }; + for unit in text.encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + bytes + } + + // --- a log that shrinks between the size and the read ---------------- + + /// How a [`ShrinkingSource`] loses bytes. + #[derive(Clone, Copy)] + enum Shrink { + /// Cut the content to this many bytes once. + Once(usize), + /// Lose one byte on every read, so the source never settles. + EveryRead, + } + + /// A source that gets shorter when a read starts at or after `trigger_pos`, + /// the way a log recreated by a restart does between the `stat` and the + /// read. The size handed to the routine is the one taken before the shrink. + struct ShrinkingSource { + inner: std::io::Cursor>, + trigger_pos: u64, + shrink: Shrink, + done: bool, + } + + impl ShrinkingSource { + fn new(content: Vec, trigger_pos: u64, shrink: Shrink) -> Self { + Self { + inner: std::io::Cursor::new(content), + trigger_pos, + shrink, + done: false, + } + } + } + + impl Read for ShrinkingSource { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.inner.position() >= self.trigger_pos { + match self.shrink { + Shrink::Once(len) if !self.done => { + self.inner.get_mut().truncate(len); + self.done = true; + } + Shrink::Once(_) => {} + Shrink::EveryRead => { + let content = self.inner.get_mut(); + content.truncate(content.len().saturating_sub(1)); + } + } + } + self.inner.read(buf) + } + } + + impl Seek for ShrinkingSource { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.inner.seek(position) + } + } + + /// Twenty eight-byte lines, `line-00` to `line-19`. + fn twenty_lines() -> String { + use std::fmt::Write as _; + (0..20).fold(String::new(), |mut text, index| { + let _ = writeln!(text, "line-{index:02}"); + text + }) + } + + fn tail_of_shrinking( + content: Vec, + trigger_pos: u64, + shrink: Shrink, + max_tail_read: u64, + decode: LogDecode, + ) -> Result { + let size = content.len() as u64; + tail_lines_from( + ShrinkingSource::new(content, trigger_pos, shrink), + size, + Path::new("shrinking-source"), + 10, + max_tail_read, + decode, + ) + } + + #[test] + fn a_log_that_shrank_before_the_probe_is_measured_again_instead_of_failing() -> Result<()> { + // 160 bytes under a 40 byte budget: the probe reads byte 119. By then + // the log is three lines (24 bytes), so that byte is gone. + let content = twenty_lines().into_bytes(); + let tail = tail_of_shrinking(content, 119, Shrink::Once(24), 40, LogDecode::PLAIN)?; + + assert_eq!(tail.lines, ["line-00", "line-01", "line-02"]); + assert_eq!( + tail.skipped_bytes, 0, + "the whole shrunk log fits the budget" + ); + Ok(()) + } + + #[test] + fn a_log_that_shrank_before_the_window_read_is_measured_again() -> Result<()> { + // The probe succeeds; the log is cut to three lines before the window + // starts reading at byte 120, which would otherwise show as no lines. + let content = twenty_lines().into_bytes(); + let tail = tail_of_shrinking(content, 120, Shrink::Once(24), 40, LogDecode::PLAIN)?; + + assert_eq!(tail.lines, ["line-00", "line-01", "line-02"]); + assert_eq!(tail.skipped_bytes, 0); + Ok(()) + } + + #[test] + fn a_log_that_shrank_but_is_still_larger_than_the_budget_is_read_from_its_new_end() -> Result<()> + { + // Cut from 160 to 80 bytes (ten lines) under a 40 byte budget: the + // tail is the last five lines of what is left, not of what was. + let content = twenty_lines().into_bytes(); + let tail = tail_of_shrinking(content, 119, Shrink::Once(80), 40, LogDecode::PLAIN)?; + + assert_eq!( + tail.lines, + ["line-05", "line-06", "line-07", "line-08", "line-09"] + ); + assert_eq!(tail.skipped_bytes, 40); + Ok(()) + } + + #[test] + fn a_sixteen_bit_log_that_shrank_is_measured_again_at_the_probe_and_the_window() -> Result<()> { + let content = utf16_le_bytes(&twenty_lines(), true); + // 322 bytes under a 64 byte budget: the probe unit is at byte 256 and + // the window starts at 258. Cut to the mark plus three lines (50). + for trigger_pos in [256, 258] { + let tail = tail_of_shrinking( + content.clone(), + trigger_pos, + Shrink::Once(50), + 64, + LogDecode::CONSOLE, + )?; + + assert_eq!( + tail.lines, + ["line-00", "line-01", "line-02"], + "shrunk at byte {trigger_pos}" + ); + } + Ok(()) + } + + #[test] + fn a_whole_file_read_of_a_log_that_shrank_is_measured_again() -> Result<()> { + // Under the budget there is no probe; the window read itself runs short. + let content = twenty_lines().into_bytes(); + let tail = tail_of_shrinking(content, 0, Shrink::Once(24), 4096, LogDecode::PLAIN)?; + + assert_eq!(tail.lines, ["line-00", "line-01", "line-02"]); + Ok(()) + } + + #[test] + fn a_log_that_keeps_shrinking_ends_in_an_empty_tail_not_an_error() -> Result<()> { + let content = twenty_lines().into_bytes(); + let tail = tail_of_shrinking(content, 0, Shrink::EveryRead, 4096, LogDecode::PLAIN)?; + + assert_eq!(tail, LogTail::default()); + Ok(()) + } + + #[test] + fn a_log_that_did_not_shrink_reads_exactly_as_before() -> Result<()> { + // The trigger is past the end, so the source never shrinks. + let content = twenty_lines().into_bytes(); + let tail = tail_of_shrinking(content, u64::MAX, Shrink::Once(0), 40, LogDecode::PLAIN)?; + + assert_eq!( + tail.lines, + ["line-15", "line-16", "line-17", "line-18", "line-19"] + ); + assert_eq!(tail.skipped_bytes, 120); + Ok(()) + } + + /// `len` bytes of `a` with a zero at the first `zeros` odd positions. + fn bytes_with_zero_odd_bytes(len: usize, zeros: usize) -> Vec { + let mut bytes = vec![b'a'; len]; + for index in (1..len).step_by(2).take(zeros) { + bytes[index] = 0; + } + bytes + } + + fn sniff(bytes: Vec) -> Result> { + let size = bytes.len() as u64; + sniff_utf16_le(&mut std::io::Cursor::new(bytes), size, Path::new("sniff")) + } + + #[test] + fn a_log_without_a_byte_order_mark_is_sixteen_bit_from_half_its_odd_bytes_being_zero() + -> Result<()> { + // 16 bytes have 8 odd positions: 4 zeros (half) is the threshold. + assert_eq!(sniff(bytes_with_zero_odd_bytes(16, 4))?, Some(false)); + assert_eq!(sniff(bytes_with_zero_odd_bytes(16, 3))?, None); + // Only the first 512 bytes count: 256 odd positions, 128 zeros... + assert_eq!(sniff(bytes_with_zero_odd_bytes(600, 128))?, Some(false)); + assert_eq!(sniff(bytes_with_zero_odd_bytes(600, 127))?, None); + // ...and a source under 8 bytes is never taken for UTF-16. + assert_eq!(sniff(bytes_with_zero_odd_bytes(6, 3))?, None); + Ok(()) + } + + #[test] + fn a_tail_read_touches_no_more_than_its_budget_of_a_large_file() -> Result<()> { + // 8 MiB under a 64 KiB budget: a whole-file read would pull 8 MiB. + let filler = format!("{}\n", "x".repeat(1023)); + let content = filler.repeat(8192).into_bytes(); + let budget = 64 * 1024; + + let (lines, bytes_read) = read_counting(content, usize::MAX, budget, LogDecode::PLAIN)?; + + assert!(!lines.is_empty()); + // The window plus the one byte probed before it. + assert!(bytes_read <= budget + 1, "read {bytes_read} bytes"); + Ok(()) + } + + #[test] + fn a_sixteen_bit_tail_read_touches_no_more_than_its_budget_of_a_large_file() -> Result<()> { + let filler = format!("\u{1b}[33m{}\u{1b}[0m\r\n", "y".repeat(500)); + let content = utf16_le_bytes(&filler.repeat(8000), true); + assert!(content.len() > 8_000_000); + let budget = 64 * 1024 + 1; + + let (lines, bytes_read) = read_counting(content, usize::MAX, budget, LogDecode::CONSOLE)?; + + assert!(!lines.is_empty()); + assert!( + lines.iter().all(|line| line == &"y".repeat(500)), + "decoded from a unit boundary, ANSI stripped, no fragment" + ); + // The window, the unit probed before it, and the 512-byte sniff of the + // start of the file. + assert!(bytes_read <= budget + 2 + 512, "read {bytes_read} bytes"); + Ok(()) + } + + /// For every budget from nothing to the whole file, the lines returned are + /// a suffix of the file's complete lines: never a fragment, never garbage. + fn assert_every_budget_returns_a_suffix_of_whole_lines( + content: &[u8], + expected: &[String], + decode: LogDecode, + ) -> Result<()> { + // A byte order mark is not part of the first line. + let bom_len = if decode.utf16_le && content.starts_with(&[0xff, 0xfe]) { + 2 + } else { + 0 + }; + for budget in 0..=content.len() as u64 + 2 { + let (lines, _) = read_counting(content.to_vec(), usize::MAX, budget, decode)?; + assert!( + lines.len() <= expected.len() && lines == expected[expected.len() - lines.len()..], + "budget {budget}: {lines:?} is not a suffix of {expected:?}" + ); + if budget + bom_len >= content.len() as u64 { + assert_eq!(lines, expected, "budget {budget} covers the whole file"); + } + } + Ok(()) + } + + #[test] + fn every_budget_returns_whole_utf8_lines() -> Result<()> { + let text = "alpha\nbravo\r\ncharlie\n\ndelta\u{2588}\nlast"; + let expected = ["alpha", "bravo", "charlie", "", "delta\u{2588}", "last"] + .map(str::to_owned) + .to_vec(); + assert_every_budget_returns_a_suffix_of_whole_lines( + text.as_bytes(), + &expected, + LogDecode::PLAIN, + )?; + assert_every_budget_returns_a_suffix_of_whole_lines( + text.as_bytes(), + &expected, + LogDecode::CONSOLE, + ) + } + + #[test] + fn every_budget_returns_whole_sixteen_bit_lines_including_odd_ones() -> Result<()> { + // An odd budget puts the seek position on the second byte of a unit, + // which must be rounded to a unit boundary rather than decoded from. + let text = "alpha\nbravo\r\ncharlie\n\ndelta\u{2588}\u{1f600}\nlast"; + let expected = [ + "alpha", + "bravo", + "charlie", + "", + "delta\u{2588}\u{1f600}", + "last", + ] + .map(str::to_owned) + .to_vec(); + for bom in [true, false] { + assert_every_budget_returns_a_suffix_of_whole_lines( + &utf16_le_bytes(text, bom), + &expected, + LogDecode::CONSOLE, + )?; + } + Ok(()) + } + + // --- decoding ------------------------------------------------------ + + fn read_bytes( + name: &str, + bytes: &[u8], + limit: usize, + decode: LogDecode, + ) -> Result> { + let path = + std::env::temp_dir().join(format!("rocm-core-tail-{name}-{}.log", unique_suffix())); + fs::write(&path, bytes)?; + let lines = tail_lines_decoded(&path, limit, TEST_MAX_TAIL_READ, decode); + fs::remove_file(&path).ok(); + lines + } + + #[test] + fn console_decode_reads_a_sixteen_bit_log_with_a_byte_order_mark() -> Result<()> { + let bytes = utf16_le_bytes("\u{1b}[32m[INFO]\u{1b}[0m first\r\nsecond\r\n", true); + + let lines = read_bytes("bom", &bytes, 10, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["[INFO] first", "second"]); + Ok(()) + } + + #[test] + fn console_decode_detects_a_sixteen_bit_log_without_a_byte_order_mark() -> Result<()> { + let bytes = utf16_le_bytes("first\nsecond\nthird", false); + + let lines = read_bytes("nobom", &bytes, 2, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["second", "third"]); + Ok(()) + } + + #[test] + fn console_decode_replaces_a_lone_surrogate_and_ignores_an_odd_trailing_byte() -> Result<()> { + let mut bytes = utf16_le_bytes("ok\n", true); + bytes.extend_from_slice(&[0x00, 0xd8, b'x', 0x00, b'\n', 0x00, 0x41]); + + let lines = read_bytes("surrogate", &bytes, 10, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["ok", "\u{FFFD}x"]); + Ok(()) + } + + #[test] + fn console_decode_leaves_a_short_or_ascii_log_as_utf8() -> Result<()> { + // Under 8 bytes there is too little to call it sixteen-bit; plain + // ASCII has no zero bytes to suggest it. + let short = read_bytes("short", b"a\0b\0c\0", 5, LogDecode::CONSOLE)?; + let ascii = read_bytes("ascii", b"plain ascii text\n", 5, LogDecode::CONSOLE)?; + + assert_eq!(short, vec!["a\0b\0c\0"]); + assert_eq!(ascii, vec!["plain ascii text"]); + Ok(()) + } + + #[test] + fn console_decode_strips_ansi_per_line_so_an_unterminated_sequence_ends_with_its_line() + -> Result<()> { + let bytes = b"\x1b[31mred\x1b[0m\ncut off \x1b[1;3\nnext line\n"; + + let lines = read_bytes("ansi", bytes, 10, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["red", "cut off ", "next line"]); + Ok(()) + } + + #[test] + fn plain_decode_neither_detects_sixteen_bit_text_nor_strips_ansi() -> Result<()> { + let ansi = read_bytes("plain-ansi", b"\x1b[32mgreen\x1b[0m\n", 5, LogDecode::PLAIN)?; + let wide = read_bytes( + "plain-wide", + &utf16_le_bytes("hi\n", true), + 5, + LogDecode::PLAIN, + )?; + + assert_eq!(ansi, vec!["\u{1b}[32mgreen\u{1b}[0m"]); + assert_eq!(wide[0], "\u{FFFD}\u{FFFD}h\0i\0"); + Ok(()) + } + + #[test] + fn strip_ansi_sequences_removes_csi_sequences_and_a_bare_escape() { + assert_eq!( + strip_ansi_sequences("\u{1b}[1;31mred\u{1b}[0m \u{1b}[2Kdone \u{1b}x"), + "red done x" + ); + assert_eq!(strip_ansi_sequences("no escapes"), "no escapes"); + } + + #[test] + fn a_sixteen_bit_limit_zero_returns_empty_without_opening_the_file() -> Result<()> { + let path = Path::new("/does/not/exist/rocm-core-tail-limit-zero-wide.log"); + assert!(tail_lines_decoded(path, 0, TEST_MAX_TAIL_READ, LogDecode::CONSOLE)?.is_empty()); + Ok(()) + } + + // --- a source that grows after its size was read ------------------- + + /// A source whose underlying data grows by `appended` on the first read + /// after the size was taken, as a log does when its writer is still busy + /// between the `stat` and the read. + struct GrowsOnFirstRead { + cursor: std::io::Cursor>, + appended: Option>, + bytes_read: std::rc::Rc>, + } + + impl GrowsOnFirstRead { + fn new(content: Vec, appended: Vec) -> (Self, std::rc::Rc>) { + let bytes_read = std::rc::Rc::new(std::cell::Cell::new(0)); + let source = Self { + cursor: std::io::Cursor::new(content), + appended: Some(appended), + bytes_read: bytes_read.clone(), + }; + (source, bytes_read) + } + } + + impl Read for GrowsOnFirstRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(appended) = self.appended.take() { + self.cursor.get_mut().extend_from_slice(&appended); + } + let read = self.cursor.read(buf)?; + self.bytes_read.set(self.bytes_read.get() + read as u64); + Ok(read) + } + } + + impl Seek for GrowsOnFirstRead { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.cursor.seek(position) + } + } + + #[test] + fn a_source_that_grows_after_its_size_was_read_is_not_read_past_that_size() -> Result<()> { + // 64 KiB of 16-byte lines, then 64 KiB more appended after the size + // snapshot. With a budget of a quarter of the snapshot the window + // starts mid-file; with a budget above the snapshot there is no seek. + // Either way the tail must end where the snapshot ended and the read + // must touch no more than the window (plus the one probe byte). + let settled = "settled-line-xx\n".repeat(4096).into_bytes(); + let appended = "APPENDED-LINE-x\n".repeat(4096).into_bytes(); + let size = settled.len() as u64; + for (label, budget) in [ + ("window inside the file", size / 4), + ("whole file", size + 1), + ] { + let (source, bytes_read) = GrowsOnFirstRead::new(settled.clone(), appended.clone()); + + let lines = tail_lines_from( + source, + size, + Path::new("growing-source"), + usize::MAX, + budget, + LogDecode::PLAIN, + )? + .lines; + + assert!( + lines.iter().all(|line| line == "settled-line-xx"), + "{label}: read past the size snapshot into {:?}", + lines.iter().find(|line| line.starts_with("APPENDED")) + ); + assert_eq!(lines.last().map(String::as_str), Some("settled-line-xx")); + assert!( + bytes_read.get() <= budget.min(size) + 1, + "{label}: read {} bytes against a budget of {budget}", + bytes_read.get() + ); + } + Ok(()) + } + + #[test] + fn a_growing_sixteen_bit_source_is_not_read_past_its_size_either() -> Result<()> { + let settled = utf16_le_bytes(&"settled-line-xx\n".repeat(1024), true); + let appended = utf16_le_bytes(&"APPENDED-LINE-x\n".repeat(1024), false); + let size = settled.len() as u64; + let (source, _) = GrowsOnFirstRead::new(settled, appended); + + let lines = tail_lines_from( + source, + size, + Path::new("growing-source"), + usize::MAX, + size / 4, + LogDecode::CONSOLE, + )? + .lines; + + assert!(!lines.is_empty()); + assert!(lines.iter().all(|line| line == "settled-line-xx")); + Ok(()) + } + + // --- a final escape-only fragment ---------------------------------- + + /// What the ComfyUI log view returned before the tail routine replaced its + /// private reader: decode the whole file, strip ANSI sequences from the + /// whole text, then take the last `limit` of `str::lines`. + fn comfyui_reader_before_the_shared_routine(bytes: &[u8], limit: usize) -> Vec { + let looks_utf16 = bytes.len() >= 8 && { + let sample = &bytes[..bytes.len().min(512)]; + let zero_odd = sample + .iter() + .enumerate() + .filter(|(index, byte)| index % 2 == 1 && **byte == 0) + .count(); + zero_odd * 4 >= sample.len() + }; + let text = if bytes.starts_with(&[0xff, 0xfe]) || looks_utf16 { + let start = if bytes.starts_with(&[0xff, 0xfe]) { + 2 + } else { + 0 + }; + let units: Vec = bytes[start..] + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + String::from_utf16_lossy(&units) + } else { + String::from_utf8_lossy(bytes).into_owned() + }; + let text = strip_ansi_sequences(&text); + let mut lines: Vec = text.lines().rev().take(limit).map(str::to_owned).collect(); + lines.reverse(); + lines + } + + /// The same text as UTF-8 and as UTF-16 with and without a byte order mark. + fn console_encodings(text: &str) -> Vec<(&'static str, Vec)> { + vec![ + ("utf-8", text.as_bytes().to_vec()), + ("utf-16 with bom", utf16_le_bytes(text, true)), + ("utf-16 without bom", utf16_le_bytes(text, false)), + ] + } + + #[test] + fn a_colour_reset_after_the_final_newline_does_not_displace_the_last_line() -> Result<()> { + for (label, bytes) in console_encodings("\u{1b}[31mfailed\n\u{1b}[0m") { + let lines = read_bytes("reset", &bytes, 1, LogDecode::CONSOLE)?; + assert_eq!(lines, vec!["failed"], "{label}"); + } + Ok(()) + } + + #[test] + fn a_final_escape_only_fragment_is_not_a_line_but_blank_lines_are() -> Result<()> { + // (text, limit, expected): the unterminated escape-only tail never + // counts; a terminated blank line, even one made only of an escape + // sequence, is a real line and keeps its place. + let cases: [(&str, usize, &[&str]); 5] = [ + ("one\ntwo\n\u{1b}[0m", 2, &["one", "two"]), + ("one\ntwo\n\u{1b}[1;3", 2, &["one", "two"]), + ("one\ntwo\n\n", 2, &["two", ""]), + ("one\ntwo\n\u{1b}[0m\n", 2, &["two", ""]), + ("one\ntwo\n\u{1b}[0mtext", 2, &["two", "text"]), + ]; + for (text, limit, expected) in cases { + for (label, bytes) in console_encodings(text) { + let lines = read_bytes("fragment", &bytes, limit, LogDecode::CONSOLE)?; + assert_eq!(lines, expected, "{label}: {text:?} limit {limit}"); + } + } + Ok(()) + } + + #[test] + fn plain_decode_keeps_a_final_escape_fragment_because_it_strips_nothing() -> Result<()> { + let lines = read_bytes("plain-reset", b"a\n\x1b[0m", 2, LogDecode::PLAIN)?; + + assert_eq!(lines, vec!["a", "\u{1b}[0m"]); + Ok(()) + } + + #[test] + fn console_decode_matches_the_comfyui_reader_it_replaced_on_small_logs() -> Result<()> { + // Inputs on which the old reader and the per-line strip agree: no + // escape sequence runs across a newline. The old reader is the + // oracle, so a difference here is a change in what a user sees. + let texts = [ + "plain\nlines\n", + "\u{1b}[31mfailed\n\u{1b}[0m", + "\u{1b}[31mfailed\u{1b}[0m\n\u{1b}[0m", + "a\n\u{1b}[0m\n\u{1b}[0m", + "a\r\nb\r\n\u{1b}[0m", + "a\n\n\n", + "a\n\u{1b}[2K", + "\u{1b}[0m", + "", + ]; + for text in texts { + for (label, bytes) in console_encodings(text) { + for limit in 1..=4 { + let expected = comfyui_reader_before_the_shared_routine(&bytes, limit); + let actual = read_bytes("oracle", &bytes, limit, LogDecode::CONSOLE)?; + assert_eq!(actual, expected, "{label}: {text:?} limit {limit}"); + } + } + } + Ok(()) + } + + // --- an empty tail: no lines, or no line that fits the bound -------- + + fn read_log_bytes( + name: &str, + bytes: &[u8], + limit: usize, + max_tail_read: u64, + decode: LogDecode, + ) -> Result { + let path = + std::env::temp_dir().join(format!("rocm-core-tail-{name}-{}.log", unique_suffix())); + fs::write(&path, bytes)?; + let tail = tail_log_decoded(&path, limit, max_tail_read, decode); + fs::remove_file(&path).ok(); + tail + } + + #[test] + fn an_empty_tail_says_whether_the_log_is_empty_or_its_newest_line_is_too_long() -> Result<()> { + // The state matrix of an empty `lines`: (content, limit, whether the + // newest line exceeds the bound, bytes the budget left unread). + let long_line = "x".repeat(300); + let long_with_newline = format!("{long_line}\n"); + let long_utf16 = utf16_le_bytes(&long_line, true); + let cases: Vec<(&str, Vec, usize, bool, u64)> = vec![ + ("empty file", Vec::new(), 5, false, 0), + ( + "limit zero on a long line", + long_line.clone().into_bytes(), + 0, + false, + 0, + ), + ( + "long line without a newline", + long_line.into_bytes(), + 5, + true, + 300 - 64, + ), + ( + "long line with a newline", + long_with_newline.clone().into_bytes(), + 5, + true, + 301 - 64, + ), + ("long sixteen-bit line", long_utf16, 5, true, 602 - 64), + ( + "long line that ends a larger log", + format!("first\n{long_with_newline}").into_bytes(), + 5, + true, + 6 + 301 - 64, + ), + ("short log", b"a\nb\n".to_vec(), 5, false, 0), + ]; + for (label, content, limit, exceeds, skipped) in cases { + let tail = read_log_bytes("empty-tail", &content, limit, 64, LogDecode::CONSOLE)?; + assert_eq!( + tail.newest_line_exceeds_the_read_bound(), + exceeds, + "{label}: {tail:?}" + ); + assert_eq!(tail.skipped_bytes, skipped, "{label}: {tail:?}"); + // The flag is only ever set for an empty list, and the empty list + // only ever comes from the cases that name it. + if exceeds || content.is_empty() || limit == 0 { + assert!(tail.lines.is_empty(), "{label}: {tail:?}"); + } + } + Ok(()) + } + + #[test] + fn a_tail_that_found_lines_never_reports_an_exceeded_bound() -> Result<()> { + let content = "line\n".repeat(100).into_bytes(); + + let tail = read_log_bytes("found", &content, 5, 64, LogDecode::PLAIN)?; + + assert_eq!(tail.lines.len(), 5); + assert!(tail.skipped_bytes > 0, "the budget did leave bytes unread"); + assert!(!tail.newest_line_exceeds_the_read_bound()); + Ok(()) + } + + #[test] + fn a_byte_order_mark_is_not_counted_as_bytes_the_budget_skipped() -> Result<()> { + let content = utf16_le_bytes("a\nb\n", true); + + let tail = read_log_bytes("bom-skipped", &content, 5, 1024, LogDecode::CONSOLE)?; + + assert_eq!(tail.lines, vec!["a", "b"]); + assert_eq!(tail.skipped_bytes, 0); + Ok(()) + } + + #[test] + fn the_service_log_summary_names_each_of_the_three_states_apart() { + let none = LogTail::default(); + let too_long = LogTail { + lines: Vec::new(), + skipped_bytes: 1, + }; + let two = LogTail { + lines: vec!["a".to_owned(), "b".to_owned()], + skipped_bytes: 7, + }; + + assert_eq!( + service_log_tool_summary("svc", &none), + "Service `svc` has no log output yet." + ); + assert_eq!( + service_log_tool_summary("svc", &too_long), + "Service `svc` has log output, but none is shown: the newest line is longer than \ + the 4 MiB a log view reads." + ); + assert_eq!( + service_log_tool_summary("svc", &two), + "Read the last 2 line(s) from service `svc`." + ); + assert_eq!( + LOG_VIEW_MAX_TAIL_READ, + 4 * 1024 * 1024, + "the notice above names this many MiB" + ); + } } diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index be7961c3aa..971474e9b9 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -37,10 +37,21 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 6, "recommended_system_ram_gb": 8, - "quantization": "none; recommended small assistant recipe", - "artifact_hint": "Lemonade model id; resolved internally to a GGUF build of this model", + "quantization": "Q4_K_M GGUF on Lemonade; bfloat16 safetensors on vLLM", + "artifact_hint": "Lemonade serves the Q4_K_M build from the companion GGUF repo; vLLM serves the safetensors repo under this id", "artifacts": [], "engine_recipes": [ + { + "engine": "lemonade", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "this id publishes safetensors only, so Lemonade is pointed at the companion GGUF repo with an explicit quantization variant" + ], + "model_id_override": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M" + }, { "engine": "vllm", "required_flags": ["--enable-auto-tool-choice", "--tool-call-parser", "hermes"], diff --git a/crates/rocm-core/src/model_readiness.rs b/crates/rocm-core/src/model_readiness.rs index ceeead4e71..27d4832dbe 100644 --- a/crates/rocm-core/src/model_readiness.rs +++ b/crates/rocm-core/src/model_readiness.rs @@ -8,10 +8,14 @@ //! after a failure. This answers the question that comes before both, from data //! the CLI already has: a curated [`ModelRecipeRecord`] (dtype, quantization, //! minimum GPU memory, device policy, preferred engines, declared alternatives) -//! composed with what the host can offer. **It fetches nothing** — no weights, -//! no metadata probe, no network call of any kind — which is the whole point: -//! today a user learns a model will not run by starting a download and waiting -//! for it to fail. +//! composed with what the host can offer. **It never fetches weights** — which +//! is the whole point: a user should not learn a model will not run by starting +//! a download and waiting for it to fail. The curated path reads nothing beyond +//! what the caller hands over. A GGUF reference *outside* the catalog is the one +//! exception, and a small one: [`attach_context_estimate`] reads the file's +//! header (for a remote file: one repository listing, then a few megabytes by +//! range request) and sizes the +//! model from it. See [`crate::gguf_fit`]. //! //! Three distinctions carry the design, and collapsing any of them produces a //! confident wrong answer: @@ -27,14 +31,22 @@ //! today. Conflating the two hands an APU user a remediation that can never //! work. See [`AcceleratorMemory::UnifiedMemoryUnreadable`]. //! -//! Curated recipes only. Answering for an arbitrary hub model needs a metadata -//! probe and a table mapping quantization schemes to available kernels, neither -//! of which has a source of truth here; such a model is reported -//! [`UndeterminedReason::ModelNotCurated`], never blocked. +//! A model outside the catalog has no recipe, so it is reported +//! [`UndeterminedReason::ModelNotCurated`], never blocked. When it is a GGUF the +//! header allows an *estimate* (weights, KV cache at a context size, compute +//! buffer) against dedicated VRAM, carried in [`ModelReadiness::estimate`] under +//! its own label rather than as a verdict: an estimate from a header is not a +//! recipe's tested minimum. use crate::ModelRecipeRecord; use crate::ModelRecipeRegistry; use crate::diagnose::{Fix, Route, upstream_route}; +use crate::gguf::parse_gguf_header; +use crate::gguf_fit::{ + CannotEstimateReason, ContextEstimate, ESTIMATE_CAVEAT, EstimateOutcome, FitRequest, + KvCacheType, SERVE_CONTEXT_NOTE, SERVE_OPTIONS_NOTE, estimate_fit, evidence_lines, tokens, +}; +use crate::gguf_source::{GgufRef, GgufSource, ProjectorNamed, parse_gguf_ref}; use serde::{Deserialize, Serialize}; /// How many fallback alternatives to offer when a recipe declares none that fit. @@ -113,7 +125,7 @@ pub enum AcceleratorMemory { impl AcceleratorMemory { /// The figure to compare a recipe minimum against, when there is one. - const fn measured_gib(self) -> Option { + pub(crate) const fn measured_gib(self) -> Option { match self { Self::Dedicated(gib) => Some(gib), Self::None | Self::Unknown | Self::UnifiedMemoryUnreadable => None, @@ -196,6 +208,14 @@ pub struct ModelReadiness { /// Informational — every curated recipe carries at least one — so they do /// not move the verdict. Degradation is decided from measurements. pub warnings: Vec, + /// A memory estimate from a GGUF file's header, for a model outside the + /// catalog. Absent for curated models and for references that are not GGUF. + /// + /// Deliberately not a [`ModelVerdict`]: `verdict` stays `undetermined`, and + /// this carries its own outcome, so no reader mistakes an estimate for a + /// recipe's tested minimum. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub estimate: Option, } impl ModelReadiness { @@ -222,6 +242,7 @@ impl ModelReadiness { route: None, undetermined_reason: Some(reason), warnings: Vec::new(), + estimate: None, } } } @@ -303,7 +324,7 @@ fn requires_gpu(recipe: &ModelRecipeRecord) -> bool { /// values come from `u32` catalog fields, so they are always exact integers /// and never reach this branch; flooring only ever shades a measured figure /// down, never a minimum. -fn format_gib(value: f64) -> String { +pub(crate) fn format_gib(value: f64) -> String { if value.fract().abs() < f64::EPSILON { format!("{value:.0} GiB") } else { @@ -392,8 +413,9 @@ fn assess( UndeterminedReason::ModelNotCurated, vec![ format!("`{model_ref}` is not in the curated recipe catalog"), - "`rocm diagnose --model` answers for curated recipes only: judging an arbitrary \ - model needs metadata this CLI does not fetch" + "there is no recipe for it, so no tested GPU memory minimum to compare; a GGUF \ + reference (a local .gguf path, or `owner/repo:QUANT` on Hugging Face) is \ + estimated from its file header instead" .to_owned(), ], ); @@ -408,8 +430,8 @@ fn assess( commands: vec!["rocm model".to_owned()], verify: format!("rocm serve {model_ref}"), notes: vec![ - "`rocm serve` accepts models outside this catalog; it just cannot say in advance \ - whether they will fit" + "`rocm serve` accepts models outside this catalog; for one that is not a GGUF \ + reference this cannot say in advance whether it will fit" .to_owned(), ], ..Fix::default() @@ -448,6 +470,7 @@ fn assess( route: None, undetermined_reason: None, warnings: recipe.warnings.clone(), + estimate: None, }; // Ordered by what pre-empts what. The engine gate comes first because a @@ -476,17 +499,7 @@ fn assess( fallback", recipe.device_policy )); - readiness.fix = Some(Fix { - summary: "make a GPU visible to ROCm, then ask again".to_owned(), - commands: vec!["rocm diagnose".to_owned()], - verify: format!("rocm diagnose --model {model_ref}"), - notes: vec![ - "`rocm diagnose` matches this machine against the known reasons a GPU does not \ - show up" - .to_owned(), - ], - ..Fix::default() - }); + readiness.fix = Some(no_gpu_fix(format!("rocm diagnose --model {model_ref}"))); return readiness; } @@ -522,17 +535,7 @@ fn assess( is a gap in what the CLI can see, not a verdict about the model", format_gib(required) )); - readiness.fix = Some(Fix { - summary: "let the CLI read this host's GPU memory, then ask again".to_owned(), - commands: vec!["amd-smi metric --json".to_owned()], - verify: format!("rocm diagnose --model {model_ref}"), - notes: vec![ - "the reading comes from `amd-smi`; if it is missing or failing, the ROCm \ - install is what needs attention first" - .to_owned(), - ], - ..Fix::default() - }); + readiness.fix = Some(read_gpu_memory_fix(model_ref)); } (Some(required), Some(available)) if available < required => { readiness.verdict = ModelVerdict::Blocked; @@ -675,6 +678,480 @@ fn alternatives_for( .collect() } +/// The context size an estimate is made for when none was asked for. +pub const DEFAULT_CONTEXT_SIZE: u64 = 4096; + +/// What the user asked of a context-size estimate. +/// +/// Each field is `None` when the flag was not passed, which is not the same as +/// passing the default: it decides whether a curated model, which has no +/// context estimate, should say so. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ContextRequest { + pub context_size: Option, + pub kv_cache_type: Option, +} + +/// Size a GGUF model outside the catalog from its file header. +/// +/// Acts only on an answer of [`UndeterminedReason::ModelNotCurated`] whose +/// reference names a GGUF. Every other answer keeps its verdict, plan and +/// alternatives; the only thing added to it is, when `--context-size` or +/// `--kv-cache-type` was passed, an evidence line saying why the flag did not +/// apply. The verdict stays `Undetermined`: what is added is [`ModelReadiness::estimate`] +/// and the evidence that goes with it. `source` is the only thing that reads a +/// file or the network, and it reads the header, never the weights. +/// +/// Three steps, each its own function: [`plan_context_estimate`] decides whether +/// there is a file to read (and, when there is not, what to say about flags that +/// were passed), [`read_context_estimate`] reads the header and sizes it, and +/// [`apply_context_estimate`] rewrites the readiness for what came back. +pub fn attach_context_estimate( + readiness: &mut ModelReadiness, + request: &ContextRequest, + host: &HostFacts, + source: &dyn GgufSource, +) { + match plan_context_estimate(readiness, request) { + ContextPlan::LeaveAsIs { note } => readiness.evidence.extend(note), + ContextPlan::Estimate { + reference, + fit_request, + } => { + let (estimate, refused_models) = + read_context_estimate(&reference, &readiness.model_ref, &fit_request, host, source); + apply_context_estimate( + readiness, + &reference, + estimate, + refused_models.as_deref(), + request, + ); + } + } +} + +/// What [`plan_context_estimate`] decided for an answer. +#[derive(Debug, PartialEq)] +enum ContextPlan { + /// Nothing is read and the answer keeps its shape. `note` is the evidence + /// line that says why a context flag the user passed was not applied; `None` + /// when no flag was passed, or for an answer that owes no explanation. + LeaveAsIs { note: Option }, + /// Read this GGUF's header and size it for this request. + Estimate { + reference: GgufRef, + fit_request: FitRequest, + }, +} + +/// Decide whether `readiness` is owed an estimate, without reading anything. +fn plan_context_estimate(readiness: &ModelReadiness, request: &ContextRequest) -> ContextPlan { + let asked = request.context_size.is_some() || request.kv_cache_type.is_some(); + let leave = |note: String| ContextPlan::LeaveAsIs { + note: asked.then_some(note), + }; + if readiness.canonical_model_id.is_some() { + return leave( + "a context size and KV cache type do not apply to a curated recipe, whose GPU memory \ + minimum is fixed; estimates for them cover GGUF models outside the catalog" + .to_owned(), + ); + } + if readiness.undetermined_reason == Some(UndeterminedReason::CatalogUnreachable) { + // Whether the reference names a curated model is exactly what could not be + // found out, so there is no telling which answer it is owed: say that the + // flags were not applied rather than dropping them without a word. + return leave( + "the context size and KV cache type were not applied: the recipe catalog could \ + not be read, so it is not known whether this is a curated model or a GGUF \ + outside the catalog" + .to_owned(), + ); + } + // Every answer that reaches here today is a catalog miss: the other + // undetermined answers carry the recipe they were about, and are left above. + // This keeps a future recipe-less answer from being estimated by accident. + if readiness.undetermined_reason != Some(UndeterminedReason::ModelNotCurated) { + return ContextPlan::LeaveAsIs { note: None }; + } + let Some(reference) = parse_gguf_ref(&readiness.model_ref) else { + return leave( + "a context size and KV cache type are estimated for GGUF models only: a local \ + .gguf path, or `owner/repo:QUANT` on Hugging Face" + .to_owned(), + ); + }; + // `owner/repo` with no quantization names a repository, not a file, and a + // repository is not read (every file in it is a different model). Without a + // context flag the answer is the plain "not in the catalog" it always was, + // reached without touching the network; with one, the user is told what + // would make it answerable. + if matches!(reference, GgufRef::Hub { variant: None, .. }) { + return leave(format!( + "`{}` names a repository, not a model file: add `:QUANT` (for example \ + `:Q4_K_M`) to estimate one of its GGUF files", + readiness.model_ref + )); + } + ContextPlan::Estimate { + reference, + fit_request: FitRequest { + context_size: request.context_size.unwrap_or(DEFAULT_CONTEXT_SIZE), + context_size_defaulted: request.context_size.is_none(), + kv_cache_type: request.kv_cache_type.unwrap_or(KvCacheType::F16), + }, + } +} + +/// Read `reference`'s header through `source` and size it against `host`. A +/// header that cannot be reached or read is an estimate with no figures, named +/// after `model_ref` as the user wrote it. When the name was refused as a +/// multimodal projector, the repository's model files come back as well, for +/// the plan to offer one. +fn read_context_estimate( + reference: &GgufRef, + model_ref: &str, + fit_request: &FitRequest, + host: &HostFacts, + source: &dyn GgufSource, +) -> (ContextEstimate, Option>) { + let read = source.open(reference).and_then(|mut opened| { + let header = + parse_gguf_header(&mut opened.reader).map_err(|error| anyhow::anyhow!("{error}"))?; + Ok((header, opened.size, opened.description)) + }); + match read { + Ok((header, size, description)) => ( + estimate_fit( + &header, + size, + &description, + fit_request, + host.accelerator_memory, + host.system_ram_gib, + ), + None, + ), + Err(error) => { + let refused_models = error + .downcast_ref::() + .map(|refusal| refusal.models.clone()); + ( + ContextEstimate::unreadable(model_ref, fit_request, &error), + refused_models, + ) + } + } +} + +/// Put `estimate` on `readiness`, and rewrite the parts of the answer that were +/// written for a model nobody had looked at: the route, the fix, the +/// alternatives and the warnings, each according to what the estimate found. +fn apply_context_estimate( + readiness: &mut ModelReadiness, + reference: &GgufRef, + estimate: ContextEstimate, + refused_models: Option<&[String]>, + request: &ContextRequest, +) { + readiness.evidence.extend(evidence_lines(&estimate)); + let ask_again = |model_ref: &str| rediagnose(model_ref, request); + // The route sends a user to the upstream tracker because the catalog lacks + // the model. A GGUF was looked at directly, so that is no longer the + // question -- and for a path that does not exist or a hub that did not + // answer, the tracker is the wrong place to take it. + readiness.route = None; + if estimate.total_gib.is_some() { + // The header was read and sized, so the "serve it directly and read the + // engine's refusal" plan answered a question that no longer applies -- + // whether or not this host could be compared against. A header that could + // not be read gets a plan of its own below (check the path, or the name + // and the network), as does a hub name refused as a projector; one that + // was read but could not be sized keeps the "serve it" plan, pointed at a + // command that accepts the reference. + // The host gaps with a next step get the plan a curated model gets: + // memory that could not be read is looked into with `amd-smi`, and no + // visible GPU with `rocm diagnose`. An APU's unreadable pool has no + // command that changes the answer, and gets none. + readiness.fix = match estimate.cannot_estimate_reason { + Some(CannotEstimateReason::AcceleratorMemoryUnknown) => { + let mut fix = read_gpu_memory_fix(&readiness.model_ref); + fix.verify = ask_again(&readiness.model_ref); + Some(fix) + } + Some(CannotEstimateReason::NoGpu) => Some(no_gpu_fix(ask_again(&readiness.model_ref))), + // A layout whose figure can be too low (state-space, hybrid, + // mixture-of-experts, or a header missing a width) was sized and + // compared, and its detail tells the reader to compare the total + // with the GPU's memory and leave headroom. No command would turn it + // into a verdict, so there is no plan to offer. + Some( + CannotEstimateReason::UnmeasuredLayout | CannotEstimateReason::MissingMetadata, + ) => None, + // An APU's pool cannot be read and no command changes that; a fit or + // a misfit is the answer itself, with its suggestions. + _ => None, + }; + if estimate.outcome == EstimateOutcome::Fits { + readiness.alternatives.clear(); + } + readiness.warnings.push(ESTIMATE_CAVEAT.to_owned()); + } else if estimate.cannot_estimate_reason == Some(CannotEstimateReason::HeaderUnreadable) + && matches!(reference, GgufRef::Local(_)) + { + // A path that could not be read is a path to check. "Serve it directly and + // read the engine's refusal" is advice for a model nobody has looked at, + // and `rocm serve ` on a file that is not there only repeats this. + // A URL is read as a path, and the "could not open" it gets does not say + // why; the form that is read is `owner/repo:QUANT`. + let lower = readiness.model_ref.to_ascii_lowercase(); + let notes = if lower.starts_with("http://") || lower.starts_with("https://") { + vec![ + "a URL is not read: name a Hugging Face file as `owner/repo:QUANT` (or \ + `owner/repo:file.gguf`), or download it and pass the local path" + .to_owned(), + ] + } else { + Vec::new() + }; + readiness.fix = Some(Fix { + summary: "check the path, and that the file is a GGUF model, then ask again".to_owned(), + verify: ask_again(&readiness.model_ref), + notes, + ..Fix::default() + }); + } else if estimate.cannot_estimate_reason == Some(CannotEstimateReason::HeaderUnreadable) + && matches!(reference, GgufRef::Hub { .. }) + { + // The header could not be fetched or read: a name that does not exist, a + // gated or private repository, no network, a busy hub, or a file that is + // not a GGUF. The estimate's detail says which. Serving it would start a + // download of the whole file to learn the same thing, so the plan is to + // act on that detail and ask again. + readiness.fix = Some(Fix { + summary: "act on the reason above (a misspelt name or quantization, no route to \ + Hugging Face, a busy hub to wait for, or a file that is not a GGUF), \ + then ask again" + .to_owned(), + verify: ask_again(&readiness.model_ref), + notes: vec![ + "a gated or private repository cannot be read, because no token is sent: \ + download the file yourself and ask about its local path" + .to_owned(), + ], + ..Fix::default() + }); + } else if estimate.cannot_estimate_reason == Some(CannotEstimateReason::NotAModel) { + readiness.fix = Some(projector_fix(reference, refused_models, request)); + } else if let Some(fix) = &mut readiness.fix { + point_fix_at_unsized_gguf(fix, reference, &ask_again(&readiness.model_ref)); + } + // A compared estimate is sized for a context `rocm serve` would not pass, + // so the answer says so, once: with the options a suggestion names, or on + // its own. + if matches!( + estimate.outcome, + EstimateOutcome::Fits | EstimateOutcome::DoesNotFit + ) { + let note = if estimate.suggestions.is_empty() { + SERVE_CONTEXT_NOTE + } else { + SERVE_OPTIONS_NOTE + }; + readiness.warnings.push(note.to_owned()); + } + readiness.estimate = Some(estimate); +} + +/// The command that asks the same GGUF question again: the same reference, +/// quoted for this platform's shell when it needs to be, and the same +/// `--context-size` and `--kv-cache-type` when they were passed, so following +/// the plan does not quietly answer a different question. +fn rediagnose(model_ref: &str, request: &ContextRequest) -> String { + rediagnose_for(model_ref, request, cfg!(windows)) +} + +/// [`rediagnose`] for a chosen platform, so both are tested on either. +/// +/// A reference made only of characters no shell treats specially is printed +/// as it is, so a plain Windows path (`C:\models\m.gguf`) still works in both +/// `cmd.exe` and PowerShell; anything else is quoted for PowerShell on Windows +/// and for a POSIX shell elsewhere. A reference that starts with `-` is joined +/// to the flag (`--model=-x.gguf`), because the argument parser would read it +/// as another flag. +fn rediagnose_for(model_ref: &str, request: &ContextRequest, windows: bool) -> String { + use std::fmt::Write as _; + let quoted = shell_word(model_ref, windows); + let mut command = if model_ref.starts_with('-') { + format!("rocm diagnose --model={quoted}") + } else { + format!("rocm diagnose --model {quoted}") + }; + if let Some(size) = request.context_size { + let _ = write!(command, " --context-size {size}"); + } + if let Some(kind) = request.kv_cache_type { + let _ = write!(command, " --kv-cache-type {kind}"); + } + command +} + +/// `model_ref` as one word of a command for this platform's shell: as it is +/// when no shell would split or expand it, quoted for PowerShell on Windows and +/// for a POSIX shell elsewhere otherwise. +fn shell_word(model_ref: &str, windows: bool) -> String { + // `~` is plain after the first character, where neither shell expands it; + // Windows short names (`ADMINI~1`) carry one in the middle of a path. + let plain = !model_ref.is_empty() + && !model_ref.starts_with('~') + && model_ref.chars().all(|c| { + c.is_ascii_alphanumeric() + || matches!(c, '/' | ':' | '.' | '_' | '-' | '=' | '~') + || (windows && c == '\\') + }); + if plain { + model_ref.to_owned() + } else if windows { + crate::quoting::powershell_quote(model_ref) + } else { + crate::quoting::posix_shell_quote(model_ref) + } +} + +/// The plan when no GPU is visible to ROCm: find out why, then ask again with +/// `verify`. Shared by a curated model that needs a GPU and a GGUF estimate. +fn no_gpu_fix(verify: String) -> Fix { + Fix { + summary: "make a GPU visible to ROCm, then ask again".to_owned(), + commands: vec!["rocm diagnose".to_owned()], + verify, + notes: vec![ + "`rocm diagnose` matches this machine against the known reasons a GPU does not show \ + up" + .to_owned(), + ], + ..Fix::default() + } +} + +/// The plan when this host's GPU memory could not be read: look at what +/// `amd-smi` reports, then ask again. Shared by a curated model and a GGUF +/// estimate, which hit the same gap. +fn read_gpu_memory_fix(model_ref: &str) -> Fix { + Fix { + summary: "let the CLI read this host's GPU memory, then ask again".to_owned(), + commands: vec!["amd-smi metric --json".to_owned()], + verify: format!("rocm diagnose --model {model_ref}"), + notes: vec![ + "the reading comes from `amd-smi`; if it is missing or failing, the ROCm install is \ + what needs attention first" + .to_owned(), + ], + ..Fix::default() + } +} + +/// The plan for a name refused as a multimodal projector. "Serve it and read +/// the engine's refusal" would point back at the file just refused, so the plan +/// is to ask about the model instead: by its exact file name when the +/// repository holds one model, and with no command when it holds several (the +/// estimate's detail lists them) or none. +fn projector_fix(reference: &GgufRef, models: Option<&[String]>, request: &ContextRequest) -> Fix { + let summary = "name the model's own GGUF file or quantization, not its multimodal projector, \ + then ask again" + .to_owned(); + // The model's name comes from the hub's listing, not from the user, so it is + // offered only when it reads back as that same hub file: a name the parser + // would take for a local path, or that a shell would expand, is not printed + // as a command. + let verify = match (reference, models) { + (GgufRef::Hub { owner, repo, .. }, Some([only])) => { + let candidate = format!("{owner}/{repo}:{only}"); + let reads_back = parse_gguf_ref(&candidate) + == Some(GgufRef::Hub { + owner: owner.clone(), + repo: repo.clone(), + variant: Some(only.clone()), + }); + if reads_back { + rediagnose(&candidate, request) + } else { + String::new() + } + } + _ => String::new(), + }; + Fix { + summary, + verify, + ..Fix::default() + } +} + +/// Rewrite the "serve it and see" plan for a GGUF reference whose header was +/// reached for but not sized. +fn point_fix_at_unsized_gguf(fix: &mut Fix, reference: &GgufRef, verify: &str) { + // The plan was written for a model that is not a GGUF reference, and its + // note says so. This one is, and the estimate says why it could not be + // sized, so the note says that instead of contradicting it. + fix.notes = vec![ + "`rocm serve` accepts models outside this catalog; the estimate above could not size \ + this GGUF, so this cannot say in advance whether it will fit" + .to_owned(), + ]; + // `rocm serve` takes `owner/repo:QUANT`, whose label has no `/`; the + // `owner/repo:dir/file.gguf` form selects an exact file for this estimate + // only, and serve would not read it as a checkpoint. Naming it in a + // `rocm serve` command would hand the user a line that fails. + if matches!(reference, GgufRef::Hub { variant: Some(variant), .. } if variant.contains('/')) { + verify.clone_into(&mut fix.verify); + fix.notes.push( + "`rocm serve` does not accept an exact file name after the colon; it takes \ + `owner/repo:QUANT`" + .to_owned(), + ); + } + // `rocm serve` does take a local file, and serves it as it is; the path is + // quoted for the shell. One that starts with `-` would be read as a flag, + // so that one is asked about again instead. + if let GgufRef::Local(path) = reference { + let path = path.to_string_lossy(); + if path.starts_with('-') { + verify.clone_into(&mut fix.verify); + } else { + fix.verify = format!("rocm serve {}", shell_word(&path, cfg!(windows))); + } + } +} + +/// What an estimate says when a device-visibility variable is set on a host +/// whose GPUs differ in memory. +pub const MASK_UNVERIFIED_NOTE: &str = "a device-visibility variable (HIP_VISIBLE_DEVICES, \ + ROCR_VISIBLE_DEVICES or CUDA_VISIBLE_DEVICES) is set and this host's GPUs do not all \ + have the same memory; how the mask's numbering maps onto amd-smi's has not been verified \ + on such a host, so the memory compared with may be that of a different card (or the host \ + may be mistaken for an integrated GPU)"; + +/// Add [`MASK_UNVERIFIED_NOTE`] to a compared estimate when `mask_set` and the +/// GPUs' totals (`gpu_totals_mb`, one per amd-smi row) are not all equal. +/// +/// Which row a mask selects is not something this binary has verified on a +/// multi-GPU machine, so the visible rows are not guessed at: if any two cards +/// differ, whichever one the mask picked might not be the one compared. An +/// estimate that compared nothing has no figure to doubt and is left alone. +pub fn note_unverified_mask(readiness: &mut ModelReadiness, mask_set: bool, gpu_totals_mb: &[u64]) { + let differ = gpu_totals_mb.windows(2).any(|pair| pair[0] != pair[1]); + if !(mask_set && differ) { + return; + } + if let Some(estimate) = readiness.estimate.as_mut() + && estimate.available_gib.is_some() + { + estimate.notes.push(MASK_UNVERIFIED_NOTE.to_owned()); + } +} + /// The word this verdict is reported under. const fn verdict_label(verdict: ModelVerdict) -> &'static str { match verdict { @@ -685,6 +1162,23 @@ const fn verdict_label(verdict: ModelVerdict) -> &'static str { } } +/// The word the report opens with. An estimate that compared its figures with +/// the GPU is labelled as one, never as a verdict; every other answer -- an +/// estimate that could not compare included -- keeps the verdict's own word. +fn header_label(readiness: &ModelReadiness) -> String { + match &readiness.estimate { + Some(estimate) if estimate.outcome == EstimateOutcome::Fits => format!( + "ESTIMATED: fits at {} (calibrated on one MI300X, not a tested minimum)", + tokens(estimate.context_size) + ), + Some(estimate) if estimate.outcome == EstimateOutcome::DoesNotFit => format!( + "ESTIMATED: does not fit at {} (calibrated on one MI300X, not a tested minimum)", + tokens(estimate.context_size) + ), + _ => verdict_label(readiness.verdict).to_owned(), + } +} + /// Render the human-facing model verdict. /// /// Deliberately laid out like `diagnose::render_report_text`: the same ` - ` @@ -700,7 +1194,7 @@ pub fn render_model_readiness_text(readiness: &ModelReadiness) -> String { out, "rocm diagnose --model {}: {}", readiness.model_ref, - verdict_label(readiness.verdict) + header_label(readiness) ); if let Some(canonical) = &readiness.canonical_model_id { let _ = writeln!(out, " recipe: {canonical}"); @@ -708,6 +1202,14 @@ pub fn render_model_readiness_text(readiness: &ModelReadiness) -> String { for line in &readiness.evidence { let _ = writeln!(out, " - {line}"); } + if let Some(estimate) = &readiness.estimate + && !estimate.suggestions.is_empty() + { + let _ = writeln!(out, " what would make it fit:"); + for suggestion in &estimate.suggestions { + let _ = writeln!(out, " - {}", suggestion.text); + } + } if !readiness.alternatives.is_empty() { let _ = writeln!(out, " what would run here instead:"); for alternative in &readiness.alternatives { @@ -738,9 +1240,20 @@ pub fn render_model_readiness_text(readiness: &ModelReadiness) -> String { let _ = writeln!(out, " report it: {:>12}: {}", route.target, route.url); } // The recipe's own warnings come last and carry no verdict weight. Putting - // them above the plan would read as reasons for it. + // them above the plan would read as reasons for it. The estimate's notes + // are about the figures, not a recipe, and are labelled as such. + if let Some(estimate) = &readiness.estimate { + for note in &estimate.notes { + let _ = writeln!(out, " note: {note}"); + } + } for warning in &readiness.warnings { - let _ = writeln!(out, " recipe note: {warning}"); + let label = if readiness.estimate.is_some() { + "note" + } else { + "recipe note" + }; + let _ = writeln!(out, " {label}: {warning}"); } out } @@ -1215,4 +1728,1134 @@ mod tests { fn format_gib_renders_a_whole_number_with_no_decimal() { assert_eq!(format_gib(8.0), "8 GiB"); } + + // ----- GGUF estimate (models outside the catalog) ----------------------- + + use crate::gguf::test_builder::llama_like; + use crate::gguf_fit::SuggestionKind; + use crate::gguf_source::{GgufRef, OpenedGguf}; + use std::io::Cursor; + use std::sync::Mutex; + + /// Serves one in-memory GGUF for every reference and records what it was + /// asked to open, so a test can show the source was (or was not) reached. + struct FakeSource { + file: Option>, + /// The size the source reports for the file: the header plus the + /// weights it pretends to carry, so no test allocates a real model. + size: u64, + opened: Mutex>, + /// When set, opening refuses the name as a multimodal projector, as + /// file selection does, listing these model files. + refusal: Option>, + } + + impl FakeSource { + fn with(file: Vec) -> Self { + Self { + size: file.len() as u64, + file: Some(file), + opened: Mutex::new(Vec::new()), + refusal: None, + } + } + + /// A llama-shaped file whose weights are `weights_mib` MiB: 8192 + /// tokens of f16 cache is exactly 1 GiB for it. + fn weights(weights_mib: u64) -> Self { + let header = llama_like(10).file(0); + Self { + size: header.len() as u64 + weights_mib * 1024 * 1024, + file: Some(header), + opened: Mutex::new(Vec::new()), + refusal: None, + } + } + + fn failing() -> Self { + Self { + file: None, + size: 0, + opened: Mutex::new(Vec::new()), + refusal: None, + } + } + + /// A hub that refuses the name as a projector, with `models` beside it. + fn refusing_a_projector(models: &[&str]) -> Self { + Self { + refusal: Some(models.iter().map(|&model| model.to_owned()).collect()), + ..Self::failing() + } + } + + fn calls(&self) -> usize { + self.opened.lock().unwrap().len() + } + } + + impl GgufSource for FakeSource { + fn open(&self, reference: &GgufRef) -> anyhow::Result { + self.opened.lock().unwrap().push(reference.clone()); + if let (Some(models), GgufRef::Hub { variant, .. }) = (&self.refusal, reference) { + return Err(anyhow::Error::new(ProjectorNamed { + variant: variant.clone().unwrap_or_default(), + projectors: vec![variant.clone().unwrap_or_default()], + models: models.clone(), + })); + } + let Some(file) = &self.file else { + anyhow::bail!("the hub could not be reached"); + }; + Ok(OpenedGguf { + size: self.size, + reader: Box::new(Cursor::new(file.clone())), + description: "fake.gguf".to_owned(), + }) + } + } + + fn not_curated(model_ref: &str) -> ModelReadiness { + let registry = builtin_model_recipe_registry(); + assess_model_readiness( + model_ref, + &ModelCatalogSource::Available(®istry), + &host_with(24.0, 64.0), + &engine_choice, + ) + } + + fn asked(context: u64) -> ContextRequest { + ContextRequest { + context_size: Some(context), + kv_cache_type: None, + } + } + + /// The decide step on its own: a GGUF outside the catalog is planned for a + /// read with the defaults filled in, and an answer that is not one is left as + /// it is, with an explanation only when a context flag was passed. + #[test] + fn the_plan_reads_only_a_gguf_outside_the_catalog_and_fills_in_the_defaults() { + let gguf = not_curated("some-lab/never-curated-7b:Q4_K_M"); + assert_eq!( + plan_context_estimate(&gguf, &ContextRequest::default()), + ContextPlan::Estimate { + reference: GgufRef::Hub { + owner: "some-lab".to_owned(), + repo: "never-curated-7b".to_owned(), + variant: Some("Q4_K_M".to_owned()), + }, + fit_request: FitRequest { + // 4096, written out: `--help` and the README state it + // independently of the constant. + context_size: 4096, + context_size_defaulted: true, + kv_cache_type: KvCacheType::F16, + }, + } + ); + let not_gguf = not_curated("some-lab/never-curated-7b"); + assert_eq!( + plan_context_estimate(¬_gguf, &ContextRequest::default()), + ContextPlan::LeaveAsIs { note: None } + ); + assert!(matches!( + plan_context_estimate(¬_gguf, &asked(8192)), + ContextPlan::LeaveAsIs { note: Some(note) } if note.contains("add `:QUANT`") + )); + // Flags that were passed are carried as given, not replaced by defaults. + let both = ContextRequest { + context_size: Some(32768), + kv_cache_type: Some(KvCacheType::Q8_0), + }; + assert!(matches!( + plan_context_estimate(&gguf, &both), + ContextPlan::Estimate { fit_request, .. } if fit_request == FitRequest { + context_size: 32768, + context_size_defaulted: false, + kv_cache_type: KvCacheType::Q8_0, + } + )); + // A curated model is never read, and is told why only when it asked. + let curated = not_curated(SMALL_MODEL_REF); + assert_eq!( + plan_context_estimate(&curated, &ContextRequest::default()), + ContextPlan::LeaveAsIs { note: None } + ); + assert!(matches!( + plan_context_estimate(&curated, &both), + ContextPlan::LeaveAsIs { note: Some(note) } if note.contains("curated recipe") + )); + } + + /// A name refused as a projector is `not_a_model`, and the plan points at + /// the model rather than at serving the projector it refused: by its exact + /// name when there is one model, and with no command to run otherwise. + #[test] + fn a_refused_projector_is_not_offered_back_as_a_serve_command() { + let refused = |models: &[&str]| { + let mut readiness = not_curated("some-lab/vision-7b:mmproj-vision-7b-f16.gguf"); + attach_context_estimate( + &mut readiness, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &FakeSource::refusing_a_projector(models), + ); + readiness + }; + let one = refused(&["vision-7b-Q4_K_M.gguf"]); + let estimate = one.estimate.as_ref().expect("an estimate"); + assert_eq!( + estimate.cannot_estimate_reason, + Some(CannotEstimateReason::NotAModel) + ); + assert!( + estimate.detail.as_deref().is_some_and(|detail| detail + .starts_with("`mmproj-vision-7b-f16.gguf` is a multimodal projector")), + "{:?}", + estimate.detail + ); + let fix = one.fix.as_ref().expect("a plan for the user"); + assert!( + fix.summary.contains("not its multimodal projector"), + "{fix:?}" + ); + assert_eq!( + fix.verify, + "rocm diagnose --model some-lab/vision-7b:vision-7b-Q4_K_M.gguf" + ); + // Asked with a context size, the command for the model asks with it too. + let mut with_flags = not_curated("some-lab/vision-7b:mmproj-vision-7b-f16.gguf"); + attach_context_estimate( + &mut with_flags, + &ContextRequest { + context_size: Some(16_384), + kv_cache_type: Some(KvCacheType::Q4_0), + }, + &host_with(24.0, 64.0), + &FakeSource::refusing_a_projector(&["vision-7b-Q4_K_M.gguf"]), + ); + assert_eq!( + with_flags.fix.as_ref().expect("a plan").verify, + "rocm diagnose --model some-lab/vision-7b:vision-7b-Q4_K_M.gguf --context-size \ + 16384 --kv-cache-type q4_0" + ); + let text = render_model_readiness_text(&one); + assert!(!text.contains("header could not be read"), "{text}"); + // No header was read, so the report does not say the figures came from one. + assert!(!text.contains("estimated from the GGUF header"), "{text}"); + assert!(!text.contains("rocm serve"), "{text}"); + // A listed name the parser would not read back as that hub file (a shell + // expansion, a space) is not printed as a command. + for unsafe_name in ["x$(id).gguf", "my model.gguf"] { + let odd = refused(&[unsafe_name]); + assert_eq!( + odd.fix.as_ref().map(|fix| fix.verify.as_str()), + Some(""), + "{unsafe_name}" + ); + } + // Several models: no single command to name, so none is given. + let several = refused(&["vision-7b-Q4_K_M.gguf", "vision-7b-Q8_0.gguf"]); + assert_eq!( + several.fix.as_ref().map(|fix| fix.verify.as_str()), + Some("") + ); + // A projector-like name that failed for another reason (an unreachable + // hub) is not blamed on the name: it keeps the unreadable answer. + let mut unreachable = not_curated("some-lab/vision-7b:mmproj-vision-7b-f16.gguf"); + attach_context_estimate( + &mut unreachable, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &FakeSource::failing(), + ); + assert_eq!( + unreachable + .estimate + .and_then(|estimate| estimate.cannot_estimate_reason), + Some(CannotEstimateReason::HeaderUnreadable) + ); + assert!( + unreachable + .fix + .is_some_and(|fix| !fix.summary.contains("multimodal projector")), + ); + } + + /// A sized estimate that could not be compared keeps a next step only where + /// one exists, the one a curated model gets: memory that could not be read + /// is looked into with `amd-smi`, no visible GPU with `rocm diagnose`; an + /// APU has no command that would change the answer. + #[test] + fn a_sized_estimate_that_could_not_compare_keeps_a_plan_only_when_one_exists() { + for (memory, expected) in [ + (AcceleratorMemory::Unknown, Some("amd-smi metric --json")), + (AcceleratorMemory::None, Some("rocm diagnose")), + (AcceleratorMemory::UnifiedMemoryUnreadable, None), + ] { + let host = HostFacts { + accelerator_memory: memory, + system_ram_gib: Some(64.0), + }; + let registry = builtin_model_recipe_registry(); + let mut readiness = assess_model_readiness( + "some-lab/never-curated-7b:Q4_K_M", + &ModelCatalogSource::Available(®istry), + &host, + &engine_choice, + ); + attach_context_estimate( + &mut readiness, + &ContextRequest { + context_size: Some(32_768), + kv_cache_type: None, + }, + &host, + &FakeSource::weights(64), + ); + let estimate = readiness.estimate.as_ref().expect("an estimate"); + assert!( + estimate.total_gib.is_some(), + "{memory:?}: the header was sized" + ); + let commands = readiness + .fix + .as_ref() + .map(|fix| fix.commands.clone()) + .unwrap_or_default(); + match expected { + Some(command) => { + assert_eq!(commands, vec![command.to_owned()], "{memory:?}"); + assert_eq!( + readiness.fix.as_ref().map(|fix| fix.verify.as_str()), + Some( + "rocm diagnose --model some-lab/never-curated-7b:Q4_K_M \ + --context-size 32768" + ), + "{memory:?}" + ); + } + None => assert!(readiness.fix.is_none(), "{memory:?}: {:?}", readiness.fix), + } + } + } + + #[test] + fn a_gguf_outside_the_catalog_gets_an_estimate_and_stays_undetermined() { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + // The catalog-miss answer offers curated models that would run here; an + // estimate that says the model fits has no use for them. + assert!( + !readiness.alternatives.is_empty(), + "the catalog-miss answer carries curated alternatives to clear" + ); + let source = FakeSource::weights(2048); + attach_context_estimate( + &mut readiness, + &asked(8192), + &host_with(24.0, 64.0), + &source, + ); + let estimate = readiness.estimate.as_ref().expect("no estimate attached"); + assert_eq!(estimate.outcome, EstimateOutcome::Fits); + assert!( + readiness.alternatives.is_empty(), + "ESTIMATED: fits offered alternatives: {:?}", + readiness.alternatives + ); + assert!(!render_model_readiness_text(&readiness).contains("what would run here instead"),); + assert_eq!(estimate.context_size, 8192); + // The closed-set verdict is not widened: an estimate is not a recipe. + assert_eq!(readiness.verdict, ModelVerdict::Undetermined); + assert_eq!( + readiness.undetermined_reason, + Some(UndeterminedReason::ModelNotCurated) + ); + assert!( + readiness.canonical_model_id.is_none() && readiness.required_gpu_memory_gib.is_none() + ); + assert!(readiness.fix.is_none() && readiness.route.is_none()); + assert_eq!(source.calls(), 1); + let text = render_model_readiness_text(&readiness); + assert!( + text.starts_with("rocm diagnose --model some-lab/never-curated-7b:Q4_K_M: ESTIMATED: fits at 8192 tokens"), + "{text}" + ); + assert!( + !text.contains("READY") && !text.contains("BLOCKED"), + "{text}" + ); + // Labelled `note:`, as a figure's caveat, not `recipe note:`: there is no + // recipe behind an estimate. + assert!( + text.lines() + .any(|line| line.starts_with(" note: these are estimates")), + "{text}" + ); + assert!(!text.contains("recipe note"), "{text}"); + } + + #[test] + fn a_gguf_that_does_not_fit_says_what_would_make_it_fit() { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + // 2 GiB of weights, 1 GiB of cache at 8192, ~1.4 GiB for the compute + // buffer and runtime: 4.4 GiB against 4, with room for ~5000 tokens. + let host = host_with(4.0, 64.0); + attach_context_estimate( + &mut readiness, + &asked(8192), + &host, + &FakeSource::weights(2048), + ); + let estimate = readiness.estimate.as_ref().unwrap(); + assert_eq!(estimate.outcome, EstimateOutcome::DoesNotFit); + // A model that does not fit keeps the curated models that would run here: + // they answer the question the user is left with. + assert!( + !readiness.alternatives.is_empty(), + "ESTIMATED: does not fit dropped the alternatives" + ); + let kinds: Vec<_> = estimate.suggestions.iter().map(|s| s.kind).collect(); + assert!(kinds.contains(&SuggestionKind::SmallerContext), "{kinds:?}"); + assert!(kinds.contains(&SuggestionKind::KvCacheType), "{kinds:?}"); + let text = render_model_readiness_text(&readiness); + assert!( + text.contains("ESTIMATED: does not fit at 8192 tokens"), + "{text}" + ); + assert!(text.contains("what would make it fit:"), "{text}"); + // Nothing printed as a command: `rocm serve` takes none of these options, + // and a runnable-looking line for them would be false. + assert!(!text.contains(" $ "), "{text}"); + assert!(!text.contains("rocm serve some-lab"), "{text}"); + assert!(!text.contains("rocm serve --"), "{text}"); + assert!( + text.contains("`rocm serve` does not take a context size"), + "{text}" + ); + } + + #[test] + fn a_bare_repository_is_answered_as_before_and_read_nowhere() { + // `owner/repo` with no quantization names a repository, not a file. With + // no context flag the answer is exactly the plain "not in the catalog" + // one, reached without touching the network or the disk. + let plain = not_curated("some-lab/never-curated-7b"); + let mut readiness = plain.clone(); + let source = FakeSource::weights(64); + attach_context_estimate( + &mut readiness, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &source, + ); + assert_eq!(readiness, plain); + assert_eq!(source.calls(), 0); + assert!(readiness.estimate.is_none() && readiness.route.is_some()); + // With a flag, the user is told what would make it answerable -- still + // without reading anything. + attach_context_estimate( + &mut readiness, + &asked(8192), + &host_with(24.0, 64.0), + &source, + ); + assert_eq!(source.calls(), 0); + assert!(readiness.estimate.is_none()); + let text = render_model_readiness_text(&readiness); + assert!( + text.contains("names a repository, not a model file: add `:QUANT`"), + "{text}" + ); + } + + /// The command a GGUF plan offers asks the same question again: the flags + /// that were passed come with it, and a path a shell would split is quoted. + #[test] + fn the_command_to_ask_again_keeps_the_flags_and_quotes_the_path() { + let mut readiness = not_curated("my models/nosuch.gguf"); + attach_context_estimate( + &mut readiness, + &ContextRequest { + context_size: Some(32_768), + kv_cache_type: Some(KvCacheType::Q8_0), + }, + &host_with(24.0, 64.0), + &FakeSource::failing(), + ); + assert_eq!( + readiness.fix.as_ref().expect("a plan").verify, + "rocm diagnose --model 'my models/nosuch.gguf' --context-size 32768 \ + --kv-cache-type q8_0" + ); + let plain = ContextRequest::default(); + assert_eq!( + rediagnose("o/r:Q4_K_M", &plain), + "rocm diagnose --model o/r:Q4_K_M" + ); + assert_eq!( + rediagnose_for("it's here.gguf", &plain, false), + r"rocm diagnose --model 'it'\''s here.gguf'" + ); + // Windows: a plain path is left as it is, so cmd.exe reads it too; + // one with a space is quoted for PowerShell, which doubles a `'`. + assert_eq!( + rediagnose_for(r"C:\models\m.gguf", &plain, true), + r"rocm diagnose --model C:\models\m.gguf" + ); + assert_eq!( + rediagnose_for(r"C:\my models\it's.gguf", &plain, true), + r"rocm diagnose --model 'C:\my models\it''s.gguf'" + ); + // A Windows short name keeps a path plain on either shell; a leading + // `~` is quoted, since a shell would expand it. + assert_eq!( + rediagnose_for(r"C:\Users\ADMINI~1\Temp\m.gguf", &plain, true), + r"rocm diagnose --model C:\Users\ADMINI~1\Temp\m.gguf" + ); + assert_eq!( + rediagnose_for("~/m.gguf", &plain, false), + "rocm diagnose --model '~/m.gguf'" + ); + // A backslash is not plain to a POSIX shell. + assert_eq!( + rediagnose_for(r"a\b.gguf", &plain, false), + r"rocm diagnose --model 'a\b.gguf'" + ); + // A reference that starts with `-` is joined to the flag. + assert_eq!( + rediagnose_for("-x.gguf", &plain, false), + "rocm diagnose --model=-x.gguf" + ); + assert_eq!( + rediagnose("$HOME/m.gguf", &plain), + "rocm diagnose --model '$HOME/m.gguf'" + ); + } + + /// A local file that was read but cannot be sized (here one shard of a + /// split model) is offered to `rocm serve`, which takes a local path, with + /// the path quoted for the shell where it needs to be. + #[test] + fn an_unsized_local_gguf_is_offered_to_rocm_serve_quoted() { + let mut readiness = not_curated("models/split.gguf"); + attach_context_estimate( + &mut readiness, + &asked(8192), + &host_with(24.0, 64.0), + &FakeSource::with(llama_like(10).u32("split.count", 3).file(0)), + ); + assert_eq!( + readiness.estimate.as_ref().unwrap().cannot_estimate_reason, + Some(CannotEstimateReason::SplitFile) + ); + let fix = readiness.fix.as_ref().expect("a plan"); + assert_eq!(fix.verify, "rocm serve models/split.gguf"); + let mut spaced = not_curated("my models/split.gguf"); + attach_context_estimate( + &mut spaced, + &asked(8192), + &host_with(24.0, 64.0), + &FakeSource::with(llama_like(10).u32("split.count", 3).file(0)), + ); + assert_eq!( + spaced.fix.as_ref().expect("a plan").verify, + "rocm serve 'my models/split.gguf'" + ); + } + + /// A context flag with a reference that is neither curated nor a GGUF says + /// the estimate is for GGUF models only, and reads nothing. + #[test] + fn a_context_flag_on_a_reference_that_is_not_a_gguf_says_why_it_was_not_applied() { + let mut readiness = not_curated("some-bare-model-name"); + let source = FakeSource::weights(64); + attach_context_estimate( + &mut readiness, + &asked(8192), + &host_with(24.0, 64.0), + &source, + ); + assert_eq!(source.calls(), 0); + assert!(readiness.estimate.is_none()); + assert!( + readiness + .evidence + .iter() + .any(|line| line.contains("estimated for GGUF models only")), + "{:?}", + readiness.evidence + ); + } + + /// A hub header that could not be fetched is not answered with `rocm serve`, + /// which would download the whole file to fail the same way: the plan is to + /// check the name and the network, and ask again. + #[test] + fn an_unreadable_hub_header_is_not_answered_with_a_download() { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &asked(8192), + &host_with(24.0, 64.0), + &FakeSource::failing(), + ); + let fix = readiness.fix.as_ref().expect("a plan"); + assert!(fix.summary.contains("no route to Hugging Face"), "{fix:?}"); + assert_eq!( + fix.verify, + "rocm diagnose --model some-lab/never-curated-7b:Q4_K_M --context-size 8192" + ); + assert!( + fix.notes.iter().any(|note| note.contains("local path")), + "{fix:?}" + ); + assert!(!render_model_readiness_text(&readiness).contains("rocm serve some-lab")); + } + + /// A pasted URL is read as a path and cannot be opened; the plan says which + /// form is read instead. + #[test] + fn a_url_is_told_which_form_is_read() { + let mut readiness = not_curated("https://huggingface.co/o/r/resolve/main/m.gguf"); + attach_context_estimate( + &mut readiness, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &FakeSource::failing(), + ); + let fix = readiness.fix.as_ref().expect("a plan"); + assert!( + fix.notes + .iter() + .any(|note| note.contains("`owner/repo:QUANT`")), + "{fix:?}" + ); + } + + /// A layout that gets figures and no verdict is sized and compared, so the + /// "serve it and see" plan and the tracker route no longer apply, and no + /// command would turn it into a verdict: there is no plan, and the detail + /// says to compare the total with the GPU's memory. + #[test] + fn a_layout_without_a_verdict_gets_its_figures_and_no_plan() { + for (layout, source) in [ + ( + "mixture-of-experts", + FakeSource::with(llama_like(10).u32("llama.expert_count", 8).file(64)), + ), + ( + "state-space", + FakeSource::with(llama_like(10).u32("llama.ssm.state_size", 16).file(64)), + ), + ] { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &asked(4096), + &host_with(24.0, 64.0), + &source, + ); + let estimate = readiness.estimate.as_ref().expect("an estimate"); + assert_eq!( + estimate.cannot_estimate_reason, + Some(CannotEstimateReason::UnmeasuredLayout), + "{layout}" + ); + assert!(estimate.total_gib.is_some(), "{layout}"); + assert!(readiness.fix.is_none(), "{layout}: {:?}", readiness.fix); + assert!(readiness.route.is_none(), "{layout}"); + let text = render_model_readiness_text(&readiness); + assert!( + text.contains("compare it with this GPU's"), + "{layout}: {text}" + ); + } + } + + #[test] + fn a_gguf_that_cannot_be_read_is_not_sent_to_the_upstream_tracker() { + // A typo'd path: the tracker cannot help with it, and the plan is to + // check the path. + let mut readiness = not_curated("models/nosuch.gguf"); + assert!( + readiness.route.is_some(), + "the catalog-miss route is the default" + ); + attach_context_estimate( + &mut readiness, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &FakeSource::failing(), + ); + let estimate = readiness.estimate.as_ref().unwrap(); + assert_eq!( + estimate.cannot_estimate_reason, + Some(CannotEstimateReason::HeaderUnreadable) + ); + assert!(readiness.route.is_none()); + // A typo'd local path is a path to check, not a model to "serve directly". + let fix = readiness.fix.as_ref().expect("a plan is still given"); + assert!(fix.summary.contains("check the path"), "{}", fix.summary); + assert!(fix.commands.is_empty()); + assert_eq!(fix.verify, "rocm diagnose --model models/nosuch.gguf"); + let text = render_model_readiness_text(&readiness); + assert!(!text.contains("report it:"), "{text}"); + assert!( + !text.contains("serve this model directly") && !text.contains("rocm serve"), + "{text}" + ); + } + + fn compared_estimate() -> ModelReadiness { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &asked(8192), + &host_with(24.0, 64.0), + &FakeSource::weights(2048), + ); + assert!(readiness.estimate.as_ref().unwrap().available_gib.is_some()); + readiness + } + + #[test] + fn a_mask_on_a_host_whose_gpus_differ_is_said_to_be_unverified() { + let notes = |readiness: &ModelReadiness| readiness.estimate.as_ref().unwrap().notes.clone(); + // Mask set, cards of two sizes: the figure may be for the other card. + let mut readiness = compared_estimate(); + note_unverified_mask(&mut readiness, true, &[24_576, 8192]); + assert!( + notes(&readiness).iter().any(|n| n == MASK_UNVERIFIED_NOTE), + "{:?}", + notes(&readiness) + ); + let text = render_model_readiness_text(&readiness); + assert!(text.contains("has not been verified"), "{text}"); + // Whichever order the rows come in. + let mut readiness = compared_estimate(); + note_unverified_mask(&mut readiness, true, &[8192, 24_576]); + assert!(notes(&readiness).iter().any(|n| n == MASK_UNVERIFIED_NOTE)); + // No mask: nothing to doubt. Same cards: whichever is picked, the figure is the same. + for (mask_set, totals) in [ + (false, vec![24_576_u64, 8192]), + (true, vec![24_576, 24_576]), + (true, vec![24_576]), + (true, vec![]), + ] { + let mut readiness = compared_estimate(); + note_unverified_mask(&mut readiness, mask_set, &totals); + assert!( + !notes(&readiness).iter().any(|n| n == MASK_UNVERIFIED_NOTE), + "{mask_set} {totals:?}" + ); + } + // An estimate that compared nothing has no figure to doubt. + let mut unmeasured = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut unmeasured, + &asked(8192), + &HostFacts { + accelerator_memory: AcceleratorMemory::Unknown, + system_ram_gib: Some(64.0), + }, + &FakeSource::weights(2048), + ); + note_unverified_mask(&mut unmeasured, true, &[24_576, 8192]); + assert!( + !notes(&unmeasured).iter().any(|n| n == MASK_UNVERIFIED_NOTE), + "{:?}", + notes(&unmeasured) + ); + } + + /// `owner/repo:dir/file.gguf` selects an exact file for the estimate; `rocm + /// serve` reads only `owner/repo:QUANT`, so a verify line naming it would fail. + /// Shown on a file that was read but could not be sized (one shard of a + /// split model), where serving it is still the plan. + #[test] + fn the_verify_command_is_one_rocm_serve_accepts() { + let verify = |model_ref: &str| { + let mut readiness = not_curated(model_ref); + attach_context_estimate( + &mut readiness, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &FakeSource::with(llama_like(10).u32("split.count", 3).file(64)), + ); + readiness + .fix + .expect("the plan stays for a file that could not be sized") + .verify + }; + assert_eq!( + verify("some-lab/never-curated-7b:Q4_K_M"), + "rocm serve some-lab/never-curated-7b:Q4_K_M" + ); + assert_eq!( + verify("some-lab/never-curated-7b:sub/model-Q4_K_M.gguf"), + "rocm diagnose --model some-lab/never-curated-7b:sub/model-Q4_K_M.gguf" + ); + } + + #[test] + fn the_default_context_is_labelled_as_a_default() { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &FakeSource::weights(64), + ); + let estimate = readiness.estimate.as_ref().unwrap(); + assert_eq!(estimate.context_size, 4096, "the default --help documents"); + assert!(estimate.context_size_defaulted); + assert_eq!(estimate.kv_cache_type, KvCacheType::F16); + let text = render_model_readiness_text(&readiness); + assert!( + text.contains("(the default; pass --context-size to change it)"), + "{text}" + ); + } + + /// The three states of host memory, for a model that would otherwise fit: + /// none produces a comparison, each says its own reason, and the figures + /// the file allows are still shown. + #[test] + fn host_memory_that_cannot_be_compared_never_yields_a_fits_or_blocked_answer() { + for (memory, reason) in [ + (AcceleratorMemory::None, CannotEstimateReason::NoGpu), + ( + AcceleratorMemory::Unknown, + CannotEstimateReason::AcceleratorMemoryUnknown, + ), + ( + AcceleratorMemory::UnifiedMemoryUnreadable, + CannotEstimateReason::UnifiedMemoryUnreadable, + ), + ] { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + let host = HostFacts { + accelerator_memory: memory, + system_ram_gib: Some(64.0), + }; + attach_context_estimate( + &mut readiness, + &asked(4096), + &host, + &FakeSource::weights(64), + ); + let estimate = readiness.estimate.as_ref().unwrap(); + assert_eq!( + estimate.outcome, + EstimateOutcome::CannotEstimate, + "{memory:?}" + ); + assert_eq!(estimate.cannot_estimate_reason, Some(reason), "{memory:?}"); + assert!( + estimate.total_gib.is_some(), + "{memory:?}: the need was not shown" + ); + assert_eq!(readiness.verdict, ModelVerdict::Undetermined); + // Sized, so the "serve it directly" plan no longer applies; any plan + // left is the host's own (see the test after this one). + assert!(readiness.route.is_none(), "{memory:?}"); + assert!( + readiness + .fix + .as_ref() + .is_none_or(|fix| !fix.summary.contains("serve this model directly")), + "{memory:?}: {:?}", + readiness.fix + ); + let text = render_model_readiness_text(&readiness); + assert!(text.contains(": UNDETERMINED"), "{memory:?}: {text}"); + assert!(!text.contains("ESTIMATED:"), "{memory:?}: {text}"); + assert!(text.contains("KV cache (f16)"), "{memory:?}: {text}"); + } + } + + #[test] + fn an_unreachable_or_unreadable_file_is_a_gap_in_the_estimate_not_a_verdict_on_the_model() { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &asked(4096), + &host_with(24.0, 64.0), + &FakeSource::failing(), + ); + let estimate = readiness.estimate.as_ref().unwrap(); + assert_eq!( + estimate.cannot_estimate_reason, + Some(CannotEstimateReason::HeaderUnreadable) + ); + assert!( + estimate + .detail + .as_deref() + .unwrap() + .contains("could not be reached") + ); + assert_eq!(estimate.total_gib, None); + // The plan is to act on that reason and ask again, not to serve a file + // whose header could not even be read. + let fix = readiness.fix.as_ref().expect("a plan"); + assert!(fix.summary.contains("act on the reason above"), "{fix:?}"); + assert!(fix.verify.starts_with("rocm diagnose --model "), "{fix:?}"); + + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &asked(4096), + &host_with(24.0, 64.0), + &FakeSource::with(b"not a gguf file at all".to_vec()), + ); + let estimate = readiness.estimate.as_ref().unwrap(); + assert_eq!( + estimate.cannot_estimate_reason, + Some(CannotEstimateReason::HeaderUnreadable) + ); + assert!( + estimate + .detail + .as_deref() + .unwrap() + .contains("not a GGUF file") + ); + } + + /// The plan's note was written for a model that is not a GGUF reference. For + /// one that is, and whose header could not be sized, it must not say so. + #[test] + fn the_plan_note_never_claims_a_gguf_is_not_one() { + let note_of = |source: &FakeSource| { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate(&mut readiness, &asked(4096), &host_with(24.0, 64.0), source); + let text = render_model_readiness_text(&readiness); + assert!(!text.contains("not a GGUF reference"), "{text}"); + readiness.fix.expect("the plan stays").notes.join("\n") + }; + // Unreadable (its own plan, about the name and the network), and read + // but not sizeable (a shard of a split model). + let unreadable = note_of(&FakeSource::failing()); + assert!(unreadable.contains("no token is sent"), "{unreadable}"); + let shard = note_of(&FakeSource::with( + llama_like(10).u32("split.count", 3).file(64), + )); + assert!(shard.contains("could not size this GGUF"), "{shard}"); + // Sized: there is no plan left to carry a note. + let mut sized = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut sized, + &asked(4096), + &host_with(24.0, 64.0), + &FakeSource::weights(64), + ); + assert!(sized.fix.is_none()); + // Without a context flag nothing is read and the original note stands: + // that reference is not asked about as a GGUF at all. + let mut plain = not_curated("lab/plain-name"); + attach_context_estimate( + &mut plain, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &FakeSource::failing(), + ); + assert!(plain.estimate.is_none()); + assert!( + plain + .fix + .unwrap() + .notes + .join("\n") + .contains("for one that is not a GGUF reference") + ); + } + + #[test] + fn flags_are_never_dropped_in_silence_when_the_catalog_could_not_be_read() { + let unreachable = || { + assess_model_readiness( + "/models/local.gguf", + &ModelCatalogSource::Unreachable { + detail: "no index".to_owned(), + }, + &host_with(24.0, 64.0), + &engine_choice, + ) + }; + let source = FakeSource::weights(64); + let mut readiness = unreachable(); + let before = readiness.evidence.clone(); + attach_context_estimate( + &mut readiness, + &asked(4096), + &host_with(24.0, 64.0), + &source, + ); + assert!(readiness.estimate.is_none()); + assert_eq!(source.calls(), 0, "which answer it is owed is not known"); + assert_eq!(readiness.evidence.len(), before.len() + 1); + assert!( + readiness + .evidence + .last() + .unwrap() + .contains("were not applied"), + "{:?}", + readiness.evidence + ); + assert_eq!( + readiness.undetermined_reason, + Some(UndeterminedReason::CatalogUnreachable) + ); + // No flag, no line: the answer is exactly what it was. + let mut readiness = unreachable(); + let before = readiness.clone(); + attach_context_estimate( + &mut readiness, + &ContextRequest::default(), + &host_with(24.0, 64.0), + &source, + ); + assert_eq!(readiness, before); + } + + /// A compared estimate says once that `rocm serve` would not use the size it + /// was sized for: with the suggestions' options when it does not fit, on its + /// own when it fits; an estimate that compared nothing says neither. + #[test] + fn a_compared_estimate_says_rocm_serve_passes_no_context() { + let warnings = |memory_gib: f64| { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &asked(8192), + &host_with(memory_gib, 64.0), + &FakeSource::weights(2048), + ); + readiness.warnings + }; + let fits = warnings(24.0); + assert!(fits.iter().any(|w| w == SERVE_CONTEXT_NOTE), "{fits:?}"); + assert!(!fits.iter().any(|w| w == SERVE_OPTIONS_NOTE), "{fits:?}"); + let misfit = warnings(2.0); + assert!(misfit.iter().any(|w| w == SERVE_OPTIONS_NOTE), "{misfit:?}"); + assert!( + !misfit.iter().any(|w| w == SERVE_CONTEXT_NOTE), + "{misfit:?}" + ); + let mut uncompared = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut uncompared, + &asked(8192), + &HostFacts { + accelerator_memory: AcceleratorMemory::Unknown, + system_ram_gib: None, + }, + &FakeSource::weights(2048), + ); + assert!( + !uncompared + .warnings + .iter() + .any(|w| w == SERVE_CONTEXT_NOTE || w == SERVE_OPTIONS_NOTE) + ); + } + + #[test] + fn a_one_token_context_reads_as_one_token() { + let mut readiness = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut readiness, + &asked(1), + &host_with(24.0, 64.0), + &FakeSource::weights(64), + ); + let text = render_model_readiness_text(&readiness); + assert!( + text.contains(": ESTIMATED: fits at 1 token (calibrated"), + "{text}" + ); + assert!(!text.contains("1 tokens"), "{text}"); + } + + #[test] + fn only_a_model_outside_the_catalog_that_names_a_gguf_reaches_the_source() { + let source = FakeSource::weights(64); + let host = host_with(24.0, 64.0); + // A curated model: no network, no estimate; and an explicit context + // flag is answered with why it does not apply, not silently ignored. + let mut curated = not_curated(SMALL_MODEL_REF); + attach_context_estimate(&mut curated, &asked(8192), &host, &source); + assert!(curated.estimate.is_none()); + assert!( + curated + .evidence + .iter() + .any(|line| line.contains("do not apply to a curated recipe")), + "{:?}", + curated.evidence + ); + let mut curated = not_curated(SMALL_MODEL_REF); + let before = curated.evidence.clone(); + attach_context_estimate(&mut curated, &ContextRequest::default(), &host, &source); + assert_eq!( + curated.evidence, before, + "an unasked curated answer must not change" + ); + // A name that is not a GGUF reference. + let mut other = not_curated("never-curated"); + let before = other.evidence.len(); + attach_context_estimate(&mut other, &ContextRequest::default(), &host, &source); + assert!(other.estimate.is_none() && other.evidence.len() == before); + attach_context_estimate(&mut other, &asked(4096), &host, &source); + assert!(other.estimate.is_none()); + assert!( + other + .evidence + .iter() + .any(|line| line.contains("GGUF models only")) + ); + // A reference of the right shape, but the catalog could not be read: + // the answer is about the catalog, and an estimate would contradict it. + let mut unreachable = assess_model_readiness( + "some-lab/never-curated-7b:Q4_K_M", + &ModelCatalogSource::Unreachable { + detail: "gone".to_owned(), + }, + &host, + &engine_choice, + ); + attach_context_estimate(&mut unreachable, &asked(4096), &host, &source); + assert!(unreachable.estimate.is_none()); + assert_eq!( + source.calls(), + 0, + "the source was reached for a model that did not qualify" + ); + } + + #[test] + fn an_answer_without_an_estimate_serializes_exactly_as_it_did_before() { + let readiness = not_curated("some-lab/never-curated-7b"); + let json = serde_json::to_value(&readiness).unwrap(); + assert!(json.get("estimate").is_none(), "{json}"); + // And a JSON answer from before the field existed still reads. + let round_trip: ModelReadiness = serde_json::from_value(json).unwrap(); + assert_eq!(round_trip, readiness); + let mut with = not_curated("some-lab/never-curated-7b:Q4_K_M"); + attach_context_estimate( + &mut with, + &asked(4096), + &host_with(24.0, 64.0), + &FakeSource::weights(64), + ); + let json = serde_json::to_value(&with).unwrap(); + assert_eq!(json["estimate"]["outcome"], "fits"); + assert_eq!(json["estimate"]["kv_cache_type"], "f16"); + assert_eq!(json["verdict"], "undetermined"); + } } diff --git a/crates/rocm-core/src/net.rs b/crates/rocm-core/src/net.rs index 2812c6d961..ca623360ff 100644 --- a/crates/rocm-core/src/net.rs +++ b/crates/rocm-core/src/net.rs @@ -67,6 +67,24 @@ pub fn http_get_text_with_auth( endpoint_api_key: Option<&str>, timeout: Duration, ) -> Result { + let response = http_get_with_auth(endpoint_url, path, endpoint_api_key, timeout)?; + if response.status != 200 { + bail!("HTTP endpoint returned status {}", response.status); + } + Ok(response.body) +} + +/// GET `path` and return the status and body, whatever the status. +/// +/// The GET sibling of [`http_post_json_with_auth`]: a caller that relays the +/// endpoint's answer (a proxy) must tell "answered with a refusal" apart from +/// "never answered", which [`http_get_text_with_auth`] folds into one error. +pub fn http_get_with_auth( + endpoint_url: &str, + path: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { let deadline = Instant::now() + timeout; let (host, port) = parse_http_endpoint(endpoint_url) .with_context(|| format!("unsupported endpoint URL `{endpoint_url}`"))?; @@ -87,10 +105,12 @@ pub fn http_get_text_with_auth( .split_once("\r\n\r\n") .context("HTTP response was missing a body")?; let status_line = headers.lines().next().unwrap_or_default(); - if !status_line.contains(" 200 ") { - bail!("HTTP endpoint returned {status_line}"); - } - Ok(body.to_owned()) + let status = http_status_code(status_line) + .with_context(|| format!("unparsable HTTP status line `{status_line}`"))?; + Ok(HttpResponseParts { + status, + body: body.to_owned(), + }) } /// POST a JSON body and return the response status line plus body. diff --git a/crates/rocm-core/src/openmpi.rs b/crates/rocm-core/src/openmpi.rs index 44b6a8843f..2f45df2da2 100644 --- a/crates/rocm-core/src/openmpi.rs +++ b/crates/rocm-core/src/openmpi.rs @@ -707,17 +707,6 @@ pub fn build_libnuma_install_plan(os_id: &str, id_like: &str) -> SystemPackageIn } } -/// The real host's `/etc/os-release`, falling back to `/usr/lib/os-release` -/// when the former is absent (per `os-release(5)`). -/// -/// Not routed through [`crate::host_path`]: the hints name package-manager -/// commands for the real machine, so a simulated host root must not choose -/// their distribution. -#[cfg(target_os = "linux")] -fn real_os_release() -> crate::hardware_root::OsRelease { - crate::hardware_root::read_os_release(crate::hardware_root::identity_path_for_str) -} - const OPENMPI_MANUAL_HINT: &str = "install your distribution's OpenMPI runtime package (providing libmpi.so / libmpi_cxx.so and mpirun)"; const LIBATOMIC_MANUAL_HINT: &str = "install your distribution's libatomic runtime package (providing libatomic.so.1)"; @@ -738,7 +727,7 @@ pub fn install_hint() -> InstallHint { #[cfg(target_os = "linux")] { install_hint_for_read( - &real_os_release(), + &crate::hardware_root::real_os_release(), build_openmpi_install_plan, OPENMPI_MANUAL_HINT, ) @@ -765,7 +754,7 @@ pub fn libatomic_install_hint() -> InstallHint { #[cfg(target_os = "linux")] { install_hint_for_read( - &real_os_release(), + &crate::hardware_root::real_os_release(), build_libatomic_install_plan, LIBATOMIC_MANUAL_HINT, ) @@ -792,7 +781,7 @@ pub fn libnuma_install_hint() -> InstallHint { #[cfg(target_os = "linux")] { install_hint_for_read( - &real_os_release(), + &crate::hardware_root::real_os_release(), build_libnuma_install_plan, LIBNUMA_MANUAL_HINT, ) @@ -1012,10 +1001,9 @@ mod tests { } /// `install_hint_for_read`/`plan_for_os_release_read` are the one seam - /// `openmpi::real_os_release()` and `system_packages::real_os_release()` - /// hand their result to — both now one-line delegations to - /// `hardware_root::read_os_release`, so reverting either back to a raw - /// `/etc/os-release` read is a type error here, not a silent regression. + /// `hardware_root::real_os_release()` hands its result to, from here and + /// from every caller in the `rocm` binary, so reverting any of them back to + /// a raw `/etc/os-release` read is a type error, not a silent regression. /// This still pins the behaviour at the seam itself: an `OsRelease` that /// says the fallback path was read produces the same plan and hint a /// primary-path read with the same content would. diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 15840e56e9..ee4a6f1a07 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -10,6 +10,7 @@ use anyhow::{Context, Result}; use serde::Serialize; +use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; @@ -34,20 +35,64 @@ pub struct AppPaths { pub cache_dir: PathBuf, } +/// The variable that moves where engine virtualenvs live (see +/// [`AppPaths::engine_envs_root`]), and the one `rocm` sets on an engine child +/// that must install into a particular root. +pub const ENGINE_ENVS_ROOT_ENV: &str = "ROCM_CLI_ENGINE_ENVS_ROOT"; + +/// The variable that moves the config root. +pub const CONFIG_DIR_ENV: &str = "ROCM_CLI_CONFIG_DIR"; +/// The variable that moves the data root (and, unless it is set too, the cache +/// root that follows it). +pub const DATA_DIR_ENV: &str = "ROCM_CLI_DATA_DIR"; +/// The variable that moves the cache root. +pub const CACHE_DIR_ENV: &str = "ROCM_CLI_CACHE_DIR"; + impl AppPaths { + /// The three `ROCM_CLI_*_DIR` variables that make a child process run on + /// these roots, in the order config, data, cache. [`Self::discover`] reads + /// the same three back, so a child started with these finds the roots its + /// parent was given. + pub fn env_vars(&self) -> [(&'static str, &Path); 3] { + [ + (CONFIG_DIR_ENV, self.config_dir.as_path()), + (DATA_DIR_ENV, self.data_dir.as_path()), + (CACHE_DIR_ENV, self.cache_dir.as_path()), + ] + } + + /// Set [`Self::env_vars`] on `command`, so the child it starts runs on these + /// roots. + pub fn apply_env(&self, command: &mut std::process::Command) { + for (key, value) in self.env_vars() { + command.env(key, value); + } + } + + /// Resolve the config, data and cache dirs from the environment and + /// `config.json`. + /// + /// Errors when no data dir is given through `ROCM_CLI_DATA_DIR` and + /// `config.json` exists but cannot be read: the managed root it records + /// decides the data dir, so guessing the default would point every command + /// at the wrong runtimes. Startup logging and `rocm diagnose` treat an error + /// here as "no paths". Engines started by `rocm` are handed + /// `ROCM_CLI_DATA_DIR` and so never reach it; an engine started by hand + /// does, and the AIM engine then keeps its state file in the working + /// directory, as it already did when no data dir could be found. pub fn discover() -> Result { - let data_dir_override = env_path_override("ROCM_CLI_DATA_DIR"); - let cache_dir_override = env_path_override("ROCM_CLI_CACHE_DIR"); + let data_dir_override = env_path_override(DATA_DIR_ENV); + let cache_dir_override = env_path_override(CACHE_DIR_ENV); let paths = Self::from_overrides(PathOverrides { - config: env_path_override("ROCM_CLI_CONFIG_DIR"), + config: env_path_override(CONFIG_DIR_ENV), data: data_dir_override.clone(), cache: cache_dir_override.clone(), })?; - Ok(Self::discover_from_paths( + Self::discover_from_paths( paths, data_dir_override.is_some(), cache_dir_override.is_some(), - )) + ) } /// The three roots from their `ROCM_CLI_*_DIR` overrides, before any @@ -85,17 +130,20 @@ impl AppPaths { .normalize_for_host()) } + /// Fails when `ROCM_CLI_DATA_DIR` is unset and `config.json` exists but + /// cannot be read, because the data dir it records is then unknown; see + /// `configured_managed_root_from_config`. fn discover_from_paths( mut paths: Self, data_dir_overridden: bool, cache_dir_overridden: bool, - ) -> Self { + ) -> Result { if !data_dir_overridden - && let Some(managed_root) = configured_managed_root_from_config(&paths) + && let Some(managed_root) = configured_managed_root_from_config(&paths)? { paths = paths.with_managed_root(managed_root, cache_dir_overridden); } - paths.normalize_for_host() + Ok(paths.normalize_for_host()) } fn normalize_for_host(mut self) -> Self { @@ -159,7 +207,7 @@ impl AppPaths { /// dropped; `engine_envs_dir_reads_its_root_from_the_environment` is what /// keeps the lookup honest meanwhile. pub fn engine_envs_root(&self) -> PathBuf { - self.engine_envs_root_from(env_path_override("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref()) + self.engine_envs_root_from(env_path_override(ENGINE_ENVS_ROOT_ENV).as_deref()) } /// [`Self::engine_envs_root`] against a caller-supplied override. @@ -194,6 +242,21 @@ impl AppPaths { self.engine_dir(engine).join("manifests") } + /// The manifest an engine's installer writes for its installed runtime. + /// + /// One definition for the writer (the engine) and for every reader of the + /// file, so a reader cannot drift to a different name: a reader that + /// misses it degrades to "no manifest", which is indistinguishable from an + /// engine that was never installed. + /// + /// This is the engine's own install manifest, not an entry in the runtime + /// registry (that is `apps/rocm`'s `runtime_manifest_path`), and the name keeps + /// the two apart: the registry has a text guard that treats any file spelling + /// that helper's name and writing a file as a second registry writer. + pub fn engine_install_manifest_path(&self, engine: &str) -> PathBuf { + self.engine_manifests_dir(engine).join("runtime.json") + } + pub fn engine_state_dir(&self, engine: &str) -> PathBuf { self.engine_dir(engine).join("state") } @@ -318,16 +381,213 @@ impl AppPaths { } } -fn configured_managed_root_from_config(paths: &AppPaths) -> Option { - let bytes = fs::read(paths.config_path()).ok()?; - let value = serde_json::from_slice::(&bytes).ok()?; - value - .get("setup")? - .get("therock_venv")? - .as_str() - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(PathBuf::from) +/// The managed root recorded in `config.json` (`setup.therock_venv`), if any. +/// +/// Agrees with [`crate::RocmCliConfig::load`] that a missing file means "nothing +/// configured" and an unreadable one is an error. Treating an unreadable file +/// as "nothing configured" would quietly move the data dir — the runtime +/// registry, the active-runtime marker, service records — back to the default, +/// so a `--prefix` install's runtimes would vanish and new ones would land in +/// the wrong place. +/// +/// Only what this one field needs is checked: a file that is valid JSON but +/// that `load` rejects for some other field still yields its recorded root. +/// That keeps the data dir right, and leaves the complaint about the other +/// field to `load`, which names it. +fn configured_managed_root_from_config(paths: &AppPaths) -> Result> { + let path = paths.config_path(); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(unreadable_config_error(&path, &error.to_string(), None)), + }; + let unreadable = |detail: String| { + unreadable_config_error(&path, &detail, recorded_root_in_damaged_config(&bytes)) + }; + // One reading for every file: a file `load` accepts holds `setup` as an + // object (or not at all) and `therock_venv` as a string (or null), and this + // reads those exactly as `load` does, so a typed parse first would only + // repeat the work. + let value = serde_json::from_slice::(&bytes) + .map_err(|error| unreadable(format!("not valid JSON: {error}")))?; + let Some(top) = value.as_object() else { + return Err(unreadable(format!("expected a JSON object, found {value}"))); + }; + let setup = match top.get("setup") { + None | Some(serde_json::Value::Null) => return Ok(None), + Some(serde_json::Value::Object(setup)) => setup, + Some(other) => { + return Err(unreadable(format!( + "setup should be an object, found {other}" + ))); + } + }; + match setup.get("therock_venv") { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::String(text)) => Ok(Some(text.trim()) + .filter(|text| !text.is_empty()) + .map(PathBuf::from)), + Some(other) => Err(unreadable(format!( + "setup.therock_venv should be a path, found {other}" + ))), + } +} + +/// The error for a `config.json` that discovery cannot read, with recovery +/// advice that is true of what each way out actually does: +/// +/// - repairing the file restores everything; +/// - `ROCM_CLI_DATA_DIR` only tells discovery where the data is. The file +/// stays broken: every command that changes settings loads it strictly and +/// still stops, and of the read-only ones some stop too while the rest load +/// it leniently and run on default settings. None of the lenient ones saves +/// what it loaded, so none of them overwrites the file; +/// - renaming the file (the message names the exact path to rename and a name +/// to give it; nothing renames it for the user) resets every setting, and +/// puts the data dir back to the default, so runtimes in a custom folder +/// need `ROCM_CLI_DATA_DIR`. +/// +/// When the damaged bytes still name the recorded folder, the advice names the +/// exact `ROCM_CLI_DATA_DIR` value that finds it. Every path is quoted and the +/// variable is named in words rather than as `NAME=value`, so the advice reads +/// the same on Windows and Unix and survives a space in a path. +fn unreadable_config_error(path: &Path, detail: &str, recorded: Option) -> anyhow::Error { + let (data_dir_hint, aside_hint) = match recorded { + Some(recorded) => { + let data_root = managed_runtime_data_root(&recorded); + ( + format!( + "It still names {} as the ROCm folder: setting the environment variable \ + ROCM_CLI_DATA_DIR to {} points commands back at it meanwhile", + quoted(&recorded), + quoted(&data_root) + ), + runtimes_need_data_dir_override(&data_root), + ) + } + None => ( + "Setting the environment variable ROCM_CLI_DATA_DIR to the folder that holds your \ + runtimes points commands back at it meanwhile" + .to_owned(), + "runtimes installed into a custom folder then need the environment variable \ + ROCM_CLI_DATA_DIR set to that folder" + .to_owned(), + ), + }; + anyhow::anyhow!( + "cannot tell where rocm-cli keeps its data: {} cannot be read ({detail}). Repair that \ + file to restore your settings. {data_hint}. Until the file is repaired, any command \ + that changes settings still stops on it, and other commands either stop too or run on \ + default settings. Renaming it to {renamed} resets every setting to its default, and \ + {aside_hint}", + quoted(path), + renamed = quoted(&config_rename_target(path)), + data_hint = data_dir_hint, + ) +} + +/// The data dir discovery settles on when `config.json` records no managed +/// root: `ROCM_CLI_DATA_DIR`, else the default. One function, so the advice +/// about renaming `config.json` cannot disagree with discovery. +fn data_dir_without_config() -> Option { + env_path_override("ROCM_CLI_DATA_DIR").or_else(default_data_dir) +} + +/// The clause naming the `ROCM_CLI_DATA_DIR` value that keeps `data_root` +/// once `config.json` no longer records it. +fn runtimes_need_data_dir_override(data_root: &Path) -> String { + // The variable is named in words, not as `NAME=value`, which is one + // shell's syntax and not a command on Windows. + format!( + "the runtimes under {} then also need the environment variable ROCM_CLI_DATA_DIR set to {}", + quoted(data_root), + quoted(data_root) + ) +} + +/// `path` in double quotes, so a path with a space is still one value when +/// the advice is read or copied. +fn quoted(path: &Path) -> String { + format!("\"{}\"", path.display()) +} + +impl AppPaths { + /// How to start from default settings by renaming `config.json`, and what + /// that does to these paths, for a command + /// that found its data dir through the file but cannot load its settings + /// (discovery succeeded, `load` rejected another field). + /// + /// Every setting goes back to its default. When the data dir came from + /// the `setup.therock_venv` the file records, it goes back to the default + /// too, so the advice names the `ROCM_CLI_DATA_DIR` value that keeps it. + #[must_use] + pub fn config_aside_advice(&self) -> String { + let without_config = + data_dir_without_config().map(|dir| normalize_runtime_path_for_host(&dir)); + config_aside_advice( + &self.config_path(), + &self.data_dir, + without_config.as_deref(), + ) + } +} + +/// The name the recovery advice tells the user to give an unreadable +/// `config.json`: a sibling, so it stays next to the original and is never +/// read as settings again. +fn config_rename_target(config_path: &Path) -> PathBuf { + let mut name = config_path + .file_name() + .map_or_else(|| OsString::from("config.json"), ToOwned::to_owned); + name.push(".broken"); + config_path.with_file_name(name) +} + +/// [`AppPaths::config_aside_advice`], with the data dir discovery falls back +/// to when there is no `config.json` passed in rather than read from the +/// environment. +fn config_aside_advice( + config_path: &Path, + data_dir: &Path, + without_config: Option<&Path>, +) -> String { + let rename = format!( + "To start from default settings instead, rename {} to {}", + quoted(config_path), + quoted(&config_rename_target(config_path)) + ); + if without_config == Some(data_dir) { + rename + } else { + format!("{rename}; {}", runtimes_need_data_dir_override(data_dir)) + } +} + +/// Best-effort recovery of `setup.therock_venv` from a `config.json` that does +/// not parse — typically one cut short. Finds the key and reads the JSON string +/// after it only if that string is complete; never guesses past a truncation. +fn recorded_root_in_damaged_config(bytes: &[u8]) -> Option { + let text = String::from_utf8_lossy(bytes); + let after_key = &text[text.find("\"therock_venv\"")? + "\"therock_venv\"".len()..]; + let after_colon = after_key.trim_start().strip_prefix(':')?.trim_start(); + if !after_colon.starts_with('"') { + return None; + } + let mut escaped = false; + for (index, ch) in after_colon.char_indices().skip(1) { + match ch { + _ if escaped => escaped = false, + '\\' => escaped = true, + '"' => { + let literal = &after_colon[..=index]; + let value = serde_json::from_str::(literal).ok()?; + let value = value.trim(); + return (!value.is_empty()).then(|| PathBuf::from(value)); + } + _ => {} + } + } + None } pub fn engine_plugin_dirs(paths: &AppPaths) -> Vec { @@ -337,12 +597,92 @@ pub fn engine_plugin_dirs(paths: &AppPaths) -> Vec { ] } +// Declared here rather than at the crate root so the property can reach the +// private, env-free `AppPaths::discover_from_paths`. +#[cfg(test)] +#[path = "persisted_paths_props.rs"] +mod persisted_paths_props; + #[cfg(test)] mod tests { use super::*; + use crate::RocmCliConfig; use crate::test_support::temp_app_paths; + /// `env_vars` and `discover` share one constant per name, so they cannot + /// disagree; the names themselves are what a child process and a user's + /// shell see, so they are pinned here. + #[test] + fn apply_env_sets_exactly_the_three_roots_on_a_command() { + let (_root, paths) = temp_app_paths("apply-env"); + let mut command = std::process::Command::new("true"); + paths.apply_env(&mut command); + let mut set: Vec<_> = command + .get_envs() + .map(|(key, value)| (key.to_owned(), value.map(std::ffi::OsStr::to_owned))) + .collect(); + let expected: Vec<_> = paths + .env_vars() + .iter() + .map(|(key, value)| ((*key).into(), Some(value.as_os_str().to_owned()))) + .collect(); + let mut expected = expected; + set.sort(); + expected.sort(); + assert_eq!(set, expected); + } + + #[test] + fn env_vars_pair_each_root_with_its_public_variable_name() { + let (_root, paths) = temp_app_paths("env-vars"); + assert_eq!( + paths.env_vars(), + [ + ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), + ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), + ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), + ] + ); + } + + /// For every `config.json` that `load` accepts, discovery reads the + /// recorded folder as `load` does: the saved `therock_venv`, trimmed, and + /// nothing when it is absent or blank. + #[test] + fn discovery_reads_the_recorded_folder_of_a_loadable_config_as_load_does() -> Result<()> { + let (root, paths) = temp_app_paths("discovery-agrees-with-load"); + for recorded in [ + None, + Some(""), + Some(" "), + Some("/data/rocm"), + Some(" /data/rocm "), + Some("/données/rocm"), + ] { + let config = RocmCliConfig { + setup: crate::SetupConfig { + therock_venv: recorded.map(PathBuf::from), + ..crate::SetupConfig::default() + }, + ..RocmCliConfig::default() + }; + config.save(&paths)?; + let loaded = RocmCliConfig::load(&paths)?; + let expected = loaded.setup.therock_venv.and_then(|root| { + let text = root.to_string_lossy().trim().to_owned(); + (!text.is_empty()).then(|| PathBuf::from(text)) + }); + assert_eq!( + configured_managed_root_from_config(&paths)?, + expected, + "recorded {recorded:?}" + ); + } + let _ = fs::remove_dir_all(&root); + Ok(()) + } + #[test] fn audit_events_path_lives_under_data_audit() { let (_root, paths) = temp_app_paths("audit-path"); @@ -355,6 +695,15 @@ mod tests { paths.data_dir.join("automations").join("proposals.jsonl") ); } + #[test] + fn engine_install_manifest_lives_in_the_engine_manifests_dir() { + let (_root, paths) = temp_app_paths("engine-runtime-manifest-path"); + assert_eq!( + paths.engine_install_manifest_path("lemonade"), + paths.engine_manifests_dir("lemonade").join("runtime.json") + ); + } + #[test] fn with_managed_root_keeps_reprovisioning_flat_from_runtime_leaf() { let data_root = PathBuf::from("/tmp/rocm-cli-reprovision"); @@ -402,22 +751,392 @@ mod tests { }))?, )?; - let discovered = AppPaths::discover_from_paths(paths.clone(), false, false); + let discovered = AppPaths::discover_from_paths(paths.clone(), false, false)?; assert_eq!(discovered.config_dir, paths.config_dir); assert_eq!(discovered.data_dir, managed_root); assert_eq!(discovered.cache_dir, managed_root.join("cache")); - let data_overridden = AppPaths::discover_from_paths(paths.clone(), true, false); + let data_overridden = AppPaths::discover_from_paths(paths.clone(), true, false)?; assert_eq!(data_overridden.data_dir, paths.data_dir); assert_eq!(data_overridden.cache_dir, paths.cache_dir); - let cache_overridden = AppPaths::discover_from_paths(paths.clone(), false, true); + let cache_overridden = AppPaths::discover_from_paths(paths.clone(), false, true)?; assert_eq!(cache_overridden.data_dir, managed_root); assert_eq!(cache_overridden.cache_dir, paths.cache_dir); fs::remove_dir_all(root).ok(); Ok(()) } + /// An unreadable `config.json` stops discovery with a message naming the + /// file and the ways out, and discovery leaves the file alone. Each way + /// out the message names is then shown to do what the message says it + /// does — including what it does *not* fix. + #[test] + fn app_paths_refuse_to_guess_the_data_dir_from_an_unreadable_config() -> Result<()> { + let (root, paths) = temp_app_paths("configured-managed-root-unreadable"); + let managed_root = root.join("managed"); + let mut config = RocmCliConfig::default(); + config.setup.therock_venv = Some(managed_root.clone()); + config.save(&paths)?; + let saved = fs::read(paths.config_path())?; + let healthy = AppPaths::discover_from_paths(paths.clone(), false, false)?; + assert_eq!(healthy.data_dir, managed_root); + + let recorded = serde_json::to_string(&managed_root)?; + let value_at = String::from_utf8_lossy(&saved) + .find(&recorded) + .expect("the saved config records the managed root"); + let cut_after_root = saved[..value_at + recorded.len() + 2].to_vec(); + let cut_inside_root = saved[..value_at + recorded.len() / 2].to_vec(); + let suggested = format!( + "setting the environment variable ROCM_CLI_DATA_DIR to \"{}\"", + managed_root.display() + ); + let rename_advice = format!( + "Renaming it to \"{}\" resets every setting", + paths.config_dir.join("config.json.broken").display() + ); + + for (label, damaged, names_the_root) in [ + ("cut short after the folder", cut_after_root, true), + ("cut short inside the folder", cut_inside_root, false), + ("not an object", br#""x""#.to_vec(), false), + ("setup retyped", br#"{"setup": "x"}"#.to_vec(), false), + ( + "therock_venv retyped", + br#"{"setup": {"therock_venv": 7}}"#.to_vec(), + false, + ), + ] { + fs::write(paths.config_path(), &damaged)?; + let error = AppPaths::discover_from_paths(paths.clone(), false, false) + .expect_err("discovery must not guess the data dir"); + let message = format!("{error:#}"); + for needle in [ + "cannot tell where rocm-cli keeps its data", + "config.json", + "Repair that file", + "any command that changes settings still stops on it", + "run on default settings", + rename_advice.as_str(), + ] { + assert!( + message.contains(needle), + "{label}: missing {needle:?} in: {message}" + ); + } + assert_eq!( + message.contains(&suggested), + names_the_root, + "{label}: whether the recorded folder is named: {message}" + ); + // What renaming does to the data dir, for the folder the file still + // names or, when it no longer names one, for any custom folder. + // That the data dir does go back to the default is the rename way + // out below. + let aside = if names_the_root { + format!( + "resets every setting to its default, and the runtimes under \"{root}\" \ + then also need the environment variable ROCM_CLI_DATA_DIR set to \"{root}\"", + root = managed_root.display() + ) + } else { + "resets every setting to its default, and runtimes installed into a custom \ + folder then need the environment variable ROCM_CLI_DATA_DIR set to that folder" + .to_owned() + }; + assert!( + message.ends_with(&aside), + "{label}: missing {aside:?} at the end of: {message}" + ); + assert_eq!( + fs::read(paths.config_path())?, + damaged, + "{label}: discovery changed config.json" + ); + + // ROCM_CLI_DATA_DIR set to the suggested folder finds the same data + // dir the healthy file did... + let overridden = AppPaths::discover_from_paths( + AppPaths { + data_dir: managed_root.clone(), + ..paths.clone() + }, + true, + false, + )?; + assert_eq!(overridden.data_dir, healthy.data_dir, "{label}"); + // ...but does not make the settings readable, as the message says. + assert!(RocmCliConfig::load(&overridden).is_err(), "{label}"); + } + + // A config.json that exists but cannot be read at all (here, a + // directory: a non-NotFound read error on every platform, no root or + // chmod needed) is an error too, not "nothing configured". + fs::remove_file(paths.config_path())?; + fs::create_dir(paths.config_path())?; + let error = AppPaths::discover_from_paths(paths.clone(), false, false) + .expect_err("an unreadable config.json must not read as absent"); + let message = format!("{error:#}"); + assert!( + message.contains("cannot tell where rocm-cli keeps its data") + && message.contains("Repair that file"), + "unexpected error: {message}" + ); + assert!(paths.config_path().is_dir(), "discovery touched the path"); + // `load` draws the same line: the directory is not "no settings yet". + assert!( + RocmCliConfig::load(&paths).is_err(), + "load read an unreadable config.json as absent" + ); + fs::remove_dir(paths.config_path())?; + + // Way out: repair the file. + fs::write(paths.config_path(), &saved)?; + let repaired = AppPaths::discover_from_paths(paths.clone(), false, false)?; + assert_eq!(repaired.data_dir, managed_root); + + // Way out: rename it to the name the message gives. Settings are back + // to defaults, and the data dir is back to the default too — which is + // why the message says a custom folder then needs ROCM_CLI_DATA_DIR. + fs::rename( + paths.config_path(), + paths.config_dir.join("config.json.broken"), + )?; + let moved_aside = AppPaths::discover_from_paths(paths.clone(), false, false)?; + assert_eq!(moved_aside.data_dir, paths.data_dir); + assert_ne!(moved_aside.data_dir, managed_root); + assert!( + RocmCliConfig::load(&moved_aside)? + .setup + .therock_venv + .is_none() + ); + fs::remove_file(paths.config_dir.join("config.json.broken"))?; + + // A file that is valid JSON but that `load` rejects for another field + // still names the right data dir; `load` reports that field itself. + let mut value = serde_json::from_slice::(&saved)?; + value["dashboard"]["tui"]["chat_top_p"] = serde_json::json!(1.5); + fs::write(paths.config_path(), serde_json::to_vec(&value)?)?; + assert!(RocmCliConfig::load(&paths).is_err()); + let other_field_bad = AppPaths::discover_from_paths(paths, false, false)?; + assert_eq!(other_field_bad.data_dir, managed_root); + + fs::remove_dir_all(root).ok(); + Ok(()) + } + /// The quoted value right after the first `label` in `text`: what a + /// reader copies when told to use the path the advice prints. + fn quoted_after<'a>(text: &'a str, label: &str) -> &'a str { + let rest = &text[text + .find(label) + .unwrap_or_else(|| panic!("no {label:?} in: {text}")) + + label.len()..]; + let rest = rest + .strip_prefix('"') + .unwrap_or_else(|| panic!("no quoted path after {label:?} in: {text}")); + &rest[..rest + .find('"') + .unwrap_or_else(|| panic!("an unclosed quote after {label:?} in: {text}"))] + } + + /// A command that found its data dir through `config.json` but cannot + /// load the settings in it (another field is bad) is told which file to + /// rename, to what, and what renaming it does. Following that advice to + /// the letter — the exact paths it prints — keeps the data dir, while + /// renaming the file without the override it names would not. Both the + /// config folder and the data folder have a space in their names, so a + /// path the advice did not quote would be read as two words. + #[test] + fn config_rename_advice_keeps_a_recorded_data_dir_when_followed() -> Result<()> { + let (root, paths) = temp_app_paths("config-aside-advice"); + let paths = AppPaths { + config_dir: root.join("my config"), + ..paths + }; + let managed_root = root.join("managed rocm"); + let mut config = RocmCliConfig::default(); + config.setup.therock_venv = Some(managed_root.clone()); + config.save(&paths)?; + let mut value = + serde_json::from_slice::(&fs::read(paths.config_path())?)?; + value["dashboard"]["tui"]["chat_top_p"] = serde_json::json!(1.5); + let damaged = serde_json::to_vec(&value)?; + fs::write(paths.config_path(), &damaged)?; + + let discovered = AppPaths::discover_from_paths(paths.clone(), false, false)?; + assert_eq!(discovered.data_dir, managed_root); + assert!(RocmCliConfig::load(&discovered).is_err()); + // `paths.data_dir` stands in for the default data dir. + let advice = config_aside_advice( + &discovered.config_path(), + &discovered.data_dir, + Some(&paths.data_dir), + ); + assert_eq!( + advice, + format!( + "To start from default settings instead, rename \"{config}\" to \ + \"{config}.broken\"; the runtimes under \"{dir}\" then also need the \ + environment variable ROCM_CLI_DATA_DIR set to \"{dir}\"", + config = paths.config_path().display(), + dir = managed_root.display() + ) + ); + // The file it names is the one discovery read — under the custom + // config folder these paths use, not a default location. Each path is + // taken from between its quotes, as a reader would copy it. + let from = PathBuf::from(quoted_after(&advice, "rename ")); + let to = PathBuf::from(quoted_after( + &advice, + &format!("\"{}\" to ", from.display()), + )); + let suggested = PathBuf::from(quoted_after(&advice, "ROCM_CLI_DATA_DIR set to ")); + assert_eq!(from, paths.config_path()); + assert_eq!(to, paths.config_dir.join("config.json.broken")); + assert_eq!(suggested, managed_root); + + fs::rename(&from, &to)?; + assert_eq!( + fs::read(&to)?, + damaged, + "the renamed file lost its contents" + ); + // Without the override the data dir moves back to the default... + let unadvised = AppPaths::discover_from_paths(paths.clone(), false, false)?; + assert_eq!(unadvised.data_dir, paths.data_dir); + // ...and with the value the advice printed it stays where it was, on + // default settings: the renamed file is not read as settings. + let advised = AppPaths::discover_from_paths( + AppPaths { + data_dir: suggested, + ..paths.clone() + }, + true, + false, + )?; + assert_eq!(advised.data_dir, discovered.data_dir); + assert_eq!( + RocmCliConfig::load(&advised)?.setup.therock_venv, + None, + "the renamed file was still read as settings" + ); + fs::remove_file(&to)?; + + // A file that records no folder: renaming it leaves the data dir + // alone, so the advice names no override. + fs::write( + paths.config_path(), + br#"{"dashboard": {"tui": {"chat_top_p": 1.5}}}"#, + )?; + let default_root = AppPaths::discover_from_paths(paths.clone(), false, false)?; + assert!(RocmCliConfig::load(&default_root).is_err()); + let advice = config_aside_advice( + &default_root.config_path(), + &default_root.data_dir, + Some(&paths.data_dir), + ); + assert_eq!( + advice, + format!( + "To start from default settings instead, rename \"{config}\" to \ + \"{config}.broken\"", + config = paths.config_path().display() + ) + ); + let from = PathBuf::from(quoted_after(&advice, "rename ")); + fs::rename( + &from, + quoted_after(&advice, &format!("\"{}\" to ", from.display())), + )?; + let renamed = AppPaths::discover_from_paths(paths, false, false)?; + assert_eq!(renamed.data_dir, default_root.data_dir); + assert!(RocmCliConfig::load(&renamed).is_ok()); + + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// The public method reads the same fallback discovery uses: paths whose + /// data dir is that fallback get no override advice, and any other data + /// dir is named as the override to set. Holds the data-dir lock because the + /// fallback reads `ROCM_CLI_DATA_DIR`, which other tests set. + #[test] + fn config_rename_advice_names_an_override_only_for_a_data_dir_discovery_would_lose() { + let _guard = crate::test_env::DATA_DIR_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (root, paths) = temp_app_paths("config-aside-advice-method"); + let rename = format!( + "To start from default settings instead, rename \"{config}\" to \"{config}.broken\"", + config = paths.config_path().display() + ); + let fallback = AppPaths { + data_dir: normalize_runtime_path_for_host( + &data_dir_without_config().expect("this host has a default data dir"), + ), + ..paths.clone() + }; + assert_eq!(fallback.config_aside_advice(), rename); + assert_eq!( + paths.config_aside_advice(), + format!( + "{rename}; the runtimes under \"{dir}\" then also need the environment variable \ + ROCM_CLI_DATA_DIR set to \"{dir}\"", + dir = paths.data_dir.display() + ) + ); + fs::remove_dir_all(root).ok(); + } + + /// A user who already set the `ROCM_CLI_DATA_DIR` the refusal suggested is + /// not told to set it again: with the variable naming the data dir, + /// renaming config.json leaves the data dir where it is, so the advice is + /// the rename alone. Drives the real environment read. + #[test] + fn config_rename_advice_names_no_override_once_rocm_cli_data_dir_is_set() { + let _guard = crate::test_env::DATA_DIR_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (root, paths) = temp_app_paths("config-aside-advice-env"); + let restore = crate::test_env::RestoredEnvVar::set("ROCM_CLI_DATA_DIR", &paths.data_dir); + let advice = paths.config_aside_advice(); + drop(restore); + fs::remove_dir_all(root).ok(); + assert_eq!( + advice, + format!( + "To start from default settings instead, rename \"{config}\" to \ + \"{config}.broken\"", + config = paths.config_path().display() + ) + ); + } + + /// The recorded folder is recovered from a damaged file only when its + /// string is complete: a value cut off part-way is never guessed at. + #[test] + fn recorded_root_is_recovered_only_from_a_complete_value() { + let escaped = br#"{"setup": {"therock_venv": "C:\\ROCm \"x\"\\venv", "completed"#; + assert_eq!( + recorded_root_in_damaged_config(escaped), + Some(PathBuf::from(r#"C:\ROCm "x"\venv"#)) + ); + for damaged in [ + &br#"{"setup": {"therock_venv": "/opt/ro"#[..], + br#"{"setup": {"therock_venv": 7"#, + br#"{"setup": {"therock_venv": " ", "#, + br#"{"setup": {"#, + ] { + assert_eq!( + recorded_root_in_damaged_config(damaged), + None, + "{}", + String::from_utf8_lossy(damaged) + ); + } + } #[test] fn cache_dir_follows_an_overridden_data_dir() -> Result<()> { let (root, _) = temp_app_paths("cache-follows-data"); @@ -473,11 +1192,11 @@ mod tests { cache: None, })?; assert_eq!( - configured_managed_root_from_config(&paths), + configured_managed_root_from_config(&paths)?, Some(managed_runtime), "the fixture must be a config the managed-root lookup reads" ); - let discovered = AppPaths::discover_from_paths(paths, true, false); + let discovered = AppPaths::discover_from_paths(paths, true, false)?; assert_eq!(discovered.data_dir, normalize_runtime_path_for_host(&data)); assert_eq!( discovered.cache_dir, diff --git a/crates/rocm-core/src/persisted_paths_props.rs b/crates/rocm-core/src/persisted_paths_props.rs new file mode 100644 index 0000000000..88ef899c22 --- /dev/null +++ b/crates/rocm-core/src/persisted_paths_props.rs @@ -0,0 +1,103 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property: the two readers of `config.json` agree about where the data dir +//! is. +//! +//! `RocmCliConfig::load` fails loudly on a file it cannot parse. +//! `AppPaths::discover` reads the same file a second way, through +//! `configured_managed_root_from_config`, for the one field that decides where +//! the runtime registry, the active-runtime marker and the service records +//! live. It used to turn any read or parse error into "no managed root", so a +//! config one reader called corrupt the other called "absent", and the data +//! dir moved to the default with no message. Discovery must now either fail +//! or still find the recorded root. +//! +//! No environment is read or written: `discover_from_paths` is the env-free +//! core of `discover`. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use proptest::prelude::*; +use proptest::test_runner::{Config as RunnerConfig, TestError, TestRunner}; + +use crate::{AppPaths, RocmCliConfig}; + +static NEXT_DIR: AtomicUsize = AtomicUsize::new(0); + +fn fresh_paths() -> (PathBuf, AppPaths) { + let root = std::env::temp_dir().join(format!( + "rocm-core-paths-props-{}-{}", + std::process::id(), + NEXT_DIR.fetch_add(1, Ordering::Relaxed) + )); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("default-data"), + cache_dir: root.join("default-cache"), + }; + (root, paths) +} + +#[test] +fn data_dir_never_silently_relocates_when_config_is_unreadable() { + let total = AtomicUsize::new(0); + let unreadable = AtomicUsize::new(0); + let mut runner = TestRunner::new(RunnerConfig { + cases: crate::test_support::proptest_cases(256), + ..RunnerConfig::default() + }); + let strategy = ("[a-z]{1,10}", 1_usize..2048); + let result = runner.run(&strategy, |(leaf, cut)| { + total.fetch_add(1, Ordering::Relaxed); + let (root, paths) = fresh_paths(); + let managed_root = root.join("prefix").join(&leaf); + let mut config = RocmCliConfig::default(); + config.setup.therock_venv = Some(managed_root.clone()); + config.active_runtime_key = Some("therock-release-gfx1151-7.13.0".to_owned()); + config.save(&paths).unwrap(); + let healthy = AppPaths::discover_from_paths(paths.clone(), false, false).unwrap(); + prop_assert_eq!(&healthy.data_dir, &managed_root); + + // A truncated file: a hand edit cut short, or one damaged before saves + // became atomic. + let bytes = std::fs::read(paths.config_path()).unwrap(); + std::fs::write(paths.config_path(), &bytes[..cut.min(bytes.len() - 1)]).unwrap(); + let strict = RocmCliConfig::load(&paths); + let discovered = AppPaths::discover_from_paths(paths, false, false); + let _ = std::fs::remove_dir_all(&root); + if strict.is_ok() { + return Ok(()); + } + unreadable.fetch_add(1, Ordering::Relaxed); + // Failing is the agreement with `load`; a data dir is only acceptable + // if it is still the recorded one. + if let Ok(discovered) = discovered { + prop_assert_eq!( + &discovered.data_dir, + &managed_root, + "config.json is unreadable to RocmCliConfig::load, yet AppPaths::discover \ + silently moved data_dir (registry, marker, services) to the default" + ); + } + Ok(()) + }); + eprintln!( + "reach: {} of {} cases left a config.json that load() rejects", + unreadable.load(Ordering::Relaxed), + total.load(Ordering::Relaxed) + ); + match result { + Ok(()) => {} + Err(TestError::Fail(reason, value)) => { + panic!("minimal failing input {value:?}\nreason: {reason}") + } + Err(TestError::Abort(reason)) => panic!("aborted: {reason}"), + } + assert!( + unreadable.load(Ordering::Relaxed) > 0, + "no case left a config.json that load() rejects" + ); +} diff --git a/crates/rocm-core/src/probe_exec.rs b/crates/rocm-core/src/probe_exec.rs index 86f807cfa4..e632612bd6 100644 --- a/crates/rocm-core/src/probe_exec.rs +++ b/crates/rocm-core/src/probe_exec.rs @@ -10,16 +10,17 @@ //! [`crate::examine`] so `fix`, `process` and `host_gpu` can reach them without //! importing the module that owns the `Examination` document. +use crate::command_capture::{CommandCapture, CommandOutcome}; +use crate::process::run_bounded; use crate::runtime_is_windows; -use std::ffi::OsString; -use std::io::Read; -use std::path::Path; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; - -/// Run a command with a timeout. Returns `(rc, stdout, stderr)`. `rc` is `127` -/// when the program can't be spawned and `124` on timeout. +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +/// Run a command with a timeout. Returns `(rc, stdout, stderr)`. `rc` is the +/// exit code, `-1` for a child killed by a signal, `127` when the program can't +/// be spawned and `124` on timeout or when the child could not be waited for. pub(crate) fn run(program: &str, args: &[&str], timeout: Duration) -> (i32, String, String) { run_with_env(program, args, &[], timeout) } @@ -40,66 +41,36 @@ pub(crate) fn run_with_env( } /// [`run`] without the UTF-8 assumption, for output that is not UTF-8. +/// +/// A thin adapter over [`run_bounded`]: the executor drains both pipes as bytes +/// (so one stray byte cannot empty a capture, which would read downstream as +/// "the command printed nothing") and bounds the wait, and [`run_result`] turns +/// how the run ended into the `rc` this module has always returned. fn run_raw( program: &str, args: &[&str], envs: &[(String, OsString)], timeout: Duration, ) -> (i32, Vec, Vec) { - let Ok(mut child) = Command::new(program) + let mut command = Command::new(program); + command .args(args) - .envs(envs.iter().map(|(key, value)| (key, value))) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - else { - return (127, Vec::new(), Vec::new()); - }; - // Bytes, then a lossy conversion at the end. `read_to_string` FAILS on - // invalid UTF-8 and the error was discarded, so a single stray byte silently - // emptied the whole capture — which reads downstream as "the command printed - // nothing", not as "the output could not be decoded". - let stdout_handle = child.stdout.take().map(|mut stdout| { - thread::spawn(move || { - let mut buf = Vec::new(); - let _ = stdout.read_to_end(&mut buf); - buf - }) - }); - let stderr_handle = child.stderr.take().map(|mut stderr| { - thread::spawn(move || { - let mut buf = Vec::new(); - let _ = stderr.read_to_end(&mut buf); - buf - }) - }); - let deadline = Instant::now() + timeout; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break Some(status), - Ok(None) => { - if Instant::now() >= deadline { - let _ = child.kill(); - let _ = child.wait(); - break None; - } - thread::sleep(Duration::from_millis(20)); - } - Err(_) => break None, - } - }; - let stdout: Vec = stdout_handle - .and_then(|handle| handle.join().ok()) - .unwrap_or_default(); - let stderr: Vec = stderr_handle - .and_then(|handle| handle.join().ok()) - .unwrap_or_default(); - let rc = match status { - Some(status) => status.code().unwrap_or(-1), - None => 124, + .envs(envs.iter().map(|(key, value)| (key, value))); + run_result(run_bounded(command, timeout, "process")) +} + +/// `(rc, stdout, stderr)` of a finished run: the exit code when there is one, +/// `-1` for a child killed by a signal, `124` for a timeout or a failure to +/// wait, and `127` for a program that could not be started. The bytes are what +/// arrived, a timeout and a pipe that failed to read included. +fn run_result(capture: CommandCapture) -> (i32, Vec, Vec) { + let rc = match capture.outcome { + CommandOutcome::Exited(code) => code, + CommandOutcome::Signaled(_) => -1, + CommandOutcome::TimedOut | CommandOutcome::WaitFailed => 124, + CommandOutcome::NotFound | CommandOutcome::SpawnFailed(_) => 127, }; - (rc, stdout, stderr) + (rc, capture.stdout_raw, capture.stderr_raw) } /// Run a command whose output is UTF-16LE, as `wsl.exe`'s is. @@ -134,27 +105,140 @@ pub(crate) fn read_host_text(path: &str) -> String { std::fs::read_to_string(crate::host_path(path)).unwrap_or_default() } -/// Whether `program` resolves on `PATH` (best-effort, no execution). +/// Whether `program` is installed, by `PATH` lookup plus a few fixed Windows +/// locations (best-effort, no execution). +/// +/// The one PATH lookup of the host probes: the examination stack's capability +/// checks and the summary stack's WSL facts both ask it, so a tool reads as +/// installed or not the same way in every part of `rocm examine`. It tries +/// each spelling [`tool_path_candidates`] gives, in each directory of `PATH`, +/// and answers `true` for the first one that is a file. +/// +/// - An empty `PATH` entry is skipped, not read as the current directory as a +/// shell or `execvp` would. A file named like the tool in the directory the +/// CLI was started from therefore does not read as the tool being installed +/// through an empty entry, which matters for the checks that gate a +/// privileged step (`sudo`, `usermod`). An explicit `.` or other relative +/// entry is an ordinary directory and is searched. The cost is that a +/// program the CLI later starts by bare name can still resolve from the +/// current directory when `PATH` has an empty entry, because the operating +/// system resolves it, not this lookup. +/// - A `PATH` that is not valid Unicode is searched: it is split as an +/// `OsString`, and the entries that are not Unicode are looked in as they are. +/// - The directories are split the way [`std::env::split_paths`] splits them on +/// this platform, so on Windows an entry may be quoted to hold a `;`. +/// - On Windows a program without an extension is tried with every `PATHEXT` +/// entry in both cases. A name that already has an extension (`python3.11`, +/// `wsl.exe`) is tried as given and nothing else; the examination's old copy +/// also tried `.exe`, `.bat` and `.cmd` after it, and no caller passes a +/// dotted name that relied on that (`wsl.exe` is found as given). +/// - On Windows `cmd`, `pnputil` and `powershell` are also found at their fixed +/// System32 locations (`pwsh` has no fixed location: its only extra candidate +/// is the bare `pwsh.exe`). An absolute candidate is tested once as it is, not +/// joined onto each `PATH` directory, so it is found whether or not `PATH` is +/// set, on `PATH` or not: the lookup then agrees with what the capture runner, +/// which starts those locations, can run. No caller asks for those names +/// today; they stay in the shared candidate list because the runner needs +/// them and one list cannot drift from the other. pub(crate) fn which(program: &str) -> bool { - let Ok(path) = std::env::var("PATH") else { - return false; - }; - let (sep, exts): (char, &[&str]) = if runtime_is_windows() { - (';', &[".exe", ".bat", ".cmd", ""]) - } else { - (':', &[""]) - }; - for dir in path.split(sep) { - if dir.is_empty() { - continue; - } - for ext in exts { - if Path::new(dir).join(format!("{program}{ext}")).is_file() { - return true; - } - } + let dirs = std::env::var_os("PATH") + .map(|path| path_dirs(&path)) + .unwrap_or_default(); + found_in(&dirs, &tool_path_candidates(program), Path::is_file) +} + +/// The directories of a `PATH` value, empty entries dropped. +fn path_dirs(path: &OsStr) -> Vec { + std::env::split_paths(path) + .filter(|dir| !dir.as_os_str().is_empty()) + .collect() +} + +/// Whether any of `candidates` is a file according to `is_file`. +/// +/// A relative candidate is joined onto each of `dirs`: directories are tried in +/// order, and within a directory the candidates in order. An absolute candidate +/// names its own location, so it is tested once as it is, after the directories, +/// and does not depend on `dirs` at all. `is_file` is injected so a test can say +/// which files exist without a file system. +fn found_in(dirs: &[PathBuf], candidates: &[String], is_file: impl Fn(&Path) -> bool) -> bool { + let in_a_dir = dirs.iter().any(|dir| { + candidates + .iter() + .filter(|candidate| !Path::new(candidate).is_absolute()) + .any(|candidate| is_file(&dir.join(candidate))) + }); + in_a_dir + || candidates + .iter() + .map(Path::new) + .filter(|candidate| candidate.is_absolute()) + .any(is_file) +} + +/// The spellings of `program` to try, in the order to try them. +/// +/// On Linux, just `program`. On Windows a program with no extension is also +/// spelled with each `PATHEXT` entry, in both cases, and `cmd`, `pnputil`, +/// `powershell` and `pwsh` add their fixed locations. Used by [`which`] and by +/// the capture runner, which starts each spelling in turn. +pub(crate) fn tool_path_candidates(program: &str) -> Vec { + path_candidates( + program, + runtime_is_windows(), + std::env::var("PATHEXT").ok().as_deref(), + &system_root_from_env(), + ) +} + +/// `SystemRoot`, else `WINDIR`, else `C:\Windows`. +fn system_root_from_env() -> String { + std::env::var("SystemRoot") + .or_else(|_| std::env::var("WINDIR")) + .unwrap_or_else(|_| r"C:\Windows".to_owned()) +} + +/// [`tool_path_candidates`] with the platform, `PATHEXT` (`None` when unset or +/// not Unicode) and system root passed in, so the Windows spellings can be +/// checked on every host. +fn path_candidates( + program: &str, + windows: bool, + pathext: Option<&str>, + system_root: &str, +) -> Vec { + if Path::new(program).extension().is_some() || !windows { + return vec![program.to_owned()]; + } + let mut names = vec![program.to_owned()]; + let pathext = pathext.unwrap_or(".COM;.EXE;.BAT;.CMD"); + for ext in pathext + .split(';') + .map(str::trim) + .filter(|ext| !ext.is_empty()) + { + names.push(format!("{program}{ext}")); + names.push(format!("{program}{}", ext.to_ascii_lowercase())); + } + names.extend(windows_absolute_tool_candidates(program, system_root)); + names.sort(); + names.dedup(); + names +} + +/// The fixed locations of the Windows tools the host probes run. +fn windows_absolute_tool_candidates(program: &str, system_root: &str) -> Vec { + let program = program.trim().to_ascii_lowercase(); + match program.as_str() { + "cmd" | "cmd.exe" => vec![format!(r"{system_root}\System32\cmd.exe")], + "pnputil" | "pnputil.exe" => vec![format!(r"{system_root}\System32\pnputil.exe")], + "powershell" | "powershell.exe" => vec![ + format!(r"{system_root}\System32\WindowsPowerShell\v1.0\powershell.exe"), + "powershell.exe".to_owned(), + ], + "pwsh" | "pwsh.exe" => vec!["pwsh.exe".to_owned()], + _ => Vec::new(), } - false } pub(crate) const SHORT: Duration = Duration::from_secs(5); @@ -200,6 +284,58 @@ mod tests { ); } + fn capture_that(outcome: CommandOutcome, detail: Option<&str>) -> CommandCapture { + CommandCapture::from_run( + vec!["tool".to_owned()], + outcome, + None, + b"out".to_vec(), + b"err".to_vec(), + detail.map(str::to_owned), + ) + } + + #[test] + fn a_run_reads_as_the_rc_run_raw_has_always_returned() { + // Each outcome with the bytes it carries: the rc is the whole translation, + // and the bytes arrive whatever the outcome (a timeout's partial output + // included), as `run_raw` returned them after joining its readers. + let cases = [ + (CommandOutcome::Exited(0), 0), + (CommandOutcome::Exited(3), 3), + (CommandOutcome::Signaled(Some(9)), -1), + (CommandOutcome::Signaled(None), -1), + (CommandOutcome::TimedOut, 124), + (CommandOutcome::WaitFailed, 124), + (CommandOutcome::NotFound, 127), + ( + CommandOutcome::SpawnFailed(std::io::ErrorKind::PermissionDenied), + 127, + ), + ]; + for (outcome, rc) in cases { + assert_eq!( + run_result(capture_that(outcome, None)), + (rc, b"out".to_vec(), b"err".to_vec()), + "{outcome:?}" + ); + } + } + + #[test] + fn a_pipe_that_failed_to_read_does_not_change_what_run_returns() { + // The executor reports a read error in `detail` and keeps the bytes that + // arrived with the child's real status; `run_with_timeout` turns that into + // an error, and `run` has always discarded it (`let _ = read_to_end`). + assert_eq!( + run_result(capture_that( + CommandOutcome::Exited(0), + Some("failed to read the output: broken pipe") + )), + (0, b"out".to_vec(), b"err".to_vec()) + ); + } + #[test] fn env_truncation_is_char_safe_across_multibyte_boundary() { // 5000 'é' chars = 10000 bytes; byte 4000 lands inside a char, which a @@ -215,3 +351,451 @@ mod tests { assert_eq!(truncate_to_chars("short".to_owned(), 4000), "short"); } } + +/// The PATH lookup ([`which`]) and the spellings it tries +/// ([`tool_path_candidates`]), which the summary stack and the examination stack +/// share. Before they were one, each stack had its own copy and the copies +/// answered differently for an empty `PATH` entry, a `PATH` that is not Unicode +/// and the Windows spellings (see `docs/host-probing.md`, "PATH lookup"); the +/// tests here pin the one answer they now give. +/// +/// The pure functions are tested on every host with the platform, `PATHEXT` and +/// the files that exist passed in. The lookup as a whole reads the process's own +/// `PATH`, and one case needs a different working directory, which a test thread +/// cannot change without disturbing every other test in the process (they spawn +/// children by bare name and call `which` themselves). So those cases run in a +/// child: this test binary again, started with the `PATH`, `PATHEXT` and working +/// directory the case needs and asked to run only +/// [`which_child_reports_its_answer`]. In a normal run that test finds no +/// program to ask about and passes without doing anything. +#[cfg(test)] +mod path_lookup_tests { + use super::{found_in, path_candidates, path_dirs, which}; + use std::cell::RefCell; + use std::ffi::{OsStr, OsString}; + use std::path::{Path, PathBuf}; + use std::process::Command; + + const PROGRAM_ENV: &str = "ROCM_CORE_TEST_WHICH_PROGRAM"; + const ANSWER_PREFIX: &str = "which-answer:"; + /// A name no host has installed, so only the planted file can answer for it. + const PROGRAM: &str = "rocm-core-which-tool"; + + fn names(candidates: &[&str]) -> Vec { + candidates.iter().map(|name| (*name).to_owned()).collect() + } + + #[test] + fn candidates_off_windows_are_the_program_alone_whatever_pathext_says() { + assert_eq!( + path_candidates("tool", false, Some(".COM;.EXE"), r"C:\Windows"), + names(&["tool"]) + ); + } + + #[test] + fn candidates_on_windows_for_a_name_with_an_extension_are_the_name_alone() { + // The early return also keeps the System32 spelling of `cmd.exe` out of + // the list: only the extension-less `cmd` reaches it. + assert_eq!( + path_candidates("cmd.exe", true, Some(".COM;.EXE"), r"C:\Windows"), + names(&["cmd.exe"]) + ); + assert_eq!( + path_candidates("tool.ps1", true, Some(".PS1"), r"C:\Windows"), + names(&["tool.ps1"]) + ); + // A dotted name is not a bare name with an unknown extension: PATHEXT is + // not tried after it, so `python3.11` is looked for as `python3.11` only. + // The examination's old copy also tried `.exe`, `.bat` and `.cmd`; no + // caller passes such a name (they are all bare, or `wsl.exe`). + assert_eq!( + path_candidates( + "python3.11", + true, + Some(".COM;.EXE;.BAT;.CMD"), + r"C:\Windows" + ), + names(&["python3.11"]) + ); + } + + #[test] + fn candidates_on_windows_try_every_pathext_entry_in_both_cases() { + // Sorted as bytes, so the upper-case spelling of an extension comes + // before its lower-case one, and the bare name before both. `.PS1` is + // outside the old examination list (`.exe`, `.bat`, `.cmd`). + assert_eq!( + path_candidates("tool", true, Some(".COM;.EXE;.PS1"), r"C:\Windows"), + names(&[ + "tool", "tool.COM", "tool.EXE", "tool.PS1", "tool.com", "tool.exe", "tool.ps1", + ]) + ); + } + + #[test] + fn candidates_on_windows_trim_pathext_entries_and_skip_empty_ones() { + assert_eq!( + path_candidates("tool", true, Some(" .BAT ;; ;.CMD;"), r"C:\Windows"), + names(&["tool", "tool.BAT", "tool.CMD", "tool.bat", "tool.cmd"]) + ); + } + + #[test] + fn candidates_on_windows_fall_back_to_the_default_pathext_when_it_is_unset() { + assert_eq!( + path_candidates("tool", true, None, r"C:\Windows"), + names(&[ + "tool", "tool.BAT", "tool.CMD", "tool.COM", "tool.EXE", "tool.bat", "tool.cmd", + "tool.com", "tool.exe", + ]) + ); + } + + #[test] + fn candidates_on_windows_add_the_fixed_system32_locations() { + let root = r"C:\Win"; + // A `PATHEXT` without `.EXE`, so `powershell.exe` and `pwsh.exe` can only + // come from the fixed list: with `.EXE` in it the expansion adds them + // anyway and these assertions would not notice the list losing them. + let with = |program: &str| path_candidates(program, true, Some(".BAT"), root); + + assert!( + with("cmd").contains(&r"C:\Win\System32\cmd.exe".to_owned()), + "{:?}", + with("cmd") + ); + assert!(with("pnputil").contains(&r"C:\Win\System32\pnputil.exe".to_owned())); + let powershell = with("powershell"); + assert!( + powershell + .contains(&r"C:\Win\System32\WindowsPowerShell\v1.0\powershell.exe".to_owned()) + ); + assert!(powershell.contains(&"powershell.exe".to_owned())); + assert!(with("pwsh").contains(&"pwsh.exe".to_owned())); + // Matched without regard to case, like the file system. + assert!(with("CMD").contains(&r"C:\Win\System32\cmd.exe".to_owned())); + // The location comes first when it sorts first, which is the order the + // capture runner starts them in. + assert_eq!( + with("cmd").first().map(String::as_str), + Some(r"C:\Win\System32\cmd.exe") + ); + } + + #[test] + fn candidates_on_windows_for_other_tools_have_no_fixed_location() { + assert_eq!( + path_candidates("python", true, Some(".EXE"), r"C:\Win"), + names(&["python", "python.EXE", "python.exe"]) + ); + } + + #[test] + fn candidates_off_windows_never_name_a_system32_location() { + assert_eq!( + path_candidates("cmd", false, Some(".EXE"), r"C:\Win"), + names(&["cmd"]) + ); + } + + #[test] + fn path_dirs_drop_every_empty_entry() { + let path = std::env::join_paths([ + OsStr::new(""), + OsStr::new("first"), + OsStr::new(""), + OsStr::new("second"), + OsStr::new(""), + ]) + .expect("no entry holds a separator"); + assert_eq!( + path_dirs(&path), + vec![PathBuf::from("first"), PathBuf::from("second")] + ); + assert!(path_dirs(OsStr::new("")).is_empty()); + } + + #[cfg(unix)] + #[test] + fn path_dirs_keep_an_entry_that_is_not_unicode() { + use std::os::unix::ffi::OsStrExt as _; + + let odd = OsStr::from_bytes(b"/odd-\xff"); + let path = std::env::join_paths([OsStr::new("/usr/bin"), odd]).expect("no separator"); + assert_eq!( + path_dirs(&path), + vec![PathBuf::from("/usr/bin"), PathBuf::from(odd)] + ); + } + + #[test] + fn found_in_asks_each_candidate_in_each_directory_in_order_and_stops_at_the_first_file() { + let dirs = [PathBuf::from("first"), PathBuf::from("second")]; + let candidates = names(&["tool", "tool.exe"]); + let asked = RefCell::new(Vec::new()); + + let found = found_in(&dirs, &candidates, |file| { + asked.borrow_mut().push(file.to_owned()); + file == Path::new("second").join("tool") + }); + + assert!(found); + assert_eq!( + asked.into_inner(), + vec![ + Path::new("first").join("tool"), + Path::new("first").join("tool.exe"), + Path::new("second").join("tool"), + ] + ); + assert!(!found_in(&dirs, &candidates, |_| false)); + assert!(!found_in(&[], &candidates, |_| true)); + } + + #[test] + fn found_in_asks_about_an_absolute_candidate_once_as_it_is_whatever_the_directories() { + // This is why a System32 location is found "without being on PATH", and + // found even when PATH is unset or all empty: it is tested on its own, + // once, not joined onto each directory. + let absolute = std::env::temp_dir().join("rocm-core-absolute-candidate"); + let candidates = [absolute.to_string_lossy().into_owned()]; + + for dirs in [ + Vec::new(), + vec![PathBuf::from("anywhere")], + vec![PathBuf::from("anywhere"), PathBuf::from("elsewhere")], + ] { + let asked = RefCell::new(Vec::new()); + let found = found_in(&dirs, &candidates, |file| { + asked.borrow_mut().push(file.to_owned()); + true + }); + // With directories listed the absolute candidate is never joined + // onto them, so the only thing asked about is the candidate itself. + assert!(found, "{} directories", dirs.len()); + assert_eq!( + asked.into_inner(), + vec![absolute.clone()], + "{} directories", + dirs.len() + ); + } + assert!(!found_in(&[], &candidates, |_| false)); + } + + #[test] + fn found_in_still_tries_relative_candidates_in_directories_beside_an_absolute_one() { + let absolute = std::env::temp_dir().join("rocm-core-absolute-candidate"); + let candidates = [ + absolute.to_string_lossy().into_owned(), + "tool.exe".to_owned(), + ]; + let dirs = [PathBuf::from("first")]; + + assert!(found_in(&dirs, &candidates, |file| file + == Path::new("first").join("tool.exe"))); + assert!(!found_in(&[], &candidates, |file| file + == Path::new("first").join("tool.exe"))); + } + + /// The child's side of [`ask_for`]: print the answer for the program named in + /// the environment. Does nothing in a normal test run. + #[test] + fn which_child_reports_its_answer() { + let Some(program) = std::env::var_os(PROGRAM_ENV) else { + return; + }; + println!("{ANSWER_PREFIX} {}", which(&program.to_string_lossy())); + } + + /// Run [`which`] for `program` in a child whose `PATH` is `path` (unset when + /// `None`), working directory is `cwd` and extra environment is `env`. + fn ask_for(program: &str, path: Option<&OsStr>, cwd: &Path, env: &[(&str, &str)]) -> bool { + let this_test = module_path!() + .split_once("::") + .expect("module path has a crate prefix") + .1; + let mut child = Command::new(std::env::current_exe().expect("path of the test binary")); + child + .args([ + "--exact", + &format!("{this_test}::which_child_reports_its_answer"), + "--nocapture", + "--test-threads=1", + ]) + .env(PROGRAM_ENV, program) + .current_dir(cwd); + match path { + Some(path) => child.env("PATH", path), + None => child.env_remove("PATH"), + }; + for (key, value) in env { + child.env(key, value); + } + let output = child.output().expect("start the test binary again"); + let stdout = String::from_utf8_lossy(&output.stdout); + let answer = stdout + .lines() + .find_map(|line| line.split_once(ANSWER_PREFIX).map(|(_, answer)| answer)) + .unwrap_or_else(|| { + panic!( + "the child printed no answer (status {}):\n{stdout}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ) + }); + answer.trim() == "true" + } + + fn ask(path: Option<&OsStr>, cwd: &Path) -> bool { + ask_for(PROGRAM, path, cwd, &[]) + } + + fn join(dirs: &[&Path]) -> OsString { + std::env::join_paths(dirs).expect("no directory name holds a path separator") + } + + /// Create `name` under `parent`, holding a file called `file_name`. + fn dir_holding(parent: &Path, name: &OsStr, file_name: &str) -> PathBuf { + let dir = parent.join(name); + std::fs::create_dir_all(&dir).expect("create the planted directory"); + crate::fake_executable::write_fake_executable(&dir.join(file_name), "#!/bin/sh\n") + .expect("plant the program"); + dir + } + + fn empty_dir(parent: &Path, name: &str) -> PathBuf { + let dir = parent.join(name); + std::fs::create_dir_all(&dir).expect("create the empty directory"); + dir + } + + #[test] + fn which_finds_a_program_in_a_path_directory_and_not_otherwise() { + let scratch = tempfile::tempdir().expect("scratch directory"); + let present = dir_holding(scratch.path(), OsStr::new("present"), PROGRAM); + let absent = empty_dir(scratch.path(), "absent"); + let cwd = empty_dir(scratch.path(), "cwd"); + + assert!( + ask(Some(&join(&[&absent, &present])), &cwd), + "a program in the second PATH directory is found" + ); + assert!( + !ask(Some(&join(&[&absent])), &cwd), + "a PATH without the program finds nothing" + ); + assert!(!ask(None, &cwd), "an unset PATH finds nothing"); + } + + #[test] + fn which_does_not_take_a_directory_named_like_the_program_for_the_program() { + let scratch = tempfile::tempdir().expect("scratch directory"); + let bin = empty_dir(scratch.path(), "bin"); + std::fs::create_dir_all(bin.join(PROGRAM)).expect("plant a directory with the name"); + assert!(!ask(Some(&join(&[&bin])), scratch.path())); + } + + /// The decision this lookup was merged on: an empty `PATH` entry is skipped, + /// so a file in the directory the CLI runs from is not "installed". The + /// summary stack's old copy read the entry as the current directory and + /// answered `true` here; the control below shows the same file does count + /// once a real entry names its directory. + #[cfg(unix)] + #[test] + fn which_does_not_read_an_empty_path_entry_as_the_current_directory() { + let scratch = tempfile::tempdir().expect("scratch directory"); + let cwd = dir_holding(scratch.path(), OsStr::new("cwd"), PROGRAM); + let bin = empty_dir(scratch.path(), "bin"); + let bin_text = bin.to_str().expect("UTF-8 scratch path"); + + for (shape, path) in [ + ("leading", format!(":{bin_text}")), + ("middle", format!("{bin_text}::{bin_text}")), + ("trailing", format!("{bin_text}:")), + ("the whole PATH", String::new()), + ] { + assert!( + !ask(Some(OsStr::new(&path)), &cwd), + "an empty entry ({shape}) must not read the current directory" + ); + } + assert!( + ask(Some(&join(&[&cwd])), &cwd), + "the same file is found when a PATH entry names its directory" + ); + } + + /// The other half of the decision: a `PATH` that is not valid Unicode is + /// still searched. The examination stack's old copy read `PATH` with + /// `std::env::var`, which fails on it, and answered `false` for every tool. + #[cfg(unix)] + #[test] + fn which_searches_a_path_that_is_not_unicode() { + use std::os::unix::ffi::OsStrExt as _; + + let scratch = tempfile::tempdir().expect("scratch directory"); + let cwd = empty_dir(scratch.path(), "cwd"); + let valid = dir_holding(scratch.path(), OsStr::new("valid"), PROGRAM); + // A directory whose own name is not Unicode (it holds the byte 0xff). + let odd = dir_holding(scratch.path(), OsStr::from_bytes(b"odd-\xff"), PROGRAM); + let missing_odd = scratch.path().join(OsStr::from_bytes(b"missing-\xff")); + + assert!( + ask(Some(&join(&[&odd])), &cwd), + "the program sits in a directory whose name is not Unicode" + ); + assert!( + ask(Some(&join(&[&missing_odd, &valid])), &cwd), + "a Unicode directory after an entry that is not Unicode is still searched" + ); + assert!( + !ask(Some(&join(&[&missing_odd])), &cwd), + "a PATH that is not Unicode does not find a program that is not on it" + ); + } + + /// A `.ps1` listed in `PATHEXT` is found, where the examination stack's old + /// copy tried only `.exe`, `.bat`, `.cmd` and the bare name. + #[cfg(windows)] + #[test] + fn which_tries_a_pathext_extension_outside_exe_bat_and_cmd() { + let scratch = tempfile::tempdir().expect("scratch directory"); + let ps1 = dir_holding(scratch.path(), OsStr::new("ps1"), &format!("{PROGRAM}.ps1")); + let pathext = [("PATHEXT", ".COM;.EXE;.BAT;.CMD;.PS1")]; + assert!(ask_for( + PROGRAM, + Some(&join(&[&ps1])), + scratch.path(), + &pathext + )); + } + + /// An entry in quotes keeps a `;` inside it, as `std::env::split_paths` + /// reads it on Windows; the examination stack's old copy split on every `;`. + #[cfg(windows)] + #[test] + fn which_searches_a_quoted_path_entry_that_holds_a_semicolon() { + let scratch = tempfile::tempdir().expect("scratch directory"); + let dir = dir_holding( + scratch.path(), + OsStr::new("semi;colon"), + &format!("{PROGRAM}.exe"), + ); + let quoted = format!("\"{}\"", dir.display()); + assert!(ask(Some(OsStr::new("ed)), scratch.path())); + } + + /// `cmd` is found at its System32 location without being on `PATH`, as the + /// summary stack's old copy found it; the examination stack's did not. + #[cfg(windows)] + #[test] + fn which_finds_cmd_at_its_system32_location_without_it_being_on_path() { + let scratch = tempfile::tempdir().expect("scratch directory"); + let empty = empty_dir(scratch.path(), "empty"); + assert!(ask_for("cmd", Some(&join(&[&empty])), scratch.path(), &[])); + // And with no PATH at all, as the capture runner could still start it. + assert!(ask_for("cmd", None, scratch.path(), &[])); + } +} diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 57da4bb1fe..dbe18a856e 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -15,6 +15,8 @@ use std::time::{Duration, Instant}; +use crate::process::ProcessProbe; + /// How often the bounded waits poll for the target's exit. const POLL_INTERVAL: Duration = Duration::from_millis(25); @@ -32,8 +34,10 @@ pub enum KillScope { /// A spawned process identified in a way that is robust to PID recycling. /// /// `start_ticks` is the kernel start-time of the process (clock ticks since -/// boot). It is `None` on platforms without `/proc`, where identity cannot be -/// verified and callers fall back to best-effort behaviour. +/// boot) on Linux, and its creation time as a `FILETIME` on Windows. It is +/// `None` on other platforms, and for a record written before Windows recorded +/// one, where identity cannot be verified and callers fall back to best-effort +/// behaviour. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProcessIdentity { pub pid: u32, @@ -69,8 +73,10 @@ pub enum IdentityState { /// was recycled. Recycled, /// The PID is live and an identity was recorded, but the current start-time - /// cannot be read, so it can be neither confirmed nor refuted. The process - /// must not be signalled, and it must not be reported as stopped. + /// cannot be read, so it can be neither confirmed nor refuted; or the PID + /// cannot be queried at all (on Windows, a live process this one may not + /// open), so it cannot be told from a process that is gone. The process must + /// not be signalled, and it must not be reported as stopped. Indeterminate, /// The PID is not live (or is a not-yet-reaped zombie). Gone, @@ -85,10 +91,61 @@ pub enum IdentityState { /// [`IdentityState::Matches`]. #[must_use] pub fn identity_state(id: &ProcessIdentity) -> IdentityState { - if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { - return IdentityState::Gone; + identity_state_with( + id, + crate::process::probe_process, + process_has_exited, + process_start_ticks, + ) +} + +/// [`identity_state`] with its three system reads supplied by the caller: whether +/// the PID can be queried and is running, whether it is a zombie, and its +/// current start-time token. The liveness gate in front of every stop, so a test +/// drives each answer a read can give on every host. +/// +/// A PID that cannot be queried is [`IdentityState::Indeterminate`], not +/// [`IdentityState::Gone`]: a failure to look is not an exit, and reading it as +/// one would let a stop report a live process as stopped. That is one of the two +/// routes to [`IdentityState::Indeterminate`] and so to an `unverified` stop; the +/// other is a recorded token that cannot be read back from a running process +/// ([`classify_live_identity`]). +/// +/// A token that was recorded and can still be read, and that differs, outranks +/// the failed query: it proves a different process holds the PID, so the +/// recorded one is gone. On Windows this narrows to the case where the process +/// opens but the exit-code query fails: the token and the probe both open the +/// PID with `PROCESS_QUERY_LIMITED_INFORMATION`, so a process that cannot be +/// opened has no readable token either, and stays [`IdentityState::Indeterminate`]. +fn identity_state_with( + id: &ProcessIdentity, + probe: impl Fn(u32) -> ProcessProbe, + has_exited: impl Fn(u32) -> bool, + start_ticks: impl Fn(u32) -> Option, +) -> IdentityState { + match probe(id.pid) { + ProcessProbe::Gone => IdentityState::Gone, + ProcessProbe::Unqueryable => match (id.start_ticks, start_ticks(id.pid)) { + (Some(recorded), Some(live)) if recorded != live => IdentityState::Recycled, + _ => IdentityState::Indeterminate, + }, + ProcessProbe::Running if has_exited(id.pid) => IdentityState::Gone, + ProcessProbe::Running => classify_live_identity(id.start_ticks, start_ticks(id.pid)), } - match (id.start_ticks, process_start_ticks(id.pid)) { +} + +/// The identity verdict for a PID that is running: what its recorded token +/// (`recorded`) says about the token read from the live process now (`live`). +/// +/// The one decision every stop path turns on, apart from the system reads, so +/// a test covers each case on every host. A process is refused only when a +/// token was recorded and the live one is provably different. A record with no +/// token (an older record, or one written where no token could be read) cannot +/// be refuted and is a match whatever the live process reads, so its stop and +/// its tree kill go ahead as they always did. A recorded token that cannot be +/// confirmed now is neither signalled nor reported stopped. +const fn classify_live_identity(recorded: Option, live: Option) -> IdentityState { + match (recorded, live) { (Some(expected), Some(actual)) => { if expected == actual { IdentityState::Matches @@ -104,6 +161,18 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { } } +/// The verdict that ends a termination before anything is signalled, when the +/// recorded identity says not to: `None` for a process that matches and so goes +/// on to be stopped. +const fn refusal_before_signalling(state: IdentityState) -> Option { + match state { + IdentityState::Gone => Some(TerminationOutcome::AlreadyGone), + IdentityState::Recycled => Some(TerminationOutcome::IdentityMismatch), + IdentityState::Indeterminate => Some(TerminationOutcome::Unverified), + IdentityState::Matches => None, + } +} + /// Whether `state` means the recorded process is definitively no longer running. /// /// [`IdentityState::Indeterminate`] is intentionally excluded: a process we can @@ -120,6 +189,15 @@ pub enum TerminationOutcome { AlreadyGone, /// The PID now belongs to a different process (recycled), so the recorded /// process has already exited; nothing was signalled. + /// + /// Counted as stopped, on every platform, because a token that is recorded + /// and readable and differs is positive evidence of a different process. The + /// engine's own token comes from the same [`process_start_ticks`] that reads + /// it back here, taken for that PID's process when it was spawned or reported + /// (a creation time does not change over a process's life), so a mismatch for + /// the real engine is not an expected outcome. It stays visible as + /// `identity_mismatch` in the report rather than as a signal sent to a + /// process nobody could prove was the server's. IdentityMismatch, /// The recorded PID is live but its identity could not be confirmed, so it /// was deliberately left untouched and its state is unknown. @@ -130,16 +208,42 @@ pub enum TerminationOutcome { Forced, /// At least one signalled process was still alive after the bounded deadline. TimedOut, + /// The recorded process itself is gone (this stop's own wait saw it exit, or + /// it was already gone), but the separate kill of the processes under it did + /// not complete (on Windows, a `taskkill` that failed, stalled, or found no + /// such process while the root was still running), so one of them may still + /// be running and holding the device. Deliberately not + /// [`TimedOut`](Self::TimedOut): the root did not outlive any deadline. + DescendantsUnconfirmed, + /// The recorded process was gone by the time the separate kill of the + /// processes under it ran (on Windows, `taskkill` found no such process and + /// the root's re-read agrees): it exited by itself after the identity check. + /// `taskkill /T` reaches children only through a live parent, and nothing + /// recorded what was under the root, so this stop cannot say whether a + /// process it left running is still there. Not a stop on its own; the + /// aggregate over the record's processes confirms it when another recorded + /// process is confirmed gone ([`terminate_recorded_service_pids`]). + ExitedBeforeTreeKill, } impl TerminationOutcome { /// Is the recorded service confirmed to be no longer running as a result? /// - /// `false` for [`TimedOut`](Self::TimedOut) (still alive) and - /// [`Unverified`](Self::Unverified) (could not be confirmed either way). + /// `false` for [`TimedOut`](Self::TimedOut) (still alive), + /// [`Unverified`](Self::Unverified) (could not be confirmed either way) and + /// [`DescendantsUnconfirmed`](Self::DescendantsUnconfirmed) (the root is gone + /// but a process under it may not be) and + /// [`ExitedBeforeTreeKill`](Self::ExitedBeforeTreeKill) (the root is gone and + /// nothing says what was under it). #[must_use] pub const fn stopped(self) -> bool { - !matches!(self, Self::TimedOut | Self::Unverified) + !matches!( + self, + Self::TimedOut + | Self::Unverified + | Self::DescendantsUnconfirmed + | Self::ExitedBeforeTreeKill + ) } /// Did the process stop without needing a forced kill (or was it already @@ -162,6 +266,8 @@ impl TerminationOutcome { Self::Graceful => "graceful", Self::Forced => "forced", Self::TimedOut => "timed_out", + Self::DescendantsUnconfirmed => "descendants_unconfirmed", + Self::ExitedBeforeTreeKill => "exited_before_tree_kill", } } } @@ -194,11 +300,8 @@ pub fn terminate_verified( grace: Duration, force: bool, ) -> TerminationOutcome { - match identity_state(id) { - IdentityState::Gone => return TerminationOutcome::AlreadyGone, - IdentityState::Recycled => return TerminationOutcome::IdentityMismatch, - IdentityState::Indeterminate => return TerminationOutcome::Unverified, - IdentityState::Matches => {} + if let Some(verdict) = refusal_before_signalling(identity_state(id)) { + return verdict; } let tree = matches!(scope, KillScope::Tree); @@ -261,6 +364,566 @@ pub fn terminate_verified( } } +/// How long a managed stop waits on a recorded process before it escalates. +/// +/// Each recorded process, tree included, gets this long to exit after `SIGTERM` +/// before the stop escalates to `SIGKILL`, and then this long again before it +/// reports a timeout. The one value behind `rocm services stop` and `rocmd`'s +/// `stop_server`: both stop the same records, so neither may wait for a +/// different time. +pub const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); + +/// What a managed stop did about one PID a service record names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedPidOutcome { + /// The PID is the stopping process's own. It is reported but never + /// signalled, and it counts as stopped: `rocmd` and an attached `rocm + /// serve` record themselves as the supervisor of the service they launched, + /// and the launcher is not the service. + OwnProcess, + /// The PID was handed to the verified termination, with this verdict. + Terminated(TerminationOutcome), +} + +impl RecordedPidOutcome { + /// The label a stop report carries: `self` for the stopping process's own + /// PID, otherwise [`TerminationOutcome::as_str`]. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::OwnProcess => "self", + Self::Terminated(outcome) => outcome.as_str(), + } + } +} + +/// One recorded PID's fate in a managed stop. +/// +/// Kept per PID because the outcomes that leave a stop unconfirmed are reported +/// differently: a `timed_out` PID was signalled and it, or a process under it, +/// outlived the deadline, while an `unverified` one was not signalled at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecordedPidStop { + pub pid: u32, + /// Which recorded PID this is: `supervisor`, `engine`, or + /// `supervisor_and_engine` when the record holds the same PID for both. + pub role: &'static str, + pub outcome: RecordedPidOutcome, +} + +impl RecordedPidStop { + /// Whether a signal went out to this PID. A `timed_out` PID counts: the + /// signal was sent and only the exit was never observed, which + /// [`stopped`](Self::stopped) reports separately. + #[must_use] + pub const fn signaled(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::TimedOut + | TerminationOutcome::DescendantsUnconfirmed + ) + ) + } + + /// Whether the stop escalated to a forced kill for this PID. That is not + /// "the forced kill worked": a forced stop only reaches `timed_out` by way + /// of the escalation, and [`stopped`](Self::stopped) says whether the + /// process actually went away. + #[must_use] + pub const fn forced(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Forced + | TerminationOutcome::TimedOut + | TerminationOutcome::DescendantsUnconfirmed + ) + ) + } + + /// Whether the recorded process is confirmed no longer running. + #[must_use] + pub const fn stopped(self) -> bool { + match self.outcome { + RecordedPidOutcome::OwnProcess => true, + RecordedPidOutcome::Terminated(outcome) => outcome.stopped(), + } + } + + /// The entry a stop report lists for this PID under `pid_outcomes`. + #[must_use] + pub fn report_entry(self) -> serde_json::Value { + serde_json::json!({ + "pid": self.pid, + "role": self.role, + "outcome": self.outcome.as_str(), + }) + } +} + +/// Terminate the processes a service record names, verifying each PID's +/// identity first, and report what happened to each. +/// +/// PIDs are recycled, so a persisted PID is not by itself evidence that the +/// recorded process still holds it. Each PID is paired with **its own** +/// start-time token (`supervisor_start_ticks` with `supervisor_pid`, +/// `engine_start_ticks` with `engine_pid`) and handed to `terminate`, which +/// production supplies as [`terminate_recorded_pid`]. When the two PIDs +/// coincide, the entry is reported once, under the role `supervisor_and_engine`, +/// and keeps a known token in preference to a missing one: a token is never read +/// across from the PID it does not belong to. The supervisor is visited first. +/// +/// A zero PID names nothing and is skipped. The stopping process's own PID is +/// reported as [`RecordedPidOutcome::OwnProcess`] and `terminate` is not called +/// for it. +/// +/// `terminate` is injectable so a caller's test can supply the verdicts it +/// cannot construct on demand (a process that survives `SIGKILL`, one whose +/// identity cannot be read). +/// +/// One verdict is settled across the record rather than per PID. A root that +/// exited by itself before its processes could be taken down +/// ([`TerminationOutcome::ExitedBeforeTreeKill`]) cannot say what it left +/// running. It becomes [`TerminationOutcome::AlreadyGone`] only when another +/// process the record names is confirmed gone (a stop, or itself such a root): +/// the record's other process is the one thing that was under the root and is +/// independently verified. A root that is the only process the record names, or +/// whose sibling is still running or unconfirmed, stays unconfirmed. Processes +/// the record does not name that were under a root can be neither found nor +/// verified, which is the same limit as for a root that was already gone when a +/// stop began. +/// +/// This relies on the engine being visited as its own role, which it is: the +/// engine PID is an entry of its own, with its own token and its own verdict, so +/// a worker the root left running is reached by the engine's own termination +/// when the record names it. Both callers fold every row (`rocm services stop` +/// in `stop_internal_managed_service_confirmed_with`, `rocmd` in +/// `stop_managed_service_with`): the stop is confirmed, and the recorded PIDs and +/// the endpoint key cleared, only when every recorded role is individually +/// stopped. A root that exited alone therefore never confirms a stop on its own +/// row, and the record is kept when the engine role is not confirmed. +pub fn terminate_recorded_service_pids( + record: &crate::ManagedServiceRecord, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> Vec { + let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); + for (pid, role, ticks) in [ + ( + Some(record.supervisor_pid), + "supervisor", + record.supervisor_start_ticks, + ), + (record.engine_pid, "engine", record.engine_start_ticks), + ] { + let Some(pid) = pid.filter(|pid| *pid != 0) else { + continue; + }; + if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { + // Only the engine entry can land here, aliasing the supervisor: the + // one PID is both, and is reported as both. + existing.1 = "supervisor_and_engine"; + if existing.2.is_none() { + existing.2 = ticks; + } + } else { + entries.push((pid, role, ticks)); + } + } + + let mut stops: Vec = entries + .into_iter() + .map(|(pid, role, ticks)| { + let outcome = if pid == std::process::id() { + RecordedPidOutcome::OwnProcess + } else { + RecordedPidOutcome::Terminated(terminate(&ProcessIdentity::new(pid, ticks))) + }; + RecordedPidStop { pid, role, outcome } + }) + .collect(); + settle_roots_that_exited_alone(&mut stops); + stops +} + +/// Whether `stop` vouches for what was under another recorded root: a recorded +/// process this stop took down ([`Graceful`](TerminationOutcome::Graceful), +/// [`Forced`](TerminationOutcome::Forced)), found already gone +/// ([`AlreadyGone`](TerminationOutcome::AlreadyGone)), or a root that exited by +/// itself ([`TerminationOutcome::ExitedBeforeTreeKill`]). Not +/// [`IdentityMismatch`](TerminationOutcome::IdentityMismatch): it says a stranger +/// holds that PID now, which is "gone" for that process but is no evidence about +/// what was under a different root. The stopping process's own PID is not a +/// service process and never counts. +const fn recorded_process_is_gone(stop: RecordedPidStop) -> bool { + matches!( + stop.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::AlreadyGone + | TerminationOutcome::ExitedBeforeTreeKill + ) + ) +} + +/// Turn each [`TerminationOutcome::ExitedBeforeTreeKill`] into +/// [`TerminationOutcome::AlreadyGone`] when some other recorded process is +/// confirmed gone, and leave it unconfirmed otherwise. Judged on the verdicts as +/// they came in, so two roots that both exited by themselves corroborate each +/// other. +fn settle_roots_that_exited_alone(stops: &mut [RecordedPidStop]) { + let corroborated: Vec = (0..stops.len()) + .map(|index| { + stops + .iter() + .enumerate() + .any(|(other, stop)| other != index && recorded_process_is_gone(*stop)) + }) + .collect(); + for (stop, corroborated) in stops.iter_mut().zip(corroborated) { + if corroborated + && matches!( + stop.outcome, + RecordedPidOutcome::Terminated(TerminationOutcome::ExitedBeforeTreeKill) + ) + { + stop.outcome = RecordedPidOutcome::Terminated(TerminationOutcome::AlreadyGone); + } + } +} + +/// Terminate one recorded process, and its descendants, after verifying its +/// identity. The production `terminate` of [`terminate_recorded_service_pids`]. +/// +/// The stop is forced: a managed stop is definitive, so it must not report +/// success while an engine worker still holds the GPU. Only a process whose +/// recorded identity is confirmed is ever reached by it. +/// +/// On Windows the descendants are taken separately, first. [`terminate_verified`] +/// cannot walk a process tree there (`process_tree_pids` returns just the root), +/// and the engine runs one level below the recorded PID, because the daemon +/// launches it through an `__engine-serve-http` process: a root-only kill would +/// leave the engine running and still holding the device. +/// +/// The tree kill runs only for a PID that passes the same identity gate as the +/// rest of the stop. A record written with a start-time token carries the +/// process's creation time on Windows ([`process_start_ticks`]), so a PID that +/// an unrelated process has since taken reads back [`IdentityState::Recycled`] +/// and neither it nor its subtree is touched. A record written before Windows +/// recorded a token has none to refute, and for it the gate only turns away a +/// PID that is not running at all: that old record can still name a recycled +/// PID, and `taskkill /T /F` then takes that process and its subtree. A stop +/// that confirms clears the record's PIDs, so that exposure shrinks to the +/// records written before the token existed and not yet stopped. +/// +/// On Windows the PID is held from before the identity check until the stop +/// ends: a process's PID is not handed to another process while any handle to it +/// is open, so the creation time that was compared is the creation time of the +/// process `taskkill` and the root termination then reach by PID. Without the +/// hold, the process could exit and an unrelated one take its PID between the +/// check and the kill. +/// +/// The descendant kill is bounded: a `taskkill` that has not finished within +/// [`MANAGED_STOP_GRACE`] is killed and reaped and counts as a tree kill that did +/// not complete, so a stalled helper cannot hang the stop. +/// +/// A tree kill that did not complete leaves the stop unconfirmed, whatever the +/// root's own termination says: [`terminate_verified`] can only watch the root +/// on Windows, so a descendant that `taskkill` did not take down is one this stop +/// cannot see exit, and reporting the stop confirmed would clear the record and +/// the endpoint key while a worker may still hold the device. +pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome { + terminate_recorded_pid_within(identity, MANAGED_STOP_GRACE) +} + +/// [`terminate_recorded_pid`] with the root's grace chosen by the caller. +/// +/// For a process that has nothing to flush, such as a launch child cleaned up +/// because its record could not be published. Everything else is the same +/// routine: the PID hold, the identity gate, and on Windows the descendant kill, +/// whose own deadline stays [`MANAGED_STOP_GRACE`]. +pub fn terminate_recorded_pid_within( + identity: &ProcessIdentity, + grace: std::time::Duration, +) -> TerminationOutcome { + terminate_recorded_pid_with( + identity, + platform_tree_kill(), + hold_pid, + identity_state, + |identity| terminate_verified(identity, KillScope::Tree, grace, true), + ) +} + +/// Whether the separate descendant kill took the tree down. +// Only Windows has the separate step, so nothing builds a value elsewhere; the +// tests build both, and the decision on them is shared by every platform. +#[cfg_attr(not(any(windows, test)), allow(dead_code))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TreeKill { + /// The kill ran and reported that every process it reached is terminated. + Completed, + /// The kill ran and reported that it found no such process (`taskkill` exits + /// 128): the root was gone by the time it looked. Not by itself a failure, + /// and not a success either; the root's own verdict decides which. + NotFound, + /// The kill could not run, or reported that it did not take every process + /// down. Which descendants survive is not known. + Failed, +} + +/// The verdict for a stop whose descendant kill ended as `tree` and whose root +/// termination answered `outcome`. +/// +/// When the tree kill did not complete ([`TreeKill::Failed`], or +/// [`TreeKill::NotFound`] while the root was still there), a root that this +/// stop saw exit ([`Graceful`](TerminationOutcome::Graceful), +/// [`Forced`](TerminationOutcome::Forced)) or that was already gone is reported +/// [`TerminationOutcome::DescendantsUnconfirmed`]: the root is gone, which is +/// what the wait observed, but nothing watched the processes under it, so the +/// stop cannot be called confirmed. It is not `timed_out`, which would claim +/// that something outlived a deadline. The verdicts that already deny the stop +/// ([`TimedOut`](TerminationOutcome::TimedOut), the root itself outlived the +/// deadline; [`Unverified`](TerminationOutcome::Unverified)) stay as they are. +/// +/// A kill that found no such process ([`TreeKill::NotFound`]) is judged by the +/// root's own re-read, which `outcome` carries. When the root is gone +/// ([`TerminationOutcome::AlreadyGone`]) it exited on its own between the +/// identity check and the kill: that is [`TerminationOutcome::ExitedBeforeTreeKill`], +/// not a stop. `taskkill /T` reaches children only through a live parent, so a +/// child the root left running is out of its reach, and the stop kept no list of +/// what was under the root to check afterwards. The aggregate over the record's +/// processes decides whether another recorded process corroborates the exit +/// ([`terminate_recorded_service_pids`]). When the root is still there, "no such +/// process" contradicts what the termination found, so the kill is treated as +/// failed. +const fn after_tree_kill(tree: TreeKill, outcome: TerminationOutcome) -> TerminationOutcome { + match (tree, outcome) { + (TreeKill::Completed, outcome) => outcome, + (TreeKill::NotFound, TerminationOutcome::AlreadyGone) => { + TerminationOutcome::ExitedBeforeTreeKill + } + ( + TreeKill::NotFound | TreeKill::Failed, + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::AlreadyGone, + ) => TerminationOutcome::DescendantsUnconfirmed, + (TreeKill::NotFound | TreeKill::Failed, outcome) => outcome, + } +} + +/// [`terminate_recorded_pid`] with its platform and system touchpoints supplied +/// by the caller, so a test drives the tree-kill step on every host: +/// `tree_kill` takes one PID's descendants down (`None` where the platform needs +/// no separate step), `hold` keeps the PID reserved, answering `None` when it +/// could not, and is dropped when the call returns, `state_of` reads the PID's +/// identity, and `terminate` is the verified termination of the root. +fn terminate_recorded_pid_with( + identity: &ProcessIdentity, + tree_kill: Option<&dyn Fn(u32) -> TreeKill>, + hold: impl FnOnce(&ProcessIdentity) -> Option, + state_of: impl Fn(&ProcessIdentity) -> IdentityState, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> TerminationOutcome { + // Before anything reads the identity, and until the end of the call. + let held = hold(identity); + if let Some(tree_kill) = tree_kill { + // The identity read that decides about the tree kill also decides about + // the stop. A refusal here (gone, recycled, unverifiable) is the verdict: + // the root is not handed to `terminate`, which re-reads the identity and + // could find a process that was unverifiable a moment ago verifiable now, + // and then stop the root alone with the tree kill skipped. + if let Some(refusal) = refusal_before_signalling(state_of(identity)) { + return refusal; + } + // An identity that matches is only worth acting on while the PID cannot + // change hands before `taskkill` reaches it. Without a hold nothing + // reserves it between this read and the kill, which takes a whole tree by + // PID, so the stop is not confirmed and nothing is signalled. A process + // that could not be opened and also cannot be read was refused above. + if held.is_none() { + return TerminationOutcome::Unverified; + } + // Taken while the root is still alive: `taskkill /T` resolves children + // through the live parent, so it cannot reach them once the root is gone. + let tree = tree_kill(identity.pid); + // `terminate` still owns the bounded wait and the verdict. A root the + // tree kill already took now reads back as `AlreadyGone`, which would + // deny a stop this command did perform, so it counts as forced; but when + // the kill found no such process the root went by itself, and + // `AlreadyGone` is the truth. + let outcome = match (tree, terminate(identity)) { + (TreeKill::NotFound, outcome) => outcome, + (_, TerminationOutcome::AlreadyGone) => TerminationOutcome::Forced, + (_, outcome) => outcome, + }; + return after_tree_kill(tree, outcome); + } + terminate(identity) +} + +/// What keeps a PID reserved while a stop works on it: on Windows an open +/// handle to the process, which stops the OS giving the PID to another process; +/// nothing elsewhere, where the stop signals by PID and `terminate_verified` +/// re-verifies it. +#[cfg(windows)] +struct PidHold { + _handle: std::os::windows::io::OwnedHandle, +} + +#[cfg(not(windows))] +struct PidHold; + +/// Hold `identity`'s PID, or `None` when the process cannot be opened. A stop +/// whose identity matches but whose PID is not held is not carried out, see +/// [`terminate_recorded_pid_with`]. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn hold_pid(identity: &ProcessIdentity) -> Option { + use std::os::windows::io::{FromRawHandle, OwnedHandle}; + use windows_sys::Win32::System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}; + + // SAFETY: `OpenProcess` takes only plain values and returns a handle or null; + // there is no pointer to keep valid. + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, identity.pid) }; + if handle.is_null() { + return None; + } + Some(PidHold { + // SAFETY: `handle` is non-null, `OpenProcess` just returned it and nothing + // else owns or closes it, so the `OwnedHandle` may take ownership and + // close it exactly once on drop. + _handle: unsafe { OwnedHandle::from_raw_handle(handle) }, + }) +} + +// The `cfg` sibling calls non-const code, so this arm cannot be `const` without +// giving the two platforms different signatures. +#[cfg(not(windows))] +#[allow(clippy::missing_const_for_fn, clippy::unnecessary_wraps)] +fn hold_pid(_identity: &ProcessIdentity) -> Option { + Some(PidHold) +} + +/// The separate descendant kill this platform needs before the verified +/// termination of a root, if any: `taskkill /T /F` on Windows, none elsewhere, +/// where [`terminate_verified`] walks the tree itself. +// The `cfg` arms differ in what they return, so the non-Windows arm cannot be a +// `const fn` without giving the two platforms different signatures. +#[allow(clippy::missing_const_for_fn)] +fn platform_tree_kill() -> Option<&'static dyn Fn(u32) -> TreeKill> { + #[cfg(windows)] + { + Some(&taskkill_tree) + } + #[cfg(not(windows))] + { + None + } +} + +/// `taskkill`'s exit code for "the process was not found". +#[cfg(any(windows, test))] +const TASKKILL_PROCESS_NOT_FOUND: i32 = 128; + +/// The verdict on one `taskkill` run, from whether it started and the exit code +/// it ended with. A clean exit (0) is a completed tree kill and 128 is +/// [`TreeKill::NotFound`], left to the root's re-read to judge. A failed launch, +/// a run that did not finish within its deadline +/// ([`std::io::ErrorKind::TimedOut`], see [`run_to_exit_code`]), any other +/// nonzero code (a process it could not terminate) and a run that ended without +/// a code are all [`TreeKill::Failed`]. Split out of the process spawn so a test +/// runs it on every host. +#[cfg(any(windows, test))] +fn classify_taskkill(run: std::io::Result>) -> TreeKill { + match run { + Ok(Some(0)) => TreeKill::Completed, + Ok(Some(TASKKILL_PROCESS_NOT_FOUND)) => TreeKill::NotFound, + Ok(_) | Err(_) => TreeKill::Failed, + } +} + +/// How long the descendant kill may run. It happens before +/// [`terminate_verified`]'s bounded waits, so without its own deadline a stalled +/// helper would hang the stop; the same bound as every other wait of a managed +/// stop. +#[cfg(windows)] +const TREE_KILL_DEADLINE: Duration = MANAGED_STOP_GRACE; + +/// Start `command` and wait for it to exit, for at most `deadline`. Returns its +/// exit code (`None` when it ended without one). A child still running at the +/// deadline is killed and reaped, so none is left behind, and the call fails +/// with [`std::io::ErrorKind::TimedOut`]. +#[cfg(any(windows, test))] +fn run_to_exit_code( + command: &mut std::process::Command, + deadline: Duration, +) -> std::io::Result> { + let mut child = command.spawn()?; + wait_for_exit_code(&mut child, deadline) +} + +/// The wait of [`run_to_exit_code`], on a child already started. Kills and reaps +/// `child` when the deadline passes, or when it cannot be polled, rather than +/// leaving it running. +#[cfg(any(windows, test))] +fn wait_for_exit_code( + child: &mut std::process::Child, + deadline: Duration, +) -> std::io::Result> { + let end = Instant::now() + deadline; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status.code()), + Ok(None) => {} + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + } + let now = Instant::now(); + if now >= end { + let _ = child.kill(); + let _ = child.wait(); + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "the process did not exit before its deadline", + )); + } + std::thread::sleep(POLL_INTERVAL.min(end.saturating_duration_since(now))); + } +} + +/// The arguments that take `pid` and everything under it down: by PID, with its +/// tree (`/T`), forced (`/F`). Split out so a test pins them on every host. +#[cfg(any(windows, test))] +fn taskkill_args(pid: u32) -> [String; 4] { + [ + "/PID".to_owned(), + pid.to_string(), + "/T".to_owned(), + "/F".to_owned(), + ] +} + +#[cfg(windows)] +fn taskkill_tree(pid: u32) -> TreeKill { + use std::process::{Command, Stdio}; + classify_taskkill(run_to_exit_code( + Command::new(crate::windows_system_tool("taskkill.exe")) + .args(taskkill_args(pid)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + TREE_KILL_DEADLINE, + )) +} + /// Poll until every process in `members` has exited, or `grace` elapses. /// /// A member is exited once its PID is gone or now belongs to a different process @@ -300,8 +963,9 @@ fn send_signal(pid: u32, signal: Signal, _tree: bool) -> bool { /// Read the kernel start-time (field 22 of `/proc//stat`) for `pid`. /// -/// Returns `None` when the value cannot be read, including on non-Linux -/// platforms, where identity verification degrades to best-effort. +/// Returns `None` when the value cannot be read, including on platforms with +/// neither `/proc` nor a creation time to read (neither Linux nor Windows), +/// where identity verification degrades to best-effort. #[cfg(target_os = "linux")] #[must_use] pub fn process_start_ticks(pid: u32) -> Option { @@ -309,7 +973,56 @@ pub fn process_start_ticks(pid: u32) -> Option { parse_start_ticks(&stat) } -#[cfg(not(target_os = "linux"))] +/// Read the creation time of `pid` (`GetProcessTimes`) as the token that tells +/// the process from a later one that reuses its PID. +/// +/// The value is the creation time as a Windows `FILETIME` (100 ns intervals +/// since 1601), not clock ticks since boot: only equality with a value read +/// earlier from the same PID means anything. `None` when the process cannot be +/// opened for querying (it is gone, or belongs to a more privileged account). +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn process_start_ticks(pid: u32) -> Option { + use windows_sys::Win32::Foundation::{CloseHandle, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + if pid == 0 { + return None; + } + // SAFETY: plain Win32 calls. The handle is checked before use and closed on + // every path, and the four out-parameters are valid for the one call. + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return None; + } + let mut creation = FILETIME::default(); + let mut exit = FILETIME::default(); + let mut kernel = FILETIME::default(); + let mut user = FILETIME::default(); + let read = GetProcessTimes( + handle, + &raw mut creation, + &raw mut exit, + &raw mut kernel, + &raw mut user, + ); + CloseHandle(handle); + (read != 0).then(|| filetime_ticks(creation.dwLowDateTime, creation.dwHighDateTime)) + } +} + +/// The 64-bit value of a Windows `FILETIME`, which a Win32 call returns as two +/// 32-bit halves. Kept apart from the FFI so a test can run it on every host. +#[cfg(any(windows, test))] +fn filetime_ticks(low: u32, high: u32) -> u64 { + (u64::from(high) << 32) | u64::from(low) +} + +#[cfg(not(any(target_os = "linux", windows)))] #[must_use] // Platform stub. The `cfg` sibling calls non-const code, so making only this // arm `const` would give the two platforms different signatures and push @@ -418,6 +1131,25 @@ mod tests { assert!(TerminationOutcome::AlreadyGone.stopped()); assert!(TerminationOutcome::IdentityMismatch.stopped()); assert!(!TerminationOutcome::TimedOut.stopped()); + // The root is gone but a process under it may not be: not a stop, not + // graceful, and not the label of a timeout. + assert!(!TerminationOutcome::DescendantsUnconfirmed.stopped()); + assert!(!TerminationOutcome::DescendantsUnconfirmed.graceful()); + assert_eq!( + TerminationOutcome::DescendantsUnconfirmed.as_str(), + "descendants_unconfirmed" + ); + assert_ne!( + TerminationOutcome::DescendantsUnconfirmed.as_str(), + TerminationOutcome::TimedOut.as_str() + ); + // A root that exited by itself before its tree kill: not a stop alone. + assert!(!TerminationOutcome::ExitedBeforeTreeKill.stopped()); + assert!(!TerminationOutcome::ExitedBeforeTreeKill.graceful()); + assert_eq!( + TerminationOutcome::ExitedBeforeTreeKill.as_str(), + "exited_before_tree_kill" + ); // Unverified: neither stopped nor graceful — we could not act. assert!(!TerminationOutcome::Unverified.stopped()); @@ -445,7 +1177,7 @@ mod tests { let _ = child.wait(); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_matches_our_own_process() { let pid = std::process::id(); @@ -454,7 +1186,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Matches); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_mismatch_when_start_ticks_differ() { // Same live PID (ours), but a different recorded start-time: a recycled @@ -474,7 +1206,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Gone); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn refuses_to_signal_on_identity_mismatch() { // Our own PID with a wrong start-time. A forced stop must NOT kill us: @@ -614,4 +1346,1221 @@ mod tests { ); reap(child); } + + // ---- The managed-stop routine `rocm` and `rocmd` share. ---- + + use std::cell::RefCell; + + /// A PID no host has in use. + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + fn record_naming( + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> crate::ManagedServiceRecord { + let paths = crate::AppPaths { + config_dir: std::path::PathBuf::from("config"), + data_dir: std::path::PathBuf::from("data"), + cache_dir: std::path::PathBuf::from("cache"), + }; + let mut record = crate::ManagedServiceRecord::new( + &paths, + "svc-managed-stop", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record + } + + /// The identities `terminate_recorded_service_pids` handed its termination + /// function, in call order, and the stops it reported. + fn stop_with( + record: &crate::ManagedServiceRecord, + outcome: TerminationOutcome, + ) -> (Vec, Vec) { + let seen = RefCell::new(Vec::new()); + let stops = terminate_recorded_service_pids(record, |identity| { + seen.borrow_mut().push(*identity); + outcome + }); + (seen.into_inner(), stops) + } + + #[test] + fn managed_stop_grace_is_ten_seconds() { + assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + } + + #[test] + fn managed_stop_terminates_supervisor_then_engine_each_with_its_own_start_time() { + let record = record_naming(SUPERVISOR_PID, Some(11), Some(ENGINE_PID), Some(22)); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ + ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + ProcessIdentity::new(ENGINE_PID, Some(22)), + ], + "the supervisor first, and no start-time read across from the other PID" + ); + assert_eq!( + stops + .iter() + .map(|stop| (stop.pid, stop.role)) + .collect::>(), + vec![(SUPERVISOR_PID, "supervisor"), (ENGINE_PID, "engine")] + ); + } + + #[test] + fn managed_stop_terminates_a_pid_recorded_twice_once_and_keeps_a_known_start_time() { + // Either entry may be the one that carries the token. + for (supervisor_ticks, engine_ticks, expected) in [ + (None, Some(33), Some(33)), + (Some(44), None, Some(44)), + (Some(44), Some(33), Some(44)), + (None, None, None), + ] { + let record = record_naming( + SUPERVISOR_PID, + supervisor_ticks, + Some(SUPERVISOR_PID), + engine_ticks, + ); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(SUPERVISOR_PID, expected)], + "{supervisor_ticks:?} / {engine_ticks:?}" + ); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + } + } + + #[test] + fn managed_stop_skips_a_zero_pid_and_an_absent_engine() { + let record = record_naming(0, None, Some(0), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert!(stops.is_empty()); + } + + #[test] + fn managed_stop_reports_its_own_pid_as_self_and_never_terminates_it() { + let own = std::process::id(); + let record = record_naming(own, None, Some(ENGINE_PID), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(ENGINE_PID, None)], + "the stopping process is never signalled" + ); + assert_eq!(stops.len(), 2); + assert_eq!(stops[0].pid, own); + assert_eq!(stops[0].role, "supervisor"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + assert_eq!(stops[0].outcome.as_str(), "self"); + assert!(!stops[0].signaled() && !stops[0].forced()); + assert!(stops[0].stopped(), "the launcher is not the service"); + assert_eq!( + stops[0].report_entry(), + serde_json::json!({"pid": own, "role": "supervisor", "outcome": "self"}) + ); + } + + #[test] + fn managed_stop_reports_a_pid_that_is_both_roles_and_its_own_as_self() { + let own = std::process::id(); + let record = record_naming(own, None, Some(own), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + } + + #[test] + fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { + use TerminationOutcome::{ + AlreadyGone, DescendantsUnconfirmed, ExitedBeforeTreeKill, Forced, Graceful, + IdentityMismatch, TimedOut, Unverified, + }; + let record = record_naming(SUPERVISOR_PID, None, None, None); + + // (outcome, signaled, forced, stopped) + let expected = [ + (AlreadyGone, false, false, true), + (IdentityMismatch, false, false, true), + (Unverified, false, false, false), + (Graceful, true, false, true), + (Forced, true, true, true), + (TimedOut, true, true, false), + // Signalled and forced by the tree kill, but not confirmed gone. + (DescendantsUnconfirmed, true, true, false), + // Nothing signalled: the root was gone before the tree kill ran. + (ExitedBeforeTreeKill, false, false, false), + ]; + for (outcome, signaled, forced, stopped) in expected { + let (_, stops) = stop_with(&record, outcome); + assert_eq!(stops.len(), 1); + let stop = stops[0]; + assert_eq!(stop.outcome, RecordedPidOutcome::Terminated(outcome)); + assert_eq!(stop.outcome.as_str(), outcome.as_str()); + assert_eq!( + (stop.signaled(), stop.forced(), stop.stopped()), + (signaled, forced, stopped), + "{outcome:?}" + ); + } + } + + /// What one `terminate_recorded_pid_with` call did, in order. + #[derive(Debug, PartialEq, Eq)] + enum Step { + Hold(u32), + /// The hold could not be taken: nothing reserves the PID. + HoldFailed(u32), + TreeKill(u32), + Terminate(u32), + Release, + } + + /// A stand-in for the Windows PID hold: records that it was taken and, when + /// it is dropped, that it was released. + struct RecordedHold<'a>(&'a RefCell>); + + impl Drop for RecordedHold<'_> { + fn drop(&mut self) { + self.0.borrow_mut().push(Step::Release); + } + } + + /// Run `terminate_recorded_pid_with` for `pid`, with a tree-kill step + /// present or not and ending as `tree`, the identity in `state`, and the + /// root's termination answering `outcome`. Returns the verdict and the steps + /// taken, the hold's release included. + fn tree_kill_run_with( + has_tree_kill: bool, + tree: TreeKill, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + tree + }; + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, None), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |identity| { + steps.borrow_mut().push(Step::Hold(identity.pid)); + Some(RecordedHold(&steps)) + }, + |_| state, + |identity| { + steps.borrow_mut().push(Step::Terminate(identity.pid)); + outcome + }, + ); + (verdict, steps.into_inner()) + } + + /// [`tree_kill_run_with`] for a tree kill that completes. + fn tree_kill_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + tree_kill_run_with(has_tree_kill, TreeKill::Completed, state, outcome) + } + + #[test] + fn the_tree_is_killed_before_the_root_is_terminated() { + let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!( + steps, + vec![ + Step::Hold(SUPERVISOR_PID), + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release, + ], + "`taskkill /T` resolves children through the live parent, so the \ + tree kill has to come first" + ); + } + + #[test] + fn a_root_the_tree_kill_already_took_is_reported_forced_not_already_gone() { + let (verdict, _) = tree_kill_run( + true, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + verdict, + TerminationOutcome::Forced, + "the stop did perform the kill; reading the root back as gone must not deny it" + ); + } + + #[test] + fn the_verdict_after_a_tree_kill_is_the_terminations_own_for_every_other_outcome() { + use TerminationOutcome::{Forced, Graceful, IdentityMismatch, TimedOut, Unverified}; + for outcome in [Graceful, Forced, TimedOut, Unverified, IdentityMismatch] { + let (verdict, _) = tree_kill_run(true, IdentityState::Matches, outcome); + + assert_eq!(verdict, outcome); + } + } + + #[test] + fn no_tree_kill_is_run_for_a_pid_that_is_gone_recycled_or_unverifiable() { + for state in [ + IdentityState::Gone, + IdentityState::Recycled, + IdentityState::Indeterminate, + ] { + let (verdict, steps) = tree_kill_run(true, state, TerminationOutcome::AlreadyGone); + + assert_eq!( + steps, + vec![Step::Hold(SUPERVISOR_PID), Step::Release], + "{state:?}: a stranger's process tree must not be taken, and the root \ + is not handed to the termination, which would re-read the identity" + ); + let expected = match state { + IdentityState::Gone => TerminationOutcome::AlreadyGone, + IdentityState::Recycled => TerminationOutcome::IdentityMismatch, + _ => TerminationOutcome::Unverified, + }; + assert_eq!( + verdict, expected, + "{state:?}: the initial refusal is the verdict" + ); + } + } + + #[test] + fn without_a_tree_kill_step_the_root_is_terminated_and_its_verdict_unmapped() { + let (verdict, steps) = tree_kill_run( + false, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + steps, + vec![ + Step::Hold(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release + ] + ); + assert_eq!(verdict, TerminationOutcome::AlreadyGone); + } + + /// Every way a recorded token and the live process's token can stand, with + /// what the stop does about each: whether the Windows tree kill runs, and the + /// verdict the stop reports (the label and whether it counts as stopped). + /// + /// The live process is running in every row; the tokens go through the same + /// decision production uses ([`classify_live_identity`] and + /// [`refusal_before_signalling`]), with the system reads and the signalling + /// supplied, so each row runs on every host. A process is refused only when + /// a token was recorded and the live one is provably different. + #[test] + fn what_a_stop_does_for_each_recorded_and_live_token() { + const LIVE: u64 = 0x01DC_0000_0000_0001; + struct Row { + name: &'static str, + recorded: Option, + live: Option, + tree_kill_runs: bool, + label: &'static str, + stopped: bool, + } + let row = |name, recorded, live, tree_kill_runs, label, stopped| Row { + name, + recorded, + live, + tree_kill_runs, + label, + stopped, + }; + let rows = [ + // The shape `service-stop-07` records, and any record older than the + // token: nothing to refute, so the whole tree is taken as before. + row("no token recorded", None, Some(LIVE), true, "forced", true), + row( + "no token recorded, none readable", + None, + None, + true, + "forced", + true, + ), + row( + "token matches", + Some(LIVE), + Some(LIVE), + true, + "forced", + true, + ), + // A recycled PID: the process is a stranger, left alone. + row( + "token differs", + Some(LIVE + 1), + Some(LIVE), + false, + "identity_mismatch", + true, + ), + // A recorded token that cannot be read back is neither signalled nor + // reported stopped. + row( + "token unreadable", + Some(LIVE), + None, + false, + "unverified", + false, + ), + ]; + for Row { + name: row, + recorded, + live, + tree_kill_runs, + label, + stopped, + } in rows + { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |id| { + steps.borrow_mut().push(Step::Hold(id.pid)); + Some(RecordedHold(&steps)) + }, + |id| classify_live_identity(id.start_ticks, live), + |id| { + steps.borrow_mut().push(Step::Terminate(id.pid)); + // What the verified termination answers: its refusal for a + // process the identity rules out, otherwise the tree kill + // has already taken the root and it reads back as gone. + refusal_before_signalling(classify_live_identity(id.start_ticks, live)) + .unwrap_or(TerminationOutcome::AlreadyGone) + }, + ); + let expected_steps = if tree_kill_runs { + vec![ + Step::Hold(SUPERVISOR_PID), + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release, + ] + } else { + // Refused on the initial read: not handed to the termination. + vec![Step::Hold(SUPERVISOR_PID), Step::Release] + }; + assert_eq!(steps.into_inner(), expected_steps, "{row}: steps"); + assert_eq!(verdict.as_str(), label, "{row}: verdict"); + assert_eq!(verdict.stopped(), stopped, "{row}: counts as stopped"); + } + } + + #[test] + fn a_recorded_token_refutes_a_live_one_only_when_both_are_known_and_differ() { + use IdentityState::{Indeterminate, Matches, Recycled}; + assert_eq!(classify_live_identity(None, None), Matches); + assert_eq!(classify_live_identity(None, Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(8)), Recycled); + assert_eq!(classify_live_identity(Some(7), None), Indeterminate); + assert_eq!(refusal_before_signalling(Matches), None); + } + + #[test] + fn only_windows_has_a_separate_tree_kill_step() { + // Ties the injected tests above to what production actually runs: the + // step they exercise is the one `terminate_recorded_pid` has on Windows + // and does without elsewhere, where `terminate_verified` walks the tree. + assert_eq!(platform_tree_kill().is_some(), cfg!(windows)); + } + + /// Run `terminate_recorded_pid_with` for a PID whose hold could not be taken + /// (the process could not be opened), the identity reading as `state`. + fn unheld_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, None), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |identity| { + steps.borrow_mut().push(Step::HoldFailed(identity.pid)); + None::> + }, + |_| state, + |identity| { + steps.borrow_mut().push(Step::Terminate(identity.pid)); + outcome + }, + ); + (verdict, steps.into_inner()) + } + + #[test] + fn a_tree_kill_is_not_run_for_a_pid_that_could_not_be_held_even_when_its_identity_matches() { + // Nothing reserves the PID between the identity read and `taskkill`, which + // takes a whole tree by PID, so a recycled PID would lose its tree. The + // stop is not confirmed, and neither the tree kill nor the termination runs. + let (verdict, steps) = unheld_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert_eq!(steps, vec![Step::HoldFailed(SUPERVISOR_PID)]); + } + + #[test] + fn an_unheld_pid_that_is_gone_recycled_or_unverifiable_keeps_its_refusal() { + // The hold only adds a refusal for an identity that matches: a process + // already known to be gone, a stranger, or unreadable keeps the verdict + // the identity read gives it. + for (state, expected) in [ + (IdentityState::Gone, TerminationOutcome::AlreadyGone), + ( + IdentityState::Recycled, + TerminationOutcome::IdentityMismatch, + ), + (IdentityState::Indeterminate, TerminationOutcome::Unverified), + ] { + let (verdict, steps) = unheld_run(true, state, TerminationOutcome::Forced); + + assert_eq!(verdict, expected, "{state:?}"); + assert_eq!(steps, vec![Step::HoldFailed(SUPERVISOR_PID)], "{state:?}"); + } + } + + #[test] + fn without_a_tree_kill_step_an_unheld_pid_is_still_terminated() { + // Elsewhere nothing is held and the termination re-verifies the identity + // itself, so the missing hold must not turn a stop into a refusal. + let (verdict, steps) = + unheld_run(false, IdentityState::Matches, TerminationOutcome::Graceful); + + assert_eq!(verdict, TerminationOutcome::Graceful); + assert_eq!( + steps, + vec![ + Step::HoldFailed(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID) + ] + ); + } + + #[test] + fn the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop() { + // The hold is what makes the creation time that was compared the + // creation time of the process the kill then reaches by PID: taken + // first, released last. `Hold` comes before anything else and `Release` + // after the root's termination, with the tree kill between. + let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!(steps.first(), Some(&Step::Hold(SUPERVISOR_PID))); + assert_eq!(steps.last(), Some(&Step::Release)); + assert_eq!( + steps + .iter() + .filter(|step| matches!(step, Step::Release)) + .count(), + 1 + ); + } + + #[test] + fn a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed() { + use TerminationOutcome::{ + AlreadyGone, DescendantsUnconfirmed, Forced, Graceful, TimedOut, Unverified, + }; + // Every root verdict that would confirm the stop becomes the one that + // says the root is gone but a process under it may still be running, not + // `timed_out`, which would say something outlived a deadline; the + // verdicts that already deny the stop are kept. + for (root, expected) in [ + (Forced, DescendantsUnconfirmed), + (Graceful, DescendantsUnconfirmed), + (AlreadyGone, DescendantsUnconfirmed), + (TimedOut, TimedOut), + (Unverified, Unverified), + ] { + let (verdict, steps) = + tree_kill_run_with(true, TreeKill::Failed, IdentityState::Matches, root); + + assert_eq!(verdict, expected, "root {root:?}"); + assert!( + !verdict.stopped(), + "root {root:?}: the stop is not confirmed" + ); + assert!( + steps.contains(&Step::Terminate(SUPERVISOR_PID)), + "the root is still terminated, best effort: {steps:?}" + ); + } + } + + /// `taskkill` answering "no such process" is judged by the root's own + /// re-read. The root that went away by itself between the identity check and + /// the kill is not reported as a kill this stop performed, and not as a stop + /// either on its own: `taskkill /T` reaches children only through a live + /// parent, so a child the root left running was out of its reach, and nothing + /// recorded what was under it. The record-level settling decides. + #[test] + fn a_tree_kill_that_found_no_process_with_the_root_gone_is_not_a_stop_by_itself() { + let (verdict, steps) = tree_kill_run_with( + true, + TreeKill::NotFound, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + verdict, + TerminationOutcome::ExitedBeforeTreeKill, + "the root exited on its own and nothing says what it left running" + ); + assert!(!verdict.stopped()); + assert!( + steps.contains(&Step::Terminate(SUPERVISOR_PID)), + "the root is re-read by the termination: {steps:?}" + ); + } + + /// The identity read that gates the tree kill is the one that decides the + /// stop. If it is unverifiable and a second read, inside the termination, + /// would find the process verifiable, the root must not be stopped alone with + /// the tree kill skipped: the initial refusal is the verdict and the + /// termination is never called. + #[test] + fn an_unverifiable_first_read_is_not_overtaken_by_a_verifiable_second_read() { + use std::cell::Cell; + let reads = Cell::new(0_u32); + let terminated = Cell::new(false); + let tree_killed = Cell::new(false); + let tree_kill = |_pid: u32| { + tree_killed.set(true); + TreeKill::Completed + }; + + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |_| Some(()), + |_| { + // Indeterminate the first time, matching from then on. + reads.set(reads.get() + 1); + if reads.get() == 1 { + IdentityState::Indeterminate + } else { + IdentityState::Matches + } + }, + |_| { + terminated.set(true); + TerminationOutcome::Forced + }, + ); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert!(!terminated.get(), "the root is not stopped alone"); + assert!(!tree_killed.get()); + assert_eq!(reads.get(), 1, "the initial read is the only one"); + } + + /// A sibling whose PID a stranger now holds ([`IdentityMismatch`]) says + /// nothing about what was under this root, so it does not corroborate it. + #[test] + fn an_identity_mismatch_sibling_does_not_corroborate_a_root_that_exited_alone() { + use TerminationOutcome::{ExitedBeforeTreeKill, IdentityMismatch}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, ExitedBeforeTreeKill), + (ENGINE_PID, IdentityMismatch), + ], + ); + + assert_eq!(verdicts, vec![ExitedBeforeTreeKill, IdentityMismatch]); + assert!(!verdicts[0].stopped()); + } + + /// What the record's processes say together about a root that exited by + /// itself. `exits` maps each recorded PID to the verdict `terminate` gives. + fn settled( + record: &crate::ManagedServiceRecord, + exits: &[(u32, TerminationOutcome)], + ) -> Vec { + terminate_recorded_service_pids(record, |identity| { + exits + .iter() + .find(|(pid, _)| *pid == identity.pid) + .map(|(_, outcome)| *outcome) + .expect("a verdict for every recorded pid") + }) + .into_iter() + .map(|stop| match stop.outcome { + RecordedPidOutcome::Terminated(outcome) => outcome, + RecordedPidOutcome::OwnProcess => panic!("not the stopping process"), + }) + .collect() + } + + /// A parent that exits by itself while its child keeps running is not a + /// stop: with only the parent recorded, nothing independently verifies the + /// child, so the stop stays unconfirmed (and the record, the stop request + /// and the key are kept by the callers). This is the case a root's exit + /// cannot settle. + #[test] + fn a_parent_that_exits_alone_with_nothing_else_recorded_is_not_a_confirmed_stop() { + use TerminationOutcome::ExitedBeforeTreeKill; + let record = record_naming(SUPERVISOR_PID, None, None, None); + + let verdicts = settled(&record, &[(SUPERVISOR_PID, ExitedBeforeTreeKill)]); + + assert_eq!(verdicts, vec![ExitedBeforeTreeKill]); + assert!(!verdicts[0].stopped()); + } + + /// The same parent with its engine recorded: the engine is the process under + /// it that is independently verified, by its own entry. Confirmed gone, the + /// stop is confirmed, which keeps the concern that a root exiting on its own + /// must not leave a stop unconfirmed for nothing. + #[test] + fn a_parent_that_exits_alone_is_confirmed_when_the_recorded_engine_is_confirmed_gone() { + use TerminationOutcome::{AlreadyGone, ExitedBeforeTreeKill, Forced, Graceful}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + for engine in [Forced, Graceful, AlreadyGone] { + let verdicts = settled( + &record, + &[(SUPERVISOR_PID, ExitedBeforeTreeKill), (ENGINE_PID, engine)], + ); + + assert_eq!(verdicts, vec![AlreadyGone, engine], "engine {engine:?}"); + assert!(verdicts.iter().all(|verdict| verdict.stopped())); + } + // And the other way round: the engine is the root that exited alone. + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, Graceful), + (ENGINE_PID, ExitedBeforeTreeKill), + ], + ); + assert_eq!(verdicts, vec![Graceful, AlreadyGone]); + } + + /// A parent that exits while the recorded engine is still running, or not + /// confirmed, stays unconfirmed: the engine is the child that survived. + #[test] + fn a_parent_that_exits_alone_stays_unconfirmed_while_the_recorded_engine_is_not_gone() { + use TerminationOutcome::{ + DescendantsUnconfirmed, ExitedBeforeTreeKill, TimedOut, Unverified, + }; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + for engine in [TimedOut, Unverified, DescendantsUnconfirmed] { + let verdicts = settled( + &record, + &[(SUPERVISOR_PID, ExitedBeforeTreeKill), (ENGINE_PID, engine)], + ); + + assert_eq!( + verdicts, + vec![ExitedBeforeTreeKill, engine], + "engine {engine:?}" + ); + assert!( + !verdicts[0].stopped(), + "the parent is not settled by {engine:?}" + ); + } + } + + /// Two roots that both exited by themselves corroborate each other: every + /// process the record names is gone. + #[test] + fn two_roots_that_both_exited_alone_are_a_confirmed_stop() { + use TerminationOutcome::{AlreadyGone, ExitedBeforeTreeKill}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, ExitedBeforeTreeKill), + (ENGINE_PID, ExitedBeforeTreeKill), + ], + ); + + assert_eq!(verdicts, vec![AlreadyGone, AlreadyGone]); + } + + /// The stopping process's own PID is not a service process: it never + /// corroborates a root that exited alone. + #[test] + fn the_stopping_process_does_not_corroborate_a_root_that_exited_alone() { + use TerminationOutcome::ExitedBeforeTreeKill; + let record = record_naming(SUPERVISOR_PID, None, Some(std::process::id()), None); + + let stops = terminate_recorded_service_pids(&record, |_| ExitedBeforeTreeKill); + + assert_eq!( + stops[0].outcome, + RecordedPidOutcome::Terminated(ExitedBeforeTreeKill) + ); + assert_eq!(stops[1].outcome, RecordedPidOutcome::OwnProcess); + assert!(!stops[0].stopped()); + } + + /// "No such process" while the root is still there is a contradiction, so it + /// is a failed kill and the stop stays unconfirmed, as for any other nonzero + /// exit. + #[test] + fn a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill() { + use TerminationOutcome::{DescendantsUnconfirmed, Forced, Graceful, TimedOut, Unverified}; + // The termination saw the root exit (Graceful, Forced) although the kill + // said there was no such process: the exit was observed, so the report + // must not say it outlived a deadline, but nothing watched what is under + // it. + for (root, expected) in [ + (Forced, DescendantsUnconfirmed), + (Graceful, DescendantsUnconfirmed), + (TimedOut, TimedOut), + (Unverified, Unverified), + ] { + let (verdict, _) = + tree_kill_run_with(true, TreeKill::NotFound, IdentityState::Matches, root); + + assert_eq!(verdict, expected, "root {root:?}"); + assert!(!verdict.stopped(), "root {root:?}"); + } + } + + /// Any other nonzero exit with the root gone still leaves the stop + /// unconfirmed: unlike "not found", it says a process could not be + /// terminated, so a descendant may survive the root. + #[test] + fn a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed() { + let (verdict, _) = tree_kill_run_with( + true, + TreeKill::Failed, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!(verdict, TerminationOutcome::DescendantsUnconfirmed); + assert!(!verdict.stopped()); + } + + /// A recorded token that can be read and differs proves another process + /// holds the PID, even when the rest of the query failed; without a readable + /// token the PID stays unverifiable. + #[test] + fn a_readable_token_mismatch_outranks_a_failed_query() { + use crate::process::ProcessProbe::Unqueryable; + let state = |recorded, live| { + identity_state_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + |_| Unqueryable, + |_| false, + |_| live, + ) + }; + + assert_eq!(state(Some(7), Some(8)), IdentityState::Recycled); + assert_eq!(state(Some(7), Some(7)), IdentityState::Indeterminate); + assert_eq!(state(Some(7), None), IdentityState::Indeterminate); + assert_eq!(state(None, Some(8)), IdentityState::Indeterminate); + assert_eq!(state(None, None), IdentityState::Indeterminate); + } + + #[test] + fn a_completed_tree_kill_leaves_the_roots_verdict_alone() { + use TerminationOutcome::{Forced, Graceful, TimedOut, Unverified}; + for outcome in [Forced, Graceful, TimedOut, Unverified] { + let (verdict, _) = + tree_kill_run_with(true, TreeKill::Completed, IdentityState::Matches, outcome); + + assert_eq!(verdict, outcome); + } + } + + /// A command that exits at once with `code`, and one that runs far longer + /// than any test waits, on whichever platform the tests run. + fn exits_with(code: i32) -> std::process::Command { + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sh"); + command.args(["-c", &format!("exit {code}")]); + command + }; + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("cmd"); + command.args(["/C", &format!("exit {code}")]); + command + }; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + command + } + + fn runs_for_thirty_seconds() -> std::process::Command { + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sleep"); + command.arg("30"); + command + }; + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("ping"); + command.args(["-n", "30", "127.0.0.1"]); + command + }; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + command + } + + #[test] + fn a_helper_that_exits_in_time_gives_its_exit_code() { + let deadline = Duration::from_secs(20); + + assert_eq!( + run_to_exit_code(&mut exits_with(0), deadline).unwrap(), + Some(0) + ); + assert_eq!( + run_to_exit_code(&mut exits_with(3), deadline).unwrap(), + Some(3) + ); + } + + #[test] + fn a_helper_that_cannot_be_started_is_an_error() { + let mut command = std::process::Command::new("rocm-core-test-no-such-program"); + + assert!(run_to_exit_code(&mut command, Duration::from_secs(1)).is_err()); + } + + /// A helper that stalls must not hold the stop: it is killed and reaped at + /// its deadline, the call ends close to it, and the verdict is a failed tree + /// kill, which leaves the stop unconfirmed. + #[test] + fn a_helper_that_stalls_is_killed_and_reaped_at_its_deadline() { + let mut command = runs_for_thirty_seconds(); + let mut child = command.spawn().expect("spawn the stalled helper"); + let pid = child.id(); + let started = Instant::now(); + + let run = wait_for_exit_code(&mut child, Duration::from_millis(200)); + let elapsed = started.elapsed(); + + let error = run.expect_err("a stalled helper does not finish"); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert!( + elapsed >= Duration::from_millis(200) && elapsed < Duration::from_secs(10), + "returned after {elapsed:?}, far from the deadline" + ); + // Reaped: its status is already collected, and the process is gone. + assert!( + matches!(child.try_wait(), Ok(Some(_))), + "the killed helper was waited for" + ); + assert!( + !crate::process_is_running(pid), + "the helper was killed, not left running" + ); + assert_eq!(classify_taskkill(Err(error)), TreeKill::Failed); + } + + #[test] + fn taskkill_is_asked_for_the_pid_its_tree_and_force() { + assert_eq!(taskkill_args(4242), ["/PID", "4242", "/T", "/F"]); + } + + #[test] + fn only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found() { + use std::io::{Error, ErrorKind}; + assert_eq!(classify_taskkill(Ok(Some(0))), TreeKill::Completed); + // A process it could not terminate. + assert_eq!(classify_taskkill(Ok(Some(1))), TreeKill::Failed); + assert_eq!(classify_taskkill(Ok(Some(5))), TreeKill::Failed); + // No such process: not a failure by itself, the root's re-read decides. + assert_eq!(classify_taskkill(Ok(Some(128))), TreeKill::NotFound); + // Ended without an exit code. + assert_eq!(classify_taskkill(Ok(None)), TreeKill::Failed); + // Could not be started, or did not finish before its deadline. + assert_eq!( + classify_taskkill(Err(Error::from(ErrorKind::NotFound))), + TreeKill::Failed + ); + assert_eq!( + classify_taskkill(Err(Error::from(ErrorKind::TimedOut))), + TreeKill::Failed + ); + } + + /// A PID that cannot be queried is neither gone nor a match, through the + /// same liveness gate every stop starts with. + #[test] + fn a_pid_that_cannot_be_queried_is_unverified_not_already_gone() { + use crate::process::ProcessProbe::{Gone, Running, Unqueryable}; + let state = |probe, zombie, recorded, live| { + identity_state_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + |_| probe, + |_| zombie, + |_| live, + ) + }; + // Whatever token was recorded, and whatever can be read of the process, + // short of a readable token that differs (`a_readable_token_mismatch_...`). + for (recorded, live) in [ + (None, None), + (None, Some(7)), + (None, Some(8)), + (Some(7), None), + (Some(7), Some(7)), + ] { + assert_eq!( + state(Unqueryable, false, recorded, live), + IdentityState::Indeterminate, + "recorded {recorded:?}, live {live:?}" + ); + } + assert_eq!(state(Gone, false, Some(7), Some(7)), IdentityState::Gone); + assert_eq!(state(Running, true, Some(7), Some(7)), IdentityState::Gone); + assert_eq!( + state(Running, false, Some(7), Some(7)), + IdentityState::Matches + ); + assert_eq!( + state(Running, false, Some(7), Some(8)), + IdentityState::Recycled + ); + assert_eq!( + state(Running, false, Some(7), None), + IdentityState::Indeterminate + ); + } + + /// The whole stop for a live process the caller may not query: not + /// signalled, no tree kill, and reported as unverified rather than as + /// stopped, so the record and the endpoint key are kept. + #[test] + fn a_stop_of_a_live_process_it_cannot_query_is_not_confirmed() { + use crate::process::ProcessProbe::Unqueryable; + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let state_of = + |id: &ProcessIdentity| identity_state_with(id, |_| Unqueryable, |_| false, |_| None); + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |_| Some(()), + state_of, + // The verified termination's own gate, with the same read. + |id| refusal_before_signalling(state_of(id)).unwrap_or(TerminationOutcome::Forced), + ); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert!( + steps.into_inner().is_empty(), + "a process that cannot be told from a stranger is not tree-killed" + ); + } + + #[test] + fn terminate_recorded_pid_reports_an_absent_pid_as_already_gone() { + let outcome = terminate_recorded_pid(&ProcessIdentity::new(SUPERVISOR_PID, None)); + + assert_eq!(outcome, TerminationOutcome::AlreadyGone); + } + + #[cfg(unix)] + #[test] + fn terminate_recorded_pid_stops_a_running_verified_process() { + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let id = ProcessIdentity::capture(child.id()); + + let outcome = terminate_recorded_pid(&id); + + assert!(outcome.stopped(), "{outcome:?}"); + assert!(matches!( + outcome, + TerminationOutcome::Graceful | TerminationOutcome::Forced + )); + reap(child); + assert!(!crate::process_is_running(id.pid)); + } + + #[test] + fn a_filetime_is_its_high_word_above_its_low_word() { + // The Windows creation time arrives as two 32-bit halves; swapping them + // or dropping the shift would make every process read as a different one. + assert_eq!( + filetime_ticks(0x89AB_CDEF, 0x0123_4567), + 0x0123_4567_89AB_CDEF + ); + assert_eq!(filetime_ticks(1, 0), 1); + assert_eq!(filetime_ticks(0, 1), 1 << 32); + assert_eq!(filetime_ticks(u32::MAX, u32::MAX), u64::MAX); + } + + /// The managed stop's own entry point refuses a PID whose recorded identity no + /// longer matches, and reaches neither it nor its tree. Our own PID with a + /// wrong start-time is what a recycled PID looks like; if the stop went + /// ahead, `taskkill /T /F` on Windows (a plain forced kill elsewhere) would + /// end this test process instead of asserting. On Windows this runs the real + /// tree-kill gate: the verdict has to come from the identity check, before + /// any `taskkill`. + #[cfg(any(target_os = "linux", windows))] + #[test] + fn terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches() { + let pid = std::process::id(); + let real = process_start_ticks(pid).expect("own start-time"); + let stale = ProcessIdentity::new(pid, Some(real.wrapping_add(1))); + + let outcome = terminate_recorded_pid(&stale); + + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!(crate::process_is_running(pid), "nothing was signalled"); + } + + /// The same refusal against a bystander that is a different process holding + /// the recorded PID (the recycled case), then the stop of that process when + /// its identity does match. Windows only: that is where a tree kill by PID + /// follows the identity gate, and where a creation-time token is read. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + let real = process_start_ticks(pid).expect("a live process has a creation time"); + assert_eq!( + process_start_ticks(pid), + Some(real), + "the creation time of one process does not change" + ); + assert_ne!( + Some(real), + process_start_ticks(std::process::id()), + "two processes do not share one" + ); + + let outcome = + terminate_recorded_pid(&ProcessIdentity::new(pid, Some(real.wrapping_add(1)))); + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!( + child.try_wait().expect("check the bystander").is_none(), + "a PID whose recorded creation time differs is a stranger and must keep running" + ); + + let outcome = terminate_recorded_pid(&ProcessIdentity::capture(pid)); + assert!(outcome.stopped(), "{outcome:?}"); + let _ = child.wait(); + assert!(!crate::process_is_running(pid)); + } + + /// A record with no token (written before Windows recorded one, and the shape + /// `service-stop-07` plants) cannot be refuted and still gets the whole stop: + /// the process ends. Windows only, where a real process is stopped by PID. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_stops_a_process_recorded_without_a_token() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + + let outcome = terminate_recorded_pid(&ProcessIdentity::new(pid, None)); + + assert!(outcome.stopped(), "{outcome:?}"); + assert_eq!(outcome.as_str(), "forced"); + assert!( + child.wait().is_ok() && !crate::process_is_running(pid), + "the process recorded without a token must be gone" + ); + } } diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 2e285e3169..5751d17c74 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -10,19 +10,11 @@ //! processes — e.g. the daemon autostart path, the managed-launch GPU claim, //! a `uv` install, and each JSONL store's append/rewrite. //! -//! [`run_with_timeout`] is the bulk of the file: run a command to completion -//! with both streams captured and a real time bound. A thread per pipe drains -//! stdout and stderr from the moment the child starts, so output size cannot -//! stall a command into its own timeout, and a reader that cannot finish is -//! abandoned rather than joined, so a grandchild holding the pipe's write end -//! open cannot outlast the deadline either. The timeout error quotes a bounded -//! head of the child's own output, because that text reaches persisted -//! automation logs. `Capture`, `run_captured`, `wait_bounded`, `Waitable` and -//! `assemble_output` exist to serve it. -//! -//! It is not yet the only such helper in the tree; converging the remaining -//! capture helpers onto it is tracked in -//! [#623](https://github.com/ROCm/rocm-cli/issues/623). +//! The bounded subprocess executor — [`run_with_timeout`], which runs a command +//! to completion with both streams captured and a real time bound, and +//! [`run_bounded`], the same executor returning a typed capture — lives in the +//! private `bounded` submodule, split out to keep this file under the module +//! size limit, and is re-exported from here under its original paths. use crate::DEFAULT_LOCAL_HOST; use anyhow::{Context, Result, bail}; @@ -35,6 +27,9 @@ use std::net::TcpListener; #[cfg(windows)] use std::os::windows::ffi::OsStrExt; use std::path::{Path, PathBuf}; +use std::process::Command; +use std::thread; +use std::time::{Duration, Instant}; #[cfg(windows)] use windows_sys::Win32::System::Threading::{ CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, CREATE_UNICODE_ENVIRONMENT, @@ -44,13 +39,9 @@ use windows_sys::Win32::System::Threading::{ WaitForSingleObject, }; -use crate::probe_exec::truncate_to_chars; -use std::io::{ErrorKind, Read}; -use std::process::{Child, Command, ExitStatus, Output, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::thread; -use std::time::{Duration, Instant}; +mod bounded; +pub use bounded::run_with_timeout; +pub(crate) use bounded::{MAX_QUOTED_OUTPUT_CHARS, run_bounded}; #[cfg(windows)] pub fn spawn_detached_no_inherit( @@ -363,24 +354,204 @@ pub fn terminate_process_tree(pid: u32) -> Result<()> { terminate_process(pid) } +/// What one look at a PID could establish. +/// +/// [`process_is_running`] answers a plain yes or no, so a process that is alive +/// but cannot be queried reads the same as one that is gone. A stop that is about +/// to report a process as exited must not make that mistake, so this keeps the +/// third answer apart. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessProbe { + /// The PID names a process that has not exited. + Running, + /// The PID names no process, or one that has exited. + Gone, + /// The PID could not be queried, so it can be neither confirmed running nor + /// confirmed gone. On Windows that includes a live process the caller may not + /// open (`ERROR_ACCESS_DENIED`). + /// + /// The cost of reading this as "may be running" is that a stopped service's + /// recorded PID, taken since by a process the caller cannot open, keeps its + /// record live: the record never reconciles to `stopped`, `services remove` + /// and `prune` refuse it, and a stop reports it unverified. That is the safe + /// direction (the alternative deletes the key and record under a process that + /// may be serving), and there is no better evidence to be had: the creation + /// time that would tell the two processes apart is read with the same access + /// right, so a process that cannot be opened has no readable token to + /// compare. `rocm services remove --force --yes` clears such a record + /// (see `docs/manual-testing.md`, "A record that stays unverified"); it + /// applies only when no recorded process is confirmed running, one could not + /// be queried, and a stop was requested. + Unqueryable, +} + +impl ProcessProbe { + /// Whether the process may still be running: everything but a confirmed + /// exit. The answer for a decision that must not treat a process it could + /// not look at as gone, such as dropping a service's endpoint key, marking + /// its record stopped or deleting it. + #[must_use] + pub const fn may_be_running(self) -> bool { + !matches!(self, Self::Gone) + } +} + +/// Windows `ERROR_INVALID_PARAMETER`: what `OpenProcess` sets for a PID that +/// names no process. +#[cfg(any(windows, test))] +const ERROR_INVALID_PARAMETER: i32 = 87; + +/// Windows `STILL_ACTIVE`: the exit code `GetExitCodeProcess` reports for a +/// process that has not exited. +#[cfg(any(windows, test))] +const STILL_ACTIVE: u32 = 259; + +/// What `GetExitCodeProcess` on a handle that opened says about the process: +/// still active is running, any other exit code is gone, and a failed query is +/// neither. Split out of the FFI, with the open-failure mapping above, so a test +/// runs the whole decision on every host. +#[cfg(any(windows, test))] +pub(crate) const fn classify_exit_query(queried: bool, exit_code: u32) -> ProcessProbe { + match (queried, exit_code) { + (false, _) => ProcessProbe::Unqueryable, + (true, STILL_ACTIVE) => ProcessProbe::Running, + (true, _) => ProcessProbe::Gone, + } +} + +/// What a failed `OpenProcess` says about the PID, from its `GetLastError` code. +/// +/// Only the code for "no such process" counts as gone. `ERROR_ACCESS_DENIED` +/// (5), which a live process the caller may not open sets, and any other or +/// missing code are [`ProcessProbe::Unqueryable`]: a failure to look is not an +/// exit. Split out of the FFI so a test runs it on every host. +#[cfg(any(windows, test))] +pub(crate) const fn classify_open_failure(code: Option) -> ProcessProbe { + match code { + Some(ERROR_INVALID_PARAMETER) => ProcessProbe::Gone, + _ => ProcessProbe::Unqueryable, + } +} + +/// The path of a Windows system tool in the system directory (`System32`). +/// +/// The directory comes from `GetSystemDirectoryW`, not from the environment, so +/// neither a directory earlier on `PATH` nor a `SystemRoot` the caller controls +/// can stand in for it; the bare name when the directory cannot be read (and off +/// Windows). `name` includes the extension (`taskkill.exe`). +#[must_use] +pub fn windows_system_tool(name: &str) -> PathBuf { + system_tool_path(windows_system_directory(), name) +} + +/// [`windows_system_tool`] for a given system directory, so a test builds the +/// path on every host. +fn system_tool_path(system_dir: Option, name: &str) -> PathBuf { + match system_dir.filter(|dir| !dir.as_os_str().is_empty()) { + Some(dir) => dir.join(name), + None => PathBuf::from(name), + } +} + +/// The Windows system directory, read with `GetSystemDirectoryW`. #[cfg(windows)] #[allow(unsafe_code)] // Win32 FFI -pub fn process_is_running(pid: u32) -> bool { +fn windows_system_directory() -> Option { + use std::os::windows::ffi::OsStringExt; + use windows_sys::Win32::System::SystemInformation::GetSystemDirectoryW; + + const MAX_PATH: u32 = 260; + let mut buffer = [0_u16; MAX_PATH as usize]; + // SAFETY: `buffer` is a live local valid for `MAX_PATH` wide characters, and + // that is the size passed, so the call cannot write past it. + let length = unsafe { GetSystemDirectoryW(buffer.as_mut_ptr(), MAX_PATH) } as usize; + // Zero is a failure; a length that does not fit means the path was too long + // for the buffer, which is not a directory to trust either. + (length > 0 && length < buffer.len()) + .then(|| PathBuf::from(std::ffi::OsString::from_wide(&buffer[..length]))) +} + +#[cfg(not(windows))] +#[allow(clippy::missing_const_for_fn)] // keeps the signature of the Windows arm +fn windows_system_directory() -> Option { + None +} + +/// E2E hook: `ROCM_E2E_UNQUERYABLE_PIDS`, a comma-separated list of PIDs, makes +/// [`probe_process`] answer [`ProcessProbe::Unqueryable`] for each of them. +/// +/// A live process this account may not open cannot be staged from outside the +/// binary, and without it the stop, the liveness refresh and `services remove` +/// have no live coverage for that case off Windows. Compiled only into the +/// `e2e-test-hooks` build; production builds never read the variable. +#[cfg(feature = "e2e-test-hooks")] +fn scripted_probe(pid: u32) -> Option { + let listed = std::env::var("ROCM_E2E_UNQUERYABLE_PIDS").ok()?; + listed + .split(',') + .filter_map(|entry| entry.trim().parse::().ok()) + .any(|listed| listed == pid) + .then_some(ProcessProbe::Unqueryable) +} + +/// Look at `pid`, keeping a process that cannot be queried apart from one that +/// is gone. See [`ProcessProbe`]. +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn probe_process(pid: u32) -> ProcessProbe { use windows_sys::Win32::Foundation::CloseHandle; + #[cfg(feature = "e2e-test-hooks")] + if let Some(scripted) = scripted_probe(pid) { + return scripted; + } if pid == 0 { - return false; + return ProcessProbe::Gone; } + // SAFETY: `OpenProcess` takes only plain values and returns a handle or null. let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; if handle.is_null() { - return false; + return classify_open_failure(std::io::Error::last_os_error().raw_os_error()); } let mut exit_code = 0; - let ok = unsafe { GetExitCodeProcess(handle, &raw mut exit_code) != 0 }; + // SAFETY: `handle` is the non-null handle opened above, still open, and + // `exit_code` is a live local that is valid for this one call to write to. + let queried = unsafe { GetExitCodeProcess(handle, &raw mut exit_code) != 0 }; + // SAFETY: `handle` is the handle opened above, closed here exactly once and + // not used afterwards. unsafe { CloseHandle(handle); } - ok && exit_code == 259 + classify_exit_query(queried, exit_code) +} + +/// Look at `pid`. `kill(pid, 0)` already tells a live process the caller may +/// not signal (`EPERM`, which counts as running) from one that is gone, so there +/// is no third answer to keep apart here. +#[cfg(not(windows))] +#[must_use] +pub fn probe_process(pid: u32) -> ProcessProbe { + #[cfg(feature = "e2e-test-hooks")] + if let Some(scripted) = scripted_probe(pid) { + return scripted; + } + if process_is_running(pid) { + ProcessProbe::Running + } else { + ProcessProbe::Gone + } +} + +/// Whether `pid` names a running process. +/// +/// On Windows a live process this one may not open reads as not running here; a +/// decision that must not take it for gone (dropping a key, marking a record +/// stopped, deleting it) asks [`probe_process`] instead and keeps +/// [`ProcessProbe::Unqueryable`] apart. +#[cfg(windows)] +pub fn process_is_running(pid: u32) -> bool { + probe_process(pid) == ProcessProbe::Running } #[cfg(not(windows))] @@ -833,422 +1004,6 @@ fn windows_environment_block(env_overrides: &[(&str, &Path)]) -> Vec { block } -/// How often the wait loop polls for the child's exit. -const POLL_INTERVAL: Duration = Duration::from_millis(50); - -/// How much a reader accumulates before publishing it to the shared buffer. -const READ_CHUNK: usize = 16 * 1024; - -/// How long the readers get to finish once the child is no longer running. -/// -/// Reaching this means something other than the child still holds the pipe's -/// write end — a grandchild the child left behind — so there is no EOF coming -/// and waiting longer would make `timeout` meaningless. -/// -/// For a child that genuinely finished the grace has only the tail to absorb: a -/// child cannot exit before writing everything, and the readers are started -/// before the wait loop. What they are not is *scheduled* before it — -/// `wait_bounded` polls `try_wait` once with no sleep in front of it, so a child -/// fast enough to exit between spawn and that first poll can be observed exited -/// while neither reader thread has run a single instruction. So the grace covers -/// reader start-up as well as the last buffer or two. 200ms absorbs both with -/// room to spare, including on the no-GPU e2e lane's 64 concurrent scenarios. -const DRAIN_GRACE: Duration = Duration::from_millis(200); - -/// How much of a timed-out child's own output the timeout error quotes. -/// -/// Bounded here rather than at any one caller because the quote is assembled -/// here: the timeout message is this function's own text, and a caller sees it -/// only as an opaque error, so this is the only place that can still tell which -/// of its characters came from the child. All four callers inherit it. -/// -/// It bounds this message by itself. A caller that quotes the `Output` handed -/// back to it is quoting raw child output of its own, persisted the same way, -/// so it has to apply a bound too -- which is why this is `pub(crate)` rather -/// than private: [`crate::amd_smi::amd_smi_json`] bounds its non-zero-exit -/// quote (the one #623 tracked) with *this* constant instead of declaring a -/// second one free to drift from it. The daemon's -/// `parse_sandbox_child_output` still predates both and is outside this scope. -/// -/// The bound matters because the message does not stop at a terminal. `rocmd` -/// renders the whole error chain into the bridge snapshot's `note`, and the -/// GPU-metrics watcher copies that note into the automation-event and audit -/// JSONL files once a minute — stores that are never rotated or capped. -/// Unbounded, a child that prints a megabyte and then stalls past its -/// timeout grows both logs by that much per minute, forever. A 151K-character -/// JSONL line was reproduced from exactly that poller. -/// -/// Sized to still hold a real stderr backtrace: the point is to stop a -/// payload-sized quote, not to shorten diagnostics. -pub(crate) const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; - -/// One captured stream: the bytes so far, whether its reader saw EOF, and the -/// read error that stopped it, if one did. -/// -/// # Lifetime of the reader thread -/// -/// The thread outlives the call that started it whenever the drain grace runs -/// out — that is the point of the grace — and it then lives until the pipe's -/// last write end closes, which is the abandoned grandchild's business, not -/// ours. What it must *not* do is keep growing: `take()` marks the capture -/// abandoned, and from that moment the reader drains to EOF and discards, so a -/// buffer nobody will read again cannot grow without bound in a daemon that -/// probes every 60 seconds. -/// -/// What that bounds is the memory, and only the memory. The *number* of these -/// threads is not bounded here: each abandoned capture leaves one parked in -/// `read` until the grandchild lets go, so every probe that meets one costs two -/// until it does. The trade is deliberate — a parked reader holds a stack and -/// no CPU, and the alternative is the one below. -/// -/// Discarding rather than closing the read end is deliberate. Dropping the pipe -/// would send the still-writing grandchild `EPIPE`/`SIGPIPE`, which for the -/// background helpers a sandboxed installer legitimately leaves behind would -/// turn a leak whose cost we know into a behaviour change we cannot predict. -#[derive(Clone)] -struct Capture { - buffer: Arc>>, - finished: Arc, - failure: Arc>>, - /// Set by `take()`: nothing will read this capture's bytes again. - /// - /// Read and written under `buffer`'s mutex, so "emptied and marked" is one - /// step as far as the reader is concerned. Checking it outside the lock - /// would leave the window where a reader that has just passed the check - /// appends to the buffer `take()` is in the middle of emptying. - abandoned: Arc, -} - -impl Capture { - /// Start draining `pipe` on its own thread. - /// - /// Publishing in chunks as they arrive, rather than returning the whole - /// buffer at EOF, is what lets the caller take partial output from a reader - /// that is still blocked. - fn draining(pipe: Option) -> Self { - let capture = Self { - buffer: Arc::new(Mutex::new(Vec::new())), - finished: Arc::new(AtomicBool::new(false)), - failure: Arc::new(Mutex::new(None)), - abandoned: Arc::new(AtomicBool::new(false)), - }; - let Some(mut pipe) = pipe else { - capture.finished.store(true, Ordering::Release); - return capture; - }; - let sink = capture.clone(); - thread::spawn(move || { - let mut chunk = [0_u8; READ_CHUNK]; - loop { - match pipe.read(&mut chunk) { - Ok(0) => break, - Ok(read) => { - // Keep draining once abandoned, but stop accumulating: - // see the type's doc. The check shares the buffer's - // lock with `take()`, so after `take()` returns this - // buffer cannot grow by another byte. - let mut buffer = sink.lock(); - if !sink.abandoned.load(Ordering::Acquire) { - buffer.extend_from_slice(&chunk[..read]); - } - } - // `read_to_end` — what `wait_with_output` used before this - // function existed — retries an interrupted read and - // returns every other error. Treating either as EOF would - // silently shorten the output instead, and the callers - // parse these bytes as JSON, where that surfaces as a - // parse error pointing at the wrong thing. - // Round again rather than break: not the end of anything. - Err(error) if error.kind() == ErrorKind::Interrupted => {} - Err(error) => { - // Recorded only while someone is still listening. Once - // `take()` has fully returned this is merely tidy — - // nothing calls `take()` twice, so a failure stored - // afterwards would just sit unread. - // - // It is load-bearing for one window, and that window is - // inside `take()` itself: `take()` marks `abandoned` - // and empties the buffer under `buffer`'s lock, then - // releases it and drains the failure slot under a - // *different* lock. A read error that lands in between - // would otherwise be handed back attached to bytes the - // caller has already been given, and `assemble_output` - // turns any failure on a non-timed-out run into a hard - // `failed to read the output of ...` — a cleanly - // finished command reported as a read failure. Taking - // `buffer`'s lock before the check is what closes it; - // a bare load could pass the check, then have `take()` - // run the whole sequence before the store. - let _buffer = sink.lock(); - if !sink.abandoned.load(Ordering::Acquire) { - *sink.failure() = Some(error); - } - break; - } - } - } - sink.finished.store(true, Ordering::Release); - }); - capture - } - - fn lock(&self) -> std::sync::MutexGuard<'_, Vec> { - self.buffer - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - fn failure(&self) -> std::sync::MutexGuard<'_, Option> { - self.failure - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - fn is_finished(&self) -> bool { - self.finished.load(Ordering::Acquire) - } - - /// Take what has been captured, leaving a still-running reader unblocked. - /// - /// The bytes come back alongside any read error rather than instead of it, - /// so a caller already reporting something else — a timeout — can still - /// quote what it managed to collect. - /// - /// This is also the point of no return for the reader: whatever it collects - /// after this is discarded, because there is no second caller to hand it to. - fn take(&self) -> (Vec, Option) { - let bytes = { - // Marked while the buffer is held, so the reader cannot append - // between the mark and the emptying and strand bytes in a buffer - // that is never read again. - let mut buffer = self.lock(); - self.abandoned.store(true, Ordering::Release); - std::mem::take(&mut *buffer) - }; - (bytes, self.failure().take()) - } - - /// Mark the capture abandoned and release what it holds, for an error path - /// that returns before `take()` would. - /// - /// `take()` is the only place that sets `abandoned`, so going through it - /// keeps "marked" and "emptied" a single step with one implementation. The - /// bytes and any read error are dropped because a caller on this path is - /// already reporting something else. - fn abandon(&self) { - drop(self.take()); - } -} - -/// Run `command` to completion with both output streams captured, giving up -/// after `timeout`. -/// -/// A thread per pipe drains stdout and stderr, and both are started *before* -/// the wait loop. That ordering is the whole point. Waiting for the child to -/// exit and only then calling `wait_with_output` — the shape this replaced — -/// deadlocks: the pipes are bounded by the OS (commonly ~64KiB, less on -/// Windows), so a child that writes more than that blocks in `write`, which -/// means it never exits, which means the drain is never reached. The only way -/// out was the timeout, so it measured how much the child printed rather than -/// how long it took, and killed successful commands mid-flight. -/// -/// Both pipes need their own reader for the same reason: draining stdout to -/// completion and only then reading stderr stalls on a child that fills stderr -/// first. -/// -/// The call returns within roughly `timeout` either way: once the child is no -/// longer running the readers get `DRAIN_GRACE` to finish and are then abandoned -/// rather than joined. Joining unconditionally would hand the schedule to a -/// grandchild the child left behind, which holds the pipe's write end open so no -/// EOF ever arrives — that is how an abandoned `sleep` comes to decide how long -/// a 2-second probe takes. -/// -/// The stdio configuration is this function's own: it sets null stdin and piped -/// stdout/stderr on `command`, overriding whatever the caller set, because -/// capturing both streams is the whole contract. Callers need not — and should -/// not — set them. The stdin half is a promise in its own right rather than -/// housekeeping: a child that reads from an inherited stdin waits for input -/// that is never coming, so it is the deadline that ends it. Pinned by -/// `a_child_that_reads_stdin_is_given_an_empty_one`, on both platforms. -/// -/// `label` names the subject in error messages (e.g. `"process"`, -/// `"sandbox process"`). -pub fn run_with_timeout(mut command: Command, timeout: Duration, label: &str) -> Result { - let mut child = command - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .with_context(|| format!("failed to spawn {label}"))?; - - // Taking the pipes here is what obliges us to assemble `Output` by hand - // below: `wait_with_output` reads the handles it owns, and these are gone. - let stdout = Capture::draining(child.stdout.take()); - let stderr = Capture::draining(child.stderr.take()); - - run_captured(child, stdout, stderr, timeout, label) -} - -/// The rest of [`run_with_timeout`] once the child is spawned and both readers -/// are running. -/// -/// Generic over [`Waitable`] rather than taking a `Child` so that the wait -/// loop's two failure paths — which are the only ways out of this function that -/// skip the normal `take()` — are reachable from a test. -fn run_captured( - mut child: C, - stdout: Capture, - stderr: Capture, - timeout: Duration, - label: &str, -) -> Result { - // The single error path between `Capture::draining` above and the `take()` - // pair below, and it must abandon both captures before it propagates. - // `take()` is what marks a reader abandoned, and an abandoned reader is - // what stops a buffer nobody will read again from growing for as long as a - // grandchild holds the pipe open. Returning the error straight out of the - // wait loop — the shape this replaced — left both readers appending - // forever, in a daemon that probes every 60 seconds. - let (status, timed_out) = match wait_bounded(&mut child, timeout, label) { - Ok(outcome) => outcome, - Err(error) => { - stdout.abandon(); - stderr.abandon(); - return Err(error); - } - }; - - let drained_by = Instant::now() + DRAIN_GRACE; - while !(stdout.is_finished() && stderr.is_finished()) && Instant::now() < drained_by { - thread::sleep(POLL_INTERVAL.min(DRAIN_GRACE / 4)); - } - assemble_output( - status, - timed_out, - stdout.take(), - stderr.take(), - label, - timeout, - ) -} - -/// Wait for `child` to exit, killing it once `timeout` has passed. -/// -/// Returns the status and whether the deadline is what ended the wait. Kept -/// free of the captures so that it has exactly one caller to get the -/// abandon-on-error handling right, rather than one per `?`. -fn wait_bounded( - child: &mut C, - timeout: Duration, - label: &str, -) -> Result<(ExitStatus, bool)> { - let deadline = Instant::now() + timeout; - loop { - let polled = child.try_wait(); - if polled.is_err() { - // The only exit from this function that is not already followed by - // a kill. Whatever stopped us polling, we are about to stop - // waiting, and the caller drops the child without ever reaping it; - // leaving it running would hand back a process nobody is watching. - // `let _` to match the timeout branch: the kill is best-effort, and - // the poll error is the one worth reporting. - let _ = child.kill(); - } - if let Some(status) = polled.with_context(|| format!("failed to poll {label}"))? { - return Ok((status, false)); - } - if Instant::now() >= deadline { - let _ = child.kill(); - let status = child - .wait() - .with_context(|| format!("failed to reap timed-out {label}"))?; - return Ok((status, true)); - } - thread::sleep(POLL_INTERVAL); - } -} - -/// The three child operations [`wait_bounded`] needs. -/// -/// A trait only so a test can fail `try_wait` or the post-kill `wait` on -/// demand: both are error paths that skip the normal `take()`, and provoking -/// either from a real child needs the kernel to refuse to reap a process. -trait Waitable { - fn try_wait(&mut self) -> std::io::Result>; - fn kill(&mut self) -> std::io::Result<()>; - fn wait(&mut self) -> std::io::Result; -} - -// `Child::` rather than `Self::` on purpose: inside this impl `Self` is -// `Child`, so `Self::try_wait` reads as the trait method being defined -// here rather than the inherent one being forwarded to. -#[allow(clippy::use_self)] -impl Waitable for Child { - fn try_wait(&mut self) -> std::io::Result> { - Child::try_wait(self) - } - - fn kill(&mut self) -> std::io::Result<()> { - Child::kill(self) - } - - fn wait(&mut self) -> std::io::Result { - Child::wait(self) - } -} - -/// Turn what the wait loop and the two readers produced into the call's result. -/// -/// Split out from [`run_with_timeout`] because it is the only way to test it. -/// Every branch here is plain logic over values already in hand — which error -/// wins, which stream gets quoted — but reaching it through a real child means -/// making a pipe fail on demand, which needs a kernel having a bad day. Given -/// the two `(bytes, error)` pairs directly, each branch is an ordinary unit -/// test; see the cases at the bottom of this file. -fn assemble_output( - status: ExitStatus, - timed_out: bool, - (stdout, stdout_failure): (Vec, Option), - (stderr, stderr_failure): (Vec, Option), - label: &str, - timeout: Duration, -) -> Result { - let output = Output { - status, - stdout, - stderr, - }; - - if timed_out { - // `command_failure_detail` is the crate's one "which stream explains - // this failure" rule — stderr, else stdout, else "no output" — shared - // since #584 so callers cannot quietly disagree about the order. - // - // Truncated, not dropped: what the child managed to say before it - // stalled is usually the whole explanation, and the timeout is named - // first either way, so a reader who only sees the head still learns - // which command gave up and why. - bail!( - "{label} exceeded {}s timeout: {}", - timeout.as_secs(), - truncate_to_chars( - crate::command_failure_detail(&output), - MAX_QUOTED_OUTPUT_CHARS, - ) - ); - } - - // Only once the timeout has had its say: a killed child whose pipe then - // fails to read is a timeout, and reporting it as a read error would name - // the symptom instead of the cause. - if let Some(error) = stdout_failure.or(stderr_failure) { - return Err(anyhow::Error::new(error)) - .with_context(|| format!("failed to read the output of {label}")); - } - - Ok(output) -} - /// Property tests for [`walk_process_tree`]; a child module so it can reach /// the walk while it stays private. #[cfg(all(test, not(windows)))] @@ -1261,6 +1016,72 @@ mod tests { use crate::test_support::workspace_test_artifact_dir; use std::process::Stdio; + /// A system tool is started from the system directory, never by a bare name + /// that `PATH` could resolve to something else; with no directory it falls + /// back to the bare name. + #[test] + fn a_system_tool_is_resolved_under_the_system_directory() { + let dir = PathBuf::from("C:\\Windows").join("System32"); + let path = system_tool_path(Some(dir.clone()), "taskkill.exe"); + assert_eq!(path, dir.join("taskkill.exe")); + assert!(path.ends_with(Path::new("System32").join("taskkill.exe"))); + assert!(!path.parent().unwrap().as_os_str().is_empty()); + + assert_eq!( + system_tool_path(None, "tasklist.exe"), + PathBuf::from("tasklist.exe") + ); + assert_eq!( + system_tool_path(Some(PathBuf::new()), "tasklist.exe"), + PathBuf::from("tasklist.exe"), + "an empty directory is no directory" + ); + } + + /// A failed `OpenProcess` is a gone process only for the code that means + /// "no such process". Access denied, which a live process the caller may not + /// open sets, and any code that is not known are not an exit. + #[test] + fn a_failed_open_is_gone_only_for_no_such_process() { + const ERROR_ACCESS_DENIED: i32 = 5; + assert_eq!(classify_open_failure(Some(87)), ProcessProbe::Gone); + assert_eq!( + classify_open_failure(Some(ERROR_ACCESS_DENIED)), + ProcessProbe::Unqueryable + ); + // The codes around the two that matter, and no code at all. + for code in [Some(0), Some(6), Some(31), Some(299), Some(-1), None] { + assert_eq!( + classify_open_failure(code), + ProcessProbe::Unqueryable, + "{code:?}" + ); + } + } + + /// An opened handle's exit code: still active is running, any other code is + /// gone, and a failed query (whatever the code it left behind) is neither. + #[test] + fn an_exit_code_query_is_running_gone_or_unqueryable() { + assert_eq!(classify_exit_query(true, 259), ProcessProbe::Running); + assert_eq!(classify_exit_query(true, 0), ProcessProbe::Gone); + assert_eq!(classify_exit_query(true, 1), ProcessProbe::Gone); + assert_eq!(classify_exit_query(false, 0), ProcessProbe::Unqueryable); + assert_eq!(classify_exit_query(false, 259), ProcessProbe::Unqueryable); + } + + /// Off Windows, a PID that is running (including one the caller may not + /// signal) probes as running and an absent one as gone, matching + /// [`process_is_running`]. + #[cfg(not(windows))] + #[test] + fn the_probe_agrees_with_process_is_running_off_windows() { + let own = std::process::id(); + assert_eq!(probe_process(own), ProcessProbe::Running); + assert_eq!(probe_process(0), ProcessProbe::Gone); + assert_eq!(probe_process(999_999_998), ProcessProbe::Gone); + } + const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; const HANDOFF_PORT_ENV: &str = "ROCM_CORE_TEST_HANDOFF_PORT"; @@ -1539,932 +1360,6 @@ mod tests { let _ = fs::remove_dir_all(&dir); } - - /// The shells every grandchild case below is run against. - /// - /// Whether a shell forks or execs decides whether a grandchild exists at - /// all, so it decides whether these tests test anything: dash forks `sleep` - /// in `sh -c "sleep 30"`, bash execs it into the shell's own process and no - /// grandchild is left holding the pipe. A test written against one is dead - /// weight on a host running the other, and which one `/bin/sh` is varies - /// across the distributions this ships on. So the scripts below are written - /// to fork under either, and each case runs under every shell present to - /// keep that true. - #[cfg(unix)] - fn grandchild_shells() -> Vec<&'static str> { - let shells: Vec<&'static str> = ["/bin/sh", "/bin/bash", "/bin/dash"] - .into_iter() - .filter(|shell| std::path::Path::new(shell).exists()) - .collect(); - assert!( - !shells.is_empty(), - "no shell to run the grandchild cases in" - ); - shells - } - - #[test] - fn a_command_that_succeeds_reports_its_output() { - let mut command = Command::new(if cfg!(windows) { "cmd" } else { "sh" }); - if cfg!(windows) { - // No space around `&`: cmd would carry a trailing one into the - // echoed text, which `trim` hides on stdout but is easier not to - // write than to explain. - command.args(["/C", "echo hello&echo oops 1>&2"]); - } else { - command.args(["-c", "echo hello; echo oops >&2"]); - } - - let output = run_with_timeout(command, Duration::from_secs(10), "process").expect("ran"); - - assert!(output.status.success()); - assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "hello"); - // Asserted on Windows too: `1>&2` is a cmd redirection like any other, - // and skipping it there left the second pipe — the one whose separate - // reader this function exists to provide — unchecked on that platform. - assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), "oops"); - } - - /// What a stdin-reading fixture prints when its read reaches EOF. - const STDIN_EOF_MARKER: &str = "eof"; - - /// The assertions shared by both platforms' empty-stdin cases. - /// - /// `command` must run a child that reads one line from stdin and prints - /// [`STDIN_EOF_MARKER`] if that read reaches EOF instead. - /// - /// The stdin set on `command` here is the read end of a pipe whose write end - /// this function holds open across the whole call, so it never reaches EOF - /// of its own accord. That is what makes the assertion two-sided rather than - /// a tautology: were `run_with_timeout` to stop setting null stdin, the - /// child would sit in `read` until the deadline killed it — exactly how a - /// child that prompts behaves against an inherited terminal — and the call - /// would come back as a timeout rather than with the marker. - /// - /// Setting stdin on the caller's side is also the contract under test: this - /// function's own stdio overrides whatever the caller set, which is what - /// lets all four callers hand over a bare `Command` and say nothing about - /// stdin at all. - /// - /// One thing it does not pin, said plainly rather than left to be - /// discovered: `null` against `inherit`. Telling those two apart needs this - /// test process's own fd 0 to be something other than EOF, and arranging - /// that is a `dup2` racing every other test in the binary. An `inherit` - /// mutant was run and does pass here. What is caught is the regression that - /// can actually happen -- no stdin setting at all, which is the line this - /// function took over from its callers. - /// - /// The race is avoidable by re-execing this test binary under a stdin pipe - /// the outer test holds open and asserting in that child, which would fail - /// the `inherit` mutant too. Not done: it buys one mutant that no plausible - /// edit produces -- nothing would replace `null` with `inherit`, the stdio - /// mode this helper exists to stop callers from being handed -- at the cost - /// of a self-re-execing test whose own harness plumbing is then the thing - /// most likely to break. - fn assert_the_child_is_given_an_empty_stdin(mut command: Command) { - let (stdin_reader, stdin_writer) = - std::io::pipe().expect("failed to create a pipe for the child's stdin"); - command.stdin(Stdio::from(stdin_reader)); - - let started = Instant::now(); - let output = run_with_timeout(command, Duration::from_secs(5), "process").expect( - "a child given an empty stdin must read EOF rather than block until the deadline", - ); - let elapsed = started.elapsed(); - // Dropped only here: releasing the write end any earlier would close the - // pipe, so the child would reach EOF through the caller's stdin and the - // test would pass whatever this function set. - drop(stdin_writer); - - assert!(output.status.success(), "{output:?}"); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - STDIN_EOF_MARKER, - "the child must be given an empty stdin rather than the one the caller set" - ); - assert!( - elapsed < Duration::from_secs(3), - "the read must hit EOF at once rather than waiting out the deadline, but the call \ - took {elapsed:?}" - ); - } - - #[cfg(unix)] - #[test] - fn a_child_that_reads_stdin_is_given_an_empty_one() { - // `read` reports EOF by returning non-zero, so the `||` branch is the - // one an empty stdin takes. A stdin left open takes neither. - let script = format!("read line || echo {STDIN_EOF_MARKER}"); - let mut command = Command::new("sh"); - command.args(["-c", script.as_str()]); - - assert_the_child_is_given_an_empty_stdin(command); - } - - #[cfg(unix)] - #[test] - fn a_genuinely_slow_command_times_out_on_time() { - for shell in grandchild_shells() { - // `sleep 30 & wait` rather than plain `sleep 30`: a lone final - // command is exec-ed into the shell itself by bash, which would - // leave nothing behind to hold the pipe and quietly turn this into - // a test of the easy case. Backgrounding forces a real grandchild - // under every shell. - // - // Killing the shell at the deadline does not kill that grandchild, - // so it keeps the inherited write end of both pipes open and no EOF - // arrives. An implementation that joins its readers therefore waits - // out the full 30 seconds and the timeout bounds nothing — which - // for the 2-second amd-smi probe is the difference between a snappy - // telemetry read and a stalled one. - let mut command = Command::new(shell); - command.args(["-c", "sleep 30 & wait"]); - - let started = Instant::now(); - let error = run_with_timeout(command, Duration::from_secs(1), "process") - .expect_err("a command that outlives its timeout must fail"); - let elapsed = started.elapsed(); - - assert!( - error.to_string().contains("exceeded 1s timeout"), - "the timeout must be reported as one, under {shell}: {error}" - ); - assert!( - elapsed < Duration::from_secs(5), - "the call must return at its deadline rather than waiting for an abandoned \ - grandchild to exit, but under {shell} it took {elapsed:?}" - ); - } - } - - #[cfg(unix)] - #[test] - fn a_grandchild_holding_the_pipe_does_not_hold_a_successful_call() { - for shell in grandchild_shells() { - // The timeout case above covers a child that is killed. This is the - // same hazard on the path that has no timeout to blame: the child - // exits 0 straight away, having left a background `sleep` holding - // the write end of both pipes, so there is no EOF for another five - // seconds. Joining the readers — or giving them an unbounded grace - // — hands the schedule to that `sleep` and a command that finished - // instantly takes 5s. `DRAIN_GRACE` is what bounds it. - let mut command = Command::new(shell); - command.args(["-c", "echo ready; sleep 5 &"]); - - let started = Instant::now(); - let output = run_with_timeout(command, Duration::from_secs(20), "process") - .expect("a child that exits cleanly must not be reported as a timeout"); - let elapsed = started.elapsed(); - - assert!(output.status.success(), "under {shell}: {output:?}"); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - "ready", - "what the child wrote before exiting must still arrive, under {shell}" - ); - // Generous against a loaded machine, and still a third of the - // grandchild's lifetime: the regression this guards takes the full - // five seconds, not a few hundred milliseconds more than the grace. - assert!( - elapsed < Duration::from_secs(3), - "the call must not wait for the grandchild, but under {shell} it took {elapsed:?}" - ); - } - } - - #[cfg(unix)] - #[test] - fn output_arriving_after_the_child_exits_is_still_collected() { - for shell in grandchild_shells() { - // What `DRAIN_GRACE` is for, and the only way to ask for it on - // demand. A child that writes and exits normally cannot pin it: its - // reader is already running and drains the pipe microseconds later, - // long before the wait loop's next poll even notices the exit, so - // taking the buffer the instant the child is seen to have exited - // still collects everything. Here a grandchild writes once the - // child is gone instead. - // - // The 0.1s sits between the two bounds that decide the outcome: - // past `POLL_INTERVAL` (50ms), the longest the wait loop can take - // to notice the exit, so taking the output there misses - // `late-canary` outright; and well inside `DRAIN_GRACE` (200ms), so - // draining collects it. Both margins are ~2x, and the grandchild - // exits immediately after writing, so a working implementation sees - // EOF and returns rather than sitting out the whole grace. - let mut command = Command::new(shell); - command.args(["-c", "echo early-canary; (sleep 0.1; echo late-canary) &"]); - - let started = Instant::now(); - let output = run_with_timeout(command, Duration::from_secs(20), "process") - .expect("a child that exits cleanly must not be reported as a timeout"); - let elapsed = started.elapsed(); - let stdout = String::from_utf8_lossy(&output.stdout); - - assert!( - stdout.contains("early-canary"), - "under {shell}, the child's own output is missing: {stdout:?}" - ); - assert!( - stdout.contains("late-canary"), - "under {shell}, output still in flight when the child exited was dropped \ - instead of drained: {stdout:?}" - ); - assert!( - elapsed < Duration::from_secs(3), - "draining must end at EOF rather than running long, but under {shell} the call \ - took {elapsed:?}" - ); - } - } - - #[test] - fn an_interrupted_read_is_retried_and_a_failed_one_is_reported() { - // Neither branch is reachable through a real child here — EINTR needs a - // signal to land mid-read, and a pipe read failing outright needs the - // kernel to be having a bad day — so the reader is fed a scripted - // stream directly. The property is the one `read_to_end` gave before - // this function replaced it: an interrupted read is not the end of the - // stream, and a failed one is not a successful short read. - struct ScriptedPipe(std::collections::VecDeque>); - - impl Read for ScriptedPipe { - fn read(&mut self, out: &mut [u8]) -> std::io::Result { - match self.0.pop_front() { - Some(Ok(bytes)) => { - out[..bytes.len()].copy_from_slice(bytes); - Ok(bytes.len()) - } - Some(Err(error)) => Err(error), - None => Ok(0), - } - } - } - - fn drain(script: Vec>) -> (Vec, Option) { - let capture = Capture::draining(Some(ScriptedPipe(script.into()))); - let deadline = Instant::now() + Duration::from_secs(10); - while !capture.is_finished() && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!(capture.is_finished(), "the reader never finished"); - capture.take() - } - - let (bytes, failure) = drain(vec![ - Ok(b"before"), - Err(std::io::Error::from(ErrorKind::Interrupted)), - Ok(b"after"), - Ok(b""), - ]); - assert_eq!( - String::from_utf8_lossy(&bytes), - "beforeafter", - "an interrupted read must be retried, not taken for the end of the stream" - ); - assert!(failure.is_none(), "{failure:?}"); - - let (bytes, failure) = drain(vec![ - Ok(b"partial"), - Err(std::io::Error::from(ErrorKind::BrokenPipe)), - Ok(b"unreachable"), - ]); - assert_eq!(String::from_utf8_lossy(&bytes), "partial"); - assert_eq!( - failure.map(|error| error.kind()), - Some(ErrorKind::BrokenPipe), - "a read that failed must be reported rather than passed off as EOF" - ); - } - - /// A pipe that never ends until the test says so. - /// - /// Stoppable rather than genuinely endless so the reader thread finishes - /// with the test instead of spinning for the life of the test binary. - struct EndlessPipe { - reads: Arc, - stop: Arc, - } - - impl Read for EndlessPipe { - fn read(&mut self, out: &mut [u8]) -> std::io::Result { - if self.stop.load(Ordering::Acquire) { - return Ok(0); - } - // Paced so the test can interleave with the reader rather than - // racing a tight loop, and so a regression accumulates a size the - // assertion can see without the test allocating gigabytes. - thread::sleep(Duration::from_millis(1)); - let filled = out.len().min(4096); - out[..filled].fill(b'x'); - self.reads.fetch_add(1, Ordering::Release); - Ok(filled) - } - } - - #[test] - fn an_abandoned_reader_stops_filling_a_buffer_nobody_will_read() { - // The shape this guards: the child exited but a grandchild still holds - // the write end, so `DRAIN_GRACE` expires, `take()` hands the caller - // what there is and the reader is left running. In `rocmd` — a daemon - // that probes amd-smi every 60s and runs MCP captures besides — a - // reader that kept appending would grow a `Vec` that nothing will ever - // read, for as long as the grandchild lives. There is no way to ask - // `run_with_timeout` for that state on demand, so the capture is driven - // directly, which is also where the invariant lives. - let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let stop = Arc::new(AtomicBool::new(false)); - let capture = Capture::draining(Some(EndlessPipe { - reads: Arc::clone(&reads), - stop: Arc::clone(&stop), - })); - - let wait_for_reads = |target: usize| { - let deadline = Instant::now() + Duration::from_secs(10); - while reads.load(Ordering::Acquire) < target && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!( - reads.load(Ordering::Acquire) >= target, - "the reader stalled before {target} reads" - ); - }; - - wait_for_reads(4); - let (bytes, failure) = capture.take(); - // Establishes that the reader really was accumulating up to this point, - // so the emptiness asserted below is the flag working rather than a - // reader that never ran. - assert!( - !bytes.is_empty(), - "the reader must collect while someone is still listening" - ); - assert!(failure.is_none(), "{failure:?}"); - - // Well past what fits in anything the implementation might buffer - // internally: 50 more 4KiB reads is ~200KB, which a regression leaves - // sitting in the buffer and a correct implementation discards. - wait_for_reads(reads.load(Ordering::Acquire) + 50); - let leaked = capture.lock().len(); - assert_eq!( - leaked, 0, - "after `take()` nothing will read this buffer again, so a reader that keeps \ - appending grows it without bound for the life of the daemon; {leaked} bytes \ - accumulated" - ); - - stop.store(true, Ordering::Release); - let deadline = Instant::now() + Duration::from_secs(10); - while !capture.is_finished() && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!(capture.is_finished(), "the reader never finished"); - } - - /// A child that never exits on its own and fails whichever of the wait - /// loop's two calls the test asks it to. - struct FailingChild { - /// `true`: `try_wait` reports an error rather than "still running", so - /// the loop's first `?` fires. `false`: it stays running until the - /// deadline, and the post-kill `wait` is what fails. - fail_poll: bool, - /// Raised by `kill`, so the test can tell a wait loop that leaves the - /// child running from one that does not. This is the whole reason - /// `kill` is on the trait rather than called on a concrete `Child`. - killed: Arc, - } - - impl Waitable for FailingChild { - fn try_wait(&mut self) -> std::io::Result> { - if self.fail_poll { - return Err(std::io::Error::from(ErrorKind::PermissionDenied)); - } - Ok(None) - } - - fn kill(&mut self) -> std::io::Result<()> { - self.killed.store(true, Ordering::Release); - Ok(()) - } - - fn wait(&mut self) -> std::io::Result { - Err(std::io::Error::from(ErrorKind::PermissionDenied)) - } - } - - /// Drive `run_captured` with a child that fails one of the wait loop's two - /// calls, and assert the error is reported, both readers are left - /// abandoned, *and* the child was killed on the way out. - /// - /// `fail_poll` picks which call fails; see [`FailingChild`]. - fn assert_wait_failure_abandons_both_readers( - fail_poll: bool, - timeout: Duration, - expected: &str, - ) { - let killed = Arc::new(AtomicBool::new(false)); - let child = FailingChild { - fail_poll, - killed: Arc::clone(&killed), - }; - let stop = Arc::new(AtomicBool::new(false)); - let out_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let err_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let stdout = Capture::draining(Some(EndlessPipe { - reads: Arc::clone(&out_reads), - stop: Arc::clone(&stop), - })); - let stderr = Capture::draining(Some(EndlessPipe { - reads: Arc::clone(&err_reads), - stop: Arc::clone(&stop), - })); - - let wait_for_reads = |counter: &Arc, target: usize| { - let deadline = Instant::now() + Duration::from_secs(10); - while counter.load(Ordering::Acquire) < target && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!( - counter.load(Ordering::Acquire) >= target, - "the reader stalled before {target} reads" - ); - }; - - // Both readers are accumulating before the call, so an empty buffer - // afterwards is the abandon working rather than a reader that never ran. - wait_for_reads(&out_reads, 4); - wait_for_reads(&err_reads, 4); - - let error = run_captured(child, stdout.clone(), stderr.clone(), timeout, "process") - .expect_err("a failed wait must be reported, not swallowed"); - assert!( - format!("{error:#}").contains(expected), - "expected {expected:?} in {error:#}" - ); - - // ~200KB more per stream: what a reader that was never marked abandoned - // accumulates into a buffer nobody will ever read again. - wait_for_reads(&out_reads, out_reads.load(Ordering::Acquire) + 50); - wait_for_reads(&err_reads, err_reads.load(Ordering::Acquire) + 50); - for (name, capture) in [("stdout", &stdout), ("stderr", &stderr)] { - let leaked = capture.lock().len(); - assert_eq!( - leaked, 0, - "an error out of the wait loop returns before the `take()` pair, so it must \ - abandon both readers itself or they grow a buffer nobody will read for as \ - long as a grandchild holds the pipe; {name} accumulated {leaked} bytes" - ); - } - - // Abandoning the readers is only half of it: the caller drops the child - // on this path without reaping it, so a wait loop that returns without - // killing leaves a process running that nobody will ever wait on again. - assert!( - killed.load(Ordering::Acquire), - "every exit from the wait loop must kill the child first; this one returned the \ - error with the child still running" - ); - - stop.store(true, Ordering::Release); - } - - #[test] - fn a_failure_to_poll_the_child_still_abandons_both_readers() { - assert_wait_failure_abandons_both_readers( - true, - Duration::from_secs(30), - "failed to poll process", - ); - } - - #[test] - fn a_failure_to_reap_a_timed_out_child_still_abandons_both_readers() { - // Zero timeout so the loop reaches the post-kill reap — the other `?` — - // on its first pass. - assert_wait_failure_abandons_both_readers( - false, - Duration::ZERO, - "failed to reap timed-out process", - ); - } - - /// A successful `ExitStatus` to feed `assemble_output`. - /// - /// `ExitStatus` has no portable public constructor; the per-platform - /// `ExitStatusExt` is the supported way to make one without spawning a - /// process purely to obtain it. - fn success_status() -> ExitStatus { - #[cfg(unix)] - use std::os::unix::process::ExitStatusExt as _; - #[cfg(windows)] - use std::os::windows::process::ExitStatusExt as _; - ExitStatus::from_raw(0) - } - - fn read_failure() -> Option { - Some(std::io::Error::from(ErrorKind::BrokenPipe)) - } - - #[test] - fn a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output() { - // The silent short read this function exists to prevent: handing back - // `Ok` with a truncated buffer sends the caller off to parse half a - // JSON document and blame the document. - let error = assemble_output( - success_status(), - false, - (b"partial".to_vec(), read_failure()), - (Vec::new(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a stream that failed to read must not come back as success"); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("failed to read the output of process"), - "{rendered:?}" - ); - } - - #[test] - fn a_failure_on_either_stream_is_reported() { - // stdout is checked first, so a stderr-only failure is a separate - // branch: `stdout_failure.or(stderr_failure)` reduced to - // `stdout_failure` would still pass the case above. - let error = assemble_output( - success_status(), - false, - (Vec::new(), None), - (b"partial".to_vec(), read_failure()), - "process", - Duration::from_secs(2), - ) - .expect_err("a stderr stream that failed to read must not come back as success"); - - assert!( - format!("{error:#}").contains("failed to read the output of process"), - "{error:#}" - ); - } - - #[test] - fn a_timeout_outranks_a_read_failure() { - // Killing the child is what broke the pipe, so a read error here is the - // symptom. Reporting it would name the consequence and hide the cause, - // and the caller would go looking for a kernel problem. - let error = assemble_output( - success_status(), - true, - (Vec::new(), read_failure()), - (b"the real explanation".to_vec(), read_failure()), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("exceeded 2s timeout"), - "the timeout is the cause and must be what is reported: {rendered:?}" - ); - assert!( - rendered.contains("the real explanation"), - "the child's own account must survive: {rendered:?}" - ); - assert!( - !rendered.contains("failed to read the output of"), - "the read error is downstream of the kill and must not displace it: {rendered:?}" - ); - } - - #[test] - fn a_timeout_quotes_stderr_in_preference_to_stdout() { - let error = assemble_output( - success_status(), - true, - (b"stdout-data".to_vec(), None), - (b"stderr-explanation".to_vec(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("stderr-explanation"), - "stderr is where a child explains itself: {rendered:?}" - ); - assert!( - !rendered.contains("stdout-data"), - "quoting both would push the explanation behind a payload: {rendered:?}" - ); - } - - #[test] - fn a_timeout_falls_back_to_stdout_and_then_to_saying_there_was_none() { - // Every other timeout fixture writes to stderr, so without these two - // the fallback chain could be reordered or deleted unnoticed — and a - // child whose whole output is stdout is exactly the amd-smi case. - let error = assemble_output( - success_status(), - true, - (b"stdout-only".to_vec(), None), - (Vec::new(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - assert!( - format!("{error:#}").contains("stdout-only"), - "a child that only wrote stdout must still be quoted: {error:#}" - ); - - let error = assemble_output( - success_status(), - true, - (Vec::new(), None), - (Vec::new(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("no output"), - "a silent child must say so rather than trailing a bare colon: {rendered:?}" - ); - assert!(rendered.contains("exceeded 2s timeout"), "{rendered:?}"); - } - - #[test] - fn a_clean_run_returns_both_streams() { - let output = assemble_output( - success_status(), - false, - (b"out".to_vec(), None), - (b"err".to_vec(), None), - "process", - Duration::from_secs(2), - ) - .expect("a clean run must succeed"); - - assert!(output.status.success()); - assert_eq!(output.stdout, b"out"); - assert_eq!(output.stderr, b"err"); - } - - #[cfg(unix)] - #[test] - fn a_child_that_floods_both_pipes_does_not_deadlock() { - // The defect this function exists to prevent. Both pipes are bounded by - // the OS (commonly ~64KiB), so a child writing past that blocks in - // `write` and never exits — and a version of this function that drains - // only after the child exits never reads a byte, so the child is killed - // at the timeout and a successful command is reported as too slow. - // - // Flooding *both* streams is deliberate: a single-pipe child still - // passes against an implementation that drains stdout fully and only - // then starts on stderr, which is its own deadlock. - // - // `sh` rather than a real subject program (amd-smi, an SDK install), - // because the property is about this function's pipe handling and - // nothing else, and a shell makes it a unit test rather than one that - // needs a multi-GPU host. - let mut command = Command::new("sh"); - command.args([ - "-c", - "yes deadlock-canary | head -c 2000000; yes deadlock-canary | head -c 2000000 >&2", - ]); - - // Short, because a regression fails by exhausting this timeout: ten - // seconds is far past the ~0.1s a working drain takes, and far short of - // stalling the suite. - let output = run_with_timeout(command, Duration::from_secs(10), "process") - .expect("a child that outruns the pipe buffer must not be killed as a timeout"); - - assert!(output.status.success(), "{output:?}"); - // Without these the test would keep passing if the child ever stopped - // producing more than one buffer, which is to say it would stop testing - // the hazard. - assert!( - output.stdout.len() > 64 * 1024, - "stdout must exceed one pipe buffer: {} bytes", - output.stdout.len() - ); - assert!( - output.stderr.len() > 64 * 1024, - "stderr must exceed one pipe buffer: {} bytes", - output.stderr.len() - ); - } - - /// How large a quoted-output assertion lets the whole error message get. - /// - /// Comfortably above `MAX_QUOTED_OUTPUT_CHARS` plus the label and marker, - /// and two orders of magnitude below the ~200KB the fixtures below feed in, - /// so the assertion answers "is it bounded at all" rather than pinning the - /// exact constant — which is a tuning decision, not a contract. - const QUOTE_ASSERTION_CEILING: usize = 4000; - - /// The flooding fixtures below write this many bytes, past one pipe buffer - /// and ~100x the quote bound. - const FLOOD_BYTES: usize = 200_000; - - const FLOOD_CANARY: &str = "flood-canary"; - - /// The assertions shared by both platforms' timeout-quote cases. - /// - /// Checked together because they trade off against each other: a bound that - /// kept nothing would satisfy the size check while making the message - /// useless, and quoting everything keeps it useful while costing the logs - /// the thing this bound exists to protect. - fn assert_timeout_quote_is_bounded_and_useful(message: &str, timeout_secs: u64) { - assert!( - message.contains(&format!("exceeded {timeout_secs}s timeout")), - "the timeout must still be named: {message:?}" - ); - assert!( - message.contains(FLOOD_CANARY), - "the child's own output must still be quoted, not dropped: {message:?}" - ); - assert!( - message.contains("[truncated]"), - "a shortened quote must say so, or a reader takes the head for the whole: {message:?}" - ); - let length = message.chars().count(); - assert!( - length <= QUOTE_ASSERTION_CEILING, - "the quote must be bounded: the message is {length} characters, from a child that \ - printed {FLOOD_BYTES} bytes" - ); - } - - #[cfg(unix)] - #[test] - fn a_timed_out_child_has_its_output_quoted_within_a_bound() { - for shell in grandchild_shells() { - // Both halves are load-bearing. The flood is what makes the quote - // large; the backgrounded `sleep` is what makes this a timeout, - // which is the only path that quotes the child's output at all. - // - // Unbounded, this message travels: `rocmd` renders the error chain - // into the bridge snapshot's `note`, and the GPU-metrics watcher - // writes that note into two JSONL logs every 60s, neither ever - // rotated or capped. A real amd-smi on a multi-GPU host prints - // well past this fixture's size. - let mut command = Command::new(shell); - command.args([ - "-c", - &format!("yes {FLOOD_CANARY} | head -c {FLOOD_BYTES} >&2; sleep 30 & wait"), - ]); - - let error = run_with_timeout(command, Duration::from_secs(1), "process") - .expect_err("a command that outlives its timeout must fail"); - - assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 1); - } - } - - /// A scratch directory holding a Windows fixture's script, and for - /// [`Self::flooding`] fixtures the payload beside it. - /// - /// Per-process unique so concurrent test binaries cannot collide on it, and - /// removed on drop so a failing assertion does not leave either behind. - #[cfg(windows)] - struct WindowsFixture { - dir: std::path::PathBuf, - } - - #[cfg(windows)] - impl WindowsFixture { - /// Write `script` as a batch file in a scratch directory of its own. - /// - /// A batch file rather than `cmd /C ""`: cmd's rules for - /// stripping the quotes Rust adds around an argument containing spaces - /// and redirections are subtle enough that the fixture would become the - /// thing under test. A bare path has none of that, and `%~dp0` inside - /// the script resolves anything beside it without the test passing a - /// path in. - fn script(name: &str, script: &str) -> Self { - let dir = std::env::temp_dir() - .join(format!("rocm-cli-process-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).expect("failed to create the fixture directory"); - std::fs::write(dir.join("fixture.bat"), script).expect("failed to write the script"); - - Self { dir } - } - - /// A [`Self::script`] fixture with a `FLOOD_BYTES` payload beside it for - /// the script to print. - fn flooding(name: &str, script: &str) -> Self { - let fixture = Self::script(name, script); - - let line = format!("{FLOOD_CANARY}\r\n"); - let mut payload = String::with_capacity(FLOOD_BYTES + line.len()); - while payload.len() < FLOOD_BYTES { - payload.push_str(&line); - } - std::fs::write(fixture.dir.join("payload.txt"), payload) - .expect("failed to write the payload"); - - fixture - } - - fn command(&self) -> Command { - let mut command = Command::new("cmd"); - command.args(["/C"]).arg(self.dir.join("fixture.bat")); - command - } - } - - #[cfg(windows)] - impl Drop for WindowsFixture { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.dir); - } - } - - #[cfg(windows)] - #[test] - fn a_child_that_floods_both_pipes_does_not_deadlock() { - // The Windows half of the unix case above, and the same property: both - // pipes are bounded by the OS here too — more tightly than on Linux — - // so an implementation that drains only after the child exits, or - // drains stdout to completion before starting on stderr, never lets - // this child finish. `type` twice is the shell-free equivalent of the - // unix `yes | head`; cmd has no `yes`. - let fixture = WindowsFixture::flooding( - "flood", - "@echo off\r\ntype \"%~dp0payload.txt\"\r\ntype \"%~dp0payload.txt\" 1>&2\r\n", - ); - - let output = run_with_timeout(fixture.command(), Duration::from_secs(30), "process") - .expect("a child that outruns the pipe buffer must not be killed as a timeout"); - - assert!(output.status.success(), "{output:?}"); - assert!( - output.stdout.len() > 64 * 1024, - "stdout must exceed one pipe buffer: {} bytes", - output.stdout.len() - ); - assert!( - output.stderr.len() > 64 * 1024, - "stderr must exceed one pipe buffer: {} bytes", - output.stderr.len() - ); - } - - #[cfg(windows)] - #[test] - fn a_timed_out_child_has_its_output_quoted_within_a_bound() { - // Covers on Windows what three separate unix cases cover there, because - // one fixture happens to exhibit all of it: `ping -n 30` runs for ~29 - // seconds, so the call times out; `ping` is a grandchild of the `cmd` - // this function kills at the deadline, and killing a process on Windows - // does not kill its children, so it keeps the inherited write ends open - // and no EOF arrives — an implementation that joins its readers waits - // the full 29 seconds; and the preceding flood makes the resulting - // quote large enough for the bound to matter. - let fixture = WindowsFixture::flooding( - "timeout", - "@echo off\r\ntype \"%~dp0payload.txt\" 1>&2\r\nping -n 30 127.0.0.1\r\n", - ); - - let started = Instant::now(); - let error = run_with_timeout(fixture.command(), Duration::from_secs(2), "process") - .expect_err("a command that outlives its timeout must fail"); - let elapsed = started.elapsed(); - - assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 2); - assert!( - elapsed < Duration::from_secs(10), - "the call must return at its deadline rather than waiting for an abandoned \ - grandchild to exit, but it took {elapsed:?}" - ); - } - - #[cfg(windows)] - #[test] - fn a_child_that_reads_stdin_is_given_an_empty_one() { - // `set /p` leaves the variable untouched when its read reaches EOF, so - // pre-setting it to the marker is cmd's equivalent of the unix - // `read || echo`. A stdin left open blocks in `set /p` instead. - // - // Both callers in `rocmd` run on Windows -- the amd-smi probe in - // `apps/rocmd/src/host_snapshot.rs` and the `rocm` capture that - // `apps/rocmd/src/common.rs` makes through `run_rocm_capture_with_env` - // -- and `run_with_timeout` has no `cfg(windows)` branch, so this covers - // the same promise with the same two-sidedness as the unix case. - let fixture = WindowsFixture::script( - "stdin", - &format!( - "@echo off\r\nset \"LINE={STDIN_EOF_MARKER}\"\r\nset /p LINE=\r\necho %LINE%\r\n" - ), - ); - - assert_the_child_is_given_an_empty_stdin(fixture.command()); - } } #[cfg(test)] diff --git a/crates/rocm-core/src/process/bounded.rs b/crates/rocm-core/src/process/bounded.rs new file mode 100644 index 0000000000..ab2b3131d0 --- /dev/null +++ b/crates/rocm-core/src/process/bounded.rs @@ -0,0 +1,1938 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The bounded subprocess executor behind [`super::run_with_timeout`] and +//! [`super::run_bounded`]. +//! +//! Run a command to completion with both streams captured and a real time +//! bound. A thread per pipe drains stdout and stderr from the moment the child +//! starts, so output size cannot stall a command into its own timeout, and a +//! reader that cannot finish is abandoned rather than joined, so a grandchild +//! holding the pipe's write end open cannot outlast the deadline either. The +//! timeout error quotes a bounded head of the child's own output, because that +//! text reaches persisted automation logs. `Capture`, `settle`, `run_captured`, +//! `wait_bounded`, `Waitable` and `assemble_output` exist to serve it. +//! +//! `run_bounded` is the same executor for a caller that wants the run as a +//! [`CommandCapture`] with a typed [`CommandOutcome`] and the raw bytes, a +//! timeout included, instead of an `Err`. The host probes' `run` and +//! `capture_optional_command*` are thin adapters over it. It is not yet the only +//! such helper in the tree; converging the others onto it is tracked in +//! [#623](https://github.com/ROCm/rocm-cli/issues/623). +//! +//! It lives in its own file, split out of `process.rs` to keep that file under +//! the module size limit; the public paths are unchanged, `process.rs` +//! re-exports them. + +use crate::command_capture::{self, CommandCapture, CommandOutcome}; +use crate::probe_exec::truncate_to_chars; +use anyhow::{Context, Result, bail}; +use std::io::{ErrorKind, Read}; +use std::process::{Child, Command, ExitStatus, Output, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +/// The longest the wait loop sleeps between polls for the child's exit. +const POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// The first sleep of the wait loop, before [`next_poll_interval`] backs it off. +/// +/// A host probe is usually a child that exits within a few milliseconds, and a +/// flat 50 ms sleep made every such call take 50 ms: the probes that older +/// helpers polled every 20 to 25 ms would then each have got slower for taking +/// this executor on, and `rocm examine` runs dozens of them one after another. +const FIRST_POLL: Duration = Duration::from_millis(5); + +/// The sleep after `current`: doubled, up to [`POLL_INTERVAL`]. +const fn next_poll_interval(current: Duration) -> Duration { + let doubled = current.saturating_mul(2); + if doubled.as_millis() > POLL_INTERVAL.as_millis() { + POLL_INTERVAL + } else { + doubled + } +} + +/// How much a reader accumulates before publishing it to the shared buffer. +const READ_CHUNK: usize = 16 * 1024; + +/// How long the readers get to finish once the child is no longer running. +/// +/// Reaching this means something other than the child still holds the pipe's +/// write end — a grandchild the child left behind — so there is no EOF coming +/// and waiting longer would make `timeout` meaningless. +/// +/// For a child that genuinely finished the grace has only the tail to absorb: a +/// child cannot exit before writing everything, and the readers are started +/// before the wait loop. What they are not is *scheduled* before it — +/// `wait_bounded` polls `try_wait` once with no sleep in front of it, so a child +/// fast enough to exit between spawn and that first poll can be observed exited +/// while neither reader thread has run a single instruction. So the grace covers +/// reader start-up as well as the last buffer or two. 200ms absorbs both with +/// room to spare, including on the no-GPU e2e lane's 64 concurrent scenarios. +/// +/// The one value for every caller, and what it costs is output: a descendant +/// that writes later than this after the child exited contributes nothing, and +/// the capture's `detail` says only that output was still arriving. Whether a +/// caller whose tool is known to do that should be able to ask for a longer +/// grace is open, listed in `docs/host-probing.md`. +const DRAIN_GRACE: Duration = Duration::from_millis(200); + +/// How much of a timed-out child's own output the timeout error quotes. +/// +/// Bounded here rather than at any one caller because the quote is assembled +/// here: the timeout message is this function's own text, and a caller sees it +/// only as an opaque error, so this is the only place that can still tell which +/// of its characters came from the child. All four callers inherit it. +/// +/// It bounds this message by itself. A caller that quotes the `Output` handed +/// back to it is quoting raw child output of its own, persisted the same way, +/// so it has to apply a bound too -- which is why this is `pub(crate)` rather +/// than private: [`crate::amd_smi::amd_smi_json`] bounds its non-zero-exit +/// quote (the one #623 tracked) with *this* constant instead of declaring a +/// second one free to drift from it. The daemon's +/// `parse_sandbox_child_output` still predates both and is outside this scope. +/// +/// The bound matters because the message does not stop at a terminal. `rocmd` +/// renders the whole error chain into the bridge snapshot's `note`, and the +/// GPU-metrics watcher copies that note into the automation-event and audit +/// JSONL files once a minute — stores that are never rotated or capped. +/// Unbounded, a child that prints a megabyte and then stalls past its +/// timeout grows both logs by that much per minute, forever. A 151K-character +/// JSONL line was reproduced from exactly that poller. +/// +/// Sized to still hold a real stderr backtrace: the point is to stop a +/// payload-sized quote, not to shorten diagnostics. +pub(crate) const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; + +/// One captured stream: the bytes so far, whether its reader saw EOF, and the +/// read error that stopped it, if one did. +/// +/// # Lifetime of the reader thread +/// +/// The thread outlives the call that started it whenever the drain grace runs +/// out — that is the point of the grace — and it then lives until the pipe's +/// last write end closes, which is the abandoned grandchild's business, not +/// ours. What it must *not* do is keep growing: `take()` marks the capture +/// abandoned, and from that moment the reader drains to EOF and discards, so a +/// buffer nobody will read again cannot grow without bound in a daemon that +/// probes every 60 seconds. +/// +/// What that bounds is the memory, and only the memory. The *number* of these +/// threads is not bounded here: each abandoned capture leaves one parked in +/// `read` until the grandchild lets go, so every probe that meets one costs two +/// until it does. The trade is deliberate — a parked reader holds a stack and +/// no CPU, and the alternative is the one below. +/// +/// Discarding rather than closing the read end is deliberate. Dropping the pipe +/// would send the still-writing grandchild `EPIPE`/`SIGPIPE`, which for the +/// background helpers a sandboxed installer legitimately leaves behind would +/// turn a leak whose cost we know into a behaviour change we cannot predict. +#[derive(Clone)] +struct Capture { + buffer: Arc>>, + finished: Arc, + failure: Arc>>, + /// Set by `take()`: nothing will read this capture's bytes again. + /// + /// Read and written under `buffer`'s mutex, so "emptied and marked" is one + /// step as far as the reader is concerned. Checking it outside the lock + /// would leave the window where a reader that has just passed the check + /// appends to the buffer `take()` is in the middle of emptying. + abandoned: Arc, +} + +impl Capture { + /// Start draining `pipe` on its own thread. + /// + /// Publishing in chunks as they arrive, rather than returning the whole + /// buffer at EOF, is what lets the caller take partial output from a reader + /// that is still blocked. + fn draining(pipe: Option) -> Self { + let capture = Self { + buffer: Arc::new(Mutex::new(Vec::new())), + finished: Arc::new(AtomicBool::new(false)), + failure: Arc::new(Mutex::new(None)), + abandoned: Arc::new(AtomicBool::new(false)), + }; + let Some(mut pipe) = pipe else { + capture.finished.store(true, Ordering::Release); + return capture; + }; + let sink = capture.clone(); + thread::spawn(move || { + let mut chunk = [0_u8; READ_CHUNK]; + loop { + match pipe.read(&mut chunk) { + Ok(0) => break, + Ok(read) => { + // Keep draining once abandoned, but stop accumulating: + // see the type's doc. The check shares the buffer's + // lock with `take()`, so after `take()` returns this + // buffer cannot grow by another byte. + let mut buffer = sink.lock(); + if !sink.abandoned.load(Ordering::Acquire) { + buffer.extend_from_slice(&chunk[..read]); + } + } + // `read_to_end` — what `wait_with_output` used before this + // function existed — retries an interrupted read and + // returns every other error. Treating either as EOF would + // silently shorten the output instead, and the callers + // parse these bytes as JSON, where that surfaces as a + // parse error pointing at the wrong thing. + // Round again rather than break: not the end of anything. + Err(error) if error.kind() == ErrorKind::Interrupted => {} + Err(error) => { + // Recorded only while someone is still listening. Once + // `take()` has fully returned this is merely tidy — + // nothing calls `take()` twice, so a failure stored + // afterwards would just sit unread. + // + // It is load-bearing for one window, and that window is + // inside `take()` itself: `take()` marks `abandoned` + // and empties the buffer under `buffer`'s lock, then + // releases it and drains the failure slot under a + // *different* lock. A read error that lands in between + // would otherwise be handed back attached to bytes the + // caller has already been given, and `assemble_output` + // turns any failure on a non-timed-out run into a hard + // `failed to read the output of ...` — a cleanly + // finished command reported as a read failure. Taking + // `buffer`'s lock before the check is what closes it; + // a bare load could pass the check, then have `take()` + // run the whole sequence before the store. + let _buffer = sink.lock(); + if !sink.abandoned.load(Ordering::Acquire) { + *sink.failure() = Some(error); + } + break; + } + } + } + sink.finished.store(true, Ordering::Release); + }); + capture + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Vec> { + self.buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn failure(&self) -> std::sync::MutexGuard<'_, Option> { + self.failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn is_finished(&self) -> bool { + self.finished.load(Ordering::Acquire) + } + + /// Take what has been captured, leaving a still-running reader unblocked. + /// + /// The bytes come back alongside any read error rather than instead of it, + /// so a caller already reporting something else — a timeout — can still + /// quote what it managed to collect. + /// + /// This is also the point of no return for the reader: whatever it collects + /// after this is discarded, because there is no second caller to hand it to. + fn take(&self) -> (Vec, Option) { + let bytes = { + // Marked while the buffer is held, so the reader cannot append + // between the mark and the emptying and strand bytes in a buffer + // that is never read again. + let mut buffer = self.lock(); + self.abandoned.store(true, Ordering::Release); + std::mem::take(&mut *buffer) + }; + (bytes, self.failure().take()) + } +} + +/// Run `command` to completion with both output streams captured, giving up +/// after `timeout`. +/// +/// A thread per pipe drains stdout and stderr, and both are started *before* +/// the wait loop. That ordering is the whole point. Waiting for the child to +/// exit and only then calling `wait_with_output` — the shape this replaced — +/// deadlocks: the pipes are bounded by the OS (commonly ~64KiB, less on +/// Windows), so a child that writes more than that blocks in `write`, which +/// means it never exits, which means the drain is never reached. The only way +/// out was the timeout, so it measured how much the child printed rather than +/// how long it took, and killed successful commands mid-flight. +/// +/// Both pipes need their own reader for the same reason: draining stdout to +/// completion and only then reading stderr stalls on a child that fills stderr +/// first. +/// +/// The call returns within roughly `timeout` either way: once the child is no +/// longer running the readers get `DRAIN_GRACE` to finish and are then abandoned +/// rather than joined. Joining unconditionally would hand the schedule to a +/// grandchild the child left behind, which holds the pipe's write end open so no +/// EOF ever arrives — that is how an abandoned `sleep` comes to decide how long +/// a 2-second probe takes. +/// +/// The stdio configuration is this function's own: it sets null stdin and piped +/// stdout/stderr on `command`, overriding whatever the caller set, because +/// capturing both streams is the whole contract. Callers need not — and should +/// not — set them. The stdin half is a promise in its own right rather than +/// housekeeping: a child that reads from an inherited stdin waits for input +/// that is never coming, so it is the deadline that ends it. Pinned by +/// `a_child_that_reads_stdin_is_given_an_empty_one`, on both platforms. +/// +/// `label` names the subject in error messages (e.g. `"process"`, +/// `"sandbox process"`). +pub fn run_with_timeout(mut command: Command, timeout: Duration, label: &str) -> Result { + let (child, stdout, stderr) = + spawn_captured(&mut command).with_context(|| format!("failed to spawn {label}"))?; + run_captured(child, stdout, stderr, timeout, label) +} + +/// [`run_with_timeout`] for a caller that wants to know how the run ended +/// rather than to have a timeout or a failure turned into an error: the same +/// executor, with the result as a [`CommandCapture`]. +/// +/// Nothing is an `Err` here. A program that cannot be started, a timeout and a +/// failure to wait are each an [`CommandOutcome`] of the capture, and a timeout +/// keeps what the child had written, so a caller that reads `stdout_raw` after +/// one gets the partial output [`run_with_timeout`] only quotes. The bounds are +/// [`run_with_timeout`]'s: the readers are abandoned after [`DRAIN_GRACE`], so +/// a descendant that keeps a pipe open does not extend the call. +pub(crate) fn run_bounded(mut command: Command, timeout: Duration, label: &str) -> CommandCapture { + let argv = std::iter::once(command.get_program()) + .chain(command.get_args()) + .map(|part| part.to_string_lossy().into_owned()) + .collect(); + match spawn_captured(&mut command) { + Err(error) => CommandCapture::from_run( + argv, + CommandOutcome::from_spawn_error(error.kind()), + None, + Vec::new(), + Vec::new(), + Some(error.to_string()), + ), + Ok((child, stdout, stderr)) => { + capture_of(argv, settle(child, stdout, stderr, timeout, label)) + } + } +} + +/// Start `command` with null stdin and both output pipes drained from the first +/// moment, which is what lets the wait loop be bounded at all. +fn spawn_captured(command: &mut Command) -> std::io::Result<(Child, Capture, Capture)> { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + // Taking the pipes here is what obliges us to assemble `Output` by hand + // later: `wait_with_output` reads the handles it owns, and these are gone. + let stdout = Capture::draining(child.stdout.take()); + let stderr = Capture::draining(child.stderr.take()); + Ok((child, stdout, stderr)) +} + +/// How a run ended and what it left behind, before it is turned into either a +/// `Result` or a [`CommandCapture`]. +enum Settled { + /// The child was waited for: it exited, or the deadline killed it. + Waited { + status: ExitStatus, + timed_out: bool, + /// Whether both readers reached end of output before the drain grace + /// ended. When not, something other than the child still holds a pipe + /// and the bytes are what arrived by then. + drained: bool, + stdout: (Vec, Option), + stderr: (Vec, Option), + }, + /// Polling or reaping the child failed, and it was killed. + WaitFailed { + error: anyhow::Error, + stdout: (Vec, Option), + stderr: (Vec, Option), + }, +} + +/// The wait, the drain grace and the `take()` of both captures: everything +/// between a child being spawned and its result being formed. +/// +/// Generic over [`Waitable`] rather than taking a `Child` so that the wait +/// loop's two failure paths — which are the only ways out of this function that +/// skip the drain grace — are reachable from a test. +fn settle( + mut child: C, + stdout: Capture, + stderr: Capture, + timeout: Duration, + label: &str, +) -> Settled { + // The single error path between `Capture::draining` and the `take()` pair + // below, and it must take both captures before it returns. `take()` is + // what marks a reader abandoned, and an abandoned reader is what stops a + // buffer nobody will read again from growing for as long as a grandchild + // holds the pipe open. Returning straight out of the wait loop — the shape + // this replaced — left both readers appending forever, in a daemon that + // probes every 60 seconds. + let (status, timed_out) = match wait_bounded(&mut child, timeout, label, thread::sleep) { + Ok(outcome) => outcome, + Err(error) => { + return Settled::WaitFailed { + error, + stdout: stdout.take(), + stderr: stderr.take(), + }; + } + }; + + let drained = wait_for_readers(&stdout, &stderr, DRAIN_GRACE, thread::sleep); + Settled::Waited { + status, + timed_out, + drained, + stdout: stdout.take(), + stderr: stderr.take(), + } +} + +/// Give both readers up to `grace` to reach end of output, looking again after +/// each `sleep` of the back-off schedule, and say whether they did. +/// +/// `sleep` is a parameter so a test can see the schedule the loop really +/// follows instead of inferring it from [`next_poll_interval`] alone. +fn wait_for_readers( + stdout: &Capture, + stderr: &Capture, + grace: Duration, + mut sleep: impl FnMut(Duration), +) -> bool { + let drained_by = Instant::now() + grace; + let mut pause = FIRST_POLL; + while !(stdout.is_finished() && stderr.is_finished()) && Instant::now() < drained_by { + sleep(pause); + pause = next_poll_interval(pause); + } + stdout.is_finished() && stderr.is_finished() +} + +/// The rest of [`run_with_timeout`] once the child is spawned and both readers +/// are running. +fn run_captured( + child: C, + stdout: Capture, + stderr: Capture, + timeout: Duration, + label: &str, +) -> Result { + match settle(child, stdout, stderr, timeout, label) { + Settled::WaitFailed { error, .. } => Err(error), + Settled::Waited { + status, + timed_out, + stdout, + stderr, + .. + } => assemble_output(status, timed_out, stdout, stderr, label, timeout), + } +} + +/// A [`Settled`] run as a [`CommandCapture`], keeping the bytes a timeout or a +/// read error would otherwise have turned into an error. +fn capture_of(argv: Vec, settled: Settled) -> CommandCapture { + match settled { + Settled::WaitFailed { + error, + stdout, + stderr, + } => CommandCapture::from_run( + argv, + CommandOutcome::WaitFailed, + None, + stdout.0, + stderr.0, + Some(format!("{error:#}")), + ), + Settled::Waited { + status, + timed_out, + drained, + stdout: (stdout, stdout_failure), + stderr: (stderr, stderr_failure), + } => { + let outcome = if timed_out { + CommandOutcome::TimedOut + } else { + command_capture::outcome_of_status(status) + }; + // What cut the output short, if anything: a pipe that failed to + // read, and/or a descendant still writing when the grace ended. The + // outcome stays what the child did, and the bytes are what arrived. + let notes: Vec = [ + stdout_failure.map(|error| format!("failed to read stdout: {error}")), + stderr_failure.map(|error| format!("failed to read stderr: {error}")), + (!drained).then(|| STILL_ARRIVING.to_owned()), + ] + .into_iter() + .flatten() + .collect(); + let detail = (!notes.is_empty()).then(|| notes.join("; ")); + CommandCapture::from_run(argv, outcome, Some(status), stdout, stderr, detail) + } + } +} + +/// Wait for `child` to exit, killing it once `timeout` has passed. +/// +/// Returns the status and whether the deadline is what ended the wait. Kept +/// free of the captures so that it has exactly one caller to get the +/// abandon-on-error handling right, rather than one per `?`. `sleep` is what +/// the loop waits with between polls, a parameter so a test can see the +/// back-off schedule it follows. +fn wait_bounded( + child: &mut C, + timeout: Duration, + label: &str, + mut sleep: impl FnMut(Duration), +) -> Result<(ExitStatus, bool)> { + let deadline = Instant::now() + timeout; + let mut pause = FIRST_POLL; + loop { + let polled = child.try_wait(); + if polled.is_err() { + // The only exit from this function that is not already followed by + // a kill. Whatever stopped us polling, we are about to stop + // waiting, and the caller drops the child without ever reaping it; + // leaving it running would hand back a process nobody is watching. + // `let _` to match the timeout branch: the kill is best-effort, and + // the poll error is the one worth reporting. + let _ = child.kill(); + } + if let Some(status) = polled.with_context(|| format!("failed to poll {label}"))? { + return Ok((status, false)); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let status = child + .wait() + .with_context(|| format!("failed to reap timed-out {label}"))?; + return Ok((status, true)); + } + sleep(pause); + pause = next_poll_interval(pause); + } +} + +/// What a capture's `detail` says when the drain grace ended with a reader still +/// waiting for end of output, so the caller can tell a complete capture from a +/// short one. +const STILL_ARRIVING: &str = "output was still arriving when the drain grace ended"; + +/// The three child operations [`wait_bounded`] needs. +/// +/// A trait only so a test can fail `try_wait` or the post-kill `wait` on +/// demand: both are error paths that skip the drain grace, and provoking either +/// from a real child needs the kernel to refuse to reap a process. +trait Waitable { + fn try_wait(&mut self) -> std::io::Result>; + fn kill(&mut self) -> std::io::Result<()>; + fn wait(&mut self) -> std::io::Result; +} + +// `Child::` rather than `Self::` on purpose: inside this impl `Self` is +// `Child`, so `Self::try_wait` reads as the trait method being defined +// here rather than the inherent one being forwarded to. +#[allow(clippy::use_self)] +impl Waitable for Child { + fn try_wait(&mut self) -> std::io::Result> { + Child::try_wait(self) + } + + fn kill(&mut self) -> std::io::Result<()> { + Child::kill(self) + } + + fn wait(&mut self) -> std::io::Result { + Child::wait(self) + } +} + +/// Turn what the wait loop and the two readers produced into the call's result. +/// +/// Split out from [`run_with_timeout`] because it is the only way to test it. +/// Every branch here is plain logic over values already in hand — which error +/// wins, which stream gets quoted — but reaching it through a real child means +/// making a pipe fail on demand, which needs a kernel having a bad day. Given +/// the two `(bytes, error)` pairs directly, each branch is an ordinary unit +/// test; see the cases at the bottom of this file. +fn assemble_output( + status: ExitStatus, + timed_out: bool, + (stdout, stdout_failure): (Vec, Option), + (stderr, stderr_failure): (Vec, Option), + label: &str, + timeout: Duration, +) -> Result { + let output = Output { + status, + stdout, + stderr, + }; + + if timed_out { + // `command_failure_detail` is the crate's one "which stream explains + // this failure" rule — stderr, else stdout, else "no output" — shared + // since #584 so callers cannot quietly disagree about the order. + // + // Truncated, not dropped: what the child managed to say before it + // stalled is usually the whole explanation, and the timeout is named + // first either way, so a reader who only sees the head still learns + // which command gave up and why. + bail!( + "{label} exceeded {}s timeout: {}", + timeout.as_secs(), + truncate_to_chars( + crate::command_failure_detail(&output), + MAX_QUOTED_OUTPUT_CHARS, + ) + ); + } + + // Only once the timeout has had its say: a killed child whose pipe then + // fails to read is a timeout, and reporting it as a read error would name + // the symptom instead of the cause. + if let Some(error) = stdout_failure.or(stderr_failure) { + return Err(anyhow::Error::new(error)) + .with_context(|| format!("failed to read the output of {label}")); + } + + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Stdio; + + /// The shells every grandchild case below is run against. + /// + /// Whether a shell forks or execs decides whether a grandchild exists at + /// all, so it decides whether these tests test anything: dash forks `sleep` + /// in `sh -c "sleep 30"`, bash execs it into the shell's own process and no + /// grandchild is left holding the pipe. A test written against one is dead + /// weight on a host running the other, and which one `/bin/sh` is varies + /// across the distributions this ships on. So the scripts below are written + /// to fork under either, and each case runs under every shell present to + /// keep that true. + #[cfg(unix)] + fn grandchild_shells() -> Vec<&'static str> { + let shells: Vec<&'static str> = ["/bin/sh", "/bin/bash", "/bin/dash"] + .into_iter() + .filter(|shell| std::path::Path::new(shell).exists()) + .collect(); + assert!( + !shells.is_empty(), + "no shell to run the grandchild cases in" + ); + shells + } + + #[test] + fn a_command_that_succeeds_reports_its_output() { + let mut command = Command::new(if cfg!(windows) { "cmd" } else { "sh" }); + if cfg!(windows) { + // No space around `&`: cmd would carry a trailing one into the + // echoed text, which `trim` hides on stdout but is easier not to + // write than to explain. + command.args(["/C", "echo hello&echo oops 1>&2"]); + } else { + command.args(["-c", "echo hello; echo oops >&2"]); + } + + let output = run_with_timeout(command, Duration::from_secs(10), "process").expect("ran"); + + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "hello"); + // Asserted on Windows too: `1>&2` is a cmd redirection like any other, + // and skipping it there left the second pipe — the one whose separate + // reader this function exists to provide — unchecked on that platform. + assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), "oops"); + } + + /// What a stdin-reading fixture prints when its read reaches EOF. + const STDIN_EOF_MARKER: &str = "eof"; + + /// The assertions shared by both platforms' empty-stdin cases. + /// + /// `command` must run a child that reads one line from stdin and prints + /// [`STDIN_EOF_MARKER`] if that read reaches EOF instead. + /// + /// The stdin set on `command` here is the read end of a pipe whose write end + /// this function holds open across the whole call, so it never reaches EOF + /// of its own accord. That is what makes the assertion two-sided rather than + /// a tautology: were `run_with_timeout` to stop setting null stdin, the + /// child would sit in `read` until the deadline killed it — exactly how a + /// child that prompts behaves against an inherited terminal — and the call + /// would come back as a timeout rather than with the marker. + /// + /// Setting stdin on the caller's side is also the contract under test: this + /// function's own stdio overrides whatever the caller set, which is what + /// lets all four callers hand over a bare `Command` and say nothing about + /// stdin at all. + /// + /// One thing it does not pin, said plainly rather than left to be + /// discovered: `null` against `inherit`. Telling those two apart needs this + /// test process's own fd 0 to be something other than EOF, and arranging + /// that is a `dup2` racing every other test in the binary. An `inherit` + /// mutant was run and does pass here. What is caught is the regression that + /// can actually happen -- no stdin setting at all, which is the line this + /// function took over from its callers. + /// + /// The race is avoidable by re-execing this test binary under a stdin pipe + /// the outer test holds open and asserting in that child, which would fail + /// the `inherit` mutant too. Not done: it buys one mutant that no plausible + /// edit produces -- nothing would replace `null` with `inherit`, the stdio + /// mode this helper exists to stop callers from being handed -- at the cost + /// of a self-re-execing test whose own harness plumbing is then the thing + /// most likely to break. + fn assert_the_child_is_given_an_empty_stdin(mut command: Command) { + let (stdin_reader, stdin_writer) = + std::io::pipe().expect("failed to create a pipe for the child's stdin"); + command.stdin(Stdio::from(stdin_reader)); + + let started = Instant::now(); + let output = run_with_timeout(command, Duration::from_secs(5), "process").expect( + "a child given an empty stdin must read EOF rather than block until the deadline", + ); + let elapsed = started.elapsed(); + // Dropped only here: releasing the write end any earlier would close the + // pipe, so the child would reach EOF through the caller's stdin and the + // test would pass whatever this function set. + drop(stdin_writer); + + assert!(output.status.success(), "{output:?}"); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + STDIN_EOF_MARKER, + "the child must be given an empty stdin rather than the one the caller set" + ); + assert!( + elapsed < Duration::from_secs(3), + "the read must hit EOF at once rather than waiting out the deadline, but the call \ + took {elapsed:?}" + ); + } + + #[cfg(unix)] + #[test] + fn a_child_that_reads_stdin_is_given_an_empty_one() { + // `read` reports EOF by returning non-zero, so the `||` branch is the + // one an empty stdin takes. A stdin left open takes neither. + let script = format!("read line || echo {STDIN_EOF_MARKER}"); + let mut command = Command::new("sh"); + command.args(["-c", script.as_str()]); + + assert_the_child_is_given_an_empty_stdin(command); + } + + #[cfg(unix)] + #[test] + fn a_genuinely_slow_command_times_out_on_time() { + for shell in grandchild_shells() { + // `sleep 30 & wait` rather than plain `sleep 30`: a lone final + // command is exec-ed into the shell itself by bash, which would + // leave nothing behind to hold the pipe and quietly turn this into + // a test of the easy case. Backgrounding forces a real grandchild + // under every shell. + // + // Killing the shell at the deadline does not kill that grandchild, + // so it keeps the inherited write end of both pipes open and no EOF + // arrives. An implementation that joins its readers therefore waits + // out the full 30 seconds and the timeout bounds nothing — which + // for the 2-second amd-smi probe is the difference between a snappy + // telemetry read and a stalled one. + let mut command = Command::new(shell); + command.args(["-c", "sleep 30 & wait"]); + + let started = Instant::now(); + let error = run_with_timeout(command, Duration::from_secs(1), "process") + .expect_err("a command that outlives its timeout must fail"); + let elapsed = started.elapsed(); + + assert!( + error.to_string().contains("exceeded 1s timeout"), + "the timeout must be reported as one, under {shell}: {error}" + ); + assert!( + elapsed < Duration::from_secs(5), + "the call must return at its deadline rather than waiting for an abandoned \ + grandchild to exit, but under {shell} it took {elapsed:?}" + ); + } + } + + #[cfg(unix)] + #[test] + fn a_grandchild_holding_the_pipe_does_not_hold_a_successful_call() { + for shell in grandchild_shells() { + // The timeout case above covers a child that is killed. This is the + // same hazard on the path that has no timeout to blame: the child + // exits 0 straight away, having left a background `sleep` holding + // the write end of both pipes, so there is no EOF for another five + // seconds. Joining the readers — or giving them an unbounded grace + // — hands the schedule to that `sleep` and a command that finished + // instantly takes 5s. `DRAIN_GRACE` is what bounds it. + let mut command = Command::new(shell); + command.args(["-c", "echo ready; sleep 5 &"]); + + let started = Instant::now(); + let output = run_with_timeout(command, Duration::from_secs(20), "process") + .expect("a child that exits cleanly must not be reported as a timeout"); + let elapsed = started.elapsed(); + + assert!(output.status.success(), "under {shell}: {output:?}"); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "ready", + "what the child wrote before exiting must still arrive, under {shell}" + ); + // Generous against a loaded machine, and still a third of the + // grandchild's lifetime: the regression this guards takes the full + // five seconds, not a few hundred milliseconds more than the grace. + assert!( + elapsed < Duration::from_secs(3), + "the call must not wait for the grandchild, but under {shell} it took {elapsed:?}" + ); + } + } + + #[cfg(unix)] + #[test] + fn output_arriving_after_the_child_exits_is_still_collected() { + for shell in grandchild_shells() { + // What `DRAIN_GRACE` is for, and the only way to ask for it on + // demand. A child that writes and exits normally cannot pin it: its + // reader is already running and drains the pipe microseconds later, + // long before the wait loop's next poll even notices the exit, so + // taking the buffer the instant the child is seen to have exited + // still collects everything. Here a grandchild writes once the + // child is gone instead. + // + // The 0.1s sits between the two bounds that decide the outcome: + // past `POLL_INTERVAL` (50ms), the longest the wait loop can take + // to notice the exit, so taking the output there misses + // `late-canary` outright; and well inside `DRAIN_GRACE` (200ms), so + // draining collects it. Both margins are ~2x, and the grandchild + // exits immediately after writing, so a working implementation sees + // EOF and returns rather than sitting out the whole grace. + let mut command = Command::new(shell); + command.args(["-c", "echo early-canary; (sleep 0.1; echo late-canary) &"]); + + let started = Instant::now(); + let output = run_with_timeout(command, Duration::from_secs(20), "process") + .expect("a child that exits cleanly must not be reported as a timeout"); + let elapsed = started.elapsed(); + let stdout = String::from_utf8_lossy(&output.stdout); + + assert!( + stdout.contains("early-canary"), + "under {shell}, the child's own output is missing: {stdout:?}" + ); + assert!( + stdout.contains("late-canary"), + "under {shell}, output still in flight when the child exited was dropped \ + instead of drained: {stdout:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "draining must end at EOF rather than running long, but under {shell} the call \ + took {elapsed:?}" + ); + } + } + + #[test] + fn an_interrupted_read_is_retried_and_a_failed_one_is_reported() { + // Neither branch is reachable through a real child here — EINTR needs a + // signal to land mid-read, and a pipe read failing outright needs the + // kernel to be having a bad day — so the reader is fed a scripted + // stream directly. The property is the one `read_to_end` gave before + // this function replaced it: an interrupted read is not the end of the + // stream, and a failed one is not a successful short read. + struct ScriptedPipe(std::collections::VecDeque>); + + impl Read for ScriptedPipe { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + match self.0.pop_front() { + Some(Ok(bytes)) => { + out[..bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + Some(Err(error)) => Err(error), + None => Ok(0), + } + } + } + + fn drain(script: Vec>) -> (Vec, Option) { + let capture = Capture::draining(Some(ScriptedPipe(script.into()))); + let deadline = Instant::now() + Duration::from_secs(10); + while !capture.is_finished() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!(capture.is_finished(), "the reader never finished"); + capture.take() + } + + let (bytes, failure) = drain(vec![ + Ok(b"before"), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"after"), + Ok(b""), + ]); + assert_eq!( + String::from_utf8_lossy(&bytes), + "beforeafter", + "an interrupted read must be retried, not taken for the end of the stream" + ); + assert!(failure.is_none(), "{failure:?}"); + + let (bytes, failure) = drain(vec![ + Ok(b"partial"), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"unreachable"), + ]); + assert_eq!(String::from_utf8_lossy(&bytes), "partial"); + assert_eq!( + failure.map(|error| error.kind()), + Some(ErrorKind::BrokenPipe), + "a read that failed must be reported rather than passed off as EOF" + ); + } + + /// A pipe that never ends until the test says so. + /// + /// Stoppable rather than genuinely endless so the reader thread finishes + /// with the test instead of spinning for the life of the test binary. + struct EndlessPipe { + reads: Arc, + stop: Arc, + } + + impl Read for EndlessPipe { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.stop.load(Ordering::Acquire) { + return Ok(0); + } + // Paced so the test can interleave with the reader rather than + // racing a tight loop, and so a regression accumulates a size the + // assertion can see without the test allocating gigabytes. + thread::sleep(Duration::from_millis(1)); + let filled = out.len().min(4096); + out[..filled].fill(b'x'); + self.reads.fetch_add(1, Ordering::Release); + Ok(filled) + } + } + + #[test] + fn an_abandoned_reader_stops_filling_a_buffer_nobody_will_read() { + // The shape this guards: the child exited but a grandchild still holds + // the write end, so `DRAIN_GRACE` expires, `take()` hands the caller + // what there is and the reader is left running. In `rocmd` — a daemon + // that probes amd-smi every 60s and runs MCP captures besides — a + // reader that kept appending would grow a `Vec` that nothing will ever + // read, for as long as the grandchild lives. There is no way to ask + // `run_with_timeout` for that state on demand, so the capture is driven + // directly, which is also where the invariant lives. + let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let capture = Capture::draining(Some(EndlessPipe { + reads: Arc::clone(&reads), + stop: Arc::clone(&stop), + })); + + let wait_for_reads = |target: usize| { + let deadline = Instant::now() + Duration::from_secs(10); + while reads.load(Ordering::Acquire) < target && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!( + reads.load(Ordering::Acquire) >= target, + "the reader stalled before {target} reads" + ); + }; + + wait_for_reads(4); + let (bytes, failure) = capture.take(); + // Establishes that the reader really was accumulating up to this point, + // so the emptiness asserted below is the flag working rather than a + // reader that never ran. + assert!( + !bytes.is_empty(), + "the reader must collect while someone is still listening" + ); + assert!(failure.is_none(), "{failure:?}"); + + // Well past what fits in anything the implementation might buffer + // internally: 50 more 4KiB reads is ~200KB, which a regression leaves + // sitting in the buffer and a correct implementation discards. + wait_for_reads(reads.load(Ordering::Acquire) + 50); + let leaked = capture.lock().len(); + assert_eq!( + leaked, 0, + "after `take()` nothing will read this buffer again, so a reader that keeps \ + appending grows it without bound for the life of the daemon; {leaked} bytes \ + accumulated" + ); + + stop.store(true, Ordering::Release); + let deadline = Instant::now() + Duration::from_secs(10); + while !capture.is_finished() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!(capture.is_finished(), "the reader never finished"); + } + + /// A child that never exits on its own and fails whichever of the wait + /// loop's two calls the test asks it to. + struct FailingChild { + /// `true`: `try_wait` reports an error rather than "still running", so + /// the loop's first `?` fires. `false`: it stays running until the + /// deadline, and the post-kill `wait` is what fails. + fail_poll: bool, + /// Raised by `kill`, so the test can tell a wait loop that leaves the + /// child running from one that does not. This is the whole reason + /// `kill` is on the trait rather than called on a concrete `Child`. + killed: Arc, + } + + impl Waitable for FailingChild { + fn try_wait(&mut self) -> std::io::Result> { + if self.fail_poll { + return Err(std::io::Error::from(ErrorKind::PermissionDenied)); + } + Ok(None) + } + + fn kill(&mut self) -> std::io::Result<()> { + self.killed.store(true, Ordering::Release); + Ok(()) + } + + fn wait(&mut self) -> std::io::Result { + Err(std::io::Error::from(ErrorKind::PermissionDenied)) + } + } + + /// Drive `run_captured` with a child that fails one of the wait loop's two + /// calls, and assert the error is reported, both readers are left + /// abandoned, *and* the child was killed on the way out. + /// + /// `fail_poll` picks which call fails; see [`FailingChild`]. + fn assert_wait_failure_abandons_both_readers( + fail_poll: bool, + timeout: Duration, + expected: &str, + ) { + let killed = Arc::new(AtomicBool::new(false)); + let child = FailingChild { + fail_poll, + killed: Arc::clone(&killed), + }; + let stop = Arc::new(AtomicBool::new(false)); + let out_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let err_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let stdout = Capture::draining(Some(EndlessPipe { + reads: Arc::clone(&out_reads), + stop: Arc::clone(&stop), + })); + let stderr = Capture::draining(Some(EndlessPipe { + reads: Arc::clone(&err_reads), + stop: Arc::clone(&stop), + })); + + let wait_for_reads = |counter: &Arc, target: usize| { + let deadline = Instant::now() + Duration::from_secs(10); + while counter.load(Ordering::Acquire) < target && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!( + counter.load(Ordering::Acquire) >= target, + "the reader stalled before {target} reads" + ); + }; + + // Both readers are accumulating before the call, so an empty buffer + // afterwards is the abandon working rather than a reader that never ran. + wait_for_reads(&out_reads, 4); + wait_for_reads(&err_reads, 4); + + let error = run_captured(child, stdout.clone(), stderr.clone(), timeout, "process") + .expect_err("a failed wait must be reported, not swallowed"); + assert!( + format!("{error:#}").contains(expected), + "expected {expected:?} in {error:#}" + ); + + // ~200KB more per stream: what a reader that was never marked abandoned + // accumulates into a buffer nobody will ever read again. + wait_for_reads(&out_reads, out_reads.load(Ordering::Acquire) + 50); + wait_for_reads(&err_reads, err_reads.load(Ordering::Acquire) + 50); + for (name, capture) in [("stdout", &stdout), ("stderr", &stderr)] { + let leaked = capture.lock().len(); + assert_eq!( + leaked, 0, + "an error out of the wait loop returns before the `take()` pair, so it must \ + abandon both readers itself or they grow a buffer nobody will read for as \ + long as a grandchild holds the pipe; {name} accumulated {leaked} bytes" + ); + } + + // Abandoning the readers is only half of it: the caller drops the child + // on this path without reaping it, so a wait loop that returns without + // killing leaves a process running that nobody will ever wait on again. + assert!( + killed.load(Ordering::Acquire), + "every exit from the wait loop must kill the child first; this one returned the \ + error with the child still running" + ); + + stop.store(true, Ordering::Release); + } + + #[test] + fn a_failure_to_poll_the_child_still_abandons_both_readers() { + assert_wait_failure_abandons_both_readers( + true, + Duration::from_secs(30), + "failed to poll process", + ); + } + + #[test] + fn a_failure_to_reap_a_timed_out_child_still_abandons_both_readers() { + // Zero timeout so the loop reaches the post-kill reap — the other `?` — + // on its first pass. + assert_wait_failure_abandons_both_readers( + false, + Duration::ZERO, + "failed to reap timed-out process", + ); + } + + /// A successful `ExitStatus` to feed `assemble_output`. + /// + /// `ExitStatus` has no portable public constructor; the per-platform + /// `ExitStatusExt` is the supported way to make one without spawning a + /// process purely to obtain it. + fn success_status() -> ExitStatus { + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt as _; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt as _; + ExitStatus::from_raw(0) + } + + #[test] + fn the_poll_interval_backs_off_from_a_few_milliseconds_to_the_flat_one() { + // The flat 50 ms this loop had would make a probe that exits at once take + // 50 ms, where the helpers it replaces polled every 20 to 25 ms. + let mut pause = FIRST_POLL; + let mut seen = vec![pause]; + for _ in 0..5 { + pause = next_poll_interval(pause); + seen.push(pause); + } + let millis: Vec = seen.iter().map(Duration::as_millis).collect(); + assert_eq!(millis, [5, 10, 20, 40, 50, 50]); + assert!(FIRST_POLL < Duration::from_millis(20)); + } + + /// A child that has not exited for its first `remaining` polls. + struct ExitsAfterPolls { + remaining: usize, + } + + impl Waitable for ExitsAfterPolls { + fn try_wait(&mut self) -> std::io::Result> { + if self.remaining == 0 { + return Ok(Some(success_status())); + } + self.remaining -= 1; + Ok(None) + } + + fn kill(&mut self) -> std::io::Result<()> { + Ok(()) + } + + fn wait(&mut self) -> std::io::Result { + Ok(success_status()) + } + } + + fn millis(slept: &[Duration]) -> Vec { + slept.iter().map(Duration::as_millis).collect() + } + + #[test] + fn the_wait_loop_sleeps_the_back_off_schedule_between_polls() { + // The sleeps are recorded, not taken, so the schedule the loop really + // follows is what is asserted. A loop that slept `POLL_INTERVAL` every + // time would charge a probe that exits at once the full 50 ms. + let mut slept = Vec::new(); + let mut child = ExitsAfterPolls { remaining: 6 }; + + let (status, timed_out) = + wait_bounded(&mut child, Duration::from_mins(1), "process", |pause| { + slept.push(pause); + }) + .expect("the child exits on the seventh poll"); + + assert!(status.success() && !timed_out); + assert_eq!(millis(&slept), [5, 10, 20, 40, 50, 50]); + } + + /// Two readers over [`EndlessPipe`]s that stop when `stop` is raised. + fn two_endless_captures(stop: &Arc) -> (Capture, Capture) { + let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let pipe = || EndlessPipe { + reads: Arc::clone(&reads), + stop: Arc::clone(stop), + }; + ( + Capture::draining(Some(pipe())), + Capture::draining(Some(pipe())), + ) + } + + #[test] + fn the_drain_wait_sleeps_the_back_off_schedule_until_both_readers_finish() { + let stop = Arc::new(AtomicBool::new(false)); + let (stdout, stderr) = two_endless_captures(&stop); + let mut slept = Vec::new(); + + let drained = wait_for_readers(&stdout, &stderr, Duration::from_secs(30), |pause| { + slept.push(pause); + if slept.len() == 3 { + // End the pipes, and give the reader threads real time to see it. + stop.store(true, Ordering::Release); + thread::sleep(Duration::from_millis(100)); + } + }); + + assert!(drained, "both readers reached the end of their pipes"); + assert_eq!(millis(&slept[..3]), [5, 10, 20]); + } + + #[test] + fn a_drain_wait_that_runs_out_of_grace_says_the_readers_were_still_going() { + let stop = Arc::new(AtomicBool::new(false)); + let (stdout, stderr) = two_endless_captures(&stop); + + let drained = wait_for_readers(&stdout, &stderr, Duration::ZERO, |_| { + unreachable!("no grace, so no sleep") + }); + + stop.store(true, Ordering::Release); + assert!(!drained); + } + + /// Drive `settle` for a child that fails the wait loop the way + /// [`FailingChild`] says, over two readers that are accumulating. + fn settle_a_failing_child(fail_poll: bool, timeout: Duration) -> (Settled, bool) { + let killed = Arc::new(AtomicBool::new(false)); + let child = FailingChild { + fail_poll, + killed: Arc::clone(&killed), + }; + let stop = Arc::new(AtomicBool::new(false)); + // One counter per stream: a single shared one is satisfied by one reader + // doing all the reads, which leaves the other stream's bytes to chance. + let out_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let err_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let pipe = |reads: &Arc| EndlessPipe { + reads: Arc::clone(reads), + stop: Arc::clone(&stop), + }; + let stdout = Capture::draining(Some(pipe(&out_reads))); + let stderr = Capture::draining(Some(pipe(&err_reads))); + let deadline = Instant::now() + Duration::from_secs(10); + while (out_reads.load(Ordering::Acquire) < 4 || err_reads.load(Ordering::Acquire) < 4) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(5)); + } + assert!( + out_reads.load(Ordering::Acquire) >= 4 && err_reads.load(Ordering::Acquire) >= 4, + "both readers must have started" + ); + + let settled = settle(child, stdout, stderr, timeout, "process"); + + stop.store(true, Ordering::Release); + (settled, killed.load(Ordering::Acquire)) + } + + #[test] + fn a_failure_to_wait_is_a_capture_that_keeps_the_bytes_that_had_arrived() { + // `run_raw` returned `124` with the bytes joined after a failed poll, so + // the capture keeps them rather than dropping them with the error, and + // the child is killed on the way out. + for (fail_poll, timeout, expected) in [ + (true, Duration::from_secs(30), "failed to poll process"), + (false, Duration::ZERO, "failed to reap timed-out process"), + ] { + let (settled, killed) = settle_a_failing_child(fail_poll, timeout); + assert!(killed, "{expected}: the child must be killed"); + + let capture = capture_of(vec!["fake".to_owned()], settled); + + assert_eq!(capture.outcome, CommandOutcome::WaitFailed, "{expected}"); + assert!(capture.status.is_none(), "{expected}"); + assert!( + capture.stdout_raw.len() >= 4096 && capture.stdout_raw.iter().all(|&b| b == b'x'), + "{expected}: stdout bytes: {}", + capture.stdout_raw.len() + ); + assert!( + capture.stderr_raw.len() >= 4096 && capture.stderr_raw.iter().all(|&b| b == b'x'), + "{expected}: stderr bytes: {}", + capture.stderr_raw.len() + ); + assert_eq!(capture.exit_status, 1, "{expected}"); + let detail = capture.detail.expect("the wait error is kept in words"); + assert!(detail.contains(expected), "{detail}"); + } + } + + fn waited( + timed_out: bool, + drained: bool, + stdout: (Vec, Option), + stderr: (Vec, Option), + ) -> Settled { + Settled::Waited { + status: success_status(), + timed_out, + drained, + stdout, + stderr, + } + } + + #[test] + fn a_timed_out_settle_is_a_capture_with_the_partial_output_and_the_outcome() { + let capture = capture_of( + vec!["fake".to_owned()], + waited( + true, + true, + (b"partial".to_vec(), None), + (b"warning".to_vec(), None), + ), + ); + + assert_eq!(capture.outcome, CommandOutcome::TimedOut); + assert_eq!(capture.stdout_raw, b"partial"); + assert_eq!(capture.stderr, "warning"); + assert_eq!(capture.exit_status, 1, "a timeout has no exit code"); + assert!(capture.detail.is_none()); + } + + #[test] + fn a_read_failure_on_either_stream_keeps_the_bytes_and_the_outcome_the_child_had() { + // `run_with_timeout` turns this into an error and drops the bytes + // (`a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output`); + // the capture keeps both, as the older helpers did with `let _ =`. + let stdout_only = capture_of( + vec!["fake".to_owned()], + waited( + false, + true, + (b"partial".to_vec(), read_failure()), + (Vec::new(), None), + ), + ); + assert_eq!(stdout_only.outcome, CommandOutcome::Exited(0)); + assert_eq!(stdout_only.stdout, "partial"); + let detail = stdout_only.detail.expect("the read error is kept in words"); + assert!(detail.contains("failed to read stdout"), "{detail}"); + assert!(!detail.contains("stderr"), "{detail}"); + + let stderr_only = capture_of( + vec!["fake".to_owned()], + waited( + false, + true, + (b"out".to_vec(), None), + (b"partial".to_vec(), read_failure()), + ), + ); + assert_eq!(stderr_only.outcome, CommandOutcome::Exited(0)); + assert_eq!(stderr_only.stderr, "partial"); + let detail = stderr_only.detail.expect("the read error is kept in words"); + assert!(detail.contains("failed to read stderr"), "{detail}"); + assert!(!detail.contains("stdout"), "{detail}"); + + let both = capture_of( + vec!["fake".to_owned()], + waited( + false, + true, + (Vec::new(), read_failure()), + (Vec::new(), read_failure()), + ), + ); + let detail = both.detail.expect("both read errors are kept in words"); + assert!( + detail.contains("failed to read stdout") && detail.contains("failed to read stderr"), + "{detail}" + ); + } + + #[test] + fn a_capture_says_when_a_reader_was_still_going_at_the_end_of_the_grace() { + let short = capture_of( + vec!["fake".to_owned()], + waited(false, false, (b"early".to_vec(), None), (Vec::new(), None)), + ); + assert_eq!(short.outcome, CommandOutcome::Exited(0)); + assert_eq!(short.stdout, "early"); + assert_eq!(short.detail.as_deref(), Some(STILL_ARRIVING)); + + let complete = capture_of( + vec!["fake".to_owned()], + waited(false, true, (b"early".to_vec(), None), (Vec::new(), None)), + ); + assert!(complete.detail.is_none()); + } + + #[cfg(unix)] + #[test] + fn output_written_after_the_grace_is_missing_and_the_capture_says_so() { + // The descendant writes 0.5 s after the child exits, which is past the + // 200 ms grace: the capture holds `early`, and `detail` says it is short. + let mut command = Command::new("sh"); + command.args(["-c", "echo early; (sleep 0.5; echo late) &"]); + + let capture = run_bounded(command, Duration::from_secs(20), "process"); + + assert_eq!(capture.outcome, CommandOutcome::Exited(0)); + assert_eq!(capture.stdout, "early\n"); + assert_eq!(capture.detail.as_deref(), Some(STILL_ARRIVING)); + } + + #[test] + fn a_program_that_cannot_be_started_is_a_capture_not_an_error() { + let missing = "/nonexistent-dir-for-the-runner-pair/tool"; + let mut command = Command::new(missing); + command.arg("--flag"); + + let capture = run_bounded(command, Duration::from_secs(5), "process"); + + assert_eq!(capture.outcome, CommandOutcome::NotFound); + assert_eq!(capture.argv, [missing, "--flag"]); + assert!(capture.stdout_raw.is_empty() && capture.stderr_raw.is_empty()); + assert_eq!(capture.exit_status, 1); + assert!(capture.status.is_none()); + assert!(capture.detail.is_some(), "the OS error is kept in words"); + } + + #[cfg(unix)] + #[test] + fn a_program_that_exists_but_cannot_be_executed_is_a_spawn_failure_not_not_found() { + use std::os::unix::fs::PermissionsExt as _; + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("not-executable"); + std::fs::write(&path, "#!/bin/sh\n").expect("write"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).expect("chmod"); + + // Not skipped for root: `execve` refuses a file with no execute bit for + // every user. + let capture = run_bounded(Command::new(&path), Duration::from_secs(5), "process"); + + assert_eq!( + capture.outcome, + CommandOutcome::SpawnFailed(ErrorKind::PermissionDenied) + ); + } + + #[cfg(unix)] + #[test] + fn a_run_is_a_capture_with_its_exit_code_or_its_signal_and_both_streams() { + let capture = run_bounded( + { + let mut command = Command::new("sh"); + command.args(["-c", "echo out; echo err >&2; exit 3"]); + command + }, + Duration::from_secs(10), + "process", + ); + assert_eq!(capture.outcome, CommandOutcome::Exited(3)); + assert_eq!(capture.exit_status, 3); + assert_eq!(capture.stdout, "out\n"); + assert_eq!(capture.stderr_raw, b"err\n"); + assert_eq!(capture.argv, ["sh", "-c", "echo out; echo err >&2; exit 3"]); + assert_eq!(capture.status.and_then(|status| status.code()), Some(3)); + assert!(capture.detail.is_none()); + + let capture = run_bounded( + { + let mut command = Command::new("sh"); + command.args(["-c", "echo partial; kill -9 $$"]); + command + }, + Duration::from_secs(10), + "process", + ); + assert_eq!(capture.outcome, CommandOutcome::Signaled(Some(9))); + assert_eq!(capture.exit_status, 1); + assert_eq!(capture.stdout, "partial\n"); + } + + #[cfg(windows)] + #[test] + fn a_run_is_a_capture_with_its_exit_code_and_both_streams() { + let mut command = Command::new("cmd"); + command.args(["/C", "echo out&echo err 1>&2&exit 3"]); + + let capture = run_bounded(command, Duration::from_secs(10), "process"); + + assert_eq!(capture.outcome, CommandOutcome::Exited(3)); + assert_eq!(capture.stdout.trim(), "out"); + assert_eq!(capture.stderr.trim(), "err"); + } + + #[cfg(unix)] + #[test] + fn a_timeout_is_a_capture_with_what_the_child_had_written() { + // `exec` so killing the shell leaves nothing on the pipe: what is under + // test is the partial output and the outcome, not the join. + let mut command = Command::new("sh"); + command.args(["-c", "echo before; exec sleep 30"]); + + // Room for a loaded host to start the shell and run its `echo` before the + // deadline, since the assertion is on what the child had written by then. + let started = Instant::now(); + let capture = run_bounded(command, Duration::from_secs(3), "process"); + + assert_eq!(capture.outcome, CommandOutcome::TimedOut); + assert_eq!(capture.stdout, "before\n"); + assert!(started.elapsed() < Duration::from_secs(10)); + } + + fn read_failure() -> Option { + Some(std::io::Error::from(ErrorKind::BrokenPipe)) + } + + #[test] + fn a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output() { + // The silent short read this function exists to prevent: handing back + // `Ok` with a truncated buffer sends the caller off to parse half a + // JSON document and blame the document. + let error = assemble_output( + success_status(), + false, + (b"partial".to_vec(), read_failure()), + (Vec::new(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a stream that failed to read must not come back as success"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains("failed to read the output of process"), + "{rendered:?}" + ); + } + + #[test] + fn a_failure_on_either_stream_is_reported() { + // stdout is checked first, so a stderr-only failure is a separate + // branch: `stdout_failure.or(stderr_failure)` reduced to + // `stdout_failure` would still pass the case above. + let error = assemble_output( + success_status(), + false, + (Vec::new(), None), + (b"partial".to_vec(), read_failure()), + "process", + Duration::from_secs(2), + ) + .expect_err("a stderr stream that failed to read must not come back as success"); + + assert!( + format!("{error:#}").contains("failed to read the output of process"), + "{error:#}" + ); + } + + #[test] + fn a_timeout_outranks_a_read_failure() { + // Killing the child is what broke the pipe, so a read error here is the + // symptom. Reporting it would name the consequence and hide the cause, + // and the caller would go looking for a kernel problem. + let error = assemble_output( + success_status(), + true, + (Vec::new(), read_failure()), + (b"the real explanation".to_vec(), read_failure()), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains("exceeded 2s timeout"), + "the timeout is the cause and must be what is reported: {rendered:?}" + ); + assert!( + rendered.contains("the real explanation"), + "the child's own account must survive: {rendered:?}" + ); + assert!( + !rendered.contains("failed to read the output of"), + "the read error is downstream of the kill and must not displace it: {rendered:?}" + ); + } + + #[test] + fn a_timeout_quotes_stderr_in_preference_to_stdout() { + let error = assemble_output( + success_status(), + true, + (b"stdout-data".to_vec(), None), + (b"stderr-explanation".to_vec(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains("stderr-explanation"), + "stderr is where a child explains itself: {rendered:?}" + ); + assert!( + !rendered.contains("stdout-data"), + "quoting both would push the explanation behind a payload: {rendered:?}" + ); + } + + #[test] + fn a_timeout_falls_back_to_stdout_and_then_to_saying_there_was_none() { + // Every other timeout fixture writes to stderr, so without these two + // the fallback chain could be reordered or deleted unnoticed — and a + // child whose whole output is stdout is exactly the amd-smi case. + let error = assemble_output( + success_status(), + true, + (b"stdout-only".to_vec(), None), + (Vec::new(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + assert!( + format!("{error:#}").contains("stdout-only"), + "a child that only wrote stdout must still be quoted: {error:#}" + ); + + let error = assemble_output( + success_status(), + true, + (Vec::new(), None), + (Vec::new(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("no output"), + "a silent child must say so rather than trailing a bare colon: {rendered:?}" + ); + assert!(rendered.contains("exceeded 2s timeout"), "{rendered:?}"); + } + + #[test] + fn a_clean_run_returns_both_streams() { + let output = assemble_output( + success_status(), + false, + (b"out".to_vec(), None), + (b"err".to_vec(), None), + "process", + Duration::from_secs(2), + ) + .expect("a clean run must succeed"); + + assert!(output.status.success()); + assert_eq!(output.stdout, b"out"); + assert_eq!(output.stderr, b"err"); + } + + #[cfg(unix)] + #[test] + fn a_child_that_floods_both_pipes_does_not_deadlock() { + // The defect this function exists to prevent. Both pipes are bounded by + // the OS (commonly ~64KiB), so a child writing past that blocks in + // `write` and never exits — and a version of this function that drains + // only after the child exits never reads a byte, so the child is killed + // at the timeout and a successful command is reported as too slow. + // + // Flooding *both* streams is deliberate: a single-pipe child still + // passes against an implementation that drains stdout fully and only + // then starts on stderr, which is its own deadlock. + // + // `sh` rather than a real subject program (amd-smi, an SDK install), + // because the property is about this function's pipe handling and + // nothing else, and a shell makes it a unit test rather than one that + // needs a multi-GPU host. + let mut command = Command::new("sh"); + command.args([ + "-c", + "yes deadlock-canary | head -c 2000000; yes deadlock-canary | head -c 2000000 >&2", + ]); + + // Short, because a regression fails by exhausting this timeout: ten + // seconds is far past the ~0.1s a working drain takes, and far short of + // stalling the suite. + let output = run_with_timeout(command, Duration::from_secs(10), "process") + .expect("a child that outruns the pipe buffer must not be killed as a timeout"); + + assert!(output.status.success(), "{output:?}"); + // Without these the test would keep passing if the child ever stopped + // producing more than one buffer, which is to say it would stop testing + // the hazard. + assert!( + output.stdout.len() > 64 * 1024, + "stdout must exceed one pipe buffer: {} bytes", + output.stdout.len() + ); + assert!( + output.stderr.len() > 64 * 1024, + "stderr must exceed one pipe buffer: {} bytes", + output.stderr.len() + ); + } + + /// How large a quoted-output assertion lets the whole error message get. + /// + /// Comfortably above `MAX_QUOTED_OUTPUT_CHARS` plus the label and marker, + /// and two orders of magnitude below the ~200KB the fixtures below feed in, + /// so the assertion answers "is it bounded at all" rather than pinning the + /// exact constant — which is a tuning decision, not a contract. + const QUOTE_ASSERTION_CEILING: usize = 4000; + + /// The flooding fixtures below write this many bytes, past one pipe buffer + /// and ~100x the quote bound. + const FLOOD_BYTES: usize = 200_000; + + const FLOOD_CANARY: &str = "flood-canary"; + + /// The assertions shared by both platforms' timeout-quote cases. + /// + /// Checked together because they trade off against each other: a bound that + /// kept nothing would satisfy the size check while making the message + /// useless, and quoting everything keeps it useful while costing the logs + /// the thing this bound exists to protect. + fn assert_timeout_quote_is_bounded_and_useful(message: &str, timeout_secs: u64) { + assert!( + message.contains(&format!("exceeded {timeout_secs}s timeout")), + "the timeout must still be named: {message:?}" + ); + assert!( + message.contains(FLOOD_CANARY), + "the child's own output must still be quoted, not dropped: {message:?}" + ); + assert!( + message.contains("[truncated]"), + "a shortened quote must say so, or a reader takes the head for the whole: {message:?}" + ); + let length = message.chars().count(); + assert!( + length <= QUOTE_ASSERTION_CEILING, + "the quote must be bounded: the message is {length} characters, from a child that \ + printed {FLOOD_BYTES} bytes" + ); + } + + #[cfg(unix)] + #[test] + fn a_timed_out_child_has_its_output_quoted_within_a_bound() { + for shell in grandchild_shells() { + // Both halves are load-bearing. The flood is what makes the quote + // large; the backgrounded `sleep` is what makes this a timeout, + // which is the only path that quotes the child's output at all. + // + // Unbounded, this message travels: `rocmd` renders the error chain + // into the bridge snapshot's `note`, and the GPU-metrics watcher + // writes that note into two JSONL logs every 60s, neither ever + // rotated or capped. A real amd-smi on a multi-GPU host prints + // well past this fixture's size. + let mut command = Command::new(shell); + command.args([ + "-c", + &format!("yes {FLOOD_CANARY} | head -c {FLOOD_BYTES} >&2; sleep 30 & wait"), + ]); + + let error = run_with_timeout(command, Duration::from_secs(1), "process") + .expect_err("a command that outlives its timeout must fail"); + + assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 1); + } + } + + /// A scratch directory holding a Windows fixture's script, and for + /// [`Self::flooding`] fixtures the payload beside it. + /// + /// Per-process unique so concurrent test binaries cannot collide on it, and + /// removed on drop so a failing assertion does not leave either behind. + #[cfg(windows)] + struct WindowsFixture { + dir: std::path::PathBuf, + } + + #[cfg(windows)] + impl WindowsFixture { + /// Write `script` as a batch file in a scratch directory of its own. + /// + /// A batch file rather than `cmd /C ""`: cmd's rules for + /// stripping the quotes Rust adds around an argument containing spaces + /// and redirections are subtle enough that the fixture would become the + /// thing under test. A bare path has none of that, and `%~dp0` inside + /// the script resolves anything beside it without the test passing a + /// path in. + fn script(name: &str, script: &str) -> Self { + let dir = std::env::temp_dir() + .join(format!("rocm-cli-process-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("failed to create the fixture directory"); + std::fs::write(dir.join("fixture.bat"), script).expect("failed to write the script"); + + Self { dir } + } + + /// A [`Self::script`] fixture with a `FLOOD_BYTES` payload beside it for + /// the script to print. + fn flooding(name: &str, script: &str) -> Self { + let fixture = Self::script(name, script); + + let line = format!("{FLOOD_CANARY}\r\n"); + let mut payload = String::with_capacity(FLOOD_BYTES + line.len()); + while payload.len() < FLOOD_BYTES { + payload.push_str(&line); + } + std::fs::write(fixture.dir.join("payload.txt"), payload) + .expect("failed to write the payload"); + + fixture + } + + fn command(&self) -> Command { + let mut command = Command::new("cmd"); + command.args(["/C"]).arg(self.dir.join("fixture.bat")); + command + } + } + + #[cfg(windows)] + impl Drop for WindowsFixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + #[cfg(windows)] + #[test] + fn a_child_that_floods_both_pipes_does_not_deadlock() { + // The Windows half of the unix case above, and the same property: both + // pipes are bounded by the OS here too — more tightly than on Linux — + // so an implementation that drains only after the child exits, or + // drains stdout to completion before starting on stderr, never lets + // this child finish. `type` twice is the shell-free equivalent of the + // unix `yes | head`; cmd has no `yes`. + let fixture = WindowsFixture::flooding( + "flood", + "@echo off\r\ntype \"%~dp0payload.txt\"\r\ntype \"%~dp0payload.txt\" 1>&2\r\n", + ); + + let output = run_with_timeout(fixture.command(), Duration::from_secs(30), "process") + .expect("a child that outruns the pipe buffer must not be killed as a timeout"); + + assert!(output.status.success(), "{output:?}"); + assert!( + output.stdout.len() > 64 * 1024, + "stdout must exceed one pipe buffer: {} bytes", + output.stdout.len() + ); + assert!( + output.stderr.len() > 64 * 1024, + "stderr must exceed one pipe buffer: {} bytes", + output.stderr.len() + ); + } + + #[cfg(windows)] + #[test] + fn a_timed_out_child_has_its_output_quoted_within_a_bound() { + // Covers on Windows what three separate unix cases cover there, because + // one fixture happens to exhibit all of it: `ping -n 30` runs for ~29 + // seconds, so the call times out; `ping` is a grandchild of the `cmd` + // this function kills at the deadline, and killing a process on Windows + // does not kill its children, so it keeps the inherited write ends open + // and no EOF arrives — an implementation that joins its readers waits + // the full 29 seconds; and the preceding flood makes the resulting + // quote large enough for the bound to matter. + let fixture = WindowsFixture::flooding( + "timeout", + "@echo off\r\ntype \"%~dp0payload.txt\" 1>&2\r\nping -n 30 127.0.0.1\r\n", + ); + + let started = Instant::now(); + let error = run_with_timeout(fixture.command(), Duration::from_secs(2), "process") + .expect_err("a command that outlives its timeout must fail"); + let elapsed = started.elapsed(); + + assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 2); + assert!( + elapsed < Duration::from_secs(10), + "the call must return at its deadline rather than waiting for an abandoned \ + grandchild to exit, but it took {elapsed:?}" + ); + } + + #[cfg(windows)] + #[test] + fn a_child_that_reads_stdin_is_given_an_empty_one() { + // `set /p` leaves the variable untouched when its read reaches EOF, so + // pre-setting it to the marker is cmd's equivalent of the unix + // `read || echo`. A stdin left open blocks in `set /p` instead. + // + // Both callers in `rocmd` run on Windows -- the amd-smi probe in + // `apps/rocmd/src/host_snapshot.rs` and the `rocm` capture that + // `apps/rocmd/src/common.rs` makes through `run_rocm_capture_with_env` + // -- and `run_with_timeout` has no `cfg(windows)` branch, so this covers + // the same promise with the same two-sidedness as the unix case. + let fixture = WindowsFixture::script( + "stdin", + &format!( + "@echo off\r\nset \"LINE={STDIN_EOF_MARKER}\"\r\nset /p LINE=\r\necho %LINE%\r\n" + ), + ); + + assert_the_child_is_given_an_empty_stdin(fixture.command()); + } +} diff --git a/crates/rocm-core/src/report.rs b/crates/rocm-core/src/report.rs index 7a4748aae9..8884040776 100644 --- a/crates/rocm-core/src/report.rs +++ b/crates/rocm-core/src/report.rs @@ -330,6 +330,14 @@ pub fn is_rocm_supported(gfx_target: &str) -> bool { /// Build the report for this machine, or refuse and say which rule refused. /// +/// `cli_version` is the leading version of the CLI that writes the report, the +/// one token `rocm version` prints before its parenthetical ref and hash. It is +/// a parameter because this crate does not know it: its own package version +/// is the frozen workspace one, and the string `rocm version` prints is +/// computed by the `rocm` binary's build script. Anything that is not a short +/// version token is published as [`UNKNOWN`] instead (ASCII letters, digits, +/// `.`, `-` and `+`, starting with a digit, at most 64 characters). +/// /// # Errors /// When the machine holds hardware that is not on the ROCm compatibility /// matrix, or hardware whose architecture could not be read. @@ -337,6 +345,7 @@ pub fn prepare_report( examination: &Examination, entry: Option<&str>, fix_offered: bool, + cli_version: &str, ) -> Result { // Every AMD GPU is checked, not just the one a finding concerns. A released // GPU sitting beside an unreleased one does not make the machine @@ -368,9 +377,10 @@ pub fn prepare_report( return Err(Refusal::UnreleasedHardware); } - // `entry` is the only value here that comes from a caller rather than from - // the machine. Checked against the catalog so that a forged or mistaken id - // cannot carry caller-supplied text onto a public tracker. + // `entry` and `cli_version` are the only values here that come from a + // caller rather than from the machine. `entry` is checked against the + // catalog so that a forged or mistaken id cannot carry caller-supplied + // text onto a public tracker; `cli_version` is checked below. let entry_recognised = entry.is_some_and(crate::fix::is_catalog_id); let entry = if entry_recognised { entry.expect("checked Some above").to_owned() @@ -402,11 +412,36 @@ pub fn prepare_report( rocm: rocm_release(examination), engine, engine_version, - cli_version: env!("CARGO_PKG_VERSION").to_owned(), + cli_version: self::cli_version(cli_version), fix_offered, }) } +/// Longest CLI version a report carries. A `vX.Y.Z` tag with a pre-release +/// or build suffix is far shorter; anything longer is not a version. +const CLI_VERSION_MAX_LEN: usize = 64; + +/// The CLI version to publish, or [`UNKNOWN`]. +/// +/// The caller passes the leading version token, never the ref and hash after +/// it: a local branch name is free text from the machine, which is exactly +/// what this module keeps off a public tracker. Checked here rather than +/// trusted, because `prepare_report` is `pub` and re-exported: only ASCII +/// letters, digits, `.`, `-` and `+` (the characters of a semantic version) +/// are accepted, and it must start with a digit. +fn cli_version(candidate: &str) -> String { + let is_version_token = candidate.len() <= CLI_VERSION_MAX_LEN + && candidate.starts_with(|c: char| c.is_ascii_digit()) + && candidate + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'+')); + if is_version_token { + candidate.to_owned() + } else { + UNKNOWN.to_owned() + } +} + /// The population-level OS major version to publish. /// /// Sourced per platform, because no single `Examination` field holds "the OS @@ -680,6 +715,19 @@ mod tests { use super::*; use crate::examine::Gpu; + /// What the `rocm` binary passes for a build from `main`. + const TEST_CLI_VERSION: &str = "0.0.0-dev"; + + /// [`prepare_report`] with the CLI version every test here uses unless it + /// is the thing under test. + fn prepare( + examination: &Examination, + entry: Option<&str>, + fix_offered: bool, + ) -> Result { + prepare_report(examination, entry, fix_offered, TEST_CLI_VERSION) + } + /// A machine whose every free-text field is a distinctive marker. /// /// The markers are what [`no_report_carries_a_value_the_machine_did_not_agree_to_publish`] @@ -761,7 +809,7 @@ mod tests { /// released has to come back with a report. #[test] fn hardware_off_the_rocm_compatibility_matrix_never_produces_a_report() { - let released = prepare_report(&machine_of_sentinels("gfx1100"), None, false); + let released = prepare(&machine_of_sentinels("gfx1100"), None, false); assert!( released.is_ok(), "premise failed: a machine holding only released hardware must produce a report, \ @@ -769,7 +817,7 @@ mod tests { ); assert_eq!( - prepare_report(&machine_of_sentinels(NOT_RELEASED), None, false), + prepare(&machine_of_sentinels(NOT_RELEASED), None, false), Err(Refusal::UnreleasedHardware), "{NOT_RELEASED} is on no compatibility matrix, so it must not be describable" ); @@ -790,7 +838,7 @@ mod tests { is_amd: true, }); assert_eq!( - prepare_report(&mixed, None, false), + prepare(&mixed, None, false), Err(Refusal::UnreleasedHardware), "a released GPU beside an unreleased one does not make the machine reportable" ); @@ -800,7 +848,7 @@ mod tests { #[test] fn hardware_that_could_not_be_identified_is_refused_rather_than_assumed() { assert_eq!( - prepare_report(&machine_of_sentinels(""), None, false), + prepare(&machine_of_sentinels(""), None, false), Err(Refusal::ArchitectureUnreadable), "nothing confirmed this hardware is on the compatibility matrix, and default-deny \ is the whole point" @@ -823,7 +871,7 @@ mod tests { wsl.is_wsl = true; assert_eq!( - prepare_report(&wsl, None, false), + prepare(&wsl, None, false), Err(Refusal::PlatformNotProbed), "a platform this CLI never inspects must say so, rather than report a finding \ about hardware nothing looked at" @@ -834,7 +882,7 @@ mod tests { let mut bare_metal = wsl; bare_metal.is_wsl = false; assert!( - prepare_report(&bare_metal, None, false).is_ok(), + prepare(&bare_metal, None, false).is_ok(), "premise failed: the same machine off WSL has an approved GPU and must report" ); } @@ -847,7 +895,7 @@ mod tests { /// an approved field. #[test] fn no_report_carries_a_value_the_machine_did_not_agree_to_publish() { - let report = prepare_report(&machine_of_sentinels("gfx1100"), Some("fix-6-path"), true) + let report = prepare(&machine_of_sentinels("gfx1100"), Some("fix-6-path"), true) .expect("a released machine must produce a report"); let serialized = serde_json::to_string(&report).expect("a report must serialize"); @@ -892,8 +940,8 @@ mod tests { machine.os_version = banner.to_owned(); machine.distro_version = "22.04".to_owned(); - let report = prepare_report(&machine, None, false) - .expect("a released machine must produce a report"); + let report = + prepare(&machine, None, false).expect("a released machine must produce a report"); assert_eq!( report.os_major, "22", "os_major must come from distro_version, not the kernel banner in os_version \ @@ -919,8 +967,8 @@ mod tests { machine.os_family = "windows".to_owned(); machine.os_version = "Microsoft Windows [Version 10.0.22631.4460]".to_owned(); - let report = prepare_report(&machine, None, false) - .expect("a released machine must produce a report"); + let report = + prepare(&machine, None, false).expect("a released machine must produce a report"); assert_eq!( report.os_major, "10", "the report groups by NT major version, so that is what it carries" @@ -987,7 +1035,7 @@ mod tests { #[test] fn every_axis_name_sits_where_its_field_lands_in_the_group_key() { type Change = (&'static str, fn(&mut Report)); - let base = prepare_report( + let base = prepare( &machine("gfx942", ("ubuntu", "22"), "7.1", ("pytorch", "2.5")), Some("fix-6-path"), false, @@ -1030,7 +1078,7 @@ mod tests { #[test] fn the_group_key_names_an_engine_with_its_release_and_an_absent_one_once() { - let with = prepare_report( + let with = prepare( &machine("gfx942", ("ubuntu", "22"), "7.1", ("pytorch", "2.5")), None, false, @@ -1046,7 +1094,7 @@ mod tests { } fn describe(machine: &Examination, entry: Option<&str>) -> String { - description(&prepare_report(machine, entry, false).expect("a released machine reports")) + description(&prepare(machine, entry, false).expect("a released machine reports")) } /// Reports are grouped by exact match on their fields, so the fields have @@ -1134,7 +1182,7 @@ mod tests { /// the numeric head survived while the build tail did not. #[test] fn the_grouping_fields_keep_the_release_and_drop_the_build() { - let report = prepare_report(&machine_of_sentinels("gfx1100"), None, false) + let report = prepare(&machine_of_sentinels("gfx1100"), None, false) .expect("a released machine must produce a report"); assert_eq!( @@ -1160,7 +1208,7 @@ mod tests { fn a_recognised_distribution_is_named_and_an_unrecognised_one_is_not() { let mut known = machine_of_sentinels("gfx1100"); known.distro_id = "Ubuntu".to_owned(); - let report = prepare_report(&known, None, false).expect("a released machine reports"); + let report = prepare(&known, None, false).expect("a released machine reports"); assert_eq!( report.distro, "ubuntu", "premise failed: a distribution on the list must be named, otherwise the case below \ @@ -1171,7 +1219,7 @@ mod tests { // rather than a distribution. let mut vendor = machine_of_sentinels("gfx1100"); vendor.distro_id = "SENTINEL-CORP-INTERNAL-IMAGE".to_owned(); - let report = prepare_report(&vendor, None, false).expect("a released machine reports"); + let report = prepare(&vendor, None, false).expect("a released machine reports"); assert_eq!(report.distro, DISTRO_OTHER); let serialized = serde_json::to_string(&report).expect("a report must serialize"); assert!( @@ -1190,7 +1238,7 @@ mod tests { fn a_missing_os_release_reads_as_unknown_not_as_an_unrecognised_distribution() { let mut machine = machine_of_sentinels("gfx1100"); machine.distro_id = String::new(); - let report = prepare_report(&machine, None, false).expect("a released machine reports"); + let report = prepare(&machine, None, false).expect("a released machine reports"); assert_eq!( report.distro, UNKNOWN, "an empty ID means the file could not be read, which is different from a file that \ @@ -1208,7 +1256,7 @@ mod tests { fn a_release_with_no_minor_or_an_unusual_build_separator_still_groups() { let mut bare_major = machine_of_sentinels("gfx1100"); bare_major.rocm_version = "7".to_owned(); - let report = prepare_report(&bare_major, None, false).expect("a released machine reports"); + let report = prepare(&bare_major, None, false).expect("a released machine reports"); assert_eq!( report.rocm, "7", "a release with no minor is still a population and must not be discarded" @@ -1216,8 +1264,7 @@ mod tests { let mut dash_delimited = machine_of_sentinels("gfx1100"); dash_delimited.rocm_version = "7-1-0".to_owned(); - let report = - prepare_report(&dash_delimited, None, false).expect("a released machine reports"); + let report = prepare(&dash_delimited, None, false).expect("a released machine reports"); assert_eq!( report.rocm, "7.1", "a dash-delimited build string must yield the same release as a dot-delimited one" @@ -1310,7 +1357,7 @@ mod tests { /// satisfy the test above on its own. #[test] fn recognising_refusals_did_not_stop_reports_being_read() { - let report = prepare_report(&machine_of_sentinels("gfx1100"), None, false) + let report = prepare(&machine_of_sentinels("gfx1100"), None, false) .expect("a released machine must produce a report"); let serialized = serde_json::to_string(&report).expect("a report must serialize"); @@ -1347,8 +1394,8 @@ mod tests { for (nth, machine) in machines.iter().enumerate() { for entry in [None, Some("fix-1-arch")] { - let report = prepare_report(machine, entry, entry.is_some()) - .expect("a released machine reports"); + let report = + prepare(machine, entry, entry.is_some()).expect("a released machine reports"); let serialized = serde_json::to_string(&report).expect("a report serializes"); assert!( matches!(read_report(&serialized), ReadOutcome::Understood(_)), @@ -1366,7 +1413,7 @@ mod tests { /// otherwise carry that text into every count built from it. #[test] fn a_report_with_a_value_outside_the_vocabulary_is_not_read() { - let genuine = prepare_report( + let genuine = prepare( &machine("gfx942", ("ubuntu", "22"), "7.1", ("pytorch", "2.5")), Some("fix-1-arch"), true, @@ -1414,7 +1461,7 @@ mod tests { fn an_engine_this_build_does_not_recognise_is_not_named_in_the_report() { let mut machine = machine_of_sentinels("gfx1100"); machine.framework = "SENTINEL-UNAPPROVED-ENGINE".to_owned(); - let report = prepare_report(&machine, None, false).expect("a released machine reports"); + let report = prepare(&machine, None, false).expect("a released machine reports"); assert_eq!( report.engine, UNKNOWN, @@ -1444,7 +1491,7 @@ mod tests { fn no_engine_found_by_a_completed_probe_reads_as_none_not_unknown() { let mut machine = machine_of_sentinels("gfx1100"); machine.framework = "unknown".to_owned(); - let report = prepare_report(&machine, None, false).expect("a released machine reports"); + let report = prepare(&machine, None, false).expect("a released machine reports"); assert_eq!( report.engine, NONE, @@ -1467,7 +1514,7 @@ mod tests { fn a_skipped_probe_reads_as_unreadable_not_as_no_engine_installed() { let mut machine = machine_of_sentinels("gfx1100"); machine.framework = "skipped".to_owned(); - let report = prepare_report(&machine, None, false).expect("a released machine reports"); + let report = prepare(&machine, None, false).expect("a released machine reports"); assert_eq!( report.engine, UNKNOWN, @@ -1491,12 +1538,11 @@ mod tests { let mut absent = machine_of_sentinels("gfx1100"); absent.rocm_path = String::new(); absent.rocm_version = String::new(); - let absent = prepare_report(&absent, None, false).expect("a released machine reports"); + let absent = prepare(&absent, None, false).expect("a released machine reports"); let mut unreadable = machine_of_sentinels("gfx1100"); unreadable.rocm_version = String::new(); - let unreadable = - prepare_report(&unreadable, None, false).expect("a released machine reports"); + let unreadable = prepare(&unreadable, None, false).expect("a released machine reports"); assert_eq!(absent.rocm, NONE); assert_eq!(unreadable.rocm, UNKNOWN); @@ -1529,7 +1575,7 @@ mod tests { windows.rocm_path = String::new(); windows.rocm_version = String::new(); - let report = prepare_report(&windows, None, false).expect("a released machine reports"); + let report = prepare(&windows, None, false).expect("a released machine reports"); assert_eq!( report.rocm, "6.2", "a Windows host with an installed SDK must report its release, not an absence" @@ -1540,15 +1586,14 @@ mod tests { let mut bare = windows.clone(); bare.hip_sdk_path = String::new(); bare.hip_sdk_version = String::new(); - let bare = prepare_report(&bare, None, false).expect("a released machine reports"); + let bare = prepare(&bare, None, false).expect("a released machine reports"); assert_eq!(bare.rocm, NONE); // And an install whose version cannot be read is still distinguishable // from one that is not there. let mut unreadable = windows; unreadable.hip_sdk_version = String::new(); - let unreadable = - prepare_report(&unreadable, None, false).expect("a released machine reports"); + let unreadable = prepare(&unreadable, None, false).expect("a released machine reports"); assert_eq!(unreadable.rocm, UNKNOWN); } @@ -1561,8 +1606,8 @@ mod tests { let mut machine = machine_of_sentinels("gfx1100"); machine.distro_version = "SENTINEL-UNPARSEABLE-RELEASE".to_owned(); - let report = prepare_report(&machine, None, false) - .expect("a released machine must produce a report"); + let report = + prepare(&machine, None, false).expect("a released machine must produce a report"); assert_eq!( report.os_major, "", "a release string that does not reduce to a bare number must not pass through as \ @@ -1587,7 +1632,7 @@ mod tests { fn an_entry_id_the_catalog_does_not_know_is_never_published_verbatim() { // Non-vacuity: a real id has to reach the report, or "the forged one // does not" is satisfied by discarding every id. - let known = prepare_report(&machine_of_sentinels("gfx1100"), Some("fix-6-path"), true) + let known = prepare(&machine_of_sentinels("gfx1100"), Some("fix-6-path"), true) .expect("a released machine must produce a report"); assert_eq!( known.entry, "fix-6-path", @@ -1601,7 +1646,7 @@ mod tests { unconditionally" ); - let forged = prepare_report( + let forged = prepare( &machine_of_sentinels("gfx1100"), Some("SENTINEL-FORGED-ENTRY"), true, @@ -1636,7 +1681,7 @@ mod tests { ); let current = serde_json::to_string( - &prepare_report(&machine_of_sentinels("gfx1100"), Some("fix-6-path"), true) + &prepare(&machine_of_sentinels("gfx1100"), Some("fix-6-path"), true) .expect("a released machine must produce a report"), ) .expect("a report must serialize"); @@ -1646,4 +1691,43 @@ mod tests { assertion above is satisfied by a reader that understands nothing" ); } + + /// The report carries the version the caller passes, not this crate's own + /// package version, which is the frozen workspace one for every build. + #[test] + fn the_report_carries_the_cli_version_its_caller_passes() { + for version in ["0.2.1", "0.2.1-rc.1", "0.2.1+build.1", "0.0.0-dev"] { + let report = prepare_report(&machine_of_sentinels("gfx1100"), None, false, version) + .expect("a released machine must produce a report"); + assert_eq!(report.cli_version, version); + } + } + + /// The caller passes the leading version token. Anything else, such as the + /// whole `rocm version` string with its local branch name, is free text + /// from the machine and is published as `unknown` rather than repeated. + #[test] + fn a_cli_version_that_is_not_a_version_token_is_published_as_unknown() { + // One character over the limit, so a limit off by one is caught. + let too_long = format!("1.{}", "0".repeat(CLI_VERSION_MAX_LEN - 1)); + assert_eq!(too_long.len(), CLI_VERSION_MAX_LEN + 1); + for candidate in [ + "0.0.0-dev (SENTINEL-BRANCH, abcdef1)", + "SENTINEL-USER", + "v0.2.1", + "", + "0.2.1/../x", + too_long.as_str(), + ] { + let report = prepare_report(&machine_of_sentinels("gfx1100"), None, false, candidate) + .expect("a released machine must produce a report"); + assert_eq!(report.cli_version, UNKNOWN, "{candidate:?}"); + } + let longest = format!("1.{}", "0".repeat(CLI_VERSION_MAX_LEN - 2)); + assert_eq!( + cli_version(&longest), + longest, + "the limit itself is a version" + ); + } } diff --git a/crates/rocm-core/src/report_delivery.rs b/crates/rocm-core/src/report_delivery.rs index c835fd3f34..8376061b2f 100644 --- a/crates/rocm-core/src/report_delivery.rs +++ b/crates/rocm-core/src/report_delivery.rs @@ -171,13 +171,15 @@ fn choose(url: String, sending: bool, env: &dyn Fn(&str) -> Option) -> D } } -/// Percent-encode for a query-string value. +/// Percent-encode for a query-string value or one URL path segment. /// /// Written out rather than taken from a crate: this is the only encoding this /// binary needs, and a signed artifact is not worth a dependency for fifteen /// lines. Everything outside the unreserved set of RFC 3986 is escaped, which -/// is stricter than necessary and wrong in no case. -fn percent_encode(value: &str) -> String { +/// is stricter than necessary and wrong in no case; that also makes it right +/// for a path segment, where `/` must be escaped too. The GGUF header reader +/// sends hub file names through it, one segment at a time. +pub(crate) fn percent_encode(value: &str) -> String { const HEX: &[u8; 16] = b"0123456789ABCDEF"; let mut out = String::with_capacity(value.len()); for byte in value.bytes() { @@ -225,7 +227,7 @@ mod tests { } fn report_of(entry: Option<&str>) -> Report { - prepare_report(&reportable_machine(), entry, false) + prepare_report(&reportable_machine(), entry, false, "0.0.0-dev") .expect("a released machine must produce a report") } diff --git a/crates/rocm-core/src/runtime.rs b/crates/rocm-core/src/runtime.rs index 3535035b07..6d3e8fbb39 100644 --- a/crates/rocm-core/src/runtime.rs +++ b/crates/rocm-core/src/runtime.rs @@ -119,6 +119,31 @@ pub fn runtime_python_executable_in_env(env_root: &Path) -> PathBuf { runtime_python_env_bin_dir(env_root).join(runtime_python_executable_name()) } +/// The environment folder an interpreter belongs to, found from the +/// interpreter's own path: the inverse of [`runtime_python_executable_in_env`] +/// for a caller that holds only the interpreter. +/// +/// An interpreter belongs to an environment only when it sits directly in that +/// environment's `bin` or `Scripts` folder, and the answer is that folder's +/// parent. Both layouts are accepted, in any letter case, whatever the host: a +/// recorded interpreter path is not always from this host's layout, and a +/// Windows file system does not tell `Scripts` from `SCRIPTS`. +/// +/// Anything else gives `None`: an interpreter in some other folder, a bare file +/// name, or one at the file system root. A guess would name a folder that is +/// not an environment, and the caller would then export it as `VIRTUAL_ENV`, +/// look for `lib/python3.*` under it, or register it as a runtime. `None` lets +/// each caller refuse in its own words instead. +pub fn runtime_python_env_root_for_executable(python: &Path) -> Option { + let bin_dir = python.parent()?; + let name = bin_dir.file_name()?.to_str()?; + if name.eq_ignore_ascii_case("bin") || name.eq_ignore_ascii_case("Scripts") { + bin_dir.parent().map(Path::to_path_buf) + } else { + None + } +} + pub fn runtime_python_activation_script(env_root: &Path) -> PathBuf { let script = if runtime_is_windows() { "activate.bat" @@ -695,6 +720,35 @@ pub fn runtime_install_root_is_protected(path: &Path) -> bool { .any(|root| runtime_path_is_same_or_inside(&path, root)) } +/// Whether an install root holds anything the install put there. +/// +/// Dot-entries do not count: the runtime's own `.rocm-cli-runtime.json` marker is +/// one, and a root that holds only that has no payload. Any other entry does, +/// whatever it is: a file, a directory, or a symlink that points nowhere. A path +/// that does not exist holds nothing, so it answers `false` rather than failing; +/// that is what lets a fresh install ask about a root it has not created yet. A +/// root that exists but cannot be listed (not a directory, no permission) is an +/// error, never a quiet `false`. +/// +/// This is the one rule for "is this install root empty" in the workspace: the +/// tarball extraction refuses a target that has a payload, a repair counts a +/// sibling as installed only when it has one, and activation validation rejects a +/// tarball runtime that has none. +pub fn install_root_has_payload(path: &Path) -> Result { + if !path.exists() { + return Ok(false); + } + let entries = + std::fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))?; + for entry in entries { + let entry = entry?; + if !entry.file_name().to_string_lossy().starts_with('.') { + return Ok(true); + } + } + Ok(false) +} + #[cfg_attr(not(test), allow(dead_code))] #[derive(Debug, Clone, Copy, Eq, PartialEq)] enum RuntimePathSeparator { @@ -1026,6 +1080,57 @@ mod tests { use super::*; use std::fs; + /// The one rule every caller shares. Paths are built with `join`, not + /// written with a separator, so the table means the same on every host. + #[test] + fn env_root_for_executable_needs_a_bin_or_scripts_folder() { + let venv = Path::new("opt").join("venv"); + for (folder, python) in [ + ("bin", "python"), + ("BIN", "python"), + ("Scripts", "python.exe"), + ("SCRIPTS", "python.exe"), + ("scripts", "python.exe"), + ] { + assert_eq!( + runtime_python_env_root_for_executable(&venv.join(folder).join(python)), + Some(venv.clone()), + "{folder}" + ); + } + for folder in ["tools", "bin2", "my-bin", "venv"] { + assert_eq!( + runtime_python_env_root_for_executable(&venv.join(folder).join("python")), + None, + "{folder}" + ); + } + // A bare file name has an empty parent, and a root-level file's parent + // is the file system root: neither is inside a `bin` or `Scripts`. + assert_eq!( + runtime_python_env_root_for_executable(Path::new("python")), + None + ); + assert_eq!( + runtime_python_env_root_for_executable( + &Path::new(std::path::MAIN_SEPARATOR_STR).join("python") + ), + None + ); + } + + /// The inverse really is the inverse: the interpreter this host's layout + /// puts in an environment leads back to that environment. + #[test] + fn env_root_for_executable_inverts_the_hosts_layout() { + let env_root = Path::new("opt").join("venv"); + let python = runtime_python_executable_in_env(&env_root); + assert_eq!( + runtime_python_env_root_for_executable(&python).as_deref(), + Some(normalize_runtime_path_for_host(&env_root).as_path()) + ); + } + fn scratch_dir(name: &str) -> PathBuf { let root = std::env::temp_dir().join(format!( "rocm-core-resolve-{name}-{}-{:?}", @@ -1819,3 +1924,135 @@ mod tests { } } } + +#[cfg(test)] +mod install_root_payload_tests { + use super::*; + use std::fs; + + /// One path per kind of install root. An unreadable directory is only built + /// when this process cannot read it anyway (a root user can, and then the case + /// says nothing). + fn cases(base: &Path) -> Vec<(&'static str, PathBuf)> { + let mut cases = vec![("missing path", base.join("missing"))]; + let dir = |name: &str| { + let path = base.join(name); + fs::create_dir_all(&path).expect("create case dir"); + path + }; + cases.push(("empty dir", dir("empty"))); + let dotfile = dir("dotfile"); + fs::write(dotfile.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); + cases.push(("dotfile only", dotfile)); + let dotdir = dir("dotdir"); + fs::create_dir_all(dotdir.join(".hidden")).expect("create dot dir"); + cases.push(("dot directory only", dotdir)); + let file = dir("file"); + fs::write(file.join("payload.bin"), b"x").expect("write payload"); + cases.push(("regular file", file)); + let sub = dir("sub"); + fs::create_dir_all(sub.join("bin")).expect("create subdir"); + cases.push(("subdirectory", sub)); + let marker_and_file = dir("marker-and-file"); + fs::write(marker_and_file.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); + fs::write(marker_and_file.join("payload.bin"), b"x").expect("write payload"); + cases.push(("marker plus a regular file", marker_and_file)); + let not_a_dir = base.join("not-a-dir"); + fs::write(¬_a_dir, b"x").expect("write file root"); + cases.push(("root is a regular file", not_a_dir)); + #[cfg(unix)] + { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dangling = dir("dangling-entry"); + symlink(base.join("nowhere"), dangling.join("link")).expect("dangling entry"); + cases.push(("dangling symlink entry", dangling)); + let link_root = base.join("link-root"); + symlink(base.join("sub"), &link_root).expect("symlink root"); + cases.push(("root is a symlink to a populated dir", link_root)); + let dangling_root = base.join("dangling-root"); + symlink(base.join("nowhere"), &dangling_root).expect("dangling root"); + cases.push(("root is a dangling symlink", dangling_root)); + let locked = dir("locked"); + fs::write(locked.join("payload.bin"), b"x").expect("write payload"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock dir"); + if fs::read_dir(&locked).is_err() { + cases.push(("unreadable dir", locked)); + } else { + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok(); + } + } + cases + } + + /// The single definition of a non-empty install root, over every input kind. + /// Each caller in `apps/rocm` goes through this function (see + /// `tarball_validation_follows_the_shared_payload_rule`, + /// `the_tarball_install_guard_follows_the_shared_payload_rule` and + /// `a_replacement_counts_as_installed_by_the_shared_payload_rule` there), so a + /// row changing here changes all three together. + #[test] + fn install_root_has_payload_answers_each_input_kind() { + let base = tempfile::tempdir().expect("scratch dir"); + let mut seen: Vec<(&str, String)> = cases(base.path()) + .into_iter() + .map(|(label, path)| { + let outcome = match install_root_has_payload(&path) { + Ok(true) => "payload", + Ok(false) => "empty", + Err(_) => "error", + }; + (label, outcome.to_owned()) + }) + .collect(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + base.path().join("locked"), + fs::Permissions::from_mode(0o755), + ) + .ok(); + } + let mut expected: Vec<(&str, String)> = vec![ + ("missing path", "empty"), + ("empty dir", "empty"), + ("dotfile only", "empty"), + ("dot directory only", "empty"), + ("regular file", "payload"), + ("subdirectory", "payload"), + ("marker plus a regular file", "payload"), + ("root is a regular file", "error"), + ] + .into_iter() + .map(|(label, outcome)| (label, outcome.to_owned())) + .collect(); + #[cfg(unix)] + { + expected.extend([ + ("dangling symlink entry", "payload".to_owned()), + ("root is a symlink to a populated dir", "payload".to_owned()), + ("root is a dangling symlink", "empty".to_owned()), + ]); + // Absent from `seen` when running as root, which reads a mode-000 + // directory anyway; then the case is neither built nor asserted. + if seen.iter().any(|(label, _)| *label == "unreadable dir") { + expected.push(("unreadable dir", "error".to_owned())); + } + } + seen.sort(); + expected.sort(); + assert_eq!(seen, expected); + } + + #[test] + fn a_root_that_cannot_be_listed_names_the_path_in_its_error() { + let base = tempfile::tempdir().expect("scratch dir"); + let file = base.path().join("not-a-dir"); + fs::write(&file, b"x").expect("write file root"); + let error = install_root_has_payload(&file).expect_err("a file cannot be listed"); + assert!( + format!("{error:#}").contains(&format!("failed to read {}", file.display())), + "{error:#}" + ); + } +} diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index aecab92d8c..7d1ec24571 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -21,9 +21,9 @@ use std::fs; use std::path::PathBuf; use std::time::Duration; -use crate::AppPaths; use crate::format_http_base_url; use crate::unix_time_millis; +use crate::{ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AppPaths}; use crate::{normalize_runtime_path_for_host, normalize_runtime_path_for_storage}; /// Engine state-file key recording when inference was first confirmed. @@ -109,6 +109,33 @@ pub struct ManagedServiceRecord { /// existed, which is the safe default (keep the key). #[serde(default)] pub stop_requested_unix_ms: Option, + /// The stop marker a liveness refresh cleared off a record that named no + /// process, kept as the evidence that it was cleared and not by a relaunch. + /// + /// A record naming no process has nothing for a pending stop to wait on, so + /// the refresh drops the marker (and the key) at once. A `starting` or + /// `failed` record is left at that status, and a stop that began on such a + /// record would then find it with no marker and nothing recorded, which is + /// also what a relaunch that has not recorded its supervisor yet looks like. + /// This holds the marker's value (the stop's own request stamp), so the stop + /// can tell its own marker having been consumed from a relaunch having + /// cleared it ([`Self::is_covered_by_stop_of`]). Cleared by a restart + /// ([`Self::reset_for_restart`]); absent from records that never had one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub settled_stop_unix_ms: Option, + /// The generation of a relaunch that has begun but not yet published its + /// processes: stamped by `rocm services restart` once its own stop has + /// confirmed and before it restores the endpoint key or spawns, and cleared + /// when the restart publishes ([`Self::reset_for_restart`]). + /// + /// Until then the record is an empty `stopped` manifest, which is also what a + /// stop's own completion looks like, so a stop that read the record earlier + /// would claim it and delete the key the restart restored for its child. A + /// stop covers the record only while the record carries the generation the + /// stop read ([`Self::is_covered_by_stop_of`]). Absent from records that are + /// not between a restart's stop and its publication. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relaunch_started_unix_ms: Option, /// When the last inference probe was attempted. Throttles re-probing of a /// service that is listed but still loading — see /// [`crate::INFERENCE_PROBE_RETRY_INTERVAL`]. Absent on records written before @@ -127,6 +154,15 @@ pub struct ManagedServiceRecord { /// readiness was gated on inference. #[serde(default)] pub inference_verified_at_unix_ms: Option, + /// Base URL of the Anthropic-compatible `/v1/messages` endpoint, once the + /// serving process has published one. Adopted from the engine state file (see + /// [`crate::anthropic_endpoint`]); `None` while the service is still coming up + /// and on records written before the endpoint existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub anthropic_endpoint_url: Option, + /// How [`Self::anthropic_endpoint_url`] is provided: `proxy`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub anthropic_endpoint_mode: Option, pub manifest_path: PathBuf, pub log_path: PathBuf, pub engine_state_path: PathBuf, @@ -245,9 +281,13 @@ impl ManagedServiceRecord { restart_count: 0, last_restart_unix_ms: None, stop_requested_unix_ms: None, + settled_stop_unix_ms: None, + relaunch_started_unix_ms: None, startup_phase: None, inference_verified_at_unix_ms: None, inference_probe_attempted_at_unix_ms: None, + anthropic_endpoint_url: None, + anthropic_endpoint_mode: None, manifest_path, log_path, engine_state_path, @@ -283,6 +323,14 @@ impl ManagedServiceRecord { pub fn reset_for_restart(&mut self) { self.inference_verified_at_unix_ms = None; self.inference_probe_attempted_at_unix_ms = None; + // The respawned process publishes its own endpoint; the old proxy port + // died with the previous run. + self.anthropic_endpoint_url = None; + self.anthropic_endpoint_mode = None; + // A relaunch is no longer the stop that was settled under it, and it is + // published now, so it is no longer waiting to be. + self.settled_stop_unix_ms = None; + self.relaunch_started_unix_ms = None; self.restart_count = self.restart_count.saturating_add(1); self.last_restart_unix_ms = Some(unix_time_millis()); } @@ -315,6 +363,74 @@ impl ManagedServiceRecord { && self.engine_start_ticks == other.engine_start_ticks } + /// Whether `self`, the record as it is now, is still the record of the + /// service that `stopping`, the copy the stop read before it signalled + /// anything, was stopping: the stop's verdict covers what it names. + /// + /// True when the two name the same processes + /// ([`Self::names_same_processes`]), and also when `self` names none because + /// the same stop was completed under it: a liveness refresh that saw every + /// recorded process gone, with the stop requested, clears the processes and + /// reads the record as `stopped` (the deferred cleanup). A record reading + /// `stopped` that names no process has nothing left that the stop did not + /// handle, so the stop's own confirmation is still true of it. + /// + /// False for a restart that recorded new processes, and equally for one that + /// is mid-launch with none recorded yet: that record reads `starting` (or + /// another live status) with the stop marker cleared, not `stopped`, and + /// claiming it stopped, or dropping the key it will serve with, would be + /// false. That holds when the stopping snapshot itself names no process too: + /// an empty record is covered only while it still carries the stop's marker, + /// once it reads `stopped`, or once a liveness refresh consumed that very + /// marker ([`Self::settled_stop_unix_ms`]), which leaves a `starting` or + /// `failed` record at its status with the marker gone. + /// + /// Never true for a record in another relaunch generation than the stop read + /// ([`Self::relaunch_started_unix_ms`]): a restart that has stopped the + /// service and not yet published its child leaves an empty `stopped` record + /// that an older stop would otherwise take for its own completion. + #[must_use] + pub fn is_covered_by_stop_of(&self, stopping: &Self) -> bool { + if self.relaunch_started_unix_ms != stopping.relaunch_started_unix_ms { + return false; + } + if self.status == "stopped" && self.names_no_process() { + return true; + } + // The same processes. When they are none, that equality is also what a + // relaunch that has not recorded its supervisor yet looks like, so it + // proves nothing: the stop's own marker is what tells the two apart, since + // a restart clears it and a stop that is still finalizing has it on disk. + // A marker that is gone is the stop's own only when the refresh that + // consumed it left its stamp behind; a restart wipes that stamp. + self.names_same_processes(stopping) + && (!stopping.names_no_process() + || self.stop_requested_unix_ms.is_some() + || (stopping.stop_requested_unix_ms.is_some() + && self.settled_stop_unix_ms == stopping.stop_requested_unix_ms)) + } + + /// Stamp the record as a relaunch that has begun and not yet published its + /// processes, as a generation no earlier stop or relaunch carried. See + /// [`Self::relaunch_started_unix_ms`]. + pub fn begin_relaunch(&mut self) { + let now = unix_time_millis(); + self.relaunch_started_unix_ms = Some( + now.max( + self.relaunch_started_unix_ms + .map_or(0, |earlier| earlier + 1), + ), + ); + } + + /// Whether the record names neither a supervisor nor an engine process. + const fn names_no_process(&self) -> bool { + self.supervisor_pid == 0 + && self.supervisor_start_ticks.is_none() + && self.engine_pid.is_none() + && self.engine_start_ticks.is_none() + } + /// Record `pid` as the engine, together with its start-time token. See /// [`Self::record_supervisor_identity`]. pub fn record_engine_identity(&mut self, pid: u32) { @@ -322,6 +438,25 @@ impl ManagedServiceRecord { self.engine_start_ticks = crate::process_start_ticks(pid); } + /// Forget the processes this record names: the supervisor and the engine, + /// each with its start-time token. + /// + /// For a stop that has confirmed every one of them gone, and for nothing + /// else. A recorded PID goes stale the instant its process exits, and a + /// later stop of the same record would signal whatever the OS has since + /// handed that number to; a PID recorded without a token cannot be + /// refuted, and on Windows none can, so the tree kill would take that + /// process and everything under it. `rocm services stop` and `rocmd`'s stop + /// both call this on the confirmed path, so the two leave the same record + /// behind. An unconfirmed stop keeps the PIDs: the record is the only thing + /// that names a survivor. + pub const fn clear_recorded_processes(&mut self) { + self.supervisor_pid = 0; + self.supervisor_start_ticks = None; + self.engine_pid = None; + self.engine_start_ticks = None; + } + pub fn normalize_paths_for_host(&mut self) { self.manifest_path = normalize_runtime_path_for_host(&self.manifest_path); self.log_path = normalize_runtime_path_for_host(&self.log_path); @@ -433,6 +568,19 @@ impl ManagedServiceRecord { { self.inference_verified_at_unix_ms = Some(verified_at); } + // The endpoint belongs to the run that published it, so it is replaced + // rather than kept: a restarted proxy listens on a different port. + let published = |key: &str| { + state + .get(key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .map(str::to_owned) + }; + if let Some(url) = published(ANTHROPIC_ENDPOINT_URL_STATE_KEY) { + self.anthropic_endpoint_url = Some(url); + self.anthropic_endpoint_mode = published(ANTHROPIC_ENDPOINT_MODE_STATE_KEY); + } Ok(self.status != previous) } @@ -502,10 +650,11 @@ impl ManagedServiceRecord { let storage_record = host_record.with_storage_paths(); let bytes = serde_json::to_vec_pretty(&storage_record) .context("failed to serialize service record")?; - crate::write_atomic(&host_record.manifest_path, &bytes, "json").with_context(|| { + let manifest_path = &host_record.manifest_path; + crate::atomic_write::write_file_atomically(manifest_path, &bytes).with_context(|| { format!( "failed to write service manifest {}", - host_record.manifest_path.display() + manifest_path.display() ) }) } @@ -514,7 +663,8 @@ impl ManagedServiceRecord { #[cfg(test)] mod tests { use super::*; - use crate::test_support::{probe_test_record, temp_app_paths}; + use crate::foreign_temp_probe::assert_write_spares_foreign_temp_files; + use crate::test_support::{probe_test_record, retired_temp_path, temp_app_paths}; /// An unknown id reads as one plain sentence pointing at the listing /// command, with no OS error chained under it. @@ -645,6 +795,33 @@ mod tests { record } + /// The serving process publishes the Anthropic endpoint in the engine state; + /// the record adopts it, and a restart drops it so the next run's port wins. + #[test] + fn engine_refresh_adopts_the_published_anthropic_endpoint() { + let record = refresh_from_state( + "anthropic", + &serde_json::json!({ + "status": "ready", + "anthropic_endpoint_url": "http://127.0.0.1:40123", + "anthropic_endpoint_mode": "proxy", + }), + ); + assert_eq!( + record.anthropic_endpoint_url.as_deref(), + Some("http://127.0.0.1:40123") + ); + assert_eq!(record.anthropic_endpoint_mode.as_deref(), Some("proxy")); + + let mut record = record; + record.reset_for_restart(); + assert_eq!(record.anthropic_endpoint_url, None); + assert_eq!(record.anthropic_endpoint_mode, None); + + let record = refresh_from_state("no-anthropic", &serde_json::json!({ "status": "ready" })); + assert_eq!(record.anthropic_endpoint_url, None); + } + /// A container engine serves under a different name than the image it pulls; /// the record adopts that name from the engine state, and every readiness /// probe asks for it rather than for the image reference. @@ -721,7 +898,8 @@ mod tests { /// been recycled. One live process stands in for both roles, as it does for /// a `rocm serve --background` launch. /// - /// Linux-only: `process_start_ticks` reads `/proc`, and is `None` elsewhere. + /// Linux-only here, because it spawns `sleep`: `process_start_ticks` gives a + /// token on Linux and on Windows (the creation time) and is `None` elsewhere. #[cfg(target_os = "linux")] #[test] fn recording_a_process_identity_writes_the_pid_with_its_own_token() { @@ -745,6 +923,269 @@ mod tests { assert_eq!(record.engine_start_ticks, expected); } + /// What a stop that confirmed its processes gone leaves behind: no PID and + /// no token for either role, and nothing else touched. + #[test] + fn clearing_the_recorded_processes_clears_each_pid_and_its_token_and_nothing_else() { + let mut record = probe_test_record(11437); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.status = "ready".to_owned(); + record.restart_count = 3; + let before = record.clone(); + + record.clear_recorded_processes(); + + assert_eq!(record.supervisor_pid, 0); + assert_eq!(record.supervisor_start_ticks, None); + assert_eq!(record.engine_pid, None); + assert_eq!(record.engine_start_ticks, None); + assert_eq!(record.service_id, before.service_id); + assert_eq!(record.status, "ready"); + assert_eq!(record.restart_count, 3); + assert_eq!(record.port, before.port); + // A cleared record no longer names what it named, so a stop that read it + // before and finds this after does not claim the same processes. + assert!(!before.names_same_processes(&record)); + assert!(record.names_same_processes(&record.clone())); + } + + /// What a stop's verdict still covers, record by record: the same processes, + /// or none because the same stop was completed under it (`stopped`, nothing + /// recorded). A restart's record is never covered: new processes recorded, or + /// still starting with none recorded yet. + #[test] + fn a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch() { + let mut stopping = probe_test_record(11438); + stopping.supervisor_pid = 4242; + stopping.supervisor_start_ticks = None; + stopping.engine_pid = Some(4243); + stopping.engine_start_ticks = Some(77); + stopping.status = "ready".to_owned(); + let record_with = |status: &str, pid: u32, engine: Option| { + let mut record = stopping.clone(); + status.clone_into(&mut record.status); + record.supervisor_pid = pid; + record.supervisor_start_ticks = None; + record.engine_pid = engine; + record.engine_start_ticks = None; + record + }; + // (what the record on disk is now, whether the stop still covers it) + let rows = [ + ("unchanged", stopping.clone(), true), + // The deferred cleanup of this very stop: every process gone, so the + // record is stopped and names none. + ( + "completed under the stop", + record_with("stopped", 0, None), + true, + ), + // A restart that recorded new processes. + ( + "relaunched, ready", + record_with("ready", 5555, Some(5556)), + false, + ), + ( + "relaunched, starting", + record_with("starting", 5555, None), + false, + ), + // A restart mid-launch that has not recorded anything yet: not a stop. + ( + "restarting, nothing recorded", + record_with("starting", 0, None), + false, + ), + ( + "recovering, nothing recorded", + record_with("recovering", 0, None), + false, + ), + ( + "ready, nothing recorded", + record_with("ready", 0, None), + false, + ), + // Stopped, but naming a process the stop did not handle. + ( + "stopped, other process", + record_with("stopped", 5555, None), + false, + ), + ]; + for (name, record, covered) in rows { + assert_eq!(record.is_covered_by_stop_of(&stopping), covered, "{name}"); + } + + // The same table from a stopping snapshot that names no process: an + // untouched record (the stop's own marker still on it, or completed to + // `stopped`) is that stop; an empty relaunch that has cleared the marker is + // not, and neither is one that has recorded processes. + let mut empty = stopping.clone(); + empty.clear_recorded_processes(); + empty.stop_requested_unix_ms = Some(1); + let with_marker = |status: &str, marker: Option, pid: u32| { + let mut record = empty.clone(); + status.clone_into(&mut record.status); + record.stop_requested_unix_ms = marker; + record.supervisor_pid = pid; + record + }; + // What a liveness refresh leaves when it consumes the stop's marker off an + // empty record: the marker gone, the status as it was, the marker's value + // kept as `settled_stop_unix_ms`. + let settled = |status: &str, stamp: Option| { + let mut record = with_marker(status, None, 0); + record.settled_stop_unix_ms = stamp; + record + }; + let rows = [ + ("empty, untouched", with_marker("ready", Some(1), 0), true), + ("empty, stopped", with_marker("stopped", None, 0), true), + // The stop's own marker consumed by a listing: not a relaunch. + ( + "empty, starting, marker consumed by a listing", + settled("starting", Some(1)), + true, + ), + ( + "empty, failed, marker consumed by a listing", + settled("failed", Some(1)), + true, + ), + // A stamp that is not this stop's (an earlier stop's), or none at all, + // is no evidence about this one. + ( + "empty, starting, another stop's marker consumed", + settled("starting", Some(2)), + false, + ), + ( + "empty, starting relaunch", + with_marker("starting", None, 0), + false, + ), + ( + "empty, recovering relaunch", + with_marker("recovering", None, 0), + false, + ), + ( + "empty, ready relaunch", + with_marker("ready", None, 0), + false, + ), + ( + "relaunched with a process", + with_marker("starting", None, 5555), + false, + ), + ]; + for (name, record, covered) in rows { + assert_eq!(record.is_covered_by_stop_of(&empty), covered, "{name}"); + } + + // Nothing is learned from two absent stamps: a snapshot without a marker + // of its own is not matched by a record that never consumed one. + let mut unmarked = empty.clone(); + unmarked.stop_requested_unix_ms = None; + assert!( + !settled("starting", None).is_covered_by_stop_of(&unmarked), + "None must not equal None as evidence" + ); + } + + /// A restart between its own confirmed stop and the publication of its child + /// leaves an empty `stopped` record. A stop that read the record before that + /// (another generation) must not cover it, however the rest of the record + /// reads, while a stop that read the same generation still does. + #[test] + fn a_stop_does_not_cover_a_record_in_another_relaunch_generation() { + let mut stopping = probe_test_record(11440); + stopping.supervisor_pid = 4242; + stopping.status = "ready".to_owned(); + let mut relaunching = stopping.clone(); + relaunching.clear_recorded_processes(); + "stopped".clone_into(&mut relaunching.status); + assert!( + relaunching.is_covered_by_stop_of(&stopping), + "premise: without a relaunch the completed stop covers it" + ); + relaunching.begin_relaunch(); + assert!( + !relaunching.is_covered_by_stop_of(&stopping), + "an older stop does not cover a record whose relaunch has begun" + ); + // A stop that read the record during the relaunch, and an empty snapshot + // of the same generation, are the same stop as before. + let mut during = relaunching.clone(); + during.stop_requested_unix_ms = Some(9); + assert!(relaunching.is_covered_by_stop_of(&during)); + // The generation is never reused, even within one millisecond. + let first = relaunching.relaunch_started_unix_ms; + relaunching.begin_relaunch(); + assert!(relaunching.relaunch_started_unix_ms > first); + } + + /// A restart supersedes the stop that was settled under it: the stamp a + /// listing left behind does not outlive the relaunch, or a stop that began + /// before it would read the relaunch's empty record as its own completion. + #[test] + fn restarting_forgets_a_settled_stop() { + let mut record = probe_test_record(11439); + record.settled_stop_unix_ms = Some(7); + record.begin_relaunch(); + record.reset_for_restart(); + assert_eq!(record.settled_stop_unix_ms, None); + assert_eq!( + record.relaunch_started_unix_ms, None, + "publishing ends the relaunch generation" + ); + } + + /// Writing a service manifest never writes through a file another writer + /// already staged next to it. The occupied names are the ones the retired + /// clock-and-pid temp naming would have picked, so a write that still used + /// one would truncate a foreign file and rename it into place. Reading the + /// manifest back would pass for either writer; the foreign files are what + /// tell them apart. + #[test] + fn writing_a_service_manifest_never_reuses_another_writers_temp_file() -> Result<()> { + let (root, paths) = temp_app_paths("record-write-foreign-temp"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-foreign-temp", + "vllm", + "Qwen/Qwen3-0.6B", + "Qwen/Qwen3-0.6B", + "127.0.0.1", + 9, + "managed", + std::process::id(), + None, + None, + Some("gpu_required".to_owned()), + ); + record.write()?; + record.startup_phase = Some("loading".to_owned()); + let manifest_path = record.manifest_path.clone(); + + assert_write_spares_foreign_temp_files( + |millis| retired_temp_path(&manifest_path, millis), + || record.write(), + )?; + + let published: ManagedServiceRecord = serde_json::from_slice(&fs::read(&manifest_path)?)?; + let _ = fs::remove_dir_all(&root); + assert_eq!(published.service_id, "svc-foreign-temp"); + assert_eq!(published.startup_phase.as_deref(), Some("loading")); + Ok(()) + } + #[test] fn engine_state_refresh_keeps_the_runtime_selector_the_launch_requested() -> Result<()> { let (root, paths) = temp_app_paths("record-runtime-pin"); diff --git a/crates/rocm-core/src/sigpipe.rs b/crates/rocm-core/src/sigpipe.rs new file mode 100644 index 0000000000..2a28b0a223 --- /dev/null +++ b/crates/rocm-core/src/sigpipe.rs @@ -0,0 +1,258 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Writing to a pipe whose reader has gone away, without being killed for it. +//! +//! Rust starts every process with SIGPIPE ignored, so a write to a closed pipe +//! returns `EPIPE`. `rocm` restores the default action at startup so it ends +//! quietly when its own output is piped into `head`, and `rocm daemon` +//! hosts the daemon in that same process. Under the default action a write to +//! a child that has already exited kills the writer before it can report why +//! the child exited, so code that writes to a child's stdin wraps the write in +//! [`with_sigpipe_ignored`]. + +/// Run `f` with SIGPIPE ignored, putting the previous disposition back afterwards. +/// +/// A write that reaches a closed pipe inside `f` then returns an `EPIPE` error +/// instead of delivering a signal. Where the disposition was already ignored +/// (Rust's startup default) this changes nothing. The disposition is +/// process-wide, so keep `f` to the write itself. Windows has no SIGPIPE and +/// runs `f` as is. +/// +/// The disposition is read and put back with `signal()`, which reports only the +/// handler (`SIG_DFL`, `SIG_IGN` or a function address). One that was installed +/// with `sigaction()` and carries flags or a signal mask would come back without +/// them. That is not a case here: this process only ever sets `SIG_DFL` (`rocm` +/// at startup) and `SIG_IGN` (this guard) for SIGPIPE. +/// +/// Guarded regions may overlap, on one thread or on several (the stderr +/// forwarder of one engine request runs while another request writes to its +/// child). The disposition in force before the first region is remembered once +/// and put back when the last region ends, so an overlap can neither restore +/// "ignored" over a default action nor leave "ignored" behind. +#[cfg(unix)] +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + let _guard = unix::IgnoreGuard::enter(); + f() +} + +#[cfg(unix)] +#[allow(unsafe_code)] // libc FFI +mod unix { + use std::sync::{Mutex, PoisonError}; + + /// How many guarded regions are open, and the disposition that was in force + /// before the first of them. + static OPEN: Mutex<(usize, libc::sighandler_t)> = Mutex::new((0, libc::SIG_DFL)); + + /// Holds SIGPIPE ignored from [`IgnoreGuard::enter`] until it is dropped, so + /// a panic inside the guarded region still restores the disposition. + pub(super) struct IgnoreGuard; + + impl IgnoreGuard { + pub(super) fn enter() -> Self { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + if open.0 == 0 { + // SAFETY: reading/installing a signal disposition is always safe. + open.1 = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + } + open.0 += 1; + Self + } + } + + impl Drop for IgnoreGuard { + fn drop(&mut self) { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + open.0 -= 1; + if open.0 == 0 { + // SAFETY: restoring the disposition captured by the first guard. + unsafe { + libc::signal(libc::SIGPIPE, open.1); + } + } + } + } +} + +#[cfg(not(unix))] +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + f() +} + +#[cfg(all(test, unix))] +mod tests { + use super::with_sigpipe_ignored; + use std::io::Write as _; + use std::os::unix::process::ExitStatusExt as _; + use std::process::{Command, ExitStatus, Stdio}; + + /// The disposition SIGPIPE has now (read by installing the same one back). + #[allow(unsafe_code)] // libc::signal + fn current_disposition() -> libc::sighandler_t { + // SAFETY: reading/installing a signal disposition is always safe. + let current = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + // SAFETY: putting back what was just read. + unsafe { + libc::signal(libc::SIGPIPE, current); + } + current + } + + /// Guarded regions that overlap, nested on one thread or interleaved across + /// two, leave SIGPIPE ignored for as long as any of them is open and put the + /// original default action back only when the last one closes. With a guard + /// that saved and restored per call, the region that ended first would + /// restore "ignored" over the second's `SIG_DFL` capture, or restore the + /// default under the region still open. Runs in a re-executed child, since + /// the disposition is process-wide and the harness leaves it ignored. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn overlapping_guards_restore_the_original_disposition_once() { + if std::env::var_os("ROCM_TEST_SIGPIPE_OVERLAP_ARM").is_some() { + use std::sync::Barrier; + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + + // Nested on one thread. + with_sigpipe_ignored(|| { + with_sigpipe_ignored(|| {}); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the inner region ended while the outer one was open" + ); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "nested"); + + // Two threads whose regions overlap, the first to enter leaving + // first. The second thread waits until the first is inside its + // region before it enters its own, so the order is the same on + // every run: were it left to the scheduler, a per-call save and + // restore would pass whenever the "second" closure happened to run + // first. + let first_in = Barrier::new(2); + let entered = Barrier::new(2); + let first_left = Barrier::new(2); + std::thread::scope(|scope| { + scope.spawn(|| { + with_sigpipe_ignored(|| { + first_in.wait(); + entered.wait(); + }); + first_left.wait(); + }); + scope.spawn(|| { + first_in.wait(); + with_sigpipe_ignored(|| { + entered.wait(); + first_left.wait(); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the first region ended while this one was open" + ); + }); + }); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "interleaved"); + return; + } + + let status = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "sigpipe::tests::overlapping_guards_restore_the_original_disposition_once", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_OVERLAP_ARM", "1") + .status() + .expect("re-exec the test arm in a child process"); + assert!(status.success(), "{status:?}"); + } + + /// A write to a dead child's stdin under the default SIGPIPE action is + /// fatal, and the same write inside [`with_sigpipe_ignored`] returns an + /// error instead. + /// + /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE + /// at Rust's default `SIG_IGN`, and flipping it process-wide would race + /// sibling tests. So this re-execs itself in fresh processes that install + /// `SIG_DFL` first, and checks BOTH directions: + /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — + /// this is the regression the plain `SIG_DFL` reset introduced; + /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns + /// an error instead — this is the fix. + /// + /// Each child reaps its short-lived grandchild before writing, so the read + /// end is closed deterministically and the outcome does not race. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { + // Grandchild helper: install the default action, spawn a process that + // exits immediately, reap it, then write to its now-closed stdin. The + // "guarded" arm wraps the write; the "unguarded" arm does not. + if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let mut grandchild = Command::new("true") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a process that exits immediately"); + let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); + grandchild.wait().expect("reap grandchild"); + // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; + // it must exceed the pipe buffer so the write actually reaches the + // closed read end rather than being swallowed by kernel buffering. + let payload = vec![b'x'; 64 * 1024]; + if mode == "guarded" { + let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); + assert!( + result.is_err(), + "guarded write to a dead pipe should return BrokenPipe, not succeed" + ); + } else { + // Unguarded: under SIG_DFL this write delivers SIGPIPE and never + // returns. If the process somehow survives, exit 0 so the parent's + // "should have been signalled" assertion fails loudly. + let _ = stdin.write_all(&payload); + } + return; + } + + let exe = std::env::current_exe().expect("current test executable"); + let run_arm = |mode: &str| -> ExitStatus { + Command::new(&exe) + .args([ + "sigpipe::tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_ARM", mode) + .status() + .expect("re-exec the test arm in a child process") + }; + + let unguarded = run_arm("unguarded"); + assert_eq!( + unguarded.signal(), + Some(libc::SIGPIPE), + "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ + SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" + ); + + let guarded = run_arm("guarded"); + assert!( + guarded.success(), + "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ + SIGPIPE reached the process instead of surfacing as an error" + ); + } +} diff --git a/crates/rocm-core/src/test_env.rs b/crates/rocm-core/src/test_env.rs index 7e5bd9da58..e1af4d09b2 100644 --- a/crates/rocm-core/src/test_env.rs +++ b/crates/rocm-core/src/test_env.rs @@ -18,6 +18,12 @@ use std::ffi::OsString; use std::path::Path; +/// Serializes the tests that set `ROCM_CLI_DATA_DIR`, and the ones whose +/// expectations depend on it (the recovery advice for an unreadable +/// `config.json` names it). Named `*_TEST_LOCK` so the env-mutation contract +/// guard recognises the discipline by suffix. +pub(crate) static DATA_DIR_ENV_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + /// Sets an environment variable and puts the previous value back on drop. /// /// This RESTORES; it does not SERIALIZE. Two tests holding one of these for the diff --git a/crates/rocm-core/src/test_support.rs b/crates/rocm-core/src/test_support.rs index 518787ee5f..6beb42599c 100644 --- a/crates/rocm-core/src/test_support.rs +++ b/crates/rocm-core/src/test_support.rs @@ -3,8 +3,9 @@ // SPDX-License-Identifier: MIT //! Shared fixtures for building a throwaway [`AppPaths`] under a per-test, -//! per-process directory, and for seeding the automation audit store that both -//! `automation.rs`'s and `jsonl_store.rs`'s tests drive. +//! per-process directory, and for seeding the automation audit store that +//! `automation.rs`'s tests drive. `jsonl_store.rs`'s tests use only the +//! throwaway directory: the engine is generic, so they carry their own record. //! //! Used from every module split out of this crate's former `lib.rs` god file //! so the naming/uniqueness scheme for test artifact directories stays in one @@ -178,36 +179,23 @@ pub(crate) fn audit_messages(records: &[AuditEventRecord]) -> Vec<&str> { records.iter().map(|r| r.message.as_str()).collect() } -/// Case count for a property test that sets its own count: `PROPTEST_CASES` -/// (proptest's own knob, which an explicit count would otherwise override) -/// when it is set and above `default`, otherwise `default`. -/// -/// Never below `default`: the counts are chosen so a property catches the -/// defects it targets with near certainty, and some of those are hit by a -/// small share of cases, so a lower count would pass with them present. `0` -/// would pass vacuously. A value that is ignored, because it is lower or does -/// not parse, is reported on stderr. `archive_props::run_property` applies the -/// same rule with its own copy, since other crates include that file by -/// `#[path]`. -pub(crate) fn proptest_cases(default: u32) -> u32 { - let requested = std::env::var("PROPTEST_CASES").ok(); - let cases = raised_case_count(requested.as_deref(), default); - if let Some(requested) = requested - && requested.parse::().ok() != Some(cases) - { - eprintln!( - "PROPTEST_CASES={requested} ignored: this property runs at least {default} cases" - ); - } - cases -} - -/// The pure half of [`proptest_cases`]: `requested` raises `default` and never -/// lowers it. -fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { - requested - .and_then(|value| value.parse().ok()) - .map_or(default, |requested: u32| requested.max(default)) +// The case-count rule for property tests that set their own count, shared with +// `rocm`, the integration tests and the Lemonade engine rather than copied. +#[path = "../tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; + +/// The temp name the retired `write_atomic` picked for `destination` at +/// `millis`: the full file name plus `.tmp--`, with this +/// process's pid. Unlike the names `atomic_write` picks, it was fixed by the +/// clock and the pid alone, so a test can occupy it ahead of time. +pub(crate) fn retired_temp_path(destination: &std::path::Path, millis: u128) -> PathBuf { + let mut name = destination + .file_name() + .expect("a destination has a file name") + .to_os_string(); + name.push(format!(".tmp-{millis}-{}", std::process::id())); + destination.with_file_name(name) } #[cfg(test)] @@ -218,13 +206,4 @@ mod tests { fn temp_app_paths_gives_each_call_its_own_root_even_for_one_label() { assert_each_call_gets_its_own_root(|| temp_app_paths("same-label").0); } - - #[test] - fn proptest_cases_are_raised_but_never_lowered() { - assert_eq!(raised_case_count(None, 256), 256); - assert_eq!(raised_case_count(Some("4096"), 256), 4096); - assert_eq!(raised_case_count(Some("16"), 256), 256); - assert_eq!(raised_case_count(Some("0"), 256), 256); - assert_eq!(raised_case_count(Some("lots"), 256), 256); - } } diff --git a/crates/rocm-core/src/uv.rs b/crates/rocm-core/src/uv.rs index 5f3b17b34c..e89a0d1121 100644 --- a/crates/rocm-core/src/uv.rs +++ b/crates/rocm-core/src/uv.rs @@ -422,6 +422,38 @@ pub fn ensure_uv_binary(paths: &AppPaths) -> Result { ) } +/// A `uv` that is already available, without acquiring one: the binary named by +/// [`UV_BINARY_ENV`], or else the managed copy a previous install left behind. +/// +/// This is [`ensure_uv_binary`] stopped short of its download. It never downloads, +/// takes a lock, or creates a folder (not even the managed tools folder), and `None` +/// is the answer when `ensure_uv_binary` would have had to fetch one. A caller that +/// must leave the machine untouched (a preview) asks this instead and goes without +/// `uv` on `None`. A managed copy is only reported once it runs, which is the same +/// test `ensure_uv_binary` applies before it reuses one. +pub fn find_uv_binary(paths: &AppPaths) -> Option { + if let Some(path) = uv_binary_override() { + return Some(path); + } + find_managed_uv(paths, &uv_version()) +} + +/// The installed managed `uv` of `version`, when it runs. Looks and creates nothing. +fn find_managed_uv(paths: &AppPaths, version: &str) -> Option { + usable_uv_in(&managed_uv_install_dir(paths, version), uv_binary_name()) +} + +/// Where the managed `uv` of `version` is installed: `tools/uv/` under the +/// data folder. Shared by the install and by [`find_uv_binary`] so that a copy one +/// installs is the copy the other finds. +fn managed_uv_install_dir(paths: &AppPaths, version: &str) -> PathBuf { + managed_uv_root(paths).join(slug(version)) +} + +fn managed_uv_root(paths: &AppPaths) -> PathBuf { + managed_tools_dir(&paths.data_dir).join("uv") +} + /// Which `uv` release to install, resolved from the environment by /// [`ensure_uv_binary`] so the install itself never reads it. struct ManagedUvRelease { @@ -444,9 +476,9 @@ struct ManagedUvRelease { /// The lock is a leaf: nothing below acquires another lock, so it cannot take part in /// a lock-order inversion with the managed-launch or daemon-autostart locks. fn install_managed_uv(paths: &AppPaths, release: &ManagedUvRelease) -> Result { - let uv_root = managed_tools_dir(&paths.data_dir).join("uv"); + let uv_root = managed_uv_root(paths); let version_slug = slug(&release.version); - let install_dir = uv_root.join(&version_slug); + let install_dir = managed_uv_install_dir(paths, &release.version); let binary_name = uv_binary_name(); if let Some(existing) = usable_uv_in(&install_dir, binary_name) { @@ -1216,6 +1248,76 @@ All installed packages are compatible Ok(std::fs::read(&archive)?) } + /// Plants `script` as the managed `uv` of `version`, where an install would have + /// left it. + #[cfg(unix)] + fn plant_managed_uv(paths: &AppPaths, version: &str, script: &str) -> PathBuf { + use std::os::unix::fs::PermissionsExt; + + let dir = managed_uv_install_dir(paths, version); + std::fs::create_dir_all(&dir).expect("create the managed uv folder"); + let binary = dir.join(uv_binary_name()); + std::fs::write(&binary, script).expect("write the stand-in uv"); + std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)) + .expect("make the stand-in uv executable"); + binary + } + + // The preview's question, answered on every platform: on a clean data folder + // there is no managed `uv`, and asking leaves no trace. `ensure_uv_binary` would + // have created `tools/uv` (for its lock) before it downloaded anything, which is + // the state this pins as untouched. + #[test] + fn looking_for_a_managed_uv_on_a_clean_host_finds_none_and_creates_nothing() { + let (root, paths) = crate::test_support::temp_app_paths("uv-find-clean"); + assert_eq!(find_managed_uv(&paths, "latest"), None); + assert!( + !managed_tools_dir(&paths.data_dir).exists(), + "looking for uv must not create the managed tools folder" + ); + let _ = std::fs::remove_dir_all(&root); + } + + // Unix-only: the stand-in `uv` is a shell script, and Windows can only run a real + // executable as `uv.exe`. + #[cfg(unix)] + #[test] + fn an_installed_managed_uv_is_found_where_the_install_puts_it() { + let (root, paths) = crate::test_support::temp_app_paths("uv-find-installed"); + let planted = plant_managed_uv(&paths, "latest", "#!/bin/sh\nexit 0\n"); + assert_eq!(find_managed_uv(&paths, "latest"), Some(planted.clone())); + // The install reuses the same copy rather than downloading: the lookup and the + // install agree on the location, so a copy one leaves is the copy the other + // finds. The release names a URL nothing listens on, so reaching for it would + // fail the call. + let reused = install_managed_uv( + &paths, + &ManagedUvRelease { + version: "latest".to_owned(), + asset: "uv-test.tar.gz".to_owned(), + url: "http://127.0.0.1:9/uv-test.tar.gz".to_owned(), + }, + ) + .expect("an installed uv is reused without a download"); + assert_eq!(reused, planted); + // Another version is a different copy. + assert_eq!(find_managed_uv(&paths, "0.0.1"), None); + let _ = std::fs::remove_dir_all(&root); + } + + #[cfg(unix)] + #[test] + fn a_managed_uv_that_does_not_run_is_not_reported() { + let (root, paths) = crate::test_support::temp_app_paths("uv-find-broken"); + plant_managed_uv(&paths, "latest", "#!/bin/sh\nexit 1\n"); + assert_eq!( + find_managed_uv(&paths, "latest"), + None, + "a copy `ensure_uv_binary` would replace is not one a preview can use" + ); + let _ = std::fs::remove_dir_all(&root); + } + // Unix-only: the fake `uv` is a shell script, and Windows can only run a real // executable as `uv.exe`. The lock and the re-check under test are // platform-independent. diff --git a/crates/rocm-core/tests/common/loopback.rs b/crates/rocm-core/tests/common/loopback.rs new file mode 100644 index 0000000000..40c05aa79f --- /dev/null +++ b/crates/rocm-core/tests/common/loopback.rs @@ -0,0 +1,108 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A loopback HTTP server for the integration tests that drive real network +//! code: it answers every request with one fixed `200 OK` until it is dropped. + +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::thread; +use std::time::Duration; + +/// Serves one fixed response on an ephemeral loopback port, to any number of +/// requests, until dropped. +/// +/// Dropping it never hangs: it wakes the accept loop with a connection of its +/// own before joining, so a client that never connected cannot leave the +/// server thread blocked in `accept`. +pub struct LoopbackServer { + port: u16, + stopping: Arc, + served: Arc, + handle: Option>, +} + +impl LoopbackServer { + pub fn start(content_type: Option<&str>, body: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let port = listener.local_addr().expect("local addr").port(); + let stopping = Arc::new(AtomicBool::new(false)); + let served = Arc::new(AtomicUsize::new(0)); + let content_type = content_type + .map(|value| format!("Content-Type: {value}\r\n")) + .unwrap_or_default(); + let handle = thread::spawn({ + let stopping = Arc::clone(&stopping); + let served = Arc::clone(&served); + move || { + for stream in listener.incoming() { + if stopping.load(Ordering::SeqCst) { + break; + } + let Ok(mut stream) = stream else { continue }; + // A client that connects and never sends cannot hold the + // accept loop, and with it `Drop`, forever. + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + read_request_head(&mut stream); + // Counted before the response is written, so a client that + // has its answer always sees its request counted. + served.fetch_add(1, Ordering::SeqCst); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\n{content_type}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(&body); + let _ = stream.flush(); + } + } + }); + Self { + port, + stopping, + served, + handle: Some(handle), + } + } + + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Requests answered so far. A test that expects a verdict about the + /// response checks this is non-zero, so a request that never reached the + /// server cannot pass as one the code under test rejected. + pub fn requests_served(&self) -> usize { + self.served.load(Ordering::SeqCst) + } +} + +impl Drop for LoopbackServer { + fn drop(&mut self) { + self.stopping.store(true, Ordering::SeqCst); + // Wake the accept loop so it sees the flag. If even that connection + // fails, leave the thread detached rather than block on it. + let woken = TcpStream::connect(("127.0.0.1", self.port)).is_ok(); + if let Some(handle) = self.handle.take() + && woken + { + let _ = handle.join(); + } + } +} + +/// Read the request line and headers, so the response is not written into a +/// request still being sent. +fn read_request_head(stream: &mut TcpStream) { + let mut head = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !head.windows(4).any(|window| window == b"\r\n\r\n") && head.len() < 64 * 1024 { + match stream.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(read) => head.extend_from_slice(&chunk[..read]), + } + } +} diff --git a/crates/rocm-core/tests/common/proptest_cases.rs b/crates/rocm-core/tests/common/proptest_cases.rs new file mode 100644 index 0000000000..50b3090e5d --- /dev/null +++ b/crates/rocm-core/tests/common/proptest_cases.rs @@ -0,0 +1,54 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The case-count rule for property tests that set their own count. This file +//! is its one implementation: `rocm`'s and `rocm-core`'s crate-internal +//! `test_support`, the `rocm-core` and `rocm-engine-protocol` integration +//! tests, and the Lemonade engine's unit tests all include it by `#[path]`. +//! `archive_props::run_property` applies the same rule inline, since that file +//! is itself included by `#[path]` into three crates. + +/// Case count for a property test: `PROPTEST_CASES` (proptest's own knob, +/// which an explicit count would otherwise override) when it is set and above +/// `default`, otherwise `default`. +/// +/// Never below `default`: each count is chosen so its property reaches the +/// cases it targets, and some of those are hit by a small share of cases, so a +/// lower count would pass with the defect present. `PROPTEST_CASES=0` would +/// otherwise pass every property without running one. +pub fn proptest_cases(default: u32) -> u32 { + case_count_from_env("PROPTEST_CASES", default) +} + +/// Case count read from `var`, raising `default` but never lowering it, for +/// the same reason as [`proptest_cases`]. `rocm`'s uninstall properties use it +/// with their own knob, `ROCM_DELETION_PROP_CASES`. A value that is ignored, +/// because it is lower or does not parse, is reported on stderr. +pub fn case_count_from_env(var: &str, default: u32) -> u32 { + let requested = std::env::var(var).ok(); + let cases = raised_case_count(requested.as_deref(), default); + if let Some(requested) = requested + && requested.parse::().ok() != Some(cases) + { + eprintln!("{var}={requested} ignored: this property runs at least {default} cases"); + } + cases +} + +/// The pure half of [`case_count_from_env`]: `requested` raises `default` and +/// never lowers it. +pub fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { + requested + .and_then(|value| value.parse().ok()) + .map_or(default, |requested: u32| requested.max(default)) +} + +#[test] +fn proptest_cases_can_raise_the_default_but_never_lower_it() { + assert_eq!(raised_case_count(None, 24), 24); + assert_eq!(raised_case_count(Some("1000"), 24), 1000); + assert_eq!(raised_case_count(Some("0"), 24), 24); + assert_eq!(raised_case_count(Some("8"), 24), 24); + assert_eq!(raised_case_count(Some("lots"), 24), 24); +} diff --git a/crates/rocm-core/tests/digest_gate_properties.rs b/crates/rocm-core/tests/digest_gate_properties.rs new file mode 100644 index 0000000000..c0aef3ca0f --- /dev/null +++ b/crates/rocm-core/tests/digest_gate_properties.rs @@ -0,0 +1,333 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the digest gate in `download_file_streaming`, the single +//! place a downloaded artifact's bytes are authenticated. +//! +//! The generator is built around *near misses*, because that is the only region +//! where this gate can plausibly fail open: a uniformly random 64-character +//! string differs from the real digest in ~every position and is rejected by +//! any implementation, correct or not. What distinguishes a correct comparison +//! from a prefix comparison, a truncating comparison, or a trimming one is a +//! candidate that shares most of the real digest. + +#[path = "common/loopback.rs"] +mod loopback; +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + +use std::time::Duration; + +use loopback::LoopbackServer; +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_core::{DOWNLOAD_CHUNK_BYTES, DownloadRequest, download_file_streaming}; +use sha2::{Digest, Sha256}; + +/// A fresh scratch directory, removed when the returned guard drops — including +/// when an assertion fails, so a failing run leaves nothing in the temp dir. +fn scratch_dir(tag: &str) -> tempfile::TempDir { + tempfile::Builder::new() + .prefix(&format!("rocm-core-digest-gate-{tag}-")) + .tempdir() + .expect("scratch dir") +} + +/// Run one download against a loopback server, returning whether the gate +/// accepted the body. +fn download_accepts(body: &[u8], expected_sha256: Option<&str>, tag: &str) -> bool { + download_outcome(body, expected_sha256, tag).0.is_ok() +} + +/// Run one download against a loopback server, returning the download's +/// result and every file it left in its otherwise empty scratch directory — +/// the destination and any partial file, whatever it is named. +/// +/// Panics if the request never reached the server: a download that failed +/// before the body arrived proves nothing about the digest gate. +fn download_outcome( + body: &[u8], + expected_sha256: Option<&str>, + tag: &str, +) -> (Result<(), String>, Vec) { + let scratch = scratch_dir(tag); + let dir = scratch.path(); + let destination = dir.join("artifact.bin"); + let server = LoopbackServer::start(None, body.to_vec()); + let url = format!("{}/artifact.bin", server.url()); + let request = DownloadRequest { + expected_sha256, + ..DownloadRequest::new(&url, &destination, Duration::from_secs(10)) + }; + let result = download_file_streaming(&request) + .map(drop) + .map_err(|error| format!("{error:#}")); + assert!( + server.requests_served() > 0, + "the download never reached the test server: {result:?}" + ); + drop(server); + let mut left_behind: Vec = std::fs::read_dir(dir) + .expect("read scratch dir") + .map(|entry| { + entry + .expect("scratch entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + left_behind.sort(); + (result, left_behind) +} + +/// Whether `result` is the digest gate's rejection, rather than success or +/// some other failure. +fn is_digest_rejection(result: &Result<(), String>) -> bool { + result + .as_ref() + .is_err_and(|error| error.contains("SHA-256 mismatch")) +} + +fn digest_of(body: &[u8]) -> String { + format!("{:x}", Sha256::digest(body)) +} + +/// How to perturb the true digest. Each variant lands in the region where a +/// sloppy comparison would wrongly accept. +#[derive(Debug, Clone, Copy)] +enum NearMiss { + /// One hex nibble changed — the smallest possible difference. + OneNibble, + /// Correct prefix, wrong suffix: accepted by a prefix comparison. + WrongSuffix, + /// Correct prefix only, truncated: accepted by a `starts_with` comparison. + TruncatedPrefix, + /// Correct digest with a trailing newline, as a `.sha256` sidecar carries it. + TrailingNewline, + /// Correct digest with surrounding spaces. + Padded, + /// Correct digest with a 65th character appended. + Overlong, + /// Correct digest with a leading zero prepended (shifts every position). + LeadingZero, + /// Empty — the "no expectation" string, which must not read as a match. + Empty, + /// All-zero digest of the right shape. + AllZeroes, +} + +const ALL_NEAR_MISSES: [NearMiss; 9] = [ + NearMiss::OneNibble, + NearMiss::WrongSuffix, + NearMiss::TruncatedPrefix, + NearMiss::TrailingNewline, + NearMiss::Padded, + NearMiss::Overlong, + NearMiss::LeadingZero, + NearMiss::Empty, + NearMiss::AllZeroes, +]; + +fn apply_near_miss(digest: &str, miss: NearMiss, position: usize, keep: usize) -> String { + let position = position % digest.len(); + let keep = 1 + keep % (digest.len() - 1); + match miss { + NearMiss::OneNibble => { + let mut out: Vec = digest.chars().collect(); + out[position] = if out[position] == '0' { '1' } else { '0' }; + out.into_iter().collect() + } + NearMiss::WrongSuffix => { + let mut out = digest[..keep].to_owned(); + out.push_str(&"f".repeat(digest.len() - keep)); + if out == digest { + out.replace_range(keep..=keep, "0"); + } + out + } + NearMiss::TruncatedPrefix => digest[..keep].to_owned(), + NearMiss::TrailingNewline => format!("{digest}\n"), + NearMiss::Padded => format!(" {digest} "), + NearMiss::Overlong => format!("{digest}a"), + NearMiss::LeadingZero => format!("0{digest}"), + NearMiss::Empty => String::new(), + NearMiss::AllZeroes => "0".repeat(64), + } +} + +/// The positional near misses — one nibble changed, a correct prefix with a +/// wrong tail, a truncated prefix — at every position and every prefix length, +/// rather than at positions drawn at random: a regression that misses only one +/// position (a comparison that skips the last nibble, say) is caught on every +/// run, not on the runs that happen to draw it. +#[test] +fn positional_near_misses_are_rejected_at_every_position() { + let body = b"artifact body for the positional near misses".to_vec(); + let truth = digest_of(&body); + let mut not_rejected = Vec::new(); + for miss in [ + NearMiss::OneNibble, + NearMiss::WrongSuffix, + NearMiss::TruncatedPrefix, + ] { + for at in 0..truth.len() { + // `apply_near_miss` reads `position` for a changed nibble and + // `keep` (as `1 + keep % 63`) for a prefix length; `at` walks both. + let candidate = apply_near_miss(&truth, miss, at, at); + assert!( + !candidate.eq_ignore_ascii_case(&truth), + "near-miss kind {miss:?} at {at} reproduced the true digest" + ); + let (result, _) = download_outcome(&body, Some(&candidate), "positional"); + if !is_digest_rejection(&result) { + not_rejected.push(format!("{miss:?} at {at}: {candidate:?} gave {result:?}")); + } + } + } + assert!( + not_rejected.is_empty(), + "the digest gate did not reject these near misses of {truth:?}:\n{}", + not_rejected.join("\n") + ); +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(24), + ..ProptestConfig::default() + })] + + /// A near-miss digest must never be accepted, whatever the body. The one + /// deliberate exception is pure ASCII case folding, which the gate + /// implements as equivalent and which is covered separately below. + /// + /// Every kind of near miss is tried for every generated case, rather than + /// one kind drawn per case: a regression that admits just one kind would + /// otherwise go unnoticed in any run that happened not to draw it. The + /// positional kinds are also walked exhaustively by + /// [`positional_near_misses_are_rejected_at_every_position`]. + #[test] + fn near_miss_digests_are_rejected( + body in prop::collection::vec(any::(), 1..512), + position in any::(), + keep in any::(), + ) { + let truth = digest_of(&body); + for miss in ALL_NEAR_MISSES { + let candidate = apply_near_miss(&truth, miss, position, keep); + prop_assert!( + !candidate.eq_ignore_ascii_case(&truth), + "near-miss kind {miss:?} reproduced the true digest" + ); + let (result, _) = download_outcome(&body, Some(&candidate), "near-miss"); + prop_assert!( + is_digest_rejection(&result), + "the digest gate did not reject {candidate:?} for an artifact whose digest is \ + {truth:?} (near-miss kind {miss:?}): {result:?}" + ); + } + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(12), + ..ProptestConfig::default() + })] + + /// The true digest is accepted, in either ASCII case, and in mixed case — + /// case-insensitivity must hold in both directions, not just for a fully + /// lowercase expectation. + #[test] + fn the_true_digest_is_accepted_in_any_ascii_case( + body in prop::collection::vec(any::(), 1..512), + mask in prop::collection::vec(any::(), 64), + ) { + let truth = digest_of(&body); + let mixed: String = truth + .chars() + .zip(mask) + .map(|(ch, upper)| if upper { ch.to_ascii_uppercase() } else { ch }) + .collect(); + prop_assert!(download_accepts(&body, Some(&mixed), "mixed-case")); + } + + /// Mutating the *body* while keeping the expected digest must be rejected, + /// and nothing — no destination, no partial file — may be left behind for a + /// later run to treat as cached. This checks the end state: the download + /// cleans up on more than one path, and either one suffices here. + #[test] + fn a_tampered_body_is_rejected( + body in prop::collection::vec(any::(), 2..512), + index in any::(), + bit in 0u8..8, + ) { + let truth = digest_of(&body); + let mut tampered = body; + let position = index.index(tampered.len()); + tampered[position] ^= 1 << bit; + let (result, left_behind) = download_outcome(&tampered, Some(&truth), "tampered"); + prop_assert!( + is_digest_rejection(&result), + "a body differing by one bit was not rejected against the original digest: {result:?}" + ); + prop_assert!( + left_behind.is_empty(), + "a rejected download left {left_behind:?} behind" + ); + } +} + +/// A body several streaming chunks long, tampered only in its last byte, is +/// rejected and leaves nothing behind, while the intact body is accepted: the +/// digest must cover every byte, not just the first buffer. (A hash over only a +/// prefix is caught by the intact half; the tampered half catches a gate that +/// is skipped or disabled for a body this large.) +#[test] +fn tampering_past_the_first_chunk_is_rejected() { + // Three chunks plus a byte, so the body crosses a chunk boundary + // regardless of how short the socket reads come back. + let body: Vec = (0..=255_u8) + .cycle() + .take(3 * DOWNLOAD_CHUNK_BYTES + 1) + .collect(); + let truth = digest_of(&body); + assert!( + download_accepts(&body, Some(&truth), "multi-chunk-intact"), + "the untampered multi-chunk body must be accepted" + ); + let mut tampered = body; + *tampered.last_mut().expect("non-empty body") ^= 1; + let (result, left_behind) = download_outcome(&tampered, Some(&truth), "multi-chunk"); + assert!( + is_digest_rejection(&result), + "a change in the last byte was not detected: {result:?}" + ); + assert!( + left_behind.is_empty(), + "a rejected download left {left_behind:?} behind" + ); +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(8), + ..ProptestConfig::default() + })] + + /// Without an expected digest the download succeeds — this is the gate's + /// documented fail-open mode, pinned here so it cannot be mistaken for + /// verification. `DownloadOutcome::sha256` is reported but nothing compares + /// it, so a caller that omits `expected_sha256` has authenticated nothing. + #[test] + fn an_absent_expectation_authenticates_nothing( + body in prop::collection::vec(any::(), 1..256), + ) { + prop_assert!( + download_accepts(&body, None, "no-expectation"), + "callers rely on a digest-less download succeeding; it just proves nothing" + ); + } +} diff --git a/crates/rocm-core/tests/integrity_properties.rs b/crates/rocm-core/tests/integrity_properties.rs new file mode 100644 index 0000000000..eb782735ab --- /dev/null +++ b/crates/rocm-core/tests/integrity_properties.rs @@ -0,0 +1,256 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the signature-verification path that decides whether a +//! downloaded artifact or metadata document is authentic. +//! +//! The generators here deliberately sample the *near-miss* region — a signature +//! with one byte changed, a signature re-encoded with leading zeros, a payload +//! with one bit flipped, a PEM mangled by Windows tooling — because a uniformly +//! random generator essentially never produces an input that reaches the +//! interesting branches of an RSA verifier. + +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + +use std::sync::OnceLock; + +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_core::{ + generate_rsa_signing_keypair, sign_rsa_pkcs1_sha256_signature, + verify_rsa_pkcs1_sha256_signature, +}; + +/// One 2048-bit keypair, generated once: key generation takes seconds in a debug build, and every +/// property below only needs *a* valid key, not a fresh one per case. +fn keypair() -> &'static (String, String) { + static KEYPAIR: OnceLock<(String, String)> = OnceLock::new(); + KEYPAIR.get_or_init(|| generate_rsa_signing_keypair().expect("keypair generation")) +} + +/// A second, unrelated keypair, for "signed by the wrong key" cases. +fn other_keypair() -> &'static (String, String) { + static KEYPAIR: OnceLock<(String, String)> = OnceLock::new(); + KEYPAIR.get_or_init(|| generate_rsa_signing_keypair().expect("keypair generation")) +} + +fn sign(payload: &[u8]) -> Vec { + sign_rsa_pkcs1_sha256_signature(&keypair().0, payload).expect("signing") +} + +fn verify(payload: &[u8], signature: &[u8]) -> bool { + verify_rsa_pkcs1_sha256_signature(&keypair().1, payload, signature, "test").is_ok() +} + +/// Payloads shaped like the documents actually verified here — arbitrary bytes, +/// near-empty bodies, and printable text such as a metadata JSON blob — capped +/// at 512 bytes so shrinking stays fast. +fn payload_strategy() -> impl Strategy> { + prop_oneof![ + 3 => prop::collection::vec(any::(), 0..512), + 1 => prop::collection::vec(any::(), 0..4), + 1 => "[ -~\\n]{0,400}".prop_map(String::into_bytes), + ] +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(64), + ..ProptestConfig::default() + })] + + /// Baseline: a genuine signature over the exact payload verifies. + #[test] + fn genuine_signature_verifies(payload in payload_strategy()) { + let signature = sign(&payload); + prop_assert!(verify(&payload, &signature)); + } + + /// Flipping a single bit of the payload must invalidate the signature. + /// This is the "tampered artifact" case: the attacker edits one byte of a + /// signed metadata document and keeps the original signature. + #[test] + fn single_bit_payload_mutation_is_rejected( + payload in prop::collection::vec(any::(), 1..256), + index in any::(), + bit in 0u8..8, + ) { + let signature = sign(&payload); + let mut tampered = payload.clone(); + let position = index.index(payload.len()); + tampered[position] ^= 1 << bit; + prop_assert!( + !verify(&tampered, &signature), + "a payload differing by one bit must not verify" + ); + } + + /// Truncating or extending the payload must invalidate the signature, even + /// when the retained prefix is byte-identical. + #[test] + fn payload_prefix_or_extension_is_rejected( + payload in prop::collection::vec(any::(), 2..256), + cut in any::(), + suffix in prop::collection::vec(any::(), 1..16), + ) { + let signature = sign(&payload); + let keep = cut.index(payload.len()); + prop_assert!(!verify(&payload[..keep], &signature)); + + let mut extended = payload; + extended.extend_from_slice(&suffix); + prop_assert!(!verify(&extended, &signature)); + } + + /// Mutating the signature — one byte changed, truncated, padded, or + /// re-encoded with leading zeros — must be rejected. Leading-zero + /// re-encoding is the classic PKCS#1 v1.5 pitfall: the integer value is + /// unchanged, so a verifier that parses to a bignum without length-checking + /// accepts it. + #[test] + fn signature_mutations_are_rejected( + payload in payload_strategy(), + index in any::(), + delta in 1u8..=255, + pad in 1usize..4, + ) { + let signature = sign(&payload); + + let mut flipped = signature.clone(); + let position = index.index(flipped.len()); + flipped[position] = flipped[position].wrapping_add(delta); + prop_assert!(!verify(&payload, &flipped), "a mutated signature must not verify"); + + prop_assert!(!verify(&payload, &signature[..signature.len() - 1])); + prop_assert!(!verify(&payload, &signature[1..])); + prop_assert!(!verify(&payload, &[])); + + let mut zero_padded = vec![0_u8; pad]; + zero_padded.extend_from_slice(&signature); + prop_assert!( + !verify(&payload, &zero_padded), + "a leading-zero re-encoding carries the same integer value and must \ + still be rejected on length" + ); + + let mut trailing = signature.clone(); + trailing.extend(std::iter::repeat_n(0_u8, pad)); + prop_assert!(!verify(&payload, &trailing)); + } + + /// A signature produced by a different key must never verify. + #[test] + fn signature_from_another_key_is_rejected(payload in payload_strategy()) { + let wrong = sign_rsa_pkcs1_sha256_signature(&other_keypair().0, &payload) + .expect("signing with the second key"); + prop_assert!(!verify(&payload, &wrong)); + } + + /// Arbitrary bytes offered as a signature must be rejected, never accepted + /// and never panic. + #[test] + fn arbitrary_bytes_are_never_a_valid_signature( + payload in payload_strategy(), + candidate in prop::collection::vec(any::(), 0..300), + ) { + prop_assert!(!verify(&payload, &candidate)); + } +} + +/// PEM mangling that real tooling produces: CRLF, lone CR, a UTF-8 BOM, a stray +/// trailing space, a blank line, a tab. Verification must still succeed for +/// each. The BOM, space, blank-line and tab styles depend on `normalize_pem`; +/// CRLF and lone CR are also accepted by the PEM parser itself, so those two +/// styles pin the outcome rather than `normalize_pem`'s handling of them. +fn mangle_pem(pem: &str, style: u8) -> String { + match style % 6 { + 0 => pem.replace('\n', "\r\n"), + 1 => pem.replace('\n', "\r"), + 2 => format!("\u{feff}{pem}"), + 3 => pem.replace('\n', " \n"), + 4 => pem.replace('\n', "\n\n"), + _ => pem.replace('\n', "\t\r\n"), + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(48), + ..ProptestConfig::default() + })] + + /// Tolerated mangling must not change *which* key is trusted. + #[test] + fn pem_mangling_preserves_the_verification_verdict( + payload in payload_strategy(), + style in any::(), + ) { + let signature = sign(&payload); + let mangled = mangle_pem(&keypair().1, style); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&mangled, &payload, &signature, "test").is_ok(), + "whitespace/BOM mangling must not break a genuine signature" + ); + + // The same mangling applied to the *wrong* key must still reject. + let wrong = mangle_pem(&other_keypair().1, style); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&wrong, &payload, &signature, "test").is_err(), + "normalization must not turn the wrong key into an accepting one" + ); + } + + /// Corrupting the base64 body of the public key must never yield a key that + /// accepts a signature made by the real key. + #[test] + fn corrupted_public_key_never_accepts( + payload in payload_strategy(), + index in any::(), + replacement in prop::sample::select( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + .chars() + .collect::>(), + ), + ) { + let signature = sign(&payload); + let pem = &keypair().1; + // Only the base64 body: everything after the header line and before + // the footer. + let body_start = pem.find('\n').expect("PEM header line") + 1; + let body_end = pem.find("-----END").expect("PEM footer"); + let body_positions: Vec = pem[body_start..body_end] + .char_indices() + .filter(|(_, ch)| ch.is_ascii_alphanumeric() || *ch == '+' || *ch == '/') + .map(|(position, _)| body_start + position) + .collect(); + let position = body_positions[index.index(body_positions.len())]; + let mut corrupted = pem.clone(); + corrupted.replace_range(position..=position, &replacement.to_string()); + if corrupted == *pem { + return Ok(()); + } + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&corrupted, &payload, &signature, "test").is_err(), + "a public key with a corrupted body must not accept the real signature" + ); + } + + /// Garbage offered as a public key must produce an error, never a success + /// and never a panic. + #[test] + fn arbitrary_public_key_text_is_rejected( + payload in payload_strategy(), + pem in prop_oneof![ + Just(String::new()), + "[ -~\\n]{0,200}", + "-----BEGIN PUBLIC KEY-----\n[A-Za-z0-9+/=\\n]{0,200}\n-----END PUBLIC KEY-----\n", + ], + ) { + let signature = sign(&payload); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&pem, &payload, &signature, "test").is_err() + ); + } +} diff --git a/crates/rocm-core/tests/model_selector_properties.rs b/crates/rocm-core/tests/model_selector_properties.rs new file mode 100644 index 0000000000..75b56089eb --- /dev/null +++ b/crates/rocm-core/tests/model_selector_properties.rs @@ -0,0 +1,626 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for model selection and for the readiness decision the CLI +//! makes from an engine's `/v1/models` response. +//! +//! Two separate risks live here. Selection must be *deterministic and +//! unambiguous*: `resolve_model_recipe` takes the first recipe whose +//! `matches_ref` returns true, so two recipes answering to the same selector +//! would make the winner an accident of catalog order. And readiness is decided +//! from a JSON document served by the engine — untrusted input — so it must +//! never panic and must not report a *different* model as the one the user +//! asked for. + +#[path = "common/loopback.rs"] +mod loopback; +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + +use std::time::Duration; + +use loopback::LoopbackServer; +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_core::{ModelRecipeRecord, builtin_model_recipes, openai_models_endpoint_has_model}; + +fn catalog() -> &'static [ModelRecipeRecord] { + use std::sync::OnceLock; + static CATALOG: OnceLock> = OnceLock::new(); + CATALOG.get_or_init(builtin_model_recipes) +} + +/// Every selector the shipped catalog advertises: canonical ids and aliases. +fn advertised_selectors() -> Vec<(String, String)> { + let mut selectors = Vec::new(); + for recipe in catalog() { + selectors.push(( + recipe.canonical_model_id.clone(), + recipe.canonical_model_id.clone(), + )); + for alias in &recipe.aliases { + selectors.push((alias.clone(), recipe.canonical_model_id.clone())); + } + } + selectors +} + +/// A `/v1/models` document listing exactly `ids`. +fn models_document(ids: &[&str]) -> String { + serde_json::json!({ + "object": "list", + "data": ids + .iter() + .map(|id| serde_json::json!({ "id": id, "object": "model" })) + .collect::>(), + }) + .to_string() +} + +/// Ask the readiness check what it makes of the document `server` serves when +/// expecting `expected`. Panics if the request never reached the server, so a +/// transport failure cannot pass as a "not ready" verdict. +fn readiness_from(server: &LoopbackServer, expected: Option<&str>) -> Result { + let before = server.requests_served(); + let outcome = + openai_models_endpoint_has_model(&server.url(), expected, None, Duration::from_secs(10)) + .map_err(|error| error.to_string()); + assert!( + server.requests_served() > before, + "the readiness check never reached the test server: {outcome:?}" + ); + outcome +} + +/// Ask the readiness check what it makes of `body` when expecting `expected`. +fn endpoint_reports(body: String, expected: Option<&str>) -> Result { + let server = LoopbackServer::start(Some("application/json"), body.into_bytes()); + readiness_from(&server, expected) +} + +#[test] +fn the_shipped_catalog_is_not_empty() { + assert!( + !catalog().is_empty(), + "these properties are vacuous without a catalog to check" + ); +} + +/// Every advertised selector must resolve to the recipe that advertises it. +/// +/// That no selector is advertised twice is the catalog schema's own check +/// (`ModelRecipeIndexDocument::validate`, which normalizes refs the same way +/// `matches_ref` does), run on the shipped catalog by its unit tests. +#[test] +fn every_advertised_selector_resolves_to_its_own_recipe() { + for (selector, owner) in advertised_selectors() { + let resolved: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&selector)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + assert_eq!( + resolved, + vec![owner.as_str()], + "selector {selector:?} must resolve to exactly its own recipe" + ); + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(512), + ..ProptestConfig::default() + })] + + /// Selector matching ignores surrounding whitespace and ASCII case: any + /// re-casing of an advertised selector, padded with spaces, tabs or + /// newlines, still selects exactly its own recipe. (That other edits do + /// not match is the next property's job.) + #[test] + fn selector_matching_tolerates_only_case_and_surrounding_whitespace( + index in any::(), + upper in prop::collection::vec(any::(), 0..80), + lead in "[ \\t\\n]{0,3}", + trail in "[ \\t\\n]{0,3}", + ) { + let selectors = advertised_selectors(); + let (selector, owner) = &selectors[index.index(selectors.len())]; + let recased: String = selector + .chars() + .enumerate() + .map(|(position, ch)| { + if upper.get(position).copied().unwrap_or(false) { + ch.to_ascii_uppercase() + } else { + ch + } + }) + .collect(); + let decorated = format!("{lead}{recased}{trail}"); + let resolved: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&decorated)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + prop_assert_eq!( + resolved, + vec![owner.as_str()], + "case/whitespace decoration of {:?} changed which recipe it selects", + selector + ); + } + + /// Selection is otherwise exact: an advertised selector with one character + /// inserted (interior whitespace included) or deleted selects no recipe at + /// all — neither its own (no fuzzy matching) nor another — unless the edit + /// happens to spell some other advertised selector, up to ASCII case. + #[test] + fn a_mistyped_selector_selects_nothing( + index in any::(), + position in any::(), + delete in any::(), + inserted in prop::sample::select( + "abcdefghijklmnopqrstuvwxyz0123456789-_./ \t".chars().collect::>() + ), + ) { + let selectors = advertised_selectors(); + let (selector, _) = &selectors[index.index(selectors.len())]; + let mut mistyped: Vec = selector.chars().collect(); + if delete { + let at = position.index(mistyped.len()); + mistyped.remove(at); + } else { + let at = position.index(mistyped.len() + 1); + // Whitespace at either end is surrounding whitespace, which the + // previous property covers and selection deliberately ignores. + prop_assume!( + !(inserted.is_whitespace() && (at == 0 || at == mistyped.len())) + ); + mistyped.insert(at, inserted); + } + let mistyped: String = mistyped.into_iter().collect(); + prop_assume!( + !selectors + .iter() + .any(|(other, _)| other.eq_ignore_ascii_case(&mistyped)) + ); + + let selected: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&mistyped)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + prop_assert!( + selected.is_empty(), + "mistyped selector {:?} (from {:?}) selected {:?}", + mistyped, + selector, + selected + ); + } +} + +/// `/v1/models` bodies that name no model matching any of the expectations +/// below, shaped the way a real engine shapes them, plus the degenerate shapes +/// a broken or hostile one could send. +/// +/// The listed ids carry no letter. Every expected name below does, and every +/// rule readiness matches by — equality, basename, catalog alias, family word — +/// needs a letter in common, so none of these documents may read as ready. +fn models_body_strategy() -> impl Strategy { + prop_oneof![ + 3 => prop::collection::vec("[0-9 ._:#/@-]{0,24}", 0..4).prop_map(|ids| { + let ids: Vec<&str> = ids.iter().map(String::as_str).collect(); + models_document(&ids) + }), + 1 => Just("{}".to_owned()), + 1 => Just(r#"{"data": []}"#.to_owned()), + 1 => Just(r#"{"data": null}"#.to_owned()), + 1 => Just(r#"{"data": [{}]}"#.to_owned()), + 1 => Just(r#"{"data": [{"id": null}]}"#.to_owned()), + 1 => Just(r#"{"data": [[]]}"#.to_owned()), + 1 => Just(r#"{"data": "not-an-array"}"#.to_owned()), + 1 => Just("[]".to_owned()), + 1 => Just("null".to_owned()), + 1 => "[ -~]{0,64}", + ] +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(48), + ..ProptestConfig::default() + })] + + /// A `/v1/models` document that names no matching model — malformed, + /// hostile, or listing only unrelated ids — must produce a clean `false` or + /// a clean error: never a panic, and never "ready". A check that reported + /// ready for any non-empty listing, ignoring the expected model, fails here. + #[test] + fn a_models_document_naming_no_matching_model_never_reports_ready( + body in models_body_strategy(), + expected in prop::sample::select(vec![ + "Qwen/Qwen3-4B-Instruct-2507", + "meta-llama/Llama-3.1-8B-Instruct", + "nonexistent/model", + ]), + ) { + let verdict = endpoint_reports(body.clone(), Some(expected)); + prop_assert_ne!( + verdict, + Ok(true), + "reported ready for {:?} from a document naming no matching model: {}", + expected, + body + ); + } + + /// Positive control for the property above: a document that does list the + /// expected model — exactly, re-cased, padded, or by its basename alone, as + /// engines commonly report it — among other entries reads as ready. Without + /// this, a check that never reported ready would pass every negative + /// property here. + /// It pins the outcome, not which matching rule produces it: several rules + /// accept these spellings, so it does not stand guard over any one of them. + #[test] + fn a_models_document_listing_the_expected_model_reports_ready( + others in prop::collection::vec("[ -~]{0,24}", 0..3), + at in any::(), + spelling in 0u8..4, + expected in prop::sample::select(vec![ + "Qwen/Qwen3-4B-Instruct-2507", + "meta-llama/Llama-3.1-8B-Instruct", + "nonexistent/model", + ]), + ) { + let listed = match spelling { + 0 => expected.to_owned(), + 1 => expected.to_ascii_uppercase(), + 2 => format!(" {expected}\t"), + _ => expected.rsplit('/').next().unwrap_or(expected).to_owned(), + }; + let mut ids = others; + let position = at.index(ids.len() + 1); + ids.insert(position, listed.clone()); + let ids: Vec<&str> = ids.iter().map(String::as_str).collect(); + let body = models_document(&ids); + prop_assert_eq!( + endpoint_reports(body, Some(expected)), + Ok(true), + "{:?} listed as {:?} was not reported ready", + expected, + listed + ); + } +} + +/// Whether `name` is a bare *family* word — a single alphabetic token of three +/// or more characters with no digit, like `qwen`, `llama` or `gemma`. +/// +/// Approximates the shape `rocm_core`'s private family matcher treats as a +/// family: readiness matches such a word against any model with a name token +/// that starts with it, so these names are excluded from the strict property +/// below and pinned separately. (Being an approximation, it only decides which +/// catalog names that property skips; the near-miss names that property also +/// checks never pass through it.) +fn is_bare_family_word(name: &str) -> bool { + let normalized: String = name + .trim() + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + let single_token = name + .trim() + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .count() + == 1; + single_token && normalized.len() >= 3 && !normalized.chars().any(|ch| ch.is_ascii_digit()) +} + +/// An engine that lists only a *different* catalog model must not satisfy a +/// request for the one the user asked for. This is the "silently serving an +/// unrelated model" case: readiness is what gates the traffic the user then +/// sends, so a false positive routes inference to the wrong weights. +/// +/// Checked exhaustively over every ordered pair of catalog entries — including +/// each entry's aliases as the served id, since an engine echoes back whatever +/// name it was launched with — rather than sampled. The catalog is small, the +/// confusable pairs in it are specific (several `Qwen*` variants differing only +/// in size and quantization), and a sampling strategy can miss exactly those. +/// +/// Each wanted model is also checked against near misses of its own name that +/// it does not answer to: the name with a quantization or variant suffix, the +/// first name token containing a digit (`qwen3`, or just `3` for a Llama), a +/// two-letter stub, and the leading word with its first letter dropped +/// (`wen`). Those are what separate an exact comparison from a substring one, +/// and a family match from one that has lost its guards or matches inside a +/// word rather than at its start. +/// +/// Two shapes that *do* cross-match today are excluded here and pinned +/// separately, so they stay visible: bare family words, by +/// [`a_bare_family_word_satisfies_readiness_for_the_whole_family`], and one +/// recipe's alias spelling another recipe's basename, by +/// [`an_alias_spelling_another_models_basename_reads_as_ready_for_it`]. +/// +/// The request count grows with the square of the catalog (every served name +/// against every wanted model), so each served name gets one loopback server +/// that answers all of its requests, rather than a server per request. +#[test] +fn listing_a_different_catalog_model_is_never_readiness() { + let recipes = catalog(); + assert!(recipes.len() > 1, "need at least two recipes to confuse"); + let mut false_positives = Vec::new(); + + for served in recipes { + let served_names = std::iter::once(&served.canonical_model_id).chain(&served.aliases); + for served_name in served_names { + if is_bare_family_word(served_name) { + continue; + } + let server = LoopbackServer::start( + Some("application/json"), + models_document(&[served_name.as_str()]).into_bytes(), + ); + for wanted in recipes { + // Skip the served recipe itself, a name the *wanted* recipe + // also answers to (a genuine shared alias, not a confusion), + // and the pinned alias/basename collision. + if wanted.canonical_model_id == served.canonical_model_id + || wanted.matches_ref(served_name) + || spells_the_basename_of(served_name, wanted) + { + continue; + } + if readiness_from(&server, Some(&wanted.canonical_model_id)) == Ok(true) { + false_positives.push(format!( + "wanted {:?} but an endpoint serving only {:?} reported ready", + wanted.canonical_model_id, served_name + )); + } + } + } + } + + for wanted in recipes { + for served_name in near_miss_names(&wanted.canonical_model_id) { + if wanted.matches_ref(&served_name) { + continue; + } + let body = models_document(&[served_name.as_str()]); + if endpoint_reports(body, Some(&wanted.canonical_model_id)) == Ok(true) { + false_positives.push(format!( + "wanted {:?} but an endpoint serving only the near miss {:?} reported ready", + wanted.canonical_model_id, served_name + )); + } + } + } + + assert!( + false_positives.is_empty(), + "readiness matched an unrelated model:\n{}", + false_positives.join("\n") + ); +} + +/// Whether `name`, as an engine would report it, is exactly the basename of +/// `recipe`'s canonical id (up to ASCII case) — the part after the last `/`, +/// which readiness accepts on its own because engines commonly report a model +/// by its basename. +fn spells_the_basename_of(name: &str, recipe: &ModelRecipeRecord) -> bool { + let basename = recipe + .canonical_model_id + .rsplit('/') + .next() + .unwrap_or(&recipe.canonical_model_id); + name.trim().eq_ignore_ascii_case(basename) +} + +/// Characterization test for a known false-positive surface, pinned so it is +/// visible rather than merely absent from the suite. +/// +/// Readiness accepts a model's basename for the model (an engine launched on +/// `Qwen/Qwen3-32B` commonly reports `Qwen3-32B`). Some catalog aliases spell +/// the basename of a *different* catalog model: `qwen3-32b` is an alias of +/// `Qwen/Qwen3-32B-FP8` and also the basename of `Qwen/Qwen3-32B`. An engine +/// launched on the FP8 recipe under that alias reports `qwen3-32b`, and that +/// reads as ready for `Qwen/Qwen3-32B` — readiness cannot tell the two apart +/// from the name alone. +/// +/// The collisions are derived from the catalog and compared with the list +/// below, so a catalog change that adds one fails here and has to be looked +/// at rather than being silently excluded from +/// [`listing_a_different_catalog_model_is_never_readiness`]. If readiness or +/// the catalog aliases are changed to remove a collision, update the list +/// along with them. +#[test] +fn an_alias_spelling_another_models_basename_reads_as_ready_for_it() { + let mut collisions = Vec::new(); + for owner in catalog() { + for alias in &owner.aliases { + for other in catalog() { + if other.canonical_model_id != owner.canonical_model_id + && !other.matches_ref(alias) + && spells_the_basename_of(alias, other) + { + collisions.push(( + alias.as_str(), + owner.canonical_model_id.as_str(), + other.canonical_model_id.as_str(), + )); + } + } + } + } + collisions.sort_unstable(); + assert_eq!( + collisions, + vec![ + ("qwen3-32b", "Qwen/Qwen3-32B-FP8", "Qwen/Qwen3-32B"), + ( + "qwen3.6-35b-a3b", + "unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M", + "Qwen/Qwen3.6-35B-A3B" + ), + ], + "(alias, recipe it belongs to, recipe whose basename it spells) — the catalog's \ + alias/basename collisions changed" + ); + + for (alias, owner, other) in collisions { + assert_eq!( + endpoint_reports(models_document(&[alias]), Some(other)), + Ok(true), + "pinning the current behavior: {owner:?} served as {alias:?} reads as ready for \ + {other:?}" + ); + } +} + +/// Readiness accepts a catalog model under any of its names: an engine that +/// reports an alias satisfies a request for the canonical id, and the reverse. +/// Checked for every alias in the shipped catalog. +#[test] +fn readiness_accepts_a_catalog_model_under_any_of_its_names() { + let mut misses = Vec::new(); + for recipe in catalog() { + for alias in &recipe.aliases { + for (served, wanted) in [ + (alias.as_str(), recipe.canonical_model_id.as_str()), + (recipe.canonical_model_id.as_str(), alias.as_str()), + ] { + let body = models_document(&[served]); + if endpoint_reports(body, Some(wanted)) != Ok(true) { + misses.push(format!("serving {served:?} did not satisfy {wanted:?}")); + } + } + } + } + assert!(misses.is_empty(), "{}", misses.join("\n")); +} + +/// Names that sit next to `id` without being it: `id` with a variant suffix, +/// the first token of its basename that contains a digit, a two-letter stub of +/// its basename, and the basename's leading letters minus the first one. +fn near_miss_names(id: &str) -> Vec { + let basename = id.rsplit('/').next().unwrap_or(id); + let mut names: Vec = ["-FP8", "-GGUF", "-Instruct", "-v2"] + .iter() + .map(|suffix| format!("{id}{suffix}")) + .collect(); + if let Some(first_token) = basename + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .find(|token| token.chars().any(|ch| ch.is_ascii_digit())) + { + names.push(first_token.to_ascii_lowercase()); + } + let stub: String = basename + .chars() + .filter(char::is_ascii_alphabetic) + .take(2) + .collect(); + if stub.len() == 2 { + names.push(stub.to_ascii_lowercase()); + } + let leading_word: String = basename + .chars() + .take_while(char::is_ascii_alphabetic) + .collect(); + if leading_word.len() >= 4 { + names.push(leading_word[1..].to_ascii_lowercase()); + } + names +} + +/// Characterization test for a known false-positive surface, pinned so it is +/// visible rather than merely absent from the suite. +/// +/// Readiness matches a bare *family* word against any model with a name token +/// that starts with it, and does so in both directions. One direction is +/// exercised by an existing unit test in `endpoint_readiness.rs`: asking for `qwen` is +/// satisfied by an engine serving `Qwen3-0.6B-GGUF`. This test pins the other: an endpoint +/// whose `/v1/models` reports only the bare id `qwen` — itself a catalog alias +/// — is reported ready for other `Qwen*` entries in the catalog, not just the +/// one it actually loaded. The test asserts that such cross-matches exist and +/// that they stay inside the family. +/// +/// So when an engine reports nothing more specific than the family word, +/// readiness cannot tell the models of that family apart: a request for one +/// `Qwen*` model reads as ready whichever `Qwen*` model the engine has +/// resident. Note the asymmetry — *selection* (`matches_ref`) is exact up to +/// case and whitespace, while the readiness check that confirms the engine is +/// serving the selected model is looser. +/// +/// If readiness is ever tightened, this test is expected to fail and should be +/// updated along with the behavior. +#[test] +fn a_bare_family_word_satisfies_readiness_for_the_whole_family() { + let body = models_document(&["qwen"]); + + let cross_matched: Vec<&str> = catalog() + .iter() + .map(|recipe| recipe.canonical_model_id.as_str()) + // Every catalog entry except the one that genuinely owns the `qwen` + // alias. + .filter(|id| { + !catalog() + .iter() + .find(|recipe| recipe.canonical_model_id == *id) + .is_some_and(|recipe| recipe.matches_ref("qwen")) + }) + .filter(|id| endpoint_reports(body.clone(), Some(id)) == Ok(true)) + .collect(); + + assert!( + !cross_matched.is_empty(), + "expected the documented family-word cross-match to still be present" + ); + assert!( + cross_matched + .iter() + .all(|id| id.to_ascii_lowercase().contains("qwen")), + "the cross-match must at least stay inside the family: {cross_matched:?}" + ); + + // The other direction, which the `endpoint_readiness.rs` unit test also exercises: asking + // for the bare family word is satisfied by an engine serving any catalog + // model of that family. + for recipe in catalog().iter().filter(|recipe| { + recipe + .canonical_model_id + .to_ascii_lowercase() + .contains("qwen") + }) { + let body = models_document(&[recipe.canonical_model_id.as_str()]); + assert_eq!( + endpoint_reports(body, Some("qwen")), + Ok(true), + "asking for `qwen` was not satisfied by a served {:?}", + recipe.canonical_model_id + ); + } +} + +/// The same behavior outside the catalog: a user can `rocm serve` any Hugging +/// Face ref, and readiness compares the engine's reported id against it. A bare +/// family word therefore satisfies a request for any model of that family, +/// whatever its size. +#[test] +fn a_bare_family_word_satisfies_readiness_for_an_uncatalogued_model() { + for (served, wanted) in [ + ("qwen", "Qwen/Qwen3-235B-A22B-Instruct"), + ("llama", "meta-llama/Llama-3.1-70B-Instruct"), + ("gemma", "google/gemma-2-27b-it"), + ] { + let body = models_document(&[served]); + assert_eq!( + endpoint_reports(body, Some(wanted)), + Ok(true), + "pinning the current behavior: serving {served:?} reads as ready for {wanted:?}" + ); + } +} diff --git a/crates/rocm-dash-collectors/Cargo.toml b/crates/rocm-dash-collectors/Cargo.toml index f8e7a826eb..5926adb3d0 100644 --- a/crates/rocm-dash-collectors/Cargo.toml +++ b/crates/rocm-dash-collectors/Cargo.toml @@ -16,7 +16,6 @@ rocm-core = { path = "../rocm-core" } serde = { version = "1", features = ["derive"] } serde_json = "1" csv = "1" -regex = "1" thiserror = "2" chrono = { version = "0.4", features = ["serde"] } tracing = "0.1" @@ -28,11 +27,6 @@ tokio = { version = "1", features = ["full"] } reqwest = { version = "0.12", default-features = false } [dev-dependencies] -# Test-only: `write_fake_executable` plants the fake `amd-smi` the timeout tests -# run without the intermittent "Text file busy" failure that `std::fs::write` -# followed by a chmod hits. Dev-dependencies are exempt from the layering -# allowlist, so the collectors keep no normal dependency on `rocm-core`. -rocm-core = { path = "../rocm-core" } tokio = { version = "1", features = ["full"] } tempfile = "3" wiremock = "0.6" diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 0d258952e1..6370f19860 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -5,7 +5,7 @@ //! amd-smi subprocess + JSON parse. //! //! Field paths and the GPU-device pre-flight are vendored from the TypeScript -//! `AmdSmiProvider` in instinct-dash. See `../wiki/entities/amd-smi.md`. +//! `AmdSmiProvider` in instinct-dash. use std::ffi::OsString; use std::io; diff --git a/crates/rocm-dash-collectors/src/ancestry.rs b/crates/rocm-dash-collectors/src/ancestry.rs new file mode 100644 index 0000000000..5d2f30613a --- /dev/null +++ b/crates/rocm-dash-collectors/src/ancestry.rs @@ -0,0 +1,230 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! GPU process → managed service, by process ancestry. +//! +//! `amd-smi process` reports the host PIDs of GPU compute processes. For a +//! managed `rocm serve` service those are the engine's workers, which descend +//! from the PID rocm-cli recorded for the engine (or for its supervisor), so a +//! walk up `/proc//stat` parent links reaches a recorded root. This is the +//! managed-service counterpart of [`crate::cgroup`], which only recognises +//! docker/containerd ids and can never name a managed service. +//! +//! The matcher is pure: the `/proc//stat` reader is injected, so the +//! whole thing is testable on any host. Only [`owner_of_pid`] reads `/proc`, +//! and it is `cfg`-gated with a non-Linux stub. + +/// How many parent links to follow before giving up. Real engines sit a few +/// levels deep (launcher, server, worker); the bound only stops a corrupt or +/// adversarial parent chain from looping. +const MAX_DEPTH: usize = 32; + +/// The lowest PID that can be a root. +/// +/// PID 0 means "not recorded" and PID 1 is init, an ancestor of every process on +/// the host: a root there would own all of them, including GPU processes that +/// belong to no service. +pub const MIN_ROOT_PID: u32 = 2; + +/// A recorded process that identifies one managed service. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Root { + /// The recorded PID (engine or supervisor). + pub pid: u32, + /// The kernel start time recorded with it, when there is one. A PID can be + /// recycled, so a root whose live start time differs is a different process + /// that merely has the same number, and must not claim anything. + pub start_ticks: Option, + /// The managed service this root belongs to. + pub owner: String, +} + +/// `(parent pid, start time in clock ticks)` from the contents of +/// `/proc//stat`, or `None` when it does not parse. +/// +/// The process name (field 2) is parenthesised and may itself contain spaces +/// and parentheses, so the fields after it are counted from the *last* `)`. +/// Field numbers below are the `proc(5)` ones: state is 3, ppid is 4 and the +/// start time is 22. +#[must_use] +pub fn parse_stat(stat: &str) -> Option<(u32, u64)> { + let rest = &stat[stat.rfind(')')? + 1..]; + let fields: Vec<&str> = rest.split_whitespace().collect(); + let ppid = fields.get(1)?.parse().ok()?; + let start = fields.get(19)?.parse().ok()?; + Some((ppid, start)) +} + +/// The managed service that owns `pid`, if `pid` is, or descends from, one of +/// `roots`. `read_stat` returns the contents of `/proc//stat` for a PID. +/// +/// A root counts only when its recorded start time (if it has one) matches the +/// live process, so a recycled PID never claims a stranger's GPU memory. An +/// unreadable link ends the walk with no owner rather than a guess. +pub fn owner_of(pid: u32, roots: &[Root], read_stat: F) -> Option +where + F: Fn(u32) -> Option, +{ + let mut current = pid; + for _ in 0..MAX_DEPTH { + let (parent, start) = parse_stat(&read_stat(current)?)?; + if let Some(root) = roots + .iter() + .find(|r| r.pid == current && r.pid >= MIN_ROOT_PID) + && root.start_ticks.is_none_or(|recorded| recorded == start) + { + return Some(root.owner.clone()); + } + if parent == 0 || parent == current { + return None; + } + current = parent; + } + None +} + +/// [`owner_of`] against the live `/proc`. +#[cfg(target_os = "linux")] +#[must_use] +pub fn owner_of_pid(pid: u32, roots: &[Root]) -> Option { + owner_of(pid, roots, |p| { + std::fs::read_to_string(format!("/proc/{p}/stat")).ok() + }) +} + +/// Non-Linux stub: there is no `/proc`, so ancestry is never available. +#[cfg(not(target_os = "linux"))] +#[must_use] +pub const fn owner_of_pid(_pid: u32, _roots: &[Root]) -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + /// A `/proc//stat` line: `comm` is field 2, then state, ppid, and 17 + /// filler fields so the start time lands at field 22. + fn stat(pid: u32, comm: &str, ppid: u32, start: u64) -> String { + let filler = vec!["0"; 17].join(" "); + format!("{pid} ({comm}) S {ppid} {filler} {start} 0 0") + } + + fn table(entries: &[(u32, u32, u64)]) -> HashMap { + entries + .iter() + .map(|&(pid, ppid, start)| (pid, stat(pid, "proc", ppid, start))) + .collect() + } + + fn root(pid: u32, start: Option, owner: &str) -> Root { + Root { + pid, + start_ticks: start, + owner: owner.into(), + } + } + + #[test] + fn parse_stat_reads_ppid_and_start_time_even_when_the_name_is_awkward() { + assert_eq!(parse_stat(&stat(10, "vllm", 4, 777)), Some((4, 777))); + // Spaces and parentheses inside the process name. + assert_eq!( + parse_stat(&stat(10, "my (odd) name", 4, 777)), + Some((4, 777)) + ); + assert_eq!(parse_stat("garbage"), None); + assert_eq!(parse_stat("10 (short) S 4"), None); + } + + #[test] + fn a_worker_resolves_to_the_service_whose_engine_is_its_ancestor() { + // 100 (supervisor) -> 200 (engine) -> 300 (TP worker) -> 301 (helper) + let t = table(&[ + (100, 1, 5), + (200, 100, 6), + (300, 200, 7), + (301, 300, 8), + (1, 0, 1), + ]); + let roots = [root(200, Some(6), "svc-a")]; + let read = |p: u32| t.get(&p).cloned(); + assert_eq!(owner_of(301, &roots, read), Some("svc-a".into())); + assert_eq!(owner_of(300, &roots, read), Some("svc-a".into())); + // The root itself counts, and a process outside the tree does not. + assert_eq!(owner_of(200, &roots, read), Some("svc-a".into())); + assert_eq!(owner_of(100, &roots, read), None); + } + + #[test] + fn two_services_are_told_apart() { + let t = table(&[(10, 1, 1), (11, 10, 2), (20, 1, 3), (21, 20, 4), (1, 0, 1)]); + let roots = [root(10, None, "a"), root(20, None, "b")]; + let read = |p: u32| t.get(&p).cloned(); + assert_eq!(owner_of(11, &roots, read), Some("a".into())); + assert_eq!(owner_of(21, &roots, read), Some("b".into())); + } + + #[test] + fn a_recycled_pid_does_not_claim_a_stranger_s_children() { + // The recorded engine had start time 6, but pid 200 is now a different + // process (start 99) that happens to own a GPU worker. + let t = table(&[(200, 1, 99), (300, 200, 7), (1, 0, 1)]); + let roots = [root(200, Some(6), "svc-a")]; + assert_eq!(owner_of(300, &roots, |p| t.get(&p).cloned()), None); + // With no recorded start time the root is taken at its word. + let loose = [root(200, None, "svc-a")]; + assert_eq!( + owner_of(300, &loose, |p| t.get(&p).cloned()), + Some("svc-a".into()) + ); + } + + #[test] + fn a_root_at_init_or_zero_never_owns_anything() { + // Every process descends from init, so a root recorded as PID 1 would + // claim every GPU process on the host, whoever it belongs to. + let t = table(&[(300, 200, 7), (200, 1, 6), (1, 0, 1)]); + let read = |p: u32| t.get(&p).cloned(); + // PID 0 documents a state the walk never reaches (a parent of 0 ends it + // before `current` is assigned); PID 1 is the one the guard decides. + for pid in [0, 1] { + let roots = [root(pid, None, "svc")]; + assert_eq!(owner_of(300, &roots, read), None, "root pid {pid}"); + } + // A real root beside a bogus one still works. + let roots = [root(1, None, "bogus"), root(200, None, "real")]; + assert_eq!(owner_of(300, &roots, read), Some("real".into())); + // A process outside every service tree has no owner even with a bogus root. + let stray = table(&[(900, 1, 50), (1, 0, 1)]); + assert_eq!( + owner_of(900, &[root(1, None, "bogus")], |p| stray.get(&p).cloned()), + None + ); + } + + #[test] + fn owner_of_pid_is_total_on_every_platform() { + // A PID that cannot exist: unreadable on Linux, a stub elsewhere. Never + // a panic, never an owner. + assert_eq!(owner_of_pid(u32::MAX, &[]), None); + assert_eq!(owner_of_pid(u32::MAX, &[root(2, None, "svc")]), None); + } + + #[test] + fn unreadable_links_and_cycles_end_without_an_owner() { + let roots = [root(1000, None, "svc")]; + // The worker's parent has vanished. + let t = table(&[(300, 200, 7)]); + assert_eq!(owner_of(300, &roots, |p| t.get(&p).cloned()), None); + // A parent chain that loops must terminate. + let looped = table(&[(5, 6, 1), (6, 5, 2)]); + assert_eq!(owner_of(5, &roots, |p| looped.get(&p).cloned()), None); + // A process that is its own parent, and init. + let selfref = table(&[(7, 7, 1)]); + assert_eq!(owner_of(7, &roots, |p| selfref.get(&p).cloned()), None); + assert_eq!(owner_of(5, &[], |_| None), None); + } +} diff --git a/crates/rocm-dash-collectors/src/docker.rs b/crates/rocm-dash-collectors/src/docker.rs index 1727995052..9bea9d5841 100644 --- a/crates/rocm-dash-collectors/src/docker.rs +++ b/crates/rocm-dash-collectors/src/docker.rs @@ -6,7 +6,6 @@ //! //! Field-extraction logic is vendored from instinct-dash `DockerService.ts` //! (HIP_VISIBLE_DEVICES → gpu_ids, `--tensor-parallel-size`/`-tp`, etc.). -//! See `../../wiki/entities/dockerode.md`. //! //! The async inherent methods are the public API. The `ServiceDiscovery` trait //! impl is sync-only and returns `Unsupported`: bollard requires a tokio @@ -295,6 +294,7 @@ fn parse_container(inspect: &ContainerInspectResponse) -> Result>, } -/// `/api/v1/health` response — liveness + the most-recently-loaded model name. +/// Lemonade health response. `all_models_loaded`, when present, is the +/// authoritative loaded set. Older servers report only `model_loaded`. #[derive(Debug, Clone, Default, Deserialize)] pub struct LemonadeHealth { #[serde(default)] pub status: Option, #[serde(default)] pub model_loaded: Option, + #[serde(default)] + pub all_models_loaded: Option>, +} + +impl LemonadeHealth { + /// Sorted, deduplicated loaded model names, only from a healthy response. + /// An empty router is healthy but does not constitute a serving model. + pub fn loaded_models(&self) -> Vec { + if self.status.as_deref() != Some("ok") { + return Vec::new(); + } + let names: Vec<&str> = match &self.all_models_loaded { + Some(models) => models + .iter() + .filter_map(|model| { + model.as_str().or_else(|| { + ["model_name", "id", "name"] + .into_iter() + .filter_map(|field| { + model.get(field).and_then(serde_json::Value::as_str) + }) + .find(|name| !name.trim().is_empty()) + }) + }) + .collect(), + None => self.model_loaded.as_deref().into_iter().collect(), + }; + names + .into_iter() + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + .collect::>() + .into_iter() + .collect() + } + + /// Singleton attribution for server-global last-request stats. Health's + /// `model_loaded` means last access (load OR inference), not last request, + /// so it cannot disambiguate multiple loaded models. A conflicting explicit + /// name also rejects the singleton fallback. + pub fn stats_model(&self) -> Option { + let loaded = self.loaded_models(); + if loaded.len() != 1 { + return None; + } + if let Some(last) = self + .model_loaded + .as_deref() + .map(str::trim) + .filter(|name| !name.is_empty()) + { + return loaded.into_iter().find(|name| name == last); + } + loaded.into_iter().next() + } +} + +/// Map a failed request to a [`CollectorError`]. A refused or unanswered +/// connection becomes `Io(ConnectionRefused)`, the typed marker for "nothing is +/// listening" that [`is_connect_error`] reads; every other failure stays a +/// `Transport` error. A timeout counts as unanswered because Windows does not +/// refuse a connection to a closed local port: the attempt hangs until the +/// request timeout. +fn send_error(url: &str, error: &reqwest::Error) -> CollectorError { + let message = format!("GET {url}: {error}"); + if error.is_connect() || error.is_timeout() { + CollectorError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionRefused, + message, + )) + } else { + CollectorError::Transport(message) + } +} + +/// True when `error` means no server accepted the connection, as opposed to a +/// server that answered with an HTTP status or a broken body. +pub fn is_connect_error(error: &CollectorError) -> bool { + matches!(error, CollectorError::Io(io) if io.kind() == std::io::ErrorKind::ConnectionRefused) } /// PURE: parse a `/api/v1/stats` body into an [`InstanceSample`]. /// -/// Lemonade reports an instantaneous `tokens_per_second` rate (mapped to `gen_tps`) rather than a +/// Lemonade reports the `tokens_per_second` rate of the most recent request (mapped to `gen_tps`; +/// it is not a live rate: the same value is reported until another request completes, so the +/// dashboard keeps showing the last request's rate until a new request replaces it) rather than a /// cumulative counter; it exposes no KV-cache / running / waiting metrics, so /// those stay `None`. Malformed/empty JSON → an all-`None` default, never panics. pub fn parse_stats(body: &str) -> InstanceSample { @@ -90,11 +172,18 @@ pub fn parse_stats_struct(body: &str) -> LemonadeStats { object_or_default(body) } -/// PURE: extract the loaded model name from a `/api/v1/health` body, if present -/// and non-empty. Malformed JSON → `None`, never panics. -pub fn parse_health_model(body: &str) -> Option { - let health: LemonadeHealth = object_or_default(body); - health.model_loaded.filter(|s| !s.is_empty()) +/// PURE: parse a health body, rejecting invalid/wrong-shape input to default. +pub fn parse_health(body: &str) -> LemonadeHealth { + let Ok(value @ serde_json::Value::Object(_)) = serde_json::from_str(body) else { + return LemonadeHealth::default(); + }; + if value + .get("all_models_loaded") + .is_some_and(|models| !models.is_array() && !models.is_null()) + { + return LemonadeHealth::default(); + } + serde_json::from_value(value).unwrap_or_default() } /// Deserialize a JSON **object** body into `T`, returning `T::default()` for any @@ -116,22 +205,27 @@ pub fn lemonade_container_id(host: &str, port: u16) -> String { format!("lemonade-{host}-{port}") } -/// PURE: build a `DiscoveredService` for a Lemonade endpoint. Lemonade is a -/// single local server (no tensor-parallel sharding metadata), so TP = 1. +/// A model identity within an external endpoint. Encoding the model's UTF-8 +/// bytes avoids separator collisions and keeps identity stable across reorder. +pub fn lemonade_model_id(host: &str, port: u16, model_name: &str) -> String { + use std::fmt::Write as _; + let mut id = format!("{}-model-", lemonade_container_id(host, port)); + for byte in model_name.as_bytes() { + let _ = write!(id, "{byte:02x}"); + } + id +} + +/// PURE: build metadata for an external loaded model. /// -/// Every call site only constructs this after `/api/v1/health` has already -/// answered (see `LemonadeCollector::discover`), so a successfully-built -/// service is by definition reachable and serving — mark it `Ready` rather -/// than the generic `Running`. +/// Callers must establish healthy loaded-model evidence first; answering HTTP +/// alone is insufficient. `model_name` must be a trimmed, non-empty name, as +/// produced by [`LemonadeHealth::loaded_models`]. Lemonade exposes no TP metadata. pub fn lemonade_service(host: &str, port: u16, model_name: &str) -> DiscoveredService { DiscoveredService { - container_id: lemonade_container_id(host, port), + container_id: lemonade_model_id(host, port, model_name), container_name: "lemonade".to_string(), - model_name: if model_name.is_empty() { - "lemonade".to_string() - } else { - model_name.to_string() - }, + model_name: model_name.to_string(), port: Some(port), tensor_parallel_size: 1, status: rocm_dash_core::metrics::InstanceStatus::Ready, @@ -139,18 +233,23 @@ pub fn lemonade_service(host: &str, port: u16, model_name: &str) -> DiscoveredSe } } -/// PURE: build a finished `Instance` from a Lemonade `/health` model name + a `/stats` body. -/// -/// This is the fixture→Instance anchor (no network). `gen_tps` flows from -/// `tokens_per_second`; KV/req fields stay `None` (Lemonade does not report them). -pub fn lemonade_instance(host: &str, port: u16, model_name: &str, stats_body: &str) -> Instance { - let svc = lemonade_service(host, port, model_name); - let sample = parse_stats(stats_body); - merge_instance(&svc, &sample, 0, 0) +/// PURE: map a health payload to one discovered service per loaded model. +/// Invalid, unhealthy or empty payloads yield no rows. The daemon runner and +/// the tests both use this single mapping. +pub fn services_from_health( + host: &str, + port: u16, + health: &LemonadeHealth, +) -> Vec { + health + .loaded_models() + .iter() + .map(|model| lemonade_service(host, port, model)) + .collect() } /// Async scrape of a local Lemonade endpoint. Network/parse failure degrades to -/// `Err`/`None` ("not reachable") — never a panic. The pure parsers above do the +/// transport errors or an empty loaded set, never a panic. The pure parsers above do the /// actual mapping; this only does I/O. #[derive(Debug, Clone)] pub struct LemonadeCollector { @@ -173,17 +272,47 @@ impl LemonadeCollector { } fn url(&self, path: &str) -> String { - format!("http://{}:{}{path}", self.host, self.port) + let host = if self.host.contains(':') && !self.host.starts_with('[') { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + format!("http://{host}:{}{path}", self.port) } - async fn get_text(&self, path: &str) -> Result { - let url = self.url(path); - let resp = self + /// GET `path`, falling back to the legacy route on a 404. `Ok(None)` means + /// neither route exists: the server does not speak the Lemonade protocol. + async fn get_text_opt(&self, path: &str, auth_optional: bool) -> Result> { + let mut url = self.url(path); + let mut resp = self .client .get(&url) .send() .await - .map_err(|e| CollectorError::Transport(format!("GET {url}: {e}")))?; + .map_err(|e| send_error(&url, &e))?; + // Router versions expose either /api/v1 or /v1. Only a missing route + // warrants compatibility fallback, never an authentication or health + // failure. This does not add a llama-server metrics protocol. + if resp.status() == reqwest::StatusCode::NOT_FOUND + && let Some(legacy_path) = path.strip_prefix("/api") + { + url = self.url(legacy_path); + resp = self + .client + .get(&url) + .send() + .await + .map_err(|e| send_error(&url, &e))?; + } + if resp.status() == reqwest::StatusCode::NOT_FOUND + || (auth_optional + && matches!( + resp.status(), + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN + )) + { + return Ok(None); + } if !resp.status().is_success() { return Err(CollectorError::Transport(format!( "GET {url}: status {}", @@ -192,40 +321,76 @@ impl LemonadeCollector { } resp.text() .await + .map(Some) .map_err(|e| CollectorError::Transport(format!("body {url}: {e}"))) } + async fn get_text(&self, path: &str) -> Result { + self.get_text_opt(path, false).await?.ok_or_else(|| { + CollectorError::Transport(format!("GET {}: status 404 Not Found", self.url(path))) + }) + } + /// Scrape `/api/v1/stats` → an `InstanceSample` (rate → `gen_tps`). pub async fn fetch_stats(&self) -> Result { Ok(parse_stats(&self.get_text(LEMONADE_STATS_PATH).await?)) } - /// Scrape `/api/v1/health` → the loaded model name (if any). - pub async fn fetch_health_model(&self) -> Result> { - Ok(parse_health_model( - &self.get_text(LEMONADE_HEALTH_PATH).await?, - )) + /// Registry-target lookup: `Ok(None)` if the Lemonade health route is absent + /// or requires credentials (401/403). The registry still supplies ownership + /// and status; this does not establish health or permit a stats scrape. + /// Transport failures and 5xx responses are still errors. External discovery + /// uses [`Self::fetch_health`] instead, where auth rejection remains an error. + pub async fn fetch_health_if_router(&self) -> Result> { + Ok(self + .get_text_opt(LEMONADE_HEALTH_PATH, true) + .await? + .map(|body| parse_health(&body))) } - /// Probe the endpoint: if `/api/v1/health` answers, return a `DiscoveredService` - /// for it (model name from health, falling back to "lemonade"); otherwise - /// `None` (endpoint absent/unreachable) — a clean no-op, no panic. - pub async fn discover(&self) -> Option { - match self.fetch_health_model().await { - Ok(model) => Some(lemonade_service( - &self.host, - self.port, - model.as_deref().unwrap_or(""), - )), - Err(_) => None, - } + /// Fetch the health payload. Invalid payloads carry no loaded models. + pub async fn fetch_health(&self) -> Result { + Ok(parse_health(&self.get_text(LEMONADE_HEALTH_PATH).await?)) } } #[cfg(test)] mod tests { + /// The dashboard config's default Lemonade port and this crate's constant are + /// two literals in two crates (this crate depends on `rocm-core`, not the + /// reverse), so nothing but this test keeps them equal. + #[test] + fn the_dashboard_default_port_is_the_collectors_lemonade_port() { + assert_eq!( + super::LEMONADE_PORT, + rocm_core::DashboardDaemonConfig::default().lemonade_port + ); + } + use super::*; + /// An IPv6 host is bracketed in the URL exactly once; IPv4 and names are not. + /// Pure, so it runs on hosts where the daemon's IPv6 tests cannot bind. + #[test] + fn url_brackets_an_ipv6_host_once() { + let url = |host: &str| { + LemonadeCollector::new(host, 13305, std::time::Duration::from_secs(1)) + .url("/api/v1/health") + }; + assert_eq!(url("::1"), "http://[::1]:13305/api/v1/health"); + assert_eq!(url("[::1]"), "http://[::1]:13305/api/v1/health"); + assert_eq!(url("127.0.0.1"), "http://127.0.0.1:13305/api/v1/health"); + assert_eq!(url("localhost"), "http://localhost:13305/api/v1/health"); + } + + /// Same path the daemon runner takes: health fetch, then the shared mapping. + async fn discovered(collector: &LemonadeCollector) -> Vec { + match collector.fetch_health().await { + Ok(health) => services_from_health(&collector.host, collector.port, &health), + Err(_) => Vec::new(), + } + } + // Representative `/api/v1/stats` response (lemonade-server.ai/docs/api). const STATS_FIXTURE: &str = r#"{ "time_to_first_token": 2.14, @@ -241,9 +406,149 @@ mod tests { "status": "ok", "version": "9.3.3", "model_loaded": "Llama-3.2-1B-Instruct-Hybrid", - "all_models_loaded": [] + "all_models_loaded": [{"model_name":"Llama-3.2-1B-Instruct-Hybrid"}] }"#; + #[test] + fn loaded_set_and_stats_attribution_are_conservative() { + let health = parse_health( + r#"{"status":"ok","all_models_loaded":[" Model-B ",{"id":"Model-A"},{"name":"Model-B"},{"model_name":"","id":"Model-C"},null,7,{}]}"#, + ); + assert_eq!(health.loaded_models(), ["Model-A", "Model-B", "Model-C"]); + assert_eq!(health.stats_model(), None); + let health = parse_health( + r#"{"status":"ok","model_loaded":"Model-B","all_models_loaded":["Model-A","Model-B"]}"#, + ); + assert_eq!( + health.stats_model(), + None, + "last access can be a load, not a request" + ); + for body in [ + r#"{"status":"ok","all_models_loaded":["Model-A"]}"#, + r#"{"status":"ok","model_loaded":"Model-A"}"#, + ] { + assert_eq!(parse_health(body).stats_model().as_deref(), Some("Model-A")); + } + assert_eq!( + parse_health( + r#"{"status":"ok","model_loaded":"Model-B","all_models_loaded":["Model-A"]}"# + ) + .stats_model(), + None + ); + // Names are compared exactly, never case-folded: the server treats `Model-A` + // and `model-a` as two models, so they are two rows, and a last-request rate + // whose name differs only in case is not attributed to either. + let health = parse_health(r#"{"status":"ok","all_models_loaded":["Model-A","model-a"]}"#); + assert_eq!(health.loaded_models(), ["Model-A", "model-a"]); + assert_eq!(health.stats_model(), None); + assert_eq!( + parse_health( + r#"{"status":"ok","model_loaded":"model-a","all_models_loaded":["Model-A"]}"# + ) + .stats_model(), + None + ); + for body in [ + r#"{"status":"ok","all_models_loaded":{},"model_loaded":"Model-A"}"#, + r#"{"status":"bad","model_loaded":"Model-A"}"#, + r#"{"model_loaded":"Model-A"}"#, + ] { + assert!(parse_health(body).loaded_models().is_empty()); + } + } + + #[tokio::test] + async fn health_if_router_distinguishes_missing_route_from_failure() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + // A plain llama-server: neither Lemonade health route exists. + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/health")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"status":"ok"}"#)) + .mount(&server) + .await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + assert!( + collector + .fetch_health_if_router() + .await + .expect("missing routes are not an error") + .is_none() + ); + assert!(collector.fetch_health().await.is_err()); + // A router answering on the legacy route is still a router. + server.reset().await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(r#"{"status":"ok","model_loaded":"Model-A"}"#), + ) + .mount(&server) + .await; + let health = collector.fetch_health_if_router().await.unwrap().unwrap(); + assert_eq!(health.loaded_models(), ["Model-A"]); + // Only registry lookup tolerates auth rejection, without claiming health. + for status in [401, 403] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(status)) + .mount(&server) + .await; + assert!(collector.fetch_health_if_router().await.unwrap().is_none()); + assert!(collector.fetch_health().await.is_err()); + } + // Unrelated server failures must still warn. + for status in [500, 503] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(status)) + .mount(&server) + .await; + assert!( + collector.fetch_health_if_router().await.is_err(), + "status {status} must stay an error" + ); + } + } + + #[tokio::test] + async fn health_http_failure_never_falls_back_to_successful_route() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + for status in [401, 403, 500, 503] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(status)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(r#"{"status":"ok","model_loaded":"Model-A"}"#), + ) + .expect(0) + .mount(&server) + .await; + let collector = LemonadeCollector::new( + "127.0.0.1", + server.address().port(), + Duration::from_secs(1), + ); + assert!(discovered(&collector).await.is_empty()); + server.verify().await; + } + } + #[test] fn parse_stats_maps_tokens_per_second_to_gen_tps() { let sample = parse_stats(STATS_FIXTURE); @@ -267,13 +572,30 @@ mod tests { } #[test] - fn parse_health_model_extracts_loaded_model() { + fn parse_health_extracts_loaded_model() { assert_eq!( - parse_health_model(HEALTH_FIXTURE).as_deref(), + parse_health(HEALTH_FIXTURE).stats_model().as_deref(), Some("Llama-3.2-1B-Instruct-Hybrid") ); } + #[test] + fn a_null_model_list_keeps_the_rest_of_the_health_body() { + // An unset optional field is an explicit `null` for FastAPI/Pydantic + // servers; the body is otherwise healthy and must not be discarded. + let health = + parse_health(r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":null}"#); + assert_eq!(health.status.as_deref(), Some("ok")); + assert_eq!(health.loaded_models().len(), 1); + // A present but wrong-shape list is still rejected whole. + for body in [ + r#"{"status":"ok","all_models_loaded":{}}"#, + r#"{"status":"ok","all_models_loaded":"x"}"#, + ] { + assert_eq!(parse_health(body).status, None, "body {body}"); + } + } + #[test] fn malformed_or_empty_input_is_graceful_not_panic() { // Garbage / empty / wrong-shape → all-None sample, no panic. @@ -288,19 +610,27 @@ mod tests { assert_eq!(sample.gen_tps, None, "body {body:?}"); assert!(sample.kv_cache_usage_pct.is_none()); } - assert_eq!(parse_health_model("not json"), None); - assert_eq!(parse_health_model(r#"{"model_loaded": ""}"#), None); + assert!(parse_health("not json").loaded_models().is_empty()); + assert!( + parse_health(r#"{"model_loaded": ""}"#) + .loaded_models() + .is_empty() + ); } #[test] fn fixture_stats_plus_health_becomes_an_instance() { // The fixture→Instance anchor: no network. Health gives the model name, // /stats gives the live rate → an Instances-tab row. - let model = parse_health_model(HEALTH_FIXTURE).expect("model"); - let inst = lemonade_instance("127.0.0.1", LEMONADE_PORT, &model, STATS_FIXTURE); + let model = parse_health(HEALTH_FIXTURE).stats_model().expect("model"); + let svc = lemonade_service("127.0.0.1", LEMONADE_PORT, &model); + let inst = rocm_dash_core::traits::merge_instance(&svc, &parse_stats(STATS_FIXTURE), 0, 0); assert_eq!(inst.model_name, "Llama-3.2-1B-Instruct-Hybrid"); assert_eq!(inst.container_name, "lemonade"); - assert_eq!(inst.container_id, "lemonade-127.0.0.1-13305"); + assert_eq!( + inst.container_id, + lemonade_model_id("127.0.0.1", 13305, &model) + ); assert_eq!(inst.port, Some(13305)); assert_eq!(inst.gen_tps, Some(33.33)); assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready); @@ -310,14 +640,177 @@ mod tests { } #[tokio::test] - async fn discover_is_clean_noop_when_endpoint_absent() { - // Probe a port with no server → connection refused → None, no panic. - // (Port 0 is never a live listener; the GET fails fast.) - let collector = LemonadeCollector::new("127.0.0.1", 1, Duration::from_millis(200)); - assert!(collector.discover().await.is_none()); + async fn discovery_rejects_invalid_health_and_healthy_empty_router() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + for body in [ + "not json", + "{}", + "[]", + "null", + r#"{"status":"error","model_loaded":"Model-A"}"#, + r#"{"status":"ok","all_models_loaded":[]}"#, + r#"{"status":"ok","all_models_loaded":[],"model_loaded":"Old-Model"}"#, + r#"{"status":"ok","model_loaded":" "}"#, + ] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200).set_body_string(body)) + .mount(&server) + .await; + assert!( + discovered(&collector).await.is_empty(), + "health {body:?} must not advertise a serving model" + ); + } + } + + #[tokio::test] + async fn discovery_recognizes_modern_loaded_model_without_legacy_field() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200).set_body_string( + r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"}]}"#, + )) + .mount(&server) + .await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + assert_eq!(discovered(&collector).await[0].model_name, "Model-A"); + } + + #[tokio::test] + async fn discovery_accepts_v1_health_only_when_api_prefix_is_absent() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(r#"{"status":"ok","model_loaded":"Model-A"}"#), + ) + .mount(&server) + .await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + assert_eq!(discovered(&collector).await[0].model_name, "Model-A"); + } + + #[tokio::test] + async fn v1_fallback_status_failure_names_the_requested_v1_url() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let port = server.address().port(); + let collector = LemonadeCollector::new("127.0.0.1", port, Duration::from_secs(1)); + let error = collector.fetch_health().await.unwrap_err().to_string(); + let wanted = format!("http://127.0.0.1:{port}/v1/health"); + assert!(error.contains(&wanted), "{error}"); + assert!(!error.contains("/api/v1/health"), "{error}"); + } + + #[tokio::test] + async fn v1_fallback_transport_failure_names_host_port_and_path() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + // 404 on the primary route, then the retry is cut off by the client + // timeout so it fails at the transport level. + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(2))) + .mount(&server) + .await; + let port = server.address().port(); + let collector = LemonadeCollector::new("127.0.0.1", port, Duration::from_millis(200)); + let error = collector.fetch_health().await.unwrap_err().to_string(); + let wanted = format!("GET http://127.0.0.1:{port}/v1/health"); + assert!(error.contains(&wanted), "{error}"); + } + + #[tokio::test] + async fn health_and_stats_return_transport_errors_when_endpoint_absent() { + // Reserve then release a random port: an absent listener yields health + // and stats transport errors, never a panic. + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let collector = LemonadeCollector::new("127.0.0.1", port, Duration::from_millis(200)); + assert!(collector.fetch_health().await.is_err()); assert!(collector.fetch_stats().await.is_err()); } + /// Windows does not refuse a connection to a closed local port: the attempt + /// hangs until the request timeout. Both outcomes mean nothing answered, so + /// both must read as "absent" and not as a broken server. + #[tokio::test] + async fn closed_and_unanswered_endpoints_both_read_as_nothing_listening() { + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let closed_port = closed.local_addr().unwrap().port(); + drop(closed); + let refused = LemonadeCollector::new("127.0.0.1", closed_port, Duration::from_millis(200)) + .fetch_health() + .await + .unwrap_err(); + assert!(is_connect_error(&refused), "{refused}"); + + // Accepts the connection at the kernel level but never answers. + let silent = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let silent_port = silent.local_addr().unwrap().port(); + let timed_out = + LemonadeCollector::new("127.0.0.1", silent_port, Duration::from_millis(200)) + .fetch_health() + .await + .unwrap_err(); + assert!(is_connect_error(&timed_out), "{timed_out}"); + drop(silent); + } + + /// An HTTP status error is a server that answered, so it is not "absent". + #[tokio::test] + async fn an_answering_server_with_an_error_status_is_not_nothing_listening() { + use wiremock::matchers::method; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let error = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)) + .fetch_health() + .await + .unwrap_err(); + assert!(!is_connect_error(&error), "{error}"); + } + /// Integration-gated: scrape a REAL local Lemonade server (install the engine /// with `rocm engines install lemonade` and start it first). Not run in CI. #[tokio::test] @@ -325,7 +818,8 @@ mod tests { async fn live_lemonade_scrape_produces_instance() { let collector = LemonadeCollector::new("127.0.0.1", LEMONADE_PORT, Duration::from_millis(1500)); - let svc = collector.discover().await.expect("lemonade reachable"); + let svcs = discovered(&collector).await; + let svc = svcs.first().expect("lemonade reachable"); assert_eq!(svc.port, Some(LEMONADE_PORT)); // A stats scrape returns a sample (gen_tps may be None until an inference). let _sample = collector.fetch_stats().await.expect("stats scrape"); diff --git a/crates/rocm-dash-collectors/src/lib.rs b/crates/rocm-dash-collectors/src/lib.rs index a58cdf851a..f196f3d7f5 100644 --- a/crates/rocm-dash-collectors/src/lib.rs +++ b/crates/rocm-dash-collectors/src/lib.rs @@ -5,9 +5,9 @@ //! IO-heavy collector implementations. Used by `rocm-dash-daemon` and, on Windows, `rocm-dash-tui`. //! //! Every collector implements one of the traits in `rocm_dash_core::traits`. -//! Stubs return `CollectorError::Unsupported` so the daemon can start with nothing wired. pub mod amd_smi; +pub mod ancestry; pub mod bench_load; pub mod bench_tail; pub mod cgroup; @@ -16,8 +16,5 @@ pub mod engine_registry; pub mod host; pub mod lemonade; pub mod parallel; -pub mod proc_scan; -pub mod sysfs; -pub mod vllm_log; pub mod vllm_prom; pub mod winperf; diff --git a/crates/rocm-dash-collectors/src/parallel.rs b/crates/rocm-dash-collectors/src/parallel.rs index 0ad1d68f9a..cfe79b221f 100644 --- a/crates/rocm-dash-collectors/src/parallel.rs +++ b/crates/rocm-dash-collectors/src/parallel.rs @@ -4,10 +4,9 @@ //! Shared primitives for async collectors that scrape N targets concurrently. //! -//! Today: [[VllmPrometheusCollector]] uses this pattern hand-rolled in the -//! daemon's runner. As future per-instance scrapers come online, they'll reuse -//! `parallel_scrape` and the `WarningBus` instead of re-implementing the -//! JoinSet glue. +//! Today: the daemon's runner calls `parallel_scrape` for the vLLM Prometheus +//! scrape. Future per-instance scrapers should reuse `parallel_scrape` instead +//! of re-implementing the JoinSet glue. use std::future::Future; use std::sync::Arc; diff --git a/crates/rocm-dash-collectors/src/proc_scan.rs b/crates/rocm-dash-collectors/src/proc_scan.rs deleted file mode 100644 index ea4c4053fd..0000000000 --- a/crates/rocm-dash-collectors/src/proc_scan.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// -// SPDX-License-Identifier: MIT - -//! /proc scan for non-container deployments (Strix Halo). Stub. - -use rocm_dash_core::traits::{CollectorError, DiscoveredService, Result, ServiceDiscovery}; - -#[derive(Debug, Default)] -pub struct ProcDiscovery; - -impl ProcDiscovery { - pub const fn new() -> Self { - Self - } -} - -impl ServiceDiscovery for ProcDiscovery { - fn name(&self) -> &'static str { - "proc" - } - - fn discover(&self) -> Result> { - // TODO: walk /proc, match comm == "llama-server" | "lemonade-server". - Err(CollectorError::Unsupported("proc discovery stub".into())) - } -} diff --git a/crates/rocm-dash-collectors/src/sysfs.rs b/crates/rocm-dash-collectors/src/sysfs.rs deleted file mode 100644 index d6381fbd17..0000000000 --- a/crates/rocm-dash-collectors/src/sysfs.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// -// SPDX-License-Identifier: MIT - -//! sysfs/hwmon collector for Strix Halo (gfx1151). Stub. - -use rocm_dash_core::metrics::{GpuMetrics, GpuSystemInfo}; -use rocm_dash_core::traits::{CollectorError, GpuCollector, GpuDevice, GpuProcess, Result}; - -#[derive(Debug, Default)] -pub struct SysfsGpuCollector; - -impl SysfsGpuCollector { - pub const fn new() -> Self { - Self - } -} - -impl GpuCollector for SysfsGpuCollector { - fn name(&self) -> &'static str { - "sysfs" - } - - fn devices(&self) -> Result> { - Err(CollectorError::Unsupported("sysfs collector stub".into())) - } - - fn metrics(&self) -> Result> { - Err(CollectorError::Unsupported("sysfs collector stub".into())) - } - - fn system_info(&self) -> Result { - Err(CollectorError::Unsupported("sysfs collector stub".into())) - } - - fn processes(&self) -> Result> { - Err(CollectorError::Unsupported("sysfs collector stub".into())) - } -} diff --git a/crates/rocm-dash-collectors/src/vllm_log.rs b/crates/rocm-dash-collectors/src/vllm_log.rs deleted file mode 100644 index a501d72fb0..0000000000 --- a/crates/rocm-dash-collectors/src/vllm_log.rs +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// -// SPDX-License-Identifier: MIT - -//! vLLM log slicer — verbatim port of `inspect_bench/log_slicer.py`. -//! See `../wiki/entities/log-slicer.md` and `../wiki/concepts/log-derived-metrics.md`. - -use regex::Regex; - -#[derive(Debug, Clone, Default, PartialEq)] -pub struct VllmPeaks { - pub prompt_tps: Option, - pub gen_tps: Option, - pub max_running_reqs: Option, - pub max_waiting_reqs: Option, - pub n_requests: u32, -} - -#[allow(clippy::struct_field_names)] // `_re` suffix is meaningful (these are Regex fields) -pub struct VllmLogSlicer { - prompt_tps_re: Regex, - gen_tps_re: Regex, - running_re: Regex, - waiting_re: Regex, - req_re: Regex, -} - -impl Default for VllmLogSlicer { - fn default() -> Self { - Self::new() - } -} - -impl VllmLogSlicer { - pub fn new() -> Self { - Self { - prompt_tps_re: Regex::new(r"Avg prompt throughput: ([0-9.]+)").unwrap(), - gen_tps_re: Regex::new(r"Avg generation throughput: ([0-9.]+)").unwrap(), - running_re: Regex::new(r"Running: ([0-9]+)").unwrap(), - waiting_re: Regex::new(r"Waiting: ([0-9]+)").unwrap(), - req_re: Regex::new(r"POST /v1/(chat/completions|completions|messages)").unwrap(), - } - } - - /// Aggregate peaks over a byte-bounded slice of the vLLM log. - pub fn parse_slice(&self, text: &str) -> VllmPeaks { - let mut out = VllmPeaks::default(); - for cap in self.prompt_tps_re.captures_iter(text) { - if let Ok(v) = cap[1].parse::() { - out.prompt_tps = Some(out.prompt_tps.map_or(v, |p| p.max(v))); - } - } - for cap in self.gen_tps_re.captures_iter(text) { - if let Ok(v) = cap[1].parse::() { - out.gen_tps = Some(out.gen_tps.map_or(v, |p| p.max(v))); - } - } - for cap in self.running_re.captures_iter(text) { - if let Ok(v) = cap[1].parse::() { - out.max_running_reqs = Some(out.max_running_reqs.map_or(v, |p| p.max(v))); - } - } - for cap in self.waiting_re.captures_iter(text) { - if let Ok(v) = cap[1].parse::() { - out.max_waiting_reqs = Some(out.max_waiting_reqs.map_or(v, |p| p.max(v))); - } - } - out.n_requests = self.req_re.find_iter(text).count() as u32; - out - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_throughput_peaks_and_request_count() { - let log = r" -[2026-05-26 10:00:00] Avg prompt throughput: 1234.5 tokens/s -[2026-05-26 10:00:01] Avg generation throughput: 67.8 tokens/s -[2026-05-26 10:00:02] Running: 8, Waiting: 2 -[2026-05-26 10:00:03] Avg prompt throughput: 2000.1 tokens/s -[2026-05-26 10:00:03] Avg generation throughput: 50.0 tokens/s -[2026-05-26 10:00:04] Running: 16, Waiting: 0 -INFO POST /v1/chat/completions 200 -INFO POST /v1/completions 200 -INFO POST /v1/messages 200 -INFO POST /v1/chat/completions 200 -"; - let slicer = VllmLogSlicer::new(); - let peaks = slicer.parse_slice(log); - assert_eq!(peaks.prompt_tps, Some(2000.1)); - assert_eq!(peaks.gen_tps, Some(67.8)); - assert_eq!(peaks.max_running_reqs, Some(16)); - assert_eq!(peaks.max_waiting_reqs, Some(2)); - assert_eq!(peaks.n_requests, 4); - } - - #[test] - fn empty_log_is_all_none() { - let slicer = VllmLogSlicer::new(); - let peaks = slicer.parse_slice(""); - assert_eq!(peaks, VllmPeaks::default()); - } -} diff --git a/crates/rocm-dash-collectors/src/vllm_prom.rs b/crates/rocm-dash-collectors/src/vllm_prom.rs index 0fb6e2eb76..1aa184d151 100644 --- a/crates/rocm-dash-collectors/src/vllm_prom.rs +++ b/crates/rocm-dash-collectors/src/vllm_prom.rs @@ -5,7 +5,7 @@ //! vLLM Prometheus `/metrics` scraper. //! //! Field paths and the kv-cache 0..1 → 0..100 scaling are vendored from -//! instinct-dash `VllmMetricsService.ts`. See `../../wiki/entities/vllm.md`. +//! instinct-dash `VllmMetricsService.ts`. //! //! Sync `InstanceMetrics::fetch` returns `Unsupported` (the underlying client //! is async); call `fetch_async` from the runner. diff --git a/crates/rocm-dash-collectors/src/winperf.rs b/crates/rocm-dash-collectors/src/winperf.rs index 7114f2c1cb..36d03aceb6 100644 --- a/crates/rocm-dash-collectors/src/winperf.rs +++ b/crates/rocm-dash-collectors/src/winperf.rs @@ -42,10 +42,12 @@ use tokio::time::timeout; // actually-hung process forever. const RUN_TIMEOUT: Duration = Duration::from_secs(15); -/// ADLX init/read/terminate shouldn't normally take anywhere near this, but a -/// wedged driver call must not freeze CPU/memory updates forever alongside it -/// — same reasoning as `RUN_TIMEOUT`, just a tighter bound since there's no -/// process spawn latency to absorb here. +/// Bounds the whole request to the resident reader — handing the request +/// over, the driver read on that thread, and the answer coming back. None of +/// it should take anywhere near this, but a wedged driver call must not +/// freeze CPU/memory updates forever alongside it — same reasoning as +/// `RUN_TIMEOUT`, just a tighter bound since there's no process spawn latency +/// to absorb here. const ADLX_TIMEOUT: Duration = Duration::from_secs(5); /// `kindkeyvalue[value2]` lines: per-engine utilization, @@ -79,32 +81,22 @@ impl WinPerfCollector { /// `counter_samples_missing_warning`) without failing the whole tick. pub async fn snapshot(&self) -> io::Result<(Vec, GpuSystemInfo, Vec)> { let out = Self::run_script().await?; - // adlx_readings() calls into amdadlx64.dll via FFI, which blocks on - // driver I/O — running it directly here would stall a tokio worker - // thread, so it runs under spawn_blocking. A native call cannot be - // cancelled, so the timeout below only stops *waiting*: a wedged - // call keeps its blocking-pool thread. `run_single_flight` therefore - // refuses to start another call while one is still running, which - // bounds the leak to that one thread instead of one per tick. A - // panic, timeout or still-wedged call surfaces as a warning; a - // driver that is absent or fails to initialise inside - // adlx_readings() degrades to no readings with no warning, since - // "no AMD driver" is an expected state indistinguishable from it. + // adlx_readings() hands its request to the resident ADLX reader + // thread and blocks until that thread answers — the FFI into + // amdadlx64.dll runs there, not here. Blocking at all would stall a + // tokio worker, so it still runs under spawn_blocking. A native call + // cannot be cancelled, so the timeout below only stops *waiting*: a + // wedged driver keeps both this blocking-pool thread and the one + // reader. `run_single_flight` therefore refuses to start another call + // while one is still running, which bounds that to a single + // outstanding request instead of one per tick. + // + // A panic, timeout or still-wedged call surfaces as a warning, as + // does the reader failing or dying (ReaderGone). A driver that is + // absent or fails to initialise answers Ok with no readings and no + // warning, since "no AMD driver" is an expected state. let (adlx, adlx_warning) = - match run_single_flight(&ADLX_IN_FLIGHT, ADLX_TIMEOUT, adlx_readings).await { - SingleFlight::Done(readings) => (readings, None), - SingleFlight::Panicked => { - (Vec::new(), Some("ADLX reading task panicked".to_string())) - } - SingleFlight::TimedOut => ( - Vec::new(), - Some(format!("ADLX query timed out after {ADLX_TIMEOUT:?}")), - ), - SingleFlight::StillRunning => ( - Vec::new(), - Some("ADLX query skipped: a previous call is still running".to_string()), - ), - }; + adlx_outcome(run_single_flight(&ADLX_IN_FLIGHT, ADLX_TIMEOUT, adlx_readings).await); let metrics = parse_metrics(&out, &adlx); let warnings = counter_samples_missing_warning(&out, &metrics) .into_iter() @@ -409,44 +401,215 @@ fn apply_adlx(reading: Option<&AdlxReading>) -> (f32, f32, Option) { ) } +/// Owns a value that is built once, on its own thread, and serves every read +/// from that thread for as long as the process lives. +/// +/// ADLX is meant to be initialised once per process. The `adlx` crate's +/// `AdlxHelper` loads `amdadlx64.dll` and calls `ADLXInitialize` in `new()`, +/// and `ADLXTerminate` plus the DLL unload in `Drop`. Doing that every tick, +/// each time on a different blocking-pool thread, loads and unloads +/// `amdadlx64.dll` (and the `d3d9`/`OPENGL32`/`dbghelp` it pulls in) over and +/// over while the driver's `atiadlxx.dll` stays resident. After a while +/// `atiadlxx.dll` fail-fasts (`0xC0000409`, `FAST_FAIL_FATAL_APP_EXIT`) +/// and takes `rocm dash` down with no panic message or stack trace. +/// +/// `init` and `read` are parameters so the "built once, same thread for every +/// read" contract is unit-testable without a driver (see the tests). +#[cfg(any(windows, test))] +struct ResidentReader { + requests: std::sync::mpsc::Sender>, +} + +/// Why the reader could not answer, as distinct from answering with no +/// readings. The difference matters: no readings is what an absent driver +/// looks like and is expected, while these mean the reader itself is in +/// trouble and the user is told. Since the reader is built once per process +/// and never replaced, a failure that reported nothing would blank +/// temperature, power and clock for the rest of the session and look exactly +/// like a machine with no AMD driver. +/// +/// Not gated: `snapshot` matches on it and the non-Windows `adlx_readings` +/// stub returns it, on every platform. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReaderGone { + /// The read step panicked. Caught, so the reader survives for the next + /// tick; reported so a driver that panics every read is not silent. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] + ReadPanicked, + /// The init step panicked. Caught for the same reason as a read panic, and + /// kept apart from it because they are not the same fault: a read panic is + /// per-tick and the next read may succeed, while this one is terminal — + /// the state was never built and this design never rebuilds it. + /// + /// Before it was caught, an init panic unwound the reader thread outright, + /// so every later tick met a closed channel and reported + /// [`Self::ThreadGone`] — indistinguishable from the thread never having + /// started. Catching it keeps the thread alive to answer and names the real + /// cause. It does **not** restore the pre-resident-reader behaviour of + /// retrying init on the next tick; that is the deliberate trade this + /// design makes, and the reason the distinct variant is worth having. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] + InitPanicked, + /// The thread never started, or exited before answering. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] + ThreadGone, +} + +type ReaderAnswer = Result, ReaderGone>; + +/// What one ADLX attempt contributes to a snapshot: the readings to use, and +/// the warning to show the user, if any. +/// +/// Split out of [`WinPerfCollector::snapshot`] purely so it can be tested. +/// `snapshot` itself calls `Self::run_script()` unconditionally, which shells +/// out to PowerShell, so no test can drive it — the same impracticality +/// `adlx_readings`'s own FFI chain is documented with. That left the mapping +/// below, including the `Done(Err(_))` arm this change introduced, as the one +/// piece of new wiring nothing exercised. It is pure and total, so testing it +/// here costs nothing and covers every arm. +/// +/// The asymmetry worth keeping: a driver that is absent or failed to +/// initialise answers `Ok` with no readings and **no** warning, because "no AMD +/// driver" is an expected state rather than a fault. Everything else is a +/// fault, and says so. +fn adlx_outcome(outcome: SingleFlight) -> (Vec, Option) { + match outcome { + SingleFlight::Done(Ok(readings)) => (readings, None), + SingleFlight::Done(Err(gone)) => (Vec::new(), Some(gone.warning())), + SingleFlight::Panicked => (Vec::new(), Some("ADLX reading task panicked".to_string())), + SingleFlight::TimedOut => ( + Vec::new(), + Some(format!("ADLX query timed out after {ADLX_TIMEOUT:?}")), + ), + SingleFlight::StillRunning => ( + Vec::new(), + Some("ADLX query skipped: a previous call is still running".to_string()), + ), + } +} + +impl ReaderGone { + fn warning(self) -> String { + match self { + Self::ReadPanicked => "ADLX read panicked".to_string(), + Self::InitPanicked => "ADLX initialisation panicked".to_string(), + Self::ThreadGone => "the ADLX reader thread is gone".to_string(), + } + } +} + +#[cfg(any(windows, test))] +impl ResidentReader { + fn spawn( + init: impl FnOnce() -> Option + Send + 'static, + read: impl Fn(&S) -> Vec + Send + 'static, + ) -> Self { + let (requests, inbox) = std::sync::mpsc::channel::>(); + let spawned = std::thread::Builder::new() + .name("adlx-reader".into()) + .spawn(move || { + // `state` is deliberately held until the request channel closes + // (never, for the process-wide reader) so it is not torn down + // and re-initialised between ticks. + // + // Init is caught for the same reason the read below is, and the + // asymmetry mattered more here: a panic escaping this line + // unwound the thread before the request loop started, so + // `inbox` dropped and every later tick reported `ThreadGone` — + // the one state that cannot be told apart from "the thread + // never started". Catching keeps the reader answering, with a + // variant that names what actually happened. + let state = std::panic::catch_unwind(std::panic::AssertUnwindSafe(init)) + .map_err(|_| ReaderGone::InitPanicked); + for reply in inbox { + // A panicking read would otherwise unwind this thread and + // leave every later tick answering from a closed channel, + // with no way to tell that from an absent driver. Catching + // it keeps the one reader alive and names the failure. + let answer = match state.as_ref() { + Ok(Some(state)) => { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| read(state))) + .map_err(|_| ReaderGone::ReadPanicked) + } + Ok(None) => Ok(Vec::new()), + Err(gone) => Err(*gone), + }; + let _ = reply.send(answer); + } + }); + if let Err(e) = spawned { + // Without the thread `inbox` is dropped, so every `read` below + // sees a closed channel and degrades to no readings. + tracing::warn!("failed to start the ADLX reader thread: {e}"); + } + Self { requests } + } + + /// One reading per GPU, or an empty `Ok` when ADLX is unavailable or + /// failed to initialise — an absent driver is an expected state, not a + /// fault. [`ReaderGone`] is reserved for the reader itself failing. + /// Blocks until the reader answers, so callers run it under + /// `spawn_blocking`. + fn read(&self) -> ReaderAnswer { + let (reply, answer) = std::sync::mpsc::channel(); + if self.requests.send(reply).is_err() { + return Err(ReaderGone::ThreadGone); + } + answer.recv().unwrap_or(Err(ReaderGone::ThreadGone)) + } +} + /// `AdlxHelper::new()` fails cleanly (`Err`, not a panic) when -/// `amdadlx64.dll` isn't found, so a total init failure can just bail to an -/// empty Vec via `.ok()?` — same graceful-degrade shape as the rest of this -/// file. A single GPU's metrics query failing, though, must NOT drop that +/// `amdadlx64.dll` isn't found, so a total init failure leaves the reader +/// answering with an empty Vec — same graceful-degrade shape as the rest of +/// this file. A single GPU's metrics query failing, though, must NOT drop that /// GPU's slot (a `filter_map` here would silently change the reading count, /// which `parse_metrics`'s single-reading guard relies on to decide whether /// an ADLX reading can be trusted at all) — it degrades to a zeroed /// `AdlxReading` instead, keeping one slot per GPU. +/// +/// Initialisation is attempted once per process. A missing driver is a static +/// condition, and retrying a failed `ADLXInitialize` would repeat the DLL +/// load/unload cycle that [`ResidentReader`] exists to avoid. #[cfg(windows)] -fn adlx_readings() -> Vec { +fn adlx_readings() -> ReaderAnswer { + static READER: std::sync::OnceLock = std::sync::OnceLock::new(); + READER + .get_or_init(|| ResidentReader::spawn(init_adlx, read_adlx)) + .read() +} + +#[cfg(windows)] +fn init_adlx() -> Option { // `adlx::helper::AdlxHelper::new()` loads `amdadlx64.dll` by bare name // (`libloading::Library::new("amdadlx64.dll")`, no search-order // hardening, no way to pass flags into the `adlx` crate). Restricting // the process-wide default DLL search path to System32 — this call // affects every subsequent bare-name `LoadLibrary` in the whole process, // not just `adlx`'s — closes the planted-DLL-in-PATH/cwd hijack vector - // for that unflagged load. Must run before the first `AdlxHelper::new()` - // call; `Once` makes repeated per-tick calls to this function a no-op - // after the first. - static HARDEN_DLL_SEARCH_PATH: std::sync::Once = std::sync::Once::new(); + // for that unflagged load. Must run before the `AdlxHelper::new()` call, + // which this function makes exactly once per process. #[allow(unsafe_code)] // Win32 FFI - HARDEN_DLL_SEARCH_PATH.call_once(|| unsafe { - // Failure here (`FALSE`) means the process keeps its default search - // order (which can include the cwd) rather than System32-only; still - // attempt the `AdlxHelper` load afterwards; a stricter search path is - // defense in depth, not a precondition for ADLX to work. - let hardened = windows_sys::Win32::System::LibraryLoader::SetDefaultDllDirectories( + let hardened = unsafe { + windows_sys::Win32::System::LibraryLoader::SetDefaultDllDirectories( windows_sys::Win32::System::LibraryLoader::LOAD_LIBRARY_SEARCH_SYSTEM32, + ) + }; + // Failure here (`FALSE`) means the process keeps its default search + // order (which can include the cwd) rather than System32-only; still + // attempt the `AdlxHelper` load afterwards; a stricter search path is + // defense in depth, not a precondition for ADLX to work. + if hardened == 0 { + tracing::warn!( + "SetDefaultDllDirectories failed; amdadlx64.dll load falls back to the default search order" ); - if hardened == 0 { - tracing::warn!( - "SetDefaultDllDirectories failed; amdadlx64.dll load falls back to the default search order" - ); - } - }); + } + adlx::helper::AdlxHelper::new().ok() +} +#[cfg(windows)] +fn read_adlx(helper: &adlx::helper::AdlxHelper) -> Vec { (|| -> Option> { - let helper = adlx::helper::AdlxHelper::new().ok()?; let system = helper.system(); let gpus = system.gpus().ok()?; let services = system.performance_monitoring_services().ok()?; @@ -479,8 +642,8 @@ fn adlx_readings() -> Vec { } #[cfg(not(windows))] -const fn adlx_readings() -> Vec { - Vec::new() +const fn adlx_readings() -> ReaderAnswer { + Ok(Vec::new()) } struct Adapter { @@ -865,6 +1028,225 @@ mod tests { assert_eq!(apply_adlx(Some(&reading)), (65.0, 12.5, Some(1900.0))); } + fn reading(temperature_c: f32) -> AdlxReading { + AdlxReading { + temperature_c, + power_w: 0.0, + clock_mhz: None, + total_vram_mb: None, + } + } + + /// The crash this guards against came from initialising and tearing down + /// ADLX on every tick. Initialisation must run exactly once however many + /// reads follow, and every read must be served by that one thread. + #[test] + fn resident_reader_initialises_once_and_reads_on_one_thread() { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + let inits = Arc::new(AtomicUsize::new(0)); + let inits_in_worker = inits.clone(); + let reader = ResidentReader::spawn( + move || { + inits_in_worker.fetch_add(1, Ordering::SeqCst); + Some(std::thread::current().id()) + }, + |init_thread| { + // Temperature encodes whether this read ran on the thread + // that initialised the state. + let same = *init_thread == std::thread::current().id(); + vec![reading(if same { 1.0 } else { 0.0 })] + }, + ); + + for _ in 0..25 { + let out = reader.read().expect("a live reader answers"); + assert_eq!(out.len(), 1); + assert!((out[0].temperature_c - 1.0).abs() < f32::EPSILON); + } + assert_eq!(inits.load(Ordering::SeqCst), 1); + } + + #[test] + fn resident_reader_degrades_to_empty_when_init_fails_and_does_not_retry() { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + let inits = Arc::new(AtomicUsize::new(0)); + let inits_in_worker = inits.clone(); + let reader = ResidentReader::spawn( + move || { + inits_in_worker.fetch_add(1, Ordering::SeqCst); + None::<()> + }, + |()| vec![reading(1.0)], + ); + + for _ in 0..5 { + // An absent driver answers Ok with nothing, never ReaderGone: + // the dashboard must not warn about a machine that simply has no + // AMD GPU. + assert!( + matches!(reader.read(), Ok(readings) if readings.is_empty()), + "a failed init must read as an absent driver, not a reader fault" + ); + } + assert_eq!(inits.load(Ordering::SeqCst), 1); + } + + /// The asymmetry raised in review on #1020: `read` was wrapped in + /// `catch_unwind` and `init` was not. + /// + /// A panic escaping `init` unwound the reader thread before the request + /// loop began, so `inbox` dropped and every later tick answered + /// `ThreadGone` — the one verdict indistinguishable from "the thread never + /// started", for a cause that is neither. Worse, the reader is built once + /// per process, so that verdict was permanent. + /// + /// Both halves are asserted, and the second is what makes this + /// discriminating: the answer must name the init panic, **and** the reader + /// must still be answering on later calls rather than having died. Before + /// the fix the first assertion fails (`ThreadGone`, not `InitPanicked`). + #[test] + fn a_panicking_init_is_named_and_leaves_the_reader_answering() { + let reader = ResidentReader::spawn( + || -> Option<()> { panic!("ADLX init blew up") }, + |()| vec![reading(1.0)], + ); + + for _ in 0..5 { + // `matches!`, not `assert_eq!`: `AdlxReading` carries no `Debug` + // or `PartialEq`, and adding them to a production struct to satisfy + // a test would be the tail wagging the dog. + assert!( + matches!(reader.read(), Err(ReaderGone::InitPanicked)), + "an init panic must be named, not reported as a thread that never started" + ); + } + assert_eq!( + ReaderGone::InitPanicked.warning(), + "ADLX initialisation panicked" + ); + } + + /// The `snapshot` mapping, which nothing reached: `snapshot` shells out to + /// PowerShell unconditionally, so the `Done(Err(_))` arm this change added + /// was wired up and never executed by a test. See `adlx_outcome`. + /// + /// Every arm is covered, including the one that must stay silent: an absent + /// driver answers `Ok` with no readings and raises no warning, because a + /// machine with no AMD GPU is not a fault to report. + #[test] + fn the_adlx_outcome_mapping_warns_for_faults_and_stays_silent_for_an_absent_driver() { + let (readings, warning) = adlx_outcome(SingleFlight::Done(Ok(vec![reading(42.0)]))); + assert_eq!(readings.len(), 1); + assert!(warning.is_none(), "a good read raises nothing"); + + let (readings, warning) = adlx_outcome(SingleFlight::Done(Ok(Vec::new()))); + assert!(readings.is_empty()); + assert!( + warning.is_none(), + "an absent driver is an expected state, not a warning" + ); + + for gone in [ + ReaderGone::ReadPanicked, + ReaderGone::InitPanicked, + ReaderGone::ThreadGone, + ] { + let (readings, warning) = adlx_outcome(SingleFlight::Done(Err(gone))); + assert!(readings.is_empty()); + assert_eq!( + warning.as_deref(), + Some(gone.warning().as_str()), + "a reader fault must surface its own wording: {gone:?}" + ); + } + + for (outcome, expected) in [ + (SingleFlight::Panicked, "ADLX reading task panicked"), + ( + SingleFlight::StillRunning, + "a previous call is still running", + ), + ] { + let (readings, warning) = adlx_outcome(outcome); + assert!(readings.is_empty()); + assert!( + warning.as_deref().is_some_and(|w| w.contains(expected)), + "expected {expected:?}, got {warning:?}" + ); + } + + let (readings, warning) = adlx_outcome(SingleFlight::TimedOut); + assert!(readings.is_empty()); + assert!( + warning.as_deref().is_some_and(|w| w.contains("timed out")), + "{warning:?}" + ); + } + + #[test] + fn a_panicking_read_is_reported_and_leaves_the_reader_usable() { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + let reads = Arc::new(AtomicUsize::new(0)); + let reads_in_worker = reads.clone(); + // Panics on the first read only, so the second shows the reader is + // still answering rather than having unwound. + let reader = ResidentReader::spawn( + || Some(()), + move |()| { + assert!( + reads_in_worker.fetch_add(1, Ordering::SeqCst) != 0, + "driver blew up" + ); + vec![reading(1.0)] + }, + ); + + // The panic is caught and named, never silently empty: the reader is + // built once per process, so a silent failure would blank + // temperature, power and clock for the whole session and read as a + // machine with no AMD driver. + assert_eq!(reader.read().err(), Some(ReaderGone::ReadPanicked)); + let after = reader.read().expect("the reader survives a caught panic"); + assert_eq!(after.len(), 1); + assert!((after[0].temperature_c - 1.0).abs() < f32::EPSILON); + assert_eq!(reads.load(Ordering::SeqCst), 2); + } + + #[test] + fn every_reader_fault_warns_in_words_that_name_adlx_and_tell_the_faults_apart() { + // `snapshot` pairs each of these with no readings, so the state the + // user lands in is "the ADLX columns are blank, and here is why". + // An absent driver is the other case and deliberately has no warning. + for gone in [ReaderGone::ReadPanicked, ReaderGone::ThreadGone] { + let warning = gone.warning(); + assert!( + warning.to_lowercase().contains("adlx"), + "a warning shown next to blank columns must name the subsystem: {warning}" + ); + } + assert_ne!( + ReaderGone::ReadPanicked.warning(), + ReaderGone::ThreadGone.warning(), + "a panicking driver and a dead reader need different text to be told apart" + ); + } + + #[test] + fn a_reader_whose_thread_is_gone_reports_it_rather_than_answering_empty() { + // A sender whose receiver has been dropped is what a dead or + // never-started reader thread looks like from the caller's side. + let (requests, inbox) = std::sync::mpsc::channel(); + drop(inbox); + let dead = ResidentReader { requests }; + assert_eq!(dead.read().err(), Some(ReaderGone::ThreadGone)); + } + #[tokio::test] async fn single_flight_returns_value_and_clears_flag() { static FLAG: AtomicBool = AtomicBool::new(false); diff --git a/crates/rocm-dash-core/src/bench_rollup.rs b/crates/rocm-dash-core/src/bench_rollup.rs index f134cfac43..8acbc2a157 100644 --- a/crates/rocm-dash-core/src/bench_rollup.rs +++ b/crates/rocm-dash-core/src/bench_rollup.rs @@ -8,10 +8,10 @@ //! it whenever the row set changes rather than relying on the upstream //! `pass_n_of_n` / `pass_at_n` CSV columns. //! -//! See `../wiki/concepts/benchmark-result-schema.md` and -//! `../wiki/entities/normalize-results.md`: rows are grouped by -//! `(cell, model, backend, concurrency)` and each group of N trials yields -//! two verdicts — strict (all N passed) and lenient (at least one passed). +//! Rows are grouped by `(cell, model, engine, tp, dtype, concurrency)` (the +//! private `RollupKey`) and each group of N trials yields two verdicts — +//! strict (all N passed) and lenient (at least one passed). `engine` is part +//! of the key, so a row with no `engine` does not group with one that has it. use std::collections::BTreeMap; diff --git a/crates/rocm-dash-core/src/bench_schema.rs b/crates/rocm-dash-core/src/bench_schema.rs index 4090b6a4de..ce5aa70359 100644 --- a/crates/rocm-dash-core/src/bench_schema.rs +++ b/crates/rocm-dash-core/src/bench_schema.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: MIT //! Benchmark row schema, vendored from instinct-agent-bench. -//! See `../wiki/concepts/benchmark-result-schema.md` and `../wiki/entities/csv-emitter.md`. use serde::{Deserialize, Serialize}; diff --git a/crates/rocm-dash-core/src/chat_template.rs b/crates/rocm-dash-core/src/chat_template.rs new file mode 100644 index 0000000000..d9757b8db8 --- /dev/null +++ b/crates/rocm-dash-core/src/chat_template.rs @@ -0,0 +1,98 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Chat-template defaults a locally served model needs on an +//! OpenAI-compatible chat request. Pure: a decision over a model id, no I/O. +//! +//! Two front ends send that request — the `rocm` CLI's local chat provider and +//! the dashboard's chat client — and both must decide this the same way. They +//! did not: the CLI turned hybrid thinking off while the dashboard left it on, +//! so the same locally served model answered in one and reasoned past the +//! request deadline in the other. The rule therefore lives here, in the leaf +//! crate both already depend on, as one copy with one set of tests. + +/// Whether a locally served `model` should have its chat template's hybrid +/// "thinking" mode turned off — i.e. whether the request body needs +/// `chat_template_kwargs.enable_thinking = false`. +/// +/// Qwen3 chat templates enable thinking by default with an effectively +/// unbounded reasoning budget, so a small quantized build on modest hardware +/// can spend an entire request generating reasoning tokens and return no +/// visible answer at all. Matching is case-insensitive and by substring, so a +/// repository-qualified and quantization-suffixed id such as +/// `Qwen/Qwen3-1.7B-GGUF:Q4_K_M` matches too. +/// +/// The decision is made on the name alone and is therefore approximate: every +/// id that spells `qwen3` anywhere matches, whether or not what is actually +/// served behind that name has a hybrid-thinking chat template. +pub fn local_model_should_disable_thinking(model: &str) -> bool { + model.to_ascii_lowercase().contains("qwen3") +} + +#[cfg(test)] +mod tests { + use super::local_model_should_disable_thinking; + + #[test] + fn qwen3_ids_match_in_every_spelling_a_serving_endpoint_reports() { + // A served model id reaches this helper in whatever shape the engine + // reports it: bare, repository-qualified, quantization-suffixed, and + // in any case. All of them name the same chat template. `qwen32b` is + // the awkward one: a catalog alias where the parameter count runs + // straight on from `qwen3` with no separator at all. + for model in [ + "qwen3", + "Qwen3-0.6B-GGUF", + "Qwen/Qwen3-8B", + "QWEN3-32B-AWQ", + "unsloth/Qwen3-1.7B-GGUF:Q4_K_M", + "qwen32b", + ] { + assert!( + local_model_should_disable_thinking(model), + "`{model}` names a hybrid-thinking chat template and must match" + ); + } + } + + #[test] + fn later_qwen3_families_match_on_the_strength_of_the_name_alone() { + // Pins the approximation the doc comment above describes, rather than + // endorsing it. `qwen3.5` and `qwen3.6-27b` are real catalog aliases + // (`crates/rocm-core/src/model_catalog.json`) that spell `qwen3` as a + // prefix, so they match on their names alone — nothing here has + // looked at the chat template actually served behind them. Narrowing + // the rule to exclude them would change behaviour for a model that + // matches today, so it has to be a deliberate decision taken against + // this test and that doc comment together. + for model in ["qwen3.5", "qwen3.6-27b"] { + assert!( + local_model_should_disable_thinking(model), + "`{model}` matches by name today; excluding it is a behaviour \ + change, not a refactor" + ); + } + } + + #[test] + fn other_models_are_left_alone() { + // Everything else keeps the server's own chat-template defaults: the + // caller sends no `chat_template_kwargs` at all, so a backend that + // rejects or misreads the key is never handed one. `qwen2.5` is the + // near miss that matters — same family, no hybrid thinking mode. + for model in [ + "local-model", + "qwen2.5-7b-instruct", + "Qwen/Qwen2.5-Coder-32B", + "llama-3.1-8b-instruct", + "gpt-4o-mini", + "", + ] { + assert!( + !local_model_should_disable_thinking(model), + "`{model}` must keep the server's own chat-template defaults" + ); + } + } +} diff --git a/crates/rocm-dash-core/src/efficiency.rs b/crates/rocm-dash-core/src/efficiency.rs index 6781c86024..9a6d19fa6d 100644 --- a/crates/rocm-dash-core/src/efficiency.rs +++ b/crates/rocm-dash-core/src/efficiency.rs @@ -5,12 +5,12 @@ //! Pure efficiency derivations (tokens-per-watt). No I/O, no rendering. //! //! This is the join that turns vLLM throughput + amd-smi power into a -//! per-instance capacity number. See `../wiki/concepts/metric-registry.md`. +//! per-instance capacity number. use std::collections::HashMap; use std::hash::BuildHasher; -use crate::metrics::{GpuMetrics, Instance}; +use crate::metrics::{GpuMetrics, Instance, ObservationFreshness, Snapshot}; /// Normalize a GPU identifier to a bare index for joining. /// @@ -49,6 +49,13 @@ pub fn gpu_ids_overlap(gpu_ids: &[String], gpus: &[GpuMetrics]) -> bool { /// a GPU has no per-container reading the split on that GPU is equal, rather /// than guessing a skew from partial data. A sole occupant gets the whole GPU. /// +/// An occupant that is still starting and has no reading while another occupant +/// does is left out of the split. (A status of `Unknown` means no readiness +/// information, not "not serving", so it stays in.) It has not +/// allocated GPU memory, so the evidence says it is drawing next to nothing, and +/// an equal share would show the serving neighbour about half its real power and +/// roughly double its real tokens per watt until the newcomer loads. +/// /// Power on a GPU no instance occupies (idle, or a process outside the managed /// set) is attributed to nobody; it still counts toward node power. An entry is /// `None` when none of its GPUs matched a GPU reporting power (id join found @@ -76,6 +83,19 @@ pub fn attribute_power_w( if occupants.is_empty() { continue; } + let reading = |k: usize| { + per_container_vram_mb + .get(&instances[k].container_id) + .is_some_and(|&mb| mb > 0) + }; + let occupants: Vec = if occupants.iter().any(|&k| reading(k)) { + occupants + .into_iter() + .filter(|&k| reading(k) || !instances[k].status.is_starting()) + .collect() + } else { + occupants + }; let weights: Option> = occupants .iter() .map(|&k| { @@ -103,6 +123,147 @@ pub fn attribute_power_w( .collect() } +/// Whole-node efficiency: aggregate generation throughput over aggregate power. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct NodeEfficiency { + /// Σ live `gen_tps` (see [`Instance::live_gen_tps`]) ÷ Σ board `power_w`, in + /// tokens per watt. + pub tokens_per_watt: f64, + /// Freshness of the throughput behind the figure (instances at zero tok/s + /// contribute nothing and are not counted): `Held` when any of it is + /// carried from a prior scrape window, `Fresh` when all of it is known to + /// be current, and `None` when some of it has no freshness metadata (a + /// legacy recording) and none is known to be held. Unknown is never + /// reported as fresh. The figure is still reported when held, so a + /// one-scrape hiccup does not blank it, but a consumer must mark it: a + /// held value expires within the observation validity window. + pub freshness: Option, +} + +impl NodeEfficiency { + /// Whether any throughput behind the figure is held. + #[must_use] + pub fn held(&self) -> bool { + self.freshness == Some(ObservationFreshness::Held) + } +} + +/// Aggregate generation throughput across instances. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct NodeThroughput { + /// Σ `gen_tps` over the instances with a finite, live reading (a Lemonade + /// last-request rate is not live), in tokens/s. A real zero counts: an idle + /// node is zero throughput, not no data. + pub tokens_per_sec: f64, + /// Freshness of the throughput behind the sum, as for + /// [`NodeEfficiency::freshness`]: instances at zero tok/s add nothing to the + /// sum and are not counted. + pub freshness: Option, +} + +impl NodeThroughput { + /// Whether any throughput behind the sum is held. + #[must_use] + pub fn held(&self) -> bool { + self.freshness == Some(ObservationFreshness::Held) + } +} + +/// Node throughput over `instances`, or `None` when no instance has a finite, +/// live `gen_tps` (a single NaN/Inf must not poison the sum). +/// +/// This is the one place the node's throughput and its freshness are worked +/// out, so every surface that shows the aggregate (the efficiency hero, the +/// throughput hero, the Hardware header, the assistant) agrees on both. An +/// instance at zero tok/s contributes nothing to the sum, so its freshness says +/// nothing about the sum's either: a held idle instance must not mark a figure +/// that comes from fresh ones as held. +/// +/// Only live throughput counts: a Lemonade last-request rate (see +/// [`Instance::live_gen_tps`]) stays unchanged while the server is idle, so it is +/// left out of the sum and out of the freshness, and an idle Lemonade server on +/// its own is no throughput reading at all. +#[must_use] +pub fn node_throughput<'a>( + instances: impl IntoIterator, +) -> Option { + let mut tokens_per_sec = 0.0_f64; + let mut any = false; + let mut held = false; + let mut unknown = false; + for inst in instances { + let Some(v) = inst.live_gen_tps() else { + continue; + }; + tokens_per_sec += v; + any = true; + if v <= 0.0 { + continue; + } + match inst.gen_tps_observation.as_ref().map(|m| m.freshness) { + Some(ObservationFreshness::Held) => held = true, + Some(ObservationFreshness::Fresh) => {} + None => unknown = true, + } + } + let freshness = if held { + Some(ObservationFreshness::Held) + } else if unknown { + None + } else { + Some(ObservationFreshness::Fresh) + }; + any.then_some(NodeThroughput { + tokens_per_sec, + freshness, + }) +} + +/// Total board power (W) of the GPUs that report a usable reading. +/// +/// A reading that is not finite or not positive is skipped, as +/// [`attribute_power_w`] does: zero means "no telemetry" (a platform without a +/// power source), and a faulty sensor reporting infinity, NaN or a negative +/// value must not turn a quotient into a plausible-looking 0 or deflate the +/// total. Every surface that shows the node's power or divides by it uses this, +/// so they agree on what counts. +#[must_use] +pub fn node_power_w(gpus: &[GpuMetrics]) -> f64 { + gpus.iter() + .map(|g| f64::from(g.power_w)) + .filter(|w| w.is_finite() && *w > 0.0) + .sum() +} + +/// Node-level efficiency for a snapshot, or `None` when it cannot be stated. +/// +/// Throughput is [`node_throughput`]; power is [`node_power_w`], so one faulty +/// sensor cannot turn the quotient into a plausible-looking 0 (infinity) or +/// deflate the total (negative). It is `None` with no traffic +/// (`Σ gen_tps == 0`), no usable power telemetry, or a non-finite quotient. +/// +/// Power here is the whole board, including GPUs no instance occupies, because +/// the question is the node's efficiency, not an instance's: per-instance +/// figures use attributed power ([`attribute_power_w`]) instead. +/// +/// A node whose only throughput is held at zero tok/s reports a fresh 0 tok/s +/// from [`node_throughput`] (nothing contributes, so nothing is held), and no +/// efficiency figure from here, since there is no throughput to divide. +#[must_use] +pub fn node_efficiency(snap: &Snapshot) -> Option { + let throughput = node_throughput(&snap.instances)?; + let power = node_power_w(&snap.gpus); + if throughput.tokens_per_sec > 0.0 && power > 0.0 { + let tokens_per_watt = throughput.tokens_per_sec / power; + tokens_per_watt.is_finite().then_some(NodeEfficiency { + tokens_per_watt, + freshness: throughput.freshness, + }) + } else { + None + } +} + /// Tokens-per-watt for one serving instance: generation throughput (tok/s) /// divided by the power (W) attributed to it by [`attribute_power_w`]. /// @@ -115,9 +276,20 @@ pub fn tokens_per_watt(gen_tps: Option, power_w: Option) -> Option 0.0).then(|| tps / power) } +/// An instance's efficiency, from its live throughput only. +/// +/// A last-request rate over the board's current (idle) power is not an +/// efficiency, so it yields `None` and never reaches the snapshot, the wire or a +/// recording. +#[must_use] +pub fn instance_tokens_per_watt(inst: &Instance, power_w: Option) -> Option { + tokens_per_watt(inst.live_gen_tps(), power_w) +} + #[cfg(test)] mod tests { use super::*; + use crate::metrics::InstanceStatus; fn gpu(device_id: &str, power_w: f32) -> GpuMetrics { GpuMetrics { @@ -135,6 +307,279 @@ mod tests { assert_eq!(normalize_gpu_id("3"), "3"); } + fn snap_with(instances: Vec, gpus: Vec) -> Snapshot { + Snapshot { + gpus, + instances, + ..Snapshot::default() + } + } + + fn serving( + id: &str, + gen_tps: Option, + freshness: Option, + ) -> Instance { + use crate::metrics::ObservationMetadata; + Instance { + container_id: id.into(), + gen_tps, + gen_tps_observation: freshness.map(|freshness| ObservationMetadata { + observed_at: chrono::Utc::now(), + freshness, + }), + ..Instance::default() + } + } + + /// An idle Lemonade server keeps reporting its last request's rate. Counted as + /// throughput it would hold the node figure up long after traffic stopped, and + /// mark it held while it lasted. + #[test] + fn node_efficiency_leaves_out_a_last_request_rate() { + let mut idle = serving("lemonade", Some(500.0), Some(ObservationFreshness::Held)); + idle.gen_tps_last_request = true; + let live = serving("vllm", Some(30.0), Some(ObservationFreshness::Fresh)); + let node = node_efficiency(&snap_with( + vec![idle.clone(), live], + vec![gpu("gpu-0", 100.0)], + )) + .expect("the live rate is traffic"); + // Only the live 30 tok/s counts: 30 / 100 W, not 530 / 100 W. + assert!((node.tokens_per_watt - 0.3).abs() < 1e-9, "{node:?}"); + assert!( + !node.held(), + "a held last-request rate must not mark the figure held" + ); + // An idle Lemonade server alone is no traffic at all. + assert_eq!( + node_efficiency(&snap_with(vec![idle], vec![gpu("gpu-0", 100.0)])), + None + ); + } + + #[test] + fn an_instance_efficiency_leaves_out_a_last_request_rate() { + let mut inst = serving("lemonade", Some(40.0), None); + assert_eq!(instance_tokens_per_watt(&inst, Some(100.0)), Some(0.4)); + inst.gen_tps_last_request = true; + assert_eq!(instance_tokens_per_watt(&inst, Some(100.0)), None); + } + + #[test] + fn node_efficiency_divides_total_throughput_by_total_board_power() { + let snap = snap_with( + vec![ + serving("a", Some(300.0), None), + serving("b", Some(200.0), None), + ], + vec![gpu("gpu-0", 400.0), gpu("gpu-1", 600.0)], + ); + // (300 + 200) / (400 + 600) = 0.5 tok/W + let n = node_efficiency(&snap).expect("some"); + assert!((n.tokens_per_watt - 0.5).abs() < 1e-9, "got {n:?}"); + assert!(!n.held()); + } + + #[test] + fn node_efficiency_counts_power_of_gpus_no_instance_occupies() { + // The node pays for an idle GPU too, so it stays in the denominator. + let snap = snap_with( + vec![serving("a", Some(100.0), None)], + vec![gpu("gpu-0", 100.0), gpu("gpu-1", 300.0)], + ); + let n = node_efficiency(&snap).expect("some"); + assert!((n.tokens_per_watt - 0.25).abs() < 1e-9, "got {n:?}"); + } + + #[test] + fn node_efficiency_is_none_without_power_or_traffic() { + let no_power = snap_with( + vec![serving("a", Some(100.0), None)], + vec![gpu("gpu-0", 0.0)], + ); + assert_eq!(node_efficiency(&no_power), None); + let no_traffic = snap_with(vec![serving("a", None, None)], vec![gpu("gpu-0", 500.0)]); + assert_eq!(node_efficiency(&no_traffic), None); + let zero_traffic = snap_with( + vec![serving("a", Some(0.0), None)], + vec![gpu("gpu-0", 500.0)], + ); + assert_eq!(node_efficiency(&zero_traffic), None); + } + + #[test] + fn a_held_idle_instance_does_not_mark_a_figure_that_comes_from_fresh_ones() { + let gpus = vec![gpu("gpu-0", 500.0)]; + let snap = snap_with( + vec![ + serving("busy", Some(250.0), Some(ObservationFreshness::Fresh)), + serving("idle", Some(0.0), Some(ObservationFreshness::Held)), + ], + gpus.clone(), + ); + let n = node_efficiency(&snap).unwrap(); + assert!( + !n.held(), + "a zero-throughput held instance must not flag the node" + ); + assert_eq!(n.freshness, Some(ObservationFreshness::Fresh)); + // ...and one with unknown freshness at zero does not make it unknown. + let snap = snap_with( + vec![ + serving("busy", Some(250.0), Some(ObservationFreshness::Fresh)), + serving("idle", Some(0.0), None), + ], + gpus, + ); + assert_eq!( + node_efficiency(&snap).unwrap().freshness, + Some(ObservationFreshness::Fresh) + ); + } + + #[test] + fn node_throughput_and_node_efficiency_agree_on_freshness() { + // The two panels that show these must never disagree: a held instance + // at zero beside a fresh one is fresh for both. + let snap = snap_with( + vec![ + serving("busy", Some(250.0), Some(ObservationFreshness::Fresh)), + serving("idle", Some(0.0), Some(ObservationFreshness::Held)), + ], + vec![gpu("gpu-0", 500.0)], + ); + let t = node_throughput(&snap.instances).unwrap(); + assert!((t.tokens_per_sec - 250.0).abs() < 1e-9); + assert!(!t.held()); + assert_eq!(t.freshness, node_efficiency(&snap).unwrap().freshness); + // A held instance that does contribute marks both. + let held = snap_with( + vec![ + serving("busy", Some(250.0), Some(ObservationFreshness::Fresh)), + serving("slow", Some(5.0), Some(ObservationFreshness::Held)), + ], + vec![gpu("gpu-0", 500.0)], + ); + assert!(node_throughput(&held.instances).unwrap().held()); + assert!(node_efficiency(&held).unwrap().held()); + } + + #[test] + fn node_throughput_distinguishes_idle_from_no_data() { + // A real zero is a figure; nothing finite is no figure. + let idle = [serving("a", Some(0.0), Some(ObservationFreshness::Fresh))]; + assert!(node_throughput(&idle).unwrap().tokens_per_sec.abs() < f64::EPSILON); + let none = [serving("a", None, None), serving("b", Some(f64::NAN), None)]; + assert_eq!(node_throughput(&none), None); + assert_eq!(node_throughput(&[]), None); + } + + #[test] + fn node_efficiency_skips_a_faulty_power_sensor_like_attribute_power_does() { + let insts = vec![serving("a", Some(250.0), Some(ObservationFreshness::Fresh))]; + // An infinite reading must not turn the quotient into 0. + let inf = snap_with( + insts.clone(), + vec![gpu("gpu-0", 500.0), gpu("gpu-1", f32::INFINITY)], + ); + let n = node_efficiency(&inf).unwrap(); + assert!((n.tokens_per_watt - 0.5).abs() < 1e-9, "got {n:?}"); + // A negative reading must not deflate the total. + let neg = snap_with( + insts.clone(), + vec![gpu("gpu-0", 500.0), gpu("gpu-1", -300.0)], + ); + let n = node_efficiency(&neg).unwrap(); + assert!((n.tokens_per_watt - 0.5).abs() < 1e-9, "got {n:?}"); + // Only faulty readings: no usable power, so no figure. A lone infinity is + // the case only the filter decides: unfiltered, tps / inf is a finite 0. + let only_bad = snap_with( + insts.clone(), + vec![gpu("gpu-0", f32::NAN), gpu("gpu-1", -5.0)], + ); + assert_eq!(node_efficiency(&only_bad), None); + let lone_inf = snap_with(insts, vec![gpu("gpu-0", f32::INFINITY)]); + assert_eq!(node_efficiency(&lone_inf), None); + } + + #[test] + fn node_power_sums_only_usable_readings() { + let gpus = [ + gpu("gpu-0", 300.0), + gpu("gpu-1", f32::INFINITY), + gpu("gpu-2", f32::NAN), + gpu("gpu-3", -40.0), + gpu("gpu-4", 0.0), + gpu("gpu-5", 200.0), + ]; + assert!((node_power_w(&gpus) - 500.0).abs() < 1e-9); + assert!(node_power_w(&[]).abs() < f64::EPSILON); + } + + #[test] + fn node_efficiency_ignores_non_finite_gen_tps() { + // One NaN sample must not poison the sum for the whole node. + let snap = snap_with( + vec![ + serving("finite", Some(250.0), Some(ObservationFreshness::Held)), + serving("poisoned", Some(f64::NAN), Some(ObservationFreshness::Held)), + ], + vec![gpu("gpu-0", 500.0)], + ); + let n = node_efficiency(&snap).expect("finite gen_tps must still produce a result"); + assert!((n.tokens_per_watt - 0.5).abs() < 1e-9, "got {n:?}"); + } + + #[test] + fn node_efficiency_is_held_when_any_contributing_throughput_is_held() { + let gpus = vec![gpu("gpu-0", 500.0)]; + let fresh = snap_with( + vec![serving("a", Some(250.0), Some(ObservationFreshness::Fresh))], + gpus.clone(), + ); + let n = node_efficiency(&fresh).unwrap(); + assert!(!n.held()); + assert_eq!(n.freshness, Some(ObservationFreshness::Fresh)); + // Unknown freshness (legacy snapshot) is neither held nor claimed + // fresh: nothing measured it. + let unknown = snap_with(vec![serving("a", Some(250.0), None)], gpus.clone()); + let n = node_efficiency(&unknown).unwrap(); + assert!(!n.held()); + assert_eq!(n.freshness, None); + // One unknown among fresh ones keeps the whole figure unknown. + let partly = snap_with( + vec![ + serving("a", Some(250.0), Some(ObservationFreshness::Fresh)), + serving("b", Some(10.0), None), + ], + gpus.clone(), + ); + assert_eq!(node_efficiency(&partly).unwrap().freshness, None); + let mixed = snap_with( + vec![ + serving("a", Some(250.0), Some(ObservationFreshness::Fresh)), + serving("b", Some(100.0), Some(ObservationFreshness::Held)), + ], + gpus.clone(), + ); + assert!(node_efficiency(&mixed).unwrap().held()); + // A held instance whose throughput is non-finite does not contribute, + // so it cannot mark the node held. + let excluded = snap_with( + vec![ + serving("a", Some(250.0), Some(ObservationFreshness::Fresh)), + serving("b", Some(f64::INFINITY), Some(ObservationFreshness::Held)), + ], + gpus, + ); + assert!(!node_efficiency(&excluded).unwrap().held()); + assert_eq!( + node_efficiency(&excluded).unwrap().freshness, + Some(ObservationFreshness::Fresh) + ); + } + fn inst(id: &str, gpu_ids: &[&str]) -> Instance { Instance { container_id: id.into(), @@ -195,6 +640,51 @@ mod tests { } } + #[test] + fn a_starting_occupant_with_no_reading_does_not_dilute_a_serving_neighbour() { + let gpus = [gpu("gpu-0", 400.0)]; + let mut serving = inst("a", &["0"]); + serving.status = InstanceStatus::Ready; + let mut loading = inst("b", &["0"]); + loading.status = InstanceStatus::Starting { phase: None }; + let insts = [serving, loading]; + // `a` has a VRAM reading; `b` is still loading and has none. + let p = attribute_power_w(&insts, &gpus, &vram(&[("a", 30_000)])); + assert_eq!(p, vec![Some(400.0), None], "the loader gets no share yet"); + // Once it has allocated memory it takes part in the split. + let p = attribute_power_w(&insts, &gpus, &vram(&[("a", 30_000), ("b", 10_000)])); + assert_eq!(p, vec![Some(300.0), Some(100.0)]); + // With no readings at all nothing is known, so the split stays equal. + let p = attribute_power_w(&insts, &gpus, &vram(&[])); + assert_eq!(p, vec![Some(200.0), Some(200.0)]); + } + + #[test] + fn a_starting_occupant_is_left_out_but_an_unmeasured_serving_one_still_forces_equal() { + let gpus = [gpu("gpu-0", 400.0)]; + let mut measured = inst("a", &["0"]); + measured.status = InstanceStatus::Ready; + let mut unmeasured = inst("b", &["0"]); + unmeasured.status = InstanceStatus::Ready; + let mut loading = inst("c", &["0"]); + loading.status = InstanceStatus::Starting { phase: None }; + let insts = [measured, unmeasured, loading]; + // Only `a` has a reading. `c` is starting with none, so it is dropped; + // `b` is serving but unmeasured (e.g. a missed /proc join), so leaving + // it out would credit `a` with the whole board: the rest stay equal. + let p = attribute_power_w(&insts, &gpus, &vram(&[("a", 30_000)])); + assert_eq!(p, vec![Some(200.0), Some(200.0), None]); + // Without the starting occupant in the picture the result is the same + // for the other two, so the starting one is what changed the split. + let p2 = attribute_power_w(&insts[..2], &gpus, &vram(&[("a", 30_000)])); + assert_eq!(p2, vec![Some(200.0), Some(200.0)]); + // An `Unknown` status is not "starting": it stays in the split. + let mut unknown = inst("d", &["0"]); + unknown.status = InstanceStatus::Unknown; + let p3 = attribute_power_w(&[insts[0].clone(), unknown], &gpus, &vram(&[("a", 30_000)])); + assert_eq!(p3, vec![Some(200.0), Some(200.0)]); + } + #[test] fn multi_gpu_instance_vram_is_spread_across_its_gpus() { // `a` spans gpu 0+1 (40 GB total -> 20 GB each); `b` is alone on gpu 1 diff --git a/crates/rocm-dash-core/src/lib.rs b/crates/rocm-dash-core/src/lib.rs index 40b13a970d..fbaa4bc20e 100644 --- a/crates/rocm-dash-core/src/lib.rs +++ b/crates/rocm-dash-core/src/lib.rs @@ -7,10 +7,12 @@ //! Pure types, traits, schemas, and the reducer for rocm-dash. //! No rendering deps. No async deps at the type boundary. //! -//! See `../wiki/concepts/tea-reducer-pattern.md` for the architectural pattern. +//! `State::apply` in `state` is the pure reducer, in the style of The Elm +//! Architecture. pub mod bench_rollup; pub mod bench_schema; +pub mod chat_template; pub mod config; pub mod efficiency; pub mod metrics; diff --git a/crates/rocm-dash-core/src/metrics.rs b/crates/rocm-dash-core/src/metrics.rs index 77d14afbe3..691d01ff45 100644 --- a/crates/rocm-dash-core/src/metrics.rs +++ b/crates/rocm-dash-core/src/metrics.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: MIT //! Live metric types. snake_case + units encoded in field names. -//! See `../wiki/concepts/metric-registry.md` and `../wiki/data/metric-field-index.md`. use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -62,7 +61,6 @@ pub struct GpuSystemInfo { pub memory_partition_mode: MemoryPartitionMode, pub compute_partition_mode: ComputePartitionMode, pub vram_per_logical_gpu_mb: u64, - pub lemond_version: Option, pub llama_server_build: Option, pub ccr_version: Option, pub llamacpp_backend: Option, @@ -198,6 +196,10 @@ pub struct ObservationMetadata { #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Instance { + /// Lifecycle ownership proven by discovery from a managed-service record. + /// Missing ownership in legacy snapshots is not permission to mutate. + #[serde(default)] + pub managed: bool, pub container_id: String, pub container_name: String, pub status: InstanceStatus, @@ -221,6 +223,11 @@ pub struct Instance { /// Omitted from JSON when `None` to preserve wire back-compat. #[serde(default, skip_serializing_if = "Option::is_none")] pub gen_tps_observation: Option, + /// True when [`Instance::gen_tps`] is the endpoint's most recent completed + /// request rate (Lemonade `/stats`), which stays unchanged while the server is + /// idle, rather than a live counter rate. Omitted from JSON when false. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub gen_tps_last_request: bool, /// Efficiency: `gen_tps` ÷ the power (W) attributed to this instance /// ([`Instance::power_w`]). `None` when throughput or GPU power telemetry /// is unavailable. @@ -256,6 +263,35 @@ pub struct Instance { pub log_file: Option, } +impl Instance { + /// External Lemonade discovery uses the engine marker, not the instance id. + /// This identifies provenance independently of the current serving status. + #[must_use] + pub fn is_external_lemonade(&self) -> bool { + !self.managed && self.container_name == "lemonade" + } + + /// The finite throughput that is current, for figures that sum instances. + /// + /// A rate that is only the endpoint's last completed request + /// ([`Instance::gen_tps_last_request`]) is excluded: it stays unchanged while + /// the server is idle, so adding it to a node total would show an idle server + /// as still generating. The Instances views show such a rate, labelled, and + /// so do the Services overlay rows and its quick preview. + #[must_use] + pub fn live_gen_tps(&self) -> Option { + self.gen_tps + .filter(|v| v.is_finite() && !self.gen_tps_last_request) + } + + /// [`Instance::tokens_per_watt`] when it derives from live throughput; `None` + /// when it derives from a last-request rate (see [`Instance::live_gen_tps`]). + #[must_use] + pub fn live_tokens_per_watt(&self) -> Option { + self.tokens_per_watt.filter(|_| !self.gen_tps_last_request) + } +} + #[cfg(test)] mod tests { use super::*; @@ -273,6 +309,10 @@ mod tests { "log_file": null }"#; let inst: Instance = serde_json::from_str(legacy).expect("legacy instance must parse"); + assert!( + !inst.managed, + "legacy snapshots do not grant lifecycle control" + ); assert_eq!(inst.gen_tps, None); assert_eq!(inst.tokens_per_watt, None); assert_eq!(inst.ttft_ms, None); @@ -294,6 +334,67 @@ mod tests { assert!((g.power_w - 5.0).abs() < f32::EPSILON); } + #[test] + fn instance_ownership_round_trips_and_defaults_to_false() { + assert!(!Instance::default().managed); + for managed in [false, true] { + let inst = Instance { + managed, + ..Instance::default() + }; + let json = serde_json::to_value(&inst).expect("serialize"); + assert_eq!(json["managed"], managed); + let back: Instance = serde_json::from_value(json).expect("deserialize"); + assert_eq!(back.managed, managed); + } + } + + #[test] + fn a_last_request_rate_is_not_live_throughput_or_live_efficiency() { + let live = Instance { + gen_tps: Some(40.0), + tokens_per_watt: Some(0.5), + ..Instance::default() + }; + assert_eq!(live.live_gen_tps(), Some(40.0)); + assert_eq!(live.live_tokens_per_watt(), Some(0.5)); + let last = Instance { + gen_tps_last_request: true, + ..live + }; + assert_eq!(last.live_gen_tps(), None); + assert_eq!(last.live_tokens_per_watt(), None); + // The rate itself is still there for views that label it. + assert_eq!(last.gen_tps, Some(40.0)); + for bad in [f64::NAN, f64::INFINITY] { + let inst = Instance { + gen_tps: Some(bad), + ..Instance::default() + }; + assert_eq!(inst.live_gen_tps(), None); + } + } + + #[test] + fn external_lemonade_predicate_ignores_ids_and_requires_engine_marker() { + for id in ["svc-looks-managed", "lemonade-looks-external"] { + for managed in [false, true] { + for marker in ["lemonade", "vllm", ""] { + let inst = Instance { + managed, + container_id: id.into(), + container_name: marker.into(), + ..Instance::default() + }; + assert_eq!( + inst.is_external_lemonade(), + !managed && marker == "lemonade" + ); + } + } + } + } + #[test] fn instance_round_trips_ttft_tpot_through_serde() { // The new live-latency fields survive a serialize → deserialize cycle. @@ -390,6 +491,17 @@ mod tests { assert_eq!(InstanceStatus::Ready.label(), "READY"); } + #[test] + fn old_recordings_with_a_retired_lemond_version_key_still_load() { + // Recorded sessions written before the field was removed still carry + // the key. This guards loading them (unknown keys are ignored); it does + // not prove the field's removal, which the compiler already enforces. + let mut value = serde_json::to_value(GpuSystemInfo::default()).expect("serialize"); + value["lemond_version"] = serde_json::json!("10.4.0"); + let info: GpuSystemInfo = serde_json::from_value(value).expect("deserialize old snapshot"); + assert_eq!(info.llama_server_build, None); + } + #[test] fn instance_status_starting_round_trips_through_serde() { // The data-carrying variant serializes as an externally-tagged object diff --git a/crates/rocm-dash-core/src/observation.rs b/crates/rocm-dash-core/src/observation.rs index 11361e3c4c..bf96cd70b6 100644 --- a/crates/rocm-dash-core/src/observation.rs +++ b/crates/rocm-dash-core/src/observation.rs @@ -132,6 +132,11 @@ pub struct GenerationObservationTracker { /// `current_value` is `None`. Set only on successful rate computation or /// valid direct-rate injection; never set on missing/transient data. last_observed_at: Option>, + /// True while the held value came from [`Self::observe_direct`]: an + /// endpoint-reported rate of the most recent request, not a live counter rate. + /// Cleared when a counter rate replaces the value, and with the value on + /// counter reset, expiry and `invalidate`. + direct: bool, } impl GenerationObservationTracker { @@ -150,9 +155,16 @@ impl GenerationObservationTracker { samples: VecDeque::with_capacity(capacity.min(16)), current_value: None, last_observed_at: None, + direct: false, } } + /// Whether the rate comes from an endpoint-reported last-request figure + /// ([`Self::observe_direct`]) rather than a cumulative counter. + pub const fn is_direct(&self) -> bool { + self.direct + } + /// The rolling rate-computation window. pub const fn rate_window(&self) -> Duration { self.rate_window @@ -192,6 +204,7 @@ impl GenerationObservationTracker { self.samples.clear(); self.current_value = None; self.last_observed_at = None; + self.direct = false; // New baseline; no rate until second sample. self.samples.push_back((observed_at, total)); return; @@ -231,6 +244,9 @@ impl GenerationObservationTracker { if rate.is_finite() && rate >= 0.0 { self.current_value = Some(rate); self.last_observed_at = Some(observed_at); + // The value is a counter rate now, not an endpoint-reported + // last-request rate, so `is_direct` must report false. + self.direct = false; } } // dt_ms == 0: identical timestamps should not reach here (guard @@ -252,6 +268,7 @@ impl GenerationObservationTracker { } self.current_value = Some(rate); self.last_observed_at = Some(observed_at); + self.direct = true; } /// Return the current rate and observation metadata for the given `now`. @@ -280,6 +297,7 @@ impl GenerationObservationTracker { self.samples.clear(); self.current_value = None; self.last_observed_at = None; + self.direct = false; return (None, None); } @@ -309,6 +327,7 @@ impl GenerationObservationTracker { self.samples.clear(); self.current_value = None; self.last_observed_at = None; + self.direct = false; } } @@ -899,6 +918,45 @@ mod tests { ); } + #[test] + fn a_counter_rate_supersedes_a_direct_rate_and_clears_the_direct_flag() { + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + assert!(!tracker.is_direct()); + tracker.observe_direct(99.0, t(0)); + assert!(tracker.is_direct()); + // A lone counter baseline computes no rate, so the direct value stands. + tracker.observe_counter(0.0, t(1)); + assert!(tracker.is_direct()); + tracker.observe_counter(50.0, t(6)); + assert!(!tracker.is_direct(), "the value is a counter rate now"); + } + + #[test] + fn every_path_that_clears_the_rate_also_clears_the_direct_flag() { + // Expiry: the daemon labels the rate "last req" from is_direct alone, so a + // flag that outlives its value renders "last req -" instead of "gen -". + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + tracker.observe_direct(42.0, t(0)); + assert!(tracker.snapshot(t(5)).0.is_some()); + assert!(tracker.is_direct(), "a live direct rate keeps its label"); + assert!(tracker.snapshot(t(12)).0.is_none()); + assert!(!tracker.is_direct(), "expired"); + + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + tracker.observe_direct(42.0, t(0)); + tracker.invalidate(); + assert!(!tracker.is_direct(), "invalidated"); + + // Counter reset: a baseline leaves the direct value standing, then a lower + // total clears it. + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + tracker.observe_direct(42.0, t(0)); + tracker.observe_counter(100.0, t(1)); + tracker.observe_counter(10.0, t(2)); + assert!(tracker.snapshot(t(2)).0.is_none()); + assert!(!tracker.is_direct(), "counter reset"); + } + // ── counter→direct→counter interleaving (M3) ───────────────────────────── #[test] diff --git a/crates/rocm-dash-core/src/partition.rs b/crates/rocm-dash-core/src/partition.rs index 6f7706623a..3aa94bb6e7 100644 --- a/crates/rocm-dash-core/src/partition.rs +++ b/crates/rocm-dash-core/src/partition.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! GPU partition mode enums. See `../wiki/concepts/gpu-partition-modes.md`. +//! GPU partition mode enums. use serde::{Deserialize, Serialize}; diff --git a/crates/rocm-dash-core/src/protocol.rs b/crates/rocm-dash-core/src/protocol.rs index 01e9888b3c..6dd2a5cbc6 100644 --- a/crates/rocm-dash-core/src/protocol.rs +++ b/crates/rocm-dash-core/src/protocol.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: MIT //! NDJSON command/event protocol between rocmdashd and rocmdash. -//! See `../wiki/comparisons/ctux-vs-rocm-dash.md` (resolved decisions). use serde::{Deserialize, Serialize}; diff --git a/crates/rocm-dash-core/src/state.rs b/crates/rocm-dash-core/src/state.rs index ec80d1ffc5..1f50df725f 100644 --- a/crates/rocm-dash-core/src/state.rs +++ b/crates/rocm-dash-core/src/state.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: MIT //! Pure reducer. `State::apply(StateEvent) -> Vec`. -//! See `../wiki/concepts/tea-reducer-pattern.md`. use std::collections::{HashMap, VecDeque}; use std::sync::Arc; diff --git a/crates/rocm-dash-core/src/traits.rs b/crates/rocm-dash-core/src/traits.rs index 3719b91ec8..25e70c798c 100644 --- a/crates/rocm-dash-core/src/traits.rs +++ b/crates/rocm-dash-core/src/traits.rs @@ -53,6 +53,9 @@ pub trait GpuCollector: Send + Sync { /// A discovered serving process or container. #[derive(Debug, Clone, Default)] pub struct DiscoveredService { + /// True only for discovery from a managed-service registry record. + /// External protocol and container discovery do not grant lifecycle control. + pub managed: bool, pub container_id: String, pub container_name: String, pub model_name: String, @@ -136,6 +139,7 @@ pub fn merge_instance( vram_total_mb: u64, ) -> Instance { Instance { + managed: svc.managed, container_id: svc.container_id.clone(), container_name: svc.container_name.clone(), status: svc.status, @@ -158,6 +162,7 @@ pub fn merge_instance( // Freshness provenance: unknown at merge time; set by the runner when // a counter-rate delta is computed. gen_tps_observation: None, + gen_tps_last_request: false, tokens_per_watt: None, power_w: None, // Derived at the runner by windowing the sample's cumulative histogram @@ -176,6 +181,20 @@ pub fn merge_instance( mod tests { use super::*; + #[test] + fn merge_instance_preserves_explicit_ownership() { + assert!(!DiscoveredService::default().managed); + for managed in [false, true] { + let svc = DiscoveredService { + managed, + container_id: "lemonade-misleading-id".into(), + ..Default::default() + }; + let inst = merge_instance(&svc, &InstanceSample::default(), 0, 0); + assert_eq!(inst.managed, managed); + } + } + #[test] fn merge_instance_propagates_quantization() { let svc = DiscoveredService { diff --git a/crates/rocm-dash-core/src/vram.rs b/crates/rocm-dash-core/src/vram.rs index 5752b45901..98fd294809 100644 --- a/crates/rocm-dash-core/src/vram.rs +++ b/crates/rocm-dash-core/src/vram.rs @@ -17,7 +17,7 @@ //! //! Graceful degradation: an instance that matches nothing (empty `gpu_ids`, //! no container entry — e.g. Lemonade) resolves to `(0, 0)`, never a panic and -//! never a confidently-wrong number. See `../wiki/concepts/metric-registry.md`. +//! never a confidently-wrong number. use std::collections::HashMap; diff --git a/crates/rocm-dash-daemon/src/demo.rs b/crates/rocm-dash-daemon/src/demo.rs index 50ede75a7e..906161474b 100644 --- a/crates/rocm-dash-daemon/src/demo.rs +++ b/crates/rocm-dash-daemon/src/demo.rs @@ -6,13 +6,14 @@ //! //! Simulates a single 8× AMD Instinct MI355X node serving inference traffic //! through three vLLM containers. Output is the same `PersistedEntry` NDJSON -//! the daemon writes when `--persist-dir` is set, so `rocm replay` (and -//! `rocm demo`) play it back unmodified. +//! the daemon's session writer produces, so `rocm dash --replay` plays it back +//! unmodified. //! //! Deterministic: the same [`DemoOptions`] always produce byte-identical //! output (seeded xorshift jitter + a fixed wallclock anchor), which the -//! tests below pin as a hard invariant. `rocm gen-demo`, `rocm demo`, and the -//! marketing screenshot/cast examples all consume this one generator. +//! tests below pin as a hard invariant. `rocm dash --demo` is the consumer: it +//! writes a session to a private file under the user data directory with +//! [`generate_file`] and replays it. use std::collections::BTreeMap; use std::f64::consts::PI; @@ -557,7 +558,6 @@ fn build_sysinfo() -> GpuSystemInfo { memory_partition_mode: MemoryPartitionMode::Nps1, compute_partition_mode: ComputePartitionMode::Spx, vram_per_logical_gpu_mb: VRAM_TOTAL_MB, - lemond_version: None, llama_server_build: None, ccr_version: None, llamacpp_backend: None, @@ -613,6 +613,7 @@ fn build_instances(t_s: f64, seed: &mut u64) -> Vec { }; Instance { + managed: false, container_id: c.container_id.into(), container_name: c.container_name.into(), status: InstanceStatus::Running, @@ -629,6 +630,7 @@ fn build_instances(t_s: f64, seed: &mut u64) -> Vec { waiting_reqs: Some(waiting), gen_tps: Some(gen_tps), gen_tps_observation: None, + gen_tps_last_request: false, tokens_per_watt: None, power_w: None, ttft_ms, diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index 1bb31a4512..2ec6364888 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -23,12 +23,15 @@ //! (`EngineKind::default_port`, `docker.rs`, `lemonade.rs`) remain only as a //! fallback for *unmanaged/external* discovery. //! -//! **Co-located scope (known limitation):** scrape targets carry only a port; -//! the runner's vLLM collector scrapes `opts.vllm_metrics_host` (default -//! `127.0.0.1`), so `record.host` is honored only for the co-located daemon -//! (the same assumption the Docker-discovery path already makes). A managed -//! service on a non-loopback host is surfaced but its metrics are scraped at the -//! local address. Per-host scrape targeting is tracked as a D7 follow-up. +//! **Lifecycle ownership:** only this registry seam marks discovered services +//! as managed. External protocol and container discovery remain read-only; +//! instance ids and engine markers are not evidence of lifecycle ownership. +//! +//! **Host scope:** managed Lemonade telemetry honors `record.host` and port. +//! The vLLM collector still scrapes `opts.vllm_metrics_host` (default +//! `127.0.0.1`) at the record's port, so a non-loopback vLLM service is surfaced +//! but its metrics use the configured collector host. External Lemonade +//! discovery uses its own configured host and port. //! //! **A registry that cannot be read in full (trade-off):** a record file that //! exists but cannot be opened (a root-owned file left by `sudo rocm serve`, a @@ -57,9 +60,10 @@ use serde::Deserialize; /// Minimal read-only view of a rocm-cli `ManagedServiceRecord` on disk. /// -/// Mirrors the stable subset the scrape seam needs; unknown fields (manifest paths, -/// pids, recipe json, …) are ignored. Every field defaults so partial/older -/// records still parse. +/// Mirrors the stable subset the scrape seam needs; the engine and supervisor +/// pids (with their start times) are kept to match GPU processes by ancestry; +/// unknown fields (manifest paths, recipe json, …) are ignored. Every field +/// defaults so partial/older records still parse. #[derive(Debug, Clone, Deserialize)] pub struct ServiceRecord { #[serde(default)] @@ -86,10 +90,46 @@ pub struct ServiceRecord { /// joins it to GPU power and VRAM. #[serde(default)] pub gpu_indices: Vec, + /// The launcher process rocm-cli recorded for the service. + #[serde(default)] + pub supervisor_pid: u32, + /// Kernel start time of the supervisor, which tells it from a recycled PID. + #[serde(default)] + pub supervisor_start_ticks: Option, + /// The engine server process, once the supervisor has seen it. + #[serde(default)] + pub engine_pid: Option, + /// Kernel start time of the engine server process. + #[serde(default)] + pub engine_start_ticks: Option, #[serde(default)] pub created_at_unix_ms: u128, } +impl ServiceRecord { + /// The recorded processes whose descendants are this service's GPU + /// processes: the engine server and its launcher. Either can be absent on + /// an old or half-written record, and a PID that is zero (not recorded) or + /// 1 (init, an ancestor of everything) is never a root, since it would claim + /// processes that belong to no service. + pub fn ancestry_roots(&self) -> Vec { + let root = |pid: u32, start_ticks| rocm_dash_collectors::ancestry::Root { + pid, + start_ticks, + owner: self.service_id.clone(), + }; + let usable = |pid: u32| pid >= rocm_dash_collectors::ancestry::MIN_ROOT_PID; + let mut roots = Vec::new(); + if let Some(pid) = self.engine_pid.filter(|p| usable(*p)) { + roots.push(root(pid, self.engine_start_ticks)); + } + if usable(self.supervisor_pid) { + roots.push(root(self.supervisor_pid, self.supervisor_start_ticks)); + } + roots + } +} + /// Service statuses worth scraping. /// /// Matches the live set the rocm-cli supervisor overlays onto a record @@ -313,6 +353,7 @@ pub fn discovered_from_record(record: &ServiceRecord) -> Option, pub seen: HashSet, pub non_vllm: HashSet, + /// Read-only Lemonade telemetry targets at registry-owned endpoints. + pub lemonade: Vec, /// How many records were skipped because they are no longer running /// (`failed`/`stopped`/unrecognized status). These are dropped from the /// scrape set on purpose - polling a dead port is pointless - but they are @@ -349,6 +392,90 @@ pub struct ManagedDiscovery { /// A live record with no bound port is NOT counted: it is skipped for a /// different reason and is not a past attempt. pub past_attempts: usize, + /// Ancestry roots of every live service surfaced, so a GPU process can be + /// attributed to the managed service it belongs to. + pub roots: Vec, + /// Ids of records that were read in this pass and found no longer running. + /// A partial read keeps what it did not see; it must still drop the roots + /// of a service it saw stop, or a dead service's PID would stay matchable. + pub retired: HashSet, +} + +/// A managed Lemonade endpoint, including both display and canonical names so +/// a CLI alias does not prevent safe last-request attribution. +#[derive(Debug, Clone)] +pub struct ManagedLemonadeTarget { + pub service_id: String, + pub host: String, + pub port: u16, + pub model_ref: String, + pub canonical_model_id: String, +} + +/// The loopback families a bind host is reachable through. `::` is a +/// dual-stack wildcard (both loopbacks); `::1` is IPv6-only; `localhost`, +/// empty, `127.0.0.1` and `0.0.0.0` are IPv4-only. No DNS is resolved. +enum HostReach { + Loopback { v4: bool, v6: bool }, + Other(String), +} + +fn normalized_host(host: &str) -> String { + host.trim() + .trim_start_matches('[') + .trim_end_matches(']') + .to_ascii_lowercase() +} + +fn host_reach(host: &str) -> HostReach { + let host = normalized_host(host); + match host.as_str() { + "" | "localhost" | "127.0.0.1" | "0.0.0.0" => HostReach::Loopback { + v4: true, + v6: false, + }, + "::" => HostReach::Loopback { v4: true, v6: true }, + "::1" => HostReach::Loopback { + v4: false, + v6: true, + }, + _ => HostReach::Other(host), + } +} + +/// Whether two co-located HTTP bindings address the same service. Wildcard +/// binds are reachable through loopback (`::` through both families); unrelated +/// remote hosts are not aliases. +pub fn endpoints_match(left_host: &str, left_port: u16, right_host: &str, right_port: u16) -> bool { + if left_port != right_port { + return false; + } + match (host_reach(left_host), host_reach(right_host)) { + (HostReach::Loopback { v4: l4, v6: l6 }, HostReach::Loopback { v4: r4, v6: r6 }) => { + (l4 && r4) || (l6 && r6) + } + (HostReach::Other(left), HostReach::Other(right)) => left == right, + _ => false, + } +} + +/// The host a collector should dial for a registry bind host: wildcards are not +/// valid destinations on every platform, so `::` is reached through `::1`. +pub fn scrape_host(host: &str) -> String { + match normalized_host(host).as_str() { + "" | "localhost" | "0.0.0.0" => "127.0.0.1".to_string(), + "::" | "::1" => "::1".to_string(), + host => host.to_string(), + } +} + +/// Whether a collector cached for `cached_host` may serve `target_host`. +/// +/// `endpoints_match` is deliberately non-transitive (`::` covers `127.0.0.1`), so +/// a match alone is not enough: the cached collector dials a fixed address, and +/// reusing it for a target in another address family probes the wrong listener. +pub fn cached_collector_reusable(cached_host: &str, target_host: &str) -> bool { + scrape_host(cached_host) == scrape_host(target_host) } /// Convert the loaded registry records into a [`ManagedDiscovery`] — the pure core of the daemon's managed-service discovery tick. @@ -356,10 +483,9 @@ pub struct ManagedDiscovery { /// Non-scrapeable records (bad status / port 0) are dropped; vLLM-engine services flow to the /// Prometheus scrape, others are flagged in `non_vllm`. /// -/// NOTE: gen_tps for managed **non-vLLM** engines (e.g. Lemonade) is not yet -/// wired — they appear in the dashboard but are excluded from the vLLM scrape -/// and not routed to a per-engine collector here. vLLM (the Phase-2 acceptance -/// target) is fully wired. Non-vLLM managed scraping is a D7 follow-up. +/// Managed Lemonade endpoints are routed to the existing Lemonade collector. +/// Servers without that protocol (including direct llama-server) keep their +/// registry row, with throughput unavailable rather than fabricated. pub fn discover_managed_services(records: &[ServiceRecord]) -> ManagedDiscovery { let mut out = ManagedDiscovery::default(); for record in records { @@ -368,10 +494,29 @@ pub fn discover_managed_services(records: &[ServiceRecord]) -> ManagedDiscovery // live record with no bound port. Only the first is a past attempt. if !is_scrapeable_status(&record.status) { out.past_attempts += 1; + out.retired.insert(record.service_id.clone()); } continue; }; + if engine_kind_for(record) == Some(EngineKind::Lemonade) { + // Keep a `::` bind as `::`: it is dual-stack, which only + // `endpoints_match` can see. The runner dials it via `scrape_host`. + let host = match record.host.trim() { + "" | "localhost" | "0.0.0.0" => "127.0.0.1".to_string(), + "[::]" => "::".to_string(), + "[::1]" => "::1".to_string(), + host => host.to_string(), + }; + out.lemonade.push(ManagedLemonadeTarget { + service_id: svc.container_id.clone(), + host, + port: record.port, + model_ref: record.model_ref.clone(), + canonical_model_id: record.canonical_model_id.clone(), + }); + } out.seen.insert(svc.container_id.clone()); + out.roots.extend(record.ancestry_roots()); if engine_kind_for(record) != Some(EngineKind::Vllm) { out.non_vllm.insert(svc.container_id.clone()); } @@ -388,6 +533,82 @@ mod tests { use std::path::PathBuf; use std::time::Duration; + #[test] + fn endpoint_matching_does_not_conflate_distinct_ports_hosts_or_ip_families() { + let same = [ + ("localhost", "127.0.0.1"), + ("0.0.0.0", "127.0.0.1"), + ("", "127.0.0.1"), + ("[::]", "::1"), + ("::", "::1"), + ("::", "127.0.0.1"), + ("[::]", "localhost"), + ("::", "0.0.0.0"), + ("::", "::"), + ("::1", "[::1]"), + ("Remote.Example", "remote.example"), + ]; + for (l, r) in same { + assert!(endpoints_match(l, 13305, r, 13305), "{l} ~ {r}"); + assert!(endpoints_match(r, 13305, l, 13305), "{r} ~ {l}"); + } + let distinct = [ + ("::1", "127.0.0.1"), + ("::1", "localhost"), + ("::1", "0.0.0.0"), + ("::1", ""), + ("::", "remote.example"), + ("127.0.0.1", "remote.example"), + ("remote.example", "other.example"), + ]; + for (l, r) in distinct { + assert!(!endpoints_match(l, 13305, r, 13305), "{l} !~ {r}"); + assert!(!endpoints_match(r, 13305, l, 13305), "{r} !~ {l}"); + } + assert!(!endpoints_match("::", 13306, "::", 13305)); + assert!(!endpoints_match("127.0.0.1", 13306, "127.0.0.1", 13305)); + } + + #[test] + fn cached_collector_is_reused_only_when_it_dials_the_same_host() { + // Same dialed address: reuse across spelling differences. + for (cached, target) in [ + ("127.0.0.1", "localhost"), + ("0.0.0.0", "127.0.0.1"), + ("::", "[::]"), + ("::1", "::"), + ("remote.example", "remote.example"), + ] { + assert!( + cached_collector_reusable(cached, target), + "{cached} -> {target}" + ); + } + // `endpoints_match` accepts these, but the cached collector would dial + // the other address family. + for (cached, target) in [("::", "127.0.0.1"), ("127.0.0.1", "::"), ("::", "0.0.0.0")] { + assert!(endpoints_match(cached, 13305, target, 13305)); + assert!( + !cached_collector_reusable(cached, target), + "{cached} -> {target}" + ); + } + } + + #[test] + fn dual_stack_bind_keeps_its_host_and_is_dialed_through_ipv6_loopback() { + let record: ServiceRecord = serde_json::from_str( + r#"{"service_id":"s","engine":"lemonade","model_ref":"m","host":"::","port":13305,"status":"ready"}"#, + ) + .unwrap(); + let disc = discover_managed_services(&[record]); + assert_eq!(disc.lemonade[0].host, "::"); + assert_eq!(scrape_host("::"), "::1"); + assert_eq!(scrape_host("[::]"), "::1"); + assert_eq!(scrape_host("0.0.0.0"), "127.0.0.1"); + assert_eq!(scrape_host("10.0.0.5"), "10.0.0.5"); + } + fn at(secs: i64) -> DateTime { Utc.timestamp_opt(secs, 0).unwrap() } @@ -738,6 +959,7 @@ mod tests { .unwrap(); let records = load_service_records(&dir); let svc = discovered_from_record(&records[0]).expect("live record converts"); + assert!(svc.managed, "registry discovery proves lifecycle ownership"); assert_eq!(svc.port, Some(8123)); assert_eq!(svc.container_id, "svc-hot"); assert_eq!(svc.model_name, "meta-llama/Llama-3.1-8B"); @@ -784,6 +1006,104 @@ mod tests { } } + #[test] + fn ancestry_roots_come_from_the_recorded_engine_and_supervisor() { + let with = record_json("svc", "vllm", 8000, "ready", 1).replace( + r#""supervisor_pid": 4242,"#, + r#""supervisor_pid": 4242, "supervisor_start_ticks": 11, "engine_pid": 4300, "engine_start_ticks": 22,"#, + ); + let rec: ServiceRecord = serde_json::from_str(&with).unwrap(); + let roots = rec.ancestry_roots(); + let got: Vec<_> = roots.iter().map(|r| (r.pid, r.start_ticks)).collect(); + assert_eq!(got, vec![(4300, Some(22)), (4242, Some(11))]); + assert!(roots.iter().all(|r| r.owner == "svc")); + // An old record: no engine yet, no start times. The supervisor alone. + let old: ServiceRecord = + serde_json::from_str(&record_json("svc", "vllm", 8000, "ready", 1)).unwrap(); + let got: Vec<_> = old + .ancestry_roots() + .iter() + .map(|r| (r.pid, r.start_ticks)) + .collect(); + assert_eq!(got, vec![(4242, None)]); + // Zero is "not recorded" and 1 is init: neither is ever a root, or it + // would own every process on the host. + for bad in [0, 1] { + let json = format!( + r#"{{"service_id":"z","engine":"vllm","port":1,"status":"ready","supervisor_pid":{bad},"engine_pid":{bad}}}"# + ); + let rec: ServiceRecord = serde_json::from_str(&json).unwrap(); + assert!( + rec.ancestry_roots().is_empty(), + "pid {bad} must not be a root" + ); + } + } + + #[test] + fn recorded_pids_survive_the_producer_to_daemon_serde_handoff() { + // Same contract as the startup-phase test above, for the pid fields the + // ancestry join reads: serialize the producer's real type and read it + // back as ours, so a rename or retype on either side fails here. + use rocm_core::{AppPaths, ManagedServiceRecord}; + + let root = PathBuf::from("/tmp/rocm-xcrate-pids"); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + let mut produced = ManagedServiceRecord::new( + &paths, + "svc-pids", + "vllm", + "M/pids", + "M/pids", + "127.0.0.1", + 8000, + "serve", + 4242, + None, + None, + None, + ); + produced.status = "ready".to_owned(); + produced.supervisor_start_ticks = Some(111); + produced.engine_pid = Some(4300); + produced.engine_start_ticks = Some(222); + + let json = serde_json::to_string(&produced).expect("producer record serializes"); + let consumed: ServiceRecord = + serde_json::from_str(&json).expect("daemon record deserializes the producer's JSON"); + let got: Vec<_> = consumed + .ancestry_roots() + .iter() + .map(|r| (r.pid, r.start_ticks, r.owner.clone())) + .collect(); + assert_eq!( + got, + vec![ + (4300, Some(222), "svc-pids".to_string()), + (4242, Some(111), "svc-pids".to_string()), + ], + "engine and supervisor pids and start times must cross the serde handoff by name" + ); + } + + #[test] + fn discover_managed_services_collects_roots_only_for_live_services() { + let live = record_json("live", "vllm", 8000, "ready", 1); + let dead = record_json("dead", "vllm", 8001, "stopped", 1).replace("4242", "5252"); + let recs: Vec = [live, dead] + .iter() + .map(|j| serde_json::from_str(j).unwrap()) + .collect(); + let disc = discover_managed_services(&recs); + let owners: Vec<_> = disc.roots.iter().map(|r| r.owner.as_str()).collect(); + assert_eq!(owners, vec!["live"], "a stopped service owns nothing"); + assert!(disc.retired.contains("dead") && !disc.retired.contains("live")); + } + #[test] fn discovered_from_record_carries_gpu_indices_as_gpu_ids() { // The recorded launch GPUs are what join an instance to GPU power and diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 4af9201368..68b2f297e3 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -10,6 +10,13 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; +mod discovery; + +use discovery::{ + DiscoverySource, DiscoveryState, Reconcile, TelemetryState, registry_read_warning, + vllm_scrape_targets, +}; + use crate::bench_ring::BenchRing; use crate::persist::SessionWriter; use crate::snapshot_ring::SnapshotRing; @@ -39,6 +46,43 @@ use tracing::{info, trace, warn}; /// Refresh GPU system info (versions, partition modes) every N seconds. const SYSINFO_REFRESH_SECS: u64 = 30; +/// Row identity and accepted model names for singleton Lemonade attribution. +#[derive(Clone)] +struct LemonadeRow { + id: String, + model_names: Vec, +} + +#[derive(Clone)] +struct LemonadeEndpoint { + host: String, + port: u16, + collector: LemonadeCollector, + rows: Vec, + /// The endpoint is the external probe (not a registry-managed target). This + /// decides how strictly health is read: an external server that starts to + /// require credentials is a failure to report, a managed target that does not + /// serve the Lemonade protocol is not. It is a property of the endpoint, not + /// of the tick. + external: bool, + /// This tick is a discovery tick: rows are rebuilt from health. External + /// endpoints also appear on scrape-only ticks, with this `false`. + discover_external: bool, +} + +impl LemonadeEndpoint { + fn warning(&self, phase: &str, error: &impl std::fmt::Display) -> String { + let mut ids: Vec<&str> = self.rows.iter().map(|row| row.id.as_str()).collect(); + ids.sort_unstable(); + format!( + "lemonade {phase} {}:{} [{}]: {error}", + self.host, + self.port, + ids.join(", ") + ) + } +} + /// Options for `run_loop`. Mirrors daemon CLI flags + config. #[derive(Debug, Clone)] pub struct RunnerOptions { @@ -58,9 +102,10 @@ pub struct RunnerOptions { pub disable_vllm_metrics: bool, /// Hostname to scrape; `127.0.0.1` for the typical co-located daemon. pub vllm_metrics_host: String, - /// Opt-in probe-based Lemonade discovery: when enabled, probe a local - /// Lemonade endpoint each discovery tick and surface it as an Instance. - /// Off by default so hosts with no Lemonade server never poll a dead port. + /// Probe-based Lemonade discovery: when enabled, probe a local Lemonade + /// endpoint each discovery tick and surface its healthy loaded models. + /// `rocm dash` passes the dashboard config value, which defaults to on; + /// only `RunnerOptions::default()` is off, for standalone use and tests. pub enable_lemonade: bool, /// Lemonade endpoint host + port (defaults `127.0.0.1:13305`). pub lemonade_host: String, @@ -72,7 +117,7 @@ pub struct RunnerOptions { /// When set, the daemon reads `ManagedServiceRecord`s each discovery tick and /// surfaces live ones as scrape targets (port from the registry), so a model /// served via `rocm serve` appears in the dashboard with live `gen_tps` - /// without Docker discovery. Off by default. + /// without Docker discovery. `None` by default; `rocm dash` sets it. pub services_dir: Option, /// Path or command name for the `amd-smi` binary. The managed ROCm SDK /// ships it inside the runtime wheel's bin directory rather than on `PATH`, @@ -318,8 +363,9 @@ pub async fn run_loop( None }; - // Opt-in Lemonade discovery (probe-based; off unless a Lemonade endpoint is - // configured). Distinct from Docker/vLLM discovery — a local server, not a + // Probe-based Lemonade discovery, gated on `opts.enable_lemonade` (the + // dashboard config default is on; `RunnerOptions::default()` is off). + // Distinct from Docker/vLLM discovery — a local server, not a // container — so it is tracked separately from the Docker known set. let lemonade = if opts.enable_lemonade { info!( @@ -335,6 +381,8 @@ pub async fn run_loop( } else { None }; + // Persistent endpoint collectors share their reqwest connection pools across cycles. + let mut managed_lemonade_collectors: HashMap<(String, u16), LemonadeCollector> = HashMap::new(); let mut bench = opts.bench_csv.as_ref().map(|p| { info!(path = %p.display(), "tailing benchmark CSV"); CsvBenchTailer::new(p.clone()) @@ -407,6 +455,7 @@ pub async fn run_loop( let mut docker_warn: Option = None; let mut registry_warn: Option = None; let mut scrape_warns: Vec = Vec::new(); + let mut lemonade_discovery_warn: Option = None; loop { ticker.tick().await; @@ -577,16 +626,6 @@ pub async fn run_loop( } } - // Lemonade discovery — probe the endpoint on the discovery cadence; add a - // Lemonade Instance when reachable, emit Gone when it disappears, and stay - // a clean no-op (no warning/panic) when no endpoint is configured. - if let Some(l) = lemonade.as_ref() - && discovery_due - { - let probed = l.discover().await; - reconcile.reconcile_lemonade(probed, &mut discovery.lemonade_id); - } - // Managed-service registry discovery — read the rocm-cli // `serve` records and surface live ones as scrape targets. The port is // the registry's authority; non-vLLM engines are tracked so the vLLM @@ -610,6 +649,14 @@ pub async fn run_loop( Ok(read) => { let disc = crate::registry::discover_managed_services(&read.records); discovery.managed_non_vllm = disc.non_vllm; + discovery.managed_roots = disc.roots; + let previous = + std::mem::replace(&mut discovery.managed_lemonade, disc.lemonade); + reconcile.reset_moved_lemonade_targets( + &disc.svcs, + &previous, + &discovery.managed_lemonade, + ); reconcile.reconcile_set( DiscoverySource::Managed, &disc.svcs, @@ -623,7 +670,13 @@ pub async fn run_loop( // The records that did read still add and refresh their // services; nothing is retired, because the one that did // not read could be any of the known ones. - let disc = crate::registry::discover_managed_services(&e.partial.records); + let mut disc = crate::registry::discover_managed_services(&e.partial.records); + let previous = discovery.keep_lemonade_through_partial_registry(&mut disc); + reconcile.reset_moved_lemonade_targets( + &disc.svcs, + &previous, + &discovery.managed_lemonade, + ); reconcile.upsert_set( DiscoverySource::Managed, &disc.svcs, @@ -638,9 +691,220 @@ pub async fn run_loop( scrape_warns.clear(); } - // Per-instance vLLM metric scrape, parallel, on its own cadence. The - // Lemonade instance (if any) is scraped via its own collector below, so - // exclude it from the vLLM Prometheus targets. Managed non-vLLM services + // Independent endpoints run concurrently, but health always precedes + // stats within each endpoint. Last-request stats have no model identity: + // only a singleton loaded set matching exactly one row can own the rate. + let scrape_now = tick_count.is_multiple_of(instance_ticks); + if discovery_due || scrape_now { + let managed_endpoint = discovery.managed_lemonade.iter().any(|target| { + crate::registry::endpoints_match( + &target.host, + target.port, + &opts.lemonade_host, + opts.lemonade_port, + ) + }); + let mut endpoints = Vec::new(); + if let Some(collector) = lemonade.as_ref() { + if managed_endpoint && discovery_due { + // Registry binding wins even if its configured model differs + // from health. Never attach external controls to that binding. + Reconcile { + state: &mut runner.state, + telemetry: &mut telemetry, + tx: &tx, + persist: persist.as_ref(), + } + .reconcile_lemonade(&[], &mut discovery.lemonade_ids); + lemonade_discovery_warn = None; + } else if !managed_endpoint && (discovery_due || !discovery.lemonade_ids.is_empty()) + { + endpoints.push(LemonadeEndpoint { + host: opts.lemonade_host.clone(), + port: opts.lemonade_port, + collector: collector.clone(), + rows: discovery + .lemonade_ids + .iter() + .filter_map(|id| { + runner.state.instances.get(id).map(|inst| LemonadeRow { + id: id.clone(), + model_names: vec![inst.model_name.clone()], + }) + }) + .collect(), + external: true, + discover_external: discovery_due, + }); + } + } + let mut groups: HashMap<(String, u16), Vec> = HashMap::new(); + for target in &discovery.managed_lemonade { + // Use the registry's endpoint equivalence without rewriting the + // selected request host or conflating IPv4 and IPv6 listeners. + // Prefer the cached representative so reorder keeps its client. + // A cached collector dials a fixed address, so it is reused + // only when it would dial the same host as this target. + let endpoint = groups + .keys() + .find(|(host, port)| { + crate::registry::endpoints_match(host, *port, &target.host, target.port) + && crate::registry::cached_collector_reusable(host, &target.host) + }) + .or_else(|| { + managed_lemonade_collectors.keys().find(|(host, port)| { + crate::registry::endpoints_match(host, *port, &target.host, target.port) + && crate::registry::cached_collector_reusable(host, &target.host) + }) + }) + .cloned() + .unwrap_or_else(|| (target.host.clone(), target.port)); + groups.entry(endpoint).or_default().push(LemonadeRow { + id: target.service_id.clone(), + model_names: vec![target.model_ref.clone(), target.canonical_model_id.clone()], + }); + } + managed_lemonade_collectors.retain(|endpoint, _| groups.contains_key(endpoint)); + if scrape_now { + for ((host, port), rows) in groups { + let collector = managed_lemonade_collectors + .entry((host.clone(), port)) + .or_insert_with(|| { + LemonadeCollector::new( + crate::registry::scrape_host(&host), + port, + Duration::from_millis(1500), + ) + }) + .clone(); + endpoints.push(LemonadeEndpoint { + host, + port, + collector, + rows, + external: false, + discover_external: false, + }); + } + } + let results = parallel_scrape(endpoints, move |endpoint| async move { + // A managed row can be a server without the Lemonade protocol + // (a direct llama-server). It stays listed with throughput + // unavailable and no standing warning; real failures still warn. + let health = if endpoint.external { + endpoint.collector.fetch_health().await? + } else if let Some(health) = endpoint.collector.fetch_health_if_router().await? { + health + } else { + return Ok((None, endpoint.rows.clone(), None, None)); + }; + let discovered = endpoint.discover_external.then(|| { + rocm_dash_collectors::lemonade::services_from_health( + &endpoint.host, + endpoint.port, + &health, + ) + }); + let rows = discovered.as_ref().map_or_else( + || endpoint.rows.clone(), + |services| { + services + .iter() + .map(|service| LemonadeRow { + id: service.container_id.clone(), + model_names: vec![service.model_name.clone()], + }) + .collect() + }, + ); + let attribution = health.stats_model(); + let mut matching = rows.iter().filter(|row| { + attribution + .as_ref() + .is_some_and(|model| row.model_names.iter().any(|name| name == model)) + }); + let attributed_id = matching + .next() + .filter(|_| matching.next().is_none()) + .map(|row| row.id.clone()); + // The stats ride with the id they belong to, so a fetch cannot + // exist without the row that attribution picked. + let stats = match attributed_id.clone() { + Some(id) if scrape_now => Some((id, endpoint.collector.fetch_stats().await)), + _ => None, + }; + Ok::<_, rocm_dash_core::traits::CollectorError>(( + discovered, + rows, + attributed_id, + stats, + )) + }) + .await; + for (endpoint, result) in results { + let (discovered, rows, attributed_id, stats) = match result { + Ok(result) => { + if endpoint.discover_external { + lemonade_discovery_warn = None; + } + result + } + Err(error) => { + // A failed external discovery still removes rows immediately, + // but reports why they disappeared. A never-seen endpoint + // with nothing listening stays quiet; one that answers with + // an HTTP or body error warns even on first contact. + // Managed failures retain only the existing hold. + if endpoint.discover_external { + let absent = endpoint.rows.is_empty() + && rocm_dash_collectors::lemonade::is_connect_error(&error); + lemonade_discovery_warn = + (!absent).then(|| endpoint.warning("health", &error)); + (Some(Vec::new()), Vec::new(), None, None) + } else { + scrape_warns.push(endpoint.warning("health", &error)); + continue; + } + } + }; + if let Some(services) = discovered { + Reconcile { + state: &mut runner.state, + telemetry: &mut telemetry, + tx: &tx, + persist: persist.as_ref(), + } + .reconcile_lemonade(&services, &mut discovery.lemonade_ids); + } + // Healthy contradictory/ambiguous evidence invalidates prior + // attribution immediately, even off the stats cadence. + for row in &rows { + if Some(&row.id) != attributed_id.as_ref() { + telemetry.gen_trackers.remove(&row.id); + } + } + if let Some((stats_id, stats)) = stats { + match stats { + Ok(sample) => { + if let Some(rate) = sample.gen_tps { + telemetry + .gen_trackers + .entry(stats_id) + .or_insert_with(|| { + GenerationObservationTracker::new(opts.instance_tick) + }) + .observe_direct(rate, cycle_at); + } + } + Err(error) => scrape_warns.push(endpoint.warning("stats", &error)), + } + } + } + } + + // Per-instance vLLM metric scrape, parallel, on its own cadence. + // Lemonade rows are scraped via their endpoint collectors above, so + // exclude them from the vLLM Prometheus targets. Managed non-vLLM services // are excluded too (their engine uses a different parser). if let Some(prom) = vllm.as_ref() && !runner.state.instances.is_empty() @@ -648,7 +912,7 @@ pub async fn run_loop( { let targets = vllm_scrape_targets( &runner.state.instances, - discovery.lemonade_id.as_deref(), + &discovery.lemonade_ids, &discovery.managed_non_vllm, ); let prom = prom.clone(); @@ -746,52 +1010,6 @@ pub async fn run_loop( } } - // Lemonade per-instance scrape — reports an instantaneous rate directly - // (`gen_tps`), so no counter-differencing. A scrape failure leaves the - // last-known fields and warns; it never panics. - if let (Some(l), Some(id)) = (lemonade.as_ref(), discovery.lemonade_id.clone()) - && tick_count.is_multiple_of(instance_ticks) - { - match l.fetch_stats().await { - Ok(sample) => { - // EAI-7960: drive the tracker with a direct rate reading when - // the sample carries one. A missing direct rate (None) does not - // erase or refresh the tracker; the held value stays valid until - // the validity window expires. - if let Some(rate) = sample.gen_tps { - telemetry - .gen_trackers - .entry(id.clone()) - .or_insert_with(|| { - GenerationObservationTracker::new(opts.instance_tick) - }) - .observe_direct(rate, cycle_at); - } - if let Some(mut inst) = runner.state.instances.get(&id).cloned() { - // gen_tps is written in the snapshot assembly below; - // set the KV/request fields immediately so they are - // present in state for subsequent snapshots. - inst.kv_cache_usage_pct = sample.kv_cache_usage_pct; - inst.running_reqs = sample.running_reqs; - inst.waiting_reqs = sample.waiting_reqs; - // See the vLLM scrape-success handler: a successful stats - // fetch proves the instance is serving; promote out of Starting. - if matches!(inst.status, InstanceStatus::Starting { .. }) { - inst.status = InstanceStatus::Ready; - } - runner.state.apply(StateEvent::InstanceUpserted(inst)); - } - } - Err(e) => { - trace!(id = %id, error = %e, "lemonade scrape failed"); - scrape_warns.push(format!("lemonade scrape: {e}")); - // EAI-7960: retain tracker during validity window on failure. - // gen_tps is assembled from the tracker snapshot below; - // no explicit clearing needed here. - } - } - } - // Per-process VRAM attribution: refresh the per-container map on the // instance cadence via one amd-smi `process` scrape, joining GPU-process // host PIDs to container ids through `/proc//cgroup`. On Err/empty @@ -805,10 +1023,18 @@ pub async fn run_loop( match g.processes().await { Ok(procs) => { procs_nonempty = !procs.is_empty(); - per_container_used = rocm_dash_core::vram::aggregate_process_vram( - &procs, - rocm_dash_collectors::cgroup::container_id_for_pid, - ); + // A container id when the process is in a docker/containerd + // cgroup, otherwise the managed service whose engine it + // descends from. Both key the same map the instances read. + per_container_used = + rocm_dash_core::vram::aggregate_process_vram(&procs, |pid| { + rocm_dash_collectors::cgroup::container_id_for_pid(pid).or_else(|| { + rocm_dash_collectors::ancestry::owner_of_pid( + pid, + &discovery.managed_roots, + ) + }) + }); } Err(e) => { // Keep the last-known map; device fallback covers the gap. @@ -832,6 +1058,10 @@ pub async fn run_loop( snapshot_latency(&mut telemetry.prev_ttft, &inst.container_id, cycle_at); (inst.tpot_ms, inst.tpot_observation) = snapshot_latency(&mut telemetry.prev_tpot, &inst.container_id, cycle_at); + inst.gen_tps_last_request = telemetry + .gen_trackers + .get(&inst.container_id) + .is_some_and(GenerationObservationTracker::is_direct); } // Derive per-instance efficiency now that this tick's GPU power is known // and gen_tps reflects the tracker state for this cycle. @@ -846,7 +1076,7 @@ pub async fn run_loop( for (inst, power_w) in instances.iter_mut().zip(attributed) { inst.power_w = power_w; inst.tokens_per_watt = - rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, power_w); + rocm_dash_core::efficiency::instance_tokens_per_watt(inst, power_w); if inst.gen_tps.is_some() && !inst.gpu_ids.is_empty() && !gpus.is_empty() @@ -872,6 +1102,7 @@ pub async fn run_loop( entry.ttft_observation = inst.ttft_observation.clone(); entry.tpot_ms = inst.tpot_ms; entry.tpot_observation = inst.tpot_observation.clone(); + entry.gen_tps_last_request = inst.gen_tps_last_request; entry.tokens_per_watt = inst.tokens_per_watt; entry.power_w = inst.power_w; } @@ -882,16 +1113,23 @@ pub async fn run_loop( // Warn only when a real process scrape happened but an instance with // live GPUs still fell back to device-summed (cgroup join missed). if procs_nonempty { - let vram_fallbacks = count_vram_join_misses(&instances, &gpus, &per_container_used); + let vram_fallbacks = count_vram_join_misses( + &instances, + &gpus, + &per_container_used, + &discovery.managed_roots, + &discovery.managed_non_vllm, + ); if vram_fallbacks > 0 { scrape_warns.push(format!( "vram attribution: {vram_fallbacks} instance(s) fell back to device-summed \ - VRAM (no per-process cgroup match; check /proc access)" + VRAM (no per-process cgroup or ancestry match; check /proc access)" )); } } warnings.extend(docker_warn.iter().cloned()); warnings.extend(registry_warn.iter().cloned()); + warnings.extend(lemonade_discovery_warn.iter().cloned()); warnings.extend(scrape_warns.iter().cloned()); let snap = Snapshot { timestamp: cycle_at, @@ -934,20 +1172,32 @@ pub async fn run_loop( /// Instances that *could* have joined the per-process VRAM map but did not. /// -/// The map is keyed by docker/containerd ids read from `/proc//cgroup`, -/// so only an instance whose id has that shape can ever hit it. A managed -/// `rocm serve` service (named by its service id) falls back to device-summed -/// VRAM by design, and counting it would raise a false "check /proc access" -/// warning on every healthy managed instance. +/// Two kinds of instance can hit the map: one whose id is a docker/containerd +/// id read from `/proc//cgroup`, and a managed `rocm serve` service that +/// has recorded `roots` (its engine/supervisor PIDs) to match by ancestry. A +/// managed service with no roots cannot join and falls back to device-summed +/// VRAM by design, so it is not counted. One that is not yet `Ready`/`Running` +/// may still be loading its model with no GPU process to attribute, so it is +/// not counted either; a serving one always holds VRAM, so a missing entry +/// means its ancestry walk failed (unreadable `/proc`, a recycled root PID). +/// A non-vLLM managed service (e.g. Lemonade on a backend `amd-smi` does not +/// list) is not counted: it may hold GPU memory without ever appearing in the +/// process list, which is not an access problem. fn count_vram_join_misses( instances: &[Instance], gpus: &[GpuMetrics], per_container_used: &HashMap, + roots: &[rocm_dash_collectors::ancestry::Root], + non_vllm: &HashSet, ) -> usize { instances .iter() .filter(|i| { - rocm_dash_collectors::cgroup::is_container_id(&i.container_id) + let can_join = rocm_dash_collectors::cgroup::is_container_id(&i.container_id) + || (matches!(i.status, InstanceStatus::Ready | InstanceStatus::Running) + && !non_vllm.contains(&i.container_id) + && roots.iter().any(|r| r.owner == i.container_id)); + can_join && !i.gpu_ids.is_empty() && rocm_dash_core::efficiency::gpu_ids_overlap(&i.gpu_ids, gpus) && !per_container_used.contains_key(&i.container_id) @@ -991,6 +1241,7 @@ pub(crate) fn merge_discovery_refresh( inst.waiting_reqs = ex.waiting_reqs; inst.gen_tps = ex.gen_tps; inst.gen_tps_observation = ex.gen_tps_observation.clone(); + inst.gen_tps_last_request = ex.gen_tps_last_request; inst.tokens_per_watt = ex.tokens_per_watt; inst.power_w = ex.power_w; inst.ttft_ms = ex.ttft_ms; @@ -1001,336 +1252,6 @@ pub(crate) fn merge_discovery_refresh( inst } -/// Per-instance telemetry the loop carries between scrapes, keyed by container id. -#[derive(Default)] -struct TelemetryState { - /// Generation-throughput trackers. Constructed lazily from - /// `opts.instance_tick`; they survive scrape failures so the last-observed - /// rate is held for the validity window before clearing. - gen_trackers: HashMap, - /// Previous TTFT / TPOT histogram readings plus a freshness tracker per - /// instance, for the windowed avg-ms derivation (mirrors `gen_trackers`). - prev_ttft: LatencyWindows, - prev_tpot: LatencyWindows, -} - -impl TelemetryState { - /// Remove all per-service telemetry state for `id`: tracker, TTFT, and - /// TPOT baselines. Call on explicit removal (Docker Gone, Lemonade Gone, - /// managed Gone) and whenever an identity change requires a clean slate. - fn purge(&mut self, id: &str) { - self.gen_trackers.remove(id); - self.prev_ttft.remove(id); - self.prev_tpot.remove(id); - } - - /// Invalidate the generation tracker for `id` (reset baseline without - /// removing the map entry) and clear the TTFT/TPOT latency windows. - /// Called when an endpoint changes within the same container/service id so - /// the new scrape address starts with a fresh rate baseline. - fn invalidate_on_endpoint_change(&mut self, id: &str) { - if let Some(t) = self.gen_trackers.get_mut(id) { - t.invalidate(); - } - self.prev_ttft.remove(id); - self.prev_tpot.remove(id); - } -} - -/// What each discovery source currently has live, so a pass can tell a new -/// service from a known one and a known one from one that has gone. -#[derive(Default)] -struct DiscoveryState { - /// Container ids the last Docker pass saw. - known_instances: HashSet, - /// Ids the last managed-service registry pass surfaced. - known_services: HashSet, - /// The subset of the registry ids whose engine is NOT vLLM, excluded from - /// the vLLM Prometheus scrape so they aren't mis-parsed. - managed_non_vllm: HashSet, - /// The id of the currently-live Lemonade instance, if any. - lemonade_id: Option, - /// How the registry reads have been going, so a condition that lasts many - /// passes is logged once rather than every discovery tick. - registry_health: RegistryHealth, -} - -impl DiscoveryState { - /// Record what a pass that read only part of the registry found. The services - /// it saw join the known set, so a later complete read can retire them, and - /// the known ones it did not see stay; the non-vLLM marks of the services it - /// saw are refreshed and the others kept, so a kept non-vLLM service is still - /// left out of the vLLM scrape. - fn keep_through_partial_registry(&mut self, disc: crate::registry::ManagedDiscovery) { - self.managed_non_vllm.retain(|id| !disc.seen.contains(id)); - self.managed_non_vllm.extend(disc.non_vllm); - self.known_services.extend(disc.seen); - } -} - -/// What the managed-service registry's reads have reported so far. A failure -/// that lasts is logged on its first pass only, the recovery once, and a set of -/// malformed records each time it changes; the dashboard's warning badge, which -/// is cheap and does not scroll, carries the failure for as long as it lasts. -#[derive(Default)] -struct RegistryHealth { - /// The previous pass could not read the registry. - failing: bool, - /// The malformed records the last successful pass reported, so an unchanged - /// set is not reported again. - reported_malformed: Vec, -} - -/// What a discovery pass should say about the registry, apart from the services -/// it found. Kept as data so [`RegistryHealth::observe`] can be tested without a -/// log subscriber. -#[derive(Debug, Default, PartialEq, Eq)] -struct RegistryChange { - /// The first failed pass of a streak, with the error. - failed: Option, - /// The first successful pass after a streak of failures. - recovered: bool, - /// The malformed records, when the set differs from the one last reported - /// and is not empty. - malformed: Vec, -} - -impl RegistryHealth { - fn observe( - &mut self, - read: &Result, - ) -> RegistryChange { - let mut change = RegistryChange::default(); - match read { - Err(e) => { - if !self.failing { - self.failing = true; - change.failed = Some(e.to_string()); - } - } - Ok(read) => { - if self.failing { - self.failing = false; - change.recovered = true; - } - if read.malformed != self.reported_malformed { - self.reported_malformed.clone_from(&read.malformed); - change.malformed.clone_from(&read.malformed); - } - } - } - change - } -} - -impl RegistryChange { - fn log(&self) { - if let Some(error) = &self.failed { - warn!( - error = %error, - "managed-service registry could not be read; keeping the known services until it can be" - ); - } - if self.recovered { - info!("managed-service registry is readable again"); - } - if !self.malformed.is_empty() { - warn!( - files = ?self.malformed, - "skipping managed-service records that are not valid JSON" - ); - } - } -} - -/// The header warning for a pass that could not read the registry. -fn registry_read_warning(error: &crate::registry::RegistryReadError) -> String { - format!("managed registry: {error}; keeping the last known services") -} - -/// Which discovery source a service came from. It only selects the log wording -/// in [`Reconcile::announce`] and [`Reconcile::retire`]; those two steps behave -/// the same for every source. How a pass decides what is new, known or gone -/// differs: Docker and the registry go through `reconcile_set`, Lemonade through -/// `reconcile_lemonade`. -#[derive(Clone, Copy)] -enum DiscoverySource { - Docker, - Lemonade, - Managed, -} - -impl DiscoverySource { - fn log_discovered(self, svc: &DiscoveredService) { - match self { - Self::Docker => info!( - id = %svc.container_id, - name = %svc.container_name, - model = %svc.model_name, - "instance discovered" - ), - Self::Lemonade => info!( - id = %svc.container_id, - model = %svc.model_name, - "lemonade instance discovered" - ), - Self::Managed => info!(id = %svc.container_id, "managed service discovered"), - } - } - - fn log_gone(self, id: &str) { - match self { - Self::Docker => info!(id = %id, "instance gone"), - Self::Lemonade => info!(id = %id, "lemonade instance gone"), - Self::Managed => info!(id = %id, "managed service gone"), - } - } -} - -/// The loop state a discovery pass changes, and the stream it announces on. -struct Reconcile<'a> { - state: &'a mut State, - telemetry: &'a mut TelemetryState, - tx: &'a broadcast::Sender, - persist: Option<&'a Arc>>, -} - -impl Reconcile<'_> { - /// Upsert a (re)discovered service. Collector-owned fields carry over from - /// the instance already in state unless `carry_live` is false, which is how - /// a changed identity starts from a clean baseline. - fn upsert(&mut self, svc: &DiscoveredService, carry_live: bool) { - let existing = self.state.instances.get(&svc.container_id).cloned(); - let inst = merge_discovery_refresh(svc, if carry_live { existing.as_ref() } else { None }); - self.state.apply(StateEvent::InstanceUpserted(inst)); - } - - /// Announce a newly discovered service. InstanceDiscovered is a - /// discovery-metadata event: it carries a clean instance, with no prior - /// collector-owned telemetry. - fn announce(&self, source: DiscoverySource, svc: &DiscoveredService) { - source.log_discovered(svc); - broadcast_and_persist( - self.tx, - self.persist, - Event::InstanceDiscovered(instance_from_discovered(svc)), - ); - } - - /// Drop a service that has gone: purge its telemetry, remove it from state - /// and announce it. - fn retire(&mut self, source: DiscoverySource, id: &str) { - source.log_gone(id); - self.telemetry.purge(id); - self.state - .apply(StateEvent::InstanceRemoved(id.to_string())); - broadcast_and_persist( - self.tx, - self.persist, - Event::InstanceGone { - container_id: id.to_string(), - }, - ); - } - - /// Reconcile a source that reports its whole set each pass (Docker, the - /// managed-service registry). `known` is what the previous pass saw and - /// `seen` what this one saw; the caller stores `seen` as the next `known`. - /// - /// A service whose port changed under the same id has a new scrape address: - /// its rate baseline and latency windows are reset and no live fields carry - /// over. It is still the same service, so it is not announced again. - fn reconcile_set( - &mut self, - source: DiscoverySource, - svcs: &[DiscoveredService], - known: &HashSet, - seen: &HashSet, - ) { - self.upsert_set(source, svcs, known); - for gone in known.difference(seen) { - self.retire(source, gone); - } - } - - /// The half of [`Self::reconcile_set`] that adds and refreshes: upsert every - /// service in `svcs` and announce the ones `known` did not have. Retires - /// nothing, which is what a pass that could only see part of its source needs - /// (see [`DiscoveryState::keep_through_partial_registry`]). - fn upsert_set( - &mut self, - source: DiscoverySource, - svcs: &[DiscoveredService], - known: &HashSet, - ) { - for svc in svcs { - let same_endpoint = self - .state - .instances - .get(&svc.container_id) - .is_none_or(|ex| ex.port == svc.port); - if !same_endpoint { - self.telemetry - .invalidate_on_endpoint_change(&svc.container_id); - } - self.upsert(svc, same_endpoint); - if !known.contains(&svc.container_id) { - self.announce(source, svc); - } - } - } - - /// Reconcile the probe-based Lemonade source, which has at most one service. - /// - /// Its `container_id` encodes the host and port, so a port or host change - /// yields a new id. The old id is retired first (its telemetry is purged, so - /// a stale rate baseline, TTFT and TPOT do not bleed across; it leaves the - /// state and is announced gone), then the new id is announced, so the state - /// and every snapshot carry one Lemonade instance and the vLLM scrape never - /// polls the old endpoint. `None` means the endpoint did not answer, which - /// retires the current one. - fn reconcile_lemonade( - &mut self, - probed: Option, - current: &mut Option, - ) { - match probed { - Some(svc) => { - let is_new_id = current.as_deref() != Some(svc.container_id.as_str()); - if is_new_id && let Some(old_id) = current.take() { - self.retire(DiscoverySource::Lemonade, &old_id); - } - self.upsert(&svc, !is_new_id); - if is_new_id { - self.announce(DiscoverySource::Lemonade, &svc); - *current = Some(svc.container_id.clone()); - } - } - None => { - if let Some(id) = current.take() { - self.retire(DiscoverySource::Lemonade, &id); - } - } - } - } -} - -/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every -/// instance with a port except the live Lemonade instance (its own collector -/// scrapes it) and the managed services whose engine is not vLLM. -fn vllm_scrape_targets( - instances: &HashMap, - lemonade_id: Option<&str>, - managed_non_vllm: &HashSet, -) -> Vec<(String, u16)> { - instances - .values() - .filter(|i| Some(i.container_id.as_str()) != lemonade_id) - .filter(|i| !managed_non_vllm.contains(&i.container_id)) - .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) - .collect() -} - /// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process /// attribution map, falling back to device-summed VRAM over the instance's /// GPUs when its container has no per-process entry. Pure — the runner does the @@ -1608,7 +1529,7 @@ mod tests { ); } - fn at(secs: i64) -> DateTime { + pub(super) fn at(secs: i64) -> DateTime { Utc.timestamp_opt(secs, 0).unwrap() } @@ -1616,7 +1537,7 @@ mod tests { const TICK: Duration = Duration::from_secs(2); - fn baseline(sum_s: f64, count: f64) -> LatencyWindow { + pub(super) fn baseline(sum_s: f64, count: f64) -> LatencyWindow { LatencyWindow { prev: Some((sum_s, count, at(10))), tracker: GenerationObservationTracker::new(TICK), @@ -1641,20 +1562,95 @@ mod tests { } #[test] - fn managed_instances_never_count_as_vram_join_misses() { + fn only_instances_that_can_join_count_as_vram_join_misses() { + use rocm_dash_collectors::ancestry::Root; let docker_id = "ab".repeat(32); + let rooted = "vllm-qwen-qwen2-5-0-5b-instru-1793361705247"; + let unrooted = "vllm-other-1793361705248"; let gpus = [gpu("gpu-0", 1000, 8000)]; - let insts = [ - inst_on("vllm-qwen-qwen2-5-0-5b-instru-1793361705247", "0"), - inst_on(&docker_id, "0"), - ]; - // A docker instance with no /proc match is a real miss; the managed - // service, which could never match, is not. + let serving = |id: &str| Instance { + status: InstanceStatus::Ready, + ..inst_on(id, "0") + }; + let insts = [serving(rooted), serving(unrooted), inst_on(&docker_id, "0")]; + let roots = [Root { + pid: 10, + start_ticks: None, + owner: rooted.into(), + }]; + // The docker instance and the rooted managed service could both have + // joined and did not; the managed service with no roots never could. let empty: HashMap = HashMap::new(); - assert_eq!(count_vram_join_misses(&insts, &gpus, &empty), 1); - // Once the docker instance joins, nothing is missed. - let joined: HashMap = [(docker_id, 512)].into(); - assert_eq!(count_vram_join_misses(&insts, &gpus, &joined), 0); + assert_eq!( + count_vram_join_misses(&insts, &gpus, &empty, &roots, &HashSet::new()), + 2 + ); + // With no roots recorded, only the docker instance is a miss. + assert_eq!( + count_vram_join_misses(&insts, &gpus, &empty, &[], &HashSet::new()), + 1 + ); + // Once both joined, nothing is missed. + let joined: HashMap = [(docker_id, 512), (rooted.to_string(), 256)].into(); + assert_eq!( + count_vram_join_misses(&insts, &gpus, &joined, &roots, &HashSet::new()), + 0 + ); + } + + #[test] + fn a_rooted_non_vllm_service_is_not_a_vram_join_miss() { + use rocm_dash_collectors::ancestry::Root; + let id = "lemonade-gguf-1793361705247"; + let gpus = [gpu("gpu-0", 1000, 8000)]; + let serving = Instance { + status: InstanceStatus::Ready, + ..inst_on(id, "0") + }; + let roots = [Root { + pid: 10, + start_ticks: None, + owner: id.into(), + }]; + let empty: HashMap = HashMap::new(); + // As a vLLM service it would be a miss; marked non-vLLM it is not, since + // its backend may hold GPU memory without `amd-smi` listing the process. + assert_eq!( + count_vram_join_misses( + std::slice::from_ref(&serving), + &gpus, + &empty, + &roots, + &HashSet::new() + ), + 1 + ); + let non_vllm: HashSet = [id.to_string()].into(); + assert_eq!( + count_vram_join_misses(&[serving], &gpus, &empty, &roots, &non_vllm), + 0 + ); + } + + #[test] + fn a_rooted_service_still_loading_is_not_a_vram_join_miss() { + use rocm_dash_collectors::ancestry::Root; + let id = "vllm-qwen-1793361705247"; + let gpus = [gpu("gpu-0", 1000, 8000)]; + let loading = Instance { + status: InstanceStatus::Starting { phase: None }, + ..inst_on(id, "0") + }; + let roots = [Root { + pid: 10, + start_ticks: None, + owner: id.into(), + }]; + let empty: HashMap = HashMap::new(); + assert_eq!( + count_vram_join_misses(&[loading], &gpus, &empty, &roots, &HashSet::new()), + 0 + ); } #[test] @@ -1841,762 +1837,4 @@ mod tests { 120 ); } - - // -- Identity/endpoint cleanup helper tests ----------------------------------- - - fn make_tracker(tick_secs: u64) -> GenerationObservationTracker { - GenerationObservationTracker::new(Duration::from_secs(tick_secs)) - } - - /// `TelemetryState::purge` removes the tracker entry, TTFT baseline, and - /// TPOT baseline for the target id and leaves other ids untouched. - #[test] - fn telemetry_state_purge_removes_all_three_maps_for_id() { - let mut state = TelemetryState::default(); - - // Populate two ids. - state.gen_trackers.insert("a".into(), make_tracker(2)); - state.gen_trackers.insert("b".into(), make_tracker(2)); - state.prev_ttft.insert("a".into(), baseline(1.0, 1.0)); - state.prev_ttft.insert("b".into(), baseline(2.0, 2.0)); - state.prev_tpot.insert("a".into(), baseline(3.0, 3.0)); - state.prev_tpot.insert("b".into(), baseline(4.0, 4.0)); - - state.purge("a"); - - // "a" must be gone from all maps. - assert!( - !state.gen_trackers.contains_key("a"), - "tracker for a must be removed" - ); - assert!( - !state.prev_ttft.contains_key("a"), - "ttft for a must be removed" - ); - assert!( - !state.prev_tpot.contains_key("a"), - "tpot for a must be removed" - ); - // "b" must be unaffected. - assert!( - state.gen_trackers.contains_key("b"), - "tracker for b must survive" - ); - assert!(state.prev_ttft.contains_key("b"), "ttft for b must survive"); - assert!(state.prev_tpot.contains_key("b"), "tpot for b must survive"); - } - - /// `TelemetryState::invalidate_on_endpoint_change` invalidates the tracker - /// (resets its window so subsequent snapshots return None) and removes the - /// latency baselines, without removing the tracker entry itself. - #[test] - fn invalidate_on_endpoint_change_resets_tracker_and_clears_latency() { - let mut state = TelemetryState::default(); - - let mut tracker = make_tracker(2); - // Give the tracker a reading so it has a non-None baseline. - tracker.observe_counter(100.0, at(10)); - tracker.observe_counter(200.0, at(12)); - state.gen_trackers.insert("svc".into(), tracker); - state.prev_ttft.insert("svc".into(), baseline(1.0, 10.0)); - state.prev_tpot.insert("svc".into(), baseline(0.5, 8.0)); - - state.invalidate_on_endpoint_change("svc"); - - // Tracker still exists but its window is reset: snapshot returns None. - let (rate, _meta) = state - .gen_trackers - .get_mut("svc") - .expect("tracker entry must remain") - .snapshot(at(12)); - assert!(rate.is_none(), "invalidated tracker must yield None rate"); - // Latency baselines must be cleared. - assert!( - !state.prev_ttft.contains_key("svc"), - "ttft baseline must be cleared on endpoint change" - ); - assert!( - !state.prev_tpot.contains_key("svc"), - "tpot baseline must be cleared on endpoint change" - ); - } - - // ── Discovery reconcile ───────────────────────────────────────────────── - // - // The Docker source cannot be driven end to end without a container - // daemon, and `reconcile_set` is the code both it and the managed-service - // registry run, so these tests exercise it directly. The registry and - // Lemonade sources are also driven end to end in - // `tests/discovery_reconcile.rs`, except the registry's retirement of a - // vanished service and Lemonade's change of id, which only these tests - // cover. - - fn discovered(id: &str, port: u16) -> DiscoveredService { - DiscoveredService { - container_id: id.into(), - container_name: format!("name-{id}"), - model_name: "model".into(), - port: Some(port), - ..Default::default() - } - } - - fn id_set(ids: &[&str]) -> HashSet { - ids.iter().map(|s| (*s).to_string()).collect() - } - - /// The reducer state, telemetry and event stream a reconcile pass works on. - struct Pass { - state: State, - telemetry: TelemetryState, - tx: broadcast::Sender, - rx: broadcast::Receiver, - } - - impl Pass { - fn new() -> Self { - let (tx, rx) = broadcast::channel(64); - Self { - state: State::default(), - telemetry: TelemetryState::default(), - tx, - rx, - } - } - - fn reconcile(&mut self) -> Reconcile<'_> { - Reconcile { - state: &mut self.state, - telemetry: &mut self.telemetry, - tx: &self.tx, - persist: None, - } - } - - /// The announcements broadcast since the last call, as `discovered ` - /// and `gone ` strings. - fn announced(&mut self) -> Vec { - let mut out = Vec::new(); - while let Ok(ev) = self.rx.try_recv() { - match ev { - Event::InstanceDiscovered(i) => { - out.push(format!("discovered {}", i.container_id)); - } - Event::InstanceGone { container_id } => { - out.push(format!("gone {container_id}")); - } - _ => {} - } - } - out - } - - /// Give `id` a live KV reading and a latency baseline, as a scrape would. - fn scrape(&mut self, id: &str) { - let mut inst = self.state.instances.get(id).cloned().expect("instance"); - inst.kv_cache_usage_pct = Some(25.0); - self.state.apply(StateEvent::InstanceUpserted(inst)); - self.telemetry - .prev_ttft - .insert(id.into(), baseline(1.0, 10.0)); - self.telemetry - .prev_tpot - .insert(id.into(), baseline(0.5, 8.0)); - } - - fn kv(&self, id: &str) -> Option { - self.state - .instances - .get(id) - .and_then(|i| i.kv_cache_usage_pct) - } - } - - #[test] - fn reconcile_set_announces_a_new_service_once() { - let mut pass = Pass::new(); - let svcs = [discovered("a", 8000)]; - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&[]), - &id_set(&["a"]), - ); - assert_eq!(pass.announced(), ["discovered a"]); - assert!(pass.state.instances.contains_key("a")); - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&["a"]), - &id_set(&["a"]), - ); - assert!( - pass.announced().is_empty(), - "a known service must not be announced again" - ); - } - - #[test] - fn reconcile_set_keeps_live_fields_while_the_endpoint_is_unchanged() { - let mut pass = Pass::new(); - let svcs = [discovered("a", 8000)]; - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&[]), - &id_set(&["a"]), - ); - pass.scrape("a"); - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&["a"]), - &id_set(&["a"]), - ); - - assert_eq!(pass.kv("a"), Some(25.0)); - assert!(pass.telemetry.prev_ttft.contains_key("a")); - assert!(pass.telemetry.prev_tpot.contains_key("a")); - } - - #[test] - fn reconcile_set_starts_clean_when_the_port_moves_under_the_same_id() { - let mut pass = Pass::new(); - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &[discovered("a", 8000)], - &id_set(&[]), - &id_set(&["a"]), - ); - pass.scrape("a"); - pass.announced(); - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &[discovered("a", 9000)], - &id_set(&["a"]), - &id_set(&["a"]), - ); - - assert_eq!(pass.state.instances["a"].port, Some(9000)); - assert_eq!(pass.kv("a"), None, "the old endpoint's reading leaked"); - assert!(!pass.telemetry.prev_ttft.contains_key("a")); - assert!(!pass.telemetry.prev_tpot.contains_key("a")); - assert!( - pass.announced().is_empty(), - "a port change is not a new or a gone service" - ); - } - - #[test] - fn reconcile_set_retires_a_vanished_service_and_keeps_the_others() { - let mut pass = Pass::new(); - let svcs = [discovered("a", 8000), discovered("b", 8001)]; - pass.reconcile().reconcile_set( - DiscoverySource::Managed, - &svcs, - &id_set(&[]), - &id_set(&["a", "b"]), - ); - pass.scrape("a"); - pass.scrape("b"); - pass.announced(); - - pass.reconcile().reconcile_set( - DiscoverySource::Managed, - &svcs[..1], - &id_set(&["a", "b"]), - &id_set(&["a"]), - ); - - assert_eq!(pass.announced(), ["gone b"]); - assert!(!pass.state.instances.contains_key("b")); - assert!(!pass.telemetry.prev_ttft.contains_key("b")); - assert!(pass.state.instances.contains_key("a")); - assert!(pass.telemetry.prev_ttft.contains_key("a")); - } - - // ── Lemonade state matrix ─────────────────────────────────────────────── - // - // One Lemonade endpoint is probed per discovery pass. Its id encodes the host - // and port, so the states a probe can observe are: nothing ever answered; the - // first answer; the same endpoint again; the host or the port changed; the - // endpoint went quiet; it came back somewhere else; and the same answer - // arriving twice in one tick. Each row below checks the announcements, the - // instances a snapshot would carry, the telemetry kept per id, and the - // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is - // visible on all four. - - fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { - DiscoveredService { - container_id: format!("lemonade-{host}-{port}"), - container_name: "lemonade".into(), - model_name: "model".into(), - port: Some(port), - ..Default::default() - } - } - - impl Pass { - /// The ids a snapshot carries. The loop builds a snapshot's instances from - /// `state.instances`, so this is the dashboard's view of the pass. - fn instance_ids(&self) -> Vec { - let mut ids: Vec = self.state.instances.keys().cloned().collect(); - ids.sort(); - ids - } - - /// The ids the vLLM Prometheus scrape would poll while `current` is the - /// live Lemonade id. - fn scrape_targets(&self, current: Option<&str>) -> Vec { - let mut ids: Vec = - vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) - .into_iter() - .map(|(id, _port)| id) - .collect(); - ids.sort(); - ids - } - - /// A live KV reading, latency baselines and a rate tracker for `id`. - fn give_live_telemetry(&mut self, id: &str) { - self.scrape(id); - self.telemetry - .gen_trackers - .insert(id.into(), make_tracker(2)); - } - - fn has_telemetry(&self, id: &str) -> bool { - self.telemetry.gen_trackers.contains_key(id) - || self.telemetry.prev_ttft.contains_key(id) - || self.telemetry.prev_tpot.contains_key(id) - } - } - - fn ids(ids: &[&str]) -> Vec { - ids.iter().map(|s| (*s).to_string()).collect() - } - - /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, - /// the loop's current Lemonade id and what the second probe announced. - fn lemonade_identity_change( - old: &DiscoveredService, - new: &DiscoveredService, - ) -> (Pass, Option, Vec) { - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(old.clone()), &mut current); - pass.give_live_telemetry(&old.container_id); - pass.announced(); - - pass.reconcile() - .reconcile_lemonade(Some(new.clone()), &mut current); - let announced = pass.announced(); - (pass, current, announced) - } - - #[test] - fn lemonade_that_never_answers_leaves_no_trace() { - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile().reconcile_lemonade(None, &mut current); - pass.reconcile().reconcile_lemonade(None, &mut current); - - assert!(pass.announced().is_empty()); - assert_eq!(current, None); - assert!(pass.instance_ids().is_empty()); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile() - .reconcile_lemonade(Some(lemonade_svc("127.0.0.1", 13305)), &mut current); - - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); - assert_eq!(current.as_deref(), Some("lemonade-127.0.0.1-13305")); - assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); - assert!( - pass.scrape_targets(current.as_deref()).is_empty(), - "the live Lemonade instance is not a vLLM scrape target" - ); - } - - #[test] - fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { - let svc = lemonade_svc("127.0.0.1", 13305); - let id = svc.container_id.clone(); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(svc.clone()), &mut current); - pass.give_live_telemetry(&id); - pass.announced(); - - pass.reconcile().reconcile_lemonade(Some(svc), &mut current); - - assert!(pass.announced().is_empty()); - assert_eq!(pass.instance_ids(), ids(&[&id])); - assert_eq!(pass.kv(&id), Some(25.0)); - assert!(pass.has_telemetry(&id)); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { - let old = lemonade_svc("127.0.0.1", 13305); - let new = lemonade_svc("10.0.0.7", 13305); - let (pass, current, announced) = lemonade_identity_change(&old, &new); - - assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!( - announced, - [ - "gone lemonade-127.0.0.1-13305", - "discovered lemonade-10.0.0.7-13305" - ] - ); - assert!(!pass.has_telemetry(&old.container_id)); - assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); - assert!( - pass.scrape_targets(current.as_deref()).is_empty(), - "the old endpoint must not be polled as a vLLM target" - ); - } - - #[test] - fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { - let old = lemonade_svc("127.0.0.1", 13305); - let new = lemonade_svc("127.0.0.1", 13306); - let (pass, current, announced) = lemonade_identity_change(&old, &new); - - assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!( - announced, - [ - "gone lemonade-127.0.0.1-13305", - "discovered lemonade-127.0.0.1-13306" - ] - ); - assert!(!pass.has_telemetry(&old.container_id)); - assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); - assert!( - pass.scrape_targets(current.as_deref()).is_empty(), - "the old endpoint must not be polled as a vLLM target" - ); - } - - #[test] - fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { - let svc = lemonade_svc("127.0.0.1", 13305); - let id = svc.container_id.clone(); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile().reconcile_lemonade(Some(svc), &mut current); - pass.give_live_telemetry(&id); - pass.announced(); - - pass.reconcile().reconcile_lemonade(None, &mut current); - - assert_eq!(pass.announced(), [format!("gone {id}")]); - assert_eq!(current, None); - assert!(pass.instance_ids().is_empty()); - assert!(!pass.has_telemetry(&id)); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { - let old = lemonade_svc("127.0.0.1", 13305); - let new = lemonade_svc("127.0.0.1", 13306); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(old.clone()), &mut current); - pass.give_live_telemetry(&old.container_id); - pass.reconcile().reconcile_lemonade(None, &mut current); - assert_eq!( - pass.announced(), - [ - "discovered lemonade-127.0.0.1-13305", - "gone lemonade-127.0.0.1-13305" - ] - ); - - pass.reconcile() - .reconcile_lemonade(Some(new.clone()), &mut current); - - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); - assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); - assert!(!pass.has_telemetry(&old.container_id)); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_answer_repeated_within_one_tick_is_announced_once() { - let svc = lemonade_svc("127.0.0.1", 13305); - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile() - .reconcile_lemonade(Some(svc.clone()), &mut current); - pass.reconcile().reconcile_lemonade(Some(svc), &mut current); - - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); - assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { - let a = lemonade_svc("127.0.0.1", 13305); - let b = lemonade_svc("127.0.0.1", 13306); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(a.clone()), &mut current); - pass.announced(); - - pass.reconcile().reconcile_lemonade(Some(b), &mut current); - assert_eq!( - pass.announced(), - [ - "gone lemonade-127.0.0.1-13305", - "discovered lemonade-127.0.0.1-13306" - ] - ); - pass.reconcile() - .reconcile_lemonade(Some(a.clone()), &mut current); - - assert_eq!( - pass.announced(), - [ - "gone lemonade-127.0.0.1-13306", - "discovered lemonade-127.0.0.1-13305" - ] - ); - assert_eq!(current.as_deref(), Some(a.container_id.as_str())); - assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile() - .reconcile_lemonade(Some(discovered("l1", 13305)), &mut current); - assert_eq!(pass.announced(), ["discovered l1"]); - assert_eq!(current.as_deref(), Some("l1")); - - pass.scrape("l1"); - pass.reconcile() - .reconcile_lemonade(Some(discovered("l1", 13305)), &mut current); - assert!( - pass.announced().is_empty(), - "same id must not be re-announced" - ); - assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); - - // A new id (the host or port changed) retires the old id, telemetry - // included, and announces the new one. - pass.reconcile() - .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); - assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); - assert_eq!(current.as_deref(), Some("l2")); - assert!(!pass.telemetry.prev_ttft.contains_key("l1")); - // The superseded instance leaves the state (and so every snapshot), not - // only its telemetry. - assert!(!pass.state.instances.contains_key("l1")); - - pass.reconcile().reconcile_lemonade(None, &mut current); - assert_eq!(pass.announced(), ["gone l2"]); - assert_eq!(current, None); - assert!(!pass.state.instances.contains_key("l2")); - - pass.reconcile().reconcile_lemonade(None, &mut current); - assert!(pass.announced().is_empty(), "nothing left to retire"); - } - - // Registry read health: one test per row of the state matrix (first - // failure, repeated failure, recovery, empty, populated, malformed new, - // malformed unchanged, malformed cleared). - - fn registry_read( - malformed: &[&str], - ) -> Result { - Ok(crate::registry::RegistryRead { - records: Vec::new(), - malformed: malformed.iter().map(PathBuf::from).collect(), - }) - } - - fn registry_failure( - why: &str, - ) -> Result { - Err(crate::registry::RegistryReadError { - path: PathBuf::from("services"), - source: std::io::Error::other(why), - partial: crate::registry::RegistryRead::default(), - }) - } - - /// State matrix for what a partially read registry does to the discovery - /// state, one assertion group per row: a service it saw joins the known set - /// and its non-vLLM mark follows the record; one it did not see stays known - /// and keeps its mark; nothing leaves either set. - #[test] - fn partial_registry_read_adds_what_it_saw_and_forgets_nothing() { - let mut state = DiscoveryState { - known_services: id_set(&["old-vllm", "old-lemonade", "flips"]), - managed_non_vllm: id_set(&["old-lemonade", "flips"]), - ..DiscoveryState::default() - }; - - state.keep_through_partial_registry(crate::registry::ManagedDiscovery { - seen: id_set(&["new-lemonade", "new-vllm", "flips"]), - non_vllm: id_set(&["new-lemonade"]), - ..Default::default() - }); - - assert_eq!( - state.known_services, - id_set(&[ - "old-vllm", - "old-lemonade", - "flips", - "new-lemonade", - "new-vllm" - ]), - "the unseen services stay known and the new ones join them" - ); - assert_eq!( - state.managed_non_vllm, - id_set(&["old-lemonade", "new-lemonade"]), - "an unseen non-vLLM service keeps its mark, a seen one follows its record" - ); - } - - #[test] - fn partial_registry_read_that_saw_nothing_changes_nothing() { - let mut state = DiscoveryState { - known_services: id_set(&["a", "b"]), - managed_non_vllm: id_set(&["b"]), - ..DiscoveryState::default() - }; - - state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); - - assert_eq!(state.known_services, id_set(&["a", "b"])); - assert_eq!(state.managed_non_vllm, id_set(&["b"])); - } - - #[test] - fn registry_health_reports_the_first_failure_of_a_streak_once() { - let mut health = RegistryHealth::default(); - - let first = health.observe(®istry_failure("disk gone")); - assert_eq!( - first.failed.as_deref(), - Some("cannot read services: disk gone"), - "the first failed pass names the error" - ); - assert!(!first.recovered); - - for pass in 2..=5 { - assert_eq!( - health.observe(®istry_failure("disk gone")), - RegistryChange::default(), - "failed pass {pass} of the same streak must stay quiet" - ); - } - } - - #[test] - fn registry_health_reports_recovery_once_and_a_new_streak_again() { - let mut health = RegistryHealth::default(); - health.observe(®istry_failure("one")); - health.observe(®istry_failure("one")); - - let recovered = health.observe(®istry_read(&[])); - assert!(recovered.recovered); - assert!(recovered.failed.is_none()); - assert_eq!( - health.observe(®istry_read(&[])), - RegistryChange::default(), - "only the first good pass after a streak is a recovery" - ); - - let again = health.observe(®istry_failure("two")); - assert_eq!(again.failed.as_deref(), Some("cannot read services: two")); - } - - #[test] - fn registry_health_is_quiet_for_an_empty_or_populated_registry() { - let mut health = RegistryHealth::default(); - // A healthy read, empty or not, says nothing: there is no failure to - // report and no failure to have recovered from. - for _ in 0..3 { - assert_eq!( - health.observe(®istry_read(&[])), - RegistryChange::default() - ); - } - } - - #[test] - fn registry_health_reports_a_malformed_set_when_it_changes() { - let mut health = RegistryHealth::default(); - - let first = health.observe(®istry_read(&["a.json"])); - assert_eq!(first.malformed, [PathBuf::from("a.json")]); - assert_eq!( - health.observe(®istry_read(&["a.json"])), - RegistryChange::default(), - "an unchanged malformed set is not reported again" - ); - - let grown = health.observe(®istry_read(&["a.json", "b.json"])); - assert_eq!( - grown.malformed, - [PathBuf::from("a.json"), PathBuf::from("b.json")], - "a newly malformed record reports the whole set" - ); - - let cleared = health.observe(®istry_read(&[])); - assert!( - cleared.malformed.is_empty(), - "clearing the set is not a warning" - ); - let back = health.observe(®istry_read(&["a.json"])); - assert_eq!( - back.malformed, - [PathBuf::from("a.json")], - "a record that goes bad again after being fixed is reported again" - ); - } - - #[test] - fn registry_health_keeps_the_malformed_set_across_a_read_failure() { - let mut health = RegistryHealth::default(); - health.observe(®istry_read(&["a.json"])); - health.observe(®istry_failure("blip")); - - let after = health.observe(®istry_read(&["a.json"])); - assert!(after.recovered); - assert!( - after.malformed.is_empty(), - "the same corrupt record is not news just because a read failed in between" - ); - } } diff --git a/crates/rocm-dash-daemon/src/runner/discovery.rs b/crates/rocm-dash-daemon/src/runner/discovery.rs new file mode 100644 index 0000000000..3ac172bfcc --- /dev/null +++ b/crates/rocm-dash-daemon/src/runner/discovery.rs @@ -0,0 +1,1306 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What the runner's discovery passes remember between ticks, and how a pass +//! reconciles what a source reports against it: the per-instance telemetry the +//! loop carries ([`TelemetryState`]), what each source has live +//! ([`DiscoveryState`]), how the managed-service registry's reads have gone +//! ([`RegistryHealth`]), and the upsert/announce/retire steps every source shares +//! ([`Reconcile`]). Split out of `runner.rs` to keep it within the module-size +//! limit; `run_loop` drives all of it. + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use rocm_dash_core::metrics::Instance; +use rocm_dash_core::observation::GenerationObservationTracker; +use rocm_dash_core::protocol::Event; +use rocm_dash_core::state::{State, StateEvent}; +use rocm_dash_core::traits::DiscoveredService; +use tokio::sync::broadcast; +use tracing::{info, warn}; + +use super::{ + LatencyWindows, broadcast_and_persist, instance_from_discovered, merge_discovery_refresh, +}; +use crate::persist::SessionWriter; + +/// Per-instance telemetry the loop carries between scrapes, keyed by container id. +#[derive(Default)] +pub(super) struct TelemetryState { + /// Generation-throughput trackers. Constructed lazily from + /// `opts.instance_tick`; they survive scrape failures so the last-observed + /// rate is held for the validity window before clearing. + pub(super) gen_trackers: HashMap, + /// Previous TTFT / TPOT histogram readings plus a freshness tracker per + /// instance, for the windowed avg-ms derivation (mirrors `gen_trackers`). + pub(super) prev_ttft: LatencyWindows, + pub(super) prev_tpot: LatencyWindows, +} + +impl TelemetryState { + /// Remove all per-service telemetry state for `id`: tracker, TTFT, and + /// TPOT baselines. Call on explicit removal (Docker Gone, Lemonade Gone, + /// managed Gone) and whenever an identity change requires a clean slate. + pub(super) fn purge(&mut self, id: &str) { + self.gen_trackers.remove(id); + self.prev_ttft.remove(id); + self.prev_tpot.remove(id); + } + + /// Invalidate the generation tracker for `id` (reset baseline without + /// removing the map entry) and clear the TTFT/TPOT latency windows. + /// Called when an endpoint changes within the same container/service id so + /// the new scrape address starts with a fresh rate baseline. + pub(super) fn invalidate_on_endpoint_change(&mut self, id: &str) { + if let Some(t) = self.gen_trackers.get_mut(id) { + t.invalidate(); + } + self.prev_ttft.remove(id); + self.prev_tpot.remove(id); + } +} + +/// What each discovery source currently has live, so a pass can tell a new +/// service from a known one and a known one from one that has gone. +#[derive(Default)] +pub(super) struct DiscoveryState { + /// Container ids the last Docker pass saw. + pub(super) known_instances: HashSet, + /// Ids the last managed-service registry pass surfaced. + pub(super) known_services: HashSet, + /// The subset of the registry ids whose engine is NOT vLLM, excluded from + /// the vLLM Prometheus scrape so they aren't mis-parsed. + pub(super) managed_non_vllm: HashSet, + /// Model identities of the externally discovered Lemonade rows. + pub(super) lemonade_ids: HashSet, + /// Registry-managed Lemonade endpoints, scraped read-only for telemetry. + pub(super) managed_lemonade: Vec, + /// Ancestry roots of the managed services the registry surfaced, used to + /// attribute GPU processes to a service when the cgroup join cannot (a + /// managed service is not a container). + pub(super) managed_roots: Vec, + /// How the registry reads have been going, so a condition that lasts many + /// passes is logged once rather than every discovery tick. + pub(super) registry_health: RegistryHealth, +} + +impl DiscoveryState { + /// Fold a partial registry read into the managed Lemonade targets and return + /// the previous set: the targets this pass read are replaced by its own, the + /// ones it did not see are kept, and one it read and found stopped is + /// dropped (it is in `retired`, never in `seen`). Keeping a stopped target + /// would poll a dead port and, if anything later bound it, group that + /// service's row with the live one and lose the live one's throughput. + pub(super) fn keep_lemonade_through_partial_registry( + &mut self, + disc: &mut crate::registry::ManagedDiscovery, + ) -> Vec { + let mut next: Vec<_> = self + .managed_lemonade + .iter() + .filter(|target| { + !disc.seen.contains(&target.service_id) + && !disc.retired.contains(&target.service_id) + }) + .cloned() + .collect(); + next.append(&mut disc.lemonade); + std::mem::replace(&mut self.managed_lemonade, next) + } + + /// Record what a pass that read only part of the registry found. The services + /// it saw join the known set, so a later complete read can retire them, and + /// the known ones it did not see stay; the non-vLLM marks of the services it + /// saw are refreshed and the others kept, so a kept non-vLLM service is still + /// left out of the vLLM scrape. + pub(super) fn keep_through_partial_registry( + &mut self, + disc: crate::registry::ManagedDiscovery, + ) { + self.managed_non_vllm.retain(|id| !disc.seen.contains(id)); + self.managed_non_vllm.extend(disc.non_vllm); + // Roots of the services this pass saw are replaced; the unseen ones + // keep theirs, as their services stay known. + // A service this pass read and found stopped loses its roots too, or its + // PID would stay matchable for as long as an unrelated record is broken. + self.managed_roots + .retain(|r| !disc.seen.contains(&r.owner) && !disc.retired.contains(&r.owner)); + self.managed_roots.extend(disc.roots); + self.known_services.extend(disc.seen); + } +} + +/// What the managed-service registry's reads have reported so far. A failure +/// that lasts is logged on its first pass only, the recovery once, and a set of +/// malformed records each time it changes; the dashboard's warning badge, which +/// is cheap and does not scroll, carries the failure for as long as it lasts. +#[derive(Default)] +pub(super) struct RegistryHealth { + /// The previous pass could not read the registry. + pub(super) failing: bool, + /// The malformed records the last successful pass reported, so an unchanged + /// set is not reported again. + pub(super) reported_malformed: Vec, +} + +/// What a discovery pass should say about the registry, apart from the services +/// it found. Kept as data so [`RegistryHealth::observe`] can be tested without a +/// log subscriber. +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct RegistryChange { + /// The first failed pass of a streak, with the error. + pub(super) failed: Option, + /// The first successful pass after a streak of failures. + pub(super) recovered: bool, + /// The malformed records, when the set differs from the one last reported + /// and is not empty. + pub(super) malformed: Vec, +} + +impl RegistryHealth { + pub(super) fn observe( + &mut self, + read: &Result, + ) -> RegistryChange { + let mut change = RegistryChange::default(); + match read { + Err(e) => { + if !self.failing { + self.failing = true; + change.failed = Some(e.to_string()); + } + } + Ok(read) => { + if self.failing { + self.failing = false; + change.recovered = true; + } + if read.malformed != self.reported_malformed { + self.reported_malformed.clone_from(&read.malformed); + change.malformed.clone_from(&read.malformed); + } + } + } + change + } +} + +impl RegistryChange { + pub(super) fn log(&self) { + if let Some(error) = &self.failed { + warn!( + error = %error, + "managed-service registry could not be read; keeping the known services until it can be" + ); + } + if self.recovered { + info!("managed-service registry is readable again"); + } + if !self.malformed.is_empty() { + warn!( + files = ?self.malformed, + "skipping managed-service records that are not valid JSON" + ); + } + } +} + +/// The header warning for a pass that could not read the registry. +pub(super) fn registry_read_warning(error: &crate::registry::RegistryReadError) -> String { + format!("managed registry: {error}; keeping the last known services") +} + +/// Which discovery source a service came from. It only selects the log wording +/// in [`Reconcile::announce`] and [`Reconcile::retire`]; those two steps behave +/// the same for every source. How a pass decides what is new, known or gone +/// differs: Docker and the registry go through `reconcile_set`, Lemonade through +/// `reconcile_lemonade`. +#[derive(Clone, Copy)] +pub(super) enum DiscoverySource { + Docker, + Lemonade, + Managed, +} + +impl DiscoverySource { + pub(super) fn log_discovered(self, svc: &DiscoveredService) { + match self { + Self::Docker => info!( + id = %svc.container_id, + name = %svc.container_name, + model = %svc.model_name, + "instance discovered" + ), + Self::Lemonade => info!( + id = %svc.container_id, + model = %svc.model_name, + "lemonade instance discovered" + ), + Self::Managed => info!(id = %svc.container_id, "managed service discovered"), + } + } + + pub(super) fn log_gone(self, id: &str) { + match self { + Self::Docker => info!(id = %id, "instance gone"), + Self::Lemonade => info!(id = %id, "lemonade instance gone"), + Self::Managed => info!(id = %id, "managed service gone"), + } + } +} + +/// The loop state a discovery pass changes, and the stream it announces on. +pub(super) struct Reconcile<'a> { + pub(super) state: &'a mut State, + pub(super) telemetry: &'a mut TelemetryState, + pub(super) tx: &'a broadcast::Sender, + pub(super) persist: Option<&'a Arc>>, +} + +impl Reconcile<'_> { + /// Upsert a (re)discovered service. Collector-owned fields carry over from + /// the instance already in state unless `carry_live` is false, which is how + /// a changed identity starts from a clean baseline. + pub(super) fn upsert(&mut self, svc: &DiscoveredService, carry_live: bool) { + let existing = self.state.instances.get(&svc.container_id).cloned(); + let inst = merge_discovery_refresh(svc, if carry_live { existing.as_ref() } else { None }); + self.state.apply(StateEvent::InstanceUpserted(inst)); + } + + /// Announce a newly discovered service. InstanceDiscovered is a + /// discovery-metadata event: it carries a clean instance, with no prior + /// collector-owned telemetry. + pub(super) fn announce(&self, source: DiscoverySource, svc: &DiscoveredService) { + source.log_discovered(svc); + broadcast_and_persist( + self.tx, + self.persist, + Event::InstanceDiscovered(instance_from_discovered(svc)), + ); + } + + /// Drop a service that has gone: purge its telemetry, remove it from state + /// and announce it. + pub(super) fn retire(&mut self, source: DiscoverySource, id: &str) { + source.log_gone(id); + self.telemetry.purge(id); + self.state + .apply(StateEvent::InstanceRemoved(id.to_string())); + broadcast_and_persist( + self.tx, + self.persist, + Event::InstanceGone { + container_id: id.to_string(), + }, + ); + } + + /// Reconcile a source that reports its whole set each pass (Docker, the + /// managed-service registry). `known` is what the previous pass saw and + /// `seen` what this one saw; the caller stores `seen` as the next `known`. + /// + /// A service whose port changed under the same id has a new scrape address: + /// its rate baseline and latency windows are reset and no live fields carry + /// over. It is still the same service, so it is not announced again. + pub(super) fn reconcile_set( + &mut self, + source: DiscoverySource, + svcs: &[DiscoveredService], + known: &HashSet, + seen: &HashSet, + ) { + self.upsert_set(source, svcs, known); + for gone in known.difference(seen) { + self.retire(source, gone); + } + } + + /// The half of [`Self::reconcile_set`] that adds and refreshes: upsert every + /// service in `svcs` and announce the ones `known` did not have. Retires + /// nothing, which is what a pass that could only see part of its source needs + /// (see [`DiscoveryState::keep_through_partial_registry`]). + pub(super) fn upsert_set( + &mut self, + source: DiscoverySource, + svcs: &[DiscoveredService], + known: &HashSet, + ) { + for svc in svcs { + let same_endpoint = self + .state + .instances + .get(&svc.container_id) + .is_none_or(|ex| ex.port == svc.port); + if !same_endpoint { + self.telemetry + .invalidate_on_endpoint_change(&svc.container_id); + } + self.upsert(svc, same_endpoint); + if !known.contains(&svc.container_id) { + self.announce(source, svc); + } + } + } + + /// Reconcile the probe-based Lemonade source: the models one endpoint + /// reports loaded. An empty set means the endpoint did not answer (or is + /// now registry-managed), which retires every current row. + /// + /// Each row's `container_id` encodes the host, the port and the model, so a + /// host or port change yields new ids. The ids that went are retired first + /// (their telemetry is purged, so a stale rate baseline, TTFT and TPOT do + /// not bleed across; they leave the state and are announced gone), then the + /// new ones are announced, so the state never carries a row for an endpoint + /// that moved and the vLLM scrape never polls the old one. `current` is + /// updated to the ids this pass saw. + pub(super) fn reconcile_lemonade( + &mut self, + services: &[DiscoveredService], + current: &mut HashSet, + ) { + let seen: HashSet = services + .iter() + .map(|svc| svc.container_id.clone()) + .collect(); + for gone in current.difference(&seen) { + self.retire(DiscoverySource::Lemonade, gone); + } + self.upsert_set(DiscoverySource::Lemonade, services, current); + *current = seen; + } + + /// Start a managed Lemonade service clean when its registry target moved to + /// another endpoint or model under the same service id: the rate baseline + /// and latency windows are reset and no live fields carry over, as for a + /// port change in [`Self::upsert_set`]. A target with no previous entry is + /// new and needs nothing reset. + pub(super) fn reset_moved_lemonade_targets( + &mut self, + svcs: &[DiscoveredService], + previous: &[crate::registry::ManagedLemonadeTarget], + current: &[crate::registry::ManagedLemonadeTarget], + ) { + for svc in svcs { + let id = &svc.container_id; + let Some(old) = previous.iter().find(|target| &target.service_id == id) else { + continue; + }; + let unchanged = current + .iter() + .find(|target| &target.service_id == id) + .is_some_and(|new| { + crate::registry::endpoints_match(&old.host, old.port, &new.host, new.port) + && old.model_ref == new.model_ref + && old.canonical_model_id == new.canonical_model_id + }); + if !unchanged && self.state.instances.contains_key(id) { + self.telemetry.invalidate_on_endpoint_change(id); + self.upsert(svc, false); + } + } + } +} + +/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every +/// instance with a port except the discovered Lemonade rows (their endpoint +/// collector scrapes them) and the managed services whose engine is not vLLM. +pub(super) fn vllm_scrape_targets( + instances: &HashMap, + lemonade_ids: &HashSet, + managed_non_vllm: &HashSet, +) -> Vec<(String, u16)> { + instances + .values() + .filter(|i| !lemonade_ids.contains(&i.container_id)) + .filter(|i| !managed_non_vllm.contains(&i.container_id)) + .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runner::tests::{at, baseline}; + use std::time::Duration; + + // -- Identity/endpoint cleanup helper tests ----------------------------------- + + fn target(id: &str, port: u16) -> crate::registry::ManagedLemonadeTarget { + crate::registry::ManagedLemonadeTarget { + service_id: id.into(), + host: "127.0.0.1".into(), + port, + model_ref: "m".into(), + canonical_model_id: "m".into(), + } + } + + /// A partial registry read replaces the targets it saw, keeps the ones it + /// did not see, and drops one it read and found stopped. + #[test] + fn a_partial_registry_read_keeps_unseen_lemonade_targets_and_drops_stopped_ones() { + let mut state = DiscoveryState { + managed_lemonade: vec![target("seen", 1), target("unseen", 2), target("stopped", 3)], + ..DiscoveryState::default() + }; + let mut disc = crate::registry::ManagedDiscovery { + svcs: Vec::new(), + seen: HashSet::from(["seen".to_string()]), + non_vllm: HashSet::new(), + lemonade: vec![target("seen", 10)], + past_attempts: 0, + roots: Vec::new(), + retired: HashSet::from(["stopped".to_string()]), + }; + let previous = state.keep_lemonade_through_partial_registry(&mut disc); + assert_eq!(previous.len(), 3, "the previous set is handed back"); + let mut now: Vec<_> = state + .managed_lemonade + .iter() + .map(|t| (t.service_id.as_str(), t.port)) + .collect(); + now.sort_unstable(); + assert_eq!(now, [("seen", 10), ("unseen", 2)]); + } + + fn make_tracker(tick_secs: u64) -> GenerationObservationTracker { + GenerationObservationTracker::new(Duration::from_secs(tick_secs)) + } + + /// `TelemetryState::purge` removes the tracker entry, TTFT baseline, and + /// TPOT baseline for the target id and leaves other ids untouched. + #[test] + fn telemetry_state_purge_removes_all_three_maps_for_id() { + let mut state = TelemetryState::default(); + + // Populate two ids. + state.gen_trackers.insert("a".into(), make_tracker(2)); + state.gen_trackers.insert("b".into(), make_tracker(2)); + state.prev_ttft.insert("a".into(), baseline(1.0, 1.0)); + state.prev_ttft.insert("b".into(), baseline(2.0, 2.0)); + state.prev_tpot.insert("a".into(), baseline(3.0, 3.0)); + state.prev_tpot.insert("b".into(), baseline(4.0, 4.0)); + + state.purge("a"); + + // "a" must be gone from all maps. + assert!( + !state.gen_trackers.contains_key("a"), + "tracker for a must be removed" + ); + assert!( + !state.prev_ttft.contains_key("a"), + "ttft for a must be removed" + ); + assert!( + !state.prev_tpot.contains_key("a"), + "tpot for a must be removed" + ); + // "b" must be unaffected. + assert!( + state.gen_trackers.contains_key("b"), + "tracker for b must survive" + ); + assert!(state.prev_ttft.contains_key("b"), "ttft for b must survive"); + assert!(state.prev_tpot.contains_key("b"), "tpot for b must survive"); + } + + /// `TelemetryState::invalidate_on_endpoint_change` invalidates the tracker + /// (resets its window so subsequent snapshots return None) and removes the + /// latency baselines, without removing the tracker entry itself. + #[test] + fn invalidate_on_endpoint_change_resets_tracker_and_clears_latency() { + let mut state = TelemetryState::default(); + + let mut tracker = make_tracker(2); + // Give the tracker a reading so it has a non-None baseline. + tracker.observe_counter(100.0, at(10)); + tracker.observe_counter(200.0, at(12)); + state.gen_trackers.insert("svc".into(), tracker); + state.prev_ttft.insert("svc".into(), baseline(1.0, 10.0)); + state.prev_tpot.insert("svc".into(), baseline(0.5, 8.0)); + + state.invalidate_on_endpoint_change("svc"); + + // Tracker still exists but its window is reset: snapshot returns None. + let (rate, _meta) = state + .gen_trackers + .get_mut("svc") + .expect("tracker entry must remain") + .snapshot(at(12)); + assert!(rate.is_none(), "invalidated tracker must yield None rate"); + // Latency baselines must be cleared. + assert!( + !state.prev_ttft.contains_key("svc"), + "ttft baseline must be cleared on endpoint change" + ); + assert!( + !state.prev_tpot.contains_key("svc"), + "tpot baseline must be cleared on endpoint change" + ); + } + + // ── Discovery reconcile ───────────────────────────────────────────────── + // + // The Docker source cannot be driven end to end without a container + // daemon, and `reconcile_set` is the code both it and the managed-service + // registry run, so these tests exercise it directly. The registry and + // Lemonade sources are also driven end to end in + // `tests/discovery_reconcile.rs`, except the registry's retirement of a + // vanished service and Lemonade's change of id, which only these tests + // cover. + + fn discovered(id: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: id.into(), + container_name: format!("name-{id}"), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + fn id_set(ids: &[&str]) -> HashSet { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// The reducer state, telemetry and event stream a reconcile pass works on. + struct Pass { + state: State, + telemetry: TelemetryState, + tx: broadcast::Sender, + rx: broadcast::Receiver, + } + + impl Pass { + fn new() -> Self { + let (tx, rx) = broadcast::channel(64); + Self { + state: State::default(), + telemetry: TelemetryState::default(), + tx, + rx, + } + } + + fn reconcile(&mut self) -> Reconcile<'_> { + Reconcile { + state: &mut self.state, + telemetry: &mut self.telemetry, + tx: &self.tx, + persist: None, + } + } + + /// The announcements broadcast since the last call, as `discovered ` + /// and `gone ` strings. + fn announced(&mut self) -> Vec { + let mut out = Vec::new(); + while let Ok(ev) = self.rx.try_recv() { + match ev { + Event::InstanceDiscovered(i) => { + out.push(format!("discovered {}", i.container_id)); + } + Event::InstanceGone { container_id } => { + out.push(format!("gone {container_id}")); + } + _ => {} + } + } + out + } + + /// Give `id` a live KV reading and a latency baseline, as a scrape would. + fn scrape(&mut self, id: &str) { + let mut inst = self.state.instances.get(id).cloned().expect("instance"); + inst.kv_cache_usage_pct = Some(25.0); + self.state.apply(StateEvent::InstanceUpserted(inst)); + self.telemetry + .prev_ttft + .insert(id.into(), baseline(1.0, 10.0)); + self.telemetry + .prev_tpot + .insert(id.into(), baseline(0.5, 8.0)); + } + + fn kv(&self, id: &str) -> Option { + self.state + .instances + .get(id) + .and_then(|i| i.kv_cache_usage_pct) + } + } + + #[test] + fn reconcile_set_announces_a_new_service_once() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000)]; + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&[]), + &id_set(&["a"]), + ); + assert_eq!(pass.announced(), ["discovered a"]); + assert!(pass.state.instances.contains_key("a")); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&["a"]), + &id_set(&["a"]), + ); + assert!( + pass.announced().is_empty(), + "a known service must not be announced again" + ); + } + + #[test] + fn reconcile_set_keeps_live_fields_while_the_endpoint_is_unchanged() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000)]; + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&[]), + &id_set(&["a"]), + ); + pass.scrape("a"); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&["a"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.kv("a"), Some(25.0)); + assert!(pass.telemetry.prev_ttft.contains_key("a")); + assert!(pass.telemetry.prev_tpot.contains_key("a")); + } + + #[test] + fn reconcile_set_starts_clean_when_the_port_moves_under_the_same_id() { + let mut pass = Pass::new(); + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &[discovered("a", 8000)], + &id_set(&[]), + &id_set(&["a"]), + ); + pass.scrape("a"); + pass.announced(); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &[discovered("a", 9000)], + &id_set(&["a"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.state.instances["a"].port, Some(9000)); + assert_eq!(pass.kv("a"), None, "the old endpoint's reading leaked"); + assert!(!pass.telemetry.prev_ttft.contains_key("a")); + assert!(!pass.telemetry.prev_tpot.contains_key("a")); + assert!( + pass.announced().is_empty(), + "a port change is not a new or a gone service" + ); + } + + #[test] + fn reconcile_set_retires_a_vanished_service_and_keeps_the_others() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000), discovered("b", 8001)]; + pass.reconcile().reconcile_set( + DiscoverySource::Managed, + &svcs, + &id_set(&[]), + &id_set(&["a", "b"]), + ); + pass.scrape("a"); + pass.scrape("b"); + pass.announced(); + + pass.reconcile().reconcile_set( + DiscoverySource::Managed, + &svcs[..1], + &id_set(&["a", "b"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.announced(), ["gone b"]); + assert!(!pass.state.instances.contains_key("b")); + assert!(!pass.telemetry.prev_ttft.contains_key("b")); + assert!(pass.state.instances.contains_key("a")); + assert!(pass.telemetry.prev_ttft.contains_key("a")); + } + + // ── Lemonade state matrix ─────────────────────────────────────────────── + // + // One Lemonade endpoint is probed per discovery pass. Its id encodes the host + // and port, so the states a probe can observe are: nothing ever answered; the + // first answer; the same endpoint again; the host or the port changed; the + // endpoint went quiet; it came back somewhere else; and the same answer + // arriving twice in one tick. Each row below checks the announcements, the + // instances a snapshot would carry, the telemetry kept per id, and the + // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is + // visible on all four. + + fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: format!("lemonade-{host}-{port}"), + container_name: "lemonade".into(), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + impl Pass { + /// The ids a snapshot carries. The loop builds a snapshot's instances from + /// `state.instances`, so this is the dashboard's view of the pass. + fn instance_ids(&self) -> Vec { + let mut ids: Vec = self.state.instances.keys().cloned().collect(); + ids.sort(); + ids + } + + /// The ids the vLLM Prometheus scrape would poll while `current` holds + /// the live Lemonade ids. + fn scrape_targets(&self, current: &HashSet) -> Vec { + let mut ids: Vec = + vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) + .into_iter() + .map(|(id, _port)| id) + .collect(); + ids.sort(); + ids + } + + /// A live KV reading, latency baselines and a rate tracker for `id`. + fn give_live_telemetry(&mut self, id: &str) { + self.scrape(id); + self.telemetry + .gen_trackers + .insert(id.into(), make_tracker(2)); + } + + fn has_telemetry(&self, id: &str) -> bool { + self.telemetry.gen_trackers.contains_key(id) + || self.telemetry.prev_ttft.contains_key(id) + || self.telemetry.prev_tpot.contains_key(id) + } + } + + fn ids(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, + /// the loop's current Lemonade ids and what the second probe announced. + fn lemonade_identity_change( + old: &DiscoveredService, + new: &DiscoveredService, + ) -> (Pass, HashSet, Vec) { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(old), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(new), &mut current); + let announced = pass.announced(); + (pass, current, announced) + } + + #[test] + fn lemonade_that_never_answers_leaves_no_trace() { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + pass.reconcile().reconcile_lemonade(&[], &mut current); + + assert!(pass.announced().is_empty()); + assert!(current.is_empty()); + assert!(pass.instance_ids().is_empty()); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile() + .reconcile_lemonade(&[lemonade_svc("127.0.0.1", 13305)], &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!( + current, + HashSet::from(["lemonade-127.0.0.1-13305".to_string()]) + ); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!( + pass.scrape_targets(¤t).is_empty(), + "the live Lemonade instance is not a vLLM scrape target" + ); + } + + #[test] + fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&svc), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(&[svc], &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(pass.instance_ids(), ids(&[&id])); + assert_eq!(pass.kv(&id), Some(25.0)); + assert!(pass.has_telemetry(&id)); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("10.0.0.7", 13305); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current, HashSet::from([new.container_id])); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-10.0.0.7-13305" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); + assert!( + pass.scrape_targets(¤t).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current, HashSet::from([new.container_id])); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); + assert!( + pass.scrape_targets(¤t).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile().reconcile_lemonade(&[svc], &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + + assert_eq!(pass.announced(), [format!("gone {id}")]); + assert!(current.is_empty()); + assert!(pass.instance_ids().is_empty()); + assert!(!pass.has_telemetry(&id)); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&old), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.reconcile().reconcile_lemonade(&[], &mut current); + assert_eq!( + pass.announced(), + [ + "discovered lemonade-127.0.0.1-13305", + "gone lemonade-127.0.0.1-13305" + ] + ); + + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&new), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + assert_eq!(current, HashSet::from([new.container_id.clone()])); + assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); + assert!(!pass.has_telemetry(&old.container_id)); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_answer_repeated_within_one_tick_is_announced_once() { + let svc = lemonade_svc("127.0.0.1", 13305); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&svc), &mut current); + pass.reconcile().reconcile_lemonade(&[svc], &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { + let a = lemonade_svc("127.0.0.1", 13305); + let b = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&a), &mut current); + pass.announced(); + + pass.reconcile().reconcile_lemonade(&[b], &mut current); + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&a), &mut current); + + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13306", + "discovered lemonade-127.0.0.1-13305" + ] + ); + assert_eq!(current, HashSet::from([a.container_id.clone()])); + assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile() + .reconcile_lemonade(&[discovered("l1", 13305)], &mut current); + assert_eq!(pass.announced(), ["discovered l1"]); + assert_eq!(current, HashSet::from(["l1".to_string()])); + + pass.scrape("l1"); + pass.reconcile() + .reconcile_lemonade(&[discovered("l1", 13305)], &mut current); + assert!( + pass.announced().is_empty(), + "same id must not be re-announced" + ); + assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); + + // A new id (the host or port changed) retires the old id, telemetry + // included, and announces the new one. + pass.reconcile() + .reconcile_lemonade(&[discovered("l2", 13306)], &mut current); + assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); + assert_eq!(current, HashSet::from(["l2".to_string()])); + assert!(!pass.telemetry.prev_ttft.contains_key("l1")); + // The superseded instance leaves the state (and so every snapshot), not + // only its telemetry. + assert!(!pass.state.instances.contains_key("l1")); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + assert_eq!(pass.announced(), ["gone l2"]); + assert!(current.is_empty()); + assert!(!pass.state.instances.contains_key("l2")); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + assert!(pass.announced().is_empty(), "nothing left to retire"); + } + + // Registry read health: one test per row of the state matrix (first + // failure, repeated failure, recovery, empty, populated, malformed new, + // malformed unchanged, malformed cleared). + + fn registry_read( + malformed: &[&str], + ) -> Result { + Ok(crate::registry::RegistryRead { + records: Vec::new(), + malformed: malformed.iter().map(PathBuf::from).collect(), + }) + } + + fn registry_failure( + why: &str, + ) -> Result { + Err(crate::registry::RegistryReadError { + path: PathBuf::from("services"), + source: std::io::Error::other(why), + partial: crate::registry::RegistryRead::default(), + }) + } + + /// State matrix for what a partially read registry does to the discovery + /// state, one assertion group per row: a service it saw joins the known set + /// and its non-vLLM mark follows the record; one it did not see stays known + /// and keeps its mark; nothing leaves either set. + #[test] + fn partial_registry_read_adds_what_it_saw_and_forgets_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["old-vllm", "old-lemonade", "flips"]), + managed_non_vllm: id_set(&["old-lemonade", "flips"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + seen: id_set(&["new-lemonade", "new-vllm", "flips"]), + non_vllm: id_set(&["new-lemonade"]), + ..Default::default() + }); + + assert_eq!( + state.known_services, + id_set(&[ + "old-vllm", + "old-lemonade", + "flips", + "new-lemonade", + "new-vllm" + ]), + "the unseen services stay known and the new ones join them" + ); + assert_eq!( + state.managed_non_vllm, + id_set(&["old-lemonade", "new-lemonade"]), + "an unseen non-vLLM service keeps its mark, a seen one follows its record" + ); + } + + #[test] + fn partial_registry_read_that_saw_nothing_changes_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["a", "b"]), + managed_non_vllm: id_set(&["b"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); + + assert_eq!(state.known_services, id_set(&["a", "b"])); + assert_eq!(state.managed_non_vllm, id_set(&["b"])); + } + + #[test] + fn registry_health_reports_the_first_failure_of_a_streak_once() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_failure("disk gone")); + assert_eq!( + first.failed.as_deref(), + Some("cannot read services: disk gone"), + "the first failed pass names the error" + ); + assert!(!first.recovered); + + for pass in 2..=5 { + assert_eq!( + health.observe(®istry_failure("disk gone")), + RegistryChange::default(), + "failed pass {pass} of the same streak must stay quiet" + ); + } + } + + #[test] + fn registry_health_reports_recovery_once_and_a_new_streak_again() { + let mut health = RegistryHealth::default(); + health.observe(®istry_failure("one")); + health.observe(®istry_failure("one")); + + let recovered = health.observe(®istry_read(&[])); + assert!(recovered.recovered); + assert!(recovered.failed.is_none()); + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default(), + "only the first good pass after a streak is a recovery" + ); + + let again = health.observe(®istry_failure("two")); + assert_eq!(again.failed.as_deref(), Some("cannot read services: two")); + } + + #[test] + fn registry_health_is_quiet_for_an_empty_or_populated_registry() { + let mut health = RegistryHealth::default(); + // A healthy read, empty or not, says nothing: there is no failure to + // report and no failure to have recovered from. + for _ in 0..3 { + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default() + ); + } + } + + #[test] + fn registry_health_reports_a_malformed_set_when_it_changes() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_read(&["a.json"])); + assert_eq!(first.malformed, [PathBuf::from("a.json")]); + assert_eq!( + health.observe(®istry_read(&["a.json"])), + RegistryChange::default(), + "an unchanged malformed set is not reported again" + ); + + let grown = health.observe(®istry_read(&["a.json", "b.json"])); + assert_eq!( + grown.malformed, + [PathBuf::from("a.json"), PathBuf::from("b.json")], + "a newly malformed record reports the whole set" + ); + + let cleared = health.observe(®istry_read(&[])); + assert!( + cleared.malformed.is_empty(), + "clearing the set is not a warning" + ); + let back = health.observe(®istry_read(&["a.json"])); + assert_eq!( + back.malformed, + [PathBuf::from("a.json")], + "a record that goes bad again after being fixed is reported again" + ); + } + + #[test] + fn registry_health_keeps_the_malformed_set_across_a_read_failure() { + let mut health = RegistryHealth::default(); + health.observe(®istry_read(&["a.json"])); + health.observe(®istry_failure("blip")); + + let after = health.observe(®istry_read(&["a.json"])); + assert!(after.recovered); + assert!( + after.malformed.is_empty(), + "the same corrupt record is not news just because a read failed in between" + ); + } + + #[test] + fn partial_registry_read_refreshes_the_roots_it_saw_and_keeps_the_unseen() { + use rocm_dash_collectors::ancestry::Root; + let root = |pid, owner: &str| Root { + pid, + start_ticks: None, + owner: owner.into(), + }; + let mut state = DiscoveryState { + known_services: id_set(&["seen", "unseen"]), + managed_roots: vec![root(10, "seen"), root(20, "unseen")], + ..DiscoveryState::default() + }; + + // `seen` was relaunched under a new engine pid; `unseen` was not read. + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + seen: id_set(&["seen"]), + roots: vec![root(11, "seen")], + ..Default::default() + }); + + let mut got: Vec<_> = state + .managed_roots + .iter() + .map(|r| (r.pid, r.owner.as_str())) + .collect(); + got.sort_unstable(); + assert_eq!( + got, + vec![(11, "seen"), (20, "unseen")], + "a seen service's roots are replaced, an unseen one's survive" + ); + } + + #[test] + fn partial_registry_read_drops_the_roots_of_a_service_it_saw_stop() { + use rocm_dash_collectors::ancestry::Root; + let root = |pid, owner: &str| Root { + pid, + start_ticks: None, + owner: owner.into(), + }; + let mut state = DiscoveryState { + managed_roots: vec![root(10, "stopped"), root(20, "unseen")], + ..DiscoveryState::default() + }; + // `stopped` was read this pass and is no longer running; an unrelated + // record failed to read, which is what makes this a partial pass. + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + retired: id_set(&["stopped"]), + ..Default::default() + }); + let owners: Vec<_> = state + .managed_roots + .iter() + .map(|r| r.owner.as_str()) + .collect(); + assert_eq!( + owners, + vec!["unseen"], + "a stopped service's PID must not stay matchable" + ); + } + + #[test] + fn partial_registry_read_that_saw_nothing_keeps_every_root() { + use rocm_dash_collectors::ancestry::Root; + let mut state = DiscoveryState { + managed_roots: vec![Root { + pid: 10, + start_ticks: Some(5), + owner: "a".into(), + }], + ..DiscoveryState::default() + }; + state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); + assert_eq!(state.managed_roots.len(), 1); + } +} diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index 570c0a656c..76d16e0dd2 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -964,13 +964,66 @@ fn lemonade_responder(path: &str) -> String { } } +fn two_model_responder(path: &str) -> String { + match path { + "/api/v1/health" => http( + 200, + "application/json", + r#"{"status":"ok","model_loaded":"Mock-Lemonade-1B","all_models_loaded":[{"model_name":"Mock-Lemonade-1B"},{"model_name":"Mock-Lemonade-2B"}]}"#, + ), + "/api/v1/stats" => http(200, "application/json", r#"{"tokens_per_second":33.5}"#), + _ => http(404, "text/plain", ""), + } +} + +/// An endpoint with two models loaded is two rows: one id per model, each +/// spelled out here (endpoint, then the model's UTF-8 bytes in hex) so that +/// collapsing them into one row per endpoint fails this test. +#[tokio::test] +async fn each_loaded_lemonade_model_is_announced_as_its_own_row() { + let server = spawn_server(two_model_responder).await; + let ids = [ + format!( + "lemonade-127.0.0.1-{}-model-4d6f636b2d4c656d6f6e6164652d3142", + server.port + ), + format!( + "lemonade-127.0.0.1-{}-model-4d6f636b2d4c656d6f6e6164652d3242", + server.port + ), + ]; + let (mut rx, _runner) = spawn_runner(lemonade_opts(server.port)); + let mut announced = Vec::new(); + while announced.len() < 2 { + let inst = next_event(&mut rx, |ev| match ev { + Event::InstanceDiscovered(inst) => Some(inst.clone()), + _ => None, + }) + .await; + announced.push((inst.container_id, inst.model_name)); + } + announced.sort(); + assert_eq!( + announced, + [ + (ids[0].clone(), "Mock-Lemonade-1B".to_string()), + (ids[1].clone(), "Mock-Lemonade-2B".to_string()), + ] + ); +} + /// A reachable Lemonade endpoint is announced once with its health-reported /// model, scraped for its rate, and announced gone when the endpoint stops /// answering, after which it leaves the snapshots. #[tokio::test] async fn lemonade_endpoint_is_announced_scraped_and_announced_gone() { let server = spawn_server(lemonade_responder).await; - let id = format!("lemonade-127.0.0.1-{}", server.port); + // The identity is spelled out, not derived from the production function, + // so a change to it fails here: endpoint, then the model's UTF-8 bytes in hex. + let id = format!( + "lemonade-127.0.0.1-{}-model-4d6f636b2d4c656d6f6e6164652d3142", + server.port + ); let (mut rx, _runner) = spawn_runner(lemonade_opts(server.port)); let announced = next_event(&mut rx, |ev| match ev { diff --git a/crates/rocm-dash-daemon/tests/telemetry_contract.rs b/crates/rocm-dash-daemon/tests/telemetry_contract.rs index c15589325d..5ac26a3b66 100644 --- a/crates/rocm-dash-daemon/tests/telemetry_contract.rs +++ b/crates/rocm-dash-daemon/tests/telemetry_contract.rs @@ -27,15 +27,24 @@ //! Scenarios 1–3, 5–8 verify already-correct or distinguishably-observable //! behaviour. All scenarios now serve as regression guards. //! +//! The `lemonade_*` tests below the matrix cover Lemonade discovery and +//! ownership at the same seam, and are not rows in it: identity and the shared +//! last-request rate across several loaded models; external discovery, its +//! warnings (a seen endpoint closing, one never seen, an HTTP error on first +//! contact, recovery) and its removal of rows; a managed record taking over, or +//! being removed from, an endpoint; managed rewrites that reset the attributed +//! rate; scrape-only ticks; and IPv4/IPv6 dual-stack binds. The test names say +//! which. +//! //! Timing: `tick_override = 200 ms`, `discovery_tick = 200 ms`, //! `instance_tick = 400 ms` (scrape every 2nd base tick). //! Expiry window boundary: `clamp(3 × 400ms, 6 s, 30 s) = 6 s`. //! -//! ## Frozen observation schema (proposed — do not implement in production during this task) +//! ## Observation schema (as shipped in `rocm-dash-core`) //! -//! The controller-reviewed additive schema for EAI-7960 observation metadata. -//! Production types (`rocm-dash-core`) are unchanged; this block is the -//! candidate contract for the implementer to reference. +//! The additive schema for EAI-7960 observation metadata. The production types +//! are `ObservationMetadata` and `ObservationFreshness` in `rocm-dash-core`'s +//! `metrics.rs`; this block is the contract the tests below assert against. //! //! ```rust,ignore //! // Existing value carrier — unchanged, remains the sole numeric source. @@ -102,6 +111,1381 @@ use rocm_dash_daemon::bench_ring::BenchRing; use rocm_dash_daemon::runner::{self, RunnerOptions}; use rocm_dash_daemon::snapshot_ring::SnapshotRing; +/// Read-only Lemonade fixture with a mutable health body. The fixture records +/// unexpected requests, including any attempt to manage or stop the server. +async fn lemonade_fixture( + health: Arc>, + unexpected: Arc, +) -> (u16, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + lemonade_fixture_on(listener, health, stats, unexpected).await +} + +/// Same fixture on a caller-chosen listener with a mutable stats body. +// Async so call-sites read `.await` like `lemonade_fixture`. +#[allow(clippy::unused_async)] +async fn lemonade_fixture_on( + listener: TcpListener, + health: Arc>, + stats: Arc>, + unexpected: Arc, +) -> (u16, tokio::task::JoinHandle<()>) { + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let health = health.clone(); + let stats = stats.clone(); + let unexpected = unexpected.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let body = if request.starts_with("GET /api/v1/health ") { + health.lock().unwrap().clone() + } else if request.starts_with("GET /api/v1/stats ") { + stats.lock().unwrap().clone() + } else { + unexpected.fetch_add(1, Ordering::Relaxed); + "{}".to_string() + }; + let response = if body == "HTTP_503" { + HTTP_503.to_string() + } else { + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + (port, task) +} + +/// With the shipped cadence (discovery more often than instance scrapes) a +/// discovery-only tick asks the endpoint for its health but must not fetch its +/// stats: those ride the instance cadence. Here discovery runs every tick and the +/// instance scrape every fifth, so stats requests stay well under the health ones. +#[tokio::test] +async fn lemonade_stats_are_fetched_on_the_instance_cadence_not_every_discovery_tick() { + let health_hits = Arc::new(AtomicU64::new(0)); + let stats_hits = Arc::new(AtomicU64::new(0)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = { + let (health_hits, stats_hits) = (health_hits.clone(), stats_hits.clone()); + tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let (health_hits, stats_hits) = (health_hits.clone(), stats_hits.clone()); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let body = if request.starts_with("GET /api/v1/health ") { + health_hits.fetch_add(1, Ordering::Relaxed); + r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":[{"model_name":"Model-A"}]}"# + } else { + stats_hits.fetch_add(1, Ordering::Relaxed); + r#"{"tokens_per_second":42.0}"# + }; + let response = format!( + "HTTP/1.1 200 OK +Content-Length: {} +Connection: close + +{body}", + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }) + }; + let (tx, _rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK * 5, + ..RunnerOptions::default() + }, + ) + .await; + let deadline = tokio::time::Instant::now() + SCENARIO_DEADLINE; + while health_hits.load(Ordering::Relaxed) < 15 { + assert!( + tokio::time::Instant::now() < deadline, + "the endpoint was never polled 15 times" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + task.abort(); + server.abort(); + let (health, stats) = ( + health_hits.load(Ordering::Relaxed), + stats_hits.load(Ordering::Relaxed), + ); + assert!(stats >= 1, "the instance cadence still fetches stats"); + assert!( + stats * 3 <= health, + "stats must follow the instance cadence (1 in 5), not every poll: {stats} stats for {health} health requests" + ); +} + +#[tokio::test] +async fn lemonade_multiple_models_keep_identity_and_do_not_share_last_request_rate() { + let health = Arc::new(Mutex::new(r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"},{"model_name":"Model-B"},{"model_name":"Model-A"}]}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health.clone(), unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let clock_dir = tempfile::tempdir().unwrap(); + let clock = clock_dir.path().join("clock"); + std::fs::write(&clock, "hold").unwrap(); + let opts = RunnerOptions { + test_clock_offset_path: Some(clock), + enable_lemonade: true, + lemonade_port: port, + disable_vllm_metrics: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + let result = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await; + if result.is_err() { + task.abort(); + server.abort(); + } + let rows = result.expect("two deduplicated loaded model rows"); + assert!(rows.iter().all(|row| row.gen_tps.is_none())); + let model_a_id = rows + .iter() + .find(|row| row.model_name == "Model-A") + .unwrap() + .container_id + .clone(); + let model_b_id = rows + .iter() + .find(|row| row.model_name == "Model-B") + .unwrap() + .container_id + .clone(); + assert_ne!(model_a_id, model_b_id); + *health.lock().unwrap() = r#"{"status":"ok","model_loaded":"Model-B","all_models_loaded":[{"model_name":"Model-B"},{"model_name":"Model-A"}]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| { + let a = snap + .instances + .iter() + .find(|row| row.container_id == model_a_id)?; + let b = snap + .instances + .iter() + .find(|row| row.container_id == model_b_id)?; + (a.gen_tps.is_none() && b.gen_tps.is_none()).then_some(()) + }) + .await + .expect("last access is not last request; multiple loaded models have no attributable rate"); + *health.lock().unwrap() = + r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"}]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == model_a_id + && snap.instances[0].gen_tps == Some(42.0)) + .then_some(()) + }) + .await + .expect("unloaded model disappears; singleton receives rate"); + // Freeze observation time: expiry cannot make a missing invalidation pass, + // even if the test is descheduled or waits through its entire deadline. + drain_snapshots(&mut rx); + *health.lock().unwrap() = + r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":["Model-A","Model-B"]}"# + .to_string(); + let ambiguous = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await + .expect("adding another model produces both rows on the next discovery"); + assert!( + ambiguous + .iter() + .all(|row| row.gen_tps.is_none() && row.gen_tps_observation.is_none()), + "the first multi-model snapshot must invalidate the singleton rate and observation" + ); + *health.lock().unwrap() = + r#"{"status":"error","all_models_loaded":["Model-A","Model-B"]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| snap.instances.is_empty().then_some(())) + .await + .expect("non-ok health never remains ready"); + *health.lock().unwrap() = r#"{"status":"ok","all_models_loaded":["Model-A"]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].container_id == model_a_id).then_some(()) + }) + .await + .expect("healthy recovery restores the same model identity"); + *health.lock().unwrap() = r#"{"status":"ok","all_models_loaded":[]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| snap.instances.is_empty().then_some(())) + .await + .expect("healthy router without loaded models is not serving"); + task.abort(); + server.abort(); + assert_eq!( + unexpected.load(Ordering::Relaxed), + 0, + "only read-only health/stats requests are allowed" + ); +} + +#[tokio::test] +async fn lemonade_managed_nondefault_port_scrapes_even_when_external_discovery_disabled() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"model-alias","canonical_model_id":"Model-A","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let opts = RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + let result = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == "managed-lemonade" + && snap.instances[0].managed + && snap.instances[0].gen_tps == Some(42.0) + && snap.instances[0].gen_tps_last_request) + .then_some(()) + }) + .await; + task.abort(); + server.abort(); + result.expect("managed Lemonade is scraped at its registry port without external discovery"); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// A successful Lemonade scrape proves the server answers health and stats, but +/// the row's status still comes from the registry record: unlike a Docker +/// container, whose first good scrape is the only sign it is serving, a managed +/// service's own record says when it is ready. The telemetry reaching the row in +/// the same snapshot is what shows the scrape did succeed. +#[tokio::test] +async fn a_successful_lemonade_scrape_leaves_a_starting_managed_row_starting() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":[{"model_name":"Model-A"}]}"# + .to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"starting-lemonade","engine":"lemonade","model_ref":"Model-A","canonical_model_id":"Model-A","host":"127.0.0.1","port":{port},"status":"starting"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let snapshot = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then(|| snap.clone()) + }) + .await; + task.abort(); + server.abort(); + let snapshot = snapshot.expect("the scrape reaches the managed row"); + let row = &snapshot.instances[0]; + assert_eq!(row.container_id, "starting-lemonade"); + assert!( + matches!( + row.status, + rocm_dash_core::metrics::InstanceStatus::Starting { .. } + ), + "a successful scrape must not promote the registry's Starting: {:?}", + row.status + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_equivalent_endpoint_records_do_not_duplicate_singleton_rate() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + for (id, host) in [("first", "127.0.0.1"), ("second", "LOCALHOST")] { + std::fs::write(dir.path().join(format!("{id}.json")), format!(r#"{{"service_id":"{id}","engine":"lemonade","model_ref":"Model-A","host":"{host}","port":{port},"status":"ready"}}"#)).unwrap(); + } + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let result = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await; + task.abort(); + server.abort(); + let rows = result.expect("both registry identities remain present"); + assert!( + rows.iter() + .all(|row| row.managed && row.gen_tps.is_none() && row.gen_tps_observation.is_none()), + "equivalent endpoint records cannot each receive its singleton rate: {rows:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_external_endpoint_deduplicates_managed_owner_then_reappears_after_record_removal() +{ + let health = Arc::new(Mutex::new(r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"},{"model_name":"Model-B"}]}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let record = dir.path().join("managed.json"); + std::fs::write(&record, format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"Model-A","host":"localhost","port":{port},"status":"ready"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let opts = RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == "managed-lemonade" + && snap.instances[0].gen_tps.is_none()) + .then_some(()) + }) + .await + .expect("managed endpoint is not duplicated or assigned ambiguous stats"); + std::fs::remove_file(record).unwrap(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2 + && snap.instances.iter().all(|row| { + !row.managed && row.container_id != "managed-lemonade" && row.gen_tps.is_none() + })) + .then_some(()) + }) + .await + .expect("removing the registry file does not stop an externally reachable server"); + task.abort(); + server.abort(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_managed_record_taking_over_endpoint_drops_existing_external_rows() { + let health = Arc::new(Mutex::new(r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"},{"model_name":"Model-B"}]}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let opts = RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2 && snap.instances.iter().all(|row| !row.managed)).then_some(()) + }) + .await + .expect("external rows are discovered before any managed record exists"); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"Model-A","host":"localhost","port":{port},"status":"ready"}}"#)).unwrap(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == "managed-lemonade" + && snap.instances[0].managed) + .then_some(()) + }) + .await + .expect("managed takeover removes the external rows and keeps the managed row"); + task.abort(); + server.abort(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_external_health_failure_warns_and_removes_rows_without_grace() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health.clone(), unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let row = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)) + .then(|| snap.instances[0].clone()) + }) + .await + .expect("external singleton established"); + assert!( + row.gen_tps_last_request, + "a Lemonade rate is the last request's and must be flagged as such" + ); + drain_snapshots(&mut rx); + *health.lock().unwrap() = "HTTP_503".to_string(); + // Discovery runs every tick here, so the rows go on the next one. A hold of + // even a few ticks (the observation validity window is 6 s, some thirty + // ticks) is a regression: count the snapshots that still carry the row. + let still_listed = AtomicU64::new(0); + let result = wait_for_snapshot(&mut rx, |snap| { + if snap.instances.is_empty() { + Some(snap.warnings.clone()) + } else { + still_listed.fetch_add(1, Ordering::Relaxed); + None + } + }) + .await; + task.abort(); + server.abort(); + let warnings = result.expect("external failure removes rows on next discovery, without grace"); + assert!( + still_listed.load(Ordering::Relaxed) <= 3, + "rows were held for {} snapshots after the health failure", + still_listed.load(Ordering::Relaxed) + ); + assert!( + warnings + .iter() + .any(|warning| warning.contains("lemonade health") + && warning.contains(&port.to_string()) + && warning.contains("503")), + "failed disappearance carries the health error: {warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_seen_external_endpoint_closing_warns_and_removes_rows() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + wait_for_snapshot(&mut rx, |snap| (snap.instances.len() == 1).then_some(())) + .await + .unwrap(); + server.abort(); + let _ = server.await; + drain_snapshots(&mut rx); + let result = wait_for_snapshot(&mut rx, |snap| { + snap.instances.is_empty().then(|| snap.warnings.clone()) + }) + .await; + let warnings = result.expect("closing a seen endpoint removes its rows"); + assert!( + warnings + .iter() + .any(|warning| warning.contains("lemonade health") + && warning.contains(&port.to_string())), + "disappearance must name the endpoint: {warnings:?}" + ); + + // The endpoint comes back on the same port: its row returns and the warning + // about its disappearance is cleared, not kept for as long as the daemon runs. + let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap(); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let (_, revived) = lemonade_fixture_on(listener, health, stats, unexpected.clone()).await; + drain_snapshots(&mut rx); + let recovered = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1).then(|| snap.warnings.clone()) + }) + .await; + task.abort(); + revived.abort(); + let warnings = recovered.expect("a recovered endpoint's row returns"); + assert!( + !warnings.iter().any(|warning| warning.contains("lemonade")), + "recovery must clear the disappearance warning: {warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_never_seen_absent_external_endpoint_is_quiet() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let snap = wait_for_snapshot(&mut rx, |snap| Some(snap.clone())).await; + task.abort(); + let snap = snap.expect("absent endpoint does not block snapshots"); + assert!(snap.instances.is_empty()); + assert!( + !snap + .warnings + .iter() + .any(|warning| warning.contains("lemonade")), + "normal absent external endpoint is not a noisy warning" + ); +} + +/// A never-seen external endpoint that answers with an HTTP error is present +/// but unusable (auth needed or broken), so it warns on first contact. +async fn never_seen_endpoint_answering_warns(status_line: &'static str, needle: &str) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let _ = socket.read(&mut bytes).await; + let response = format!( + "HTTP/1.1 {status_line}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let warnings = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|w| w.contains("lemonade")) + .then(|| snap.warnings.clone()) + }) + .await; + task.abort(); + server.abort(); + let warnings = warnings.expect("first-contact HTTP error warns"); + assert!( + warnings.iter().any(|w| w.contains("lemonade health") + && w.contains(&port.to_string()) + && w.contains(needle)), + "warning names the endpoint and status: {warnings:?}" + ); +} + +#[tokio::test] +async fn lemonade_never_seen_endpoint_answering_401_warns() { + never_seen_endpoint_answering_warns("401 Unauthorized", "401").await; +} + +#[tokio::test] +async fn lemonade_never_seen_endpoint_answering_503_warns() { + never_seen_endpoint_answering_warns("503 Service Unavailable", "503").await; +} + +#[tokio::test] +async fn lemonade_protected_managed_endpoint_keeps_registry_status_without_telemetry_warning() { + for status in ["401 Unauthorized", "403 Forbidden"] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let unexpected = Arc::new(AtomicU64::new(0)); + let seen = unexpected.clone(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let seen = seen.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + if !request.starts_with("GET /api/v1/health ") + || request.to_ascii_lowercase().contains("authorization:") + { + seen.fetch_add(1, Ordering::Relaxed); + } + let _ = socket.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes()).await; + }); + } + }); + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"protected","engine":"lemonade","model_ref":"Model-A","host":"127.0.0.1","port":{port},"status":"starting"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let snapshot = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1).then(|| snap.clone()) + }) + .await; + task.abort(); + server.abort(); + let snapshot = snapshot.unwrap(); + let row = &snapshot.instances[0]; + assert!(row.managed); + assert_eq!(row.container_id, "protected"); + assert!(matches!( + row.status, + rocm_dash_core::metrics::InstanceStatus::Starting { .. } + )); + assert!(row.gen_tps.is_none() && row.gen_tps_observation.is_none()); + assert!( + !snapshot + .warnings + .iter() + .any(|warning| warning.contains("lemonade")), + "protected telemetry is unavailable, not a recurring health failure: {:?}", + snapshot.warnings + ); + assert_eq!( + unexpected.load(Ordering::Relaxed), + 0, + "no stats, fallback, credentials or lifecycle requests" + ); + } +} + +/// A slow health response can finish only after the other endpoint's stats +/// request. Both serial orders fail: fast health waits for slow health to start. +#[tokio::test] +async fn lemonade_independent_endpoints_overlap_and_health_failures_warn() { + let slow_started = Arc::new(tokio::sync::Notify::new()); + let fast_stats = Arc::new(tokio::sync::Notify::new()); + let fast_stats_count = Arc::new(AtomicU64::new(0)); + let failed = Arc::new(AtomicBool::new(false)); + let unexpected = Arc::new(AtomicU64::new(0)); + let mut servers = Vec::new(); + let dir = tempfile::tempdir().unwrap(); + for (id, model, slow) in [("slow", "Model-S", true), ("fast", "Model-F", false)] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + std::fs::write(dir.path().join(format!("{id}.json")), format!(r#"{{"service_id":"{id}","engine":"lemonade","model_ref":"{model}","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + let slow_started = slow_started.clone(); + let fast_stats = fast_stats.clone(); + let fast_stats_count = fast_stats_count.clone(); + let failed = failed.clone(); + let unexpected = unexpected.clone(); + let health_succeeded = Arc::new(AtomicBool::new(false)); + servers.push(tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let slow_started = slow_started.clone(); + let fast_stats = fast_stats.clone(); + let fast_stats_count = fast_stats_count.clone(); + let failed = failed.clone(); + let unexpected = unexpected.clone(); + let health_succeeded = health_succeeded.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let response = if request.starts_with("GET /api/v1/health ") { + health_succeeded.store(false, Ordering::SeqCst); + if slow && failed.load(Ordering::SeqCst) { + HTTP_503.to_string() + } else { + let overlap = if slow { + slow_started.notify_one(); + timeout(Duration::from_secs(1), fast_stats.notified()).await + } else { + timeout(Duration::from_secs(1), slow_started.notified()).await + }; + if overlap.is_ok() { + health_succeeded.store(true, Ordering::SeqCst); + http_ok(&format!(r#"{{"status":"ok","model_loaded":"{model}"}}"#)) + } else { + HTTP_503.to_string() + } + } + } else if request.starts_with("GET /api/v1/stats ") { + if !health_succeeded.swap(false, Ordering::SeqCst) { + unexpected.fetch_add(1, Ordering::Relaxed); + } + if !slow { + fast_stats_count.fetch_add(1, Ordering::SeqCst); + fast_stats.notify_one(); + } + http_ok(r#"{"tokens_per_second":42.0}"#) + } else { + unexpected.fetch_add(1, Ordering::Relaxed); + HTTP_503.to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + })); + } + let (tx, mut rx) = broadcast::channel(64); + let clock = dir.path().join("clock"); + std::fs::write(&clock, "hold").unwrap(); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + discovery_tick: TICK, + instance_tick: TICK, + test_clock_offset_path: Some(clock), + ..RunnerOptions::default() + }, + ) + .await; + let overlap = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await; + if overlap.is_err() { + task.abort(); + for server in &servers { + server.abort(); + } + } + let rows = overlap.expect("both registry rows appear in the first scrape snapshot"); + assert!( + rows.iter().all(|row| row.gen_tps == Some(42.0)), + "independent endpoint health and stats must overlap in the same cycle, not run serially: {rows:?}" + ); + drain_snapshots(&mut rx); + let previous_fast_stats = fast_stats_count.load(Ordering::SeqCst); + failed.store(true, Ordering::SeqCst); + // Allow fast health through while slow health is now failing immediately. + slow_started.notify_one(); + let failure = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|warning| { + warning.contains("lemonade health") + && warning.contains("slow") + && warning.contains("503") + }) + .then(|| snap.instances.clone()) + }) + .await; + task.abort(); + for server in servers { + server.abort(); + } + let rows = + failure.expect("health failures must produce endpoint/row-context snapshot warnings"); + assert!( + fast_stats_count.load(Ordering::SeqCst) > previous_fast_stats, + "independent fast endpoint completes another stats scrape during slow health failure" + ); + assert!( + rows.iter().all(|row| row.gen_tps == Some(42.0)), + "failed health holds existing rates; independent fast endpoint still scrapes" + ); + assert_eq!( + unexpected.load(Ordering::Relaxed), + 0, + "only health-before-stats reads are permitted" + ); +} + +/// A managed record whose identity (model or host) is rewritten while a rate is +/// attributed must drop that rate and its observation. The test clock is held so +/// expiry cannot clear it, and the stats route stops reporting a rate after the +/// rewrite so only the identity reset can remove the held value. +async fn assert_record_rewrite_resets_attributed_rate( + write_initial: fn(&std::path::Path, u16), + rewrite: fn(&std::path::Path, u16), +) { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let listener = TcpListener::bind("0.0.0.0:0").await.unwrap(); + let (port, server) = + lemonade_fixture_on(listener, health, stats.clone(), unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let record = dir.path().join("managed.json"); + write_initial(&record, port); + let clock_dir = tempfile::tempdir().unwrap(); + let clock = clock_dir.path().join("clock"); + std::fs::write(&clock, "hold").unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + test_clock_offset_path: Some(clock), + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then_some(()) + }) + .await + .map_err(|e| format!("rate attributed: {e}"))?; + // Control: with the stats rate gone and identity unchanged, the held + // value survives, so a later None can only come from the reset. + *stats.lock().unwrap() = "{}".to_string(); + for _ in 0..4 { + let held = wait_for_snapshot(&mut rx, |snap| Some(snap.instances.clone())).await?; + if held.len() != 1 || held[0].gen_tps != Some(42.0) { + return Err(format!( + "held rate lost without an identity change: {held:?}" + )); + } + } + rewrite(&record, port); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].gen_tps.is_none() + && snap.instances[0].gen_tps_observation.is_none()) + .then_some(()) + }) + .await + .map_err(|e| format!("identity rewrite must reset the attributed rate: {e}")) + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +fn write_managed_record(path: &std::path::Path, model_ref: &str, host: &str, port: u16) { + std::fs::write(path, format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"{model_ref}","canonical_model_id":"Model-A","host":"{host}","port":{port},"status":"ready"}}"#)).unwrap(); +} + +#[tokio::test] +async fn lemonade_managed_model_rewrite_resets_attributed_rate() { + assert_record_rewrite_resets_attributed_rate( + |path, port| write_managed_record(path, "alias-one", "127.0.0.1", port), + |path, port| write_managed_record(path, "alias-two", "127.0.0.1", port), + ) + .await; +} + +/// Rewriting `canonical_model_id` to a name health does not report leaves the row +/// matching nothing, so the attributed rate is cleared because attribution is +/// lost. The same-endpoint identity clause is isolated by +/// `lemonade_managed_canonical_rewrite_that_still_matches_health_resets_rate`. +#[tokio::test] +async fn lemonade_managed_canonical_rewrite_that_unmatches_health_clears_rate() { + assert_record_rewrite_resets_attributed_rate( + |path, port| write_managed_record(path, "alias-one", "127.0.0.1", port), + |path, port| { + write_managed_record(path, "alias-one", "127.0.0.1", port); + let mut record: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + record["canonical_model_id"] = serde_json::json!("Model-B"); + std::fs::write(path, serde_json::to_vec(&record).unwrap()).unwrap(); + }, + ) + .await; +} + +/// Only `canonical_model_id` changes, and the row still matches health through +/// its `model_ref`. Nothing else then clears the rate (the row is not +/// unmatched), so removing the canonical-id clause from the same-endpoint check +/// leaves the old rate attributed to the rewritten service. +#[tokio::test] +async fn lemonade_managed_canonical_rewrite_that_still_matches_health_resets_rate() { + fn write(path: &std::path::Path, canonical: &str, port: u16) { + std::fs::write(path, format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"Model-A","canonical_model_id":"{canonical}","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + } + assert_record_rewrite_resets_attributed_rate( + |path, port| write(path, "Model-A", port), + |path, port| write(path, "Model-B", port), + ) + .await; +} + +#[tokio::test] +async fn lemonade_managed_host_rewrite_resets_attributed_rate() { + assert_record_rewrite_resets_attributed_rate( + |path, port| write_managed_record(path, "alias-one", "127.0.0.1", port), + |path, port| write_managed_record(path, "alias-one", "127.0.0.2", port), + ) + .await; +} + +/// With discovery three times slower than scraping, a health failure on a +/// scrape-only tick holds the external rows and warns; the next discovery tick +/// with the same failure removes them. The fixture counts health requests (one +/// per tick) and fails from the fourth, so the first failing ticks are +/// deterministically scrape-only (discovery runs on ticks 1, 3, 6, ...). +/// +/// `failure` is the status the health route starts answering. A 401/403 must +/// behave like the 503: the external endpoint is checked as strictly on a +/// scrape-only tick as on a discovery tick, so the warning appears on the first +/// failing tick rather than waiting for the next discovery tick. +async fn scrape_only_health_failure_holds_external_rows(failure: &'static str) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let hits = Arc::new(AtomicU64::new(0)); + let server_hits = hits.clone(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let hits = server_hits.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let response = if request.starts_with("GET /api/v1/health ") { + if hits.fetch_add(1, Ordering::SeqCst) + 1 >= 4 { + format!( + "HTTP/1.1 {failure}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ) + } else { + http_ok(r#"{"status":"ok","model_loaded":"Model-A"}"#) + } + } else if request.starts_with("GET /api/v1/stats ") { + http_ok(r#"{"tokens_per_second":42.0}"#) + } else { + HTTP_503.to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + let (tx, mut rx) = broadcast::channel(256); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: 3 * TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + let mut saw_held = false; + loop { + let (rows, warned) = wait_for_snapshot(&mut rx, |snap| { + Some(( + snap.instances.len(), + snap.warnings.iter().any(|w| w.contains("lemonade health")), + )) + }) + .await?; + match (rows, warned) { + (1, true) => saw_held = true, + (0, true) => { + return if saw_held { + Ok(()) + } else { + Err("rows removed by a scrape-only tick".to_string()) + }; + } + (0, false) if hits.load(Ordering::SeqCst) >= 4 => { + return Err("rows removed without a warning".to_string()); + } + _ => {} + } + } + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); +} + +#[tokio::test] +async fn lemonade_scrape_only_tick_holds_external_rows_until_next_discovery() { + scrape_only_health_failure_holds_external_rows("503 Service Unavailable").await; +} + +/// An already-known external endpoint that starts requiring credentials is +/// reported on the scrape-only tick that finds out, not one discovery interval +/// later (the managed-target tolerance for 401/403 does not apply to it). +#[tokio::test] +async fn lemonade_scrape_only_tick_warns_when_a_known_external_endpoint_requires_credentials() { + for status in ["401 Unauthorized", "403 Forbidden"] { + scrape_only_health_failure_holds_external_rows(status).await; + } +} + +/// A known external endpoint whose stats route fails while health is fine and one +/// model is loaded (so the rate is attributable): the failure is reported, the row +/// stays, and nothing but reads reaches the server. +#[tokio::test] +async fn lemonade_stats_failure_on_an_attributed_endpoint_warns_and_keeps_the_row() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new("HTTP_503".to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (port, server) = lemonade_fixture_on(listener, health, stats, unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let seen = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|w| w.contains("lemonade stats")) + .then(|| (snap.instances.clone(), snap.warnings.clone())) + }) + .await; + task.abort(); + server.abort(); + let (rows, warnings) = seen.expect("a stats failure is reported"); + assert_eq!(rows.len(), 1, "the row stays listed: {rows:?}"); + assert!(rows[0].gen_tps.is_none(), "{rows:?}"); + assert!( + warnings.iter().any(|w| w.contains("lemonade stats") + && w.contains(&port.to_string()) + && w.contains("503")), + "the warning names the endpoint and the status: {warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// A managed target whose health route answers with a genuine 5xx (not the +/// 404/401/403 that mean "no router" or "credentials needed") is a failure to +/// report: the registry row stays with its registry status, and the warning +/// names the endpoint and the managed service. +#[tokio::test] +async fn lemonade_managed_endpoint_health_5xx_warns_and_keeps_the_registry_row() { + let health = Arc::new(Mutex::new("HTTP_503".to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"managed-5xx","engine":"lemonade","model_ref":"Model-A","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let seen = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|w| w.contains("lemonade health")) + .then(|| (snap.instances.clone(), snap.warnings.clone())) + }) + .await; + task.abort(); + server.abort(); + let (rows, warnings) = seen.expect("a managed health 5xx is reported"); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert!( + rows[0].managed && rows[0].container_id == "managed-5xx", + "{rows:?}" + ); + assert!(rows[0].gen_tps.is_none(), "{rows:?}"); + assert!( + warnings.iter().any(|w| w.contains("lemonade health") + && w.contains(&port.to_string()) + && w.contains("managed-5xx") + && w.contains("503")), + "{warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// Bind an IPv6 listener for a test that exercises IPv6 handling. +/// +/// A failed bind fails the test: a bare `return` reads as a pass, and these two +/// tests are the only cover for the `::` to `::1` mapping and for the collector +/// cache's reuse across address families. A machine with no IPv6 loopback opts +/// out with `ROCM_TEST_NO_IPV6=1`, which prints that the test did not run. +async fn bind_ipv6(addr: &str) -> Option { + match TcpListener::bind(addr).await { + Ok(listener) => Some(listener), + Err(error) if std::env::var_os("ROCM_TEST_NO_IPV6").is_some() => { + eprintln!("SKIPPED (ROCM_TEST_NO_IPV6 set): {addr} bind failed: {error}"); + None + } + Err(error) => panic!( + "IPv6 bind {addr} failed: {error}; this test needs IPv6 loopback (set ROCM_TEST_NO_IPV6=1 on a machine that has none)" + ), + } +} + +/// A dual-stack `::` bind at the external endpoint's port is the same service as +/// the IPv4 external probe; it must not also appear as an external row. +#[tokio::test] +async fn lemonade_dual_stack_managed_bind_does_not_duplicate_external_row() { + let Some(listener) = bind_ipv6("[::]:0").await else { + return; + }; + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture_on(listener, health, stats, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + write_managed_record(&dir.path().join("managed.json"), "Model-A", "::", port); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| { + snap.instances + .iter() + .any(|row| row.container_id == "managed-lemonade" && row.gen_tps == Some(42.0)) + .then_some(()) + }) + .await?; + for _ in 0..5 { + let rows = wait_for_snapshot(&mut rx, |snap| Some(snap.instances.clone())).await?; + if rows.len() != 1 || rows[0].container_id != "managed-lemonade" { + return Err(format!("duplicate external row: {rows:?}")); + } + } + Ok(()) + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// A managed `::` record replaced by a `127.0.0.1` record on the same port +/// must not keep dialing the old IPv6 address: the IPv4-only listener has to +/// come up healthy with no standing warning. +#[tokio::test] +async fn lemonade_managed_ipv6_record_replaced_by_ipv4_record_on_same_port_recovers() { + // Take an IPv6 loopback port, then the same port on IPv4. Another process + // can hold the IPv4 side of a given port, so try a few ports before giving up. + let mut pair = None; + for _ in 0..20 { + let Some(v6) = bind_ipv6("[::1]:0").await else { + return; + }; + let port = v6.local_addr().unwrap().port(); + if let Ok(v4) = TcpListener::bind(("127.0.0.1", port)).await { + pair = Some((v6, v4, port)); + break; + } + } + let (v6, v4, port) = pair.expect("no port free on both IPv6 and IPv4 loopback after 20 tries"); + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (_, v6_server) = + lemonade_fixture_on(v6, health.clone(), stats.clone(), unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let record = dir.path().join("managed.json"); + write_managed_record(&record, "Model-A", "::", port); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then_some(()) + }) + .await + .map_err(|e| format!("ipv6 record healthy: {e}"))?; + // The old listener goes away and an IPv4-only service takes the port. + v6_server.abort(); + let _ = v6_server.await; + let (_, _v4_server) = + lemonade_fixture_on(v4, health.clone(), stats.clone(), unexpected.clone()).await; + // The replacement lands in one step, so no discovery tick sees an empty + // registry that would drop the cached collector on its own. + write_managed_record(&record, "Model-A", "127.0.0.1", port); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then_some(()) + }) + .await + .map_err(|e| format!("ipv4 record healthy: {e}"))?; + for _ in 0..4 { + let warnings = wait_for_snapshot(&mut rx, |snap| Some(snap.warnings.clone())).await?; + if warnings.iter().any(|w| w.contains("lemonade")) { + return Err(format!("standing warning after takeover: {warnings:?}")); + } + } + Ok::<(), String>(()) + } + .await; + task.abort(); + outcome.unwrap(); +} + +/// A managed direct llama-server row (the server answers only `/health` and +/// `/v1/models`, with no stats route) stays listed across repeated scrapes with +/// `gen_tps` left at `None` and no standing Lemonade warning. +#[tokio::test] +async fn lemonade_managed_direct_llama_server_row_has_no_standing_warning() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let response = if request.starts_with("GET /health ") { + http_ok(r#"{"status":"ok"}"#) + } else if request.starts_with("GET /v1/models ") { + http_ok(r#"{"data":[{"id":"Model-A"}]}"#) + } else { + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + let dir = tempfile::tempdir().unwrap(); + write_managed_record( + &dir.path().join("direct.json"), + "Model-A", + "127.0.0.1", + port, + ); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| (snap.instances.len() == 1).then_some(())).await?; + for _ in 0..6 { + let (rows, warnings) = wait_for_snapshot(&mut rx, |snap| { + Some((snap.instances.clone(), snap.warnings.clone())) + }) + .await?; + if rows.len() != 1 || rows[0].gen_tps.is_some() { + return Err(format!("row must stay visible without a rate: {rows:?}")); + } + if warnings.iter().any(|w| w.contains("lemonade")) { + return Err(format!("standing warning on a direct server: {warnings:?}")); + } + } + Ok(()) + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); +} + // ── Timing constants ──────────────────────────────────────────────────────── const TICK: Duration = Duration::from_millis(200); @@ -796,21 +2180,158 @@ esac .unwrap_or_else(|e| panic!("limit and attributed power never appeared: {e}")); assert!((limit - 750.0).abs() < f32::EPSILON, "limit was {limit}"); - // Neither service joins the per-container VRAM map (managed services can - // not), so the board is split equally and the two shares add back up to + // Neither service joins the per-container VRAM map (these records carry no + // pids, so there is no ancestry root to match), so the board is split equally and the two shares add back up to // what the GPU actually draws. assert!((a - 200.0).abs() < 1e-6, "shared-a got {a} W"); assert!((b - 200.0).abs() < 1e-6, "shared-b got {b} W"); // amd-smi lists a GPU process, so the VRAM scrape is "real", but that - // process is in no container the cgroup join can name. Managed services - // can never join that map by design, so this must not read as a /proc - // problem. + // process is in no container the cgroup join can name, and these services + // have no ancestry roots, so they could not join by design and this must + // not read as a /proc problem. assert!( !warnings.iter().any(|w| w.starts_with("vram attribution")), "false vram-attribution warning for healthy managed services: {warnings:?}" ); } +// ── Managed-service VRAM attribution by process ancestry (ROCMAI-941) ─────── + +/// A real child of `parent`, found through `/proc`, once the shell has forked it. +#[cfg(target_os = "linux")] +fn first_child_of(parent: u32) -> u32 { + for _ in 0..100 { + if let Ok(kids) = std::fs::read_to_string(format!("/proc/{parent}/task/{parent}/children")) + && let Some(pid) = kids.split_whitespace().next().and_then(|p| p.parse().ok()) + { + return pid; + } + std::thread::sleep(Duration::from_millis(50)); + } + panic!("process {parent} never forked a child"); +} + +/// Two fake "engine" processes, killed on drop, so a panic while waiting for a +/// fork (or in any assert) does not leave the shells and their sleeping +/// children behind. +#[cfg(target_os = "linux")] +struct Engines([std::process::Child; 2]); + +#[cfg(target_os = "linux")] +impl Drop for Engines { + fn drop(&mut self) { + for e in &mut self.0 { + let _ = e.kill(); + let _ = e.wait(); + } + } +} + +/// Contract: GPU processes that `amd-smi` lists are attributed to the managed +/// service they descend from, by walking parent links to the PID rocm-cli +/// recorded for it. Two services share one 400 W GPU; each gets its own VRAM +/// (not the whole device) and a power share weighted by it, and the dashboard +/// raises no "vram attribution" warning for them. +/// +/// Uses real processes: each "engine" is a shell with a real child standing in +/// for a GPU worker, so the `/proc//stat` walk is the production one. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn managed_services_get_their_own_vram_and_a_weighted_power_split() { + use rocm_core::fake_executable::write_fake_executable; + use std::process::{Command, Stdio}; + + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + std::fs::create_dir_all(&services_dir).unwrap(); + + // `; true` stops the shell exec-ing straight into `sleep`, so it really forks. + let spawn = |script: &str| { + Command::new("sh") + .args(["-c", script]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn fake process") + }; + // own-a: the recorded engine forks the worker directly. own-b: only the + // supervisor is recorded (as before the supervisor has seen its engine) and + // the worker sits two hops down, behind an unregistered intermediate shell. + let engines = Engines([ + spawn("sleep 120; true"), + spawn("sh -c 'sleep 120; true'; true"), + ]); + let workers: Vec = vec![ + first_child_of(engines.0[0].id()), + first_child_of(first_child_of(engines.0[1].id())), + ]; + + for (id, pids) in [ + ("own-a", format!(r#""engine_pid":{}"#, engines.0[0].id())), + ( + "own-b", + format!(r#""supervisor_pid":{}"#, engines.0[1].id()), + ), + ] { + let json = format!( + r#"{{"service_id":"{id}","engine":"vllm","model_ref":"M/{id}","canonical_model_id":"M/{id}","host":"127.0.0.1","port":9,"status":"ready","gpu_indices":[0],{pids},"created_at_unix_ms":0}}"# + ); + std::fs::write(services_dir.join(format!("{id}.json")), json).unwrap(); + } + + let fake = tmp.path().join("amd-smi-fake"); + write_fake_executable( + &fake, + format!( + r#"#!/bin/sh +case "$1" in + metric) echo '{{"gpu_data":[{{"gpu":0,"mem_usage":{{"used_vram":{{"value":4000}},"total_vram":{{"value":8000}}}},"usage":{{"gfx_activity":{{"value":50}}}},"temperature":{{"hotspot":{{"value":60}}}},"power":{{"socket_power":{{"value":400.0,"unit":"W"}}}}}}]}}' ;; + process) echo '{{"gpu_data":[{{"gpu":0,"process_list":[{{"process_info":{{"pid":{a},"memory_usage":{{"vram_mem":{{"value":3000,"unit":"MB"}}}}}}}},{{"process_info":{{"pid":{b},"memory_usage":{{"vram_mem":{{"value":1000,"unit":"MB"}}}}}}}}]}}]}}' ;; + *) echo '{{}}' ;; +esac +"#, + a = workers[0], + b = workers[1] + ), + ) + .unwrap(); + + let (tx, mut rx) = broadcast::channel::(64); + let opts = RunnerOptions { + amd_smi_binary: Some(fake.into_os_string()), + amd_smi_skip_device_preflight: true, + ..fast_opts(services_dir) + }; + let _runner = spawn_runner(tx, opts).await; + + let seen = wait_for_snapshot(&mut rx, |snap| { + let get = |id: &str| snap.instances.iter().find(|i| i.container_id == id); + let (a, b) = (get("own-a")?, get("own-b")?); + Some(( + a.vram_used_mb, + b.vram_used_mb, + a.power_w?, + b.power_w?, + snap.warnings.clone(), + )) + }) + .await; + + drop(engines); + let (a_vram, b_vram, a_w, b_w, warnings) = + seen.unwrap_or_else(|e| panic!("per-service VRAM and power never appeared: {e}")); + + // Each service shows what its own GPU process holds, not the device total. + assert_eq!((a_vram, b_vram), (3000, 1000), "VRAM must be per service"); + // Power follows that VRAM split of the 400 W board. + assert!((a_w - 300.0).abs() < 1e-6, "own-a got {a_w} W"); + assert!((b_w - 100.0).abs() < 1e-6, "own-b got {b_w} W"); + assert!( + !warnings.iter().any(|w| w.starts_with("vram attribution")), + "unexpected vram-attribution warning: {warnings:?}" + ); +} + // ── Scenario 5: Omitted counter → None (not zero, not panic) ─────────────── /// Contract: when the `/metrics` endpoint returns HTTP 200 but the body omits diff --git a/crates/rocm-dash-tui/examples/gen_cast.rs b/crates/rocm-dash-tui/examples/gen_cast.rs index 490480958c..36e6a706b0 100644 --- a/crates/rocm-dash-tui/examples/gen_cast.rs +++ b/crates/rocm-dash-tui/examples/gen_cast.rs @@ -3,13 +3,17 @@ // SPDX-License-Identifier: MIT //! Render the TUI against a synthetic session NDJSON to an asciicast v2 -//! recording. The marketing page replays it as a real animated terminal -//! demo — no live terminal, no external capture tools, no drift. +//! recording — no live terminal and no external capture tools. //! //! Usage: //! cargo run --release --example gen_cast -p rocm-dash-tui -- \ -//! --input /tmp/mi355x-demo.ndjson \ -//! --output ../marketing/asciinema/rocm-dash.cast +//! --input session.ndjson \ +//! --output rocm-dash.cast +//! +//! `--input` is a `PersistedEntry` NDJSON session recording, the format +//! `rocm dash --replay` reads, such as the `session-*.ndjson` that +//! `rocm dash --demo` writes under the `demo/` folder of the user data +//! directory. Whatever the input, the cast is marked as simulated data. //! //! Each frame is the actual `ui::draw` framebuffer for a point in a //! scripted storyboard (tab + modal walk), converted to a truecolor ANSI @@ -45,13 +49,15 @@ const DEFAULT_BG: (u8, u8, u8) = (13, 15, 18); about = "Render TUI storyboard to an asciicast v2 recording" )] struct Args { - /// Synthetic session file (produced by `make demo`). - #[arg(long, default_value = "/tmp/mi355x-demo.ndjson")] + /// Session NDJSON recording to render (`PersistedEntry` lines). + #[arg(long)] input: PathBuf, - /// Output cast file. Parent dir created if absent. Default targets the - /// repo-root `marketing/asciinema/` directory when run from `app/`. - #[arg(long, default_value = "../marketing/asciinema/rocm-dash.cast")] + /// Output cast file. Parent dir created if absent. The default is relative + /// to the working directory. It is git-ignored only when produced at the + /// repository root; a run from elsewhere writes the file there instead, + /// uncovered by that rule. + #[arg(long, default_value = "rocm-dash.cast")] output: PathBuf, /// Terminal grid (cols × rows) the cast is rendered at. @@ -366,6 +372,17 @@ const fn resolve_color(c: Option, default: (u8, u8, u8)) -> (u8, u8, u8) mod tests { use super::*; + #[test] + fn input_is_required_and_output_defaults_to_the_working_directory() { + let err = Args::try_parse_from(["gen_cast"]) + .err() + .expect("no recording is produced at a fixed path, so --input has no default"); + assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument); + let args = Args::try_parse_from(["gen_cast", "--input", "session.ndjson"]) + .expect("--input alone is enough"); + assert_eq!(args.output, PathBuf::from("rocm-dash.cast")); + } + #[test] fn generated_cast_is_marked_simulated() { let mut state = build_state(); diff --git a/crates/rocm-dash-tui/examples/gen_screenshots.rs b/crates/rocm-dash-tui/examples/gen_screenshots.rs index 77bd598741..fb77942d69 100644 --- a/crates/rocm-dash-tui/examples/gen_screenshots.rs +++ b/crates/rocm-dash-tui/examples/gen_screenshots.rs @@ -3,13 +3,17 @@ // SPDX-License-Identifier: MIT //! Render the TUI against a synthetic session NDJSON and dump each view to -//! a standalone SVG. The screenshots embed cleanly in the marketing page -//! and demo decks — no live terminal, no manual capture, no drift. +//! a standalone SVG — no live terminal and no manual capture. //! //! Usage: //! cargo run --release --example gen_screenshots -p rocm-dash-tui -- \ -//! --input /tmp/mi355x-demo.ndjson \ -//! --output-dir marketing/screenshots +//! --input session.ndjson \ +//! --output-dir screenshots +//! +//! `--input` is a `PersistedEntry` NDJSON session recording, the format +//! `rocm dash --replay` reads, such as the `session-*.ndjson` that +//! `rocm dash --demo` writes under the `demo/` folder of the user data +//! directory. Whatever the input, the views are marked as simulated data. //! //! Each output is the actual `ui::draw` framebuffer for a specific //! AppState configuration (tab + modal + theme). Cells get run-length @@ -30,19 +34,17 @@ use rocm_dash_tui::app::{ActiveTab, AppState, ConnState, Modal}; use rocm_dash_tui::ui; #[derive(Parser)] -#[command( - name = "gen_screenshots", - about = "Render TUI views to SVG for marketing" -)] +#[command(name = "gen_screenshots", about = "Render TUI views to SVG")] struct Args { - /// Synthetic session file (produced by `make demo`). - #[arg(long, default_value = "/tmp/mi355x-demo.ndjson")] + /// Session NDJSON recording to render (`PersistedEntry` lines). + #[arg(long)] input: PathBuf, - /// Output directory. Created if absent. SVGs are named per view. - /// Default targets the repo-root `marketing/screenshots/` directory - /// when run from the `app/` workspace. - #[arg(long, default_value = "../marketing/screenshots")] + /// Output directory. Created if absent. SVGs are named per view. The + /// default is relative to the working directory. It is git-ignored only + /// when produced at the repository root; a run from elsewhere writes the + /// files there instead, uncovered by that rule. + #[arg(long, default_value = "screenshots")] output_dir: PathBuf, /// How many snapshots to replay before capturing. Higher = fuller @@ -447,6 +449,17 @@ fn xml_escape(s: &str) -> String { mod tests { use super::*; + #[test] + fn input_is_required_and_output_dir_defaults_to_the_working_directory() { + let err = Args::try_parse_from(["gen_screenshots"]) + .err() + .expect("no recording is produced at a fixed path, so --input has no default"); + assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument); + let args = Args::try_parse_from(["gen_screenshots", "--input", "session.ndjson"]) + .expect("--input alone is enough"); + assert_eq!(args.output_dir, PathBuf::from("screenshots")); + } + #[test] fn generated_screenshot_is_marked_simulated() { let mut state = build_state("default-dark", &[], 0); diff --git a/crates/rocm-dash-tui/src/agent/clients.rs b/crates/rocm-dash-tui/src/agent/clients.rs index ae42afd3e1..33d34c9368 100644 --- a/crates/rocm-dash-tui/src/agent/clients.rs +++ b/crates/rocm-dash-tui/src/agent/clients.rs @@ -7,13 +7,14 @@ //! //! Split out of `agent.rs` to keep the `AgentClient` seam focused. The shared //! seam types (`StateSnapshot`, `AgentError`, `AgentClient`, `InferenceParams`, -//! `REQUEST_TIMEOUT`) stay in `agent/mod.rs`; the rig `Tool` registration these +//! `request_timeout`) stay in `agent/mod.rs`; the rig `Tool` registration these //! backends call into lives in `agent::tools`. use std::sync::{Arc, Mutex}; use std::time::Instant; use async_trait::async_trait; +use rocm_dash_core::chat_template::local_model_should_disable_thinking; use tracing::{error, info, warn}; use crate::app::{ChatRole, ChatTurn}; @@ -26,7 +27,7 @@ use tokio::sync::mpsc::UnboundedSender; use super::tools::{ FiredLog, register_rocm_mutating_tools, register_rocm_read_tools, register_telemetry_tools, }; -use super::{AgentClient, AgentError, InferenceParams, REQUEST_TIMEOUT, StateSnapshot}; +use super::{AgentClient, AgentError, InferenceParams, StateSnapshot, request_timeout}; /// Max tool-calling turns the model may take before producing a final answer. const MAX_TOOL_TURNS: usize = 5; @@ -37,17 +38,31 @@ const MAX_TOOL_TURNS: usize = 5; /// Used as the fallback when no explicit `--max-tokens` is configured. const MAX_AGENT_TOKENS: u64 = 1024; -/// Apply the optional sampling controls to a fresh Rig `AgentBuilder`. +/// Apply the optional sampling controls, plus any chat-template defaults the +/// named model needs, to a fresh Rig `AgentBuilder`. /// /// `max_tokens` always resolves to a concrete value (the CLI override or the /// shared [`MAX_AGENT_TOKENS`] default). `temperature` maps to the builder's -/// native `.temperature()`; `top_p` has no dedicated builder method in -/// rig-core, so it rides in `.additional_params({"top_p": ..})`, which the -/// provider merges into the request body. Both are set only when supplied, so -/// an unset knob leaves the request untouched. +/// native `.temperature()`. The rest have no dedicated builder method in +/// rig-core and ride in `.additional_params(..)`, which the provider flattens +/// into the request body: +/// +/// - `top_p`, when supplied; +/// - `chat_template_kwargs.enable_thinking = false`, when `model` names a +/// model whose chat template would otherwise reason without a budget (see +/// [`local_model_should_disable_thinking`]). `model` is `Some` only for the +/// OpenAI-compatible backend, which is the one that can be pointed at a +/// locally served model. +/// +/// Both are collected into ONE object and set with a single +/// `.additional_params(..)` call, because rig-core 0.38.1's builder *replaces* +/// that field rather than merging into it — calling it twice silently drops +/// whatever the first call set. An empty object is not sent at all, so a +/// request with neither knob set stays byte-for-byte what it was. fn apply_inference_params( builder: rig::agent::AgentBuilder, params: &InferenceParams, + model: Option<&str>, ) -> rig::agent::AgentBuilder where M: rig::completion::CompletionModel, @@ -57,8 +72,18 @@ where if let Some(temperature) = params.temperature { builder = builder.temperature(f64::from(temperature)); } + let mut additional = serde_json::Map::new(); if let Some(top_p) = params.top_p { - builder = builder.additional_params(serde_json::json!({ "top_p": top_p })); + additional.insert("top_p".to_string(), serde_json::json!(top_p)); + } + if model.is_some_and(local_model_should_disable_thinking) { + additional.insert( + "chat_template_kwargs".to_string(), + serde_json::json!({ "enable_thinking": false }), + ); + } + if !additional.is_empty() { + builder = builder.additional_params(serde_json::Value::Object(additional)); } builder } @@ -166,7 +191,8 @@ pub fn annotate_reply(reply: String, fired: &[String]) -> String { } /// Drive a built rig prompt request to completion under the shared -/// [`REQUEST_TIMEOUT`], then annotate the reply with the tools that fired. +/// [`request_timeout`] budget, then annotate the reply with the tools that +/// fired. /// /// The shared `complete()` tail, called from [`run_agent_request`] — the one /// call site all three backends (RigAgentClient, ChatGptAgentClient, @@ -188,20 +214,35 @@ async fn finish_agent_request( req: F, fired: &FiredLog, ) -> Result +where + F: std::future::IntoFuture>, + E: std::fmt::Display, +{ + finish_agent_request_within(request_timeout(), backend, req, fired).await +} + +/// [`finish_agent_request`] with the budget passed in, so a test can drive the +/// timeout path without touching the process environment. +async fn finish_agent_request_within( + budget: std::time::Duration, + backend: &'static str, + req: F, + fired: &FiredLog, +) -> Result where F: std::future::IntoFuture>, E: std::fmt::Display, { let started = Instant::now(); info!(backend, "chat request start"); - let reply = match tokio::time::timeout(REQUEST_TIMEOUT, req.into_future()).await { + let reply = match tokio::time::timeout(budget, req.into_future()).await { Err(_) => { warn!( backend, elapsed_ms = started.elapsed().as_millis() as u64, "chat request timed out" ); - return Err(AgentError::Timeout); + return Err(AgentError::Timeout(budget.as_secs())); } Ok(Err(e)) => { error!( @@ -240,9 +281,17 @@ where /// shared site. A backend with ordering constraints (ChatGPT's auth must not /// run on an empty history) does its own cheap `history.is_empty()` guard /// first — see `ChatGptAgentClient::complete` — without needing the split. +/// +/// `model` carries the model id for the OpenAI-compatible backend, the one +/// that can be pointed at a locally served model, so +/// [`apply_inference_params`] can add the chat-template defaults such a model +/// needs. The hosted backends pass `None`: their models are never served +/// locally, and an unrecognised key on a hosted request is a needless risk. +#[allow(clippy::too_many_arguments)] async fn run_agent_request( agent: rig::agent::AgentBuilder, params: &InferenceParams, + model: Option<&str>, executor: Option<&SharedRocmToolExecutor>, approval_tx: Option<&UnboundedSender>, history: &[ChatTurn], @@ -262,7 +311,7 @@ where let snap = Arc::new(snapshot); let fired: FiredLog = Arc::new(Mutex::new(Vec::new())); - let agent = apply_inference_params(agent, params); + let agent = apply_inference_params(agent, params, model); // Telemetry tools (shared registration site). let agent = register_telemetry_tools(agent, &snap, &fired); // Read-only ROCm machine-inspection tools (forward across the seam). @@ -373,6 +422,7 @@ where run_agent_request( agent, &self.params, + Some(self.model.as_str()), self.executor.as_ref(), self.approval_tx.as_ref(), history, @@ -627,6 +677,7 @@ where run_agent_request( agent, &self.params, + None, self.executor.as_ref(), self.approval_tx.as_ref(), history, @@ -722,6 +773,7 @@ where run_agent_request( agent, &self.params, + None, self.executor.as_ref(), self.approval_tx.as_ref(), history, @@ -852,6 +904,95 @@ mod tests { ROCM_MUTATING_TOOL_NAMES, ROCM_READ_TOOL_NAMES, TELEMETRY_TOOL_NAMES, fixture_snapshot, }; + /// A request still running when its budget ends is reported as a timeout + /// that names the budget it was actually given, not a fixed number. + #[tokio::test] + async fn a_request_that_outlives_its_budget_times_out_naming_that_budget() { + let fired: FiredLog = Arc::new(Mutex::new(Vec::new())); + let never = std::future::pending::>(); + + let err = + finish_agent_request_within(std::time::Duration::from_secs(1), "test", never, &fired) + .await + .expect_err("a request that never answers must time out"); + + assert!(matches!(err, AgentError::Timeout(1)), "{err:?}"); + } + + /// The other half of the budget: a reply that arrives after a delay but + /// inside the budget reaches `annotate_reply` rather than the timeout arm. + /// The budget is passed in here, so this pins the in-budget branch, not the + /// default's size — `the_budget_and_its_upper_bound_are_the_values_the_docs_promise` + /// in `agent` is what holds the 2 minutes. + #[tokio::test] + async fn a_slow_reply_inside_its_budget_is_returned_not_cancelled() { + let fired: FiredLog = Arc::new(Mutex::new(Vec::new())); + let slow = async { + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + Ok::("hello".to_owned()) + }; + + let reply = + finish_agent_request_within(std::time::Duration::from_secs(30), "test", slow, &fired) + .await + .expect("a reply inside the budget must be returned"); + + assert!(reply.contains("hello"), "{reply}"); + } + + /// Serializes the tests that write `ROCM_DASH_CHAT_TIMEOUT_SECS`, as the + /// repo's environment-mutation contract (`xtask`'s `env_mutation_contract`) + /// requires of any test that calls `set_var`. + static CHAT_TIMEOUT_ENV_TEST_LOCK: Mutex<()> = Mutex::new(()); + + /// The wiring the two tests above cannot see: that the real request path + /// takes its budget from `ROCM_DASH_CHAT_TIMEOUT_SECS`. They hand + /// `finish_agent_request_within` a budget directly, and `parse_request_timeout` + /// is tested on its own, so a `finish_agent_request` that went back to a fixed + /// constant would pass every one of them. This drives `finish_agent_request` + /// itself, with a request that never answers, and expects the 1 s the + /// environment asked for, not the 2 minute default. + /// + /// A plain `#[test]` that builds its own runtime, rather than a + /// `#[tokio::test]`: the lock has to be held across the whole run, and a + /// `std` mutex guard held over an `.await` is what `clippy::await_holding_lock` + /// rejects. + #[test] + fn the_request_path_honours_the_chat_timeout_environment_variable() { + let _guard = CHAT_TIMEOUT_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // SAFETY: serialized against the other writers by + // `CHAT_TIMEOUT_ENV_TEST_LOCK`. The tests that only read the variable, + // through `finish_agent_request`, are offline and finish in milliseconds, + // inside the 1 s it holds for. + #[allow(unsafe_code)] + unsafe { + std::env::set_var(crate::agent::REQUEST_TIMEOUT_ENV, "1"); + } + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let fired: FiredLog = Arc::new(Mutex::new(Vec::new())); + let never = std::future::pending::>(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("build a runtime for the request"); + + let err = runtime + .block_on(finish_agent_request("test", never, &fired)) + .expect_err("a request that never answers must time out"); + + assert!(matches!(err, AgentError::Timeout(1)), "{err:?}"); + })); + // Restored before the result is surfaced, so a failed assertion cannot + // leave the override set for the rest of the test binary. + #[allow(unsafe_code)] + unsafe { + std::env::remove_var(crate::agent::REQUEST_TIMEOUT_ENV); + } + result.unwrap(); + } + /// A throwaway, never-pre-existing directory for /// `ChatGptAgentClient::new_in`'s OAuth token cache, so a test builds a /// ChatGPT client without ever touching the developer's real `$HOME` (and @@ -1782,6 +1923,7 @@ mod tests { ¶ms, None, None, + None, &history, fixture_snapshot(), "mock-test", @@ -1879,6 +2021,7 @@ mod tests { ¶ms, None, None, + None, &history, fixture_snapshot(), "mock-test", @@ -1972,6 +2115,7 @@ mod tests { &InferenceParams::default(), None, None, + None, &[], fixture_snapshot(), "mock-test", @@ -2017,6 +2161,7 @@ mod tests { let reply = run_agent_request( agent, &InferenceParams::default(), + None, Some(&exec), Some(&tx), &history, @@ -2249,6 +2394,56 @@ mod tests { /// if the test builds its client with `DEFAULT_PREAMBLE` itself. const TEST_PREAMBLE: &str = "test-only preamble, distinct from the production default"; + /// Build a `RigAgentClient` whose HTTP client is a + /// [`rig::test_utils::RecordingHttpClient`] scripted with one successful + /// OpenAI chat completion, returning both so the caller can drive a real + /// `complete()` and then read the request that reached the wire. + /// + /// `RigAgentClient`'s `H` (HTTP client) type parameter — defaulted to + /// the live `reqwest::Client` for production — exists for exactly this. + fn rig_client_over_recorded_response( + model: &str, + params: InferenceParams, + ) -> ( + RigAgentClient, + rig::test_utils::RecordingHttpClient, + ) { + use rig::test_utils::RecordingHttpClient; + + let response_body = serde_json::to_vec(&json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": model, + "system_fingerprint": null, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "GPU 0 is healthy."}, + "logprobs": null, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 1, "total_tokens": 2}, + })) + .expect("serialize stub OpenAI completion response"); + let http_client = RecordingHttpClient::new(response_body); + let rig_client = rig::providers::openai::CompletionsClient::builder() + .api_key("sk-test") + .base_url("http://rocm-dash-tui-test.invalid") + .http_client(http_client.clone()) + .build() + .expect("build client over the recording HTTP double"); + + let client = RigAgentClient { + client: rig_client, + model: model.to_string(), + preamble: TEST_PREAMBLE.to_string(), + params, + executor: None, + approval_tx: None, + }; + (client, http_client) + } + /// Asserts the `model`/`temperature`/`max_tokens` fields a captured wire /// request carries, shared by the two forwards-params-to-the-wire tests /// below. Each caller separately asserts where its provider places the @@ -2280,52 +2475,19 @@ mod tests { /// actually passes it `&self.params` rather than, say, /// `&InferenceParams::default()`, or the right tag — a swap at that one /// call site stays green under every other test in this file; only a - /// real `complete()` round-trip catches it. - /// - /// `RigAgentClient`'s `H` (HTTP client) type parameter — defaulted to - /// the live `reqwest::Client` for production — exists for exactly this: - /// swap in `rig::test_utils::RecordingHttpClient` and drive the actual - /// `complete()` method over a scripted response instead of a live - /// endpoint. + /// real `complete()` round-trip catches it — over + /// [`rig_client_over_recorded_response`]'s scripted HTTP double rather + /// than a live endpoint. #[tokio::test] async fn rig_agent_client_complete_forwards_params_to_the_wire() { - use rig::test_utils::RecordingHttpClient; - - let response_body = serde_json::to_vec(&json!({ - "id": "chatcmpl-test", - "object": "chat.completion", - "created": 0, - "model": "test-model", - "system_fingerprint": null, - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "GPU 0 is healthy."}, - "logprobs": null, - "finish_reason": "stop", - }], - "usage": {"prompt_tokens": 1, "total_tokens": 2}, - })) - .expect("serialize stub OpenAI completion response"); - let http_client = RecordingHttpClient::new(response_body); - let rig_client = rig::providers::openai::CompletionsClient::builder() - .api_key("sk-test") - .base_url("http://rocm-dash-tui-test.invalid") - .http_client(http_client.clone()) - .build() - .expect("build client over the recording HTTP double"); - - let client = RigAgentClient { - client: rig_client, - model: "test-model".to_string(), - preamble: TEST_PREAMBLE.to_string(), - params: InferenceParams { + let (client, http_client) = rig_client_over_recorded_response( + "test-model", + InferenceParams { temperature: Some(0.4), top_p: None, max_tokens: Some(256), }, - executor: None, - approval_tx: None, - }; + ); let history = [ChatTurn::user("check gpu 0")]; let (reply, tags) = complete_capturing_backend_tags(&client, &history).await; @@ -2357,6 +2519,88 @@ mod tests { ); } + /// Drive one `RigAgentClient::complete` round-trip over a scripted + /// OpenAI-compatible response and hand back the JSON body that reached + /// the wire. The chat-template tests below care only about the body, so + /// they skip the reply/backend-tag assertions + /// `rig_agent_client_complete_forwards_params_to_the_wire` makes over the + /// same seam. + async fn rig_wire_request_body(model: &str, params: InferenceParams) -> serde_json::Value { + let (client, http_client) = rig_client_over_recorded_response(model, params); + let history = [ChatTurn::user("check gpu 0")]; + complete_capturing_backend_tags(&client, &history).await; + + let requests = http_client.requests(); + assert_eq!( + requests.len(), + 1, + "exactly one request should have been sent" + ); + serde_json::from_slice(&requests[0].body).expect("request body is JSON") + } + + /// A model whose chat template reasons without a budget must be told to + /// stop: the request body carries + /// `chat_template_kwargs.enable_thinking = false`, so the answer the + /// operator asked for is generated instead of an unbounded reasoning + /// trace that can consume the whole request deadline. + /// + /// The `top_p` half of this is not incidental. rig-core 0.38.1's + /// `AgentBuilder::additional_params` **replaces** rather than merges, so + /// adding the chat-template object as a second call would silently drop + /// `top_p` — a configured sampling knob vanishing off the wire, with + /// nothing else in this file to notice. Both must arrive together. + #[tokio::test] + async fn rig_agent_client_disables_unbounded_thinking_without_dropping_top_p() { + let body = rig_wire_request_body( + "unsloth/Qwen3-1.7B-GGUF:Q4_K_M", + InferenceParams { + temperature: Some(0.4), + top_p: Some(0.2), + max_tokens: Some(256), + }, + ) + .await; + + assert_eq!( + body["chat_template_kwargs"]["enable_thinking"].as_bool(), + Some(false), + "a hybrid-thinking model must be sent enable_thinking=false: {body}" + ); + assert_eq!( + body["top_p"].as_f64(), + Some(f64::from(0.2_f32)), + "top_p must survive alongside chat_template_kwargs, not be replaced by it: {body}" + ); + } + + /// The other half of the contract: every other model keeps the serving + /// endpoint's own chat-template defaults. The key is absent from the body + /// entirely — not sent as `true` — so a hosted backend that rejects an + /// unrecognised field is never handed one. + #[tokio::test] + async fn rig_agent_client_sends_no_chat_template_kwargs_for_other_models() { + let body = rig_wire_request_body( + "test-model", + InferenceParams { + temperature: Some(0.4), + top_p: Some(0.2), + max_tokens: Some(256), + }, + ) + .await; + + assert!( + body.get("chat_template_kwargs").is_none(), + "a model with no hybrid thinking mode must get no chat_template_kwargs: {body}" + ); + assert_eq!( + body["top_p"].as_f64(), + Some(f64::from(0.2_f32)), + "top_p must still reach the wire on its own: {body}" + ); + } + /// As `rig_agent_client_complete_forwards_params_to_the_wire`, for /// `AnthropicAgentClient::complete`'s call site. #[tokio::test] diff --git a/crates/rocm-dash-tui/src/agent/mod.rs b/crates/rocm-dash-tui/src/agent/mod.rs index e79a0a90b8..1af9d1fff6 100644 --- a/crates/rocm-dash-tui/src/agent/mod.rs +++ b/crates/rocm-dash-tui/src/agent/mod.rs @@ -62,9 +62,34 @@ pub use tools::{ UpdateCheckRocmTool, WatcherDisableRocmTool, WatcherEnableRocmTool, }; -/// One-shot request budget. A hung backend becomes a timeout error turn, never -/// a frozen pane. -pub const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(45); +/// Default one-shot request budget. A hung backend becomes a timeout error +/// turn, never a frozen pane. +/// +/// Sized for a locally served model on modest hardware: the first message +/// prefills the whole system prompt plus tool schemas on a cold prompt cache +/// (about 22 s for roughly 3k tokens on an integrated GPU), and the budget +/// covers every tool-calling turn of the request, not just that prefill. +pub const DEFAULT_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(2); + +/// Environment variable that overrides [`DEFAULT_REQUEST_TIMEOUT`], in whole +/// seconds. +pub const REQUEST_TIMEOUT_ENV: &str = "ROCM_DASH_CHAT_TIMEOUT_SECS"; + +/// Longest budget an override may set, so a typo cannot park a chat turn for +/// hours. +const MAX_REQUEST_TIMEOUT_SECS: u64 = 3600; + +/// The request budget in effect: [`REQUEST_TIMEOUT_ENV`] when it holds a whole +/// number of seconds in `1..=3600`, otherwise [`DEFAULT_REQUEST_TIMEOUT`]. +pub fn request_timeout() -> std::time::Duration { + parse_request_timeout(std::env::var(REQUEST_TIMEOUT_ENV).ok().as_deref()) +} + +fn parse_request_timeout(raw: Option<&str>) -> std::time::Duration { + raw.and_then(|value| value.trim().parse::().ok()) + .filter(|secs| (1..=MAX_REQUEST_TIMEOUT_SECS).contains(secs)) + .map_or(DEFAULT_REQUEST_TIMEOUT, std::time::Duration::from_secs) +} /// Optional sampling controls forwarded to the chat backend. /// @@ -94,8 +119,8 @@ pub enum AgentError { Build(String), #[error("chat sign-in failed: {0}")] Auth(String), - #[error("request timed out after {}s", REQUEST_TIMEOUT.as_secs())] - Timeout, + #[error("request timed out after {0}s (set {REQUEST_TIMEOUT_ENV} to allow longer)")] + Timeout(u64), #[error("chat request failed: {0}")] Request(String), } @@ -187,3 +212,82 @@ fn fixture_snapshot() -> StateSnapshot { bench_rows: vec![row], } } + +#[cfg(test)] +mod request_timeout_tests { + use std::time::Duration; + + use super::{ + AgentError, DEFAULT_REQUEST_TIMEOUT, MAX_REQUEST_TIMEOUT_SECS, REQUEST_TIMEOUT_ENV, + parse_request_timeout, + }; + + #[test] + fn an_unset_override_leaves_the_default_budget() { + assert_eq!(parse_request_timeout(None), DEFAULT_REQUEST_TIMEOUT); + } + + // Pins the two numbers the rest of this module only compares against + // themselves. Without these, restoring the 45 s budget this change exists + // to replace leaves every other test green. Both values are published in + // README.md, docs/testing.md and docs/manual-testing.md section 4d; change + // them together or the docs drift silently. + #[test] + fn the_budget_and_its_upper_bound_are_the_values_the_docs_promise() { + assert_eq!( + DEFAULT_REQUEST_TIMEOUT.as_secs(), + 120, + "the default budget is 2 minutes; 45 s is the cut-off this change removed" + ); + assert_eq!( + MAX_REQUEST_TIMEOUT_SECS, 3600, + "an override may reach one hour" + ); + } + + #[test] + fn a_whole_number_of_seconds_overrides_the_default() { + assert_eq!(parse_request_timeout(Some("300")), Duration::from_mins(5)); + assert_eq!( + parse_request_timeout(Some(" 90 ")), + Duration::from_secs(90), + "surrounding whitespace from a shell is not a reason to reject it" + ); + assert_eq!( + parse_request_timeout(Some(&MAX_REQUEST_TIMEOUT_SECS.to_string())), + Duration::from_secs(MAX_REQUEST_TIMEOUT_SECS) + ); + } + + #[test] + fn an_unusable_override_falls_back_to_the_default_instead_of_disabling_the_budget() { + for bad in [ + "", + "0", + "-5", + "abc", + "1.5", + "99999", + &(MAX_REQUEST_TIMEOUT_SECS + 1).to_string(), + ] { + assert_eq!( + parse_request_timeout(Some(bad)), + DEFAULT_REQUEST_TIMEOUT, + "{bad:?}" + ); + } + } + + #[test] + fn the_timeout_message_names_the_budget_and_the_variable_that_changes_it() { + let msg = AgentError::Timeout(120).to_string(); + assert!(msg.contains("after 120s"), "{msg}"); + // `parse_request_timeout` is what `request_timeout()` feeds + // `REQUEST_TIMEOUT_ENV` into, and the unit tests above prove it + // honours a valid value; this pins that the message advertises that + // same variable. That the request path really reads it is proven by + // `the_request_path_honours_the_chat_timeout_environment_variable` + // in `clients.rs`. + assert!(msg.contains(REQUEST_TIMEOUT_ENV), "{msg}"); + } +} diff --git a/crates/rocm-dash-tui/src/agent/snapshot.rs b/crates/rocm-dash-tui/src/agent/snapshot.rs index 43ebe0d5b2..fd1e071bab 100644 --- a/crates/rocm-dash-tui/src/agent/snapshot.rs +++ b/crates/rocm-dash-tui/src/agent/snapshot.rs @@ -10,7 +10,7 @@ use serde_json::{Value, json}; -use rocm_dash_core::metrics::{GpuMetrics, ObservationFreshness, ObservationMetadata}; +use rocm_dash_core::metrics::{GpuMetrics, ObservationFreshness}; use super::StateSnapshot; @@ -43,11 +43,31 @@ pub fn gpu_status_json(snap: &StateSnapshot, gpu_index: Option) -> Value } } +/// The whole-node efficiency object shared by `list_instances` and +/// `tokens_per_watt`, or `null` when it cannot be stated. +/// +/// It is computed from `StateSnapshot::latest` (the daemon's last snapshot), the +/// same source the Hardware and Observe tabs read, so it matches those screens. +/// The per-instance entries come from `StateSnapshot::instances`, the +/// event-driven map the Instances tab reads, which can be a tick ahead of or +/// behind `latest`; the two are separate views, not one. +fn node_json(snap: &StateSnapshot) -> Value { + snap.latest + .as_ref() + .and_then(rocm_dash_core::efficiency::node_efficiency) + .map_or(Value::Null, |n| { + json!({ + "tokens_per_watt": n.tokens_per_watt, + "freshness": freshness_label(n.freshness), + }) + }) +} + /// `"fresh"` | `"held"` | `null` (unknown, e.g. a legacy snapshot). -fn freshness_label(obs: Option<&ObservationMetadata>) -> Value { - obs.map_or(Value::Null, |m| { +fn freshness_label(freshness: Option) -> Value { + freshness.map_or(Value::Null, |f| { Value::String( - match m.freshness { + match f { ObservationFreshness::Fresh => "fresh", ObservationFreshness::Held => "held", } @@ -60,7 +80,11 @@ fn freshness_label(obs: Option<&ObservationMetadata>) -> Value { /// freshness metadata, and observed_at timestamp. /// /// Fields added by EAI-7960: -/// - `gen_tps`: live generation throughput from the Instance (same as daemon) +/// - `gen_tps`: generation throughput from the Instance (same as daemon) +/// - `gen_tps_last_request`: `true` when `gen_tps` is only the endpoint's most +/// recent completed request (Lemonade), which stays unchanged while the server +/// is idle, so it must not be read as live throughput; `tokens_per_watt` is +/// derived from it /// - `tokens_per_watt`: daemon-provided efficiency (never recomputed here) /// - `attributed_power_w`: this instance's share of its GPUs' board power /// (split among instances sharing a GPU). Not the same as `gpu_status`'s @@ -97,25 +121,33 @@ pub fn list_instances_json(snap: &StateSnapshot) -> Value { "running_reqs": i.running_reqs, "waiting_reqs": i.waiting_reqs, "gen_tps": i.gen_tps, + "gen_tps_last_request": i.gen_tps_last_request, "tokens_per_watt": i.tokens_per_watt, "attributed_power_w": i.power_w, "ttft_ms": i.ttft_ms, - "ttft_freshness": freshness_label(i.ttft_observation.as_ref()), + "ttft_freshness": freshness_label(i.ttft_observation.as_ref().map(|m| m.freshness)), "tpot_ms": i.tpot_ms, - "tpot_freshness": freshness_label(i.tpot_observation.as_ref()), + "tpot_freshness": freshness_label(i.tpot_observation.as_ref().map(|m| m.freshness)), "freshness": freshness, "observed_at": observed_at, }) }) .collect(); - json!({ "instances": arr, "instance_count": arr.len() }) + json!({ "instances": arr, "instance_count": arr.len(), "node": node_json(snap) }) } /// Per-instance tokens-per-watt from the daemon-provided field. /// -/// Uses `Instance::tokens_per_watt` directly so this tool matches the screen -/// truth exactly (the daemon computes the same value the TUI displays). Also -/// includes gen_tps, freshness, and observed_at for completeness. +/// Per-instance values come straight from `Instance::tokens_per_watt`, which +/// the daemon computes (never recomputed here), and include gen_tps, freshness +/// and observed_at. +/// +/// `node` is different: it is computed here, from the latest snapshot, by the +/// same function the Hardware and Observe tabs use (Σ tok/s over Σ board +/// power). Its `freshness` is `"fresh"`, `"held"`, or `null` when it is unknown +/// (no freshness metadata, e.g. an old recording), and `node` itself is `null` +/// when it cannot be stated (no traffic, zero measured traffic, or no power +/// telemetry), which is "no figure", not zero. pub fn tokens_per_watt_json(snap: &StateSnapshot) -> Value { let arr: Vec = snap .instances @@ -137,13 +169,14 @@ pub fn tokens_per_watt_json(snap: &StateSnapshot) -> Value { json!({ "name": i.container_name, "gen_tps": i.gen_tps, + "gen_tps_last_request": i.gen_tps_last_request, "tokens_per_watt": i.tokens_per_watt, "freshness": freshness, "observed_at": observed_at, }) }) .collect(); - json!({ "instances": arr }) + json!({ "instances": arr, "node": node_json(snap) }) } /// Pass^N / Pass@N rollup over the cached bench rows, reusing the core rollup. @@ -232,6 +265,63 @@ mod tests { assert_eq!(v["instances"][0]["tokens_per_watt"], 2.0); } + #[test] + fn tokens_per_watt_json_reports_the_node_figure_with_freshness() { + use rocm_dash_core::metrics::{ObservationMetadata, Snapshot}; + let mut snap = fixture_snapshot(); + let mk = |freshness| Instance { + container_name: "a".into(), + gen_tps: Some(300.0), + gen_tps_observation: Some(ObservationMetadata { + observed_at: chrono::Utc::now(), + freshness, + }), + ..Default::default() + }; + let gpu = GpuMetrics { + device_id: "gpu-0".into(), + power_w: 600.0, + ..GpuMetrics::default() + }; + snap.latest = Some(Snapshot { + instances: vec![mk(ObservationFreshness::Held)], + gpus: vec![gpu.clone()], + ..Snapshot::default() + }); + let v = tokens_per_watt_json(&snap); + assert_eq!(v["node"]["tokens_per_watt"], 0.5); + assert_eq!(v["node"]["freshness"], "held"); + snap.latest = Some(Snapshot { + instances: vec![mk(ObservationFreshness::Fresh)], + gpus: vec![gpu], + ..Snapshot::default() + }); + assert_eq!(tokens_per_watt_json(&snap)["node"]["freshness"], "fresh"); + // Unknown freshness (an old recording) is null, never "fresh". + snap.latest = Some(Snapshot { + instances: vec![Instance { + container_name: "a".into(), + gen_tps: Some(300.0), + ..Default::default() + }], + gpus: vec![GpuMetrics { + device_id: "gpu-0".into(), + power_w: 600.0, + ..GpuMetrics::default() + }], + ..Snapshot::default() + }); + let v = tokens_per_watt_json(&snap); + assert_eq!(v["node"]["tokens_per_watt"], 0.5); + assert!( + v["node"]["freshness"].is_null(), + "freshness must not be fabricated" + ); + // Nothing to state: null, not zero. + snap.latest = Some(Snapshot::default()); + assert!(tokens_per_watt_json(&snap)["node"].is_null()); + } + #[test] fn bench_summary_json_rolls_up_groups() { let v = bench_summary_json(&fixture_snapshot()); @@ -265,6 +355,23 @@ mod tests { ); } + /// A Lemonade rate is the last completed request's, not live throughput; the + /// agent must be able to tell, in both tools that report `gen_tps`. + #[test] + fn instance_tools_carry_the_last_request_flag() { + for last_request in [true, false] { + let mut snap = + fixture_with_observation(rocm_dash_core::metrics::ObservationFreshness::Fresh); + snap.instances[0].gen_tps_last_request = last_request; + for v in [list_instances_json(&snap), tokens_per_watt_json(&snap)] { + assert_eq!( + v["instances"][0]["gen_tps_last_request"], last_request, + "{v}" + ); + } + } + } + #[test] fn list_instances_json_reports_latency_freshness_and_attributed_power() { use chrono::Utc; @@ -293,6 +400,16 @@ mod tests { assert_eq!(i["attributed_power_w"], 123.0); } + #[test] + fn list_instances_json_carries_the_same_node_object() { + let snap = fixture_snapshot(); + assert_eq!( + list_instances_json(&snap)["node"], + tokens_per_watt_json(&snap)["node"], + "the two tools must report one node figure" + ); + } + #[test] fn gpu_status_json_reports_the_power_limit_or_null() { use rocm_dash_core::metrics::Snapshot; diff --git a/crates/rocm-dash-tui/src/agent/tools.rs b/crates/rocm-dash-tui/src/agent/tools.rs index 0c96d22d6a..97f9784571 100644 --- a/crates/rocm-dash-tui/src/agent/tools.rs +++ b/crates/rocm-dash-tui/src/agent/tools.rs @@ -10,6 +10,12 @@ //! seam types (`StateSnapshot`, `AgentError`, `AgentClient`) stay in //! `agent/mod.rs`; the JSON summarization these tools call lives in //! `agent::snapshot`. +//! +//! This is also where a read-only tool result is shaped for the model: +//! [`model_facing_tool_result`] drops the command output the bin's MCP +//! envelope carries twice and bounds what is left to +//! [`TOOL_RESULT_CHAR_BUDGET`], so one large result cannot exhaust a locally +//! served model's context. See `docs/llm-tool-use.md`. use std::sync::{Arc, Mutex}; @@ -113,6 +119,7 @@ impl Tool for ListInstancesTool { name: Self::NAME.to_string(), description: "List discovered serving instances: name, model, status, \ KV-cache usage %, running/waiting request counts, \ + the whole-node `node` figure (see tokens_per_watt), \ generation tok/s, TTFT/TPOT (each with a fresh/held \ flag; held means the last value, not a live reading), \ and attributed_power_w (this instance's share of its \ @@ -171,7 +178,14 @@ impl Tool for TokensPerWattTool { name: Self::NAME.to_string(), description: "Per-instance efficiency: generation tokens/sec divided by \ the instance's attributed share of its GPUs' power (W). \ - GPUs shared by several instances are split between them." + GPUs shared by several instances are split between them. \ + Also returns `node`: whole-node tokens/sec over total \ + board power (whole-board, so it is not the sum of the \ + per-instance values), with freshness `fresh`, `held` \ + (carried from a prior scrape, expires within about \ + 30 s) or null when unknown. `node` is null, meaning \ + no figure and not zero, when there is no traffic or \ + no power telemetry." .to_string(), parameters: json!({ "type": "object", "properties": {} }), } @@ -208,16 +222,209 @@ pub const TELEMETRY_TOOL_NAMES: [&str; 4] = [ // stop has finished). `doctor`/`examine` can re-register the configured setup // runtime in the runtime registry when its entry is missing. `update_check` and // `install_sdk_dry_run` cache package-index metadata under the cache folder. -// `update_check` can also, on a host with no usable Python on PATH, download a -// managed `uv` and Python into the data folder and record that Python in the -// saved config; `install_sdk_dry_run` prepares no Python and saves no Python -// path: on such a host it reports the Python a real install would prepare. (A -// ROCm 10 preview still resolves package versions with `uv` and can fetch the -// managed `uv` into the data folder.) The boundary type +// Neither prepares a Python or saves a Python path: on a host with no usable +// Python on PATH they report the Python a real update or install would prepare +// (`update_check` runs `rocm update`, whose text report then carries a +// `python_note:` line saying so), and only a real apply or install, which these +// tools never run, prepares it. `install_sdk_dry_run` fetches no `uv` either: a +// ROCm 10 preview runs a `uv` that is already installed to resolve the package +// versions and, with none, prints the lines that depend on them as not resolved, +// so it creates no `tools` folder in the data folder. (`update_check` on a ROCm +// 10 or newer runtime still resolves with `uv` and can fetch the managed `uv` +// into the data folder, though never a Python.) The boundary type // (`SharedRocmToolExecutor`) is plain data — importing it does NOT pull // `rocm-core` into this crate; the `agent` module stays the sole `rig` namer. // --------------------------------------------------------------------------- +/// How many characters of one tool result may reach the model. +/// +/// A tool result is not free: it is sent back as a message and counts against +/// the model's context on that request and on every later turn of the same +/// conversation. A locally served model is typically configured with a 32k +/// context (llama.cpp's default `n_ctx`), of which the system prompt and the +/// tool schemas already take a few thousand tokens, so the remaining room has +/// to cover several turns. 16 KiB is roughly 4.5k tokens — one large result +/// still fits alongside the prompt and a handful of turns, where an uncapped +/// one (`rocm_command` with `["model"]` measured 204,470 characters, about +/// 58k tokens) cannot fit in such a context at all and the provider rejects +/// the whole request. +pub(crate) const TOOL_RESULT_CHAR_BUDGET: usize = 16 * 1024; + +/// Shape one executor result for the model: drop the command output the +/// envelope carries twice, then bound what is left to `budget`. +/// +/// Both halves exist because the result is JSON the dash hands to the model +/// verbatim, unlike the CLI chat path (`apps/rocm/src/chat/turn.rs`), which +/// reads only the envelope's text. +fn model_facing_tool_result(value: Value, budget: usize) -> Value { + cap_tool_result(drop_duplicated_command_output(value), budget) +} + +/// Drop `structuredContent.stdout` / `.stderr` when the envelope's own text +/// already carries them. +/// +/// The bin's `internal_mcp_tool_result_from_command` puts a command's captured +/// output in `content[0].text` *and* again in `structuredContent.stdout` (and +/// `.stderr`). The MCP envelope keeps both — `rocm mcp-call` and the approval +/// replay path read the structured fields — but the model only needs one copy, +/// and charging it twice for the same bytes is what makes a big read-only +/// result twice as expensive as it has to be. Only the duplicated keys go; a +/// tool whose data lives *only* in `structuredContent` (`engines`, +/// `bridge_snapshot`, `automations`, `gpu_snapshot`) is untouched. +fn drop_duplicated_command_output(mut value: Value) -> Value { + let text = tool_result_content_text(&value); + if text.is_empty() { + return value; + } + let Some(structured) = value + .get_mut("structuredContent") + .and_then(Value::as_object_mut) + else { + return value; + }; + for key in ["stdout", "stderr"] { + let duplicated = structured + .get(key) + .and_then(Value::as_str) + .is_some_and(|s| { + let trimmed = s.trim(); + trimmed.is_empty() || text.contains(trimmed) + }); + if duplicated { + structured.remove(key); + } + } + value +} + +/// The text items of an MCP tool-result envelope, joined. Empty when the value +/// is not such an envelope (a bare `{"error": …}`, say). +fn tool_result_content_text(value: &Value) -> String { + value + .get("content") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|item| item.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n") + }) + .unwrap_or_default() +} + +/// Everything the envelope carries, rendered as plain text. +/// +/// Used only when a result has to be truncated, so the model reads prose +/// rather than a JSON document cut off mid-escape. Nothing is dropped +/// silently: the text items come first, then whatever `structuredContent` +/// still holds after [`drop_duplicated_command_output`]. +fn tool_result_plain_text(value: &Value) -> String { + let mut parts = Vec::new(); + let text = tool_result_content_text(value); + if !text.is_empty() { + parts.push(text); + } + match value.get("structuredContent") { + Some(structured) if !structured.is_null() => { + parts.push( + serde_json::to_string_pretty(structured).unwrap_or_else(|_| structured.to_string()), + ); + } + _ => {} + } + if parts.is_empty() { + serde_json::to_string_pretty(value).unwrap_or_else(|_| value.to_string()) + } else { + parts.join("\n") + } +} + +/// The line appended to a truncated result. +/// +/// Explicit about being a partial result and about how much is missing, so +/// neither the model nor the user reading the model's answer can take a cut +/// list for a complete one. +fn truncation_marker(shown_chars: usize, total_chars: usize) -> String { + format!( + "\n[output truncated: {shown_chars} of {total_chars} characters shown, \ + {omitted} omitted. This is NOT the whole result — say so when you use it, \ + and run the matching `rocm` command in a terminal for the full output.]", + omitted = total_chars.saturating_sub(shown_chars) + ) +} + +/// Bound a tool result to `budget` characters of serialized JSON. +/// +/// A result that already fits is returned untouched, so the common case keeps +/// its exact shape. One that does not is replaced by the plain-text rendering +/// cut on a line boundary plus [`truncation_marker`]; `budget` is measured +/// against the *serialized* value, which is what actually reaches the model, +/// so JSON escaping cannot push it back over. +fn cap_tool_result(value: Value, budget: usize) -> Value { + if value.to_string().len() <= budget { + return value; + } + // The truncated form is a fresh object, so whatever the rest of the chat + // path reads off the envelope has to be carried over deliberately. + // `isError` is the success/failure signal (`turn.rs` reads it), and a + // command that failed must not read as a success merely because its + // output was too long to keep. + let is_error = value.get("isError").and_then(Value::as_bool); + let full = tool_result_plain_text(&value); + let total_chars = full.chars().count(); + // Candidate cuts: the start, then the end of every line, so a cut never + // lands inside a line (and never inside a UTF-8 character). + let mut cuts = vec![0usize]; + cuts.extend(full.match_indices('\n').map(|(index, _)| index + 1)); + if !full.is_empty() && !full.ends_with('\n') { + cuts.push(full.len()); + } + // Largest cut that still fits. `best` is only ever set to a cut that was + // measured as fitting, so the returned value is within budget even where + // the length is not perfectly monotonic in the cut (the marker's digit + // count shifts by one across a power of ten). + let (mut low, mut high, mut best) = (0usize, cuts.len() - 1, 0usize); + while low <= high { + let mid = low.midpoint(high); + if truncated_tool_result(&full, cuts[mid], total_chars, is_error) + .to_string() + .len() + <= budget + { + best = mid; + low = mid + 1; + } else if mid == 0 { + break; + } else { + high = mid - 1; + } + } + truncated_tool_result(&full, cuts[best], total_chars, is_error) +} + +/// The truncated form of `full` kept up to the byte offset `cut`. `is_error` +/// is the envelope's `isError`, carried over when it had one so truncating a +/// failed command's output does not turn it into a success. +fn truncated_tool_result( + full: &str, + cut: usize, + total_chars: usize, + is_error: Option, +) -> Value { + let kept = &full[..cut]; + let shown_chars = kept.chars().count(); + let mut truncated = json!({ + "truncated": true, + "tool_output": format!("{kept}{}", truncation_marker(shown_chars, total_chars)), + }); + if let Some(is_error) = is_error { + truncated["isError"] = json!(is_error); + } + truncated +} + /// Shared body for every read-only ROCm tool: forward the intent to the injected /// executor (None ⇒ a clear "not available in this mode" message; ApprovalRequired /// ⇒ a "needs approval" error and nothing runs — `rocm_command` lands here for @@ -243,7 +450,7 @@ async fn run_rocm_read_tool( return json!({ "error": "ROCm tools are unavailable in this mode (demo/replay/mock)." }); }; match tokio::task::spawn_blocking(move || exec.execute(name, &args)).await { - Ok(RocmToolOutcome::Result(v)) => v, + Ok(RocmToolOutcome::Result(v)) => model_facing_tool_result(v, TOOL_RESULT_CHAR_BUDGET), Ok(RocmToolOutcome::Error(s)) => json!({ "error": s }), Ok(RocmToolOutcome::ApprovalRequired(_)) => { json!({ "error": "this action requires approval (interactive chat only)" }) @@ -406,17 +613,24 @@ rocm_read_tool!( rocm_read_tool!( RocmCommandRocmTool, "rocm_command", + // The examples are deliberately the cheap commands. A small local model + // copies the example in a tool description literally, and the example used + // to be `["model"]` — the single largest read-only result there is, which + // on a 32k-context model used to end the turn with a provider + // context-overflow error instead of an answer. "Run a READ-ONLY rocm CLI subcommand and return its output. Allowed: \ - model/models, config show, runtimes (list), logs, daemon status. Pass the \ - argv as `args`, e.g. [\"model\"] or [\"config\",\"show\"]. Mutating \ - subcommands are rejected.", + config show, runtimes (list), logs, daemon status, model/models. Pass the \ + argv as `args`, e.g. [\"config\",\"show\"] or [\"runtimes\",\"list\"]. \ + [\"model\"] lists the recommended models and [\"model\",\"--verbose\"] \ + adds the per-recipe diagnostics, which are long. Mutating subcommands are \ + rejected.", { "type": "object", "properties": { "args": { "type": "array", "items": { "type": "string" }, - "description": "Argv for the rocm subcommand, e.g. [\"model\"]." + "description": "Argv for the rocm subcommand, e.g. [\"config\",\"show\"]." } }, "required": ["args"] @@ -908,6 +1122,201 @@ mod tests { ); } + /// A command-backed MCP envelope of roughly `text_chars` characters, + /// shaped exactly as the bin's `internal_mcp_tool_result_from_command` + /// builds one: the command output in `content[0].text` and again in + /// `structuredContent.stdout`. + fn command_envelope(text_chars: usize) -> (String, serde_json::Value) { + let line = "Qwen/Qwen3.5-4B Q4_K_M fits this GPU\n"; + let stdout = line.repeat(text_chars.div_ceil(line.len())); + let envelope = json!({ + "content": [{ "type": "text", "text": format!("Ran `rocm` command.\n\n{stdout}") }], + "structuredContent": { + "argv": ["rocm", "model"], + "exit_status": 0, + "stdout": stdout, + "stderr": "", + }, + "isError": false, + }); + (stdout, envelope) + } + + /// The `[output truncated: A of B characters shown, C omitted` figures. + fn truncation_figures(text: &str) -> (usize, usize, usize) { + let marker = text + .split_once("[output truncated: ") + .unwrap_or_else(|| panic!("no truncation marker in:\n{text}")) + .1; + let numbers = marker + .split(|c: char| !c.is_ascii_digit()) + .filter(|part| !part.is_empty()) + .take(3) + .map(|part| part.parse::().expect("marker figure")) + .collect::>(); + (numbers[0], numbers[1], numbers[2]) + } + + #[tokio::test] + async fn an_oversized_read_only_tool_result_is_capped_before_the_model_sees_it() { + // A ~200 KB result is what `rocm_command` with `["model"]` used to + // return (204,470 characters measured on a developer host). Sent whole + // to a locally served model it exceeds the entire context and the + // provider refuses the request, so the user gets an HTTP 400 instead + // of an answer. + let (_, envelope) = command_envelope(200 * 1024); + assert!( + envelope.to_string().len() > 200 * 1024, + "the fixture must be bigger than the budget or it proves nothing" + ); + let tool = RocmCommandRocmTool { + executor: Some(Arc::new(FakeExec(envelope)) as SharedRocmToolExecutor), + fired: Arc::new(Mutex::new(Vec::new())), + }; + + let out = tool.call(json!({ "args": ["model"] })).await.expect("ok"); + + // What the model is handed is the serialized value, so that is what + // the budget is measured against. + let rendered = out.to_string(); + assert!( + rendered.len() <= TOOL_RESULT_CHAR_BUDGET, + "a tool result of {} characters reached the model; the budget is {TOOL_RESULT_CHAR_BUDGET}", + rendered.len() + ); + // It says it is partial, rather than reading as a complete answer. + assert_eq!(out["truncated"], json!(true), "{out}"); + let text = out["tool_output"].as_str().expect("tool_output text"); + assert!( + text.contains("[output truncated:") && text.trim_end().ends_with(']'), + "the marker must close the result so the model cannot miss it: {text}" + ); + // …and says how much is missing, consistently. + let (shown, total, omitted) = truncation_figures(text); + assert_eq!(shown + omitted, total, "the marker's figures must add up"); + assert!( + omitted > total / 2, + "a 200 KB result cut to {TOOL_RESULT_CHAR_BUDGET} characters must report most of it omitted, got {omitted} of {total}" + ); + // The part that is shown is real output, cut on a line boundary. + assert!(text.contains("Qwen/Qwen3.5-4B"), "{text}"); + assert!( + text.split_once("\n[output truncated:") + .is_some_and(|(kept, _)| kept.is_empty() || kept.ends_with('\n')), + "the cut must land on a line boundary: {text}" + ); + } + + #[tokio::test] + async fn truncating_a_failed_command_keeps_it_a_failure() { + // The capped form is a fresh object, so `isError` survives only if it + // is carried over. A command that failed with a lot of output would + // otherwise reach the model looking like a success, and the chat path + // reads that field to decide exactly that. + let (_, mut envelope) = command_envelope(200 * 1024); + envelope["isError"] = json!(true); + envelope["structuredContent"]["exit_status"] = json!(1); + let tool = RocmCommandRocmTool { + executor: Some(Arc::new(FakeExec(envelope)) as SharedRocmToolExecutor), + fired: Arc::new(Mutex::new(Vec::new())), + }; + + let out = tool.call(json!({ "args": ["model"] })).await.expect("ok"); + + assert_eq!( + out["truncated"], + json!(true), + "the fixture must truncate: {out}" + ); + assert_eq!( + out["isError"], + json!(true), + "a failed command must still read as failed after truncation: {out}" + ); + } + + #[tokio::test] + async fn truncating_a_successful_command_keeps_it_a_success() { + // The mirror of the case above: carrying `isError` over must not + // invent a failure for output that was merely long. + let (_, envelope) = command_envelope(200 * 1024); + let tool = RocmCommandRocmTool { + executor: Some(Arc::new(FakeExec(envelope)) as SharedRocmToolExecutor), + fired: Arc::new(Mutex::new(Vec::new())), + }; + + let out = tool.call(json!({ "args": ["model"] })).await.expect("ok"); + + assert_eq!( + out["truncated"], + json!(true), + "the fixture must truncate: {out}" + ); + assert_eq!(out["isError"], json!(false), "{out}"); + } + + #[tokio::test] + async fn a_command_backed_result_reaches_the_model_once_not_twice() { + // The bin's envelope carries the command output twice — once as the + // text a reader sees, once as `structuredContent.stdout` — because + // `rocm mcp-call` consumers read the structured field. The dash hands + // the whole value to the model, so without this the model pays for + // every byte twice. + let (stdout, envelope) = command_envelope(2 * 1024); + let before = envelope.to_string().len(); + let tool = RocmCommandRocmTool { + executor: Some(Arc::new(FakeExec(envelope)) as SharedRocmToolExecutor), + fired: Arc::new(Mutex::new(Vec::new())), + }; + + let out = tool + .call(json!({ "args": ["config", "show"] })) + .await + .expect("ok"); + + // One copy survives, and it is the readable one. + let text = out["content"][0]["text"].as_str().expect("content text"); + assert!(text.contains(stdout.trim()), "{out}"); + assert!( + out["structuredContent"].get("stdout").is_none(), + "the duplicated copy must not reach the model: {out}" + ); + assert!( + out["structuredContent"].get("stderr").is_none(), + "an empty stderr is pure overhead for the model: {out}" + ); + // The fields that are not duplicates stay, so the model can still see + // what ran and whether it worked. + assert_eq!(out["structuredContent"]["exit_status"], json!(0), "{out}"); + assert_eq!(out["structuredContent"]["argv"][1], json!("model"), "{out}"); + assert!( + out.to_string().len() < before / 2 + 512, + "dropping the duplicate should roughly halve the result: {} vs {before}", + out.to_string().len() + ); + } + + #[tokio::test] + async fn a_result_within_the_budget_keeps_its_exact_shape() { + // The cap must not reshape ordinary results: a tool whose data lives + // only in `structuredContent` (engines, bridge_snapshot, automations) + // would lose it if every result were flattened to text. + let value = json!({ + "content": [{ "type": "text", "text": "Found 3 engine entries." }], + "structuredContent": { "engines": [{ "id": "lemonade", "available": true }] }, + "isError": false, + }); + let tool = EnginesRocmTool { + executor: Some(Arc::new(FakeExec(value.clone())) as SharedRocmToolExecutor), + fired: Arc::new(Mutex::new(Vec::new())), + }; + let out = tool.call(json!({})).await.expect("ok"); + assert_eq!( + out, value, + "an in-budget result must pass through unchanged" + ); + } + /// Recording executor for the mutating-tool surfacing test: `execute` /// returns `ApprovalRequired`; `execute_approved` records a call (which must /// NOT happen during the rig tool loop). diff --git a/crates/rocm-dash-tui/src/app/chat_runtime.rs b/crates/rocm-dash-tui/src/app/chat_runtime.rs index 2b4d8cd019..002b7f0e99 100644 --- a/crates/rocm-dash-tui/src/app/chat_runtime.rs +++ b/crates/rocm-dash-tui/src/app/chat_runtime.rs @@ -11,6 +11,10 @@ //! one `drain_*` method per one-shot edge on `AppState` (`provider_switch`, //! `chat_endpoint_rebuild`, `chat_dispatch`) consumes that edge. The detect and //! persist edges do not touch the backend and stay in `event_loop`. +//! +//! [`chat_error_message`] sits on the dispatch edge: it turns a failed round +//! trip into the line the transcript shows, rewriting a provider context +//! overflow into plain language and leaving every other error as it is. use tokio::sync::mpsc; @@ -27,6 +31,82 @@ use super::{AppState, ProviderSwitch}; type SharedAgent = std::sync::Arc; +/// Turn a backend error into the line the Chat tab shows. +/// +/// Most errors are already readable and pass through unchanged. The one that +/// is not is a context overflow: the provider answers HTTP 400 with its own +/// JSON body, and the transcript used to show the whole thing — status line, +/// error type, token counts and all — which tells a user nothing about what +/// happened or what to do next. That one case is rewritten into a sentence +/// that names the cause and a way forward, and the raw body is logged so the +/// detail is still available in `rocm-cli.log`. +fn chat_error_message(raw: &str) -> String { + let Some(message) = context_overflow_message(raw) else { + return raw.to_owned(); + }; + // Only on the rewritten path: an error shown verbatim is already logged + // by the backend, and this is the one case where what the user sees no + // longer contains what a maintainer would need. + tracing::warn!(error = %raw, "chat request exceeded the model's context"); + message +} + +/// The plain-language form of a provider context-overflow error, or `None` +/// when `raw` is some other failure. +/// +/// Matches on the provider's own error type rather than on the HTTP status: +/// a 400 covers many unrelated mistakes, and only this one has a cause the +/// user can act on. The token counts come out of the same body; a body that +/// omits them still gets the sentence, without the numbers. +fn context_overflow_message(raw: &str) -> Option { + if !raw.contains("exceed_context_size_error") { + return None; + } + let counts = json_number_field(raw, "n_prompt_tokens") + .zip(json_number_field(raw, "n_ctx")) + .map(|(used, available)| { + format!( + " ({} of {} tokens)", + group_thousands(used), + group_thousands(available) + ) + }) + .unwrap_or_default(); + Some(format!( + "A ROCm check returned more text than this model can read{counts}. \ + Try a more specific question, or a model with a larger context." + )) +} + +/// Read `"": ` out of a JSON fragment embedded in a longer +/// message. Deliberately not a full parse: the body arrives wrapped in the +/// backend's own error prose, so there is no single value to deserialize. +fn json_number_field(raw: &str, field: &str) -> Option { + let needle = format!("\"{field}\""); + let after = raw.split_once(&needle)?.1; + let after = after.trim_start().strip_prefix(':')?.trim_start(); + let digits = after + .split(|c: char| !c.is_ascii_digit()) + .next() + .filter(|digits| !digits.is_empty())?; + digits.parse().ok() +} + +/// `53820` → `53,820`. The counts are the point of the message, and an +/// ungrouped five-digit number next to another one is hard to compare at a +/// glance. +fn group_thousands(value: u64) -> String { + let digits = value.to_string(); + let mut out = String::with_capacity(digits.len() + digits.len() / 3); + for (index, ch) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + out.push(','); + } + out.push(ch); + } + out +} + pub(super) struct ChatRuntime { /// The backend the next submit is routed to; `None` when none could be /// built, in which case a submit surfaces an error turn. @@ -345,7 +425,7 @@ impl ChatRuntime { let msg = match agent.complete(&history, snapshot).await { Ok(text) => ClientMsg::ChatReply { text }, Err(e) => ClientMsg::ChatError { - message: e.to_string(), + message: chat_error_message(&e.to_string()), }, }; let _ = reply_tx.send(msg); @@ -710,6 +790,98 @@ mod tests { } } + // --- error messages ------------------------------------------------------ + + /// The body a local llama.cpp-style server answers a too-large request + /// with, wrapped in the backend prose the Chat tab used to show verbatim. + const CONTEXT_OVERFLOW_ERROR: &str = "chat request failed: CompletionError: HttpError: \ + Invalid status code 400 Bad Request with message: \ + {\"error\":{\"code\":400,\"message\":\"request (53820 tokens) exceeds the available \ + context size (32768 tokens), try increasing it\",\ + \"type\":\"exceed_context_size_error\",\"n_prompt_tokens\":53820,\"n_ctx\":32768}}"; + + /// In-memory `tracing` writer so a test can read back what was logged. + #[derive(Clone, Default)] + struct LogCapture(Arc>>); + impl std::io::Write for LogCapture { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + impl tracing_subscriber::fmt::MakeWriter<'_> for LogCapture { + type Writer = Self; + fn make_writer(&self) -> Self::Writer { + self.clone() + } + } + + #[test] + fn a_context_overflow_reads_as_plain_language_and_keeps_the_raw_body_in_the_log() { + let logs = LogCapture::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_ansi(false) + .finish(); + let guard = tracing::subscriber::set_default(subscriber); + let message = chat_error_message(CONTEXT_OVERFLOW_ERROR); + drop(guard); + + // What the user reads says what happened and what to do about it… + assert_eq!( + message, + "A ROCm check returned more text than this model can read \ + (53,820 of 32,768 tokens). Try a more specific question, or a model \ + with a larger context." + ); + // …and none of the provider's wire detail. + for noise in ["HttpError", "400", "exceed_context_size_error", "n_ctx"] { + assert!( + !message.contains(noise), + "`{noise}` leaked into the chat message: {message}" + ); + } + // The detail is not lost: it is in the log a maintainer reads. + let logged = String::from_utf8(logs.0.lock().unwrap().clone()).expect("utf-8 log"); + assert!( + logged.contains("exceed_context_size_error") && logged.contains("n_prompt_tokens"), + "the raw provider body must still reach rocm-cli.log, got: {logged}" + ); + } + + #[test] + fn a_context_overflow_without_token_counts_still_reads_as_plain_language() { + let message = chat_error_message( + "chat request failed: HttpError: 400 {\"error\":{\"type\":\"exceed_context_size_error\"}}", + ); + assert!( + message.starts_with("A ROCm check returned more text"), + "{message}" + ); + assert!( + !message.contains('('), + "with no counts in the body the sentence carries no numbers: {message}" + ); + } + + #[test] + fn every_other_backend_error_is_left_alone() { + // Only the one error with an actionable cause is rewritten; rewriting + // the rest would hide detail the user needs to report a real fault. + for raw in [ + "chat request failed: connection refused", + "request timed out after 120s", + "failed to build chat client: no endpoint configured", + // A 400 that is not a context overflow keeps its own text. + "chat request failed: HttpError: Invalid status code 400 Bad Request", + ] { + assert_eq!(chat_error_message(raw), raw); + } + } + #[tokio::test] async fn a_submit_with_no_backend_surfaces_the_no_backend_error_turn() { let (tx, mut rx) = channel(); diff --git a/crates/rocm-dash-tui/src/app/event_loop.rs b/crates/rocm-dash-tui/src/app/event_loop.rs index 8709b4a53e..8738a2cf40 100644 --- a/crates/rocm-dash-tui/src/app/event_loop.rs +++ b/crates/rocm-dash-tui/src/app/event_loop.rs @@ -67,6 +67,62 @@ fn seed_registry_probe_and_hosting( state.launcher_hosted = launcher_hosted; } +/// Whether the dashboard polls the managed-service registry for its instance +/// list. Only Windows has no daemon to send instances; a focused host draws no +/// instance list, and `--demo`/`--replay` (`replay`) must never show this +/// host's real services. `windows` is a parameter rather than a `cfg!` so the +/// gate itself is unit-tested on every platform. +const fn polls_managed_registry(focus: Option, replay: bool, windows: bool) -> bool { + windows && !replay && !should_skip_daemon(focus) +} + +/// Logged when a registry read panics under `spawn_blocking`. Named so the +/// test asserting the warning fires cannot pass against a message that was +/// silently reworded or downgraded below WARN. +const MANAGED_REGISTRY_PROBE_PANIC: &str = + "managed-service registry probe panicked; instance list not updated this poll"; + +/// Read the managed-service registry every `period` and post the live +/// services as `ClientMsg::ManagedInstances`. The probe does a blocking disk +/// read, so it runs under `spawn_blocking` rather than on the event loop — a +/// slow or antivirus-scanned services directory then delays only the next +/// poll, never a frame. The first poll is immediate, so a running service is +/// listed as soon as the dashboard opens. Ends when the receiver is dropped. +fn spawn_managed_registry_poller( + probe: crate::registry_probe::SharedManagedRegistryProbe, + tx: mpsc::UnboundedSender, + period: Duration, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = interval(period); + // A slow read must not queue a burst of catch-up polls behind it. + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + ticker.tick().await; + let probe = probe.clone(); + let snapshot = match tokio::task::spawn_blocking(move || probe.snapshot()).await { + Ok(snapshot) => snapshot, + Err(e) => { + // The probe panicked; keep polling rather than freezing the + // list — but say so. A probe that panics every tick would + // otherwise leave the instance list silently empty for the + // whole session with nothing in the log to explain it. + // Same shape as `client.rs`'s per-tick Windows + // perf-counter failure. + tracing::warn!(error = %e, "{MANAGED_REGISTRY_PROBE_PANIC}"); + continue; + } + }; + if tx + .send(ClientMsg::ManagedInstances(snapshot.instances)) + .is_err() + { + break; + } + } + }) +} + pub async fn run(args: ResolvedArgs) -> color_eyre::Result<()> { // Install the termination-signal watcher BEFORE switching the terminal into // raw/alternate-screen mode. A signal that arrives during startup must find @@ -234,6 +290,8 @@ async fn event_loop(terminal: &mut Tui, args: &ResolvedArgs) -> color_eyre::Resu args.managed_registry_probe.clone(), args.launcher_hosted, ); + // Managed Lemonade install inventory, captured by the bin at launch. + state.lemonade_managed_installed = args.lemonade_managed_installed; // Focused host: open exactly the overlay for the requested flow (Examine // also auto-runs its read-only job). `Focus::Setup` opens the onboarding // overlay — the same wizard `rocm bootstrap setup` routes to. @@ -243,6 +301,11 @@ async fn event_loop(terminal: &mut Tui, args: &ResolvedArgs) -> color_eyre::Resu // Both `--demo` (a generated session replayed) and `--replay ` present // non-live data, so mark the session simulated for the honesty chrome. state.simulated = state.replay.is_some(); + if polls_managed_registry(args.focus, state.simulated, cfg!(windows)) + && let Some(probe) = args.managed_registry_probe.clone() + { + spawn_managed_registry_poller(probe, chat_tx.clone(), Duration::from_secs(1)); + } let mut chat = ChatRuntime::start(&mut state, args, &chat_tx).await; @@ -273,6 +336,7 @@ async fn event_loop(terminal: &mut Tui, args: &ResolvedArgs) -> color_eyre::Resu } Some(ClientMsg::Event(ev)) => state.apply_event(*ev), Some(ClientMsg::HostGpuSnapshot(snap)) => state.push_host_gpu_snapshot(*snap), + Some(ClientMsg::ManagedInstances(live)) => state.set_managed_instances(live), Some(ClientMsg::ReplaySeek) => state.reset_for_seek(), Some(ClientMsg::ReplayPosition { elapsed_s, total_s }) => { if let Some(r) = state.replay.as_mut() { @@ -978,6 +1042,7 @@ mod tests { std::sync::Arc::new(StaticManagedRegistrySnapshot(ManagedRegistrySnapshot { has_live_service: true, managed_ids: std::iter::once("svc-a".to_string()).collect(), + ..Default::default() })); seed_registry_probe_and_hosting(&mut s, Some(probe.clone()), true); @@ -989,4 +1054,266 @@ mod tests { Some(true) ); } + + #[test] + fn polls_managed_registry_state_matrix() { + // The only caller is `event_loop`, which only the Linux e2e lane + // drives, and the Windows arm is the one that matters. `windows` is + // injected so every row runs on every platform's CI job. + assert!( + polls_managed_registry(None, false, true), + "Windows dashboard polls" + ); + assert!( + !polls_managed_registry(None, false, false), + "Linux has a daemon" + ); + assert!( + !polls_managed_registry(None, true, true), + "--demo/--replay must not show this host's real services" + ); + for focus in [Focus::Setup, Focus::Serve, Focus::Examine] { + assert!( + !polls_managed_registry(Some(focus), false, true), + "a focused host draws no instance list: {focus:?}" + ); + } + } + + fn probe_with( + ids: &[&str], + ) -> ( + crate::registry_probe::SharedManagedRegistryProbe, + Vec, + ) { + use crate::registry_probe::{ManagedRegistrySnapshot, StaticManagedRegistrySnapshot}; + let instances: Vec<_> = ids + .iter() + .map(|id| rocm_dash_core::metrics::Instance { + container_id: (*id).to_string(), + status: rocm_dash_core::metrics::InstanceStatus::Ready, + port: Some(11_400), + ..Default::default() + }) + .collect(); + let probe = std::sync::Arc::new(StaticManagedRegistrySnapshot(ManagedRegistrySnapshot { + instances: instances.clone(), + ..Default::default() + })); + (probe, instances) + } + + #[tokio::test] + async fn managed_registry_poller_posts_the_registry_list_immediately_and_repeatedly() { + // The first poll must not wait a full period: a service already + // running when the dashboard opens is listed at once, not after + // "Nothing running" has been on screen for a second. + // + // The period is deliberately much longer than the first read's deadline + // (1s vs 250ms), because the deadline only discriminates if one period + // does not fit inside it: with a period as short as the bound, a poller + // that slept a full period before its first read would clear it anyway + // and the test would pass on the very bug it exists to catch. + const PERIOD: Duration = Duration::from_secs(1); + const FIRST_READ_DEADLINE: Duration = Duration::from_millis(250); + + let (probe, expected) = probe_with(&["svc-a"]); + let (tx, mut rx) = mpsc::unbounded_channel(); + let handle = spawn_managed_registry_poller(probe, tx, PERIOD); + + // Poll 0 is the immediate one and gets the tight bound; poll 1 only has + // to prove the loop repeats, so it is allowed a full period plus slack. + for poll in 0..2 { + let deadline = if poll == 0 { + FIRST_READ_DEADLINE + } else { + PERIOD + FIRST_READ_DEADLINE + }; + let msg = tokio::time::timeout(deadline, rx.recv()) + .await + .unwrap_or_else(|_| { + panic!("poll {poll} did not arrive within {deadline:?} (period {PERIOD:?})") + }) + .expect("channel open"); + let ClientMsg::ManagedInstances(got) = msg else { + panic!("unexpected message: {msg:?}"); + }; + assert_eq!(got.len(), expected.len()); + assert_eq!(got[0].container_id, "svc-a"); + } + + drop(rx); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("the poller stops once the receiver is gone") + .expect("poller did not panic"); + } + + #[tokio::test] + async fn managed_registry_poller_survives_a_panicking_probe() { + // A probe panic (under `spawn_blocking`) must not end the poll loop and + // freeze the instance list for the rest of the session. + #[derive(Debug, Default)] + struct PanicsOnce(std::sync::atomic::AtomicBool); + impl crate::registry_probe::ManagedRegistryProbe for PanicsOnce { + fn snapshot(&self) -> crate::registry_probe::ManagedRegistrySnapshot { + assert!( + self.0.swap(true, std::sync::atomic::Ordering::SeqCst), + "first read fails" + ); + crate::registry_probe::ManagedRegistrySnapshot::default() + } + } + + // This test reaches the poller's `warn!` callsite too, so it must warm + // the interest cache for the same reason + // `managed_registry_poller_warns_when_the_probe_panics` does — see + // `warm_tracing_interest_cache`. Without the call here, whichever of + // the two tests registers that callsite first decides whether it is + // ever enabled, and this one has no subscriber of its own. + warm_tracing_interest_cache(); + + let (tx, mut rx) = mpsc::unbounded_channel(); + let handle = spawn_managed_registry_poller( + std::sync::Arc::new(PanicsOnce::default()), + tx, + Duration::from_millis(20), + ); + + let msg = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("a later poll arrived") + .expect("channel open"); + assert!(matches!(msg, ClientMsg::ManagedInstances(_)), "{msg:?}"); + handle.abort(); + } + + /// Minimal `tracing::Subscriber` recording each event's level and message + /// text, so a test can assert that a SPECIFIC diagnostic fired — not + /// merely that something was logged. Counting events alone would stay + /// green if the `warn!` were downgraded to `debug!` or its text removed. + /// `agent/clients.rs`'s `CapturedWarnEvents` does the same for that + /// module's warnings; it is private to that test module, so this is the + /// local equivalent rather than a shared helper. + #[derive(Default)] + struct CapturedWarnings(std::sync::Mutex>); + + impl CapturedWarnings { + fn warned(&self, message: &str) -> bool { + self.0 + .lock() + .unwrap() + .iter() + .any(|(level, text)| *level == tracing::Level::WARN && text == message) + } + } + + impl tracing::Subscriber for CapturedWarnings { + fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + + fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {} + + fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {} + + fn event(&self, event: &tracing::Event<'_>) { + struct MessageVisitor(Option); + impl tracing::field::Visit for MessageVisitor { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + // tracing wraps the message in an internal type whose + // `Debug` forwards to `Display`, so `{value:?}` is the + // message text itself, unquoted. + if field.name() == "message" { + self.0 = Some(format!("{value:?}")); + } + } + } + let mut visitor = MessageVisitor(None); + event.record(&mut visitor); + if let Some(message) = visitor.0 { + self.0 + .lock() + .unwrap() + .push((*event.metadata().level(), message)); + } + } + + fn enter(&self, _span: &tracing::span::Id) {} + + fn exit(&self, _span: &tracing::span::Id) {} + } + + /// Register one always-interested process-wide default subscriber before + /// any test can reach the poller's `warn!` callsite. + /// + /// tracing-core caches a callsite's `Interest` the first time that + /// callsite executes, by asking the dispatcher in scope on whichever + /// thread happens to get there first. Under threaded `cargo test` that is + /// a race between this module's two panicking-probe tests, and the other + /// one carries no subscriber — so without this, "never interested" can be + /// cached before the capturing test ever installs its own, and the + /// assertion below fails for a reason that has nothing to do with the + /// production code. A permanent global default whose `enabled()` is + /// always `true` makes "nobody is interested yet" impossible from the + /// first registration onward. Its own buffer is never read; only the + /// thread-local override installed in the test is. Same remedy, and same + /// reasoning, as `agent/clients.rs`'s `warm_tracing_interest_cache`. + fn warm_tracing_interest_cache() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + let _ = tracing::subscriber::set_global_default(CapturedWarnings::default()); + }); + } + + #[tokio::test] + async fn managed_registry_poller_warns_when_the_probe_panics() { + // Surviving the panic (above) is not enough: a probe that panics on + // every tick degrades this feature for a whole session, and without a + // log line the operator sees only a permanently empty instance list + // with nothing anywhere to explain it. + #[derive(Debug)] + struct AlwaysPanics; + impl crate::registry_probe::ManagedRegistryProbe for AlwaysPanics { + fn snapshot(&self) -> crate::registry_probe::ManagedRegistrySnapshot { + panic!("registry read blew up"); + } + } + + // Thread-local, not global: `#[tokio::test]` runs a current-thread + // runtime, so the poller task is polled on this very thread and the + // `warn!` dispatches through this subscriber. The global default the + // warming installs is only there to keep the callsite's cached + // `Interest` from being decided by a subscriber-less thread. + warm_tracing_interest_cache(); + let captured = std::sync::Arc::new(CapturedWarnings::default()); + let guard = tracing::subscriber::set_default(captured.clone()); + + let (tx, _rx) = mpsc::unbounded_channel(); + let handle = spawn_managed_registry_poller( + std::sync::Arc::new(AlwaysPanics), + tx, + Duration::from_millis(20), + ); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while !captured.warned(MANAGED_REGISTRY_PROBE_PANIC) { + assert!( + tokio::time::Instant::now() < deadline, + "a panicking probe must log {MANAGED_REGISTRY_PROBE_PANIC:?} at WARN, not fail silently" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + handle.abort(); + drop(guard); + } } diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index 9bbc7ef147..599d6bfe8c 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -81,7 +81,16 @@ pub use types::{ pub(crate) use types::{ChatProvider, PendingApproval, SlashOutcome, SlashToolRequest}; /// How many snapshots to keep for sparklines. -pub const HISTORY_CAP: usize = 240; +/// +/// The braille sparkline packs two samples into each character column, so this +/// bounds how much of a panel the graph can ever cover: 600 samples fill an +/// inner width of 300 columns. That covers the half-width CPU box on a very +/// wide terminal and a full-width stacked panel up to about 305 columns; past +/// that the graph still stops short of the left edge. At 240 it topped out at +/// 120 columns, which a full-width panel exceeds on any ordinary terminal. +/// Snapshots arrive about once a second, so this is also roughly 10 minutes of +/// history before the oldest samples start falling off the left. +pub const HISTORY_CAP: usize = 600; /// How many benchmark rows to keep client-side for the bench panel. pub const BENCH_CAP: usize = 200; @@ -240,6 +249,8 @@ pub struct AppState { /// that actually exits the process. See the field doc on /// `ResolvedArgs::launcher_hosted` and `actions::request_quit`. pub(crate) launcher_hosted: bool, + /// Managed installation inventory captured at launch; not endpoint liveness. + pub lemonade_managed_installed: Option, /// Last body area used by the most recent draw. Mouse hit-tests resolve /// pointer coordinates against this rect (filled by `ui::draw`). pub last_body_area: Option, @@ -407,6 +418,7 @@ impl AppState { services_past_attempts: 0, managed_registry_probe: None, launcher_hosted: false, + lemonade_managed_installed: None, last_body_area: None, last_tab_bar_area: None, last_header_badge: None, @@ -599,14 +611,34 @@ impl AppState { /// charts/history exactly like a real snapshot, but deliberately leaves /// `instances` and `has_received_snapshot` untouched. The Windows /// in-process ticker (`client.rs`) always sends an empty `instances` - /// stand-in (no instance-registry daemon exists there yet), so treating - /// it as authoritative the way `push_snapshot` does would permanently - /// blind the issue #145 quit-confirm prompt to models actually being - /// served — see `has_live_instance_with`'s Windows note. + /// stand-in (it has no daemon to discover them), so treating it as + /// authoritative the way `push_snapshot` does would blind the issue #145 + /// quit-confirm prompt to models actually being served — see + /// `has_live_instance_with`'s Windows note. Instances arrive separately, + /// from the managed-service registry, through + /// [`set_managed_instances`](Self::set_managed_instances). pub(crate) fn push_host_gpu_snapshot(&mut self, snap: Snapshot) { self.push_history_and_latest(snap); } + /// Replace `instances` with the managed services the registry poller + /// (`event_loop::spawn_managed_registry_poller`, Windows only) last read, + /// delivered as `ClientMsg::ManagedInstances`. `has_received_snapshot` + /// stays untouched, so the issue #145 quit-confirm gate keeps reading the + /// registry live too. + /// + /// The map is refilled in place rather than rebuilt: a fresh `HashMap` + /// gets a fresh random hasher, so the iteration order — which picks the + /// Home tile's model — would change on every poll. + pub(crate) fn set_managed_instances(&mut self, live: Vec) { + self.instances.clear(); + self.instances.extend( + live.into_iter() + .map(|inst| (inst.container_id.clone(), inst)), + ); + self.clamp_selectors(); + } + /// Shared tail of [`push_snapshot`](Self::push_snapshot) and /// [`push_host_gpu_snapshot`](Self::push_host_gpu_snapshot): the /// history/latest bookkeeping both share, independent of how (or @@ -723,6 +755,111 @@ mod tests { assert_eq!(s.history.len(), 1); } + /// Windows has no daemon to discover instances, so the registry poller's + /// list is what fills `instances`: a served model appears when the poll + /// reports it and drops off once a later poll no longer does. + #[test] + fn set_managed_instances_follows_the_polled_registry_list() { + use crate::registry_probe::{ManagedRegistrySnapshot, StaticManagedRegistrySnapshot}; + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + + // `managed: true` is not decoration. The poller builds these through + // `merge_instance`, which copies `managed` from the registry record, + // and `discover_managed_services` only ever emits `managed: true`, so + // a polled instance is always managed. Hand-building it with + // `..Default::default()` alone would leave the field `false` — a shape + // the poller cannot produce — and `has_stoppable_managed_instance_with` + // screens on exactly that field (`is_listed_service`), because the + // Services overlay `s` jumps to lists only managed rows. + let served = Instance { + container_id: "lemonade-qwen".to_string(), + model_name: "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M".to_string(), + status: InstanceStatus::Ready, + port: Some(11436), + managed: true, + ..Default::default() + }; + let mut s = st(); + s.managed_registry_probe = Some(std::sync::Arc::new(StaticManagedRegistrySnapshot( + ManagedRegistrySnapshot { + has_live_service: true, + managed_ids: std::iter::once(served.container_id.clone()).collect(), + instances: vec![served.clone()], + }, + ))); + + s.set_managed_instances(vec![served.clone()]); + + assert_eq!(s.instances.len(), 1); + let inst = &s.instances["lemonade-qwen"]; + assert_eq!(inst.model_name, served.model_name); + assert_eq!(inst.port, Some(11436)); + assert!( + !s.has_received_snapshot, + "the registry poll must not flip has_received_snapshot" + ); + let snap = s.live_registry_snapshot(); + assert!(s.has_live_instance_with(&snap)); + assert!( + s.has_stoppable_managed_instance_with(&snap), + "a registry-sourced instance must be offered for stopping on quit" + ); + + // Shrink the list out from under a selection that points past the new + // end. `instance_sel` has to be nonzero first, or the clamp inside + // `set_managed_instances` is unobservable — with the selector already + // at its default 0 the call could be dropped entirely and every + // assertion below would still hold. + let second = Instance { + container_id: "lemonade-llama".to_string(), + status: InstanceStatus::Ready, + port: Some(11437), + ..Default::default() + }; + s.set_managed_instances(vec![served, second]); + s.instance_sel = 1; + + s.set_managed_instances(Vec::new()); + + assert!( + s.instances.is_empty(), + "a service the registry no longer reports must drop off" + ); + assert_eq!( + s.instance_sel, 0, + "the row selection must be clamped to the shrunken list, not left \ + pointing past its end" + ); + } + + /// The Home tile shows the first serving instance in `instances`' + /// iteration order, so applying the same registry contents poll after + /// poll must not reorder them — otherwise the tile alternates models. + #[test] + fn set_managed_instances_keeps_instance_order_stable_across_polls() { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + + let polled = || -> Vec { + (0..8) + .map(|n| Instance { + container_id: format!("svc-{n}"), + status: InstanceStatus::Ready, + port: Some(11_400 + n), + ..Default::default() + }) + .collect() + }; + let mut s = st(); + + s.set_managed_instances(polled()); + let first: Vec = s.instances.keys().cloned().collect(); + for _ in 0..5 { + s.set_managed_instances(polled()); + let again: Vec = s.instances.keys().cloned().collect(); + assert_eq!(again, first, "instance order changed between polls"); + } + } + #[test] fn actions_does_not_import_from_scrollbar() { // scrollbar.rs depends on actions.rs (KeyAction/handle_mouse/tab_bar_hit, diff --git a/crates/rocm-dash-tui/src/app/quit_confirm.rs b/crates/rocm-dash-tui/src/app/quit_confirm.rs index c0c6f7096e..053d28542f 100644 --- a/crates/rocm-dash-tui/src/app/quit_confirm.rs +++ b/crates/rocm-dash-tui/src/app/quit_confirm.rs @@ -38,7 +38,9 @@ impl AppState { /// there either — the live read stays authoritative for the entire /// session rather than only a brief startup window — the registry read is /// cheap and re-run every time this is called, so there is no staleness - /// to outlive. Once a snapshot lands + /// to outlive. (`instances` itself is refilled from the same registry by + /// a background poller once a second — `set_managed_instances` — so it + /// can trail this live read by about a second.) Once a snapshot lands /// on a host where the daemon does connect, `instances` is authoritative /// and the read's *result* is no longer consulted for THIS question (a /// model that legitimately stops later in the session is still reflected @@ -104,13 +106,25 @@ impl AppState { /// call sites also ask a sibling registry-backed question in the same /// decision and need the two answered from one read, not two independent /// ones a registry write in between could answer differently. + /// + /// Both halves of ownership must hold: the live registry read names the + /// instance, and the daemon's broadcast already marks it `managed`, because + /// the Services overlay lists only broadcast-managed rows + /// ([`is_listed_service`](crate::ui::services_manager::is_listed_service)). + /// An instance started a moment ago can be on disk before the next broadcast + /// says so; it is not offered here, because `s` could not focus a row for it. + /// The quit prompt still prints `rocm services stop --yes` for it (that + /// line keys off the registry alone), and the offer appears on the broadcast + /// that catches up. pub(crate) fn has_stoppable_managed_instance_with( &self, snapshot: &crate::registry_probe::ManagedRegistrySnapshot, ) -> bool { - self.instances - .values() - .any(|i| i.status.is_serving() && snapshot.managed_ids.contains(&i.container_id)) + self.instances.values().any(|i| { + crate::ui::services_manager::is_listed_service(i) + && i.status.is_serving() + && snapshot.managed_ids.contains(&i.container_id) + }) } /// Whether the confirm-before-quit prompt is open. Mirrors @@ -194,13 +208,15 @@ impl AppState { /// that normal resolution instead. /// /// The short-circuit above calls - /// [`has_stoppable_managed_instance_with`](Self::has_stoppable_managed_instance_with) - /// on exactly the same predicate the row-finding `position` closure below - /// re-checks per row (`is_serving() && managed_ids.contains(..)`), so it - /// cannot change which outcome this returns — only skip it sooner. Its - /// value is avoiding `service_rows`'s allocation and sort on every `s` - /// keypress when nothing is stoppable; it is not a correctness guard, and - /// there is nothing for the two checks to drift on. + /// [`has_stoppable_managed_instance_with`](Self::has_stoppable_managed_instance_with), + /// which applies the same ownership test `service_rows` lists by + /// (`is_listed_service`) plus the per-row check the `position` closure below + /// repeats (`is_serving() && managed_ids.contains(..)`). Because `service_rows` + /// lists only broadcast-managed instances, a predicate without that test would + /// say "stoppable" for an instance the rows do not contain and the key would + /// be dropped silently, so the two must keep agreeing; the test + /// `a_registry_managed_instance_the_broadcast_does_not_yet_mark_managed_is_not_offered` + /// pins that. pub(crate) fn quit_confirm_open_services_key( &mut self, code: crossterm::event::KeyCode, @@ -369,6 +385,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status, ..Default::default() }, @@ -400,6 +417,7 @@ mod tests { "svc-a".to_string(), Instance { container_id: "svc-a".to_string(), + managed: true, status: InstanceStatus::Stopped, ..Default::default() }, @@ -408,6 +426,7 @@ mod tests { "svc-b".to_string(), Instance { container_id: "svc-b".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -440,6 +459,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status, ..Default::default() }, @@ -473,6 +493,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -519,6 +540,42 @@ mod tests { assert!(s.services.is_none()); } + /// A managed instance started a moment ago is on disk (`managed_ids`) before the + /// daemon's next broadcast marks it `managed`. The Services overlay lists only + /// broadcast-managed rows, so `s` cannot focus one for it: the stop offer must + /// not be made, and `s` must say it did not handle the key (so the prompt + /// resolves it normally) instead of claiming a stoppable instance and dropping it. + #[test] + fn a_registry_managed_instance_the_broadcast_does_not_yet_mark_managed_is_not_offered() { + use crossterm::event::KeyCode; + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + for (managed, offered) in [(false, false), (true, true)] { + let mut s = st(); + s.instances.insert( + "svc-new".to_string(), + Instance { + container_id: "svc-new".to_string(), + managed, + status: InstanceStatus::Ready, + ..Default::default() + }, + ); + with_registry_probe(&mut s, true, &["svc-new"]); + assert_eq!( + s.has_stoppable_managed_instance_with(&s.live_registry_snapshot()), + offered, + "managed={managed}" + ); + s.open_quit_confirm(); + assert_eq!( + s.quit_confirm_open_services_key(KeyCode::Char('s')), + offered, + "managed={managed}" + ); + assert_eq!(s.services.is_some(), offered, "managed={managed}"); + } + } + #[test] fn quit_confirm_open_services_key_ignores_other_keys_and_a_closed_prompt() { use crossterm::event::KeyCode; diff --git a/crates/rocm-dash-tui/src/app/test_support.rs b/crates/rocm-dash-tui/src/app/test_support.rs index 0920e4a796..2114292b9f 100644 --- a/crates/rocm-dash-tui/src/app/test_support.rs +++ b/crates/rocm-dash-tui/src/app/test_support.rs @@ -37,6 +37,7 @@ pub(crate) fn with_registry_probe( ManagedRegistrySnapshot { has_live_service, managed_ids: managed_ids.iter().map(|s| (*s).to_string()).collect(), + ..Default::default() }, ))); } @@ -71,5 +72,6 @@ pub(super) fn args_with_anthropic_key(key: Option<&str>) -> ResolvedArgs { services_past_attempts: 0, managed_registry_probe: None, launcher_hosted: false, + lemonade_managed_installed: None, } } diff --git a/crates/rocm-dash-tui/src/app/types.rs b/crates/rocm-dash-tui/src/app/types.rs index 27fb3962ae..d28442959c 100644 --- a/crates/rocm-dash-tui/src/app/types.rs +++ b/crates/rocm-dash-tui/src/app/types.rs @@ -143,6 +143,10 @@ pub struct ResolvedArgs { /// serves still warns (see dash-29). See /// `registry_probe::ManagedRegistryProbe`. pub managed_registry_probe: Option, + /// Launch-time managed Lemonade installation verdict from the bin's built-in + /// Detect method. None means unavailable/unknown, never not installed. + /// Separate from live external-serving telemetry and GPU system metadata. + pub lemonade_managed_installed: Option, } impl ResolvedArgs { @@ -184,7 +188,8 @@ impl ConnState { /// True for the Windows in-process collector (see /// `client::connect_and_run`'s `#[cfg(windows)]` arm), which tags its /// `daemon_version` this way since there's no real `rocmdashd` behind - /// it — and so no instance registry it could ever populate. + /// it — its instances come only from the managed-service registry + /// (`AppState::set_managed_instances`), never from discovery. pub fn is_in_process(&self) -> bool { matches!(self, Self::Connected { version, .. } if version.starts_with(Self::IN_PROCESS_PREFIX)) } diff --git a/crates/rocm-dash-tui/src/client.rs b/crates/rocm-dash-tui/src/client.rs index 069e2e86b3..1c63b76050 100644 --- a/crates/rocm-dash-tui/src/client.rs +++ b/crates/rocm-dash-tui/src/client.rs @@ -41,10 +41,14 @@ pub enum ClientMsg { /// Deliberately NOT `Event(Snapshot)` — that path treats `instances` as /// authoritative truth and latches `has_received_snapshot` forever, but /// this snapshot's `instances` is always an empty stand-in (Windows has - /// no instance registry daemon yet), so routing it through there would - /// permanently blind the issue #145 quit-confirm prompt to models - /// actually being served. See `AppState::push_host_gpu_snapshot`. + /// no instance registry daemon yet). `AppState::push_host_gpu_snapshot` + /// leaves `instances` alone; they arrive as `ManagedInstances`. HostGpuSnapshot(Box), + /// Windows only: the managed services the registry poller + /// (`event_loop::spawn_managed_registry_poller`) last read, off the UI + /// thread. Replaces `AppState::instances` — see + /// `AppState::set_managed_instances`. + ManagedInstances(Vec), /// Replay-only: the scrubber jumped backward. The TUI must wipe any /// state derived from past events before the burst-emitted events for /// the new playhead position arrive. @@ -230,9 +234,9 @@ fn build_host_gpu_snapshot( /// instead of connecting anywhere: `HostCollector` (already cross-platform) /// for CPU/memory, `WinPerfCollector` (OS perf counters, no AMD tool needed) /// for GPU. Sends `Connected` like the unix arm, but then a stream of -/// `ClientMsg::HostGpuSnapshot` rather than `Event::Snapshot` — deliberately -/// not routed through `Event::Snapshot`, to avoid disturbing the "quit while -/// serving" confirmation prompt (see `AppState::push_host_gpu_snapshot`). +/// `ClientMsg::HostGpuSnapshot` rather than `Event::Snapshot`; instances come +/// from the managed-service registry poller, not from here (see +/// `ClientMsg::ManagedInstances`). #[cfg(windows)] async fn connect_and_run(connect: &str, tx: UnboundedSender) -> anyhow::Result<()> { use rocm_dash_collectors::host::HostCollector; diff --git a/crates/rocm-dash-tui/src/jobs.rs b/crates/rocm-dash-tui/src/jobs.rs index 8f964bc71d..a3a1b55169 100644 --- a/crates/rocm-dash-tui/src/jobs.rs +++ b/crates/rocm-dash-tui/src/jobs.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Async job-bridge runtime (Phase 3 Wave 0). +//! Async job-bridge runtime. //! //! Interprets [`SideEffect::SpawnJob`]: launches the child process, streams its //! stdout+stderr back as [`StateEvent::JobLine`], finishes with diff --git a/crates/rocm-dash-tui/src/llm.rs b/crates/rocm-dash-tui/src/llm.rs index f83f85e687..b5627f9616 100644 --- a/crates/rocm-dash-tui/src/llm.rs +++ b/crates/rocm-dash-tui/src/llm.rs @@ -4,7 +4,7 @@ //! Chat LLM configuration: a pure precedence resolver plus a std-only TCP liveness probe. //! -//! No HTTP client here — the Rig-built `AgentClient` (Phase 3) +//! No HTTP client here — the Rig-built `AgentClient` //! lives in `agent/clients.rs`. Keeping detection in the TUI crate preserves the //! core's render/async-free boundary. diff --git a/crates/rocm-dash-tui/src/registry_probe.rs b/crates/rocm-dash-tui/src/registry_probe.rs index 87614553db..17a1967cec 100644 --- a/crates/rocm-dash-tui/src/registry_probe.rs +++ b/crates/rocm-dash-tui/src/registry_probe.rs @@ -18,16 +18,20 @@ //! whose daemon client never sends an instance snapshot (Windows — it //! connects and streams live GPU/CPU/memory every tick, but has no //! `rocmdashd` socket to source an instance list from, so -//! `has_received_snapshot` never flips and `AppState::instances` stays empty -//! for the life of the session). A live probe has neither problem — there is -//! no window for it to go stale in. +//! `has_received_snapshot` never flips). A live probe has neither problem — +//! there is no window for it to go stale in. On Windows it is also the source +//! of `AppState::instances` itself: a background poller +//! (`event_loop::spawn_managed_registry_poller`) reads +//! [`ManagedRegistrySnapshot::instances`] off the UI thread once a second. use std::collections::HashSet; use std::sync::Arc; +use rocm_dash_core::metrics::Instance; + /// Live classification of the managed-service registry, as of the moment /// [`ManagedRegistryProbe::snapshot`] was called. -#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default)] pub struct ManagedRegistrySnapshot { /// Whether any managed-service record is currently a live scrape target — /// the same statuses `Instance::status.is_serving()` treats as "actively @@ -45,6 +49,11 @@ pub struct ManagedRegistrySnapshot { /// Docker- or Lemonade-discovered instance from an external daemon, which /// that command cannot stop. pub managed_ids: HashSet, + /// The live scrape targets themselves (the records `managed_ids` names), + /// as dashboard instances with no telemetry. The Windows in-process ticker + /// has no daemon to send instances, so the registry poller takes them from + /// here. + pub instances: Vec, } /// rocm-core/rocm-dash-daemon-free live registry-read boundary. @@ -63,12 +72,15 @@ pub struct ManagedRegistrySnapshot { /// daemon round-trip or network I/O. In the dashboard, [`snapshot`](Self::snapshot) /// is called every time something needs `live_registry_snapshot`'s answer: /// `request_quit` on the quit keystroke, `quit_confirm_request` on every -/// render while the quit-confirm prompt is open, and -/// `quit_confirm_open_services_key` on the `s` keystroke while it's open. -/// The launcher's own `has_live_instance` check calls it on every render -/// too, but through `StaticManagedRegistrySnapshot` — a read cached once per -/// menu pass — so that per-frame call never reaches an implementation's -/// actual registry I/O; only the dashboard's calls above do. +/// render while the quit-confirm prompt is open, +/// `quit_confirm_open_services_key` on the `s` keystroke while it's open, +/// and — on Windows only — the registry poller +/// (`event_loop::spawn_managed_registry_poller`), once a second under +/// `spawn_blocking`. The launcher's own +/// `has_live_instance` check calls it on every render too, but through +/// `StaticManagedRegistrySnapshot` — a read cached once per menu pass — so +/// that per-frame call never reaches an implementation's actual registry I/O; +/// only the dashboard's calls above do. pub trait ManagedRegistryProbe: std::fmt::Debug + Send + Sync { /// Read the registry now and classify it from the record's own `status` /// field, the same statuses `Instance::status.is_serving()` treats as @@ -122,6 +134,7 @@ mod tests { let probe = StaticManagedRegistrySnapshot(ManagedRegistrySnapshot { has_live_service: true, managed_ids: ids.clone(), + ..Default::default() }); let snap = probe.snapshot(); assert!(snap.has_live_service); diff --git a/crates/rocm-dash-tui/src/tool_exec.rs b/crates/rocm-dash-tui/src/tool_exec.rs index 0ae43ff45f..8ad611e742 100644 --- a/crates/rocm-dash-tui/src/tool_exec.rs +++ b/crates/rocm-dash-tui/src/tool_exec.rs @@ -7,8 +7,8 @@ //! Plain-data signatures ONLY (serde_json / std / serde). The bin (`apps/rocm`, //! which owns `rocm-core` and the tool engine) implements [`RocmToolExecutor`]; //! the dash holds it as `Option>` and never depends on -//! `rocm-core`. Phase 2 stored the seam; Phase 3 used it for read-only tools; -//! Phase 4 adds the mutating "execute approved" path + the approval descriptor. +//! `rocm-core`. The trait has a read-only `execute` and a mutating +//! `execute_approved` path, plus the approval descriptor. use std::sync::Arc; diff --git a/crates/rocm-dash-tui/src/ui/approval.rs b/crates/rocm-dash-tui/src/ui/approval.rs index 66d080905b..6fa82ffef9 100644 --- a/crates/rocm-dash-tui/src/ui/approval.rs +++ b/crates/rocm-dash-tui/src/ui/approval.rs @@ -2,12 +2,11 @@ // // SPDX-License-Identifier: MIT -//! Approval-gate seam (Phase 3 Wave 0). +//! Approval-gate seam. //! //! This is a **render + event seam ONLY**. The decision *logic* — what an //! approval actually does (run a CLI command, enable full access, approve a -//! proposal) — stays CLI-side per the working agreement -//! (`rocm-cli-unification-working-agreements.md` §1). This module renders an +//! proposal) — stays CLI-side. This module renders an //! [`ApprovalRequest`] and reports the user's [`ApprovalVerdict`]; the caller //! maps the verdict onto a CLI-side action. //! diff --git a/crates/rocm-dash-tui/src/ui/automations_manager.rs b/crates/rocm-dash-tui/src/ui/automations_manager.rs index b54ee199ca..acf6e4ab1e 100644 --- a/crates/rocm-dash-tui/src/ui/automations_manager.rs +++ b/crates/rocm-dash-tui/src/ui/automations_manager.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Automations manager overlay (Phase 3 Wave 3). +//! Automations manager overlay. //! //! Lists the background checks ("watchers", plumbed in from the bin which owns //! `rocm-core`) and toggles them via the `rocm automations …` verbs: diff --git a/crates/rocm-dash-tui/src/ui/bench.rs b/crates/rocm-dash-tui/src/ui/bench.rs deleted file mode 100644 index 23372f5ad5..0000000000 --- a/crates/rocm-dash-tui/src/ui/bench.rs +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// -// SPDX-License-Identifier: MIT - -//! Benchmark result table + filters. Stub. diff --git a/crates/rocm-dash-tui/src/ui/command_screen.rs b/crates/rocm-dash-tui/src/ui/command_screen.rs index dca012b8bc..e52936af09 100644 --- a/crates/rocm-dash-tui/src/ui/command_screen.rs +++ b/crates/rocm-dash-tui/src/ui/command_screen.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Command runner overlay (Phase 3 Wave 3). +//! Command runner overlay. //! //! A general escape hatch: type any `rocm …` subcommand and run it through the //! job-bridge. Because arbitrary subcommands can't be statically classified as diff --git a/crates/rocm-dash-tui/src/ui/config_manager.rs b/crates/rocm-dash-tui/src/ui/config_manager.rs index 278104ac4d..9f124dbc5a 100644 --- a/crates/rocm-dash-tui/src/ui/config_manager.rs +++ b/crates/rocm-dash-tui/src/ui/config_manager.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Config + provider manager overlay (Phase 3 Wave 3). +//! Config + provider manager overlay. //! //! Folds the frozen `config_manager` and `provider_manager` into one screen: //! diff --git a/crates/rocm-dash-tui/src/ui/core_bars.rs b/crates/rocm-dash-tui/src/ui/core_bars.rs index 097217204a..b30da7c59e 100644 --- a/crates/rocm-dash-tui/src/ui/core_bars.rs +++ b/crates/rocm-dash-tui/src/ui/core_bars.rs @@ -4,10 +4,23 @@ //! Per-CPU-core vertical braille bars. //! -//! Each core renders as one character column (2 dots wide) showing its current -//! utilization as a thermometer fill from the bottom. Designed for ≤ ~200 cores -//! visible in a terminal that is at least that wide. Cores that don't fit on -//! screen are dropped from the right edge. +//! Each core renders as a thermometer fill from the bottom. The cores tile the +//! width of the area rather than taking one column each, so the strip spans its +//! box instead of ending in a short stub on the left. The strip has three +//! shapes, by how much width each core can have: +//! +//! - **Two or more columns each**: every bar is the same width and separated +//! from its neighbour, with the left-over columns widening the *gaps* rather +//! than the bars. Two cores under equal load must draw the same size, and +//! when bars are separated a one-column difference is plain to see. +//! - **About one column each**: bars tile proportionally and touch, so the +//! strip reads as a continuous profile across the cores. Widths differ by up +//! to a column here, which is not a signal when nothing is separated, and +//! gutters are dropped entirely rather than applied to some bars and not +//! others. +//! - **More cores than columns**: each column covers a group of cores and +//! draws the busiest of them. Every core reaches a column, so a single +//! pegged core on a many-core host cannot be hidden by its quiet neighbours. //! //! Inspired by btop's per-core mini bars. @@ -56,41 +69,105 @@ impl<'a> CoreBars<'a> { } } +impl CoreBars<'_> { + /// Paint one bar of `value` across columns `[start, end)` of `area`. + fn draw_bar(&self, buf: &mut Buffer, area: Rect, start: usize, end: usize, value: f32) { + let rows = area.height as usize; + let total_dot_rows = rows * 4; + let v = value.clamp(0.0, self.max); + // Every core draws at least the bottom dot row: an idle core is a + // baseline, not a gap, otherwise a few percent of `max` rounds to + // zero dots and the bar strip breaks into fragments. + let lit = ((v / self.max) * total_dot_rows as f32).round().max(1.0) as usize; + let first_lit = total_dot_rows.saturating_sub(lit); + let style = match self.gradient { + Some(stops) => self.style.fg(lerp3_t(stops, f64::from(v / self.max))), + None => self.style, + }; + for cy in 0..rows { + let row_top = cy * 4; + let mut mask = 0u8; + for i in 0..4 { + let dot_row = row_top + i; + if dot_row >= first_lit && dot_row < total_dot_rows { + mask |= 1 << i; + } + } + if mask == 0 { + continue; + } + // Use both left and right dot columns for a fuller bar look. + let s = braille_both_cols(mask).to_string(); + for cx in start..end { + buf.set_string(area.x + cx as u16, area.y + cy as u16, &s, style); + } + } + } +} + impl Widget for CoreBars<'_> { fn render(self, area: Rect, buf: &mut Buffer) { if area.width == 0 || area.height == 0 || self.values.is_empty() { return; } - let cols = (area.width as usize).min(self.values.len()); - let rows = area.height as usize; - let total_dot_rows = rows * 4; + let width = area.width as usize; + let n = self.values.len(); - for cx in 0..cols { - let v = self.values[cx].clamp(0.0, self.max); - let lit = ((v / self.max) * total_dot_rows as f32).round() as usize; - let first_lit = total_dot_rows.saturating_sub(lit); - let style = match self.gradient { - Some(stops) if v > 0.0 => self.style.fg(lerp3_t(stops, f64::from(v / self.max))), - _ => self.style, - }; - for cy in 0..rows { - let row_top = cy * 4; - let mut mask = 0u8; - for i in 0..4 { - let dot_row = row_top + i; - if dot_row >= first_lit && dot_row < total_dot_rows { - mask |= 1 << i; - } - } - if mask == 0 { - continue; - } - // Use both left and right dot columns for a fuller bar look. - let s = braille_both_cols(mask).to_string(); - buf.set_string(area.x + cx as u16, area.y + cy as u16, &s, style); + if n > width { + // One column per group of cores, showing the busiest of the group. + // Giving the column to a single representative core instead would + // let a pegged core vanish behind idle neighbours. + for cx in 0..width { + let (lo, hi) = core_group(cx, n, width); + let busiest = self.values[lo..hi] + .iter() + .copied() + .fold(f32::NEG_INFINITY, f32::max); + self.draw_bar(buf, area, cx, cx + 1, busiest); } + return; } + for (core, &value) in self.values.iter().enumerate() { + let (start, end) = slot(core, n, width); + self.draw_bar(buf, area, start, end, value); + } + } +} + +/// Cores `[lo, hi)` sharing column `cx` when cores outnumber columns. +/// +/// The groups tile `0..n`, so every core lands in exactly one column. The +/// `lo + 1` arm is defensive rather than reachable: `render` only calls this +/// with `n > width`, where `n / width >= 1` makes every group non-empty. +const fn core_group(cx: usize, n: usize, width: usize) -> (usize, usize) { + let lo = cx * n / width; + let hi = (cx + 1) * n / width; + (lo, if hi > lo { hi } else { lo + 1 }) +} + +/// Columns `[start, end)` that core `core` of `n` draws into, across `width`. +/// +/// Requires `width >= n >= 1`; `render` routes the other case to `core_group`. +/// +/// With two or more columns per core every bar is `width / n - 1` wide and the +/// `width % n` left-over columns go into the gaps, so bars stay uniform and +/// only the spacing varies. With about one column per core the bars tile +/// proportionally and touch: a gutter there would have to come out of some +/// bars and not others, and a 1-wide bar flush against its neighbour reads as +/// one wider bar, which misstates the core count. +const fn slot(core: usize, n: usize, width: usize) -> (usize, usize) { + debug_assert!(n > 0, "slot() divides by the core count"); + let slot_w = width / n; + if slot_w < 2 { + return (core * width / n, (core + 1) * width / n); } + // Proportional start, uniform width. Starting each bar at its exact share + // of the width spreads the spare columns evenly down the strip; handing + // them to the first `width % n` cores instead would bunch every wide gap + // into the left half and leave the right half visibly denser. + let bar_w = slot_w - 1; + let start = core * width / n; + (start, start + bar_w) } const fn braille_both_cols(mask: u8) -> char { @@ -129,4 +206,230 @@ mod tests { let c = braille_both_cols(0b1000); assert_eq!(c as u32 - 0x2800, 0x40 | 0x80); } + + #[test] + fn separated_bars_are_all_the_same_width() { + // 32 cores across 170 columns: 5 columns each with 10 left over. The + // spare columns must widen the gaps, not ten of the bars — two cores + // under equal load have to draw the same size. + let (n, width) = (32, 170); + let slots: Vec<(usize, usize)> = (0..n).map(|c| slot(c, n, width)).collect(); + let widths: Vec = slots.iter().map(|&(s, e)| e - s).collect(); + assert!( + widths.iter().all(|&w| w == widths[0]), + "bars differ in width: {widths:?}" + ); + assert_eq!(slots[0].0, 0, "strip does not start at the left edge"); + // The last bar ends within one slot of the right edge: its own gutter, + // plus whatever the proportional start rounded away. + assert!( + slots[n - 1].1 + width / n >= width, + "strip stops short of the right edge: ends at {}", + slots[n - 1].1 + ); + for (core, pair) in slots.windows(2).enumerate() { + let ((_, end), (next_start, _)) = (pair[0], pair[1]); + assert!(next_start > end, "core {core} has no gutter"); + } + // Gaps stay within a column of each other, so the strip has an even + // rhythm rather than a sparse half and a dense half. + let gaps: Vec = slots.windows(2).map(|p| p[1].0 - p[0].1).collect(); + let (lo, hi) = (gaps.iter().min().unwrap(), gaps.iter().max().unwrap()); + assert!(hi - lo <= 1, "gaps vary too much: {gaps:?}"); + } + + /// How many neighbouring core pairs have no blank column between them. + fn touching_pairs(n: usize, width: usize) -> usize { + (0..n - 1) + .filter(|&c| slot(c + 1, n, width).0 == slot(c, n, width).1) + .count() + } + + #[test] + fn gutters_are_all_or_nothing_across_the_strip() { + // The strip is one of two shapes, never a mixture. + // + // At 2+ columns per core every bar is separated. At fewer, slots + // alternate 1 and 2 columns wide, and taking a gutter only from the + // 2-wide ones would leave *some* pairs touching, so 32 cores would + // read as 16 wider bars. All touching is a contiguous histogram and + // is honest about the core count; a mixture is not. + assert_eq!(touching_pairs(32, 170), 0, "wide strip should be separated"); + assert_eq!( + touching_pairs(32, 48), + 31, + "narrow strip should be contiguous, not partly gapped" + ); + } + + #[test] + fn every_core_keeps_a_bar_in_the_contiguous_regime() { + let (n, width) = (32, 48); + let widths: Vec = (0..n) + .map(|c| { + let (s, e) = slot(c, n, width); + e - s + }) + .collect(); + assert!(widths.iter().all(|&w| w >= 1), "a core lost its bar"); + } + + #[test] + fn the_baseline_bar_takes_the_gradient_start_color() { + use ratatui::buffer::Buffer; + use ratatui::layout::Rect; + // Same contract as the sparkline: an idle core's baseline is part of + // the chart, so it carries the gradient's low color. A `v > 0.0` + // guard on the gradient would pass without this. + let values = [0.0_f32]; + let mut buf = Buffer::empty(Rect::new(0, 0, 1, 1)); + let (low, mid, high) = ( + Color::Rgb(10, 0, 0), + Color::Rgb(0, 20, 0), + Color::Rgb(0, 0, 30), + ); + CoreBars::new(&values) + .max(100.0) + .style(Style::default().fg(Color::Rgb(99, 99, 99))) + .gradient(low, mid, high) + .render(buf.area, &mut buf); + assert_eq!(buf.cell((0, 0)).unwrap().style().fg, Some(low)); + assert_eq!(buf.cell((0, 0)).unwrap().symbol(), "\u{28c0}"); + } + + #[test] + fn narrow_area_degrades_to_one_column_per_core() { + // One column per core leaves nothing to spare, so the gutter goes + // rather than the bar. + for core in 0..8 { + assert_eq!(slot(core, 8, 8), (core, core + 1)); + } + } + + #[test] + fn core_groups_cover_every_core_exactly_once() { + // 8 cores into 4 columns. The groups must tile 0..n: a core that falls + // in no group is a core the user cannot see. + let (n, width) = (8, 4); + let mut next = 0; + for cx in 0..width { + let (lo, hi) = core_group(cx, n, width); + assert_eq!(lo, next, "column {cx} skips a core"); + assert!(hi > lo, "column {cx} covers no core"); + next = hi; + } + assert_eq!(next, n, "the last cores reach no column"); + } + + #[test] + fn a_pegged_core_is_visible_even_when_cores_outnumber_columns() { + use ratatui::buffer::Buffer; + use ratatui::layout::Rect; + // Peg core 1, the *second* member of column 0's group, and core 7, the + // last member of column 3's. A column that drew its first core rather + // than its busiest would miss both, so this fails for `values[lo]` as + // well as for dropping all but one core. + let mut values = [0.0_f32; 8]; + values[1] = 100.0; + values[7] = 100.0; + let mut buf = Buffer::empty(Rect::new(0, 0, 4, 1)); + CoreBars::new(&values).max(100.0).render(buf.area, &mut buf); + assert_eq!( + buf.cell((0, 0)).unwrap().symbol(), + "\u{28ff}", + "column 0 missed the pegged core it shares with core 0" + ); + assert_eq!( + buf.cell((3, 0)).unwrap().symbol(), + "\u{28ff}", + "column 3 missed the pegged core at the end of its group" + ); + // Columns whose cores are all idle still read as a baseline. + assert_eq!(buf.cell((1, 0)).unwrap().symbol(), "\u{28c0}"); + assert_eq!(buf.cell((2, 0)).unwrap().symbol(), "\u{28c0}"); + } + + #[test] + fn a_group_draws_its_busiest_core_not_the_sum_of_them() { + use ratatui::buffer::Buffer; + use ratatui::layout::Rect; + // Two unequal non-zero cores in one group. Every other test leaves a + // single non-zero value per group, where the sum and the max agree, so + // a `max` -> `sum` slip would draw the same thing and go unnoticed. + // 30 + 40 would saturate the bar; the busiest of them must not. + let mut values = [0.0_f32; 8]; + values[0] = 30.0; + values[1] = 40.0; + assert_eq!(core_group(0, 8, 4), (0, 2), "group shape changed"); + let mut buf = Buffer::empty(Rect::new(0, 0, 4, 4)); + CoreBars::new(&values).max(100.0).render(buf.area, &mut buf); + // Over 4 cell rows the bar has 16 dot rows. The busiest core, 40%, + // lights 6 of them and so reaches cell row 2 but not row 1; their sum, + // 70%, lights 11 and would reach row 1. Row 0 stays blank either way, + // which is why it cannot be the assertion. + assert_eq!( + buf.cell((0, 1)).unwrap().symbol(), + " ", + "the group drew the sum of its cores, not the busiest" + ); + assert_ne!( + buf.cell((0, 2)).unwrap().symbol(), + " ", + "the group did not draw its busiest core" + ); + } + + #[test] + fn a_pegged_core_survives_groups_of_uneven_size() { + use ratatui::buffer::Buffer; + use ratatui::layout::Rect; + // 10 cores into 4 columns: groups are uneven, so no fixed offset + // within a group is safe to pick. Column 1 covers cores 2, 3 and 4; + // core 3 is the middle one, which neither `values[lo]` (core 2) nor + // `values[hi - 1]` (core 4) would show. + let mut values = [0.0_f32; 10]; + values[3] = 100.0; + let mut buf = Buffer::empty(Rect::new(0, 0, 4, 1)); + CoreBars::new(&values).max(100.0).render(buf.area, &mut buf); + assert_eq!(core_group(1, 10, 4), (2, 5), "group shape changed"); + assert_eq!( + buf.cell((1, 0)).unwrap().symbol(), + "\u{28ff}", + "a pegged core in the middle of its group was not shown" + ); + } + + #[test] + fn two_columns_per_core_is_where_bars_separate() { + // The threshold `slot` switches on. At exactly two columns per core the + // bars must separate; one below, they tile and touch. Moving the test + // to `>= 3` or `>= 1` breaks one of these two. + assert_eq!(slot(0, 4, 8), (0, 1), "2 cols/core should leave a gutter"); + assert_eq!(slot(1, 4, 8), (2, 3)); + assert_eq!(touching_pairs(4, 8), 0, "2 cols/core should be separated"); + // 7 columns for 4 cores is one short of the threshold. + assert_eq!(touching_pairs(4, 7), 3, "below it the strip is contiguous"); + } + + #[test] + fn bars_fill_their_slot_and_idle_cores_draw_a_baseline() { + use ratatui::buffer::Buffer; + use ratatui::layout::Rect; + // Two cores across 10 columns: one busy, one idle at a fraction of a + // dot row. Both must be visible, each 4 columns wide plus a gutter. + let values = [90.0_f32, 0.4]; + let mut buf = Buffer::empty(Rect::new(0, 0, 10, 1)); + CoreBars::new(&values).max(100.0).render(buf.area, &mut buf); + // An untouched cell keeps the buffer's blank, not a blank braille cell. + let drawn: Vec = (0..10) + .map(|x| buf.cell((x, 0)).unwrap().symbol() != " ") + .collect(); + assert_eq!( + drawn, + [true, true, true, true, false, true, true, true, true, false] + ); + // The busy core fills its slot; the idle one is a baseline, not a bar. + assert_eq!(buf.cell((0, 0)).unwrap().symbol(), "\u{28ff}"); + assert_eq!(buf.cell((5, 0)).unwrap().symbol(), "\u{28c0}"); + } } diff --git a/crates/rocm-dash-tui/src/ui/dock.rs b/crates/rocm-dash-tui/src/ui/dock.rs index 3ba1ea738f..bc83343811 100644 --- a/crates/rocm-dash-tui/src/ui/dock.rs +++ b/crates/rocm-dash-tui/src/ui/dock.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Wide-layout triptych rails (Phase 6). +//! Wide-layout triptych rails. //! //! When the terminal is wide enough, `ui::draw` splits the body into a left GPU //! wall, the existing center tab body, and a persistent right dock. The right @@ -225,7 +225,10 @@ pub fn context_rail(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) } lines.push(Line::raw("")); lines.push(header("GPU STATE")); - if let Some(s) = state.latest.as_ref() { + // Same rule the sparkline below uses: a snapshot with no GPUs has nothing + // to report, and folding over an empty list would print a measured-looking + // "peak util 0.0%" for a host that never reported one. + if let Some(s) = state.latest.as_ref().filter(|s| !s.gpus.is_empty()) { let util = s .gpus .iter() @@ -264,16 +267,14 @@ pub fn context_rail(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) } } // A small live trace so the rail has motion when telemetry is present. - let util_hist: Vec = state - .history - .iter() - .map(|s| { - s.gpus - .iter() - .map(|g| f64::from(g.gpu_utilization_pct)) - .fold(0.0, f64::max) as u64 - }) - .collect(); + // `gpu_history` owns the GPU-less skip: without it a host with no GPU + // draws a flat 0% trace instead of an empty one. + let util_hist: Vec = crate::ui::widgets::gpu_history(state.history.iter(), |s| { + s.gpus + .iter() + .map(|g| f64::from(g.gpu_utilization_pct)) + .fold(0.0, f64::max) + }); let body = inner.height.saturating_sub(lines.len() as u16); f.render_widget(Paragraph::new(lines), inner); if body >= 1 && !util_hist.is_empty() { @@ -494,6 +495,69 @@ mod tests { ); } + #[test] + fn a_gpu_less_host_reports_no_telemetry_rather_than_a_measured_zero() { + // Both halves of the GPU STATE section: the peak-util line and the + // trace under it. Folding over an empty `gpus` list yields 0, and the + // widget draws a baseline for every sample it is handed, so without + // the skip a host with no GPU reads as one idling at 0%. + let mut s = AppState::new("t".into(), "default-dark".into()); + s.latest = Some(Snapshot::default()); + for _ in 0..8 { + s.history.push_back(Snapshot::default()); + } + let backend = TestBackend::new(DOCK_W, 24); + let mut term = Terminal::new(backend).unwrap(); + let theme = s.theme; + term.draw(|f| context_rail(f, f.area(), &s, &theme)) + .unwrap(); + let out = flat(&term); + assert!( + out.contains("no telemetry"), + "a GPU-less host should report nothing: {out:?}" + ); + assert!( + !out.contains("peak util"), + "a GPU-less host reported a measured peak: {out:?}" + ); + // The trace under it, too: the widget draws a baseline for any sample + // it is handed, so an unfiltered series would paint one here. + assert!( + !out.chars().any(|c| ('\u{2800}'..='\u{28ff}').contains(&c)), + "a GPU-less host drew a trace: {out:?}" + ); + } + + #[test] + fn a_gpu_bearing_history_still_draws_its_trace() { + // The other side of the same filter: real GPU snapshots must survive + // it, so reverting the call site to an unfiltered fold is not the only + // thing this catches. + let mut s = AppState::new("t".into(), "default-dark".into()); + let with_gpu = || Snapshot { + gpus: vec![GpuMetrics { + gpu_utilization_pct: 60.0, + ..Default::default() + }], + ..Default::default() + }; + s.latest = Some(with_gpu()); + for _ in 0..8 { + s.history.push_back(with_gpu()); + } + let backend = TestBackend::new(DOCK_W, 24); + let mut term = Terminal::new(backend).unwrap(); + let theme = s.theme; + term.draw(|f| context_rail(f, f.area(), &s, &theme)) + .unwrap(); + let out = flat(&term); + assert!(out.contains("peak util"), "no peak util line: {out:?}"); + assert!( + !out.contains("no telemetry"), + "a GPU-bearing host was reported as having none: {out:?}" + ); + } + #[test] fn context_rail_shows_sections() { let mut s = AppState::new("t".into(), "default-dark".into()); diff --git a/crates/rocm-dash-tui/src/ui/engine_manager.rs b/crates/rocm-dash-tui/src/ui/engine_manager.rs index 5a1cd82037..802a645d90 100644 --- a/crates/rocm-dash-tui/src/ui/engine_manager.rs +++ b/crates/rocm-dash-tui/src/ui/engine_manager.rs @@ -2,13 +2,12 @@ // // SPDX-License-Identifier: MIT -//! Engine manager overlay (Phase 3 Wave 1). +//! Engine manager overlay. //! //! Lists the serving engines ROCm CLI knows about and runs use/install/reinstall //! **through the approval gate and the job-bridge** — never inline. Installs -//! stream their output into the shared job console (the Wave-0 `running_job` -//! replacement). This is the third operational screen on the proven primitives; -//! it feeds the serve wizard (which engine to launch a model with). +//! stream their output into the shared job console. It feeds the serve wizard +//! (which engine to launch a model with). //! //! The approval *decision* is the user's, captured by the render+event seam; the //! CLI owns the actual mutation (`rocm config set-default-engine` / diff --git a/crates/rocm-dash-tui/src/ui/examine_manager.rs b/crates/rocm-dash-tui/src/ui/examine_manager.rs index 40138296b0..67c21878c4 100644 --- a/crates/rocm-dash-tui/src/ui/examine_manager.rs +++ b/crates/rocm-dash-tui/src/ui/examine_manager.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Examine overlay (Phase 3 Wave 2). +//! Examine overlay. //! //! Runs `rocm examine` — a read-only environment check — through the job-bridge //! and shows its streamed output in the shared job console. That run is diff --git a/crates/rocm-dash-tui/src/ui/exec.rs b/crates/rocm-dash-tui/src/ui/exec.rs index 5da3542ef0..48a9fda0a3 100644 --- a/crates/rocm-dash-tui/src/ui/exec.rs +++ b/crates/rocm-dash-tui/src/ui/exec.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Shared helpers for launching `rocm` sub-commands from operational screens (Phase 3 Wave 1). +//! Shared helpers for launching `rocm` sub-commands from operational screens. //! //! Every screen that routes a mutating action through the //! approval gate + job-bridge resolves the binary the same way, so the logic diff --git a/crates/rocm-dash-tui/src/ui/folder_browser.rs b/crates/rocm-dash-tui/src/ui/folder_browser.rs index b26c490c9a..b6e5adf618 100644 --- a/crates/rocm-dash-tui/src/ui/folder_browser.rs +++ b/crates/rocm-dash-tui/src/ui/folder_browser.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Reusable folder browser (Phase 3 Wave 0). +//! Reusable folder browser. //! //! A shared input primitive: pick (or create-under) a directory. Dependency of //! serve / install / runtime / onboarding. Models the frozen rocm-cli diff --git a/crates/rocm-dash-tui/src/ui/format.rs b/crates/rocm-dash-tui/src/ui/format.rs index cbd8ebcdbe..01eb031621 100644 --- a/crates/rocm-dash-tui/src/ui/format.rs +++ b/crates/rocm-dash-tui/src/ui/format.rs @@ -201,7 +201,7 @@ pub fn duration(seconds: f64) -> String { } /// Power in watts. One decimal, always trailing `W`. -pub fn watts(value: f32) -> String { +pub fn watts(value: f64) -> String { format!("{value:.1} W") } @@ -230,6 +230,30 @@ pub const HELD_MARKER: &str = "*"; /// panels rather than inventing view-specific descriptions. pub const HELD_LEGEND: &str = "* = held (prior scrape window)"; +/// Explains the tok/s of a Lemonade row: the endpoint reports the rate of its most +/// recent completed request, which stays unchanged while the server is idle. +/// +/// Short enough to join the held legend inside an 80-column table. +pub const LAST_REQ_LEGEND: &str = "Lemonade: last req tok/s, not live"; + +/// The label for a throughput figure (compact rows and the instance detail). +/// +/// `last req` for the rate of a Lemonade endpoint's most recent request +/// ([`rocm_dash_core::metrics::Instance::gen_tps_last_request`]), `gen` for a +/// live rate. +#[must_use] +pub const fn gen_rate_label(last_request: bool) -> &'static str { + if last_request { "last req" } else { "gen" } +} + +/// [`gen_rate_label`] for a column with a fixed budget: as wide as `gen`, so a +/// last-request rate is never clipped to a different number. A legend +/// ([`LAST_REQ_LEGEND`]) explains it. +#[must_use] +pub const fn gen_rate_label_short(last_request: bool) -> &'static str { + if last_request { "req" } else { "gen" } +} + /// Compact gen_tps for narrow table cells (matches the existing bare-number format). /// /// - `None` → `"—"` (unavailable; em-dash matches the rest of the table) diff --git a/crates/rocm-dash-tui/src/ui/install_manager.rs b/crates/rocm-dash-tui/src/ui/install_manager.rs index 6a25a1a742..e2d4cdb6df 100644 --- a/crates/rocm-dash-tui/src/ui/install_manager.rs +++ b/crates/rocm-dash-tui/src/ui/install_manager.rs @@ -2,12 +2,12 @@ // // SPDX-License-Identifier: MIT -//! Install overlay (Phase 3 Wave 2). +//! Install overlay. //! //! A compact form that builds a `rocm install sdk …` invocation (TheRock ROCm //! wheels). Dry-run is read-only and spawns straight through the job-bridge; a //! real install is mutating and routes through the approval gate first. The -//! install folder is picked with the Wave-0 [`FolderBrowser`]. Streaming-install +//! install folder is picked with the [`FolderBrowser`]. Streaming-install //! archetype, zero thread::spawn/try_recv. use crossterm::event::{KeyCode, KeyEvent}; diff --git a/crates/rocm-dash-tui/src/ui/instance_detail.rs b/crates/rocm-dash-tui/src/ui/instance_detail.rs index 41b24be0e2..ab3871d117 100644 --- a/crates/rocm-dash-tui/src/ui/instance_detail.rs +++ b/crates/rocm-dash-tui/src/ui/instance_detail.rs @@ -63,25 +63,35 @@ pub(super) fn draw_detail(f: &mut Frame, area: Rect, state: &AppState, theme: &T return 0; } - // Vertical: summary (4 lines: status/id/port/tp · model/gpus/tpw/gen · partition/quant/vram · freshness) + // Vertical: summary (4 lines: status/id/port/tp · model/gpus/tpw/gen · partition/quant/vram · freshness, + // taller when a line wraps at this width: the last line is the one that + // qualifies the rate, so it must never be what falls off) // | recent throughput (recent non-idle ticks, ROCMAI-196) | body (min) | footer (1) let snap_ts = state.latest.as_ref().map(|s| s.timestamp); let activity = recent_activity_rows(&state.history, &inst.container_id, RECENT_ACTIVITY_CAP); let has_observed_throughput = has_observed_throughput(&state.history, &inst.container_id); - let (activity_h, visible_rows) = activity_layout(activity.len(), inner.height); + let summary = summary_lines(inner.width, inst, snap_ts, theme); + let summary_h = u16::try_from( + Paragraph::new(summary.clone()) + .wrap(Wrap { trim: false }) + .line_count(inner.width), + ) + .unwrap_or(u16::MAX) + .clamp(SUMMARY_ROWS, (inner.height / 2).max(SUMMARY_ROWS)); + let (activity_h, visible_rows) = activity_layout(activity.len(), inner.height, summary_h); let total_rows = activity.len(); let activity = &activity[..visible_rows]; let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(4), + Constraint::Length(summary_h), Constraint::Length(activity_h), Constraint::Min(0), Constraint::Length(1), ]) .split(inner); - render_summary(f, chunks[0], inst, snap_ts, theme); + render_summary(f, chunks[0], summary); render_recent_activity( f, chunks[1], @@ -96,6 +106,9 @@ pub(super) fn draw_detail(f: &mut Frame, area: Rect, state: &AppState, theme: &T max_scroll } +/// Rows of the summary block when none of its lines wraps. +const SUMMARY_ROWS: u16 = 4; + /// Max rows shown in the detail modal's "recent throughput" summary. const RECENT_ACTIVITY_CAP: usize = 5; @@ -120,8 +133,8 @@ fn body_floor(inner_h: u16) -> u16 { /// budget, so a capped panel shows fewer (newest) rows instead of being /// handed rows it can't draw. A panel too short for its header plus one row /// (or the placeholder line) collapses to `(0, 0)` so the body gets the rows. -fn activity_layout(n_rows: usize, inner_h: u16) -> (u16, usize) { - let avail = inner_h.saturating_sub(4 + 1 + body_floor(inner_h)); +fn activity_layout(n_rows: usize, inner_h: u16, summary_h: u16) -> (u16, usize) { + let avail = inner_h.saturating_sub(summary_h + 1 + body_floor(inner_h)); // 2 border rows + (1 header row + 1 per entry), or 1 placeholder row when // there's none yet; `panel::bento` adds a top-padding row once the box is // tall enough (`padding_for`), which is reused here rather than re-derived. @@ -153,7 +166,10 @@ pub(crate) struct ActivityRow { /// `container_id` had non-zero generation throughput — i.e. was actually /// serving, not idle. These are per-tick samples, not discrete serving runs. /// The live "gen:" gauge in `render_summary` already covers the idle case, so -/// idle ticks are skipped. Repeated Held ticks of one scrape collapse to one row. Returns up to `cap` rows, newest first. +/// idle ticks are skipped. A Lemonade last-request rate +/// ([`rocm_dash_core::metrics::Instance::gen_tps_last_request`]) is not serving +/// either: it is one finished request's rate re-reported on every tick, so it is +/// skipped like an idle tick. Repeated Held ticks of one scrape collapse to one row. Returns up to `cap` rows, newest first. /// Pure — exposed for tests. pub(crate) fn recent_activity_rows( history: &std::collections::VecDeque, @@ -169,7 +185,7 @@ pub(crate) fn recent_activity_rows( else { continue; }; - if !inst.gen_tps.is_some_and(|v| v.is_finite() && v > 0.0) { + if !inst.live_gen_tps().is_some_and(|v| v > 0.0) { continue; } // A Held tick repeats an earlier scrape; age it from when the @@ -297,18 +313,17 @@ fn render_recent_activity( f.render_widget(table, inner); } -fn render_summary( - f: &mut Frame, - area: Rect, +fn summary_lines( + width: u16, inst: &Instance, snap_ts: Option>, theme: &Theme, -) { +) -> Vec> { let (status_color, status_text) = status_meta(inst.status, theme); let muted = Style::default().fg(theme.muted); let fg = Style::default().fg(theme.fg); - let id_w = (area.width as usize).saturating_sub(16).max(8); + let id_w = (width as usize).saturating_sub(16).max(8); let container_id = trunc(&inst.container_id, id_w); let partition = inst.partition_info.as_deref().unwrap_or("-"); let quant = inst.quantization.as_deref().unwrap_or("-"); @@ -356,13 +371,16 @@ fn render_summary( Span::styled("tok/W: ", muted), Span::styled( format::tokens_per_watt_cell( - inst.tokens_per_watt, + inst.live_tokens_per_watt(), inst.gen_tps_observation.as_ref(), ), Style::default().fg(theme.accent), ), Span::raw(" "), - Span::styled("gen: ", muted), + Span::styled( + format!("{}: ", format::gen_rate_label(inst.gen_tps_last_request)), + muted, + ), Span::styled( format::gen_tps_compact(inst.gen_tps, inst.gen_tps_observation.as_ref()), fg, @@ -387,7 +405,10 @@ fn render_summary( Span::styled(observed_str, Style::default().fg(theme.muted)), ]), ]; + lines +} +fn render_summary(f: &mut Frame, area: Rect, lines: Vec>) { let p = Paragraph::new(lines).wrap(Wrap { trim: false }); f.render_widget(p, area); } @@ -600,7 +621,8 @@ fn render_footer(f: &mut Frame, area: Rect, inst: &Instance, theme: &Theme, scro mod tests { use super::*; use crate::ui::tabs::instances::tests::{ - SNAP_SECS, buffer_text, mk_inst, mk_inst_obs, mk_snap, mk_state, obs, state_with_snap, + SNAP_SECS, buffer_text, last_req_inst, mk_inst, mk_inst_obs, mk_snap, mk_state, obs, + state_with_snap, }; use chrono::TimeZone; use rocm_dash_core::metrics::{ObservationFreshness, ObservationMetadata}; @@ -1018,6 +1040,56 @@ mod tests { ); } + #[test] + fn detail_names_a_last_request_rate() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for (last_request, label) in [(true, "last req: 37.5 tok/s"), (false, "gen: 37.5 tok/s")] { + let state = state_with_snap(last_req_inst("rate-detail", last_request)); + let mut term = Terminal::new(TestBackend::new(160, 48)).unwrap(); + term.draw(|f| { + draw_detail(f, f.area(), &state, &state.theme); + }) + .unwrap(); + let out = buffer_text(&term); + assert!(out.contains(label), "{label:?} missing:\n{out}"); + assert_eq!(out.contains("last req:"), last_request, "{out}"); + // The value already carries its unit, so the label must not repeat it. + assert!(!out.contains("tok/s:"), "unit doubled:\n{out}"); + } + } + + /// The summary block grows with a wrapped line: at no width may the line that + /// qualifies the rate (freshness and age) fall off the bottom, whatever the + /// rate's label length (`last req:` once pushed it out at 97-102 columns). + #[test] + fn detail_keeps_the_freshness_line_at_every_width() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for last_request in [true, false] { + let mut inst = last_req_inst("a-realistic-container-id-0123456789", last_request); + inst.model_name = "gpt-oss-20b-mxfp4-instruct".into(); + inst.gpu_ids = vec!["0".into(), "1".into()]; + inst.partition_info = Some("SPX/NPS1".into()); + let state = state_with_snap(inst); + for width in 80..=200u16 { + let mut term = Terminal::new(TestBackend::new(width, 48)).unwrap(); + term.draw(|f| { + draw_detail(f, f.area(), &state, &state.theme); + }) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("gen freshness:") && out.contains("observed:"), + "last_request={last_request} width={width}: +{out}" + ); + } + } + } + #[test] fn detail_held_shows_held_freshness_label() { use ratatui::Terminal; @@ -1151,6 +1223,35 @@ mod tests { assert_eq!(rows[1].gen_tps, Some(10.0)); } + /// An idle Lemonade server keeps reporting its last request's rate. That is + /// data (so the panel says "idle", not "no throughput data"), but it is not + /// activity, and must not be listed as an unlabelled tok/s sample. + #[test] + fn a_last_request_rate_is_not_listed_as_recent_activity() { + let mut idle_lemonade = mk_inst("x"); + idle_lemonade.gen_tps = Some(45.0); + idle_lemonade.gen_tps_last_request = true; + let mut history: std::collections::VecDeque<_> = std::collections::VecDeque::new(); + for tick in 0..3 { + let mut snap = mk_snap(vec![idle_lemonade.clone()]); + snap.timestamp = Utc.timestamp_opt(1_700_000_000 + 30 * tick, 0).unwrap(); + history.push_back(snap); + } + assert!(recent_activity_rows(&history, "id-x", RECENT_ACTIVITY_CAP).is_empty()); + assert!( + has_observed_throughput(&history, "id-x"), + "the rate was reported, so the panel reads idle, not no data" + ); + // The same readings as a live rate are activity. + idle_lemonade.gen_tps_last_request = false; + let mut live = std::collections::VecDeque::new(); + live.push_back(mk_snap(vec![idle_lemonade])); + assert_eq!( + recent_activity_rows(&live, "id-x", RECENT_ACTIVITY_CAP).len(), + 1 + ); + } + #[test] fn recent_activity_rows_caps_to_requested_len() { let mut history: std::collections::VecDeque<_> = std::collections::VecDeque::new(); @@ -1529,8 +1630,8 @@ mod tests { fn activity_layout_never_exceeds_budget_and_keeps_newest_rows() { for inner_h in 0..=60u16 { for n in 0..=RECENT_ACTIVITY_CAP { - let (h, k) = activity_layout(n, inner_h); - let avail = inner_h.saturating_sub(4 + 1 + body_floor(inner_h)); + let (h, k) = activity_layout(n, inner_h, SUMMARY_ROWS); + let avail = inner_h.saturating_sub(SUMMARY_ROWS + 1 + body_floor(inner_h)); assert!(h <= avail, "n={n} inner_h={inner_h}: h={h} > {avail}"); assert!(k <= n); if n > 0 && h > 0 { @@ -1543,16 +1644,16 @@ mod tests { } } // Spacious: all five rows. Mid band: partial, but sized to match. - assert_eq!(activity_layout(5, 48).1, 5); - let (h, k) = activity_layout(5, 24); + assert_eq!(activity_layout(5, 48, SUMMARY_ROWS).1, 5); + let (h, k) = activity_layout(5, 24, SUMMARY_ROWS); assert!( (1..5).contains(&k), "mid-height band shows a partial panel: {k}" ); assert!(h > 0); // Placeholder: shown when it fits, collapsed otherwise. - assert_eq!(activity_layout(0, 48), (3, 0)); - assert_eq!(activity_layout(0, 8), (0, 0)); + assert_eq!(activity_layout(0, 48, SUMMARY_ROWS), (3, 0)); + assert_eq!(activity_layout(0, 8, SUMMARY_ROWS), (0, 0)); } #[test] diff --git a/crates/rocm-dash-tui/src/ui/instance_list.rs b/crates/rocm-dash-tui/src/ui/instance_list.rs deleted file mode 100644 index 4f5e55f492..0000000000 --- a/crates/rocm-dash-tui/src/ui/instance_list.rs +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// -// SPDX-License-Identifier: MIT - -//! j/k-navigable instance list with status badges. Stub. diff --git a/crates/rocm-dash-tui/src/ui/job_console.rs b/crates/rocm-dash-tui/src/ui/job_console.rs index 7df8480d25..15cb9d69cf 100644 --- a/crates/rocm-dash-tui/src/ui/job_console.rs +++ b/crates/rocm-dash-tui/src/ui/job_console.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Job console widget (Phase 3 Wave 0). +//! Job console widget. //! //! Renders a [`JobState`] from the reducer: a status header, the streamed //! output ring, and key hints. This is the shared "running job" surface every diff --git a/crates/rocm-dash-tui/src/ui/launcher.rs b/crates/rocm-dash-tui/src/ui/launcher.rs index 346c4c746b..e447ea3413 100644 --- a/crates/rocm-dash-tui/src/ui/launcher.rs +++ b/crates/rocm-dash-tui/src/ui/launcher.rs @@ -321,10 +321,15 @@ fn launcher_state(theme_name: &str, serving: Vec) -> AppState { .iter() .map(|inst| inst.container_id.clone()) .collect(); + // No `instances` here: that field feeds the Windows registry poller + // (`app::event_loop::spawn_managed_registry_poller`), which reads the + // bin-injected live probe rather than this static one, and the launcher + // fills `state.instances` directly on the next line anyway. state.managed_registry_probe = Some(std::sync::Arc::new(StaticManagedRegistrySnapshot( ManagedRegistrySnapshot { has_live_service, managed_ids, + ..Default::default() }, ))); state.instances = serving @@ -879,6 +884,7 @@ mod tests { "vllm-1".into(), Instance { container_id: "vllm-1".into(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -892,6 +898,7 @@ mod tests { ManagedRegistrySnapshot { has_live_service: true, managed_ids: std::collections::HashSet::from(["vllm-1".to_string()]), + ..Default::default() }, ))); s.open_quit_confirm(); diff --git a/crates/rocm-dash-tui/src/ui/logs_view.rs b/crates/rocm-dash-tui/src/ui/logs_view.rs index ff5f40afc1..53f6e71530 100644 --- a/crates/rocm-dash-tui/src/ui/logs_view.rs +++ b/crates/rocm-dash-tui/src/ui/logs_view.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Logs overlay (Phase 3 Wave 3). +//! Logs overlay. //! //! Browses recent ROCm CLI logs via `rocm logs [--search WORDS]` — read-only, so //! no approval gate. An optional search box filters before running. Output diff --git a/crates/rocm-dash-tui/src/ui/mod.rs b/crates/rocm-dash-tui/src/ui/mod.rs index 8604392cc3..367de2f173 100644 --- a/crates/rocm-dash-tui/src/ui/mod.rs +++ b/crates/rocm-dash-tui/src/ui/mod.rs @@ -4,7 +4,6 @@ pub mod approval; pub mod automations_manager; -pub mod bench; pub mod bench_run; pub mod command_screen; pub mod config_manager; @@ -21,7 +20,6 @@ mod header; mod help_modal; pub mod install_manager; mod instance_detail; -pub mod instance_list; pub mod job_console; pub mod launcher; pub mod logs_view; @@ -466,6 +464,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, diff --git a/crates/rocm-dash-tui/src/ui/model_picker.rs b/crates/rocm-dash-tui/src/ui/model_picker.rs index 0072970e3c..aa2c8db33e 100644 --- a/crates/rocm-dash-tui/src/ui/model_picker.rs +++ b/crates/rocm-dash-tui/src/ui/model_picker.rs @@ -2,12 +2,13 @@ // // SPDX-License-Identifier: MIT -//! Model picker (Phase 3 Wave 1). +//! Model picker. //! //! A reusable, filterable list of model recipes — the sub-step the serve wizard //! opens from its Model field so the user can pick a built-in recipe instead of -//! typing a model name. Selecting a recipe fills the model id and (when known) -//! pre-selects the recipe's preferred engine. +//! typing a model name. Selecting a recipe returns its summary; the serve +//! wizard keeps only the id and fills the Model field with it. The recipe's +//! preferred engine is shown in the list but never applied. //! //! Recipes are passed in as plain TUI-local [`ModelRecipeSummary`] values — the //! bin (`apps/rocm`, which has `rocm-core`) adapts the full `ModelRecipeRecord` diff --git a/crates/rocm-dash-tui/src/ui/onboarding.rs b/crates/rocm-dash-tui/src/ui/onboarding.rs index 0afcd56cf8..ea38f4d669 100644 --- a/crates/rocm-dash-tui/src/ui/onboarding.rs +++ b/crates/rocm-dash-tui/src/ui/onboarding.rs @@ -2,19 +2,18 @@ // // SPDX-License-Identifier: MIT -//! Onboarding wizard (Phase 3 Wave 2, minimal). +//! Onboarding wizard (minimal). //! -//! The first-run flow, rebuilt on the Wave-0 primitives. This is the **minimal** -//! version the phase-3 plan calls for: get a clean machine to a working ROCm -//! runtime two ways — +//! The first-run flow, kept deliberately **minimal**: get a clean machine to a +//! working ROCm runtime two ways — //! //! - **Install ROCm SDK** — a one-shot gated `rocm install sdk` with a Configure //! step to pick the channel (Release/Nightly), an optional version pin //! (`--build-date` / `--version`), and an optional install folder -//! (`--prefix`, via the same Wave-0 [`FolderBrowser`] the Adopt path uses); +//! (`--prefix`, via the same [`FolderBrowser`] the Adopt path uses); //! defaults to `--channel release --format wheel` installed into the //! default managed folder. -//! - **Adopt existing folder** — pick an existing ROCm env with the Wave-0 +//! - **Adopt existing folder** — pick an existing ROCm env with the //! [`FolderBrowser`], then approve `rocm runtimes adopt`. //! //! Reinstall / uninstall / show-log sub-modals (the full frozen onboarding), diff --git a/crates/rocm-dash-tui/src/ui/quit_confirm.rs b/crates/rocm-dash-tui/src/ui/quit_confirm.rs index 3e55a03361..a8077c06ba 100644 --- a/crates/rocm-dash-tui/src/ui/quit_confirm.rs +++ b/crates/rocm-dash-tui/src/ui/quit_confirm.rs @@ -88,11 +88,11 @@ fn quit_confirm_body(state: &AppState) -> Vec { quit_confirm_body_with(state, &snapshot) } -/// Body text for the quit-confirm prompt: name the models still serving -/// (mirroring the serving rows of the Serving tab's "Running now" listing in -/// `tabs::pane::live_lines`'s `OpenServices` arm, as plain lines instead of -/// styled spans; that listing also shows starting services, which this gate -/// deliberately does not count), and how to stop each one that this host's own registry +/// Body text for the quit-confirm prompt: name every model still serving, +/// managed or not (the Serving tab's "Managed running" listing in +/// `tabs::pane::live_lines`'s `OpenServices` arm is managed-only, and also shows +/// starting services, which this gate deliberately does not count, so this +/// listing can be longer or shorter than it), and how to stop each one that this host's own registry /// manages (a live read, passed in as `snapshot` so the caller controls how /// many times the registry is actually read — see `quit_confirm_request`) — /// not a blanket command, since a Docker- or Lemonade-discovered instance @@ -140,11 +140,10 @@ pub(super) fn quit_confirm_body_with( let mut body = Vec::new(); if total == 0 { // `total == 0` here only because `nothing_is_currently_being_served_with` - // returned `false` above: either the startup-race fallback (issue - // #145) is active — the daemon's first snapshot hasn't landed yet — - // or the connection never will succeed at all (Windows: the - // dashboard-embedded daemon client's connect path is unix-only, so - // `instances` stays empty for the whole session). `instances` can't + // returned `false` above: the startup-race fallback (issue #145) is + // active — the daemon's first snapshot hasn't landed yet, or (Windows, + // which has no daemon and refills `instances` from the registry each + // tick) a service was registered since the last tick. `instances` can't // name anything in either case, but the live registry read // (`managed_ids`) needs no daemon snapshot — it is this host's own // `rocm serve` registry, read straight off disk — so it can still @@ -190,10 +189,10 @@ pub(super) fn quit_confirm_body_with( // `values()` happened to yield, so the listing is at least stable // across renders. `s` does not focus "the Nth instance named here": // `quit_confirm_open_services_key` always focuses the FIRST managed, - // still-serving row in `service_rows`' own order, which can include - // an unmanaged (Docker/Lemonade) entry this listing also shows ahead - // of it — so the row `s` focuses need not be the Nth line printed - // below. + // still-serving row in `service_rows`' own order, and `service_rows` + // lists managed services only — so the row `s` focuses need not be the + // Nth line printed below, because this listing can also show an + // unmanaged (Docker/Lemonade) entry that the overlay does not. let mut sorted: Vec<_> = running().collect(); sorted.sort_by(|a, b| a.container_id.cmp(&b.container_id)); for i in sorted.into_iter().take(5) { @@ -248,11 +247,10 @@ mod tests { #[test] fn quit_confirm_body_prints_the_stop_command_during_the_startup_fallback_when_the_registry_names_an_id() { - // On a host whose daemon client never sends an instance snapshot - // (Windows — it connects and streams live GPU/CPU/memory every tick, - // but has no `rocmdashd` socket to source an instance list from, so - // `has_received_snapshot` never flips for the whole session), - // `instances` never fills, so without this the quit prompt + // Before `instances` has been filled (the daemon's first snapshot + // has not landed, or on Windows the registry gained a service since + // the last registry poll), `has_received_snapshot` is false and + // `instances` is empty, so without this the quit prompt // could only ever print the generic "a model was serving" sentence — // no model name (expected: `instances` is the only source for that), // but also no stop command and no id, even though the live registry @@ -379,6 +377,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -411,6 +410,41 @@ mod tests { assert_eq!(launcher.title, dashboard.title); } + /// Just started: the registry names the instance but the broadcast has not + /// marked it managed yet. The prompt must not offer `s` (the Services overlay + /// has no row for it); it still prints the command that stops it. + #[test] + fn quit_confirm_request_does_not_offer_s_for_an_instance_the_broadcast_has_not_marked_managed() + { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("t".into(), "default-dark".into()); + state.instances.insert( + "svc-new".to_string(), + Instance { + container_id: "svc-new".to_string(), + model_name: "m".to_string(), + managed: false, + status: InstanceStatus::Running, + ..Default::default() + }, + ); + with_registry_probe(&mut state, true, &["svc-new"]); + let request = quit_confirm_request(&state, true); + assert!( + !request.body.iter().any(|l| l.contains("Press s")), + "{:?}", + request.body + ); + assert!( + request + .body + .iter() + .any(|l| l.contains("rocm services stop svc-new --yes")), + "{:?}", + request.body + ); + } + #[test] fn quit_confirm_request_offers_the_stop_hint_only_for_the_dashboard_with_something_stoppable() { use rocm_dash_core::metrics::{Instance, InstanceStatus}; @@ -419,6 +453,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -481,6 +516,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, model_name: "Qwen2.5-7B".to_string(), status: InstanceStatus::Running, ..Default::default() @@ -563,8 +599,8 @@ mod tests { // `values()` happens to yield. This does not mean `s` // (`AppState::quit_confirm_open_services_key`) focuses the Nth // bullet printed here: it always focuses the FIRST managed, - // still-serving row in `service_rows`' own order, which can include - // an unmanaged entry this listing also shows. + // still-serving row in `service_rows`' own order, and `service_rows` + // lists managed services only. // `quit_confirm_body_caps_the_listing_at_five_with_an_overflow_line` // inserts ids (`svc-0`..`svc-5`) that already sort the same way // whether by id or insertion order, so it can't catch a regression diff --git a/crates/rocm-dash-tui/src/ui/runtime_manager.rs b/crates/rocm-dash-tui/src/ui/runtime_manager.rs index 98d2e443da..dcbacb05c4 100644 --- a/crates/rocm-dash-tui/src/ui/runtime_manager.rs +++ b/crates/rocm-dash-tui/src/ui/runtime_manager.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Runtime manager overlay (Phase 3 Wave 2). +//! Runtime manager overlay. //! //! Lists the registered ROCm runtimes (plumbed in from the bin, which owns //! `rocm-core` — this crate stays core-free) and runs the @@ -12,7 +12,7 @@ //! job-bridge. //! - **activate / rollback / uninstall** — mutating, routed through the approval //! gate first. -//! - **adopt** — pick an existing ROCm env folder with the Wave-0 +//! - **adopt** — pick an existing ROCm env folder with the //! [`FolderBrowser`], then approve `rocm runtimes adopt`. //! - **import** — type a manifest path, then approve `rocm runtimes import`. //! diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 2ea58fde01..172dbc8099 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -2,13 +2,13 @@ // // SPDX-License-Identifier: MIT -//! Serve wizard overlay (Phase 3 Wave 1). +//! Serve wizard overlay. //! -//! The headline operational screen rebuilt on the Wave-0 primitives: a compact -//! **model-first** form that builds a `rocm serve …` invocation and runs it -//! **through the approval gate and the job-bridge** — never inline, never with -//! a legacy `std::thread::spawn` + `try_recv`. A served model launched here -//! surfaces in the services manager and the dashboard's live `gen_tps`. +//! The headline operational screen: a compact **model-first** form that builds +//! a `rocm serve …` invocation and runs it **through the approval gate and the +//! job-bridge** — never inline, never with a legacy `std::thread::spawn` + +//! `try_recv`. A served model launched here surfaces in the services manager +//! and the dashboard's live `gen_tps`. //! //! Progressive disclosure: the default form is Model → `Advanced settings` → //! Launch. Everything else (engine, device policy, host, port, mode) is @@ -19,7 +19,7 @@ //! policy, the loopback host, the port, and managed mode. //! //! The model field can be typed directly (a recipe name, alias, or path) or -//! filled from the reusable Wave-0 [`FolderBrowser`] for a local model path +//! filled from the reusable [`FolderBrowser`] for a local model path //! (`Tab` on the Model field). The approval *decision* is the user's, captured //! by the render+event seam; the CLI owns the actual launch — the read-only //! chat invariant is untouched. diff --git a/crates/rocm-dash-tui/src/ui/services_manager.rs b/crates/rocm-dash-tui/src/ui/services_manager.rs index 24441b3b7d..4c6f429b91 100644 --- a/crates/rocm-dash-tui/src/ui/services_manager.rs +++ b/crates/rocm-dash-tui/src/ui/services_manager.rs @@ -2,13 +2,12 @@ // // SPDX-License-Identifier: MIT -//! Services manager overlay (Phase 3 Wave 1). +//! Services manager overlay. //! -//! The first operational screen rebuilt on the Wave-0 primitives: it lists the -//! managed inference services the daemon surfaces (model · port · status · live -//! `gen_tps`) and runs stop/restart **through the approval gate and the -//! job-bridge** — never inline. This is the pattern the remaining Wave-1 screens -//! (serve_wizard / model_picker / engine_manager) reuse. +//! It lists the managed inference services the daemon surfaces (model · port · +//! status · live `gen_tps`, or the last request's rate) and runs stop/restart +//! **through the approval gate and the job-bridge** — never inline. The serve wizard and the engine manager +//! follow the same pattern. //! //! Mutating actions invoke the CLI (`rocm services stop|restart --yes`) as //! a background job; the approval *decision* is the user's, captured by the @@ -91,14 +90,24 @@ pub struct ServiceRow { pub gen_tps: Option, /// Freshness metadata for `gen_tps`; `None` for legacy snapshots. pub gen_tps_observation: Option, + /// `gen_tps` is only the endpoint's most recent request, not a live rate. + pub gen_tps_last_request: bool, } -/// Build the sorted service list from the daemon-surfaced instances. +/// The one ownership predicate for lifecycle surfaces: only registry-owned +/// instances are listed and counted, because those are what the controls act on. +#[must_use] +pub const fn is_listed_service(i: &Instance) -> bool { + i.managed +} + +/// Build the sorted lifecycle list from registry-owned instances only. pub fn service_rows( instances: &HashMap, ) -> Vec { let mut rows: Vec = instances .values() + .filter(|i| is_listed_service(i)) .map(|i| ServiceRow { id: i.container_id.clone(), model: i.model_name.clone(), @@ -106,6 +115,7 @@ pub fn service_rows( status: i.status, gen_tps: i.gen_tps, gen_tps_observation: i.gen_tps_observation.clone(), + gen_tps_last_request: i.gen_tps_last_request, }) .collect(); rows.sort_by(|a, b| a.id.cmp(&b.id)); @@ -128,6 +138,20 @@ pub fn on_key( return Vec::new(); }; + // Discovery may remove the target or replace it with an external row while + // the modal is open. Revalidate ownership before handling any verdict. + // The key is not consumed: it falls through and is handled as a normal key. + if sm + .approval + .as_ref() + .is_some_and(|pending| !rows.iter().any(|row| row.id == pending.service_id)) + { + sm.approval = None; + } + + // Keep navigation aligned with the refreshed managed-only list. + sm.selected = sm.selected.min(rows.len().saturating_sub(1)); + // 1) Approval modal has focus. if let Some(pending) = sm.approval.as_mut() { let (choice, verdict) = approval_key(key.code, pending.choice); @@ -310,19 +334,24 @@ pub fn draw_services_manager( // it here introduces no circular dependency. let max_visible = inner.height.saturating_sub(1 + note_rows) as usize; let window = visible_list_window(selected, rows.len(), max_visible); - let any_held = rows.get(window).into_iter().flatten().any(|r| { + let any_held = rows.get(window.clone()).into_iter().flatten().any(|r| { r.gen_tps.is_some_and(f64::is_finite) && r.gen_tps_observation .as_ref() .is_some_and(|m| m.freshness == ObservationFreshness::Held) }); + let any_last_req = rows + .get(window) + .into_iter() + .flatten() + .any(|r| r.gen_tps_last_request && r.gen_tps.is_some_and(f64::is_finite)); let body = Layout::default() .direction(Direction::Vertical) .constraints([ Constraint::Min(1), Constraint::Length(note_rows), Constraint::Length(1), - Constraint::Length(u16::from(any_held)), + Constraint::Length(u16::from(any_held || any_last_req)), ]) .split(inner); @@ -360,7 +389,8 @@ pub fn draw_services_manager( ), Span::styled( format!( - "gen {}", + "{} {}", + format::gen_rate_label_short(r.gen_tps_last_request), format::gen_tps_compact(r.gen_tps, r.gen_tps_observation.as_ref()) ), Style::default().fg(theme.fg), @@ -404,10 +434,16 @@ pub fn draw_services_manager( body[2], ); - if any_held { + if any_held || any_last_req { f.render_widget( Paragraph::new(Line::from(Span::styled( - format::HELD_LEGEND, + match (any_held, any_last_req) { + (true, true) => { + format!("{} · {}", format::HELD_LEGEND, format::LAST_REQ_LEGEND) + } + (true, false) => format::HELD_LEGEND.to_owned(), + _ => format::LAST_REQ_LEGEND.to_owned(), + }, Style::default().fg(theme.muted), ))), body[3], @@ -415,7 +451,9 @@ pub fn draw_services_manager( } // Approval modal sits on top of the list. - if let Some(pending) = &sm.approval { + if let Some(pending) = &sm.approval + && rows.iter().any(|row| row.id == pending.service_id) + { draw_approval(f, area, &pending.request, pending.choice, theme); } } @@ -438,6 +476,7 @@ mod tests { fn inst(id: &str, model: &str, tps: Option) -> Instance { Instance { + managed: true, container_id: id.into(), container_name: id.into(), model_name: model.into(), @@ -455,6 +494,197 @@ mod tests { m } + #[test] + fn external_only_lifecycle_keys_do_not_offer_approval_or_spawn_jobs() { + let mut external = inst("svc-external", "external-model", Some(12.0)); + external.managed = false; + let insts = HashMap::from([(external.container_id.clone(), external)]); + for action in ['s', 'r'] { + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + for pressed in [action, 'y'] { + let fx = on_key( + &mut services, + &mut jobs, + &insts, + key(KeyCode::Char(pressed)), + ); + assert!(fx.is_empty(), "external service must never emit effects"); + let sm = services.as_ref().unwrap(); + assert!(sm.approval.is_none(), "external service is read-only"); + assert!(sm.active_job.is_none()); + assert!(jobs.jobs.is_empty()); + } + let out = render(services.as_ref().unwrap(), &jobs, &insts); + assert!(out.contains("No managed services."), "{out}"); + assert!(!out.contains("external-model"), "{out}"); + } + } + + #[test] + fn rows_use_positive_ownership_not_id_or_engine_markers() { + let mut insts = instances(); + let mut external = inst("svc-looks-managed", "external", None); + external.managed = false; + insts.insert("external".into(), external); + let mut managed = inst("lemonade-owned", "managed", None); + managed.container_name = "lemonade".into(); + insts.insert("managed".into(), managed); + let rows = service_rows(&insts); + assert_eq!( + rows.iter().map(|r| r.id.as_str()).collect::>(), + vec!["lemonade-owned", "svc-a", "svc-b"] + ); + let out = render(&ServicesManagerState::default(), &State::default(), &insts); + assert!(out.contains("lemonade-owned"), "{out}"); + assert!(!out.contains("svc-looks-managed"), "{out}"); + } + + #[test] + fn removed_or_unmanaged_target_invalidates_pending_approval() { + for action in ['s', 'r'] { + for replace_with_external in [false, true] { + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + let mut telemetry = State::default(); + let managed = inst("svc-a", "managed", None); + telemetry.apply(StateEvent::InstanceUpserted(managed.clone())); + on_key( + &mut services, + &mut jobs, + &telemetry.instances, + key(KeyCode::Char(action)), + ); + assert!(services.as_ref().unwrap().approval.is_some()); + telemetry.apply(StateEvent::InstanceRemoved("svc-a".into())); + if replace_with_external { + let mut external = managed; + external.managed = false; + telemetry.apply(StateEvent::InstanceUpserted(external)); + } + let out = render(services.as_ref().unwrap(), &jobs, &telemetry.instances); + assert!(out.contains("No managed services."), "{out}"); + assert!( + !out.contains("Review:"), + "stale approval must not render: {out}" + ); + let fx = on_key( + &mut services, + &mut jobs, + &telemetry.instances, + key(KeyCode::Char('y')), + ); + assert!(fx.is_empty(), "stale approval must not emit effects"); + assert!(services.as_ref().unwrap().approval.is_none()); + assert!(services.as_ref().unwrap().active_job.is_none()); + assert!(jobs.jobs.is_empty()); + } + } + } + + #[test] + fn stale_approval_does_not_swallow_the_next_key() { + // `j` after the stale approval is dropped moves the selection. + let mut insts = instances(); + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + assert_eq!( + services + .as_ref() + .unwrap() + .approval + .as_ref() + .unwrap() + .service_id, + "svc-a" + ); + // The target disappears while the modal is open. + insts.remove("a"); + insts.insert("c".into(), inst("svc-c", "other", None)); + insts.insert("d".into(), inst("svc-d", "other", None)); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('j'))); + assert!(fx.is_empty()); + let sm = services.as_ref().unwrap(); + assert!(sm.approval.is_none()); + assert_eq!(sm.selected, 1, "j must be handled as navigation"); + + // `s` after a stale approval stages a fresh approval for the selection. + let mut services = Some(ServicesManagerState::default()); + let mut insts = instances(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + insts.remove("a"); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + assert!(fx.is_empty()); + let pending = services.as_ref().unwrap().approval.as_ref().unwrap(); + assert_eq!(pending.service_id, "svc-b"); + // `y` still approves only the fresh target, never the stale one. + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('y'))); + assert!(!fx.is_empty()); + assert_eq!(jobs.jobs.len(), 1); + assert!( + jobs.jobs.contains_key("svc-stop-svc-b"), + "{:?}", + jobs.jobs.keys() + ); + } + + #[test] + fn stale_approval_then_y_cannot_spawn_a_job() { + let mut insts = instances(); + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + insts.clear(); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('y'))); + assert!(fx.is_empty()); + assert!(jobs.jobs.is_empty()); + assert!(services.as_ref().unwrap().approval.is_none()); + } + + #[test] + fn selection_is_clamped_when_managed_rows_shrink() { + let mut insts = instances(); + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('j'))); + assert_eq!(services.as_ref().unwrap().selected, 1); + // Only one managed row remains; the old selection is out of range. + insts.remove("b"); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + let sm = services.as_ref().unwrap(); + assert_eq!(sm.selected, 0); + let pending = sm.approval.as_ref().expect("a valid target is staged"); + assert_eq!(pending.service_id, "svc-a"); + } + + #[test] + fn restart_requires_approval_then_spawns_job() { + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + let insts = instances(); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('r'))); + assert!(fx.is_empty()); + assert!(jobs.jobs.is_empty()); + let sm = services.as_ref().unwrap(); + assert_eq!( + sm.approval.as_ref().unwrap().action, + LifecycleAction::Restart + ); + let out = render(sm, &jobs, &insts); + assert!(out.contains("restart service"), "{out}"); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('y'))); + assert_eq!(fx.len(), 1); + assert!(matches!(&fx[0], SideEffect::SpawnJob { args, .. } + if args == &["services", "restart", "svc-a", "--yes"])); + assert!(services.as_ref().unwrap().approval.is_none()); + assert_eq!( + services.as_ref().unwrap().active_job.as_deref(), + Some("svc-restart-svc-a") + ); + assert_eq!(jobs.jobs.len(), 1); + } + #[test] fn rows_are_sorted_by_id() { let rows = service_rows(&instances()); @@ -657,6 +887,63 @@ mod tests { assert!(out.contains("s stop"), "lifecycle hints"); } + #[test] + fn snapshot_labels_a_last_request_rate() { + let mut insts = HashMap::new(); + for (id, last_request) in [("live", false), ("idle", true)] { + let mut i = inst(id, &format!("model-{id}"), Some(42.0)); + i.gen_tps_last_request = last_request; + insts.insert(id.into(), i); + } + let rows = service_rows(&insts); + assert!(rows[0].gen_tps_last_request && !rows[1].gen_tps_last_request); + let out = render(&ServicesManagerState::default(), &State::default(), &insts); + assert!(out.contains("req 42"), "{out}"); + assert!(out.contains("gen 42"), "{out}"); + assert!(out.contains(format::LAST_REQ_LEGEND), "{out}"); + } + + /// The rate shares `gen`'s column budget, so at the widths where the `gen` + /// value fits, the last-request value fits too (it used to be clipped to a + /// different number, `last req 1` for 137.5). + #[test] + fn a_last_request_rate_is_not_clipped_where_a_live_rate_fits() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + let theme = Theme::from_name("default-dark"); + for last_request in [false, true] { + let mut i = inst("svc", "model-svc", Some(137.5)); + i.gen_tps_last_request = last_request; + let insts = HashMap::from([("svc".to_string(), i)]); + let mut term = Terminal::new(TestBackend::new(98, 28)).unwrap(); + term.draw(|f| { + draw_services_manager( + f, + f.area(), + &ServicesManagerState::default(), + &insts, + &State::default(), + 0, + &theme, + ); + }) + .unwrap(); + let out: String = term + .backend() + .buffer() + .content() + .iter() + .map(ratatui::buffer::Cell::symbol) + .collect(); + let label = format::gen_rate_label_short(last_request); + assert!( + out.contains(&format!("{label} 137.5")), + "last_request={last_request}: +{out}" + ); + } + } + #[test] fn snapshot_shows_held_legend_when_a_row_is_held() { use rocm_dash_core::metrics::{InstanceStatus, ObservationFreshness, ObservationMetadata}; @@ -664,6 +951,7 @@ mod tests { insts.insert( "held".into(), Instance { + managed: true, container_id: "held".into(), container_name: "svc-held".into(), model_name: "llama3".into(), @@ -719,6 +1007,7 @@ mod tests { m.insert( id.clone(), Instance { + managed: true, container_id: id.clone(), container_name: id, model_name: "llama3".into(), diff --git a/crates/rocm-dash-tui/src/ui/sparkline.rs b/crates/rocm-dash-tui/src/ui/sparkline.rs index abadeb933a..aa2499a849 100644 --- a/crates/rocm-dash-tui/src/ui/sparkline.rs +++ b/crates/rocm-dash-tui/src/ui/sparkline.rs @@ -48,8 +48,9 @@ impl<'a> BrailleSparkline<'a> { /// Color each character cell by the larger of its two samples, mapped /// across the gradient (low value = stops[0], peak = stops[2]). When - /// set, overrides `style.fg` per cell. Leaves `style.fg` as the fallback - /// for empty/zero cells. + /// set, overrides `style.fg` per cell, including the zero-value baseline + /// so an idle graph reads as one line. Leaves `style.fg` as the fallback + /// for cells with no sample at all. #[must_use] pub const fn gradient(mut self, start: Color, mid: Color, end: Color) -> Self { self.gradient = Some([start, mid, end]); @@ -66,24 +67,28 @@ impl Widget for BrailleSparkline<'_> { let rows = area.height as usize; let total_dot_rows = rows * 4; - // Right-align: show the most recent `cols * 2` samples. + // Right-align: show the most recent `cols * 2` samples. When the + // history is shorter than that, offset it to the right edge so the + // graph grows leftwards instead of clinging to the left border. let bins = cols * 2; let start = self.data.len().saturating_sub(bins); let slice = &self.data[start..]; + let x_off = cols.saturating_sub(slice.len().div_ceil(2)); - for cx in 0..cols { - let left = slice.get(cx * 2).copied(); - let right = slice.get(cx * 2 + 1).copied(); + for cx in x_off..cols { + let i = (cx - x_off) * 2; + let left = slice.get(i).copied(); + let right = slice.get(i + 1).copied(); // Color the whole character column by the larger of its two // samples — peaks render with the gradient end color even when // their neighbor is low. let cell_value = left.unwrap_or(0).max(right.unwrap_or(0)); let style = match self.gradient { - Some(stops) if cell_value > 0 => { + Some(stops) => { let t = cell_value as f64 / self.max as f64; self.style.fg(lerp3_t(stops, t)) } - _ => self.style, + None => self.style, }; for cy in 0..rows { let row_top = cy * 4; @@ -101,13 +106,20 @@ impl Widget for BrailleSparkline<'_> { /// Returns a 4-bit mask of which dots in this character row are lit /// (bit 0 = top dot in the cell, bit 3 = bottom). +/// +/// A cell with no sample draws nothing. Every sample that exists draws at +/// least the bottom dot row, so the graph is a continuous baseline that rises +/// under load rather than scattered fragments: at the panel sizes we use, a +/// few percent of `max` otherwise rounds to zero dots and vanishes. fn lit_dots(value: Option, max: u64, total_dot_rows: usize, row_top: usize) -> u8 { let Some(v) = value else { return 0; }; let v = v.min(max); // Fill from the bottom. A value at `max` lights every dot row. - let lit = ((v as f64 / max as f64) * total_dot_rows as f64).round() as usize; + let lit = ((v as f64 / max as f64) * total_dot_rows as f64) + .round() + .max(1.0) as usize; let first_lit_row = total_dot_rows.saturating_sub(lit); let mut mask = 0u8; @@ -189,9 +201,17 @@ mod tests { } #[test] - fn zero_value_lights_nothing() { - let m = lit_dots(Some(0), 100, 4, 0); - assert_eq!(m, 0); + fn zero_value_lights_the_baseline_dot_row() { + // A present sample always draws the bottom dot row (bit 3) so the + // graph stays continuous at idle; only an absent sample is blank. + assert_eq!(lit_dots(Some(0), 100, 4, 0), 0b1000); + assert_eq!(lit_dots(None, 100, 4, 0), 0); + } + + #[test] + fn sub_dot_value_still_draws_the_baseline() { + // 2.8% of a 3-row cell rounds to 0 dots; it must not vanish. + assert_eq!(lit_dots(Some(2), 100, 12, 8), 0b1000); } #[test] @@ -222,6 +242,73 @@ mod tests { assert_eq!(fg, Some(stops.2)); } + #[test] + fn short_history_renders_against_the_right_edge() { + // A history shorter than the area is the normal case for the first + // minutes of a session. It must grow leftwards from the right edge, + // not sit in a clump on the left with dead space beside it. + let data = vec![100u64; 4]; // 4 samples = 2 character columns + let mut buf = Buffer::empty(Rect::new(0, 0, 10, 1)); + BrailleSparkline::new(&data) + .max(100) + .render(buf.area, &mut buf); + let drawn: Vec = (0..10) + .map(|x| buf.cell((x, 0)).unwrap().symbol() != " ") + .collect(); + assert_eq!( + drawn, + [ + false, false, false, false, false, false, false, false, true, true + ] + ); + } + + #[test] + fn odd_sample_count_keeps_its_last_half_column() { + // 3 samples occupy 1.5 character columns. Rounding the offset down + // instead of up would drop the newest sample off the right edge, so + // this pins `div_ceil` rather than `/ 2`. + let data = vec![100u64; 3]; + let mut buf = Buffer::empty(Rect::new(0, 0, 10, 1)); + BrailleSparkline::new(&data) + .max(100) + .render(buf.area, &mut buf); + let drawn: Vec = (0..10) + .map(|x| buf.cell((x, 0)).unwrap().symbol() != " ") + .collect(); + assert_eq!( + drawn, + [ + false, false, false, false, false, false, false, false, true, true + ] + ); + // The last column carries one sample, so only its left dots are lit. + assert_eq!(buf.cell((9, 0)).unwrap().symbol(), "\u{2847}"); + } + + #[test] + fn the_baseline_cell_takes_the_gradient_start_color() { + // The zero-value baseline is part of the graph, so it must carry the + // gradient's low color rather than falling through to `style.fg`. + // Without this a `cell_value > 0` guard on the gradient would pass. + let data = vec![0u64]; + let mut buf = Buffer::empty(Rect::new(0, 0, 1, 1)); + let (low, mid, high) = ( + Color::Rgb(10, 0, 0), + Color::Rgb(0, 20, 0), + Color::Rgb(0, 0, 30), + ); + BrailleSparkline::new(&data) + .max(100) + .style(Style::default().fg(Color::Rgb(99, 99, 99))) + .gradient(low, mid, high) + .render(buf.area, &mut buf); + assert_eq!(buf.cell((0, 0)).unwrap().style().fg, Some(low)); + // One sample fills only the cell's left dot column; its right half has + // no sample at all, so those dots stay dark. + assert_eq!(buf.cell((0, 0)).unwrap().symbol(), "\u{2840}"); + } + #[test] fn gradient_disabled_uses_flat_style() { use ratatui::buffer::Buffer; diff --git a/crates/rocm-dash-tui/src/ui/tabs/bench.rs b/crates/rocm-dash-tui/src/ui/tabs/bench.rs index d948db8bf3..77247edc5a 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/bench.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/bench.rs @@ -315,19 +315,44 @@ fn sparkline_max(data: &[u64]) -> u64 { } } +/// The prompt_tps series and its panel title. +/// +/// Rows without a measurement are left out of the series rather than folded in +/// as zero: the widget draws a baseline for every sample it is handed, so a +/// `unwrap_or(0.0)` would render "not measured" as a measured zero. The title +/// then says how many of the rows actually carry one. +/// +/// Shared with the tests so they exercise this, rather than a copy of it. +/// +/// Takes an iterator rather than a slice: `bench_rows` is a `VecDeque` that +/// evicts with `pop_front`, so once it has wrapped it is physically split and +/// `as_slices().0` holds only the older half. A slice parameter quietly invited +/// that call and dropped the newest rows from both the series and the count. +fn prompt_tps_series<'a, I>(rows: I) -> (Vec, String) +where + I: IntoIterator, +{ + let mut n = 0usize; + let data: Vec = rows + .into_iter() + .inspect(|_| n += 1) + .filter_map(|r| r.prompt_tps) + .map(|v| v.max(0.0) as u64) + .collect(); + let title = if data.len() == n { + format!("prompt_tps · last {n} rows") + } else { + format!("prompt_tps · {} of last {n} rows", data.len()) + }; + (data, title) +} + fn draw_sparkline(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { - let n = state.bench_rows.len(); - let title = format!("prompt_tps · last {n} rows"); + let (data, title) = prompt_tps_series(state.bench_rows.iter()); let inner = panel::bento(f, area, Some(&title), BoxRole::Success, false, theme); if inner.height == 0 || inner.width == 0 { return; } - - let data: Vec = state - .bench_rows - .iter() - .map(|r| r.prompt_tps.unwrap_or(0.0).max(0.0) as u64) - .collect(); let max = sparkline_max(&data); // Higher prompt_tps is better, so use a "cool" gradient that ramps from // muted accent up through bright accent into ok-green for peak values — @@ -360,6 +385,114 @@ mod tests { } } + #[test] + fn unmeasured_rows_stay_out_of_the_series_and_the_title_says_so() { + // Exercises `prompt_tps_series`, the same function `draw_sparkline` + // uses, so reverting it to `unwrap_or(0.0)` turns this red. + // The widget draws a baseline for every sample it is handed, so a row + // with no prompt_tps must not reach it as a zero: "not measured" and + // "measured 0 tok/s" would become the same picture. + let rows = [ + BenchmarkRow { + prompt_tps: Some(120.0), + ..Default::default() + }, + BenchmarkRow { + prompt_tps: None, + ..Default::default() + }, + BenchmarkRow { + prompt_tps: Some(90.0), + ..Default::default() + }, + ]; + let (data, title) = prompt_tps_series(rows.iter()); + assert_eq!(data, vec![120, 90], "an unmeasured row reached the widget"); + assert_eq!(title, "prompt_tps · 2 of last 3 rows"); + + // With every row measured the title drops the qualifier. + let all_measured: Vec = (0..3) + .map(|_| BenchmarkRow { + prompt_tps: Some(50.0), + ..Default::default() + }) + .collect(); + let (data, title) = prompt_tps_series(all_measured.iter()); + assert_eq!(data, vec![50, 50, 50]); + assert_eq!(title, "prompt_tps · last 3 rows"); + } + + #[test] + fn the_drawn_panel_counts_a_wrapped_deque_in_full() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + // Guards the call site, not just the helper: a unit test on + // `prompt_tps_series` cannot see what `draw_sparkline` hands it, so + // reinstating `as_slices().0` there would leave that test green. + let mut state = AppState::new("t".into(), "default-dark".into()); + // Built at a small capacity so it is physically split after a few + // evictions; `AppState`'s own deque reserves BENCH_CAP and would stay + // contiguous until hundreds of rows had gone through it. + let mut rows: std::collections::VecDeque = + std::collections::VecDeque::with_capacity(4); + for i in 0..7u32 { + if rows.len() == 4 { + rows.pop_front(); + } + rows.push_back(BenchmarkRow { + prompt_tps: Some(f64::from(i) * 10.0), + ..Default::default() + }); + } + assert!( + !rows.as_slices().1.is_empty(), + "fixture no longer wraps, so it cannot catch the bug" + ); + state.bench_rows = rows; + let backend = TestBackend::new(60, 8); + let mut term = Terminal::new(backend).unwrap(); + let theme = state.theme; + term.draw(|f| draw_sparkline(f, f.area(), &state, &theme)) + .unwrap(); + let screen: String = term + .backend() + .buffer() + .content() + .iter() + .map(ratatui::buffer::Cell::symbol) + .collect(); + assert!( + screen.contains("last 4 rows"), + "panel did not count the whole deque: {screen}" + ); + } + + #[test] + fn a_wrapped_deque_still_contributes_its_newest_rows() { + use std::collections::VecDeque; + // `bench_rows` evicts with `pop_front`, so a long session leaves the + // deque physically split. Reading only `as_slices().0` keeps the older + // half and silently drops the newest rows, which is exactly the half a + // "last N rows" panel exists to show. + let mut rows: VecDeque = VecDeque::with_capacity(4); + for i in 0..7u32 { + if rows.len() == 4 { + rows.pop_front(); + } + rows.push_back(BenchmarkRow { + prompt_tps: Some(f64::from(i) * 10.0), + ..Default::default() + }); + } + assert!( + !rows.as_slices().1.is_empty(), + "fixture no longer wraps, so it cannot catch the bug" + ); + let (data, title) = prompt_tps_series(rows.iter()); + assert_eq!(data, vec![30, 40, 50, 60], "newest rows were dropped"); + assert_eq!(title, "prompt_tps · last 4 rows"); + } + #[test] fn cfg_token_renders_tp_and_dtype() { assert_eq!(cfg_token(&group("A", Some(4), Some("fp8"))), "4·fp8"); diff --git a/crates/rocm-dash-tui/src/ui/tabs/chat.rs b/crates/rocm-dash-tui/src/ui/tabs/chat.rs index ffb32550a3..a3a1736fe8 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/chat.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/chat.rs @@ -4,9 +4,11 @@ //! Chat tab — a TUI-local conversation surface. //! -//! Phase 1 is render-only with a local echo backend; later phases wire a Rig-built agent behind the -//! `AgentClient` trait. The transcript and input buffer are plain TUI state on -//! `AppState` (`rocm-dash-core` carries no chat types). +//! Until the user accepts the detected endpoint (`ChatConsent`), the tab shows a +//! consent gate or an empty state; once accepted it draws the transcript and +//! the input line of the real chat, whose backend sits behind the `AgentClient` +//! trait. The transcript and input buffer are plain TUI state on `AppState` +//! (`rocm-dash-core` carries no chat types). use ratatui::Frame; use ratatui::layout::{Constraint, Direction, Layout, Rect}; diff --git a/crates/rocm-dash-tui/src/ui/tabs/hardware.rs b/crates/rocm-dash-tui/src/ui/tabs/hardware.rs index 2474b32437..242fcacab1 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/hardware.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/hardware.rs @@ -5,8 +5,9 @@ //! Hardware Observe sub-panel — full-screen detail of host CPU/memory + per-GPU panels. //! //! Layout (vertical): -//! 1. CPU panel (~10 rows): aggregate braille sparkline over history plus -//! per-core bars at the bottom. +//! 1. CPU (~10 rows): two boxes. Overall utilization over time, then one bar +//! per core right now. They are different chart types on different axes, so +//! they get their own borders and titles rather than sharing a panel. //! 2. Memory + Swap (~3 rows): two side-by-side gauges. //! 3. Per-GPU panels (remaining): one bordered block per GPU with stats, //! firmware/partition info, and a util sparkline. @@ -26,10 +27,29 @@ use crate::ui::gradient::GradientGauge; use crate::ui::panel::{self, BoxRole}; use crate::ui::sparkline::BrailleSparkline; use crate::ui::theme::Theme; -use crate::ui::widgets::{ - gpu_stats_line, instances_on_gpu, node_efficiency, power_style, temperature_style, trunc, -}; - +use crate::ui::widgets::{gpu_stats_line, instances_on_gpu, power_style, temperature_style, trunc}; + +/// Height of the overall-CPU box when the two CPU boxes are stacked: border +/// (2) + `panel::padding_for`'s top pad (1, taken at height >= 5) + sparkline +/// (3). +/// +/// Two bordered boxes spend five of the slot's ten rows on chrome, so the +/// charts share the remaining five and neither can have what it had when they +/// shared one box (4 rows of graph, 3 of bars). This 3/2 split loses it evenly. +/// The alternative, 4 rows of graph and 1 of bars, quantises the strip at 25% +/// per dot, and stacking is not rare enough to accept that: a 120-column +/// terminal with 32 cores leaves 55 columns a side, short of the 64 the strip +/// needs, so an ordinary terminal stacks. +const CPU_OVERALL_H: u16 = 6; +/// Height of the per-core box when stacked: border (2) + bars (2). Below the +/// height-5 threshold, so this box takes no top pad. +const CPU_CORES_H: u16 = 4; +/// Columns a `panel::bento` box spends on chrome: border (2) + left pad (2) + +/// right pad (1), per `panel::padding_for`. +const PANEL_CHROME_W: u16 = 5; +/// Narrowest history graph worth splitting the CPU slot for: 30 columns is 60 +/// samples, about a minute of trend at the daemon's 1 Hz. +const MIN_GRAPH_W: u16 = 30; /// Height of the GPU-section summary header (partition + efficiency). const GPU_HEADER_H: u16 = 1; /// Minimum slot height for a "full" GPU panel: border (2) + stats (1) + @@ -116,50 +136,118 @@ fn draw_rate_block( f.render_widget(Paragraph::new(line), inner); } +/// CPU: overall utilization over time, and per-core load right now. +/// +/// These are two different charts. The sparkline's x axis is time (newest at +/// the right); the core bars' x axis is core index and carries no history at +/// all. Stacked bare inside one box they read as a single broken graph, so +/// each gets its own bordered box, its own title, and a role color of its own. fn draw_cpu(f: &mut Frame, area: Rect, state: &AppState, snap: &Snapshot, theme: &Theme) { let n_cores = snap.host.cpu_per_core_pct.len(); - let title = format!( - "CPU · {} · {} cores", - format::pct(snap.host.cpu_overall_pct), - n_cores - ); - let inner = panel::bento(f, area, Some(&title), BoxRole::Secondary, false, theme); - if inner.height == 0 { + if n_cores == 0 { + draw_cpu_overall(f, area, state, snap, theme, false); + return; + } + // Side by side keeps both boxes the full height of the slot, so each chart + // gets about twice the dot rows stacking leaves it, and the GPU section + // below keeps all of its rows. Stacking is the fallback for widths where + // halving would starve one of the two. + if cpu_fits_side_by_side(area, n_cores) { + let cols = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Ratio(1, 2), Constraint::Ratio(1, 2)]) + .split(area); + draw_cpu_overall(f, cols[0], state, snap, theme, false); + draw_cpu_cores(f, cols[1], snap, theme); return; } + if area.height < CPU_OVERALL_H + CPU_CORES_H { + // Too short for two boxes: keep the one carrying history, since the + // per-core bars only ever show the current instant. The core count + // moves into its title so the figure survives the dropped box. + draw_cpu_overall(f, area, state, snap, theme, true); + return; + } + let rows = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(CPU_OVERALL_H), + Constraint::Length(CPU_CORES_H), + ]) + .split(area); + draw_cpu_overall(f, rows[0], state, snap, theme, false); + draw_cpu_cores(f, rows[1], snap, theme); +} - let split = if n_cores == 0 || inner.height < 4 { - Layout::default() - .direction(Direction::Vertical) - .constraints([Constraint::Min(1)]) - .split(inner) +/// Whether the CPU slot is wide enough to put the two boxes side by side. +/// +/// Each half must leave the graph enough width to read as a trend rather than +/// a stub, and the core strip **two** columns per core. One per core would fit +/// the bars but not their gutters (see `core_bars::slot`), so the strip would +/// come out as one fused block; a strip that cannot be read as separate cores +/// is not worth halving the graph for. +/// +/// The core count rules this out more often than the terminal width does: at +/// 120 columns a 32-core host already falls short (55 a side against the 64 it +/// needs), so stacking is the ordinary case on a laptop-sized terminal rather +/// than a narrow-terminal fallback. +const fn cpu_fits_side_by_side(area: Rect, n_cores: usize) -> bool { + let half_inner = (area.width / 2).saturating_sub(PANEL_CHROME_W); + half_inner >= MIN_GRAPH_W && half_inner as usize >= 2 * n_cores +} + +/// Overall CPU utilization over the snapshot history. +/// +/// `with_core_count` folds the core count into this box's title for the +/// degenerate layout where the per-core box is dropped, so the count the old +/// combined title always carried is not simply lost. +fn draw_cpu_overall( + f: &mut Frame, + area: Rect, + state: &AppState, + snap: &Snapshot, + theme: &Theme, + with_core_count: bool, +) { + let pct = format::pct(snap.host.cpu_overall_pct); + let title = if with_core_count { + format!("CPU · {pct} · {} cores", snap.host.cpu_per_core_pct.len()) } else { - let core_rows = 3u16.min(inner.height.saturating_sub(2)); - let agg_rows = inner.height.saturating_sub(core_rows).max(1); - Layout::default() - .direction(Direction::Vertical) - .constraints([Constraint::Length(agg_rows), Constraint::Length(core_rows)]) - .split(inner) + format!("CPU · {pct}") }; - + let inner = panel::bento(f, area, Some(&title), BoxRole::Secondary, false, theme); + if inner.height == 0 { + return; + } let data: Vec = state .history .iter() .map(|s| s.host.cpu_overall_pct.clamp(0.0, 100.0) as u64) .collect(); - let spark = BrailleSparkline::new(&data) - .max(100) - .style(Style::default().fg(theme.accent)) - .gradient(theme.ok, theme.warn, theme.err); - f.render_widget(spark, split[0]); - - if split.len() == 2 { - let bars = CoreBars::new(&snap.host.cpu_per_core_pct) + f.render_widget( + BrailleSparkline::new(&data) + .max(100) + .style(Style::default().fg(theme.accent)) + .gradient(theme.ok, theme.warn, theme.err), + inner, + ); +} + +/// Per-core load at this instant, one bar per core. +fn draw_cpu_cores(f: &mut Frame, area: Rect, snap: &Snapshot, theme: &Theme) { + let n_cores = snap.host.cpu_per_core_pct.len(); + let title = format!("Per-core · {n_cores} cores"); + let inner = panel::bento(f, area, Some(&title), BoxRole::Primary, false, theme); + if inner.height == 0 { + return; + } + f.render_widget( + CoreBars::new(&snap.host.cpu_per_core_pct) .max(100.0) .style(Style::default().fg(theme.ok)) - .gradient(theme.ok, theme.warn, theme.err); - f.render_widget(bars, split[1]); - } + .gradient(theme.ok, theme.warn, theme.err), + inner, + ); } fn draw_memory_row(f: &mut Frame, area: Rect, snap: &Snapshot, theme: &Theme) { @@ -281,6 +369,24 @@ fn gpu_section_header(snap: &Snapshot, theme: &Theme) -> Line<'static> { .fg(theme.accent) .add_modifier(Modifier::BOLD), )]; + let eff = rocm_dash_core::efficiency::node_efficiency(snap); + // Spelled out rather than the `*` the tabs with a footer use: this header + // has no room for the legend, and a marker with nothing to define it is + // worse than none. It sits right after the GPU count, ahead of the + // partition and VRAM text, so a narrow terminal clips those and never the + // marker that says the figure is not live. + let held = if eff.is_some_and(|n| n.held()) { + " (held)" + } else { + "" + }; + spans.push(Span::styled( + format!( + " · eff {}{held}", + format::tokens_per_watt(eff.map(|n| n.tokens_per_watt)) + ), + Style::default().fg(if eff.is_some() { theme.ok } else { theme.muted }), + )); if let Some(si) = snap.gpu_system_info.as_ref() { spans.push(Span::styled( format!( @@ -296,16 +402,11 @@ fn gpu_section_header(snap: &Snapshot, theme: &Theme) -> Line<'static> { )); } } - let total_power: f64 = snap.gpus.iter().map(|g| f64::from(g.power_w)).sum(); + let total_power = rocm_dash_core::efficiency::node_power_w(&snap.gpus); spans.push(Span::styled( format!(" · {:.1} kW", total_power / 1000.0), Style::default().fg(theme.fg), )); - let eff = node_efficiency(snap); - spans.push(Span::styled( - format!(" · eff {}", format::tokens_per_watt(eff)), - Style::default().fg(if eff.is_some() { theme.ok } else { theme.muted }), - )); Line::from(spans) } @@ -601,7 +702,9 @@ mod tests { use crate::app::{ActiveTab, AppState}; use ratatui::Terminal; use ratatui::backend::TestBackend; - use rocm_dash_core::metrics::{GpuSystemInfo, SystemMetrics}; + use rocm_dash_core::metrics::{ + GpuSystemInfo, ObservationFreshness, ObservationMetadata, SystemMetrics, + }; use rocm_dash_core::partition::{ComputePartitionMode, MemoryPartitionMode}; /// Render `draw` to a TestBackend and return the buffer as a flat string. @@ -665,6 +768,171 @@ mod tests { } } + fn snap_with_cores(n: usize) -> Snapshot { + Snapshot { + host: SystemMetrics { + cpu_overall_pct: 2.8, + cpu_per_core_pct: vec![1.0; n], + ..Default::default() + }, + ..Default::default() + } + } + + /// First row of a `render_to_string` buffer, which is where box titles sit. + fn top_row(out: &str, width: usize) -> String { + out.chars().take(width).collect() + } + + #[test] + fn cpu_overall_and_per_core_get_their_own_boxes() { + // The two charts share no axis: the sparkline is utilization over + // time, the bars are per-core load right now. Stacked bare in one box + // they read as a single broken graph, so each gets its own border and + // title. + let s = state_with_snapshot(snap_with_cores(32)); + let out = render_to_string(&s, 90, 20); + assert!(out.contains("CPU · 2.8%"), "overall box title missing"); + assert!( + out.contains("Per-core · 32 cores"), + "per-core box title missing" + ); + } + + #[test] + fn cpu_boxes_sit_side_by_side_when_wide() { + // Side by side keeps both boxes the full height of the CPU slot, which + // is what buys back the dot rows stacking spends on a second border. + // Both titles landing on the same row is what proves the layout. + let s = state_with_snapshot(snap_with_cores(32)); + let out = render_to_string(&s, 170, 20); + let top = top_row(&out, 170); + assert!( + top.contains("CPU · 2.8%"), + "overall title not on the top row" + ); + assert!( + top.contains("Per-core · 32 cores"), + "per-core title not beside it: {top}" + ); + } + + #[test] + fn cpu_boxes_stack_when_halving_would_starve_the_graph() { + // Narrow terminal: half the slot is too little for a trend, so the + // boxes stack and each takes the full width instead. + let s = state_with_snapshot(snap_with_cores(32)); + let out = render_to_string(&s, 60, 20); + let top = top_row(&out, 60); + assert!( + top.contains("CPU · 2.8%"), + "overall title not on the top row" + ); + assert!( + !top.contains("Per-core"), + "per-core box still beside the graph: {top}" + ); + assert!( + out.contains("Per-core · 32 cores"), + "per-core box lost rather than stacked" + ); + } + + #[test] + fn many_cores_keep_the_full_width_strip() { + // A 96-core host needs every column for its bars, so even a wide + // terminal stacks rather than halving the strip. + assert!(!cpu_fits_side_by_side(Rect::new(0, 0, 170, 10), 96)); + assert!(cpu_fits_side_by_side(Rect::new(0, 0, 170, 10), 32)); + // Narrow enough that half a slot is a stub, not a trend. + assert!(!cpu_fits_side_by_side(Rect::new(0, 0, 60, 10), 4)); + } + + #[test] + fn a_half_narrower_than_the_minimum_graph_width_stacks() { + // The other boundary in the gate. Width 70 leaves exactly MIN_GRAPH_W + // a side, so it splits; one column less does not. Without this a + // `>=` -> `>` slip here would pass. + assert_eq!((70_u16 / 2).saturating_sub(PANEL_CHROME_W), MIN_GRAPH_W); + assert!(cpu_fits_side_by_side(Rect::new(0, 0, 70, 10), 4)); + assert!(!cpu_fits_side_by_side(Rect::new(0, 0, 69, 10), 4)); + } + + #[test] + fn side_by_side_needs_two_columns_per_core_not_one() { + // 48 cores in 170 columns: each half has 80 inner columns, enough for + // one column per core but not for the gutters between them, so the + // strip would render as one fused block. Stack instead. + assert!(!cpu_fits_side_by_side(Rect::new(0, 0, 170, 10), 48)); + // The boundary: 2 columns per core is the point it becomes readable. + assert!(cpu_fits_side_by_side(Rect::new(0, 0, 170, 10), 40)); + assert!(!cpu_fits_side_by_side(Rect::new(0, 0, 170, 10), 41)); + } + + #[test] + fn a_dropped_per_core_box_moves_the_core_count_into_the_cpu_title() { + // The old combined title always carried the core count. When the + // per-core box is dropped for want of rows, the count must not go + // with it. + let s = state_with_snapshot(snap_with_cores(32)); + let out = render_to_string(&s, 38, 9); + assert!(!out.contains("Per-core"), "per-core box kept without room"); + assert!( + out.contains("CPU · 2.8% · 32 cores"), + "core count lost with the dropped box" + ); + } + + #[test] + fn the_stacked_split_leaves_both_charts_more_than_one_row() { + // Five rows of chart after chrome, split so neither collapses: the + // graph gets 3 (box 6 = border 2 + top pad 1 + 3) and the strip 2 (box + // 4 is under the height-5 pad threshold). A 4/1 split would quantise + // the strip at 25% per dot on an ordinary 120-column terminal, which + // stacks at 32 cores. + assert_eq!(CPU_OVERALL_H, 6); + assert_eq!(CPU_CORES_H, 4); + assert_eq!(CPU_OVERALL_H + CPU_CORES_H, 10, "CPU slot is 10 rows"); + assert!(!cpu_fits_side_by_side(Rect::new(0, 0, 120, 10), 32)); + } + + #[test] + fn cpu_keeps_the_history_graph_when_too_short_to_split() { + // Stacked and out of rows: one box has to go. Keep the one carrying + // history, since the per-core bars only ever show the current instant. + let s = state_with_snapshot(snap_with_cores(32)); + let out = render_to_string(&s, 38, 9); + assert!(out.contains("CPU · 2.8%"), "overall box dropped instead"); + assert!(!out.contains("Per-core"), "per-core box kept without room"); + } + + #[test] + fn cpu_is_a_single_box_when_the_host_reports_no_cores() { + let s = state_with_snapshot(snap_with_cores(0)); + let out = render_to_string(&s, 90, 20); + assert!(out.contains("CPU · 2.8%")); + assert!(!out.contains("Per-core"), "empty per-core box rendered"); + } + + #[test] + fn an_idle_gpu_draws_a_baseline_too_not_an_empty_panel() { + // The widget change reaches every sparkline, not just the CPU panel. + // A GPU sitting at 2% must show a continuous trace in its own panel + // for the same reason the CPU graph does. + let mut s = state_with_snapshot(snap_with_gpus(1)); + for _ in 0..40 { + let mut snap = snap_with_gpus(1); + snap.gpus[0].gpu_utilization_pct = 2.0; + s.history.push_back(snap); + } + s.latest.as_mut().unwrap().gpus[0].gpu_utilization_pct = 2.0; + let out = render_to_string(&s, 120, 30); + assert!( + out.contains('\u{28c0}') || out.contains('\u{2840}'), + "no baseline braille anywhere in the GPU section" + ); + } + fn mk_gpu(id: &str, util: f32, temp: f32, power: f32) -> GpuMetrics { GpuMetrics { device_id: id.into(), @@ -852,6 +1120,95 @@ mod tests { assert!(text.contains("0.50 tok/W"), "missing efficiency: {text}"); } + fn header_text_with_freshness(freshness: Option) -> String { + let theme = Theme::default_dark(); + let mut inst = mk_instance("a", &["0"], Some(250.0)); + inst.gen_tps_observation = freshness.map(|freshness| ObservationMetadata { + observed_at: chrono::Utc::now(), + freshness, + }); + let snap = Snapshot { + gpus: vec![mk_gpu("gpu-0", 50.0, 65.0, 500.0)], + instances: vec![inst], + ..Default::default() + }; + gpu_section_header(&snap, &theme) + .spans + .iter() + .map(|s| s.content.as_ref()) + .collect() + } + + #[test] + fn gpu_section_header_marks_a_held_efficiency_and_only_then() { + // 250 tok/s over 500 W = 0.50 tok/W in every case. + let held = header_text_with_freshness(Some(ObservationFreshness::Held)); + assert!( + held.contains("0.50 tok/W (held)"), + "held must be marked: {held}" + ); + let fresh = header_text_with_freshness(Some(ObservationFreshness::Fresh)); + assert!(fresh.contains("0.50 tok/W"), "missing efficiency: {fresh}"); + assert!(!fresh.contains("held"), "fresh must not be marked: {fresh}"); + // Unknown freshness (a legacy snapshot) is never presented as held. + let legacy = header_text_with_freshness(None); + assert!( + legacy.contains("0.50 tok/W") && !legacy.contains("held"), + "{legacy}" + ); + } + + #[test] + fn a_held_efficiency_survives_an_80_column_terminal() { + // The full header with a partitioned system is wider than 80 columns, and + // a `Paragraph` clips its line at the right edge. The marker must not be + // the part that goes, or a held figure reads as live. + let theme = Theme::default_dark(); + let mut inst = mk_instance("a", &["0"], Some(250.0)); + inst.gen_tps_observation = Some(ObservationMetadata { + observed_at: chrono::Utc::now(), + freshness: ObservationFreshness::Held, + }); + let snap = Snapshot { + gpus: vec![mk_gpu("gpu-0", 50.0, 65.0, 500.0)], + gpu_system_info: Some(sysinfo_partitioned()), + instances: vec![inst], + ..Default::default() + }; + let mut term = Terminal::new(TestBackend::new(80, 1)).unwrap(); + term.draw(|f| f.render_widget(Paragraph::new(gpu_section_header(&snap, &theme)), f.area())) + .unwrap(); + let row: String = (0..80) + .map(|x| term.backend().buffer()[(x, 0)].symbol().to_string()) + .collect(); + assert!( + row.contains("0.50 tok/W (held)"), + "marker clipped at 80 columns: {row:?}" + ); + } + + #[test] + fn gpu_section_header_kw_skips_a_faulty_power_sensor() { + // 500 W next to an infinite and a negative reading: the header must say + // 0.5 kW, not `inf kW` or a total deflated by the negative one. + let theme = Theme::default_dark(); + let snap = Snapshot { + gpus: vec![ + mk_gpu("gpu-0", 50.0, 65.0, 500.0), + mk_gpu("gpu-1", 50.0, 65.0, f32::INFINITY), + mk_gpu("gpu-2", 50.0, 65.0, -300.0), + ], + ..Default::default() + }; + let text: String = gpu_section_header(&snap, &theme) + .spans + .iter() + .map(|s| s.content.as_ref()) + .collect(); + assert!(text.contains("0.5 kW"), "usable total missing: {text}"); + assert!(!text.contains("inf") && !text.contains("NaN"), "{text}"); + } + #[test] fn gpu_section_header_efficiency_dash_without_traffic() { let theme = Theme::default_dark(); diff --git a/crates/rocm-dash-tui/src/ui/tabs/home.rs b/crates/rocm-dash-tui/src/ui/tabs/home.rs index e49cdeb872..d7ff77ddce 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/home.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/home.rs @@ -11,9 +11,7 @@ //! with a hint at the tab/key that would produce something, so no tile is a //! dead end. //! -//! ponytail: Home is added behind the existing default this phase (P2). It is -//! reachable by Tab / digit `1` but is NOT the default tab yet — P3 repoints -//! the default and folds the telemetry tabs into Observe. +//! Home is the default tab (`ActiveTab::Home`) and is reachable by Tab / digit `1`. use ratatui::Frame; use ratatui::layout::{Constraint, Direction, Layout, Rect}; @@ -21,6 +19,7 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::Paragraph; +use rocm_dash_core::metrics::Instance; use rocm_dash_core::state::JobStatus; use crate::app::{AppState, ConnState, UpdateStatus}; @@ -48,15 +47,25 @@ fn vram_used_pct(snap: &rocm_dash_core::metrics::Snapshot) -> f64 { } /// History series helper: map each snapshot to a 0..=100 magnitude via `pick`. +/// +/// Delegates the GPU-less skip to [`crate::ui::widgets::gpu_history`], which +/// owns that rule for every panel that draws a GPU series. fn history f64>( state: &AppState, pick: F, ) -> Vec { - state - .history - .iter() - .map(|s| pick(s).clamp(0.0, 100.0) as u64) - .collect() + crate::ui::widgets::gpu_history(state.history.iter(), pick) +} + +/// The POWER trace: the node's usable board power over the recent snapshots, +/// scaled to a nominal 1 kW ceiling (power can exceed 100 W, the trace is 0-100). +/// +/// It uses the same filtered sum as the headline label beside it, so a faulty +/// sensor cannot show in one and not the other. +fn power_trace(state: &AppState) -> Vec { + history(state, |s| { + rocm_dash_core::efficiency::node_power_w(&s.gpus) / 10.0 + }) } /// Label for the node-load sparkline: no data → `—`; simulated telemetry is @@ -104,6 +113,25 @@ fn snap_util(s: &rocm_dash_core::metrics::Snapshot) -> f64 { .fold(0.0, f64::max) } +/// Sums the finite values `pick` yields across instances, or `None` when no +/// instance contributed one. The `Option` is the point: "nothing reported a +/// value" has to stay distinct from "every reporter measured zero", which a +/// bare sum cannot express. A single NaN/Infinity sample is dropped rather +/// than allowed to poison the total, as `gen_tps_cell` does per instance. +/// `Iterator::sum` is deliberately avoided — its `f64` identity is `-0.0`, so +/// an empty iterator sums to `-0.0` and renders as `"-0"`. +fn instance_aggregate(state: &AppState, pick: F) -> Option +where + F: Fn(&rocm_dash_core::metrics::Instance) -> Option, +{ + state + .instances + .values() + .filter_map(pick) + .filter(|v| v.is_finite()) + .fold(None, |acc, v| Some(acc.unwrap_or(0.0) + v)) +} + /// A labeled one-row instrument: `LABEL ▁▂▃▅▆▇ value`. Port of the mock's /// `mini_spark`; `accent` picks the flat throughput tint, else value-gradient. fn mini_spark( @@ -293,7 +321,7 @@ fn draw_activity(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { fn draw_hero_band(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { let title = node_throughput_title(state); let hero = card(f, area, &title, BoxRole::Secondary, theme); - if hero.width >= 8 && hero.height >= 6 { + if hero.width >= 8 && hero.height >= 1 { let hcols = Layout::default() .direction(Direction::Horizontal) .constraints([Constraint::Percentage(50), Constraint::Percentage(50)]) @@ -304,11 +332,22 @@ fn draw_hero_band(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } } +/// One-row slots for a hero column, top to bottom. Slots past the column's +/// height come back zero-height, so a short terminal drops the bottom rows +/// (the legends first) rather than the whole hero. +fn hero_rows(area: Rect) -> [Rect; 6] { + std::array::from_fn(|i| { + let i = u16::try_from(i).unwrap_or(u16::MAX); + if i < area.height { + Rect::new(area.x, area.y + i, area.width, 1) + } else { + Rect::new(area.x, area.y, area.width, 0) + } + }) +} + fn draw_hero_left(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { - let lh = Layout::default() - .direction(Direction::Vertical) - .constraints([Constraint::Length(1); 6]) - .split(area); + let lh = hero_rows(area); f.render_widget( Paragraph::new(Line::from(Span::styled( "GPU UTILIZATION", @@ -339,31 +378,29 @@ fn draw_hero_left(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { // Tokens/watt: summed across running instances when available. // Mark held if any contributing instance has a held gen_tps observation // (tok/W derives from gen_tps; aggregate inherits held status). - // A single NaN/Infinity instance would otherwise poison the whole sum - // (and, for `any_tpw_held`, count as held without ever being displayed) — - // filtered out the same way `tokens_per_watt` is guarded per-instance - // elsewhere in this tab. - let tpw: f64 = state - .instances - .values() - .filter_map(|i| i.tokens_per_watt) - .filter(|v| v.is_finite()) - .sum(); + // `live_tokens_per_watt` leaves out an efficiency derived from a Lemonade + // last-request rate, for the same reason as the T/S hero. + let tpw = instance_aggregate(state, Instance::live_tokens_per_watt); let any_tpw_held = state.instances.values().any(|i| { - i.tokens_per_watt.is_some_and(f64::is_finite) + i.live_tokens_per_watt().is_some_and(f64::is_finite) && i.gen_tps_observation .as_ref() .is_some_and(|m| m.freshness == rocm_dash_core::metrics::ObservationFreshness::Held) }); - let tpw_label = if tpw > 0.0 { - let marker = if any_tpw_held { - crate::ui::format::HELD_MARKER - } else { - "" - }; - format!("⎓ {tpw:.1} tokens / watt{marker}") - } else { - "⎓ tokens / watt —".to_string() + // The marker and the legend that explains it share fate: the legend row + // is the first thing a short hero drops, and a bare "*" with nothing on + // screen to define it is worse than an unannotated prior-window value. + let legend_fits = lh[5].height > 0; + let tpw_label = match tpw { + None => "⎓ tokens / watt —".to_string(), + Some(v) => { + let marker = if any_tpw_held && legend_fits { + crate::ui::format::HELD_MARKER + } else { + "" + }; + format!("⎓ {v:.1} tokens / watt{marker}") + } }; f.render_widget( Paragraph::new(Line::from(Span::styled( @@ -375,10 +412,15 @@ fn draw_hero_left(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { lh[4], ); // Show the shared HELD_LEGEND only when the tok/W aggregate is actually - // held AND rendered with a marker — `tpw > 0.0` mirrors the gate on - // `tpw_label` above so a zero-throughput aggregate (which prints - // "tokens / watt —" with no marker) never shows an unexplained legend. - if tpw > 0.0 && any_tpw_held { + // held, so an unmeasured aggregate (which prints "tokens / watt —" with no + // marker) never shows an unexplained legend. + // + // `any_tpw_held` already implies `tpw.is_some()`: it requires some instance + // with a finite `tokens_per_watt`, and `instance_aggregate` folds exactly + // those into `Some` over the same `state.instances` set. The extra + // `tpw.is_some() &&` term was unreachable-false, so it is gone rather than + // left to read as a real second condition. + if any_tpw_held { f.render_widget( Paragraph::new(Line::from(Span::styled( format::HELD_LEGEND, @@ -390,10 +432,7 @@ fn draw_hero_left(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } fn draw_hero_right(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { - let rh = Layout::default() - .direction(Direction::Vertical) - .constraints([Constraint::Length(1); 6]) - .split(area); + let rh = hero_rows(area); // In demo/replay this is not a live feed — say so rather than "LIVE". let (feed_label, feed_color) = if state.simulated { ("SIMULATED · last 60s", theme.warn) @@ -417,9 +456,7 @@ fn draw_hero_right(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { .map(|g| f64::from(g.temperature_c)) .fold(0.0, f64::max) }); - let power: f64 = latest.map_or(0.0, |s| { - s.gpus.iter().map(|g| f64::from(g.power_w)).sum::() - }); + let power = latest.map_or(0.0, |s| rocm_dash_core::efficiency::node_power_w(&s.gpus)); // While there's no GPU telemetry at all, show "—" rather than 0%/0°C/0W // — those look like real (broken) readings rather than a loading/unknown // state. The same applies per-metric once data has arrived but a given @@ -441,7 +478,7 @@ fn draw_hero_right(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { let power_label = if unavailable || power == 0.0 { "—".to_string() } else { - format::watts(power as f32) + format::watts(power) }; mini_spark( f, @@ -471,33 +508,25 @@ fn draw_hero_right(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { rh[3], "POWER", &power_label, - // Power can exceed 100; scale to a nominal 1kW ceiling for the trace. - &history(state, |s| { - s.gpus.iter().map(|g| f64::from(g.power_w)).sum::() / 10.0 - }), + &power_trace(state), false, theme, ); - // A single NaN/Infinity instance would otherwise poison the whole sum - // (rendering "NaN"/"inf" in the hero) and could mark the aggregate held - // without ever contributing a displayed value — guarded the same way - // `gen_tps_cell` guards a single instance's value. - let tps: f64 = state - .instances - .values() - .filter_map(|i| i.gen_tps) - .filter(|v| v.is_finite()) - .sum(); - let any_tps_held = state.instances.values().any(|i| { - i.gen_tps.is_some_and(f64::is_finite) - && i.gen_tps_observation - .as_ref() - .is_some_and(|m| m.freshness == rocm_dash_core::metrics::ObservationFreshness::Held) - }); - let tps_str = if any_tps_held { - format!("{:.0}{}", tps, crate::ui::format::HELD_MARKER) - } else { - format!("{tps:.0}") + // The same aggregate, and the same notion of "held", as the Observe heroes: + // a held instance at zero tok/s must not mark a figure that comes from fresh + // ones. + let throughput = rocm_dash_core::efficiency::node_throughput(state.instances.values()); + let tps = throughput.map(|t| t.tokens_per_sec); + let any_tps_held = throughput.is_some_and(|t| t.held()); + // The held marker is suppressed with the legend row that explains it, as + // in `draw_hero_left`. + let legend_fits = rh[5].height > 0; + let tps_str = match tps { + None => "—".to_string(), + Some(v) if any_tps_held && legend_fits => { + format!("{:.0}{}", v, crate::ui::format::HELD_MARKER) + } + Some(v) => format!("{v:.0}"), }; mini_spark(f, rh[4], "T/S ", &tps_str, &[], true, theme); // Show the shared HELD_LEGEND only when the tok/s aggregate is actually @@ -1151,6 +1180,42 @@ mod tests { } } + #[test] + fn hero_drops_only_its_lowest_rows_in_a_short_terminal() { + // A ~22-row terminal leaves the Home tab fewer rows than the six the + // hero's columns ask for. The readings must still draw, and they must + // go bottom-first: at every height the visible rows form a prefix of + // this list, so one is missing only when all below it are too. That is + // the promise docs/manual-testing.md 4c makes, pinned without naming + // the height each row happens to appear at (POWER at 13, T/S at 15 — + // thresholds that would otherwise drift silently). + let s = state_with_gpu(); + let top_down = ["VRAM", "TEMP", "POWER", "T/S"]; + for h in 8u16..=24 { + let out = render(&s, 160, h); + assert!( + out.contains("GPU UTILIZATION"), + "the hero must never render as an empty card, at height {h}:\n{out}" + ); + let present: Vec = top_down.iter().map(|l| out.contains(l)).collect(); + let kept = present.iter().take_while(|p| **p).count(); + assert!( + present[kept..].iter().all(|p| !*p), + "the hero dropped {} but kept a row below it, at height {h}:\n{out}", + top_down[kept] + ); + } + // Without this the prefix check above would pass on a hero that + // renders none of these rows at any height. + let roomy = render(&s, 160, 24); + for label in top_down { + assert!( + roomy.contains(label), + "{label} missing from a hero with room for it:\n{roomy}" + ); + } + } + fn instance_with_obs(gen_tps: f64, obs: Option) -> Instance { Instance { container_id: "m".into(), @@ -1185,6 +1250,35 @@ mod tests { s } + /// The hero's T/S value, i.e. the text after the "T/S" label up to the + /// panel border on that row. + fn hero_tps_value(out: &str) -> &str { + out.split("T/S") + .nth(1) + .and_then(|s| s.split('│').next()) + .unwrap_or_else(|| panic!("T/S row missing: {out:?}")) + .trim() + } + + #[test] + fn hero_tps_shows_dash_not_negative_zero_when_nothing_reports_throughput() { + // A GPU is reported but no instance is serving: the Windows in-process + // collectors never send instances, and a Linux host with nothing + // running is in the same state. + let out = render(&state_with_gpu(), 160, 30); + assert_eq!(hero_tps_value(&out), "—", "got:\n{out}"); + } + + #[test] + fn hero_tps_shows_zero_when_an_instance_reports_zero_throughput() { + let out = render( + &state_with_instance(instance_with_obs(0.0, Some(fresh_obs()))), + 160, + 30, + ); + assert_eq!(hero_tps_value(&out), "0", "got:\n{out}"); + } + #[test] fn home_held_legend_visible_when_tpw_and_tps_held() { let out = render( @@ -1209,6 +1303,148 @@ mod tests { ); } + /// The text of the screen row holding `needle`, for a `cols`-wide render. + fn row_with(out: &str, cols: usize, needle: &str) -> String { + let at = out + .find(needle) + .unwrap_or_else(|| panic!("{needle:?} not rendered:\n{out}")); + let chars: Vec = out.chars().collect(); + let row = out[..at].chars().count() / cols; + chars[row * cols..(row + 1) * cols].iter().collect() + } + + /// An idle Lemonade server keeps its last request's rate. The same numbers on + /// a live instance fill the T/S and tokens/watt heroes; as a last-request rate + /// they must not, and held metadata on it adds no marker or legend. + #[test] + fn home_heroes_leave_out_a_last_request_rate() { + let live = render( + &state_with_instance(instance_with_obs(300.0, Some(fresh_obs()))), + 160, + 30, + ); + assert!(row_with(&live, 160, "T/S").contains("300"), "{live}"); + assert!(live.contains("1.0 tokens / watt"), "{live}"); + + for obs in [fresh_obs(), held_obs()] { + let mut inst = instance_with_obs(300.0, Some(obs)); + inst.gen_tps_last_request = true; + let out = render(&state_with_instance(inst), 160, 30); + let tps_row = row_with(&out, 160, "T/S"); + assert!(!tps_row.contains("300"), "{tps_row}"); + assert!(out.contains("tokens / watt —"), "{out}"); + assert!(!out.contains("1.0 tokens / watt"), "{out}"); + assert!(!out.contains(format::HELD_LEGEND), "{out}"); + } + } + + /// Beside a live instance the totals are non-zero, so whether they are marked + /// held is decided by the held check alone: a held last-request rate is left + /// out of the totals and must not mark them held either. + #[test] + fn a_held_last_request_rate_does_not_mark_live_totals_held() { + let mut s = state_with_instance(instance_with_obs(300.0, Some(fresh_obs()))); + let mut last = instance_with_obs(150.0, Some(held_obs())); + last.container_id = "last".into(); + last.gen_tps_last_request = true; + s.instances.insert(last.container_id.clone(), last); + let out = render(&s, 160, 30); + let tps_row = row_with(&out, 160, "T/S"); + assert!(tps_row.contains("300"), "{tps_row}"); + assert!(!tps_row.contains(format::HELD_MARKER), "{tps_row}"); + assert!(out.contains("1.0 tokens / watt"), "{out}"); + assert!( + !out.contains(&format!("tokens / watt{}", format::HELD_MARKER)), + "{out}" + ); + assert!(!out.contains(format::HELD_LEGEND), "{out}"); + } + + #[test] + fn hero_never_shows_a_held_marker_without_its_legend() { + // Squeezing the hero drops the legend row before the value rows, so + // every height has to be checked, not just the roomy one. + let s = state_with_instance(instance_with_obs(300.0, Some(held_obs()))); + let marked_tps = format!("{:.0}{}", 300.0, format::HELD_MARKER); + let marked_tpw = format!("{:.1} tokens / watt{}", 1.0, format::HELD_MARKER); + let mut saw_unmarked_value = false; + for h in 8u16..=30 { + let out = render(&s, 160, h); + if out.contains(&marked_tps) || out.contains(&marked_tpw) { + assert!( + out.contains(format::HELD_LEGEND), + "held marker with no legend to explain it at height {h}:\n{out}" + ); + } + // The reading itself must survive the legend being dropped. + if out.contains("T/S") && !out.contains(format::HELD_LEGEND) { + assert!( + out.contains("300"), + "the T/S reading must stay once the marker is dropped, at height {h}:\n{out}" + ); + saw_unmarked_value = true; + } + } + assert!( + saw_unmarked_value, + "no height rendered T/S without the legend — the squeeze this guards never happened" + ); + } + + #[test] + fn home_power_label_skips_a_faulty_sensor_like_the_trace_beside_it() { + // One GPU at 420 W and one reporting infinity: the label must show the + // usable 420 W, not `inf W`, and agree with the sparkline's filtered sum. + let mut state = state_with_gpu(); + if let Some(snap) = state.latest.as_mut() { + let mut bad = snap.gpus[0].clone(); + bad.device_id = "gpu-1".into(); + bad.power_w = f32::INFINITY; + snap.gpus.push(bad); + } + // ...and the trace beside it, drawn from the same snapshot, skips the + // sensor too: 420 W / 10 = 42, where a raw sum would clamp to 100. + if let Some(snap) = state.latest.clone() { + state.history.push_back(snap); + } + assert_eq!( + power_trace(&state), + vec![42], + "the trace must skip it as well" + ); + let out = render(&state, 160, 30); + assert!(out.contains("420.0 W"), "usable power missing:\n{out}"); + assert!( + !out.contains("inf W") && !out.contains("NaN W"), + "a faulty sensor reached the label:\n{out}" + ); + } + + #[test] + fn home_throughput_is_not_marked_held_by_a_held_instance_at_zero_tok_s() { + // The same rule as the Observe heroes: an instance at zero tok/s adds + // nothing to the throughput, so its freshness must not mark it. + let mut state = state_with_instance(instance_with_obs(300.0, Some(fresh_obs()))); + let mut idle = instance_with_obs(0.0, Some(held_obs())); + idle.container_id = "idle".into(); + idle.container_name = "idle".into(); + state.instances.insert(idle.container_id.clone(), idle); + let out = render(&state, 160, 30); + assert_eq!(hero_tps_value(&out), "300", "got:\n{out}"); + // ...while a held instance that does contribute still marks it. + let mut state = state_with_instance(instance_with_obs(300.0, Some(fresh_obs()))); + let mut slow = instance_with_obs(5.0, Some(held_obs())); + slow.container_id = "slow".into(); + slow.container_name = "slow".into(); + state.instances.insert(slow.container_id.clone(), slow); + let out = render(&state, 160, 30); + assert_eq!( + hero_tps_value(&out), + format!("305{}", format::HELD_MARKER), + "got:\n{out}" + ); + } + #[test] fn home_held_legend_absent_when_all_fresh() { let out = render( @@ -1235,27 +1471,52 @@ mod tests { ); } - #[test] - fn home_tpw_legend_absent_when_aggregate_is_zero_even_if_held() { - // tokens_per_watt sums to 0.0 (the "tokens / watt —" branch, no marker - // rendered) while gen_tps is absent (so the tok/s side never fires - // either). Held metadata alone must not add an unexplained legend. - let inst = Instance { + /// An instance whose only reading is `tokens_per_watt`, held. + fn tpw_only_instance(tokens_per_watt: Option) -> Instance { + Instance { container_id: "m".into(), container_name: "m".into(), status: InstanceStatus::Running, model_name: "m".into(), gpu_ids: vec!["0".into()], gen_tps: None, - tokens_per_watt: Some(0.0), + tokens_per_watt, gen_tps_observation: Some(held_obs()), ..Default::default() - }; - let out = render(&state_with_instance(inst), 160, 30); + } + } + + #[test] + fn home_tpw_shows_a_measured_zero_rather_than_a_dash() { + // An instance that reports 0.0 tok/W measured it; only "no instance + // reported one at all" is unmeasured. Mirrors the tok/s side, which + // renders a reported 0 as "0" and reserves "—" for nothing reporting. + let out = render(&state_with_instance(tpw_only_instance(Some(0.0))), 160, 30); assert!( - !out.contains(format::HELD_LEGEND), - "HELD_LEGEND must not appear when the tok/W aggregate is zero; got:\n{out}" + out.contains(&format!("0.0 tokens / watt{}", format::HELD_MARKER)), + "a measured zero tok/W must render as 0.0, not —; got:\n{out}" ); + assert!( + out.contains(format::HELD_LEGEND), + "the marked zero aggregate must still be explained; got:\n{out}" + ); + } + + #[test] + fn home_tpw_legend_absent_when_no_instance_reports_a_finite_value() { + // The invariant the aggregate change must not break: held metadata + // alone, with nothing finite to mark, must not conjure a legend. + for tpw in [None, Some(f64::NAN), Some(f64::INFINITY)] { + let out = render(&state_with_instance(tpw_only_instance(tpw)), 160, 30); + assert!( + out.contains("tokens / watt —"), + "unmeasured tok/W must read —; got for {tpw:?}:\n{out}" + ); + assert!( + !out.contains(format::HELD_LEGEND), + "HELD_LEGEND must not appear with nothing marked; got for {tpw:?}:\n{out}" + ); + } } #[test] @@ -1708,4 +1969,27 @@ mod tests { theme.accent ); } + + #[test] + fn history_skips_snapshots_with_no_gpus() { + // Every `pick` folds or sums over `gpus`, so a GPU-less snapshot comes + // out as 0. Since the widget now draws a baseline for every sample it + // is handed, letting those through would paint a host with no GPU as + // one idling at 0 °C and 0 W. + let mut s = AppState::new("t".into(), "default-dark".into()); + s.history.push_back(Snapshot::default()); + s.history.push_back(Snapshot::default()); + assert!( + history(&s, snap_util).is_empty(), + "a GPU-less snapshot reached the series" + ); + + let mut with_gpu = Snapshot::default(); + with_gpu.gpus.push(GpuMetrics { + gpu_utilization_pct: 50.0, + ..Default::default() + }); + s.history.push_back(with_gpu); + assert_eq!(history(&s, snap_util), vec![50]); + } } diff --git a/crates/rocm-dash-tui/src/ui/tabs/instances.rs b/crates/rocm-dash-tui/src/ui/tabs/instances.rs index 62ebdaf811..bcd7639ce7 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/instances.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/instances.rs @@ -10,7 +10,7 @@ use ratatui::Frame; use ratatui::layout::{Constraint, Direction, Layout, Rect}; use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; -use ratatui::widgets::{Cell, Paragraph, Row, Table}; +use ratatui::widgets::{Cell, Paragraph, Row, Table, Wrap}; use rocm_dash_core::metrics::{ Instance, InstanceStatus, ObservationFreshness, ObservationMetadata, @@ -90,7 +90,11 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { || (is_held(&inst.ttft_observation) && inst.ttft_ms.is_some_and(f64::is_finite)) || (is_held(&inst.tpot_observation) && inst.tpot_ms.is_some_and(f64::is_finite)) }); - let (table_area, legend_area) = if any_held { + let any_last_req = instances + .iter() + .take(visible_rows) + .any(|inst| inst.gen_tps_last_request && inst.gen_tps.is_some_and(f64::is_finite)); + let (table_area, legend_area) = if any_held || any_last_req { let split = Layout::default() .direction(Direction::Vertical) .constraints([Constraint::Min(0), Constraint::Length(1)]) @@ -113,8 +117,12 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { let rows = instances.iter().enumerate().map(|(i, inst)| { let model = trunc(&inst.model_name, 22); let tps = format::gen_tps_cell(inst.gen_tps, inst.gen_tps_observation.as_ref()); - let tpw = - format::tokens_per_watt_cell(inst.tokens_per_watt, inst.gen_tps_observation.as_ref()); + // A last-request rate has no efficiency to show: the figure beside it + // is idle board power over a finished request, not a current rate. + let tpw = format::tokens_per_watt_cell( + inst.live_tokens_per_watt(), + inst.gen_tps_observation.as_ref(), + ); let ttft = format::latency_cell(inst.ttft_ms, inst.ttft_observation.as_ref()); let tpot = format::latency_cell(inst.tpot_ms, inst.tpot_observation.as_ref()); let power = inst @@ -171,7 +179,13 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { if let Some(legend_area) = legend_area { f.render_widget( Paragraph::new(Line::from(Span::styled( - format::HELD_LEGEND, + match (any_held, any_last_req) { + (true, true) => { + format!("{} · {}", format::HELD_LEGEND, format::LAST_REQ_LEGEND) + } + (true, false) => format::HELD_LEGEND.to_owned(), + _ => format::LAST_REQ_LEGEND.to_owned(), + }, Style::default().fg(theme.muted), ))), legend_area, @@ -179,13 +193,16 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } } -/// Sort instances deterministically by container_name so that an index from -/// AppState always maps to the same row in the table. +/// Sort instances deterministically by container_name, then container_id, so +/// that an index from AppState always maps to the same row in the table. The id +/// matters: every external Lemonade model shares the name `lemonade`. pub(in crate::ui) fn sorted_instances( instances: &std::collections::HashMap, ) -> Vec<&Instance> { let mut v: Vec<&Instance> = instances.values().collect(); - v.sort_by(|a, b| a.container_name.cmp(&b.container_name)); + v.sort_by(|a, b| { + (&a.container_name, &a.container_id).cmp(&(&b.container_name, &b.container_id)) + }); v } @@ -206,20 +223,34 @@ pub(in crate::ui) const fn status_meta( fn draw_empty(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { let body = if state.conn.is_in_process() { - "no instance registry on this platform · vLLM containers aren't available here" + "no managed services running · serve one from 3 Serving · vLLM containers aren't available here" } else { match &state.conn { ConnState::Connected { .. } => { - "no instances · start daemon with --enable-docker on a host with vLLM containers" + "no serving models discovered · open Serving to launch a managed model or check an external endpoint" } _ => "waiting for daemon…", } }; let inner = panel::bento(f, area, Some("Instances"), BoxRole::Neutral, false, theme); + // Wrapped, not one clipped line. An unwrapped `Paragraph` truncates at the + // panel's right edge rather than wrapping, and the longer branches here + // outrun the columns an 80-column terminal leaves usable inside this + // border: the in-process one (94 chars) stopped dead at + // "· vLLM containers ar", and the `Connected` one (99) runs well past the + // same cliff, losing the half that names the external-endpoint path. The + // narrower the pane, the more of the remedy goes too. + // `Wrap { trim: false }` is what every other prose paragraph in this UI + // uses (`ui/tabs/pane.rs`, `ui/tabs/hardware.rs`, `ui/help_modal.rs`, + // `ui/warnings_modal.rs`, `ui/approval.rs`), so the empty state now + // behaves like the rest of them; shortening the string instead would have + // had to drop either the remedy or the vLLM explanation, and would only + // move the width at which the clipping returns. let p = Paragraph::new(Line::from(Span::styled( body, Style::default().fg(theme.muted), - ))); + ))) + .wrap(Wrap { trim: false }); f.render_widget(p, inner); } @@ -230,6 +261,7 @@ pub(in crate::ui) mod tests { pub(in crate::ui) fn mk_inst(name: &str) -> Instance { Instance { + managed: false, container_id: format!("id-{name}"), container_name: name.to_string(), status: InstanceStatus::Running, @@ -246,6 +278,7 @@ pub(in crate::ui) mod tests { waiting_reqs: None, gen_tps: None, gen_tps_observation: None, + gen_tps_last_request: false, tokens_per_watt: None, power_w: None, ttft_ms: None, @@ -273,20 +306,38 @@ pub(in crate::ui) mod tests { use ratatui::backend::TestBackend; // The Windows in-process collector (see `ConnState::is_in_process`) - // has no instance registry at all, ever — distinct from "a real - // daemon is connected but nothing is running" or "still connecting". + // has no daemon: its instances come only from the managed-service + // registry, so an empty list means nothing is served — distinct from + // "a real daemon is connected but nothing is running" or "still + // connecting". let mut state = mk_state(HashMap::new(), 0); state.conn = ConnState::Connected { host: "localhost".into(), version: ConnState::in_process_version("0.0.0"), }; + // 80 columns — a realistic terminal, the same width + // `ui/tabs/mod.rs`'s panel test uses — NOT a width picked to make the + // assertion pass. The message is 94 chars and the panel's border and + // padding narrow that further, so at 80 columns it survives only + // because `draw_empty` wraps: unwrapped, the render stops dead at + // "· vLLM containers ar" and the user is never told why the vLLM + // containers they are looking for cannot appear here. let mut term = Terminal::new(TestBackend::new(80, 20)).unwrap(); term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) .unwrap(); let grid = buffer_text(&term); + // Asserted whole, tab digit included. `ActiveTab::from_digit` maps '3' + // to Serving today; a tab reorder that left this line behind would send + // the user to the wrong tab, and nothing else here would catch it. + // Read back through `flatten_wrapped_prose` because the message now + // occupies more than one row: asserting only the first row's worth + // would be the same mistake as widening the backend. + let flat = flatten_wrapped_prose(&grid); assert!( - grid.contains("no instance registry on this platform"), - "should name the platform limitation, not claim a daemon could be started; got:\n{grid}" + flat.contains( + "no managed services running · serve one from 3 Serving · vLLM containers aren't available here" + ), + "should say nothing is served AND where to serve one from instead, not claim a daemon could be started; got:\n{grid}" ); assert!( !grid.contains("start daemon"), @@ -294,6 +345,31 @@ pub(in crate::ui) mod tests { ); } + /// Read a rendered panel back as the one sentence the user reads. + /// + /// A wrapped `Paragraph` lays its prose across several buffer rows, each + /// padded to the terminal's width and fenced by the panel's box-drawing + /// border. Treating every border glyph and every run of blanks as a single + /// space rejoins the rows at exactly the single spaces the source string + /// had, so a wrapped message can still be asserted verbatim — and a + /// *clipped* one still cannot, because the clipped characters are simply + /// not in the buffer to rejoin. Only U+2500–U+257F (box drawing) is + /// blanked, so the message's own `·` (U+00B7) survives. + fn flatten_wrapped_prose(grid: &str) -> String { + grid.chars() + .map(|c| { + if matches!(u32::from(c), 0x2500..=0x257F) { + ' ' + } else { + c + } + }) + .collect::() + .split_whitespace() + .collect::>() + .join(" ") + } + #[test] fn status_meta_maps_each_variant() { let theme = Theme::default_dark(); @@ -339,6 +415,28 @@ pub(in crate::ui) mod tests { ); } + /// Every external Lemonade model on an endpoint shares the container name + /// `lemonade`, so the name alone leaves their order to the map's hash order, + /// and the selection index would land on a different model from one frame + /// to the next. The id breaks the tie. + #[test] + fn sorted_instances_breaks_a_name_tie_by_id() { + for _ in 0..32 { + // A fresh map each pass: each one gets its own hash seed. + let mut m = HashMap::new(); + for id in ["lemonade-m-c", "lemonade-m-a", "lemonade-m-b"] { + let mut inst = mk_inst("lemonade"); + inst.container_id = id.into(); + m.insert(inst.container_id.clone(), inst); + } + let ids: Vec<&str> = sorted_instances(&m) + .iter() + .map(|i| i.container_id.as_str()) + .collect(); + assert_eq!(ids, ["lemonade-m-a", "lemonade-m-b", "lemonade-m-c"]); + } + } + #[test] fn selected_instance_lookup_by_sorted_index() { // HashMap insertion order is irrelevant — sorted_instances orders by name. @@ -357,6 +455,7 @@ pub(in crate::ui) mod tests { services_past_attempts: 0, managed_registry_probe: None, launcher_hosted: false, + lemonade_managed_installed: None, connect: "test".into(), conn: ConnState::Initial, latest: None, @@ -978,4 +1077,127 @@ pub(in crate::ui) mod tests { "zero gen_tps must render as '0', not '—'; got:\n{out}" ); } + + pub(in crate::ui) fn last_req_inst(name: &str, last_request: bool) -> Instance { + let mut inst = mk_inst_obs(name, Some(37.5), Some(obs(ObservationFreshness::Fresh, 0))); + inst.gen_tps_last_request = last_request; + inst + } + + #[test] + fn table_labels_a_last_request_rate_and_only_then() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for (last_request, expect_legend) in [(true, true), (false, false)] { + let state = state_with_snap(last_req_inst("rate", last_request)); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!(out.contains("38"), "the value itself still shows:\n{out}"); + assert_eq!( + out.contains(format::LAST_REQ_LEGEND), + expect_legend, + "last_request={last_request}:\n{out}" + ); + } + } + + /// The legend explains a rate that is on screen: a last-request flag on a row + /// with no finite rate has nothing to explain. + #[test] + fn table_has_no_last_request_legend_without_a_rate_to_explain() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for gen_tps in [None, Some(f64::NAN)] { + let mut inst = last_req_inst("rate", true); + inst.gen_tps = gen_tps; + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + !out.contains(format::LAST_REQ_LEGEND), + "gen_tps={gen_tps:?}: +{out}" + ); + } + } + + /// A held rate that is also a last-request rate shows both legends, joined. + #[test] + fn table_shows_the_held_and_last_request_legends_together() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut inst = mk_inst_obs( + "both", + Some(37.5), + Some(obs(ObservationFreshness::Held, 30)), + ); + inst.gen_tps_last_request = true; + let state = state_with_snap(inst); + // 80 columns is the width that matters: the joined legend must fit. + let mut term = Terminal::new(TestBackend::new(80, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + let both = format!("{} · {}", format::HELD_LEGEND, format::LAST_REQ_LEGEND); + assert!(out.contains(&both), "{out}"); + } + + /// The legend wording is what the user reads (and what the e2e screen + /// scenario looks for): it must name Lemonade and the last-request rate. + #[test] + fn the_last_request_legend_names_what_it_explains() { + assert_eq!( + format::LAST_REQ_LEGEND, + "Lemonade: last req tok/s, not live" + ); + } + + /// An efficiency derived from a last-request rate is not shown on the row, + /// whether or not the rate itself is (replayed recordings can carry one). + #[test] + fn table_leaves_out_an_efficiency_derived_from_a_last_request_rate() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for gen_tps in [Some(37.5), None] { + let mut inst = last_req_inst("rate", true); + inst.gen_tps = gen_tps; + inst.tokens_per_watt = Some(0.12); + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + !out.contains("0.12"), + "gen_tps={gen_tps:?}: +{out}" + ); + } + } + + #[test] + fn the_connected_empty_state_names_the_external_endpoint_path_at_80_columns() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut state = mk_state(HashMap::new(), 0); + state.conn = ConnState::Connected { + host: "localhost".into(), + version: "0.0.0".into(), + }; + let mut term = Terminal::new(TestBackend::new(80, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let grid = buffer_text(&term); + assert!(grid.contains("external"), "{grid}"); + assert!(grid.contains("endpoint"), "{grid}"); + } } diff --git a/crates/rocm-dash-tui/src/ui/tabs/observe.rs b/crates/rocm-dash-tui/src/ui/tabs/observe.rs index f200ef3b81..5d82108948 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/observe.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/observe.rs @@ -24,9 +24,9 @@ use ratatui::text::{Line, Span}; use ratatui::widgets::Paragraph; use crate::app::{AppState, ConnState}; +use crate::ui::format; use crate::ui::panel::{self, BoxRole}; use crate::ui::theme::Theme; -use crate::ui::{format, widgets}; /// True when we have no live daemon telemetry to show — i.e. not connected to a /// running daemon, or connected but no snapshot has arrived yet. The @@ -67,7 +67,7 @@ pub fn draw(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } /// Two hero panels: Node efficiency (tok/watt + trend) and Node throughput -/// (Σ gen_tps + total power W). Reuses `widgets::node_efficiency` and the +/// (Σ gen_tps + total power W). Reuses `rocm_dash_core::efficiency::node_efficiency` and the /// braille sparkline; honest `—` when a metric is unavailable (color is never /// the only signal — every value carries a text label + unit). fn draw_hero_band(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { @@ -93,22 +93,16 @@ fn draw_efficiency_hero(f: &mut Frame, area: Rect, state: &AppState, theme: &The if inner.height == 0 { return; } - // Track aggregate held: an instance contributes if it has a finite gen_tps - // AND its observation metadata says Held. Mirrors the finite-value filter - // `node_efficiency` applies below, so this can't mark the aggregate held - // over an instance that `node_efficiency` itself excludes as NaN/Inf. - // Any contributing Held → whole aggregate is held. - let any_held = state.latest.as_ref().is_some_and(|snap| { - snap.instances.iter().any(|i| { - i.gen_tps.is_some_and(f64::is_finite) - && i.gen_tps_observation.as_ref().is_some_and(|m| { - m.freshness == rocm_dash_core::metrics::ObservationFreshness::Held - }) - }) - }); - let eff = state.latest.as_ref().and_then(widgets::node_efficiency); + // One shared implementation: the value, and whether any throughput behind it + // is held, come from the same function the Hardware tab and the assistant use. + let node = state + .latest + .as_ref() + .and_then(rocm_dash_core::efficiency::node_efficiency); + let eff = node.map(|n| n.tokens_per_watt); + let any_held = node.is_some_and(|n| n.held()); let value_base = eff.map_or_else(|| "—".to_string(), |v| format!("{v:.2} tok/W")); - let value = if any_held && eff.is_some() { + let value = if any_held { format!("{value_base}{}", format::HELD_MARKER) } else { value_base @@ -118,8 +112,8 @@ fn draw_efficiency_hero(f: &mut Frame, area: Rect, state: &AppState, theme: &The let trend: Vec = state .history .iter() - .filter_map(widgets::node_efficiency) - .map(|v| (v * 1000.0).round().max(0.0) as u64) + .filter_map(rocm_dash_core::efficiency::node_efficiency) + .map(|n| (n.tokens_per_watt * 1000.0).round().max(0.0) as u64) .collect(); // Layout: value | sparkline (min) | label/legend. @@ -162,6 +156,16 @@ fn draw_efficiency_hero(f: &mut Frame, area: Rect, state: &AppState, theme: &The ); } +/// What the node-throughput hero shows, worked out once from the snapshot. +#[derive(Default)] +struct ThroughputHero { + tps: f64, + has_tps: bool, + power: f64, + has_power: bool, + any_held: bool, +} + /// Node-throughput hero: Σ generation tok/s across instances + total board /// power (W). Both honest `—` when absent. fn draw_throughput_hero(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { @@ -176,30 +180,29 @@ fn draw_throughput_hero(f: &mut Frame, area: Rect, state: &AppState, theme: &The if inner.height == 0 { return; } - let (tps, has_tps, power, has_power, any_held) = - state - .latest - .as_ref() - .map_or((0.0, false, 0.0_f32, false, false), |snap| { - let mut tps = 0.0; - let mut has_tps = false; - let mut any_held = false; - for inst in &snap.instances { - // Finite-value filter: a single NaN/Inf sample must not - // poison the whole sum, mirroring `node_efficiency`. - if let Some(v) = inst.gen_tps.filter(|v| v.is_finite()) { - tps += v; - has_tps = true; - if inst.gen_tps_observation.as_ref().is_some_and(|m| { - m.freshness == rocm_dash_core::metrics::ObservationFreshness::Held - }) { - any_held = true; - } - } - } - let power: f32 = snap.gpus.iter().map(|g| g.power_w).sum(); - (tps, has_tps, power, !snap.gpus.is_empty(), any_held) - }); + let ThroughputHero { + tps, + has_tps, + power, + has_power, + any_held, + } = state + .latest + .as_ref() + .map_or_else(ThroughputHero::default, |snap| { + // The same aggregate, and the same notion of "held", as the + // efficiency hero beside it. + let throughput = rocm_dash_core::efficiency::node_throughput(&snap.instances); + let power = rocm_dash_core::efficiency::node_power_w(&snap.gpus); + ThroughputHero { + tps: throughput.map_or(0.0, |t| t.tokens_per_sec), + has_tps: throughput.is_some(), + power, + // Zero or unusable power means no telemetry, not "0 W". + has_power: power > 0.0, + any_held: throughput.is_some_and(|t| t.held()), + } + }); // Aggregate formatted via shared formatter: NaN/Inf guard + held marker. let tps_str = format::gen_tps_aggregate(has_tps.then_some(tps), any_held); let power_str = if has_power { @@ -458,6 +461,55 @@ mod tests { } } + fn last_request_instance( + model: &str, + gen_tps: f64, + obs: Option, + ) -> Instance { + Instance { + gen_tps_last_request: true, + ..instance_with_obs(model, gen_tps, obs) + } + } + + /// Only the two hero panels, so the Instances table (which labels a + /// last-request rate and keeps its own per-row tok/W) cannot satisfy or break + /// an assertion about them. + fn render_heroes(state: &AppState) -> String { + let mut term = Terminal::new(TestBackend::new(160, 12)).unwrap(); + term.draw(|f| draw_hero_band(f, f.area(), state, &state.theme)) + .unwrap(); + term.backend() + .buffer() + .content() + .iter() + .map(ratatui::buffer::Cell::symbol) + .collect() + } + + /// An idle Lemonade server keeps reporting its last request's rate. The Node + /// efficiency and Node throughput heroes must not present it as current: the + /// same rate as a live instance fills them (`1.00 tok/W`, a `tok/s` figure), as a + /// last-request rate they stay at `—`, and held metadata on it adds no marker + /// or legend. + #[test] + fn observe_heroes_leave_out_a_last_request_rate() { + use crate::ui::format; + let live = connected_with_instances(vec![instance_with_obs("m", 300.0, Some(fresh_obs()))]); + let out = render_heroes(&live); + assert!(out.contains("1.00 tok/W"), "baseline: {out}"); + assert!(out.contains(" tok/s"), "baseline: {out}"); + + for obs in [fresh_obs(), held_obs()] { + let last = connected_with_instances(vec![last_request_instance("m", 300.0, Some(obs))]); + let out = render_heroes(&last); + assert!(!out.contains("1.00 tok/W"), "{out}"); + assert!(!out.contains(" tok/s"), "{out}"); + assert!(!out.contains(format::HELD_LEGEND), "{out}"); + assert!(!out.contains(format::HELD_MARKER), "{out}"); + } + } + // ── EAI-7960: HELD_LEGEND rendering and efficiency hero held marker ─────── #[test] @@ -494,12 +546,74 @@ mod tests { out.contains(format::HELD_MARKER), "HELD_MARKER must appear in efficiency hero when any instance is held; got:\n{out}" ); + // The throughput hero prints a `*` too, so pin the efficiency one: the + // marker must sit right after the tok/W figure. + assert!( + out.contains(&format!("tok/W{}", format::HELD_MARKER)), + "efficiency figure must carry the held marker; got:\n{out}" + ); assert!( out.contains(format::HELD_LEGEND), "HELD_LEGEND must appear in a hero when held; got:\n{out}" ); } + #[test] + fn throughput_hero_power_is_a_dash_when_gpus_report_no_usable_power() { + // GPUs that read exactly 0 W have no power telemetry (a platform with no + // power source). The hero used to print `0.0 W` for them; it now says + // `—`, like the other power figures. A usable reading still prints. + let busy = instance_with_obs("busy", 300.0, Some(fresh_obs())); + let mut state = connected_with_instances(vec![busy]); + if let Some(snap) = state.latest.as_mut() { + for g in &mut snap.gpus { + g.power_w = 0.0; + } + } + let out = render(&state, 160, 50); + assert!( + out.contains("power —"), + "no-telemetry power must be a dash:\n{out}" + ); + assert!(!out.contains("power 0.0 W"), "must not print 0.0 W:\n{out}"); + let with_power = + connected_with_instances(vec![instance_with_obs("busy", 300.0, Some(fresh_obs()))]); + assert!(render(&with_power, 160, 50).contains("power 300.0 W")); + } + + #[test] + fn the_two_heroes_agree_for_a_held_instance_at_zero_throughput() { + use crate::ui::format; + // A held instance at 0 tok/s beside a fresh one with real throughput: + // neither hero may show the held marker, and they must say the same. + let busy = instance_with_obs("busy", 300.0, Some(fresh_obs())); + let idle = instance_with_obs("idle", 0.0, Some(held_obs())); + let state = connected_with_instances(vec![busy, idle]); + let out = render(&state, 160, 50); + // The Instances table below marks the held idle row, and prints its + // legend, correctly; only the two heroes are under test, so look at + // their own text. Each shows its figure unmarked and its plain footer + // (not the held legend). + assert!( + out.contains("300.0 tok/s") + && !out.contains(&format!("300.0 tok/s{}", format::HELD_MARKER)), + "{out}" + ); + assert!( + out.contains("1.00 tok/W") + && !out.contains(&format!("1.00 tok/W{}", format::HELD_MARKER)), + "{out}" + ); + assert!( + out.contains("tokens per watt · whole node"), + "efficiency hero footer: {out}" + ); + assert!( + out.contains("Σ generation throughput · all instances"), + "throughput hero footer: {out}" + ); + } + #[test] fn observe_efficiency_hero_no_held_marker_for_fresh_instances() { use crate::ui::format; diff --git a/crates/rocm-dash-tui/src/ui/tabs/pane.rs b/crates/rocm-dash-tui/src/ui/tabs/pane.rs index 0909fb74cb..30b6babc42 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/pane.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/pane.rs @@ -6,9 +6,10 @@ //! //! Each domain tab is a left **Actions** bento (its locked verb list) plus a //! right **`Details: `** bento that previews the selected operation. The -//! verb rows open EXISTING managers through the EXISTING execution seam -//! (`KeyAction::Open*` → `apply_action` → `RocmToolExecutor` / `ui/approval.rs`) -//! — no second approval path, no reimplementation of the managers. +//! verb rows open EXISTING managers: a verb produces a `KeyAction::Open*`, and +//! `apply_action` (in `app/actions.rs`) opens the matching overlay. Nothing +//! here executes anything — no second approval path, no reimplementation of +//! the managers. //! //! `rocm.rs` and `serving.rs` only declare their verb tables; all rendering, //! hit-testing, and the list|detail geometry live here so the two tabs cannot @@ -388,23 +389,56 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> (Vec { let mut lines = vec![head("Engines".into())]; - let lemon = info().and_then(|i| i.lemond_version.clone()); + // External endpoint evidence is live; managed installation evidence + // is a separate launch-time Detect snapshot. GPU metadata carries + // neither verdict and must never imply an absent engine. + let external_running = state + .instances + .values() + .any(|i| i.is_external_lemonade() && i.status.is_serving()); + let lemon = if external_running { + "Lemonade — externally running" + } else { + match state.lemonade_managed_installed { + Some(true) => "Lemonade — managed runtime installed", + Some(false) => "Lemonade — managed runtime not installed", + None => "Lemonade — unknown, run `rocm engines list`", + } + }; lines.push(Line::from(vec![ - mark(lemon.is_some()), - Span::styled( - lemon.map_or_else( - || "Lemonade — not installed".into(), - |v| format!("Lemonade {v}"), - ), - Style::default().fg(theme.fg), - ), + mark(external_running || state.lemonade_managed_installed == Some(true)), + Span::styled(lemon, Style::default().fg(theme.fg)), ])); - // The daemon doesn't surface vLLM detection yet — open the manager to - // check rather than claiming a status we don't have. + if external_running { + let managed = match state.lemonade_managed_installed { + Some(true) => "Managed runtime — installed", + Some(false) => "Managed runtime — not installed", + None => "Managed runtime — unknown, run `rocm engines list`", + }; + lines.push(Line::from(Span::styled( + managed, + Style::default().fg(theme.muted), + ))); + } + // The daemon doesn't surface vLLM detection yet. `rocm engines list` + // prints each engine's Detect result, so point there rather than + // claiming a status we don't have. lines.push(Line::from(vec![ Span::styled("· ", Style::default().fg(theme.muted)), - Span::styled("vLLM — open to check", Style::default().fg(theme.muted)), + Span::styled( + "vLLM — unknown, run `rocm engines list`", + Style::default().fg(theme.muted), + ), ])); + // The managed-runtime verdict is a launch-time snapshot: installing + // from the Engines overlay does not update it. The test + // `managed_install_verdict_is_not_changed_by_live_events` proves it. + if state.lemonade_managed_installed.is_some() { + lines.push(Line::from(Span::styled( + "Install status is read at startup. Restart the dashboard to refresh.", + Style::default().fg(theme.muted), + ))); + } (lines, false) } KeyAction::OpenServeWizard => { @@ -429,13 +463,18 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> (Vec { // Starting services are listed (with their stage) so a fresh serve - // is visible before it passes its readiness probe. - let running = format::running_instances(state); - let mut lines = vec![head(format!("Running now — {}", running.len()))]; + // is visible before it passes its readiness probe. Only services this + // host manages are listed: the Services overlay this pane opens has no + // row for an external or discovered endpoint. + let running: Vec<_> = format::running_instances(state) + .into_iter() + .filter(|i| crate::ui::services_manager::is_listed_service(i)) + .collect(); + let mut lines = vec![head(format!("Managed running — {}", running.len()))]; let mut any_held = false; if running.is_empty() { lines.push(Line::from(Span::styled( - " none running", + " no managed services running", Style::default().fg(theme.muted), ))); } else { @@ -468,7 +507,8 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> (Vec (Vec (Vec::new(), false), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn engine_text(state: &AppState) -> String { + live_lines(KeyAction::OpenEngineManager, state, &Theme::default()) + .0 + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + } + + fn glyph_of(state: &AppState, needle: &str) -> String { + let (lines, _) = live_lines(KeyAction::OpenEngineManager, state, &Theme::default()); + let line = lines + .iter() + .find(|l| l.to_string().contains(needle)) + .unwrap_or_else(|| panic!("no line containing {needle:?}")); + line.spans[0].content.to_string() + } + + #[test] + fn lemonade_line_glyph_is_check_only_when_installed() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(true); + assert_eq!( + glyph_of(&state, "Lemonade — managed runtime installed"), + "✓ " + ); + state.lemonade_managed_installed = Some(false); + assert_eq!( + glyph_of(&state, "Lemonade — managed runtime not installed"), + "· " + ); + state.lemonade_managed_installed = None; + assert_eq!(glyph_of(&state, "Lemonade — unknown"), "· "); + } + + #[test] + fn external_running_lemonade_line_has_check_glyph() { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.instances.insert( + "x".into(), + Instance { + managed: false, + container_id: "x".into(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }, + ); + assert_eq!(glyph_of(&state, "Lemonade — externally running"), "✓ "); + } + + #[test] + fn startup_snapshot_note_shown_only_with_a_verdict() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + assert!(!engine_text(&state).contains("read at startup")); + state.lemonade_managed_installed = Some(false); + let text = engine_text(&state); + // This pins the wording only. That the status really is captured once + // at startup is proved by + // `managed_install_verdict_is_not_changed_by_live_events`. + assert!( + text.contains("Install status is read at startup. Restart the dashboard to refresh."), + "{text}" + ); + assert!( + text.contains("vLLM — unknown, run `rocm engines list`"), + "{text}" + ); + } + + #[test] + fn managed_install_verdict_is_not_changed_by_live_events() { + // Proves the claim the pane's "read at startup" note makes: the verdict + // is set once from the startup args (`event_loop.rs`) and no later + // daemon event rewrites it, so the note is true. + use rocm_dash_core::metrics::{Instance, InstanceStatus, Snapshot}; + use rocm_dash_core::protocol::Event; + + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(false); + let managed = Instance { + managed: true, + container_id: "svc".into(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }; + state.apply_event(Event::InstanceDiscovered(managed)); + state.apply_event(Event::Snapshot(Snapshot::default())); + state.apply_event(Event::InstanceGone { + container_id: "svc".into(), + }); + assert_eq!(state.lemonade_managed_installed, Some(false)); + assert!(engine_text(&state).contains("Lemonade — managed runtime not installed")); + } + + #[test] + fn installed_stopped_managed_lemonade_is_installed_without_gpu_metadata() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(true); + assert!(state.instances.is_empty()); + assert!(engine_text(&state).contains("Lemonade — managed runtime installed")); + assert!(!engine_text(&state).contains("running")); + } + + #[test] + fn no_instances_and_no_managed_runtime_reports_lemonade_not_installed() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(false); + assert!(engine_text(&state).contains("Lemonade — managed runtime not installed")); + } + + #[test] + fn external_running_and_stopped_instances_do_not_invent_managed_installation() { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + let id = "lemonade-localhost-13305-model-61".to_owned(); + state.instances.insert( + id.clone(), + Instance { + container_id: id.clone(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }, + ); + let text = engine_text(&state); + assert!(text.contains("Lemonade — externally running")); + assert!(text.contains("Managed runtime — unknown, run `rocm engines list`")); + state.lemonade_managed_installed = Some(true); + assert!(engine_text(&state).contains("Managed runtime — installed")); + state.instances.get_mut(&id).unwrap().status = InstanceStatus::Stopped; + assert!(engine_text(&state).contains("Lemonade — managed runtime installed")); + assert!(!engine_text(&state).contains("externally running")); + } + + #[test] + fn external_lemonade_verdict_uses_ownership_not_id_prefix() { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + let id = "svc-ordinary-id".to_owned(); + state.instances.insert( + id.clone(), + Instance { + managed: false, + container_id: id.clone(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }, + ); + assert!(engine_text(&state).contains("Lemonade — externally running")); + let external = state.instances.remove(&id).unwrap(); + assert!(!engine_text(&state).contains("externally running")); + state.instances.insert( + "lemonade-looks-external".into(), + Instance { + managed: true, + container_id: "lemonade-looks-external".into(), + ..external + }, + ); + assert!(!engine_text(&state).contains("externally running")); + } + + fn services_text(state: &AppState) -> String { + live_lines(KeyAction::OpenServices, state, &Theme::default()) + .0 + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + } + + fn serving(id: &str, managed: bool) -> rocm_dash_core::metrics::Instance { + rocm_dash_core::metrics::Instance { + managed, + container_id: id.into(), + model_name: format!("model-{id}"), + status: rocm_dash_core::metrics::InstanceStatus::Ready, + ..rocm_dash_core::metrics::Instance::default() + } + } + + #[test] + fn services_preview_ignores_external_only_instances() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + for id in ["ext-a", "ext-b"] { + state.instances.insert(id.into(), serving(id, false)); + } + let text = services_text(&state); + assert!(text.contains("Managed running — 0"), "{text}"); + assert!(text.contains("no managed services running"), "{text}"); + assert!(!text.contains("model-ext"), "{text}"); + // Same set the overlay would list. + assert!(crate::ui::services_manager::service_rows(&state.instances).is_empty()); + } + + #[test] + fn services_preview_labels_a_last_request_rate() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + for (id, last_request) in [("live", false), ("idle", true)] { + let mut inst = serving(id, true); + inst.gen_tps = Some(42.0); + inst.gen_tps_last_request = last_request; + state.instances.insert(id.into(), inst); + } + let text = services_text(&state); + let line_of = |model: &str| { + text.lines() + .find(|l| l.contains(model)) + .unwrap_or_else(|| panic!("{model} missing in {text}")) + .to_owned() + }; + assert!(line_of("model-idle").contains("last req 42"), "{text}"); + assert!(line_of("model-live").contains("gen 42"), "{text}"); + } + + #[test] + fn services_preview_counts_only_managed_instances() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.instances.insert("ext".into(), serving("ext", false)); + state.instances.insert("own".into(), serving("own", true)); + let text = services_text(&state); + assert!(text.contains("Managed running — 1"), "{text}"); + assert!(text.contains("model-own"), "{text}"); + assert!(!text.contains("model-ext"), "{text}"); + } + + #[test] + fn unknown_lemonade_installation_is_not_claimed_absent() { + let state = AppState::new("unix:/unused".into(), "dark".into()); + let (lines, _) = live_lines(KeyAction::OpenEngineManager, &state, &Theme::default()); + let text = lines + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n"); + assert!( + text.contains("Lemonade — unknown, run `rocm engines list`"), + "{text}" + ); + assert!( + !text.contains("not installed"), + "missing GPU metadata is not installation evidence" + ); + } +} diff --git a/crates/rocm-dash-tui/src/ui/tabs/serving.rs b/crates/rocm-dash-tui/src/ui/tabs/serving.rs index ae63e93bf3..74f226a234 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/serving.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/serving.rs @@ -210,6 +210,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -219,7 +220,7 @@ mod tests { let out = render(&s, 120, 28); // The running instance is shown inline in the detail pane — no popup. assert!( - out.contains("Running now"), + out.contains("Managed running"), "live running header missing: {out:?}" ); assert!( @@ -242,6 +243,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Ready, port: Some(8000), @@ -250,7 +252,7 @@ mod tests { ); let out = render(&s, 120, 28); assert!( - out.contains("Running now — 1"), + out.contains("Managed running — 1"), "ready instance not counted as running: {out:?}" ); assert!( @@ -267,6 +269,7 @@ mod tests { s.serving_sel = 2; let mk = |name: &str, status| Instance { container_name: name.into(), + managed: true, model_name: name.into(), status, ..Default::default() @@ -283,7 +286,7 @@ mod tests { s.instances .insert("b".into(), mk("Dead-Model", InstanceStatus::Stopped)); let out = render(&s, 120, 28); - assert!(out.contains("Running now — 1"), "{out:?}"); + assert!(out.contains("Managed running — 1"), "{out:?}"); assert!(out.contains("Starting-Model"), "{out:?}"); assert!(out.contains("downloading"), "{out:?}"); assert!(!out.contains("none running"), "{out:?}"); @@ -296,6 +299,7 @@ mod tests { ) -> rocm_dash_core::metrics::Instance { rocm_dash_core::metrics::Instance { container_name: name.into(), + managed: true, model_name: name.into(), status: rocm_dash_core::metrics::InstanceStatus::Starting { phase }, ..Default::default() @@ -315,7 +319,7 @@ mod tests { s.instances .insert("a".into(), starting_inst("Docker-Model", None)); let out = render(&s, 120, 28); - assert!(out.contains("Running now — 1"), "{out:?}"); + assert!(out.contains("Managed running — 1"), "{out:?}"); assert!( out.contains("docker-model") || out.contains("Docker-Model"), "{out:?}" @@ -346,6 +350,7 @@ mod tests { "b".into(), Instance { container_name: "Z-Serving".into(), + managed: true, model_name: "Z-Serving".into(), status: InstanceStatus::Ready, ..Default::default() @@ -365,7 +370,7 @@ mod tests { .insert(format!("s{n}"), starting_inst(&format!("Start-{n}"), None)); } let out = render(&s, 120, 40); - assert!(out.contains("Running now — 7"), "{out:?}"); + assert!(out.contains("Managed running — 7"), "{out:?}"); assert!(out.contains("…and 2 more"), "{out:?}"); } @@ -399,6 +404,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -437,6 +443,7 @@ mod tests { format!("vllm-{n}"), Instance { container_name: format!("vllm-{n}"), + managed: true, model_name: format!("Model-{n}"), status: InstanceStatus::Running, port: Some(8000 + n), @@ -450,7 +457,7 @@ mod tests { ); } // area.height=8 → bento inner.height=5 for the detail column, tall - // enough for summary + blank + "Running now" header + the first + // enough for summary + blank + "Managed running" header + the first // running row, but far short of the old inline legend's position // after 5 rows + the "…and N more" line. let out = render(&s, 160, 8); @@ -472,6 +479,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -500,6 +508,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -530,6 +539,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), diff --git a/crates/rocm-dash-tui/src/ui/theme.rs b/crates/rocm-dash-tui/src/ui/theme.rs index d875802791..b90827a168 100644 --- a/crates/rocm-dash-tui/src/ui/theme.rs +++ b/crates/rocm-dash-tui/src/ui/theme.rs @@ -11,7 +11,7 @@ //! sourced from each project's published palette. Theme catalogue //! inspired by [ansicolor.com](https://ansicolor.com/). //! -//! Pattern borrowed from ctux (see `../../../wiki/sources/ctux.md`). +//! Pattern borrowed from ctux. use ratatui::style::{Color, Modifier, Style}; use rocm_dash_core::metrics::InstanceStatus; diff --git a/crates/rocm-dash-tui/src/ui/update_manager.rs b/crates/rocm-dash-tui/src/ui/update_manager.rs index 49e9c80896..4b6c6ac2aa 100644 --- a/crates/rocm-dash-tui/src/ui/update_manager.rs +++ b/crates/rocm-dash-tui/src/ui/update_manager.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Update overlay (Phase 3 Wave 2). +//! Update overlay. //! //! Wraps `rocm update`: a read-only check (and a `--apply --dry-run` preview) //! that run straight through the job-bridge, plus the mutating `--apply` / diff --git a/crates/rocm-dash-tui/src/ui/widgets.rs b/crates/rocm-dash-tui/src/ui/widgets.rs index f429ecdbda..28c6afb500 100644 --- a/crates/rocm-dash-tui/src/ui/widgets.rs +++ b/crates/rocm-dash-tui/src/ui/widgets.rs @@ -129,29 +129,6 @@ pub fn instances_on_gpu<'a>(device_id: &str, instances: &'a [Instance]) -> Vec<& .collect() } -/// Node-level energy efficiency: total generation throughput divided by total board power, in tokens per watt. -/// -/// `None` when there is no traffic -/// (`sum gen_tps == 0`) or no power telemetry (`sum power_w == 0`), or when the -/// result is non-finite. -pub fn node_efficiency(snap: &Snapshot) -> Option { - // A single NaN/Inf `gen_tps` sample must not poison the whole sum — - // same guard commit 0bf99ac added to `home.rs`'s hero aggregates. - let tps: f64 = snap - .instances - .iter() - .filter_map(|i| i.gen_tps) - .filter(|v| v.is_finite()) - .sum(); - let power: f64 = snap.gpus.iter().map(|g| f64::from(g.power_w)).sum(); - if tps > 0.0 && power > 0.0 { - let eff = tps / power; - eff.is_finite().then_some(eff) - } else { - None - } -} - /// One-line GPU stats: id, util, vram, temp, power. pub fn gpu_stats_line<'a>(g: &'a GpuMetrics, theme: &Theme) -> Line<'a> { let vram_pct = if g.vram_total_mb > 0 { @@ -186,6 +163,25 @@ pub fn gpu_stats_line<'a>(g: &'a GpuMetrics, theme: &Theme) -> Line<'a> { ]) } +/// Map snapshots to a sparkline series, skipping those with no GPUs. +/// +/// Every GPU `pick` folds or sums over `gpus`, so a GPU-less snapshot yields 0. +/// The braille widgets draw a baseline for every sample they are handed, so +/// letting those through paints a host with no GPU as one sitting at 0 — a +/// measured zero rather than an absent reading. One rule, one place: `home`, +/// `dock` and the GPU panels all go through here. +pub fn gpu_history<'a, I, F>(snapshots: I, pick: F) -> Vec +where + I: IntoIterator, + F: Fn(&Snapshot) -> f64, +{ + snapshots + .into_iter() + .filter(|s| !s.gpus.is_empty()) + .map(|s| pick(s).clamp(0.0, 100.0) as u64) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -203,14 +199,6 @@ mod tests { } } - fn gpu(device_id: &str, power_w: f32) -> GpuMetrics { - GpuMetrics { - device_id: device_id.into(), - power_w, - ..Default::default() - } - } - #[test] fn trunc_keeps_short_strings() { assert_eq!(trunc("abc", 5), "abc"); @@ -318,56 +306,6 @@ mod tests { assert!(instances_on_gpu("gpu-7", &[]).is_empty()); } - #[test] - fn node_efficiency_divides_tps_by_power() { - let snap = Snapshot { - gpus: vec![gpu("gpu-0", 400.0), gpu("gpu-1", 600.0)], - instances: vec![ - inst("a", &["0"], Some(300.0)), - inst("b", &["1"], Some(200.0)), - ], - ..Default::default() - }; - // (300 + 200) / (400 + 600) = 0.5 tok/W - let eff = node_efficiency(&snap).expect("some"); - assert!((eff - 0.5).abs() < 1e-9, "got {eff}"); - } - - #[test] - fn node_efficiency_none_without_power_or_traffic() { - // no power - let no_power = Snapshot { - gpus: vec![gpu("gpu-0", 0.0)], - instances: vec![inst("a", &["0"], Some(100.0))], - ..Default::default() - }; - assert_eq!(node_efficiency(&no_power), None); - // no traffic - let no_traffic = Snapshot { - gpus: vec![gpu("gpu-0", 500.0)], - instances: vec![inst("a", &["0"], None)], - ..Default::default() - }; - assert_eq!(node_efficiency(&no_traffic), None); - } - - #[test] - fn node_efficiency_ignores_non_finite_gen_tps() { - // A NaN sample from one instance must not poison the sum for the - // whole node — the same class of bug commit 0bf99ac fixed for - // `home.rs`'s hero aggregates. - let snap = Snapshot { - gpus: vec![gpu("gpu-0", 500.0)], - instances: vec![ - inst("finite", &["0"], Some(250.0)), - inst("poisoned", &["0"], Some(f64::NAN)), - ], - ..Default::default() - }; - let eff = node_efficiency(&snap).expect("finite gen_tps must still produce a result"); - assert!((eff - 0.5).abs() < 1e-9, "got {eff}"); - } - #[test] fn apply_char_key_appends_chars_in_order() { let mut buf = String::from("ab"); @@ -428,4 +366,31 @@ mod tests { )); assert_eq!(buf, "a"); } + + #[test] + fn gpu_history_skips_snapshots_with_no_gpus() { + // One rule for home, dock and the GPU panels: a snapshot with no GPUs + // is absent from the series, not a measured zero. Every `pick` folds + // over `gpus`, so letting one through would draw a baseline for a host + // that reported nothing. + let with_gpu = |util: f32| Snapshot { + gpus: vec![GpuMetrics { + gpu_utilization_pct: util, + ..Default::default() + }], + ..Default::default() + }; + let snaps = [Snapshot::default(), with_gpu(40.0), Snapshot::default()]; + let series = gpu_history(snaps.iter(), |s| { + s.gpus + .iter() + .map(|g| f64::from(g.gpu_utilization_pct)) + .fold(0.0, f64::max) + }); + assert_eq!(series, vec![40], "a GPU-less snapshot reached the series"); + assert!( + gpu_history([Snapshot::default()].iter(), |_| 0.0).is_empty(), + "a host with no GPU at all should draw nothing" + ); + } } diff --git a/crates/rocm-dash-tui/tests/dash_characterization.rs b/crates/rocm-dash-tui/tests/dash_characterization.rs index fc7f8e659a..43fa0cf537 100644 --- a/crates/rocm-dash-tui/tests/dash_characterization.rs +++ b/crates/rocm-dash-tui/tests/dash_characterization.rs @@ -181,9 +181,11 @@ fn observe_empty_state_shows_placeholders() { }; s.latest = Some(synthetic_snapshot()); let out = render(&mut s, 160, 44); + // @id:dash-lemonade-external-protocol-transitions proves model absence + // together with this message and navigation to the named Serving view. assert!( - out.contains("no instances"), - "empty instances placeholder: {out:?}" + out.contains("no serving models discovered"), + "empty serving-model placeholder: {out:?}" ); } diff --git a/crates/rocm-deps/src/lib.rs b/crates/rocm-deps/src/lib.rs index e8044d7602..ee95efbf0d 100644 --- a/crates/rocm-deps/src/lib.rs +++ b/crates/rocm-deps/src/lib.rs @@ -6,9 +6,9 @@ //! helpers that derive artifact names and URLs from them. //! //! The pins live in the workspace-root `runtime-deps.toml` and are turned into -//! the constants below by `build.rs`. Every consumer — the engine adapters and -//! the dashboard TUI alike — derives what it needs from those constants, so a -//! runtime version is spelled exactly once in the repository and two crates +//! the constants below by `build.rs`. Every consumer — the `rocm` binary and +//! the Lemonade engine adapter — derives what it needs from those constants, so +//! a runtime version is spelled exactly once in the repository and two crates //! cannot disagree about it. // `pub const LEMONADE_VERSION: &str = "..."`, one per `runtime-deps.toml` field. diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index 19bd5f483a..cdb00339fa 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -10,7 +10,21 @@ publish.workspace = true [lints] workspace = true +[features] +# Compiles `fake_engine`, a stand-in engine child for tests, into the `rocm` and +# `rocmd` test builds. Never enabled in a release build. +test-support = [] + [dependencies] +anyhow.workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true + +[dev-dependencies] +# Property tests for the engine protocol decoders (`tests/protocol_properties.rs`). +proptest = { version = "1", default-features = false, features = ["std"] } + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test installs in a re-executed child. +libc.workspace = true diff --git a/crates/rocm-engine-protocol/src/fake_engine.rs b/crates/rocm-engine-protocol/src/fake_engine.rs new file mode 100644 index 0000000000..dc90a42449 --- /dev/null +++ b/crates/rocm-engine-protocol/src/fake_engine.rs @@ -0,0 +1,174 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A stand-in engine child for tests. +//! +//! It is a `/bin/sh` script that records how it was launched (arguments, +//! environment, stdin) into a scratch directory and then runs a caller-supplied +//! body, so a test can assert what an engine IPC client put on the child's +//! command line, environment and stdin, and shape what comes back (a response +//! envelope, a crash, stderr output). +//! +//! Test-only: compiled for this crate's own tests and for the `rocm` and +//! `rocmd` test builds through the `test-support` feature, so the two engine +//! IPC callers are exercised against the same fake rather than each growing a +//! private one. Unix-only, because the fake is a shell script; the pure parts +//! of the client are tested on every platform without it. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; + +static NEXT_ENGINE: AtomicU32 = AtomicU32::new(0); + +/// `value` as a single-quoted POSIX shell word. Inside single quotes nothing is +/// special except the quote itself, which cannot be escaped there: close the +/// literal, emit an escaped quote, and reopen it (`'\''`). An apostrophe in a +/// response (`can't stop`) or in `TMPDIR` would otherwise end the literal early. +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', r"'\''")) +} + +pub struct FakeEngine { + dir: PathBuf, + script: PathBuf, +} + +impl FakeEngine { + /// Write the script. `reads_stdin` makes the fake drain its stdin into + /// `stdin` first (what a well-behaved engine does); pass `false` for an + /// engine that exits without reading its request. `body` runs after the + /// recording and may use `$dir`, the scratch directory. + /// + /// The script is written through [`rocm_core::fake_executable`], so running + /// it straight away cannot fail with `ETXTBSY` while other tests on parallel + /// threads spawn children of their own. + pub fn new(label: &str, reads_stdin: bool, body: &str) -> Self { + Self::new_in(&std::env::temp_dir(), label, reads_stdin, body) + } + + /// [`FakeEngine::new`] with its scratch directory created under `base` + /// instead of the system temporary directory. + pub fn new_in(base: &Path, label: &str, reads_stdin: bool, body: &str) -> Self { + let dir = base.join(format!( + "rocm-fake-engine-{label}-{}-{}", + std::process::id(), + NEXT_ENGINE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&dir).expect("create the fake engine's scratch directory"); + let script = dir.join("engine.sh"); + let read_stdin = if reads_stdin { + "cat > \"$dir/stdin\"\n" + } else { + "" + }; + rocm_core::fake_executable::write_fake_executable( + &script, + format!( + "#!/bin/sh\ndir={}\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", + shell_quote(&dir.to_string_lossy()) + ), + ) + .expect("write the fake engine script"); + Self { dir, script } + } + + /// A fake that reads its request and answers with `response` (a complete + /// response envelope as JSON text), exiting with `status`. + pub fn answering(label: &str, response: &str, status: i32) -> Self { + Self::new( + label, + true, + &format!("printf '%s' {}\nexit {status}", shell_quote(response)), + ) + } + + pub fn path(&self) -> &Path { + &self.script + } + + /// The arguments the fake was started with, one per entry. + pub fn argv(&self) -> Vec { + fs::read_to_string(self.dir.join("argv")) + .expect("the fake engine never recorded its arguments") + .lines() + .map(str::to_owned) + .collect() + } + + /// Everything the fake read from its stdin, verbatim. + pub fn stdin(&self) -> String { + fs::read_to_string(self.dir.join("stdin")).expect("the fake engine never read its stdin") + } + + /// The value of `key` in the fake's environment, if it was set. + pub fn env(&self, key: &str) -> Option { + let prefix = format!("{key}="); + fs::read_to_string(self.dir.join("env")) + .expect("the fake engine never recorded its environment") + .lines() + .find_map(|line| line.strip_prefix(&prefix).map(str::to_owned)) + } +} + +impl Drop for FakeEngine { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + use std::process::{Command, Stdio}; + + /// Run `engine` the way a client does: request on stdin, answer on stdout. + fn run(engine: &FakeEngine) -> (std::process::ExitStatus, String, String) { + let mut child = Command::new(engine.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"request\n").unwrap(); + let output = child.wait_with_output().unwrap(); + ( + output.status, + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) + } + + #[test] + fn shell_quote_survives_each_character_a_literal_can_hold() { + assert_eq!(shell_quote("plain"), "'plain'"); + assert_eq!(shell_quote("can't"), r"'can'\''t'"); + assert_eq!(shell_quote("''"), r"''\'''\'''"); + assert_eq!(shell_quote(""), "''"); + } + + #[test] + fn a_response_with_an_apostrophe_is_printed_verbatim() { + let response = r#"{"ok":false,"error":{"code":"x","message":"can't stop"}}"#; + let engine = FakeEngine::answering("fake-apostrophe-response", response, 0); + let (status, stdout, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stdout, response); + assert_eq!(stderr, "", "the script must parse"); + } + + #[test] + fn a_scratch_directory_with_an_apostrophe_still_records_the_launch() { + let base = + std::env::temp_dir().join(format!("rocm-fake-engine-it's-{}", std::process::id())); + let engine = FakeEngine::new_in(&base, "fake-apostrophe-dir", true, "exit 0"); + let (status, _, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stderr, "", "the script must parse"); + assert_eq!(engine.stdin(), "request\n"); + drop(engine); + let _ = fs::remove_dir_all(&base); + } +} diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6ca773ccd6..6b3f4065bf 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -2,6 +2,15 @@ // // SPDX-License-Identifier: MIT +#[cfg(all(unix, any(test, feature = "test-support")))] +pub mod fake_engine; + +mod stdio_client; +pub use stdio_client::{ + ENGINE_PROGRESS_STDERR_ENV, EngineStdioOptions, decode_engine_response, + engine_request_envelope, run_engine_stdio, +}; + use serde::Deserialize; use serde::Serialize; use serde_json::Value; @@ -280,6 +289,17 @@ impl GpuSelection { } } +/// The two arguments `flag value` when `value` is set, none when it is not, for +/// building the command line of an engine or supervisor child. Shared so `rocm` +/// and `rocmd` spell optional flags the same way. +#[must_use] +pub fn optional_arg(flag: &str, value: Option<&str>) -> Vec { + match value { + Some(value) => vec![flag.to_owned(), value.to_owned()], + None => Vec::new(), + } +} + /// Render an explicit GPU ordinal as a comma-separated `HIP_VISIBLE_DEVICES` /// value. Returns `None` when the slice is empty (no pinning requested). #[must_use] @@ -892,6 +912,15 @@ mod tests { assert!(GpuSelection::parse_cli_value("0,2,3").is_err()); } + #[test] + fn optional_arg_is_the_flag_and_value_or_nothing() { + assert_eq!( + optional_arg("--runtime-id", Some("therock-release:gfx120X-all")), + ["--runtime-id", "therock-release:gfx120X-all"] + ); + assert!(optional_arg("--runtime-id", None).is_empty()); + } + #[test] fn gpu_indices_helpers_handle_auto_and_pinned() { assert_eq!(gpu_indices_to_csv(&[]), None); diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs new file mode 100644 index 0000000000..2dd776d15b --- /dev/null +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -0,0 +1,2052 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The client side of the engine stdio protocol: run an engine as a child +//! process, send it one [`EngineRequestEnvelope`] on stdin as a single line of +//! JSON, and read one [`EngineResponseEnvelope`] back from stdout. +//! +//! `rocm` and `rocmd` both talk to engines this way and share this one +//! implementation. They differ in *which* process they start (`rocm` runs the +//! resolved engine binary with `stdio`, or answers a built-in engine in its own +//! process without a child at all; `rocmd` runs `rocm __engine-stdio `), +//! and that stays with each caller. What is shared is everything that follows +//! the choice of program: the request framing, the environment, how stderr is +//! handled, the write guard against SIGPIPE, what a failing exit status or an +//! unusable answer is reported as, and decoding the response. The few things +//! the callers want differently are the fields of [`EngineStdioOptions`]. +//! +//! The request and the child's lifetime are not bounded by a timeout: an engine +//! install legitimately runs for minutes, and no caller has ever set one. +//! +//! Nor is the wait for the pipes after the child has exited, and that is a known +//! limit rather than an oversight. The answer is read to the end of the child's +//! stdout, and the stderr collector is joined, so an engine that leaves a +//! grandchild holding either pipe open makes the request wait for that grandchild +//! to exit. Both clients this one replaces behaved the same way (`rocm` read +//! stdout to the end then joined its stderr thread; `rocmd` used +//! `wait_with_output`), so nothing new is exposed to it, but the daemon is long +//! lived and a hung request holds its caller. Bounding it exactly would mean +//! reading both pipes on their own threads, waiting for the child itself without a +//! limit and for the pipes only up to a deadline after it exits, and choosing what +//! to report when the deadline passes (an answer already complete? a partial +//! one?). That is a timeout policy for an engine contract nothing here sets, so +//! this change leaves it to be decided on its own. The one wait this client adds +//! itself, for the stderr of a child whose request could not be written, is +//! bounded by `EXITED_STDERR_GRACE`. +//! +//! A request that cannot be written is the other place the answer-wins rule does +//! not apply: if the engine answered and exited without draining a request larger +//! than a pipe buffer, the write fails, the child is killed and its stdout is +//! never read, so the error carries its stderr and not its answer. Every engine +//! here (`lemonade`, `vllm`, `aim`, and the `rocm __engine-stdio` verb that serves +//! the built-in ones) reads stdin to the end before it answers, so reaching this +//! needs an engine that breaks the protocol. Decoding such an answer would mean +//! waiting, up to a deadline, for the child to finish printing it and to exit +//! before deciding it is complete, and an engine that has stopped reading is no +//! engine to wait for, so it is left as a documented limit. +//! +//! Forwarding the child's stderr guards each write against SIGPIPE on its own +//! (see `forward_best_effort`), and the guard is held for as long as that write +//! blocks. If the sink is stalled rather than closed (a terminal that has stopped +//! reading), SIGPIPE therefore stays ignored process-wide for that time, and a +//! concurrent `print!` to a closed pipe on another thread fails with an error +//! (which `print!` turns into a panic) instead of ending the process quietly. The +//! guard is not held across the reads between lines, so this needs a stalled +//! sink, not just a quiet engine. It is accepted: a guard that could be taken +//! back mid-write does not exist, and without one a closed sink would end +//! `rocm` before the request could report why. +//! +//! These three are deliberate tradeoffs, not oversights: the wait on pipes held +//! by a grandchild is unbounded, an answer from an engine that exits without +//! draining a large request is not decoded, and SIGPIPE stays ignored +//! process-wide while a stalled stderr forward blocks. Only the first is +//! partly cheap to bound: the stderr side would be one deadline on +//! `StderrCollector::finish`, but stdout is read to its end on the calling thread, +//! so a bound there needs a second reader thread and a decision on what to report +//! when it fires, and bounding one pipe and not the other would be a half +//! measure. The stderr is read from the moment the child starts, on its own +//! thread, so an engine that fills it before draining a large request cannot +//! deadlock the write. + +use std::ffi::OsStr; +use std::io::{self, Read, Write}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc; +use std::sync::{Arc, Mutex, PoisonError}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, with_sigpipe_ignored}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::{ + ENDPOINT_API_KEY_FILE_ENV, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, +}; + +/// Set to `1` on an engine child whose caller wants its progress lines on +/// stderr as they happen (an engine install). +pub const ENGINE_PROGRESS_STDERR_ENV: &str = "ROCM_ENGINE_PROGRESS_STDERR"; + +/// What a caller wants different about the child [`run_engine_stdio`] starts. +/// +/// The default asks for nothing beyond the request itself: the child inherits +/// the caller's environment and its stderr is only collected. +#[derive(Debug, Clone, Copy, Default)] +pub struct EngineStdioOptions<'a> { + /// Run the child on these roots, by setting the `ROCM_CLI_*_DIR` variables + /// ([`AppPaths::env_vars`]). `None` leaves the child to inherit whatever the + /// caller's own environment says. + pub app_paths: Option<&'a AppPaths>, + /// The root engine virtualenvs go under for this request, set as + /// [`ENGINE_ENVS_ROOT_ENV`]. Only an install into a particular root needs it. + pub env_root: Option<&'a Path>, + /// A key file to hand the engine through [`ENDPOINT_API_KEY_FILE_ENV`], so + /// its HTTP probe of a protected endpoint authenticates. The caller decides + /// whether a file holds a usable key; this only carries it. + pub endpoint_key_file: Option<&'a Path>, + /// With no `endpoint_key_file`, also CLEAR [`ENDPOINT_API_KEY_FILE_ENV`] + /// rather than leave whatever this process inherited to pass through. + /// + /// For a child that must not see a credential path at all — `rocm`'s + /// external engine plugins are third-party binaries. Off by default, + /// because "no key file for this request" and "this child may not have one" + /// are different statements: `rocmd` passes `None` for a loopback service + /// that simply needs no key, and its children are its own. + pub withhold_inherited_key_file: bool, + /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). + pub request_progress: bool, + /// Write the child's stderr to this process's stderr as it arrives, so a + /// terminal user watches an install. Either way the whole of it is kept and + /// shown in the error when the request fails. + pub forward_stderr: bool, + /// Say, on this process's stderr, when the child answered and then exited + /// with a failing status. The answer is used either way; this only leaves a + /// trace of the status, which the caller would otherwise never see. For a + /// daemon, whose stderr is its log; a terminal user has no use for it. + pub note_failing_exit_after_answer: bool, +} + +/// Build the request envelope for `method` from `request`. +pub fn engine_request_envelope( + method: EngineMethod, + request: &T, +) -> Result { + Ok(EngineRequestEnvelope { + method, + payload: serde_json::to_value(request) + .context("failed to encode engine request payload")?, + }) +} + +/// The command [`run_engine_stdio`] runs: `program` with `args`, all three +/// standard streams piped, and the environment `options` asks for. Separate so +/// a test can read the command line and environment without starting anything. +fn engine_stdio_command(program: &Path, args: I, options: &EngineStdioOptions<'_>) -> Command +where + I: IntoIterator, + S: AsRef, +{ + let mut command = Command::new(program); + command + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(paths) = options.app_paths { + paths.apply_env(&mut command); + } + if let Some(env_root) = options.env_root { + command.env(ENGINE_ENVS_ROOT_ENV, env_root); + } + if let Some(key_file) = options.endpoint_key_file { + command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); + } else if options.withhold_inherited_key_file { + // Cleared, not merely left unset: a value this process inherited must + // not reach a third-party plugin binary just because nothing + // overwrote it. Only for callers that ask — `rocmd` passes `None` for + // a loopback service that needs no key, and clearing there would + // change what its own children see. See + // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. + command.env_remove(ENDPOINT_API_KEY_FILE_ENV); + } + if options.request_progress { + command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); + } + command +} + +/// Send `envelope` to the engine child `program args…` and return what it +/// answered. +/// +/// A failing exit status is an error only when the child printed no answer: an +/// engine that reports its own failure in an error envelope and then exits +/// non-zero is read as that envelope, which says more than the status does. The +/// same holds for a success envelope (`ok: true`): a child that answers +/// successfully and then exits non-zero is reported as the success it answered, +/// and the failing exit is not reported or logged anywhere, by design, because +/// the answer is what the caller asked for. +/// Every other failure names what went wrong and carries the child's stderr. +pub fn run_engine_stdio( + program: &Path, + args: I, + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, +) -> Result +where + I: IntoIterator, + S: AsRef, +{ + run_engine_stdio_forwarding( + program, + args, + options, + envelope, + options.forward_stderr.then(io::stderr), + ) +} + +/// [`run_engine_stdio`] with the sink the child's stderr is forwarded to chosen +/// by the caller (`None` forwards nothing), so a test can see what is forwarded +/// without capturing this process's own stderr. [`run_engine_stdio`] is the one +/// place the `forward_stderr` option becomes a sink. +fn run_engine_stdio_forwarding( + program: &Path, + args: I, + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, + forward: Option, +) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let mut child = engine_stdio_command(program, args, options) + .spawn() + .with_context(|| format!("failed to spawn engine stdio process {}", program.display()))?; + + let stdin = child.stdin.take(); + let mut stdin = require_pipe(&mut child, stdin, "stdin")?; + // Read the child's stderr from the start, on its own thread: an engine that + // fills the stderr pipe before it has drained a request larger than a pipe + // buffer would otherwise block on that write while this process blocks on + // its own, with nobody reading either side. + let stderr_collector = child + .stderr + .take() + .map(|stderr| StderrCollector::start(stderr, forward)); + // Ignore SIGPIPE for the duration of the write. Under the default action, an + // engine that exits before reading its stdin would kill this process with + // SIGPIPE (`rocm` runs that way, and so does the daemon hosted in it); + // ignored, the write returns EPIPE and fails as an ordinary error. + let written = with_sigpipe_ignored(|| { + serde_json::to_writer(&mut stdin, envelope).map_err(io::Error::from)?; + stdin.write_all(b"\n") + }); + if let Err(error) = written { + return Err(abandon_unwritten_request( + child, + stderr_collector, + &error, + EXITED_STDERR_GRACE, + )); + } + // End of the request: the engine reads to EOF before it answers. + drop(stdin); + + let stdout_pipe = child.stdout.take(); + let stdout_pipe = require_pipe(&mut child, stdout_pipe, "stdout")?; + let stdout = match read_response(&mut child, stdout_pipe) { + Ok(stdout) => stdout, + Err(error) => { + return Err(read_failure_with_stderr( + error, + stderr_collector, + EXITED_STDERR_GRACE, + )); + } + }; + let status = child + .wait() + .context("failed waiting for engine stdio response")?; + let stderr = stderr_collector + .map(|collector| collector.finish(None)) + .unwrap_or_default(); + + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if !status.success() && stdout.is_empty() { + let exited = exit_note(status); + if stderr.is_empty() { + bail!("{exited}"); + } + bail!("{exited}: {stderr}"); + } + let answer = serde_json::from_slice(&stdout) + .with_context(|| unparseable_answer_message(status, &stderr)); + // The note says the answer was used, so it is only made for an answer that + // parsed; an unparseable one is reported by the error above, with the status. + if answer.is_ok() && !status.success() && options.note_failing_exit_after_answer { + note_failing_exit_after_answer(&mut io::stderr(), status); + } + answer +} + +/// Say on `sink` (this process's stderr, in production) that the child answered +/// and then failed. Written like the stderr forwarding, through the SIGPIPE guard +/// and ignoring a failed write: a closed stderr must neither end a daemon hosted +/// in `rocm` (SIGPIPE at its default action) nor panic it, as `eprintln!` would. +fn note_failing_exit_after_answer(sink: &mut impl Write, status: ExitStatus) { + let note = format!( + "{} after printing its answer; the answer was used\n", + exit_note(status) + ); + forward_best_effort(sink, note.as_bytes()); +} + +/// How a failing exit is worded wherever the status is reported. Built from the +/// exit code (or the signal that ended the process) rather than from the status's +/// own `Display`, which already says "exit status: N" on Unix and "exit code: N" +/// on Windows. +fn exit_note(status: ExitStatus) -> String { + if let Some(code) = status.code() { + return format!("engine stdio process exited with code {code}"); + } + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + if let Some(signal) = status.signal() { + return format!("engine stdio process was terminated by signal {signal}"); + } + } + "engine stdio process ended without an exit code".to_owned() +} + +/// The error for an answer that is not a response envelope. A child that also +/// exited with a failing status says so, because that is often why the answer is +/// cut short; the stderr it wrote follows when there is any. +fn unparseable_answer_message(status: ExitStatus, stderr: &str) -> String { + let mut message = "failed to parse engine response envelope".to_owned(); + if !status.success() { + message = format!("{message} ({})", exit_note(status)); + } + if !stderr.is_empty() { + message = format!("{message}; stderr: {stderr}"); + } + message +} + +/// `pipe` of `child`, or an error that also kills and reaps the child: a child +/// whose request cannot be sent must not be left running, as on every other +/// failure path. Unreachable while the command pipes all three streams, which is +/// why it is a function of its own, testable with a pipe that is missing. +fn require_pipe(child: &mut Child, pipe: Option, name: &str) -> Result { + pipe.ok_or_else(|| { + let _ = child.kill(); + let _ = child.wait(); + anyhow::anyhow!("engine stdio child did not expose {name}") + }) +} + +/// The error for a stdout that could not be read (the child is already killed +/// and reaped), with what its stderr had said, as every other failure carries +/// it: a crashing or signalled engine is where the explanation matters most. +fn read_failure_with_stderr( + error: anyhow::Error, + stderr: Option, + grace: Duration, +) -> anyhow::Error { + let stderr = stderr + .map(|collector| collector.finish(Some(Instant::now() + grace))) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if stderr.is_empty() { + return error; + } + anyhow::anyhow!("{error:#}; stderr: {stderr}") +} + +/// How long the stderr of a child that was killed, or had exited, is waited for +/// when its request could not be written. +const EXITED_STDERR_GRACE: Duration = Duration::from_secs(1); + +/// The request could not be written to `child`'s stdin, so it will never answer: +/// do not leave it running or unreaped (a long-lived caller would collect one +/// zombie per failure), and report why it was not listening. +/// +/// The child is killed (nothing to do for one that has already exited) and +/// waited for, then whatever it wrote to stderr is added to the error: an engine +/// that exits at once usually says why there, and that is the explanation the +/// caller wants. The kill does not wait on `try_wait` showing an exit first, +/// because the pipe closes while the process is still exiting, so a check made +/// right after the failed write can still see it running. The read is bounded +/// by `grace`: a grandchild that inherited the pipe keeps it open past the exit, +/// and this error path must not block on it. +fn abandon_unwritten_request( + mut child: Child, + stderr: Option, + write_error: &io::Error, + grace: Duration, +) -> anyhow::Error { + let _ = child.kill(); + let _ = child.wait(); + let deadline = Instant::now() + grace; + let stderr = stderr + .map(|collector| collector.finish(Some(deadline))) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + let mut message = format!("failed to write engine request: {write_error}"); + if !stderr.is_empty() { + message = format!("{message}; stderr: {stderr}"); + } + anyhow::anyhow!(message) +} + +/// A child's stderr being collected on a thread of its own, from the moment it +/// is started, whatever the main thread is doing meanwhile. +/// +/// The thread outlives the collector when a request fails and a descendant of +/// the engine still holds the pipe: nobody will read what it collects, but it +/// must keep draining the pipe (closing it would hand that descendant a SIGPIPE +/// it did not get before). So dropping the collector marks the capture +/// abandoned, which releases what was retained and makes the thread discard +/// whatever arrives later, in chunks of a fixed size, so a long-lived daemon does +/// not grow with every failed request. +struct StderrCollector { + shared: Arc>, + /// Receives one message when the stderr has been read to its end. + finished: mpsc::Receiver<()>, +} + +/// What the collecting thread has read, and whether anyone still wants it. +#[derive(Default)] +struct Captured { + bytes: Vec, + /// Set when the collector is dropped. Read and written under the same lock as + /// `bytes`, so "emptied and marked" is one step as far as the reader is + /// concerned and the buffer cannot grow after the drop returns. + abandoned: bool, +} + +impl StderrCollector { + /// Start reading `stderr`, forwarding what arrives to `forward` when there is + /// one (see [`collect_stderr_into`]). + fn start( + stderr: impl Read + Send + 'static, + forward: Option, + ) -> Self { + let shared = Arc::new(Mutex::new(Captured::default())); + let (done, finished) = mpsc::channel(); + let sink = Arc::clone(&shared); + thread::spawn(move || { + collect_stderr_into(stderr, forward, &sink); + let _ = done.send(()); + }); + Self { shared, finished } + } + + /// What was collected, once the stderr has ended, or what has arrived by + /// `deadline` when there is one and the stderr has not ended by then (a + /// grandchild holding the pipe open). With no deadline this waits for the + /// end, which is the unbounded wait the module comment states. + fn finish(&self, deadline: Option) -> Vec { + match deadline { + None => { + let _ = self.finished.recv(); + } + Some(deadline) => { + let _ = self + .finished + .recv_timeout(deadline.saturating_duration_since(Instant::now())); + } + } + std::mem::take(&mut lock(&self.shared).bytes) + } +} + +impl Drop for StderrCollector { + fn drop(&mut self) { + let mut captured = lock(&self.shared); + captured.abandoned = true; + captured.bytes = Vec::new(); + } +} + +fn lock(shared: &Mutex) -> std::sync::MutexGuard<'_, Captured> { + shared.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Read the child's whole answer from `stdout`. +/// +/// A failed read would leave the child running with nobody waiting on it, and +/// this client lives in the long-lived daemon, so the child is killed and +/// reaped before the error is returned, as it is when the request cannot be +/// written. The stderr collector is not joined on that path: a grandchild that +/// inherited the pipe could hold it open, and the thread ends by itself once +/// the pipe closes. +fn read_response(child: &mut Child, mut stdout: impl Read) -> Result> { + let mut response = Vec::new(); + if let Err(error) = stdout.read_to_end(&mut response) { + let _ = child.kill(); + let _ = child.wait(); + return Err(anyhow::Error::new(error).context("failed to read engine stdio response")); + } + Ok(response) +} + +/// Read `stderr` to the end line by line, optionally writing each line to +/// `forward` (this process's stderr, in production) as it arrives, and return +/// everything read. A failed forward is ignored: a closed terminal must not +/// cost the request its answer. That holds for the signal as well as the error: +/// under SIGPIPE's default action (`rocm` runs that way) a write to a closed +/// pipe ends the whole process before `write_all` can return, so each forward +/// runs inside [`with_sigpipe_ignored`] (see [`forward_best_effort`]). A read error ends the +/// collection and is itself recorded as the last line. +#[cfg(test)] +fn collect_stderr(stderr: impl Read, forward: Option) -> Vec { + let shared = Mutex::new(Captured::default()); + collect_stderr_into(stderr, forward, &shared); + shared + .into_inner() + .unwrap_or_else(PoisonError::into_inner) + .bytes +} + +/// [`collect_stderr`], publishing into `shared` as each chunk arrives, not when a +/// line is complete, so a caller that stops waiting has everything that had been +/// written (a diagnostic with no trailing newline included) and not only the +/// complete lines. Once the capture is abandoned the thread keeps reading, so the +/// pipe stays drained, but keeps and forwards nothing. +fn collect_stderr_into( + mut stderr: impl Read, + mut forward: Option, + shared: &Mutex, +) { + let mut chunk = [0_u8; 4096]; + let mut carry = Utf8Carry::default(); + loop { + let bytes: Vec = match stderr.read(&mut chunk) { + Ok(0) => break, + Ok(read) => chunk[..read].to_vec(), + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => { + let message = format!("failed to read engine progress: {error}\n").into_bytes(); + publish(shared, forward.as_mut(), &mut carry, &message); + break; + } + }; + publish(shared, forward.as_mut(), &mut carry, &bytes); + } + // What the pipe ended in the middle of a character of is forwarded lossily. + if let Some(sink) = forward.as_mut() + && !lock(shared).abandoned + { + forward_best_effort(sink, &carry.finish()); + } +} + +/// Keep `bytes` and forward the complete UTF-8 of them, unless the capture has +/// been abandoned. The kept copy is the raw bytes; only what is forwarded is +/// held back at an incomplete trailing character. +fn publish( + shared: &Mutex, + forward: Option<&mut impl Write>, + carry: &mut Utf8Carry, + bytes: &[u8], +) { + { + let mut captured = lock(shared); + if captured.abandoned { + return; + } + captured.bytes.extend_from_slice(bytes); + } + if let Some(sink) = forward { + let complete = carry.push(bytes); + if !complete.is_empty() { + forward_best_effort(sink, &complete); + } + } +} + +/// U+FFFD in UTF-8, as bytes: a literal here would be a non-ASCII string in +/// production code, which the output contract forbids. +const REPLACEMENT_CHARACTER: &[u8] = &[0xEF, 0xBF, 0xBD]; + +/// Turns arbitrary chunks of a byte stream into text that is valid UTF-8 at every +/// boundary, for a sink that requires it. Rust's stdio on a Windows console +/// refuses a write that is not valid UTF-8, so a chunk that ends in the middle of +/// a multi-byte character would fail, and the forwarding discards that error, +/// losing valid output. The incomplete trailing sequence is carried to the next +/// chunk instead; bytes that can never be valid are replaced with U+FFFD, and +/// what is still carried when the stream ends is flushed the same way. +#[derive(Default)] +struct Utf8Carry { + pending: Vec, +} + +impl Utf8Carry { + /// The part of `pending + bytes` that is complete UTF-8 (with invalid bytes + /// replaced), keeping an incomplete trailing sequence for the next call. + fn push(&mut self, bytes: &[u8]) -> Vec { + self.pending.extend_from_slice(bytes); + let mut out = Vec::with_capacity(self.pending.len()); + let mut rest = std::mem::take(&mut self.pending); + loop { + match std::str::from_utf8(&rest) { + Ok(valid) => { + out.extend_from_slice(valid.as_bytes()); + return out; + } + Err(error) => { + let valid_up_to = error.valid_up_to(); + out.extend_from_slice(&rest[..valid_up_to]); + match error.error_len() { + // The input ends inside a character: wait for the rest. + None => { + self.pending = rest[valid_up_to..].to_vec(); + return out; + } + Some(len) => { + out.extend_from_slice(REPLACEMENT_CHARACTER); + rest.drain(..valid_up_to + len); + } + } + } + } + } + } + + /// What is carried at the end of the stream, replaced lossily. + fn finish(&mut self) -> Vec { + let pending = std::mem::take(&mut self.pending); + String::from_utf8_lossy(&pending).into_owned().into_bytes() + } +} + +/// Write `bytes` to the forwarding sink and drop the result: a sink that has +/// gone away must not cost the request its answer. SIGPIPE is ignored for the +/// write, so a closed pipe comes back as the error that is then discarded +/// rather than as a signal that kills the process. +fn forward_best_effort(sink: &mut impl Write, bytes: &[u8]) { + let _ = with_sigpipe_ignored(|| sink.write_all(bytes)); +} + +/// Turn an engine's answer into the response type the caller asked for: the +/// engine's own error when it reported one, otherwise the decoded `data`. +pub fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result +where + R: DeserializeOwned, +{ + if !envelope.ok { + let error = envelope.error.map_or_else( + || "unknown engine error".to_owned(), + |value| format!("{}: {}", value.code, value.message), + ); + bail!("{error}"); + } + let data = envelope + .data + .context("engine response envelope did not contain data")?; + serde_json::from_value(data).context("failed to decode engine response payload") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EngineErrorDetail, StopResponse}; + use serde_json::{Value, json}; + use std::collections::BTreeMap; + + /// The environment `command` would add to its child, as plain strings. + fn added_env(command: &Command) -> BTreeMap { + command + .get_envs() + .filter_map(|(key, value)| { + Some(( + key.to_string_lossy().into_owned(), + value?.to_string_lossy().into_owned(), + )) + }) + .collect() + } + + fn args_of(command: &Command) -> Vec { + command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect() + } + + #[test] + fn the_default_options_add_nothing_to_the_childs_environment() { + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions::default(), + ); + assert_eq!(command.get_program(), "/an/engine"); + assert_eq!(args_of(&command), ["stdio"]); + assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); + } + + /// A child told it gets no endpoint key must have the variable CLEARED, + /// not merely left unset. + /// + /// `rocm` passes `endpoint_key_file: None` for every external engine + /// plugin — a third-party binary — and the path it would otherwise carry + /// points at a mode-0600 file holding a live endpoint credential. Merely + /// not setting it would still hand the value through whenever this process + /// inherited one. + /// + /// Asserts the distinction directly: `Command` records `env_remove` as an + /// explicit `(key, None)` entry, so a plain "is it absent" check would pass + /// even with the clearing deleted. + #[test] + fn an_external_plugin_child_is_not_handed_the_endpoint_key_file() { + let key_var = std::ffi::OsStr::new(ENDPOINT_API_KEY_FILE_ENV); + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions { + app_paths: None, + env_root: None, + endpoint_key_file: None, + withhold_inherited_key_file: true, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ); + + let entry = command + .get_envs() + .find(|(key, _)| *key == key_var) + .map(|(_, value)| value); + assert_eq!( + entry, + Some(None), + "the key-file variable must be explicitly CLEARED when no key file \ + is passed, not merely left unset" + ); + + // The other direction, which is what keeps this scoped: a caller that + // did not ask (`rocmd`, for a loopback service that simply needs no + // key) must leave the child's inherited value alone. + let kept = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions { + app_paths: None, + env_root: None, + endpoint_key_file: None, + withhold_inherited_key_file: false, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ); + assert!( + kept.get_envs().all(|(key, _)| key != key_var), + "a caller that did not ask to withhold must not touch the variable" + ); + } + + #[test] + fn each_option_sets_exactly_its_own_variable() { + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + let command = engine_stdio_command( + Path::new("/an/engine"), + ["__engine-stdio", "vllm"], + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), + withhold_inherited_key_file: false, + request_progress: true, + forward_stderr: true, + note_failing_exit_after_answer: false, + }, + ); + assert_eq!(args_of(&command), ["__engine-stdio", "vllm"]); + assert_eq!( + added_env(&command), + BTreeMap::from( + [ + ("ROCM_CLI_CONFIG_DIR", "/c"), + ("ROCM_CLI_DATA_DIR", "/d"), + ("ROCM_CLI_CACHE_DIR", "/k"), + ("ROCM_CLI_ENGINE_ENVS_ROOT", "/envs"), + ("ROCM_SERVE_API_KEY_FILE", "/keys/svc.endpoint-key"), + ("ROCM_ENGINE_PROGRESS_STDERR", "1"), + ] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ) + ); + + for (name, options) in [ + ( + "ROCM_CLI_ENGINE_ENVS_ROOT", + EngineStdioOptions { + env_root: Some(Path::new("/envs")), + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_SERVE_API_KEY_FILE", + EngineStdioOptions { + endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: false, + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_ENGINE_PROGRESS_STDERR", + EngineStdioOptions { + request_progress: true, + ..EngineStdioOptions::default() + }, + ), + ] { + let command = engine_stdio_command(Path::new("/an/engine"), ["stdio"], &options); + let env = added_env(&command); + assert_eq!( + env.keys().map(String::as_str).collect::>(), + [name], + "an option must set only its own variable" + ); + } + } + + #[test] + fn the_variable_names_are_the_ones_the_engines_read() { + assert_eq!(ENGINE_ENVS_ROOT_ENV, "ROCM_CLI_ENGINE_ENVS_ROOT"); + assert_eq!(ENDPOINT_API_KEY_FILE_ENV, "ROCM_SERVE_API_KEY_FILE"); + assert_eq!(ENGINE_PROGRESS_STDERR_ENV, "ROCM_ENGINE_PROGRESS_STDERR"); + } + + #[test] + fn the_request_envelope_is_the_method_and_the_encoded_payload() { + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap(); + assert_eq!(envelope.method, EngineMethod::Stop); + assert_eq!(envelope.payload, json!({"service_id": "svc"})); + assert_eq!( + serde_json::to_string(&envelope).unwrap(), + r#"{"method":"stop","payload":{"service_id":"svc"}}"# + ); + } + + #[test] + fn decode_engine_response_reports_each_way_an_answer_can_be_unusable() { + let failed = EngineResponseEnvelope { + ok: false, + data: None, + error: Some(EngineErrorDetail { + code: "stop_failed".to_owned(), + message: "it broke".to_owned(), + }), + }; + let error = decode_engine_response::(failed).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let unexplained = EngineResponseEnvelope { + ok: false, + data: None, + error: None, + }; + let error = decode_engine_response::(unexplained).unwrap_err(); + assert_eq!(error.to_string(), "unknown engine error"); + + let empty = EngineResponseEnvelope { + ok: true, + data: None, + error: None, + }; + let error = decode_engine_response::(empty).unwrap_err(); + assert_eq!( + error.to_string(), + "engine response envelope did not contain data" + ); + + let wrong_shape = EngineResponseEnvelope::success(json!({"stopped": "yes"})); + let error = decode_engine_response::(wrong_shape).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to decode engine response payload" + ); + } + + /// A source fed by the test, one chunk per message, that says when it is + /// dropped (the collecting thread has ended). + struct Fed { + chunks: mpsc::Receiver>, + dropped: mpsc::Sender<()>, + } + impl Read for Fed { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.chunks.recv() { + Ok(chunk) => { + buffer[..chunk.len()].copy_from_slice(&chunk); + Ok(chunk.len()) + } + Err(_) => Ok(0), + } + } + } + impl Drop for Fed { + fn drop(&mut self) { + let _ = self.dropped.send(()); + } + } + + /// After a request has failed and its collector is gone, a descendant that + /// still writes to the pipe must not make the daemon grow: what was retained + /// is released at once, and what arrives later is drained and discarded (and + /// not forwarded), in chunks of a fixed size. The test keeps a handle on the + /// shared buffer to see that. + #[test] + fn an_abandoned_collector_releases_what_it_held_and_discards_what_arrives_later() { + let (feed, chunks) = mpsc::channel(); + let (dropped, ended) = mpsc::channel(); + let forwarded = Arc::new(Mutex::new(Vec::new())); + let sink = { + let forwarded = Arc::clone(&forwarded); + FnSink(move |bytes: &[u8]| { + forwarded.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + }) + }; + let collector = StderrCollector::start(Fed { chunks, dropped }, Some(sink)); + let shared = Arc::clone(&collector.shared); + + feed.send(b"before the failure\n".to_vec()).unwrap(); + // Wait for the thread to have published it, not for the stderr to end. + let deadline = Instant::now() + Duration::from_secs(10); + while lock(&shared).bytes.is_empty() { + assert!(Instant::now() < deadline, "the chunk was never published"); + thread::sleep(Duration::from_millis(2)); + } + assert_eq!(lock(&shared).bytes, b"before the failure\n"); + + drop(collector); + assert!(lock(&shared).bytes.is_empty(), "what was held is released"); + assert!(lock(&shared).abandoned); + + for _ in 0..50 { + feed.send(vec![b'x'; 4096]).unwrap(); + } + drop(feed); + ended + .recv_timeout(Duration::from_secs(10)) + .expect("the thread kept draining until the pipe closed"); + let retained = lock(&shared).bytes.len(); + assert_eq!( + retained, 0, + "bytes retained after the capture was abandoned" + ); + assert_eq!( + forwarded.lock().unwrap().as_slice(), + b"before the failure\n", + "nothing is forwarded after the request has ended" + ); + } + + /// The multi-byte text the tests split: two, three and four byte characters. + const MULTIBYTE: &str = "caf\u{e9} \u{2192} \u{1f4e6} done\n"; + + /// A stream split at every possible boundary is forwarded whole: every chunk + /// handed to the sink is valid UTF-8 on its own, and together they are the + /// input. Pure, so it runs on the Windows job too. + #[test] + fn text_split_across_chunks_is_forwarded_whole_and_valid_at_every_boundary() { + let bytes = MULTIBYTE.as_bytes(); + for first in 1..bytes.len() { + for second in first..bytes.len() { + let mut carry = Utf8Carry::default(); + let mut forwarded = Vec::new(); + for part in [&bytes[..first], &bytes[first..second], &bytes[second..]] { + let out = carry.push(part); + assert!( + std::str::from_utf8(&out).is_ok(), + "split {first}/{second}: a chunk was not valid UTF-8: {out:?}" + ); + forwarded.extend(out); + } + forwarded.extend(carry.finish()); + assert_eq!(forwarded, bytes, "split {first}/{second}: bytes were lost"); + } + } + } + + /// Bytes that can never be text are replaced, and a character cut off by the + /// end of the stream is flushed lossily, never dropped. + #[test] + fn bytes_that_are_not_text_are_replaced_and_a_cut_off_tail_is_flushed() { + let mut carry = Utf8Carry::default(); + assert_eq!( + carry.push(b"ok \xff\xfe done"), + "ok \u{fffd}\u{fffd} done".as_bytes() + ); + // A lead byte of a three-byte character with nothing after it. + assert_eq!(carry.push(b"tail \xe2\x86"), b"tail "); + assert_eq!(carry.finish(), "\u{fffd}".as_bytes()); + assert!(carry.finish().is_empty()); + } + + /// The collector forwards a character split across reads in one piece, and + /// keeps the raw bytes for the error. The source hands out one byte at a time. + #[test] + fn the_collector_forwards_a_split_character_whole_and_keeps_the_raw_bytes() { + struct OneByte(std::vec::IntoIter); + impl Read for OneByte { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.0.next() { + Some(byte) => { + buffer[0] = byte; + Ok(1) + } + None => Ok(0), + } + } + } + let chunks = std::cell::RefCell::new(Vec::>::new()); + let sink = FnSink(|bytes: &[u8]| { + chunks.borrow_mut().push(bytes.to_vec()); + Ok(bytes.len()) + }); + let collected = collect_stderr( + OneByte(MULTIBYTE.as_bytes().to_vec().into_iter()), + Some(sink), + ); + assert_eq!(collected, MULTIBYTE.as_bytes()); + let chunks = chunks.into_inner(); + assert!( + chunks + .iter() + .all(|chunk| std::str::from_utf8(chunk).is_ok()) + ); + assert_eq!(chunks.concat(), MULTIBYTE.as_bytes()); + } + + /// A status that is neither an exit nor a signal (a stopped process, on + /// Unix) has neither a code nor a signal to name, and is reported as that. + #[cfg(unix)] + #[test] + fn a_status_with_no_code_and_no_signal_gets_the_neutral_wording() { + use std::os::unix::process::ExitStatusExt as _; + // Stopped by SIGSTOP: WIFSTOPPED, which is neither WIFEXITED nor WIFSIGNALED. + let stopped = ExitStatus::from_raw(0x137f); + assert_eq!(stopped.code(), None); + assert_eq!(stopped.signal(), None); + assert_eq!( + exit_note(stopped), + "engine stdio process ended without an exit code" + ); + } + + /// A source that does not answer until released, then answers once and ends. + struct Gated { + release: mpsc::Receiver<()>, + released: bool, + } + impl Read for Gated { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.released { + return Ok(0); + } + self.release.recv().unwrap(); + self.released = true; + buffer[..5].copy_from_slice(b"late\n"); + Ok(5) + } + } + + /// A collector whose stderr is still open at the deadline reports what had + /// arrived and is not waited for; the reading thread is left alone and the + /// rest is there once the stderr does end. + #[test] + fn a_collector_stops_waiting_at_its_deadline_and_keeps_collecting() { + let (release, released) = mpsc::channel(); + let collector = StderrCollector::start( + Gated { + release: released, + released: false, + }, + None::, + ); + let started = Instant::now(); + let early = collector.finish(Some(Instant::now() + Duration::from_millis(50))); + assert!(early.is_empty()); + assert!(started.elapsed() < Duration::from_secs(5)); + + release.send(()).unwrap(); + assert_eq!(collector.finish(None), b"late\n"); + } + + /// An expired deadline returns at once with what is there, however much is + /// queued behind it. + #[test] + fn an_expired_deadline_returns_what_is_collected_without_waiting() { + let (release, released) = mpsc::channel(); + let collector = StderrCollector::start( + Gated { + release: released, + released: false, + }, + None::, + ); + let started = Instant::now(); + assert!(collector.finish(Some(Instant::now())).is_empty()); + assert!(started.elapsed() < Duration::from_secs(1)); + drop(release); + } + + /// A stderr whose reads fail. + struct Fails; + impl Read for Fails { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("pipe went away")) + } + } + + /// A sink that hands every write to a closure: one that refuses them is a + /// closed terminal, one that records them shows what was forwarded. + struct FnSink(F); + impl io::Result> Write for FnSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + (self.0)(bytes) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn a_closure_sink_writes_through_and_flushes_nothing() { + let mut sink = FnSink(|bytes: &[u8]| Ok(bytes.len())); + assert_eq!(sink.write(b"abc").unwrap(), 3); + sink.flush().unwrap(); + } + + #[test] + fn stderr_is_collected_whole_and_a_read_error_is_recorded() { + let collected = collect_stderr(&b"first\nsecond"[..], None::>); + assert_eq!(collected, b"first\nsecond"); + + let collected = String::from_utf8(collect_stderr(Fails, None::>)).unwrap(); + assert_eq!( + collected, + "failed to read engine progress: pipe went away\n" + ); + } + + #[test] + fn forwarded_stderr_is_written_out_and_still_collected() { + let mut shown = Vec::new(); + let collected = collect_stderr(&b"step 1\nstep 2\n"[..], Some(&mut shown)); + assert_eq!(shown, b"step 1\nstep 2\n"); + assert_eq!(collected, b"step 1\nstep 2\n"); + + // A sink that refuses writes (a closed terminal) costs nothing. + let collected = collect_stderr( + &b"still kept\n"[..], + Some(FnSink(|_: &[u8]| { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + })), + ); + assert_eq!(collected, b"still kept\n"); + } + + #[cfg(unix)] + mod child { + use super::*; + use crate::fake_engine::FakeEngine; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn stop_request() -> EngineRequestEnvelope { + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap() + } + + fn run(engine: &FakeEngine, options: &EngineStdioOptions<'_>) -> Result { + decode_engine_response(run_engine_stdio( + engine.path(), + ["stdio"], + options, + &stop_request(), + )?) + } + + #[test] + fn a_request_is_one_newline_terminated_line_on_the_childs_stdin() { + let engine = FakeEngine::answering("client-framing", STOPPED, 0); + let answer = run(&engine, &EngineStdioOptions::default()).unwrap(); + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn a_response_with_an_apostrophe_reaches_the_caller_intact() { + // The fake writes its answer through a shell literal; an apostrophe + // in valid JSON must not end that literal early. + let engine = FakeEngine::answering( + "client-apostrophe", + r#"{"ok":false,"error":{"code":"stop_failed","message":"can't stop"}}"#, + 0, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: can't stop"); + } + + #[test] + fn the_options_reach_the_running_child() { + let engine = FakeEngine::answering("client-options", STOPPED, 0); + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + run( + &engine, + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: false, + request_progress: true, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ) + .unwrap(); + assert_eq!(engine.env("ROCM_CLI_DATA_DIR").as_deref(), Some("/d")); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/envs") + ); + assert_eq!( + engine.env("ROCM_SERVE_API_KEY_FILE").as_deref(), + Some("/keys/k") + ); + assert_eq!( + engine.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + } + + #[test] + fn a_nonzero_exit_with_no_answer_reports_the_status_and_stderr() { + let engine = FakeEngine::new("client-crash", true, "echo boom >&2\nexit 3"); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with code 3: boom" + ); + + let silent = FakeEngine::new("client-silent-crash", true, "exit 4"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "engine stdio process exited with code 4"); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let engine = FakeEngine::answering( + "client-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("client-answers-then-exits-2", STOPPED, 2); + assert_eq!( + run(&engine, &EngineStdioOptions::default()).unwrap(), + json!({"stopped": true}) + ); + } + + /// The status of a child that answered and then failed is not part of the + /// answer, so a caller that asks for it (the daemon) gets a note on this + /// process's stderr, and one that does not (the CLI) gets nothing. Run in + /// a re-executed test binary so the parent can read that stderr; the + /// decision itself is unchanged: the answer is used. + #[test] + fn a_failing_exit_after_an_answer_is_noted_only_when_asked_for() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_NOTE_ARM") { + let engine = FakeEngine::answering("client-note", STOPPED, 7); + let options = EngineStdioOptions { + note_failing_exit_after_answer: arm == "on", + ..EngineStdioOptions::default() + }; + let answer = run(&engine, &options).unwrap(); + assert_eq!(answer, json!({"stopped": true})); + return; + } + for (arm, expected) in [("on", true), ("off", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::a_failing_exit_after_an_answer_is_noted_only_when_asked_for", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + let note = "engine stdio process exited with code 7 after printing its answer; the answer was used"; + assert_eq!(stderr.contains(note), expected, "{arm}:\n{stderr}"); + } + } + + /// The note is only made for an answer that parsed: a child that prints + /// something unreadable and fails is reported by the error (with its + /// status), and must not also log that "the answer was used". Both cases + /// assert the outcome and the stderr together, in a re-executed binary. + #[test] + fn the_note_is_made_only_for_an_answer_that_parsed() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_NOTE_PARSE_ARM") { + let (body, status) = if arm == "parsed" { + (format!("printf '%s' '{STOPPED}'"), 7) + } else { + ("printf 'not json'".to_owned(), 7) + }; + let engine = + FakeEngine::new("client-note-parse", true, &format!("{body}\nexit {status}")); + let options = EngineStdioOptions { + note_failing_exit_after_answer: true, + ..EngineStdioOptions::default() + }; + let outcome = run(&engine, &options); + if arm == "parsed" { + assert_eq!(outcome.unwrap(), json!({"stopped": true})); + } else { + assert_eq!( + outcome.unwrap_err().to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with code 7)" + ); + } + return; + } + for (arm, expected) in [("parsed", true), ("unparseable", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::the_note_is_made_only_for_an_answer_that_parsed", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_PARSE_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + stderr.contains("the answer was used"), + expected, + "{arm}:\n{stderr}" + ); + } + } + + /// With stderr a closed pipe and SIGPIPE at its default action (how `rocm` + /// runs, and the daemon with it), the note must not end the process or + /// panic: the answer is still returned. Re-executes this test in a process + /// whose stderr is a pipe with no reader. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process() { + if std::env::var_os("ROCM_TEST_CLIENT_NOTE_CLOSED_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let engine = FakeEngine::answering("client-note-closed", STOPPED, 7); + let options = EngineStdioOptions { + note_failing_exit_after_answer: true, + ..EngineStdioOptions::default() + }; + let answer = run(&engine, &options).expect("the answer is still returned"); + assert_eq!(answer, json!({"stopped": true})); + return; + } + let (reader, writer) = io::pipe().unwrap(); + drop(reader); + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_CLOSED_ARM", "1") + .stderr(Stdio::from(writer)) + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE ended the process: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + + /// A child whose stdin or stdout pipe is missing is killed and reaped, as + /// on every other failure path, not left running. Linux-only for the + /// `/proc` entry a zombie would keep. + #[cfg(target_os = "linux")] + #[test] + fn a_missing_pipe_kills_and_reaps_the_child() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let proc_entry = format!("/proc/{}", child.id()); + assert!(Path::new(&proc_entry).exists()); + + let error = + require_pipe(&mut child, None::, "stdin").unwrap_err(); + + assert_eq!(error.to_string(), "engine stdio child did not expose stdin"); + assert!(!Path::new(&proc_entry).exists(), "killed and reaped"); + } + + /// A pipe that is there is handed back and the child left alone. + #[test] + fn a_present_pipe_is_returned_and_the_child_left_running() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + assert_eq!(require_pipe(&mut child, Some(5), "stdout").unwrap(), 5); + assert!(child.try_wait().unwrap().is_none()); + child.kill().unwrap(); + child.wait().unwrap(); + } + + /// A failed read of the answer reports what the engine had said on + /// stderr, with the read error, like every other failure; the message and + /// its content are asserted together. + #[test] + fn a_failed_read_of_the_answer_reports_the_stderr_too() { + let collector = StderrCollector::start( + io::Cursor::new(b"engine crashed: out of memory\n".to_vec()), + None::, + ); + // The thread ends at once on a cursor; wait for its text. + let error = read_failure_with_stderr( + anyhow::Error::new(io::Error::other("pipe went away")) + .context("failed to read engine stdio response"), + Some(collector), + Duration::from_secs(5), + ); + assert_eq!( + error.to_string(), + "failed to read engine stdio response: pipe went away; stderr: engine crashed: out of memory" + ); + + // With nothing on stderr the error is the read error, unchanged. + let quiet = read_failure_with_stderr( + anyhow::anyhow!("failed to read engine stdio response"), + None, + Duration::from_secs(1), + ); + assert_eq!(quiet.to_string(), "failed to read engine stdio response"); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let engine = FakeEngine::new( + "client-garbage", + true, + "printf 'not json'\necho oops >&2\nexit 0", + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + + let silent = FakeEngine::new("client-garbage-quiet", true, "printf 'not json'"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + + /// A child ended by a signal has no exit code; it is reported by the signal, + /// with or without a half-written answer, and never as a code of its own + /// invention. + #[test] + fn a_child_ended_by_a_signal_is_reported_by_the_signal() { + let killed = FakeEngine::new("client-killed", true, "kill -KILL $$"); + let error = run(&killed, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process was terminated by signal 9" + ); + + let cut_short = FakeEngine::new( + "client-killed-mid-answer", + true, + "printf '{\"ok\"'\nkill -KILL $$", + ); + let error = run(&cut_short, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process was terminated by signal 9)" + ); + } + + /// A child that fails and also leaves something that is not an envelope on + /// stdout is reported with its exit status (the status says the answer was + /// cut short, where the parse error alone would not), as a child that + /// printed nothing is. The same unreadable answer from a child that exited + /// cleanly carries no status, so the note is the condition's own and not + /// added to every parse failure. + #[test] + fn an_unparseable_answer_from_a_failing_child_names_the_exit_status() { + let crashed = FakeEngine::new( + "client-garbage-crash", + true, + "printf 'not json'\necho oops >&2\nexit 4", + ); + let error = run(&crashed, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with code 4); stderr: oops" + ); + + let silent = FakeEngine::new( + "client-garbage-crash-quiet", + true, + "printf 'not json'\nexit 5", + ); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with code 5)" + ); + + let clean = FakeEngine::new("client-garbage-clean", true, "printf 'not json'\nexit 0"); + let error = run(&clean, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + + /// Runs `work` on its own thread and fails the test if it has not finished + /// within `limit`, so a step that should not block turns red instead of + /// hanging the run. + fn finishes_within( + limit: Duration, + work: impl FnOnce() -> T + Send + 'static, + ) -> T { + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let _ = sender.send(work()); + }); + receiver + .recv_timeout(limit) + .unwrap_or_else(|_| panic!("did not finish within {limit:?}")) + } + + /// An engine that fills its stderr before it has drained a request larger + /// than a pipe buffer does not deadlock the client: the stderr is read from + /// the start, so the engine's write completes while the request is still + /// being written. (With the collector started after the write, the engine + /// blocks on a full stderr pipe and the client on a full stdin pipe.) + #[test] + fn an_engine_that_writes_a_lot_of_stderr_before_reading_its_request_does_not_deadlock() { + let engine = FakeEngine::new( + "client-stderr-first", + false, + &format!( + "head -c 300000 /dev/zero | tr '\\0' x >&2\ncat > /dev/null\nprintf '%s' '{STOPPED}'" + ), + ); + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"blob": "y".repeat(1 << 20)})) + .unwrap(); + let program = engine.path().to_path_buf(); + let answer = finishes_within(Duration::from_secs(20), move || { + run_engine_stdio( + &program, + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + }) + .expect("the engine answered"); + assert!(answer.ok); + drop(engine); + } + + /// A request that cannot be written because the engine is gone reports + /// what the engine said on stderr, not only that the write failed: the + /// engine's own words are the explanation. The request is larger than a + /// pipe buffer, so the write reaches the closed read end. + #[test] + fn a_request_to_an_engine_that_exited_reports_what_it_said_on_stderr() { + let engine = FakeEngine::new("client-no-listen", false, "echo 'no thanks' >&2\nexit 3"); + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"blob": "x".repeat(1 << 20)})) + .unwrap(); + let error = run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .expect_err("the engine never read its request"); + let message = error.to_string(); + assert!( + message.starts_with("failed to write engine request: "), + "{message}" + ); + assert!(message.ends_with("; stderr: no thanks"), "{message}"); + } + + /// The child of a failed write is killed and reaped when it is still + /// running, which a failed write does not show it to have stopped. + /// Linux-only for the `/proc` entry, which a zombie keeps until waited on. + #[cfg(target_os = "linux")] + #[test] + fn a_request_that_cannot_be_written_kills_and_reaps_a_child_that_is_still_running() { + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let proc_entry = format!("/proc/{}", child.id()); + assert!(Path::new(&proc_entry).exists(), "the child is running"); + + let error = finishes_within(Duration::from_secs(10), move || { + let mut child = child; + let stderr = child + .stderr + .take() + .map(|pipe| StderrCollector::start(pipe, None::)); + abandon_unwritten_request( + child, + stderr, + &io::Error::from(io::ErrorKind::BrokenPipe), + Duration::from_secs(1), + ) + }); + + assert!( + error + .to_string() + .starts_with("failed to write engine request: "), + "{error}" + ); + assert!( + !error.to_string().contains("stderr"), + "a child that said nothing adds no stderr: {error}" + ); + assert!( + !Path::new(&proc_entry).exists(), + "the child must be killed and reaped, not left running or as a zombie" + ); + } + + /// A grandchild that holds the child's stderr open past its exit cannot + /// make the error path wait for it: what had been written is reported + /// and the rest is not waited for. The diagnostic has no trailing newline, + /// as an engine's last words often do not, and is still reported: the + /// collector publishes what has arrived, not only complete lines. + #[test] + fn the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace() { + let mut child = Command::new("sh") + .args(["-c", "printf said >&2; sleep 5 & exit 3"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + // The engine has said its piece and exited before the failure is + // handled, as it has in the case this bounds. + child.wait().unwrap(); + let started = Instant::now(); + + let stderr = child + .stderr + .take() + .map(|pipe| StderrCollector::start(pipe, None::)); + let error = finishes_within(Duration::from_secs(4), move || { + abandon_unwritten_request( + child, + stderr, + &io::Error::from(io::ErrorKind::BrokenPipe), + Duration::from_millis(300), + ) + }); + + let waited = started.elapsed(); + assert!( + waited < Duration::from_secs(4), + "waited for the grandchild: {waited:?}" + ); + assert!(error.to_string().ends_with("; stderr: said"), "{error}"); + } + + /// An answer whose read fails kills the child and reaps it, so a daemon + /// that hits this repeatedly collects neither a running engine nor a + /// zombie per failure. A zombie keeps its `/proc` entry until it is + /// waited on, so the entry being gone shows the reap and not only the + /// kill (hence Linux only). + #[cfg(target_os = "linux")] + #[test] + fn a_failed_read_of_the_answer_kills_and_reaps_the_child() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + let proc_entry = format!("/proc/{pid}"); + assert!(Path::new(&proc_entry).exists(), "the child is running"); + + let error = read_response(&mut child, Fails).unwrap_err(); + + assert_eq!(error.to_string(), "failed to read engine stdio response"); + assert_eq!( + format!("{error:#}"), + "failed to read engine stdio response: pipe went away" + ); + assert!( + !Path::new(&proc_entry).exists(), + "the child must be killed and reaped, not left running or as a zombie" + ); + // The status was collected by the reap, so it is still reported. + assert_eq!( + child.try_wait().unwrap().and_then(|status| status.signal()), + Some(libc::SIGKILL) + ); + } + + /// The success path of the same helper hands the bytes back and leaves + /// the child alone. + #[test] + fn a_successful_read_of_the_answer_leaves_the_child_to_the_caller() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + + let bytes = read_response(&mut child, &b"{\"ok\":true}"[..]).unwrap(); + + assert_eq!(bytes, b"{\"ok\":true}"); + assert!(child.try_wait().unwrap().is_none(), "still running"); + child.kill().unwrap(); + child.wait().unwrap(); + } + + #[test] + fn a_program_that_cannot_start_is_named_in_the_error() { + let error = run_engine_stdio( + Path::new("/no/such/engine"), + ["stdio"], + &EngineStdioOptions::default(), + &stop_request(), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "failed to spawn engine stdio process /no/such/engine" + ); + } + + /// What the running child writes to its stderr reaches the forwarding + /// sink as it arrives. (That nothing is forwarded without the option is + /// the `off` arm of the test below.) + #[test] + fn a_childs_stderr_reaches_the_forwarding_sink_when_there_is_one() { + let progress = "downloading 1/2\ndownloading 2/2\n"; + let body = format!("printf '%s' '{progress}' >&2\nprintf '%s' '{STOPPED}'"); + let engine = FakeEngine::new("client-forward-sink", true, &body); + + let forwarded = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let sink = { + let forwarded = std::sync::Arc::clone(&forwarded); + FnSink(move |bytes: &[u8]| { + forwarded.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + }) + }; + let answer = run_engine_stdio_forwarding( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &stop_request(), + Some(sink), + ) + .unwrap(); + assert!(answer.ok); + assert_eq!(forwarded.lock().unwrap().as_slice(), progress.as_bytes()); + } + + /// The public entry point sends the child's stderr to THIS process's + /// stderr when `forward_stderr` is set, and does not when it is not. That + /// is the one place the option takes effect, and it is what shows a user + /// an engine install's progress. The test binary re-executes itself so the + /// parent can read the stderr the arm writes. + #[test] + fn forward_stderr_sends_the_childs_stderr_to_this_processs_stderr() { + const LINE: &str = "engine progress line for the terminal"; + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_FORWARD_OPTION_ARM") { + let engine = FakeEngine::new( + "client-forward-option", + true, + &format!("echo '{LINE}' >&2\nprintf '%s' '{STOPPED}'"), + ); + let options = EngineStdioOptions { + forward_stderr: arm == "on", + ..EngineStdioOptions::default() + }; + run_engine_stdio(engine.path(), ["stdio"], &options, &stop_request()).unwrap(); + return; + } + + for (arm, expected) in [("on", true), ("off", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forward_stderr_sends_the_childs_stderr_to_this_processs_stderr", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_FORWARD_OPTION_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + stderr.contains(LINE), + expected, + "forward_stderr {arm}: the child's stderr on this process's stderr:\n{stderr}" + ); + } + } + + /// Progress forwarded to a stderr whose reader has gone away is dropped, not + /// fatal, also where SIGPIPE is at its default action (`rocm` runs that + /// way). The `Closed` sink above cannot show this: it returns the error + /// itself, where the real failure is a signal that ends the process before + /// any error is returned. So this re-executes itself in a fresh process + /// that installs `SIG_DFL` and forwards into one end of a real pipe whose + /// other end is closed. Two arms, one per forwarding write in + /// `collect_stderr`: a line of the child's stderr, and the note recorded + /// when reading that stderr fails. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_FORWARD_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let (reader, writer) = io::pipe().unwrap(); + drop(reader); + let collected = if arm == "line" { + collect_stderr(&b"progress\n"[..], Some(writer)) + } else { + collect_stderr(Fails, Some(writer)) + }; + // Surviving is the point; the text is still kept for the error. + assert!(!collected.is_empty()); + return; + } + + for arm in ["line", "read-error"] { + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_FORWARD_ARM", arm) + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "{arm}: SIGPIPE killed the process instead of the write failing: {status:?}" + ); + assert!(status.success(), "{arm}: {status:?}"); + } + } + + /// Whether SIGPIPE is ignored right now, read by installing the same + /// disposition back. + #[allow(unsafe_code)] // libc::signal + fn sigpipe_is_ignored() -> bool { + // SAFETY: reading/installing a signal disposition is always safe. + let current = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + // SAFETY: putting back what was just read. + unsafe { + libc::signal(libc::SIGPIPE, current); + } + current == libc::SIG_IGN + } + + /// Forwarding guards each write on its own and never holds SIGPIPE ignored + /// across the reads between them. The reads block for as long as the engine + /// is quiet (an install runs for minutes), and the disposition is + /// process-wide, so a guard around the whole loop would leave every other + /// thread's write to a closed stdout returning an error for that whole time + /// instead of ending `rocm` quietly. Each line costs one pair of + /// `signal` calls, which is nothing next to the write itself. A fresh + /// process with `SIG_DFL` records the disposition seen inside every read + /// and every write, and after the loop. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn forwarding_guards_each_write_and_never_the_reads_between_them() { + use std::cell::RefCell; + use std::rc::Rc; + + type Seen = Rc>>; + + struct Source { + lines: Vec<&'static [u8]>, + seen: Seen, + } + impl Read for Source { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + self.seen.borrow_mut().push(("read", sigpipe_is_ignored())); + if self.lines.is_empty() { + return Ok(0); + } + let line = self.lines.remove(0); + buffer[..line.len()].copy_from_slice(line); + Ok(line.len()) + } + } + + if std::env::var_os("ROCM_TEST_CLIENT_GUARD_SCOPE_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let seen: Seen = Rc::default(); + let source = Source { + lines: vec![b"one\n", b"two\n", b"three\n"], + seen: Rc::clone(&seen), + }; + let sink_seen = Rc::clone(&seen); + let sink = FnSink(move |bytes: &[u8]| { + sink_seen.borrow_mut().push(("write", sigpipe_is_ignored())); + Ok(bytes.len()) + }); + collect_stderr(source, Some(sink)); + let seen = seen.borrow(); + let writes = seen.iter().filter(|(what, _)| *what == "write").count(); + assert_eq!(writes, 3, "{seen:?}"); + for (what, ignored) in seen.iter() { + assert_eq!( + *ignored, + *what == "write", + "SIGPIPE must be ignored inside each write and only there: {seen:?}" + ); + } + assert!( + !sigpipe_is_ignored(), + "the default action is back after the loop" + ); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forwarding_guards_each_write_and_never_the_reads_between_them", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_GUARD_SCOPE_ARM", "1") + .status() + .unwrap(); + assert!(status.success(), "{status:?}"); + } + + /// A request written to an engine that has already exited comes back as an + /// ordinary error, also where SIGPIPE is at its default action. Re-executes + /// this test in a fresh process that installs `SIG_DFL` first, because the + /// harness leaves SIGPIPE ignored and changing it process-wide would race + /// sibling tests; a write that is not guarded kills that process, and the + /// parent reports the signal. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_CLIENT_SIGPIPE_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let engine = FakeEngine::new("client-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let envelope = engine_request_envelope( + EngineMethod::Stop, + &json!({"blob": "x".repeat(1 << 20)}), + ) + .unwrap(); + let error = run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } +} diff --git a/crates/rocm-engine-protocol/tests/protocol_properties.rs b/crates/rocm-engine-protocol/tests/protocol_properties.rs new file mode 100644 index 0000000000..8aa54b9971 --- /dev/null +++ b/crates/rocm-engine-protocol/tests/protocol_properties.rs @@ -0,0 +1,1033 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the engine protocol: the types a plugin's stdout is +//! decoded into, and the helpers that turn untrusted filesystem and CLI input +//! into identifiers the CLI then acts on. +//! +//! Decoding of arbitrary bytes, arbitrary text and byte-level mutations of +//! valid encodings is smoke-tested for panics only: every type here uses +//! derived `Deserialize`, so those tests guard against a future hand-written +//! decoder or an unwrap on the decode path, not against wrong values. What a +//! decode must produce is pinned by the round-trip, version-skew and golden +//! document tests. + +#[path = "../../rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; + +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_engine_protocol::{ + DetectResponse, DevicePolicy, EngineCapabilities, EngineDeviceAvailability, EngineMethod, + EngineRecipeEndpointHint, EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, + EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, HealthcheckResponse, + LaunchRequest, LogsRequest, LogsResponse, ResolveModelRequest, + engine_id_from_plugin_binary_name, gpu_indices_to_csv, is_public_bind_host, launch_gpu_indices, +}; +use std::collections::BTreeMap; + +/// Decode `bytes` as `T`, asserting only that it does not panic. +macro_rules! decode_without_panic { + ($type:ty, $bytes:expr) => { + let _ = serde_json::from_slice::<$type>($bytes); + }; +} + +/// Every envelope and payload type the protocol decodes, in either direction: +/// what a plugin writes (responses) and what it reads (requests). +fn decode_all_without_panic(bytes: &[u8]) { + decode_without_panic!(EngineRequestEnvelope, bytes); + decode_without_panic!(EngineResponseEnvelope, bytes); + decode_without_panic!(DetectResponse, bytes); + decode_without_panic!(HealthcheckResponse, bytes); + decode_without_panic!(LaunchRequest, bytes); + decode_without_panic!(LogsRequest, bytes); + decode_without_panic!(LogsResponse, bytes); + decode_without_panic!(ResolveModelRequest, bytes); + decode_without_panic!(EngineRecipeHint, bytes); + decode_without_panic!(GpuSelection, bytes); +} + +/// `any::()` deliberately excludes NaN and the infinities, so a naive +/// generator never samples the values JSON has no representation for — exactly +/// the region where a float round trip can break. Sample them explicitly. +fn tokens_per_sec_strategy() -> impl Strategy> { + proptest::option::of(prop_oneof![ + 6 => any::(), + 1 => Just(f32::NAN), + 1 => Just(f32::INFINITY), + 1 => Just(f32::NEG_INFINITY), + 1 => Just(f32::MIN_POSITIVE), + 1 => Just(-0.0_f32), + ]) +} + +fn healthcheck_strategy() -> impl Strategy { + ( + "[ -~]{0,20}", + any::(), + "[ -~]{0,20}", + any::(), + any::(), + proptest::option::of("[ -~]{0,30}"), + tokens_per_sec_strategy(), + ) + .prop_map( + |( + status, + model_loaded, + device, + uptime_sec, + queue_depth, + last_error, + tokens_per_sec, + )| { + HealthcheckResponse { + status, + model_loaded, + device, + uptime_sec, + queue_depth, + last_error, + tokens_per_sec, + } + }, + ) +} + +fn recipe_hint_strategy() -> impl Strategy { + ( + "[ -~]{0,10}", + "[ -~]{0,10}", + prop::collection::vec("[ -~]{0,12}", 0..4), + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..4), + // Often empty: an empty map is omitted on the wire, so both the + // omitted and the present spelling have to round-trip. + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..3), + proptest::option::of(( + "[ -~]{0,8}", + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..3), + )), + prop::collection::vec(("[ -~]{0,8}", "[ -~]{0,8}"), 0..3), + prop::collection::vec("[ -~]{0,8}", 0..3), + ) + .prop_map( + |( + contract_version, + engine, + required_flags, + parser_settings, + extra_env, + preferred_endpoint, + unsupported, + notes, + )| EngineRecipeHint { + contract_version, + engine, + required_flags, + parser_settings, + extra_env, + preferred_endpoint: preferred_endpoint.map(|(endpoint_mode, settings)| { + EngineRecipeEndpointHint { + endpoint_mode, + settings, + } + }), + unsupported_combinations: unsupported + .into_iter() + .map( + |(combination, reason)| EngineRecipeUnsupportedCombinationHint { + combination, + reason, + }, + ) + .collect(), + notes, + }, + ) +} + +fn optional_text() -> impl Strategy> { + proptest::option::of("[ -~]{0,10}") +} + +fn detect_response_strategy() -> impl Strategy { + ( + ( + any::(), + optional_text(), + optional_text(), + optional_text(), + proptest::option::of(any::()), + optional_text(), + optional_text(), + optional_text(), + ), + prop::collection::vec( + ( + "[ -~]{0,8}", + any::(), + proptest::option::of("[ -~]{0,8}"), + ), + 0..4, + ), + ( + any::<[bool; 5]>(), + "[ -~]{0,8}", + prop::collection::vec("[ -~]{0,8}", 0..4), + ), + ) + .prop_map( + |( + ( + installed, + env_id, + runtime_kind, + runtime_executable, + managed_env, + python_version, + torch_version, + transformers_version, + ), + devices, + ( + [ + cpu, + rocm_gpu, + openai_compatible, + tool_calling, + reasoning_parser, + ], + quantized_models, + notes, + ), + )| DetectResponse { + installed, + env_id, + runtime_kind, + runtime_executable, + managed_env, + python_version, + torch_version, + transformers_version, + available_devices: devices + .into_iter() + .map(|(kind, available, reason)| EngineDeviceAvailability { + kind, + available, + reason, + }) + .collect(), + capabilities: EngineCapabilities { + cpu, + rocm_gpu, + openai_compatible, + tool_calling, + quantized_models, + reasoning_parser, + }, + notes, + }, + ) +} + +/// Byte-level mutations of a valid encoding: the only generator that reliably +/// reaches field-level decode logic. +fn mutate(json: &str, position: usize, kind: u8) -> Vec { + let mut bytes = json.as_bytes().to_vec(); + if bytes.is_empty() { + return bytes; + } + let position = position % bytes.len(); + match kind % 6 { + 0 => bytes[position] = b'"', + 1 => bytes[position] = b'{', + 2 => bytes[position] = 0xff, + 3 => { + bytes.truncate(position); + } + 4 => { + bytes.remove(position); + } + _ => bytes.insert(position, b'\\'), + } + bytes +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(512), + ..ProptestConfig::default() + })] + + /// Encode then decode must be the identity for every value the type holds. + #[test] + fn healthcheck_response_roundtrips(response in healthcheck_strategy()) { + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: HealthcheckResponse = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.status, response.status); + prop_assert_eq!(decoded.model_loaded, response.model_loaded); + prop_assert_eq!(decoded.device, response.device); + prop_assert_eq!(decoded.uptime_sec, response.uptime_sec); + prop_assert_eq!(decoded.queue_depth, response.queue_depth); + prop_assert_eq!(decoded.last_error, response.last_error); + // JSON has no representation for NaN or the infinities, so `serde_json` + // encodes them as `null` and they come back as `None`. Every *finite* + // value must round-trip exactly; a non-finite one is documented here as + // collapsing to "unknown" rather than surviving or erroring. + match response.tokens_per_sec { + Some(value) if !value.is_finite() => prop_assert_eq!( + decoded.tokens_per_sec, + None, + "a non-finite rate has no JSON representation and reads back as unknown" + ), + expected => prop_assert_eq!( + decoded.tokens_per_sec.map(f32::to_bits), + expected.map(f32::to_bits), + "a finite tokens_per_sec must survive an encode/decode round trip" + ), + } + } + + #[test] + fn recipe_hint_roundtrips(hint in recipe_hint_strategy()) { + let encoded = serde_json::to_string(&hint).expect("encode"); + let decoded: EngineRecipeHint = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(&decoded, &hint); + // Duplicate-serve reuse compares a recipe hint's JSON as a string, so an + // empty `extra_env` must stay off the wire: a hint encodes exactly as it + // did before the field existed. + let value = serde_json::to_value(&hint).expect("encode"); + prop_assert_eq!( + value.get("extra_env").is_some(), + !hint.extra_env.is_empty(), + "extra_env must be omitted exactly when it is empty" + ); + } + + #[test] + fn detect_response_roundtrips(response in detect_response_strategy()) { + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: DetectResponse = serde_json::from_str(&encoded).expect("decode"); + // `DetectResponse` has no `PartialEq`; compare the whole struct through + // its JSON value instead of a hand-picked subset of fields. (Being a + // round trip, this cannot see a renamed field.) + prop_assert_eq!( + serde_json::to_value(&decoded).expect("re-encode"), + serde_json::to_value(&response).expect("encode") + ); + } + + /// `EngineResponseEnvelope::success` serializes caller data with `expect`; + /// it must not panic for the response types the engines actually send, and + /// the data must arrive intact on the other side of the wire. + #[test] + fn response_envelope_success_carries_the_data_intact(response in healthcheck_strategy()) { + let envelope = EngineResponseEnvelope::success(&response); + prop_assert!(envelope.ok); + let encoded = serde_json::to_string(&envelope).expect("encode"); + let decoded: EngineResponseEnvelope = serde_json::from_str(&encoded).expect("decode"); + prop_assert!(decoded.ok); + prop_assert!(decoded.error.is_none()); + // Compared as the response it decodes back into, re-encoded: `serde_json` + // parses floats to the nearest f64 only approximately, so comparing the + // raw `Value`s would report a parser rounding as a lost field. + let data = decoded.data.expect("a success envelope carries data"); + let carried: HealthcheckResponse = serde_json::from_value(data).expect("data decodes"); + prop_assert_eq!( + serde_json::to_string(&carried).expect("re-encode"), + serde_json::to_string(&response).expect("encode"), + "the envelope must carry the caller's data unchanged" + ); + } + + /// Smoke test: mutating a valid encoding must yield either a clean decode + /// or a clean error — never a panic. This can only fail on a panic. + #[test] + fn mutated_valid_encodings_never_panic( + response in healthcheck_strategy(), + hint in recipe_hint_strategy(), + position in any::(), + kind in any::(), + ) { + for json in [ + serde_json::to_string(&response).expect("encode"), + serde_json::to_string(&hint).expect("encode"), + serde_json::to_string(&EngineResponseEnvelope::success(&response)).expect("encode"), + ] { + decode_all_without_panic(&mutate(&json, position, kind)); + } + } + + /// Fully arbitrary bytes must never panic a decoder. + #[test] + fn arbitrary_bytes_never_panic(bytes in prop::collection::vec(any::(), 0..256)) { + decode_all_without_panic(&bytes); + } + + #[test] + fn arbitrary_text_never_panics(text in "[ -~\\n\\t]{0,256}") { + decode_all_without_panic(text.as_bytes()); + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(1024), + ..ProptestConfig::default() + })] + + /// A plugin binary name is attacker-influenced (anyone who can drop a file + /// into a plugin directory). The derived engine id is later used to build + /// paths and command lines, so it must never carry a separator or a + /// `.`/`..` traversal component, and the helper must never panic on a + /// multi-byte name. + #[test] + fn engine_id_is_never_a_traversal_component( + name in prop_oneof![ + "[ -~]{0,24}", + "rocm-engine-[ -~]{0,16}", + "rocm-engine-[A-Za-z0-9._/\\\\-]{0,12}(\\.exe|\\.EXE|)", + "\\PC{0,16}", + // The traversal names themselves, which no random strategy above + // reaches: an id of exactly `.` or `..`, with and without `.exe`. + prop::sample::select(vec![ + "rocm-engine-.", + "rocm-engine-..", + "rocm-engine-..exe", + "rocm-engine-...exe", + "rocm-engine-.EXE", + ]) + .prop_map(str::to_owned), + ], + ) { + if let Some(id) = engine_id_from_plugin_binary_name(&name) { + // The character set below has no separator, so it also rules out + // `/` and `\`. That well-formed names do yield their id, which a + // helper that never returned one would fail, is pinned by + // `plugin_binary_names_yield_their_engine_id`. + prop_assert!(!id.is_empty()); + prop_assert!(id != "." && id != "..", "engine id must not traverse: {id:?}"); + prop_assert!( + id.chars().all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-')), + "engine id must stay in the documented character set: {id:?}" + ); + } + } + + /// `--gpu` is direct user input. Parsing must never panic on arbitrary + /// text, and a successful parse must round-trip through the CLI spelling + /// it reports. Random text almost never parses, so this is mostly a panic + /// check; which values must parse, and to what, is pinned by + /// `gpu_cli_values_parse_as_documented`. + #[test] + fn gpu_selection_parse_never_panics_and_roundtrips(value in "[ -~]{0,16}") { + match GpuSelection::parse_cli_value(&value) { + Ok(GpuSelection::Auto) => { + prop_assert!(launch_gpu_indices(Some(&GpuSelection::Auto)).is_empty()); + } + Ok(GpuSelection::Index(index)) => { + prop_assert_eq!( + GpuSelection::parse_cli_value(&index.to_string()), + Ok(GpuSelection::Index(index)) + ); + prop_assert_eq!(launch_gpu_indices(Some(&GpuSelection::Index(index))), vec![index]); + prop_assert_eq!( + gpu_indices_to_csv(&[index]), + Some(index.to_string()) + ); + } + Err(_) => {} + } + } + + /// A GPU ordinal must never reach `HIP_VISIBLE_DEVICES` as anything but a + /// plain comma-separated list of decimal integers — the value is + /// interpolated into a child process environment. + #[test] + fn gpu_csv_is_always_plain_decimal(indices in prop::collection::vec(any::(), 0..8)) { + match gpu_indices_to_csv(&indices) { + None => prop_assert!(indices.is_empty()), + Some(csv) => { + prop_assert!(!indices.is_empty()); + let parsed: Result, _> = + csv.split(',').map(str::parse::).collect(); + prop_assert_eq!(parsed.as_deref(), Ok(indices.as_slice())); + } + } + } + + /// The loopback allowlist decides whether an endpoint gets an API key. + /// Anything that is not one of the three exact accepted spellings must be + /// classified public (key required) — misclassifying a public bind as + /// loopback opens an unauthenticated listener. + /// + /// A random host string essentially never lands near those spellings, so + /// most cases are drawn from the near-miss region: each loopback spelling + /// padded, re-cased, bracketed or extended, plus other addresses that + /// reach this host but are not on the list. + #[test] + fn only_exact_loopback_spellings_skip_the_api_key( + host in prop_oneof![ + 3 => ( + prop::sample::select(LOOPBACK_SPELLINGS.to_vec()), + 0u8..8, + ) + .prop_map(|(spelling, kind)| near_loopback(spelling, kind)), + 1 => prop::sample::select(OTHER_LOCAL_ADDRESSES.to_vec()).prop_map(str::to_owned), + 1 => "[ -~]{0,24}", + ], + ) { + let loopback = !is_public_bind_host(&host); + let allowlisted = LOOPBACK_SPELLINGS.contains(&host.as_str()); + prop_assert!( + loopback == allowlisted, + "host {host:?} was classified {} but only the exact spellings {LOOPBACK_SPELLINGS:?} \ + may skip the key", + if loopback { "loopback" } else { "public" } + ); + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] + + /// A `LogsRequest` carries a `tail_lines` count straight from the caller into + /// the engines' tail logic; it must round-trip exactly, including the + /// extremes an engine could legitimately be asked for. + #[test] + fn logs_request_roundtrips( + service_id in "[ -~]{0,24}", + tail_lines in proptest::option::of(prop_oneof![ + Just(0usize), + Just(usize::MAX), + 0usize..10_000, + ]), + ) { + let request = LogsRequest { service_id: service_id.clone(), tail_lines }; + let encoded = serde_json::to_string(&request).expect("encode"); + let decoded: LogsRequest = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.service_id, service_id); + prop_assert_eq!(decoded.tail_lines, tail_lines); + } + + #[test] + fn logs_response_roundtrips( + log_path in "[ -~]{0,24}", + lines in prop::collection::vec("\\PC{0,40}", 0..8), + ) { + let response = LogsResponse { + log_path: log_path.clone(), + recent_lines: lines.clone(), + }; + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: LogsResponse = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.log_path, log_path); + prop_assert_eq!(decoded.recent_lines, lines); + } + + /// `host` and `port` from a launch request are what the CLI later classifies + /// for API-key enforcement, so they must survive the wire unchanged. + #[test] + fn launch_request_roundtrips_host_and_port( + host in "[ -~]{0,24}", + port in any::(), + selection in proptest::option::of(prop_oneof![ + Just(GpuSelection::Auto), + any::().prop_map(GpuSelection::Index), + ]), + device_policy in proptest::option::of(prop_oneof![ + Just(DevicePolicy::GpuRequired), + Just(DevicePolicy::GpuPreferred), + Just(DevicePolicy::CpuOnly), + ]), + ) { + let request = LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: "model".to_owned(), + host: host.clone(), + port, + device_policy: device_policy.clone(), + endpoint_mode: None, + engine_recipe: None, + gpu_selection: selection.clone(), + }; + let encoded = serde_json::to_string(&request).expect("encode"); + let decoded: LaunchRequest = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(&decoded.host, &host); + prop_assert_eq!(decoded.port, port); + prop_assert_eq!(decoded.gpu_selection, selection); + prop_assert_eq!(decoded.device_policy, device_policy); + } + + /// A request envelope decodes exactly when its method is one of the + /// documented wire names — an unknown method is rejected rather than + /// falling back to some default — and a decoded method re-encodes to the + /// spelling it arrived as. Half the cases use a real method name, the rest + /// near misses of one or arbitrary words. + #[test] + fn request_envelope_accepts_exactly_the_known_methods( + method in prop_oneof![ + 2 => prop::sample::select(METHOD_WIRE_NAMES.to_vec()).prop_map(str::to_owned), + 1 => (prop::sample::select(METHOD_WIRE_NAMES.to_vec()), 0u8..4).prop_map( + |(name, kind)| match kind { + 0 => name.to_ascii_uppercase(), + 1 => format!("{name}s"), + 2 => name.replace('_', ""), + _ => format!(" {name}"), + } + ), + 1 => "[a-z_]{0,16}", + ], + payload in "[ -~]{0,24}", + ) { + let json = serde_json::json!({ "method": method, "payload": payload }).to_string(); + let parsed = serde_json::from_str::(&json); + let known = METHOD_WIRE_NAMES.contains(&method.as_str()); + prop_assert_eq!( + parsed.is_ok(), + known, + "method {:?} decoded: {}", + method, + parsed.is_ok() + ); + if let Ok(envelope) = parsed { + let reencoded = serde_json::to_value(&envelope).expect("encode"); + prop_assert_eq!( + reencoded.get("method").and_then(serde_json::Value::as_str), + Some(method.as_str()), + "a decoded method must re-encode to the same wire spelling" + ); + } + } +} + +/// The documented wire name of every engine method, written out rather than +/// derived from the enum. +const METHOD_WIRE_NAMES: [&str; 9] = [ + "detect", + "install", + "capabilities", + "resolve_model", + "launch", + "healthcheck", + "endpoint", + "stop", + "logs", +]; + +/// The three spellings the CLI accepts as loopback, written out here rather than +/// shared with the code under test. +const LOOPBACK_SPELLINGS: [&str; 3] = ["127.0.0.1", "localhost", "::1"]; + +/// Addresses that reach this host, or every interface, without being one of the +/// accepted spellings: all must be treated as public. +const OTHER_LOCAL_ADDRESSES: [&str; 9] = [ + "0.0.0.0", + "::", + "[::1]", + "::ffff:127.0.0.1", + "127.0.0.2", + "127.000.0.1", + "0:0:0:0:0:0:0:1", + "localhost.localdomain", + "", +]; + +/// One near miss of a loopback spelling; kind 0 is the spelling itself. +fn near_loopback(spelling: &str, kind: u8) -> String { + match kind { + 0 => spelling.to_owned(), + 1 => format!(" {spelling}"), + 2 => format!("{spelling}\n"), + 3 => spelling.to_ascii_uppercase(), + 4 => { + let mut chars = spelling.chars(); + chars + .next() + .map(|first| first.to_ascii_uppercase().to_string() + chars.as_str()) + .unwrap_or_default() + } + 5 => format!("[{spelling}]"), + 6 => format!("{spelling}:8080"), + _ => format!("{spelling}."), + } +} + +/// Present four times in five, so that a meaningful share of documents carry +/// every required field and take the decode-succeeds path. +fn mostly_present( + value: impl Strategy + 'static, +) -> impl Strategy> { + proptest::option::weighted(0.8, value) +} + +/// A map strategy over the fields of a valid response, so that fields can be +/// dropped individually — version skew as an engine would produce it, not as +/// random bytes would. +fn field_skew_strategy() -> impl Strategy { + ( + mostly_present("[ -~]{0,8}"), + mostly_present(any::()), + mostly_present("[ -~]{0,8}"), + mostly_present(any::()), + mostly_present(any::()), + "[ -~]{0,8}", + ) + .prop_map( + |(status, model_loaded, device, uptime, queue_depth, extra)| { + let mut object = serde_json::Map::new(); + if let Some(status) = status { + object.insert("status".to_owned(), serde_json::Value::String(status)); + } + if let Some(model_loaded) = model_loaded { + object.insert( + "model_loaded".to_owned(), + serde_json::Value::Bool(model_loaded), + ); + } + if let Some(device) = device { + object.insert("device".to_owned(), serde_json::Value::String(device)); + } + if let Some(uptime) = uptime { + object.insert("uptime_sec".to_owned(), serde_json::Value::from(uptime)); + } + if let Some(queue_depth) = queue_depth { + object.insert( + "queue_depth".to_owned(), + serde_json::Value::from(queue_depth), + ); + } + object.insert( + "unknown_future_field".to_owned(), + serde_json::Value::String(extra), + ); + serde_json::Value::Object(object) + }, + ) +} + +/// The fields `HealthcheckResponse` cannot decode without. +const HEALTHCHECK_REQUIRED_FIELDS: [&str; 5] = [ + "status", + "model_loaded", + "device", + "uptime_sec", + "queue_depth", +]; + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] + + /// Version skew: a newer engine sends extra fields and may omit ones this + /// build requires. The document decodes exactly when every required field + /// is present — the unknown field it always carries must be ignored — and + /// a decoded response carries the values that were sent. + #[test] + fn version_skew_decodes_exactly_when_required_fields_are_present(value in field_skew_strategy()) { + let bytes = serde_json::to_vec(&value).expect("encode"); + let all_required_present = HEALTHCHECK_REQUIRED_FIELDS + .iter() + .all(|field| value.get(field).is_some()); + let decoded = serde_json::from_slice::(&bytes); + prop_assert_eq!( + decoded.is_ok(), + all_required_present, + "decode result {:?} for {}", + decoded.as_ref().map(|_| ()).map_err(ToString::to_string), + value + ); + if let Ok(response) = decoded { + prop_assert_eq!(Some(response.status.as_str()), value["status"].as_str()); + prop_assert_eq!(Some(response.model_loaded), value["model_loaded"].as_bool()); + prop_assert_eq!(Some(response.device.as_str()), value["device"].as_str()); + prop_assert_eq!(Some(response.uptime_sec), value["uptime_sec"].as_u64()); + prop_assert_eq!( + Some(u64::from(response.queue_depth)), + value["queue_depth"].as_u64() + ); + prop_assert_eq!(response.last_error, None); + } + decode_all_without_panic(&bytes); + } +} + +/// Golden documents for the wire spellings the round trips cannot see: device +/// policy and GPU selection values, and a literal healthcheck response as an +/// engine binary built from another revision would send it. +#[test] +fn protocol_values_use_their_documented_wire_spellings() { + for (policy, wire) in [ + (DevicePolicy::GpuRequired, "gpu_required"), + (DevicePolicy::GpuPreferred, "gpu_preferred"), + (DevicePolicy::CpuOnly, "cpu_only"), + ] { + assert_eq!( + serde_json::to_value(&policy).expect("encode"), + serde_json::json!(wire) + ); + } + assert_eq!( + serde_json::to_value(GpuSelection::Auto).expect("encode"), + serde_json::json!("auto") + ); + assert_eq!( + serde_json::to_value(GpuSelection::Index(1)).expect("encode"), + serde_json::json!({ "index": 1 }) + ); + + let response: HealthcheckResponse = serde_json::from_value(serde_json::json!({ + "status": "ready", + "model_loaded": true, + "device": "gpu", + "uptime_sec": 12, + "queue_depth": 3, + "last_error": "boom", + "tokens_per_sec": 1.5, + })) + .expect("a healthcheck document decodes"); + assert_eq!(response.status, "ready"); + assert!(response.model_loaded); + assert_eq!(response.device, "gpu"); + assert_eq!(response.uptime_sec, 12); + assert_eq!(response.queue_depth, 3); + assert_eq!(response.last_error.as_deref(), Some("boom")); + assert_eq!(response.tokens_per_sec, Some(1.5)); +} + +/// Non-finite token rates have no JSON representation: they encode as `null` +/// and read back as unknown, rather than failing the whole healthcheck. Pinned +/// deterministically, since the round-trip property only samples them. +#[test] +fn non_finite_token_rates_read_back_as_unknown() { + for rate in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let response: HealthcheckResponse = serde_json::from_value(serde_json::json!({ + "status": "ready", + "model_loaded": true, + "device": "gpu", + "uptime_sec": 1, + "queue_depth": 0, + })) + .expect("decode"); + let response = HealthcheckResponse { + tokens_per_sec: Some(rate), + ..response + }; + let encoded = serde_json::to_string(&response).expect("a non-finite rate still encodes"); + let decoded: HealthcheckResponse = serde_json::from_str(&encoded).expect("decode"); + assert_eq!( + decoded.tokens_per_sec, None, + "{rate} read back as {:?}", + decoded.tokens_per_sec + ); + } +} + +/// `--gpu` values as a user types them, with the selection each must produce. +#[test] +fn gpu_cli_values_parse_as_documented() { + for (value, expected) in [ + ("auto", Ok(GpuSelection::Auto)), + ("AUTO", Ok(GpuSelection::Auto)), + ("", Ok(GpuSelection::Auto)), + (" ", Ok(GpuSelection::Auto)), + ("0", Ok(GpuSelection::Index(0))), + (" 1 ", Ok(GpuSelection::Index(1))), + ("4294967295", Ok(GpuSelection::Index(u32::MAX))), + ] { + assert_eq!(GpuSelection::parse_cli_value(value), expected, "{value:?}"); + } + let multi = GpuSelection::parse_cli_value("1,2").expect_err("multiple GPUs are rejected"); + assert!( + multi.contains("multiple GPUs is not supported"), + "a list of GPUs must say why it is rejected, got {multi:?}" + ); + for value in ["-1", "4294967296", "gpu0", "1.0", "auto1"] { + assert!( + GpuSelection::parse_cli_value(value).is_err(), + "{value:?} must be rejected" + ); + } +} + +/// Literal request documents as the CLI sends them, and a detect response +/// without the optional runtime fields: each must decode to the values it +/// spells. This catches a renamed or removed field; it cannot catch a lost +/// `#[serde(default)]` on an `Option`, which serde treats as optional anyway. +#[test] +fn request_and_detect_documents_have_their_documented_wire_shape() { + let launch: LaunchRequest = serde_json::from_value(serde_json::json!({ + "service_id": "svc", + "env_id": null, + "runtime_id": null, + "model_ref": "org/model", + "host": "127.0.0.1", + "port": 8080, + "device_policy": "gpu_required", + "endpoint_mode": null, + "gpu_selection": { "index": 1 }, + })) + .expect("a launch request decodes"); + assert_eq!(launch.service_id, "svc"); + assert_eq!(launch.model_ref, "org/model"); + assert_eq!(launch.host, "127.0.0.1"); + assert_eq!(launch.port, 8080); + assert_eq!(launch.device_policy, Some(DevicePolicy::GpuRequired)); + assert_eq!(launch.gpu_selection, Some(GpuSelection::Index(1))); + assert!(launch.engine_recipe.is_none()); + + let logs: LogsRequest = + serde_json::from_value(serde_json::json!({ "service_id": "svc", "tail_lines": 50 })) + .expect("a logs request decodes"); + assert_eq!(logs.service_id, "svc"); + assert_eq!(logs.tail_lines, Some(50)); + + let detect: DetectResponse = serde_json::from_value(serde_json::json!({ + "installed": true, + "env_id": "env", + "python_version": null, + "torch_version": null, + "transformers_version": null, + "available_devices": [{ "kind": "gpu", "available": true }], + "capabilities": { + "cpu": false, + "rocm_gpu": true, + "openai_compatible": true, + "tool_calling": false, + "quantized_models": "none", + "reasoning_parser": false, + }, + "notes": ["n"], + })) + .expect("a detect response without the optional runtime fields decodes"); + assert!(detect.installed); + assert_eq!(detect.env_id.as_deref(), Some("env")); + assert!(detect.runtime_kind.is_none() && detect.managed_env.is_none()); + assert_eq!(detect.available_devices.len(), 1); + assert!(detect.available_devices[0].available); + assert!(detect.capabilities.rocm_gpu && !detect.capabilities.cpu); + assert_eq!(detect.notes, vec!["n".to_owned()]); +} + +/// The error envelope an engine sends, as a literal document: a swapped code +/// and message, or a failure that reads as success, must fail here. +#[test] +fn failure_envelope_has_its_documented_wire_shape() { + assert_eq!( + serde_json::to_value(EngineResponseEnvelope::failure("c", "m")).expect("encode"), + serde_json::json!({ "ok": false, "error": { "code": "c", "message": "m" } }) + ); +} + +/// The plugin binary naming rule, positively: the `rocm-engine-` prefix with +/// an optional `.exe` in either case yields the id, and anything else none. +#[test] +fn plugin_binary_names_yield_their_engine_id() { + for name in [ + "rocm-engine-foo", + "rocm-engine-foo.exe", + "rocm-engine-foo.EXE", + ] { + assert_eq!( + engine_id_from_plugin_binary_name(name).as_deref(), + Some("foo"), + "{name}" + ); + } + for name in ["rocm-engin-foo", "engine-foo", "foo", "rocm-engine-"] { + assert_eq!(engine_id_from_plugin_binary_name(name), None, "{name}"); + } +} + +/// The documented wire name of each method. The `match` is exhaustive, so a +/// new method fails to compile until it is listed here; the variant list in +/// the test below and [`METHOD_WIRE_NAMES`] are maintained by hand and are not +/// compile-enforced. +const fn method_wire_name(method: &EngineMethod) -> &'static str { + match method { + EngineMethod::Detect => "detect", + EngineMethod::Install => "install", + EngineMethod::Capabilities => "capabilities", + EngineMethod::ResolveModel => "resolve_model", + EngineMethod::Launch => "launch", + EngineMethod::Healthcheck => "healthcheck", + EngineMethod::Endpoint => "endpoint", + EngineMethod::Stop => "stop", + EngineMethod::Logs => "logs", + } +} + +/// Every method's wire spelling, written out: the plugin protocol is a +/// contract with separately built engine binaries, so a renamed method must +/// fail here rather than at runtime. (The round-trip properties cannot see a +/// rename, since encoding and decoding change together; the golden documents +/// above pin a few more spellings, not the whole wire format.) +#[test] +fn engine_methods_use_their_documented_wire_names() { + for wire in METHOD_WIRE_NAMES { + let envelope: EngineRequestEnvelope = + serde_json::from_value(serde_json::json!({ "method": wire })) + .unwrap_or_else(|error| panic!("{wire:?} must decode: {error}")); + assert_eq!(method_wire_name(&envelope.method), wire); + } + for method in [ + EngineMethod::Detect, + EngineMethod::Install, + EngineMethod::Capabilities, + EngineMethod::ResolveModel, + EngineMethod::Launch, + EngineMethod::Healthcheck, + EngineMethod::Endpoint, + EngineMethod::Stop, + EngineMethod::Logs, + ] { + let wire = method_wire_name(&method); + assert!( + METHOD_WIRE_NAMES.contains(&wire), + "{wire:?} is missing from METHOD_WIRE_NAMES" + ); + assert_eq!( + serde_json::to_value(&method).expect("encode"), + serde_json::Value::from(wire) + ); + let envelope: EngineRequestEnvelope = + serde_json::from_value(serde_json::json!({ "method": wire })) + .unwrap_or_else(|error| panic!("{wire:?} must decode: {error}")); + assert_eq!(envelope.method, method); + } +} + +// `BTreeMap` keys in a recipe hint come off the wire; duplicates in the JSON +// must resolve deterministically rather than depending on iteration order. +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] + + #[test] + fn duplicate_parser_setting_keys_resolve_last_wins( + key in "[a-z]{1,6}", + first in "[ -~]{0,8}", + second in "[ -~]{0,8}", + ) { + let json = format!( + r#"{{"contract_version":"0.1.0","engine":"vllm","parser_settings":{{{key}:{first},{key}:{second}}}}}"#, + key = serde_json::to_string(&key).expect("key"), + first = serde_json::to_string(&first).expect("first"), + second = serde_json::to_string(&second).expect("second"), + ); + let decoded: EngineRecipeHint = serde_json::from_str(&json).expect("decode"); + let expected: BTreeMap = BTreeMap::from([(key, second)]); + prop_assert_eq!(decoded.parser_settings, expected); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 57c4af8a39..689b1a62a2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -6,347 +6,886 @@ SPDX-License-Identifier: MIT # Architecture -This is the living module map for rocm-cli. It's a contributor-facing reference to where things live and why, updated in the same PR as the code it documents. It documents what exists now; the few moves that are still deferred are called out in the entries they affect. Phase labels such as "Phase 5 of ROCMAI-27's sequencing" name the original modularization effort's ordering and are kept only so a module can be traced to the ticket that moved it. It is also not a design-history document: entries describe current structure, not the decisions or debates that produced it. +This is the living module map for rocm-cli: a contributor-facing reference to where things live and why, updated in the same PR as the code it documents. It documents the current structure, not the decisions or debates that produced it. -> Before relying on any entry below, verify current file and function boundaries directly (e.g. `grep`) rather than trusting this doc's wording. Module boundaries shift as the codebase grows, and a stale-but-plausible-looking note is worse than an explicit prompt to check. Every concrete file citation in the Module map's per-crate inventories below — except "the old agent.rs" in the `rocm-dash-tui` entry (historical prose about a deleted file) and `report.json` in the `crates/e2e-report` entry (cucumber's untracked generated output) — is a relative markdown link rather than a bare filename, checked offline by [lychee](https://github.com/lycheeverse/lychee) in CI (the `docs-links` job, config in `lychee.toml`) — a moved or deleted file fails CI immediately, naming the broken link. That check doesn't check that the surrounding prose (e.g. which extraction pattern a module follows) is still accurate; the caution above still applies to everything but link resolution. +> Before relying on any entry below, verify current file and function boundaries directly (e.g. `grep`) rather than trusting this doc's wording. Module boundaries shift as the codebase grows, and a stale-but-plausible-looking note is worse than an explicit prompt to check. A citation of a specific file in this document is a relative markdown link at its first mention in a table row or bullet; later mentions of the same file in that row or bullet are bare, and so are generic mentions (such as `main.rs`, `lib.rs`, `mod.rs` or the workspace `Cargo.toml`), runtime or generated files (such as `config.json` or `report.json`) and files that are not tracked in this repository. The links are checked offline by [lychee](https://github.com/lycheeverse/lychee) in CI (the `docs-links` job, config in [`lychee.toml`](../lychee.toml)), so a moved or deleted file fails CI immediately and names the broken link. That check does not verify that the surrounding prose is still accurate; the caution above still applies to everything but link resolution. + +A second check runs the other way: `cargo xtask check-architecture-doc` ([`architecture_doc.rs`](../xtask/src/architecture_doc.rs)) fails when a production source file, meaning a `.rs` file under `apps/*/src`, `crates/*/src` or `engines/*/src`, has no link in this document, so a new module cannot land without a link here, normally its row. It covers the repository's own tooling and test harness the same way: every `.rs` file under `xtask/src`, `tests/e2e-cucumber/src` and `tests/e2e-cucumber/tests` needs a link too. Only a link that renders as one counts: a path inside an HTML comment, a code block or a code span does not, and neither does a malformed link (one missing its `[` or `)`, or with an escaped bracket), an image, or the outer brackets of a link that contains another link. Write the row's file cell as an inline `[text](path)` link with its text on one line; `` delimiters around the destination are accepted, but reference-style links, a `?query` suffix and root-relative paths are not. `build.rs` files and the `tests/` and `examples/` trees of the product crates are out of scope. It runs in the `clippy` CI job and in the `architecture-doc` prek hook, at commit and at push. Test-only helpers (`test_support.rs`, `test_env.rs`, `proptests.rs`, `*_proptests.rs`, `*_props.rs`, `*_properties.rs`) may have a row but need none in a production crate; in the `xtask` and `tests/e2e-cucumber` trees every file needs one, whatever its name. There is no exemption list: a file in scope with no link fails the check. The check reads no prose, so keeping a row's description true is up to the change that edits the file. ## Module organization convention -New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: +New subcommands and subsystems default to their own file from day one. They should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns exist in the codebase; use whichever fits. The Class column of the module map below records which one a file was extracted under, where that is recorded. + +- **Full domain extraction** (`domain`): a subsystem's domain implementation moves into its own file that owns its own types, not just relocated functions. Every `apps/rocm` module is a private `mod x;` regardless of pattern, so privacy does not tell the two patterns apart. Domain-extracted modules are typically reached through qualified paths (e.g. `comfyui::run(...)`), and so are mechanically relocated modules that own no types (e.g. [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs), [`logging.rs`](../apps/rocm/src/logging.rs)). The exception, reached unqualified through `use crate::x::{...}` re-imports that match the dispatch call sites' pre-extraction form, is limited to the clusters that moved as a dispatch-adjacent group; the block of `use crate::x::{...}` re-imports near the top of [`main.rs`](../apps/rocm/src/main.rs) is the current list of modules reached that way. Where a subsystem has a dedicated clap subcommand, its command enum usually lives in [`cli.rs`](../apps/rocm/src/cli.rs) with the rest of the `Cli`/`Command` definition (e.g. `ComfyuiCommand`, `RuntimesCommand`), the way [`apps/rocmd`](../apps/rocmd/src/cli.rs) keeps its command tree in a `cli.rs` of its own. The dispatch function, the match over that tree, does not stay with the enum: `comfyui::run` lives in [`comfyui.rs`](../apps/rocm/src/comfyui.rs), and `engines()` and `runtimes()` live in [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). Only the top-level `dispatch()`, the match that sends each `Command` variant to its handler, stays in `main.rs`. [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) and [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) own their command enum as well as their dispatch. Not every domain-extracted module has a command enum at all ([`providers.rs`](../apps/rocm/src/providers.rs) has none). In library crates (`crates/rocm-core`) a new subsystem is `pub mod x;` plus a `pub use x::{...};` re-export, since it is part of the crate's public API. This is the default for new subsystems. Examples: [`therock.rs`](../apps/rocm/src/therock.rs) (TheRock SDK install orchestration, with its other clusters in `therock/` child modules and no command enum of its own), [`comfyui.rs`](../apps/rocm/src/comfyui.rs) and [`providers.rs`](../apps/rocm/src/providers.rs); `crates/rocm-core`'s [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs). +- **Mechanical relocation** (`relocation`): a `pub(crate) fn` moves out verbatim, with shared types and config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Example: [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs). When the cluster is large enough to own types, they move with it instead of staying in `main.rs` (e.g. [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`driver_install.rs`](../apps/rocm/src/driver_install.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)). It is still mechanical relocation rather than domain extraction, because the call sites kept their pre-extraction unqualified re-import form when the code moved. Where a subcommand's dispatch function lives is not what decides it: a domain-extraction module can host its own (see `comfyui::run` above). -- **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is usually what distinguishes this pattern from mechanical relocation below, though the Phase 6a variant described next shows ownership alone doesn't fully tell them apart — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart either. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules with no owned types are reached the same way (e.g. [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs), [`logging.rs`](../apps/rocm/src/logging.rs)). The exception — reached unqualified through `use crate::x::{...}` re-imports, matching their dispatch call sites' pre-extraction form — is limited to the clusters that moved as a dispatch-adjacent group; the block of `use crate::x::{...}` re-imports near the top of [`main.rs`](../apps/rocm/src/main.rs) is the current list of the modules it reaches that way (a few of them re-imported for reasons other than dispatch, as the comment above the block explains). [`service_launch.rs`](../apps/rocm/src/service_launch.rs) (as of Phase 6b, ROCMAI-91) uses the same unqualified re-import form for its cross-module callers, but it is not a dispatch-adjacent cluster in the same sense — one of its callers, `automations.rs`, is not a dispatch call site, and its pre-extraction form was a qualified `crate::` call rather than an unqualified import, so the re-import is new rather than kept. This call-site form, not type ownership or dispatch-fn placement, is what actually separates the Phase 6a variant below from full domain extraction. Where a subsystem has a dedicated clap subcommand, its command *enum* usually lives in [`cli.rs`](../apps/rocm/src/cli.rs) as part of the `Command`/`Cli` definition (e.g. `ComfyuiCommand`, `RuntimesCommand`), the way [`apps/rocmd`](../apps/rocmd/src/cli.rs) keeps its command tree in a `cli.rs` of its own — but the dispatch function, the match over that tree, does not always stay in `main.rs` with it: `comfyui()` does, while `engines()` (Phase 6a) and `runtimes()` (Phase 6c, ROCMAI-89) moved out to `engines_cmd.rs`/`runtimes_cmd.rs` respectively, following the mechanical-relocation re-import form below rather than full domain extraction's qualified-path form. Not every domain-extracted module has a dedicated command enum at all (`providers.rs` has none; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: [`apps/rocm/src/therock.rs`](../apps/rocm/src/therock.rs) (TheRock SDK install orchestration, with its update planning and other clusters in `therock/` child modules and no command enum of its own), [`comfyui.rs`](../apps/rocm/src/comfyui.rs) (`ComfyuiCommand`), [`providers.rs`](../apps/rocm/src/providers.rs) (no dedicated command enum); `crates/rocm-core`'s [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs)/[`examine.rs`](../crates/rocm-core/src/examine.rs). -- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Example: [`apps/rocm/src/update_cmd.rs`](../apps/rocm/src/update_cmd.rs). Variant: when the cluster is large enough to own types (part of `main.rs`'s Phase 6a split, ROCMAI-82, `main.rs`'s Phase 6b split, ROCMAI-91, `main.rs`'s Phase 6c split, ROCMAI-89, and Phase 6e's follow-up, ROCMAI-726), those types move with it instead of staying in `main.rs` — [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s types are used nowhere else, but [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s `DriverInstallResult`/`DriverInstallError` are `pub(crate)` and read from [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs)'s `install()` (`result.output`/`result.executed`, `error.source`/`error.executed`); [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs)'s `FreeformExecution` is `pub(crate)` and read from `main.rs`'s `execute_freeform_next_action()` (`execution.action`) — it was already `pub(crate)` before the split, unlike the two functions that return/consume it, which widened from module-private; [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `ServeEngineSelection`/`ServeArgs` both stay `pub(crate)` and are used from `main.rs` (`ServeArgs`) and [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`assess_model_for_host` calls `select_serve_engine` and reads the resulting `ServeEngineSelection`'s `.engine`/`.source` fields) — and `ServeGenerationDefaults` moved out with the engine-recipe overrides to [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs), where it and its fields are `pub(crate)` because `serve()` builds it; [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are private, and its `ManagedLaunchReport` is `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) for the interactive summary; [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)` and read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; its sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)'s `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`. The pattern is still "mechanical relocation" rather than full domain extraction because call sites keep their pre-extraction unqualified-re-import form — the exception named above — rather than switching to the qualified paths full-domain-extraction modules use. The top-level dispatch fn (`dispatch()`) staying in `main.rs` is not what makes it so: full domain extraction usually keeps its dispatch fn in `main.rs` too (see `comfyui()` above). +Moving a `proptest!` block to another file leaves its saved failure cases behind unless they are carried over: proptest keys them by the test's source file (`/src/.rs` reads `/proptest-regressions/.txt`, where `src/` is the directory holding `main.rs` or `lib.rs`) and does not warn when the file stops matching. Carry them in the same commit, and how depends on how much moved. When every property test of the file moves and the new file has no regression file yet, `git mv` the old one to the name the new file needs. When only some move, the tests left behind still read the old file, so keep it, and copy the saved cases to the new file's regression file (appending to it when it already exists, so its own cases stay). An `xtask` test fails the build on a regression file no test reads; it cannot notice a regression file that was moved away from tests that still need it, so check a split's diff for that. See [Property tests](testing.md#property-tests). -A flat size limit backs this convention: `cargo xtask check-module-size` ([`module_size.rs`](../xtask/src/module_size.rs)) reports every source file with more than 1,500 production lines, and there is one limit for every file, with no per-file ceilings, baseline or exemption. Production lines exclude test-gated items (inline `#[cfg(test)]` modules and the like), and test-only files are not source files; the exact definition is at the top of that file. Today the check is advisory: the `clippy` CI job and the `cargo-module-size` pre-commit hook print the offenders without failing, until the files over the limit have been split. A PR that would push a file past 1,500 production lines splits it in the same change (a pure move, the tests moving with the code, this document updated in the same PR); raising the limit or exempting a file is not an option for a feature PR. A change that merely edits a file that is already over the limit does not by itself have to split it: those files are split under their own tickets, and the check becomes blocking only once none is left. `too_many_lines = "allow"` in the workspace `Cargo.toml` stays a deliberate function-level choice; this gate is about files. +A flat size limit backs this convention: `cargo xtask check-module-size` ([`module_size.rs`](../xtask/src/module_size.rs)) fails when any source file has more than 1,500 production lines, and there is one limit for every file, with no per-file ceilings, baseline or exemption. Production lines exclude test-gated items (inline `#[cfg(test)]` modules and the like), and test-only files are not source files; the exact definition is at the top of that file. The check blocks merges: the `clippy` CI job, a required check that also runs on merge-queue runs, fails when any file is over the limit, so a file over it has to be split before anything else merges. The `cargo-module-size` pre-commit hook is the local mirror of that step: it runs the same check and refuses a commit that stages Rust source while a file is over the limit, without rewriting or restaging anything. It measures the working tree, so it also fails when a Rust file under a crate's `src/` is declared by no `mod`, even an untracked scratch file (the error says to declare or delete it). Because it runs at pre-commit and not pre-push, it also refuses the intermediate commits when an over-limit file is split across several commits, where CI only sees the final state: do the split in one commit, or commit the intermediate steps with `SKIP=cargo-module-size git commit ...`. It runs only for contributors who installed the hooks, so CI stays the authority. A PR that would push a file past 1,500 production lines splits it in the same change (a pure move, the tests moving with the code, this document updated in the same PR); raising the limit or exempting a file is not an option for a feature PR. `too_many_lines = "allow"` in the workspace `Cargo.toml` stays a deliberate function-level choice; this gate is about files. ## Module map -Scoped to the crates that make up the shipped CLI/daemon/dashboard/engine surface, plus `crates/e2e-report` (a shared exception: it's HTML/markdown reporting consumed only by `xtask` and `tests/e2e-cucumber`, but it's still one of the modularization effort's target files, so it's mapped below). Dev-tooling and test-harness workspace members (`xtask`, `tests/e2e-cucumber` themselves) are otherwise out of scope — they're not part of the modularization effort's inventory. +Scoped to the crates that make up the shipped CLI, daemon, dashboard and engine surface, plus `crates/e2e-report`, `xtask` and `tests/e2e-cucumber`. `crates/e2e-report` is HTML and markdown reporting consumed only by `xtask` and `tests/e2e-cucumber`; those two are the repository's tooling and test harness, and `check-architecture-doc` checks that every file in them has a row. -Each per-crate module inventory lists one file per line. Flat inventories (every one below except `rocm-dash-tui`'s directory-split `agent/` and `app/` lists, `e2e-report`'s directory-split `consolidated/` list and the `engines/aim` list) are sorted alphabetically by filename, so a newly-extracted module has a predictable insertion point and doesn't force a rewrite of its neighbors; the `agent/`, `app/` and `consolidated/` lists instead keep `mod.rs` first, then the other files in logical (not alphabetical) order. +Each crate has a table with a row per source file (`rocm-dash-tui` and `tests/e2e-cucumber` have one table per directory), sorted by path with the crate roots (`main.rs`, `lib.rs`) first and a directory's `mod.rs` first within its directory. The Class column records how a file came to be, not a rule a new file is judged by: `domain` (full domain extraction or a child module of one, the default for a new subsystem), `relocation` (mechanical relocation) and `flat` (everything else: crate roots, files with no extraction pattern recorded, and test-only support). The `xtask` and `tests/e2e-cucumber` tables are `flat` throughout, since they hold tooling and test files rather than extracted subsystems, so the column carries no information for them. Under each table, **Boundaries** lists some of the facts a row cannot show, such as layering and leaf modules, deliberate privacy and shared sources of truth. It is a selection, not an inventory; the source files carry the rest. ### `apps/rocm` — main CLI binary -[`main.rs`](../apps/rocm/src/main.rs) is the crate root. Besides smaller helpers, it holds `main()`, `run()` and `dispatch()`, the match over the clap `Cli`/`Command` tree that [`cli.rs`](../apps/rocm/src/cli.rs) defines; the builders that turn that tree into a `clap::Command` (`cli_command()`, `completion_command()`); the command handlers still defined here (`comfyui()`, `version()` and `launch_default()`, the no-subcommand arm); freeform-invocation routing (`parse_freeform_invocation`, `should_treat_as_freeform`, `run_freeform`, `execute_freeform_next_action`, `treat_as_natural_language`); a few crate-root items several modules share (`status_for_readiness`, `device_policy_name`, `engine_inventory`, `ROLLBACK_RECOVERY_HINT`); the `use crate::x::{...}` re-import block described above, which also re-imports the `cli.rs` types so other modules keep reaching them as `crate::Command`, `crate::Provider` and so on; and the crate's shared `mod tests` block. - -- [`automations.rs`](../apps/rocm/src/automations.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-210): the `rocm automations` dispatch handler (`automations()`) plus the `rocm automations` status report it prints (`render_automations_text`), the `ProposalKind` classification that report renders through (`proposal_kind`, `fallback_tool_for_automation_action`, `proposal_plain_summary`, `proposal_plain_reason`), the small plain-language label helpers (`proposal_status_label`, `proposal_bool_argument`, `automation_event_plain_summary`, `audit_event_plain_summary`, `watcher_mode_plain_label`, `watcher_last_check`, `watcher_plain_name`, `watcher_plain_trigger`, `watcher_plain_action`, `watcher_policy_note`), and the unrelated `rocm daemon --status` summary (`render_daemon_text`). `render_automations_text` and `render_daemon_text` are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs)'s `dispatch()` calls `render_daemon_text` directly (re-imported alongside `automations` via `use crate::automations::{automations, render_daemon_text};`; `render_automations_text` has no `dispatch()` call site of its own, reached only through `automations()` above, now local to this file), and [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool summaries call both. Every other item in this list, including `watcher_policy_note` (called from `automations()`'s `Enable` arm and from `render_automations_text`, both in this same file), has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do. -- [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) — a further-extracted variant of full domain extraction: it owns its clap command enum (`BootstrapCommand`) and dispatch function too, rather than leaving the enum in [`cli.rs`](../apps/rocm/src/cli.rs) and the dispatch in [`main.rs`](../apps/rocm/src/main.rs). -- [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) — mechanical relocation, no owned types: the host snapshot behind the internal bridge surface (`build_codex_bridge_snapshot`, `build_codex_bridge_gpu_snapshot`, `builtin_codex_bridge_engine_inventory`), whose engine list is derived from `main.rs`'s `engine_inventory`, the single table of built-in engine ids that `rocm examine`, `rocm update` and the `--engine` help test read as well. All three are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs)'s `dispatch()` calls `build_codex_bridge_snapshot` through a re-import, [`chat.rs`](../apps/rocm/src/chat.rs)'s internal MCP tools reach them as `crate::` through the same re-import. It has no tests of its own. -- [`chat.rs`](../apps/rocm/src/chat.rs) — full domain extraction (Phase 6d, ROCMAI-88), mirrors [`providers.rs`](../apps/rocm/src/providers.rs). Owns the chat/MCP-bridge cluster's types (`DeterministicModelRecipe`, `DeterministicServiceRow`, `DeterministicEngineRow`, and others — all module-private), its execution logic, deterministic tool-output summarization (including `render_services_tool_result_text`), and the MCP tool-bridge entry points (`run_internal_mcp_call` and its helpers). The tool-call validation and approval policy (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `ChatRocmCommandAction`, `ChatToolApprovalRequest`) lives in [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs), which this module calls into and re-exports from. Callers in other modules, production and test-only, reach this module's items, and the re-exported ones, through the `chat::` module boundary. As a child module of the crate root (`mod chat;` in `main.rs`), Rust's module-privacy rule already makes most of `main.rs`'s private helpers visible to `chat.rs` without any visibility change, when `chat.rs` calls into `main.rs`. The other direction needs real widening: `chat.rs`'s own functions reached from other modules — `run_internal_mcp_call` (a production call from `main.rs`), `deterministic_comfyui_tool_summary` (reached only by `comfyui.rs`'s own test module), among others — stay `pub(crate)`; the third PR's visibility audit tightened every other function to module-private, so each function still `pub(crate)` has a confirmed external caller, production or test-only. The cluster's regression tests live in `chat.rs`'s own `mod tests`, moved out of `main.rs`'s shared `mod tests` block across the second and third PRs; `serve_negative_sampling_space_form_reaches_range_validator`, a shared clap value-parser wiring test living alongside its sibling command-parsing tests, has since moved with them to the test module in [`cli.rs`](../apps/rocm/src/cli.rs); the four structural checks that read [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs)'s source for the `Command::Chat` handler's wiring (`command_chat_interactive_routes_to_dash_run_chat_not_tui`, `command_chat_interactive_notes_dropped_provider_flag`, `command_chat_honors_chat_mock_and_keeps_prompt_passthrough`, `command_chat_reads_prompt_from_piped_stdin`) live in that module's own `mod tests`. -- [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm chat`'s command handler (`run_chat_command`, the body of the `Command::Chat` arm of `dispatch()`), which hands off to the dashboard chat (`dash::run_chat`) in a terminal with no `--prompt` and otherwise prints the reply or status screen through `chat::render_chat_prompt_text`/`chat::render_chat_text`. It is a module of its own, not part of [`chat.rs`](../apps/rocm/src/chat.rs), because it calls into both `chat.rs` and [`dash.rs`](../apps/rocm/src/dash.rs) while `dash.rs` already calls into `chat.rs`; this module depends on both and neither depends on it. `Provider` lives in [`cli.rs`](../apps/rocm/src/cli.rs) and `ChatInferenceParams` stays in [`main.rs`](../apps/rocm/src/main.rs); `run_chat_command` is `pub(crate)` and re-imported there for `dispatch()`. Four source-text tests in this module's own `mod tests` read this file and assert on the handler's text, so a rename of the function or of the calls they look for has to be made there too. -- [`chat_host_facts.rs`](../apps/rocm/src/chat_host_facts.rs) — full domain extraction. -- [`cli.rs`](../apps/rocm/src/cli.rs) — the `rocm` command line definition: `Cli`, `Command`, the subcommand enums nested under it (`BenchCommand`, `InstallTarget`, `EnginesCommand`, `RuntimesCommand`, `StorageCommand`, `ComfyuiCommand`, `ServicesCommand`, `AutomationsCommand`, `ConfigCommand`, `SetupCommand`) and the clap value enums behind `--provider`, `--device`, `--framework` and similar arguments (`Provider`, `DevicePolicyArg`, `FrameworkArg`, `InstallFormat`, `WatcherModeArg`, `TelemetryModeArg`, `SandboxToolArg`, `PermissionsModeArg`), with `SUPPORTED_ENGINES` (the `--engine` choices) and the conversions from a value enum to its domain type. Like [`apps/rocmd`](../apps/rocmd/src/cli.rs)'s `cli.rs`, it holds definitions only: the `dispatch()` that matches on `Command` stays in [`main.rs`](../apps/rocm/src/main.rs). Declaration order sets the order of every nested `--help` command list (the top-level list is sorted alphabetically by `cli_command()` in `main.rs`), so reordering variants reorders help output. Its items are `pub(crate)` and re-imported in `main.rs`, so other modules keep reaching them as `crate::Command`, `crate::Provider` and so on; [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) and [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) own their enums and appear in `Command` as a single variant each. The tests that parse the tree or render its help live in its own `mod tests`. -- [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs) — mechanical relocation, no owned types: the CLI audit and lifecycle log (`record_cli_audit_event`, plus the helpers behind it that write `logs/cli-lifecycle.log` and the per-action files under `logs/cli/`). Its production code depends only on `rocm-core`. `record_cli_audit_event`, `cli_lifecycle_log_path` and `sanitize_log_value` are `pub(crate)` and reached from other modules as `crate::` through a re-import in [`main.rs`](../apps/rocm/src/main.rs); every other item is private. The two regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. -- [`cli_progress.rs`](../apps/rocm/src/cli_progress.rs) — shared CLI-output component (`Spinner`, `AnimatedSpinner`). -- [`cli_report.rs`](../apps/rocm/src/cli_report.rs) — shared CLI-output component (`ActionReport`, `write_marked_row`, `JsonOrText`/`print_json_or_text`). -- [`comfyui.rs`](../apps/rocm/src/comfyui.rs) — full domain extraction; serves the `comfyui` subcommand, whose command enum (`ComfyuiCommand`) lives in [`cli.rs`](../apps/rocm/src/cli.rs) and dispatch fn stays in [`main.rs`](../apps/rocm/src/main.rs) (unlike [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) above, which owns both). It owns the ComfyUI types and constants, the install and state paths, the manifest and state files, `status` and `logs` rendering, and the helpers that run a Python child and capture its output. Choosing the ROCm runtime and building its environment ([`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs)), the process lifecycle ([`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs)) and the install ([`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs)) are in child modules, listed below with the other `apps/rocm` modules; `start`, `stop` and `install` are re-exported from them so callers still write `comfyui::start`, `comfyui::stop` and `comfyui::install`. -- [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm config`'s command handler (`config()`), the provider-key prompt it uses (`read_provider_key_from_user`) and the text `rocm config show` prints (`render_config_text`, `telemetry_policy_summary`). `ConfigCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). `config()` and `render_config_text` are `pub(crate)`: `dispatch()` calls `config()` and [`chat.rs`](../apps/rocm/src/chat.rs)'s read-only handling of `rocm config show` calls `render_config_text`, both through re-imports in `main.rs`; the other two functions are private. The two `render_config_text` regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. -- [`dash.rs`](../apps/rocm/src/dash.rs) — full domain extraction. -- [`dash_seam.rs`](../apps/rocm/src/dash_seam.rs) — full domain extraction. -- [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: the command handlers for `rocm diagnose` and `rocm fix` (`diagnose()`, `fix()`) and the report step behind `rocm diagnose --report` (`show_prepared_report`, `perform_delivery`, `established_entry`). `diagnose()` and `fix()` are `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`; the other three functions are private. `diagnose()` gets its `--model` verdict from [`model_fit.rs`](../apps/rocm/src/model_fit.rs), and `fix()` returns a failing exit code through `FixExitCode`, which stays in `main.rs` with the other exit-code plumbing. The report and delivery regression tests live in this module's own `mod tests`; the exit-code tests stay in `main.rs`. -- [`driver_install.rs`](../apps/rocm/src/driver_install.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6a split, ROCMAI-82; see the Phase 6a variant note above): `install_driver()`/`reconcile_driver_install()`, running an approved plan's shell commands, and the install state persisted to `state.json` (`DriverInstallState`, `DriverReconciliationState` and the passive-check types). The plan it runs lives in [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs), which this module imports from and which does not import from it; the os-release the plan is built from comes from `real_os_release` in [`system_packages.rs`](../apps/rocm/src/system_packages.rs). Its `pub(crate)` surface is `install_driver` and `reconcile_driver_install`, `DriverInstallResult` and `DriverInstallError` (read from `install_cmd.rs`'s `install()`), `driver_install_state_path`, and a re-export of `host_install_driver_only_reports` for [`chat.rs`](../apps/rocm/src/chat.rs) and [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs). -- [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs) — full domain extraction, crate-level (split out of [`driver_install.rs`](../apps/rocm/src/driver_install.rs)): the plan `rocm install driver` builds and prints. It owns the model (`DriverInstallPlan` with `execution_commands`, `DriverPlanCommand`, `DriverCommandPhase`, and `PrivilegeEscalation`, which decides whether a generated shell line gets a `sudo ` prefix), `render_driver_install_plan` with its approval label, the helpers the builders share (`driver_command`, `driver_root_preflight_checks`, `resolve_shell_default_template`, `is_safe_shell_token`), and the builders: `build_driver_install_plan_read`, which `install_driver` calls with the real host's os-release, and `build_driver_install_plan_at` beneath it, with the apt, dnf and SLES plans, the `ID_LIKE` fallback, the repository URL helpers, `resolve_repo_version_expr` (the `ROCM_CLI_AMDGPU_DRIVER_VERSION` override and the retired version variables it refuses), and the refusal plans for an unreadable os-release or an invalid override. It also owns `install_driver_only_reports_on`/`host_install_driver_only_reports` (true on native Windows, where the plan only reports); `driver_install.rs` re-exports the latter. It declares the child module [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) and re-exports its `wsl_rocdxg_driver_plan`. Production code in this file uses only `std` and `rocm_core`, and nothing here depends on `driver_install.rs`. The model types and their fields, `execution_commands`, `PrivilegeEscalation::detect`, `driver_command`, `render_driver_install_plan`, `build_driver_install_plan_read`, `install_driver_only_reports_on` and `host_install_driver_only_reports` are `pub(crate)`, as is `invalid_driver_version_override_plan`, which a test that stays in `driver_install.rs` builds a plan with; the rest is private. Its test module is `pub(crate)` so `driver_install.rs`'s tests can reuse its `test_examine` and `os_release_found` helpers. It holds the plan-builder tests, including the one that keeps the supported-distro matrix in step with `dkms_driver_plan_is_supported` in `rocm-core`, and the tests of the template resolver and the shell-token filter. -- [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) — full domain extraction, a child of `driver_plan.rs` that depends on the plan model, `driver_command`, `driver_root_preflight_checks` and `resolve_shell_default_template` in its parent, and on `truthy_env` in [`therock.rs`](../apps/rocm/src/therock.rs) (split out of [`driver_install.rs`](../apps/rocm/src/driver_install.rs)): the WSL2 rocDXG plan. `wsl_rocdxg_driver_plan` builds the plan that installs the `rocdxg-roct` package, `resolve_rocdxg_verification` decides how that download is authenticated (a supplied digest, the digest pinned for the version in `ROCDXG_PINNED_DIGESTS`, or the explicit opt-out), and `wsl_rocdxg_refusal_plan` is the plan that installs nothing and says why when it cannot be authenticated or the version is not a well-formed package version. `ROCDXG_PINNED_DIGESTS` is the trust anchor an unattended root install is checked against, so a change to it is a security-relevant diff; it moved byte for byte. `wsl_rocdxg_driver_plan` is `pub(crate)` and re-exported by `driver_plan.rs` (its builders call it, and tests in `driver_install.rs` use it as a plan fixture); everything else in production code is private. Its tests are the `wsl_rocdxg_*` plan tests, including the one that runs the generated digest step against a real file on Unix. The test-only `scoped_rocdxg_env` helper that clears the rocDXG environment lives here too, `pub(super)` for the plan-builder tests in `driver_plan.rs`. -- [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. -- [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. -- [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the stdio request/response envelope used to talk to an external engine binary or the in-process built-in handler (`engine_request`, `engine_request_with_env_root`, `decode_engine_response`, the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `decode_engine_response`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. -- [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. -- [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. -- [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it outside its own tests. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. One test here, `linux_only_engine_runtime_status_is_explicit_on_native_windows`, also calls `model_registry_adapter_availability_note` from `model_catalog_text.rs`, which is `pub(crate)` for that reason; its primary subject is `engine_runtime_status_label`, so it stays. -- [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) — mechanical relocation, no owned types: the answers a freeform request can get without running a command — `render_freeform_read_only_answer`, which answers a single approval-free `examine`, `comfyui status` or `comfyui logs` action, and `render_freeform_examine_answer`, the plain-language ROCm status it prints for `examine`. `run_freeform` in [`main.rs`](../apps/rocm/src/main.rs) is the only caller of the first, which is `pub(crate)` and re-imported there; the second is private. Besides `chat.rs` (`any_substring`, re-exported there from [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs)), it depends on `comfyui.rs`, `runtimes_cmd.rs` and `therock.rs`, none of which calls back into it. No unit test calls either function directly. -- [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) — mechanical relocation, dispatch-adjacent, owns a `pub(crate)` type (as of Phase 6e's follow-up, ROCMAI-726; see the Phase 6a variant note above for the owned-types pattern). Holds the freeform natural-language planner's plan-*execution*/consent half that [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) below deliberately left out: `validate_freeform_execution_action()`, `prepare_freeform_execution()`, `apply_freeform_execution_consent()`, `render_freeform_execution_header()`, and the `pub(crate)` `FreeformExecution` struct that `prepare_freeform_execution` returns and `render_freeform_execution_header` consumes. `main.rs`'s `execute_freeform_next_action()` and `run_freeform()` stay in `main.rs` — both are dispatch-level, not part of deciding whether a plan's next action is safe to execute — and `execute_freeform_next_action()` calls in via the unqualified re-import `use crate::freeform_execution::{prepare_freeform_execution, render_freeform_execution_header};` (`run_freeform()` only calls `execute_freeform_next_action()`, never this module directly), matching [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s dispatch-adjacent form rather than `freeform_planner.rs`'s own qualified-path form. `prepare_freeform_execution` takes the already-built `plan` by value rather than rebuilding it from `request`/`paths`/`config` (ROCMAI-725's single-build guarantee, inherited unchanged from `main.rs`); `prepare_freeform_execution`/`render_freeform_execution_header` are `pub(crate)` for that cross-module call; `FreeformExecution` was already `pub(crate)` (read directly by `main.rs` and this module's own tests); `validate_freeform_execution_action`/`apply_freeform_execution_consent` have no caller outside this module and stay module-private, unchanged from before the move. The cluster's six regression tests moved out of `main.rs`'s shared `mod tests` into this module's own `mod tests`, inheriting the qualified `freeform_planner::` path they already used to reach `freeform_planner.rs`'s items as `main.rs`'s former tests — this module reaches in from outside using the same qualified path `main.rs` and `providers.rs`'s tests use (unlike `chat.rs`/`comfyui.rs`, which import it unqualified). -- [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction`, `StructuredRequestPlan` and `PlannedToolCall`, which are `pub(crate)`, and `ProviderPlannerToolCall`, `pub(crate)` together with `validate_provider_planner_tool_call` and `planner_failure_note` so [`chat.rs`](../apps/rocm/src/chat.rs)'s command-preview property tests can drive the planner's guards), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow ([`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `render_launch_summary` → `host_default_engine` is another call into this module, unrelated to planning: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, while [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs)'s `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` from [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) directly as a no-CPU-fallback safety check on provider-proposed `serve` args; `validate_provider_planner_tool_call` calls that same function in its own `serve` branch, and hands an `install sdk` call to `tool_policy::validate_chat_rocm_command_safety`. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context` and `provider_planner_response_to_plan` are `pub(crate)` for these reasons, as is `plan_driver_request` (read from [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs)'s test module); `format_structured_tool_call` lives in [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) and is re-exported here with `pub(crate) use`, so the callers above keep their `freeform_planner::` path, while the planner's own argv lookups call `rocm_argv`, its phrase and engine readers call [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) and its safety check calls `tool_policy.rs`, directly: the planner no longer imports `chat`, and only `chat` imports the planner (`plan_freeform_request_with_context`); `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller. -- [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`).Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. -- [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm install`'s command handler (`install()`, for the `sdk` and `driver` targets), the `--version`/`--build-date` selector parsing and display it uses (`therock_install_version_selector`, `therock_install_version_selector_display`) and `sdk_install_request`, which maps the parsed `install sdk` arguments onto a `therock::SdkInstallRequest` and moved here from [`chat.rs`](../apps/rocm/src/chat.rs) with its two tests. `InstallTarget` and `InstallFormat` live in [`cli.rs`](../apps/rocm/src/cli.rs). `install()` is `pub(crate)` and re-imported in `main.rs` for `dispatch()`; `therock_install_version_selector` is `pub(crate)` too, and `chat.rs`'s `render_install_sdk_dry_run_for_args` reaches it as `crate::therock_install_version_selector` through the same kind of re-import, which is why `sdk_install_request` lives here and not in `chat.rs`: this module does not depend on `chat.rs`. `sdk_install_request` keeps the `pub(crate)` it had, although only `install()` and the tests call it now. -- [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. -- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. -- [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs) — mechanical relocation out of `examine_report_text.rs`: the `rocm model` (alias `models`) catalog renderer. `render_model_registry_text_with_context_and_host` prints the curated list grouped by hardware target (`model_catalog_platform_heading`, `model_catalog_header`, `model_recipe_memory_label`, `model_recipe_gpu_fit_label`), and `render_model_registry_verbose_text_with_context_and_host` prints the per-recipe diagnostics behind `rocm model --verbose` (`append_model_recipe_registry_source`, `append_model_recipe_metadata_lines`, `append_model_engine_recipe_settings_lines`, `append_model_recipe_staleness_lines`, `append_model_artifact_lines`, `append_artifact_source_policy_lines`, `artifact_source_policy_label`, `append_model_host_ram_fit_lines`, `append_model_fit_lines`, `append_manual_alternative_lines`, `manual_alternative_recommendations`, `recipe_is_manual_fit`, `append_model_engine_support_lines`, `model_registry_adapter_availability_note`, `recipe_display_ref`, `preferred_engine_action_target`, `model_registry_reason`, plus the small formatters `format_list_or_none`, `format_string_map_or_none`, `format_gib` and `once_per_call`). `format_bytes`, the human-readable size formatter, lives here too. The two renderers and `format_bytes` are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs) imports them for the `Command::Model` arm, and [`chat.rs`](../apps/rocm/src/chat.rs) (the verbose renderer), [`storage.rs`](../apps/rocm/src/storage.rs) and [`automations.rs`](../apps/rocm/src/automations.rs) (`format_bytes`) reach them as `crate::`. `model_registry_adapter_availability_note` is `pub(crate)` only because a test in `examine_report_text.rs` calls it. The module has no edge to `examine_report_text.rs`; it reaches the engine-availability helpers in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs), `PlatformRuleOut` and `engine_ruled_out_by_platform_with` at the crate root, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `select_serve_engine`. The catalog's regression tests live in this module's own `mod tests`; they keep the names they had in `examine_report_text.rs`. The `format_bytes` property test's saved failure case lives in [`model_catalog_text.txt`](../apps/rocm/proptest-regressions/model_catalog_text.txt), which proptest finds by the test's source file, so that file has to be renamed whenever the test moves. -- [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) and [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which read `PlatformRuleOut` and the platform gate (`engine_ruled_out_by_platform` and `engine_ruled_out_by_platform_with` respectively), do not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::` (`model_catalog_text.rs` reaches `PlatformRuleOut` and `engine_ruled_out_by_platform_with` the same way); every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. -- [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) — text extraction from a natural-language request, shared by `chat.rs`'s prompt-intent heuristics (`fallback_rocm_tool_call_for_prompt`, `fallback_config_tool_call_for_prompt`, `install_sdk_chat_approval_for_prompt` and the `prompt_*` predicates) and the freeform planner, in a leaf module that depends only on `therock::RuntimeVersionSelector` (`tool_policy.rs` does not use it): `any_substring`, the install-folder, TheRock build-date and TheRock version readers (`requested_install_prefix_from_prompt`, `requested_therock_build_date_from_prompt`, `requested_therock_version_from_prompt`, with `requested_bare_install_prefix_from_prompt`, `clean_requested_install_prefix`, `looks_like_user_path` and `looks_like_therock_runtime_version` private beside them) from `chat.rs`, and `infer_engine_from_request` from `freeform_planner.rs`. They moved verbatim, together with `infer_engine_from_request`'s regression test, so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `any_substring`, which [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) still reaches as `chat::any_substring`. -- [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) — full domain extraction. -- [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). Keeps the request/response types, the `ProviderAdapter` trait and the `provider_*` dispatch, the `LocalProvider` and `RemoteProvider` implementations, local-service selection (`select_local_chat_service`, `ready_local_services`), the remote model and endpoint resolution, the `ureq` remote transport and the `SseBytesEmitter` trait. The ROCm tool schema, the local HTTP/SSE transport and the provider wire formats live in child modules under `providers/`, below; `providers.rs` imports the request-body builders, response parsers and SSE emitters it calls from `providers/wire.rs`. -- [`providers/tool_schema.rs`](../apps/rocm/src/providers/tool_schema.rs) — full domain extraction, a leaf child of `providers.rs`: the ROCm tool catalog a provider advertises to a model, as OpenAI function definitions (`rocm_openai_tool_definitions`) and reshaped for Anthropic's `tools` field (`rocm_anthropic_tool_definitions`). Both are `pub(super)`, read by the request-body builders in `providers/wire.rs`; `rocm_openai_tool`, the one-tool constructor behind them, is private. Depends on nothing else in `providers`. -- [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edges back into `providers` are `OpenAiSseEmitter` from `providers/wire.rs`, the `SseBytesEmitter` trait the emitter is pushed bytes through (which stays in `providers.rs`), and the shared `ProviderStreamEvent` type; nothing in `wire.rs` reads `local_http.rs`, so the edge is one-way. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. -- [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) — full domain extraction, a child of `providers.rs`: the OpenAI-compatible and Anthropic wire formats. The request-body builders (`openai_chat_request_body`, `openai_stream_chat_request_body`, `openai_local_chat_request_body`, `openai_local_stream_chat_request_body`, `anthropic_chat_request_body`, `anthropic_stream_chat_request_body`, and the helpers behind them, including `local_openai_compatible_request`, `apply_local_model_request_defaults` and `local_model_should_disable_thinking`), the parsers for non-streaming responses (`parse_openai_chat_message`, `parse_anthropic_chat_message`), and the SSE emitters with their line parsers (`OpenAiSseEmitter` and `AnthropicSseEmitter`, which implement the `SseBytesEmitter` trait that stays in `providers.rs`; `parse_openai_sse_line`, `parse_anthropic_sse_line`). The builders, the two chat-message parsers and the two emitters (with their `new` and `finish`) are `pub(super)`, called from `LocalProvider` and `RemoteProvider`; `OpenAiSseEmitter` is also read by `providers/local_http.rs`. Depends on the shared request/response types in `providers.rs` and on `providers/tool_schema.rs`, and on nothing else. The tests of the builders and parsers live in this module's own `mod tests`, with the tool-catalog assertions that read through the request bodies, and [`providers/wire/sse_emitter_proptests.rs`](../apps/rocm/src/providers/wire/sse_emitter_proptests.rs), the split-invariance and panic-guard properties of the emitters and line parsers, is declared from this module; the stream tests that run an emitter against a mock server stay in `providers.rs`'s `mod tests`. -- [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) — the argv helpers that `chat.rs`, `tool_policy.rs` and the freeform planner share, in a leaf module whose only `crate::` dependency is the leaf [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs): `chat_cli_arg_value`, `chat_cli_arg_value_checked`, `chat_cli_has_flag` and `ensure_flag` (from `chat.rs`), and `format_structured_tool_call` (which quotes each argument with `shell_quote`), `serve_args_request_cpu_device` and `device_policy_is_cpu_mode` (from `freeform_planner.rs`). They moved so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `ensure_flag` (read by `freeform_execution.rs`) and `freeform_planner.rs` re-exports `format_structured_tool_call`, so every other caller keeps its path. -- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. -- [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and imported by [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), so the import edge between the two runs one way, from `engines_cmd.rs` to this module. This module (and its torch-alignment child) does not import `engines_cmd.rs`: the engine environment and runtime-selection helpers it needs (`engine_manages_own_runtime`, `env_root_for_engine_install`, `runtime_key_for_python`, and `runtime_manifest_for_selector` from the torch-alignment child) are imported from [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. -- [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. -- [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. -- [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`cli.rs`](../apps/rocm/src/cli.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. -- [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. -- [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` does). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs), [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) is deliberately not one of them: it imports from `runtime_adopt.rs` and `runtime_registry.rs` directly, because going through this module would lead back to it via `runtime_services.rs`. The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. -- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. -- [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. -- [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. -- [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, the `RecordedPidOutcome` and `RecordedStopTermination` types, `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and this module do import each other (this module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from it, and `service_launch.rs` takes `run_internal_sandbox_tool` from here), the pairing `services_text.rs` had with `service_launch.rs` before the split; `services_text.rs` still reaches `background_helper_note` in `service_launch.rs`, so that pair remains too, and breaking either is a structure change rather than a move. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. -- [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. -- [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. -- [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. -- [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). -- [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. -- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. -- [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat.rs`](../apps/rocm/src/chat.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. -- [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) — shared CLI-output component (`shell_quote`): the CLI's name for `rocm-core`'s POSIX quoter, [`posix_shell_quote`](../crates/rocm-core/src/quoting.rs). `rocm-dash-tui` keeps a copy as `ui::exec::quote_display_arg` in [`exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs), since the crate-edge check forbids it `rocm-core`; a per-character walk and a property test here hold the two to the same output. It covers the remote `ssh` command line, the chat approval card and the plan for a free-form request, the install dry-run `command:` line, the [`service_hints.rs`](../apps/rocm/src/service_hints.rs) hints, the runtime-activation report, the `rocm services logs ` lines `rocm services` lists and the `rocm comfyui install --runtime-id` advice a failed ComfyUI reinstall prints — not every `rocm …` hint the CLI prints; older ones such as `rocm logs ` still interpolate their value bare. Also holds `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. -- [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. -- [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and `real_os_release`, the real host's os-release that every one of these plans is built from. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions and `real_os_release` are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), and [`driver_install.rs`](../apps/rocm/src/driver_install.rs) imports `real_os_release` from here directly. `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `real_os_release` ignores a simulated host root. -- [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token`, `slugify` and `has_nontrivial_directory_contents`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read six of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key`, `wheel_runtime_key` and `has_nontrivial_directory_contents` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) lives in [`cli.rs`](../apps/rocm/src/cli.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). -- [`therock/version.rs`](../apps/rocm/src/therock/version.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): the version grammar behind runtime selection and update checks. `parse_version` is the strict gate an install is routed on; `parse_version_for_ordering`, `compare_version_strings` and `version_relation` implement the PEP 440 ordering and the "is this a different version" question; `parse_host_version` reads a host-reported ROCm version through the same parser; `runtime_version_display`, `runtime_version_build_date`, `normalize_requested_build_date` and the calendar helpers behind them handle build dates; `is_stable_runtime_version` and the `ParsedVersion`/`VersionOrderKey`/`VersionStage` types back them. The items `therock.rs` and its property-test module use are `pub(super)` (including the `ParsedVersion` and `VersionOrderKey` fields they read); the parsing and calendar helpers behind them are private; and `runtime_version_display` is `pub(crate)`, re-exported by `therock.rs` for the other modules that render versions. Depends on nothing else in `therock`. `therock_version_proptests.rs` stays declared from `therock.rs` because it also reaches update, wheel and registry code. -- [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`) and the stage-then-publish atomic file writes (`write_file_atomically`, `stage_file_for_atomic_publish`, `temp_sibling_path`, and the platform `publish_temp_file`/`replace_file_windows` pair) for the metadata cache entries and the staged metadata public key, the startup update-check record and the active-runtime marker; runtime registry manifests do not go through them. `write_file_atomically` is `pub(crate)` and re-exported by `therock.rs` for [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which writes the active-runtime marker with it; the rest is `pub(super)` or private. The `cfg(windows)` publish and replace paths compile only on the Windows lane. -- [`therock/process.rs`](../apps/rocm/src/therock/process.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for scratch directories (split out of `therock.rs`): running and capturing child commands (`run_command*`, `capture_command_output*`, `capture_python_stdout*`, `run_uv_progress_command`, and the Windows temp-file variants with `windows_child_path`), the bounded wait (`wait_with_output_bounded`), the progress-line output with its `--json` suppression guard (`SuppressProgressOutput`, `progress_line`), and unpacking the SDK tarball (`extract_tarball`, `extract_tarball_and_discard_archive`, which is why it sits here and not in `io.rs`: it spawns `tar` through `run_command`). Its `archive_properties` property tests run on Unix only. The process-wide environment lock these tests take is the single `PROCESS_ENV_TEST_LOCK` in [`test_support.rs`](../apps/rocm/src/test_support.rs), shared with the tests left in `therock.rs`. -- [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for atomic writes and on `slugify` in the parent for cache file names (split out of `therock.rs`): HTTP fetching for installs. `download_text_cached` is the metadata GET with its on-disk etag cache (`CachedHttp*`) and detached-signature verification (`MetadataSignaturePolicy`, `PINNED_METADATA_PUBLIC_KEY_PEM`, the unset trust root that `cargo xtask verify-pinned-keys` checks against the canonical published key, so its path is pinned in `xtask`); `download_file` is the streaming artifact download; `head_content_length` and `preflight_tarball_space` size the download and check free space first; `http_get` is the shared request. `truthy_env` is `pub(crate)` and re-exported by `therock.rs` for [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs); `download_text_cached`, `download_file`, `preflight_tarball_space` and the `CachedHttpText` struct (with its `text` field) that `download_text_cached` returns are `pub(super)` for the install and resolution code that stays in `therock.rs`; everything else is private. -- [`therock/source.rs`](../apps/rocm/src/therock/source.rs) — full domain extraction, a child of `therock.rs` that depends only on `therock/version.rs` (split out of `therock.rs`): where an install reads from. `TheRockChannel` (release or nightly) and `SourceLayout` (the canonical stream, or ROCm 10's separately hosted next layout) are the two axes; `resolve_source` maps a pair to the `ResolvedSource` wheel-index and tarball-catalog bases, honouring a `ROCM_CLI_THEROCK_*_BASE` override only behind the explicit opt-in (`therock_base_override_allowed`, `select_base_override`, `env_override_base`); `select_source_layout`, `requested_source_layout` and `next_layout_requested` decide the layout from a pin; `RuntimeVersionSelector` is the version or build date a user pins (`pub(crate)`, re-exported by `therock.rs`). `FamilyResolution` lives here rather than in `therock/family.rs` because choosing the layout reads it; that keeps the dependency one-way, with `family.rs` building on this file. `TheRockChannel`, `SourceLayout`, `ResolvedSource` and `FamilyResolution` (with their methods and fields), `RuntimeVersionSelector`'s `describe` and `matches_version`, `resolve_source`, the three layout functions, and the `THEROCK_SOURCE_LAYOUT_GENERATION` and `THEROCK_NEXT_MIN_MAJOR` constants are `pub(super)` for `therock.rs` and its siblings; the base-override helpers and the other constants (the per-channel and next-layout base URLs, and the next layout's generation tag) are private to this file. -- [`therock/family.rs`](../apps/rocm/src/therock/family.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` for `TheRockChannel` and `FamilyResolution` (split out of `therock.rs`): which package family a request targets. `resolve_family` picks it from `--family`, `ROCM_CLI_THEROCK_FAMILY`, a managed runtime or the host GPU, in that order, and refuses a GPU with no ROCm build (`unsupported_host_gpu_message`, `unsupported_requested_target_message`) instead of substituting another family; `family_override_or_recovered_arch` and `raw_arch_agreeing_with_family` recover the exact arch behind a grouped family; `family_resolution_hint` is the recovery text appended when no runtime is found. `resolve_family`, `family_override_or_recovered_arch` and `family_resolution_hint` are `pub(super)`; the rest is private. -- [`therock/python.rs`](../apps/rocm/src/therock/python.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/io.rs`, `therock/process.rs` and `therock/source.rs` (split out of `therock.rs`): the Python an install runs on. `python_requirement` maps a `SourceLayout` to the interpreter the layout needs (`PythonRequirement`: wheel tag and `uv` version), which is why `source.rs` had to move first; `resolve_python_launcher` finds one in order of an explicit `ROCM_CLI_PYTHON`, `PATH`, the saved managed Python, then `ensure_managed_python`, which installs one through `uv` and records it in the managed-Python manifest; the resolver reads its environment through `PythonResolverEnv`, so tests steer it with a thread-local override (`OverridePythonResolverEnv`) rather than rewriting `PATH`; `wheel_compatibility_for_python` and `current_platform_wheel_tags` report the wheel tags an interpreter and platform accept (`WheelCompatibility`); `ensure_uv_venv` creates, or recreates when the interpreter tag no longer matches, the runtime's virtual environment. `venv_python_path` stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The items `therock.rs` reaches, and `split_runtime_path`, which `therock/probe.rs` reads, are `pub(super)`; the rest is private, including `ensure_managed_python` and `resolve_python_launcher_in` (the resolver's entry point that takes its environment as an argument). Its Unix-only tests drive fake interpreters and `uv` scripts. -- [`therock/probe.rs`](../apps/rocm/src/therock/probe.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/process.rs` and `therock/python.rs` (split out of `therock.rs`): asking an installed runtime about itself through its own interpreter. `probe_rocm_sdk_runtime` runs `ROCM_SDK_PROBE_SCRIPT` and parses what `rocm_sdk` reports into the `RocmSdkPythonProbe` that [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) owns and validates; `probe_runtime_devices` runs `RUNTIME_DEVICE_PROBE_SCRIPT` to count the devices the runtime's torch can open and whether one can run a kernel (`RuntimeDeviceProbe`); `probe_torch_alignment` reads the installed torch and the one an engine pins from distribution metadata without importing torch (`TorchAlignmentProbe`); and `split_local_version`, `requirement_pinned_version` and `install_pinned_package` are the torch-pin helpers [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) uses to put the SDK's torch back after an engine install. The `pub(crate)` items are re-exported by `therock.rs` for [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs); `probe_rocm_sdk_runtime_for_target`, which `install_wheel_runtime` calls with the device payload it just chose, is `pub(super)`. -- [`therock/resolve.rs`](../apps/rocm/src/therock/resolve.rs) — a child of `therock.rs` that holds no logic (split out of `therock.rs`): only the `mod` declarations for the artifact-resolution modules under `therock/resolve/` and the `pub(super) use` lines through which `therock.rs` reads them. It is the parent the `resolve/` files hang from, so a module there is `pub(in crate::therock)` for the items `therock.rs` reaches and private otherwise. -- [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. -- [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. -- [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. -- [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key`, `wheel_runtime_key` and `has_nontrivial_directory_contents`, which name and inspect the install a plan targets, stay in `therock.rs`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. -- [`therock/test_support.rs`](../apps/rocm/src/therock/test_support.rs) — test-only (`#[cfg(test)]`), a child of `therock.rs`: fixtures shared by the `therock` child modules' tests and the tests left in `therock.rs` (`test_paths`, `test_runtime_manifest`, `write_test_runtime_manifest`, `family_for_test`). Separate from the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs), which holds the fixtures every module's tests share, such as `PROCESS_ENV_TEST_LOCK`. -- [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) — the chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run immediately and which need the user's approval first. It owns `ChatRocmCommandAction` (read-only, or approval with the titles the approval prompt shows) and `ChatToolApprovalRequest`, the validators (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `validate_required_chat_string`, `validate_optional_chat_integer`, `reject_control_characters_in_tool_arguments` (which refuses a model-supplied tool name or argument carrying a terminal control character, for every tool except `rocm_command`, whose argv `normalized_chat_rocm_command_args` screens entry by entry), `validate_chat_rocm_command_safety`, which refuses a public-bind, non-loopback or CPU `serve` and a system install folder (`chat_install_prefix_is_system`)), the classifiers (`chat_rocm_command_action`, `chat_rocm_command_action_from_args`, `chat_install_driver_action`, `canonicalize_chat_rocm_command`, `normalized_chat_rocm_command_args`, `chat_tool_call_is_read_only`), `chat_tool_approval_request`, and the readers that turn a tool call into its argv and one-line command (`rocm_chat_tool_requested_args`, `rocm_chat_tool_requested_command`, with `push_optional_json_cli_arg` and `json_string`). It moved verbatim from `chat.rs` with the tests that exercise only these items. It depends on [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `providers::ChatToolCall`, `service_launch::is_loopback_host`, `services_text::validate_service_id`, `shell_quote::has_terminal_control_character`, `therock::RuntimeVersionSelector` and `driver_install::host_install_driver_only_reports`, none of which imports `chat`, so `freeform_planner.rs` calls `validate_chat_rocm_command_safety` here instead of through `chat`. `chat.rs` calls into it and re-exports `validate_chat_tool_call`, `chat_tool_call_is_read_only` and `chat_tool_approval_request` (read by `dash_seam.rs`), and `ChatRocmCommandAction`, `chat_install_driver_action`, `chat_rocm_command_action_from_args` and `rocm_chat_tool_requested_args`, which only other modules' tests read as `crate::chat::` (`driver_plan.rs`, `system_packages.rs`). -- [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns types: `uninstall()`, plus the installer `PATH` undo and running-executable deletion. The shared plan type and its file-removal helpers live in [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) below instead, because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them too. The two modules depend on each other: `uninstall_plan.rs` reads this module's `pub(crate)` `PathEdit`, `PATH_EDIT_RECORD`, `plan_path_edits`, `plan_is_empty`, `removal_count` and `render_extra_plan`. -- [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) — mechanical relocation, shares crate-visible types (Phase 6f, ROCMAI-212): the uninstall-plan types (`UninstallOptions`, `UninstallPlan`, `UninstallPlanEntry`), building the plan (`build_uninstall_plan`, `plan_installed_binaries`, and the installed-binary/shared-cache discovery helpers behind them), rendering it (`render_uninstall_dry_run`, `render_uninstall_plan`), the `[y/N]` confirmation prompt (`confirm_uninstall`), and the filesystem removal primitive both [`uninstall.rs`](../apps/rocm/src/uninstall.rs) and [`storage.rs`](../apps/rocm/src/storage.rs) call to carry a plan out (`remove_path`, and the entry-resolution helpers behind it). Kept separate from `uninstall.rs` rather than folded into it, since `uninstall.rs`'s own doc comment and name both describe `uninstall()`'s exclusive logic (the installer `PATH` undo and running-executable deletion), while this module's plan/render/removal helpers are shared with `storage.rs`. `main.rs` (`UninstallOptions`), `uninstall.rs`, `storage.rs`, `runtimes_cmd.rs`, `chat.rs`, and [`deletion_properties.rs`](../apps/rocm/src/deletion_properties.rs) (Unix-only property tests) each import the subset of these `pub(crate)` types and helpers they need; see this module's own doc comment for which symbol each one uses. -- [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types (Phase 6f, ROCMAI-210): `rocm update`'s command handler (`update()`, the body of the `update` arm of `dispatch()`) and its rendering/activation logic — the read-only status report (`render_update_text`, `append_update_surfaces`), the predicate deciding whether `update` should route into the mutating path instead of that report (`update_should_preview_or_apply`), applying a runtime update (`apply_runtime_update`, `append_update_activate_summary`), and the update-source/installed-runtime pickers behind it (`select_runtime_update_source`, `select_installed_update_runtime`). `const ROLLBACK_RECOVERY_HINT` stays in `main.rs` as a shared crate-root constant, because callers outside this module also use it ([`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s `RuntimesCommand::Activate` handler and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)'s `render_sdk_install_success`); this module reaches it via its `use crate::{...};` import, the same way `gpu_policy.rs`'s `device_policy_name` stays in `main.rs`. `update()` is `pub(crate)` and called from [`main.rs`](../apps/rocm/src/main.rs)'s `dispatch()` through a re-import (`use crate::update_cmd::update;`), and calls `update_should_preview_or_apply`, `apply_runtime_update` and `render_update_text` itself; `render_update_text` is also `pub(crate)` and read from [`chat.rs`](../apps/rocm/src/chat.rs) (the deterministic `rocm update` tool summary), while `apply_runtime_update` and `update_should_preview_or_apply` keep the `pub(crate)` they had when `dispatch()` called them directly, though only `update()` calls them now; `select_runtime_update_source` is also `pub(crate)`, read from [`therock_version_proptests.rs`](../apps/rocm/src/therock_version_proptests.rs)'s property tests via `crate::update_cmd::select_runtime_update_source`. `append_update_surfaces`, `append_update_activate_summary`, and `select_installed_update_runtime` have no callers outside this module and stay private. The cluster's regression tests live in `update_cmd.rs`'s own `mod tests`, reaching shared fixtures (`test_paths`, `test_runtime_manifest_for_update`, `write_test_pip_runtime`) through `crate::tests::`/`crate::test_support::` the same way other mechanically relocated modules do. - -Further `apps/rocm` modules, not part of the `main.rs` split: - -- [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) — test-only (`#[cfg(test)]`): checks that every `rocm …`/`rocmd …` command the repository advises — in `--help` text, production string literals, the markdown docs and skills, and the VHS tapes — exists and parses, by running it through the parser of the binary it names (`rocm`'s clap `Cli`, or `rocmd`'s `parse_args`). -- [`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): `rocm comfyui install`, including its dry run and the reinstall that replaces ComfyUI's code (through [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs)) while refusing to do so under a running ComfyUI. It downloads and unpacks the source archive, installs ComfyUI's Python dependencies into the selected runtime with uv (filtering out and pinning the runtime's own torch stack, and streaming uv's output to the install log) and runs the AMD GPU check. The manifest, the install log paths and the GPU probe stay in `comfyui.rs`. It imports `runtime_env` and, for the running-ComfyUI guard, `process`; neither imports it. Its unit tests and the Unix-only `archive_properties` property tests for the source unpack live in the module; the fixtures they share with the tests left in `comfyui.rs` stay there. -- [`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): the lifecycle of the ComfyUI process this CLI launches. `start` checks the installed folder, runs the GPU check, launches ComfyUI in the background (a detached child on Linux, a hidden PowerShell launcher on Windows) and waits for its port; `stop` ends the saved process tree. `evaluate_running_state` tells a running, a still-loading and a stopped ComfyUI apart for `status`, and `running_from`, `refuse_while_running` and `running_advice` are the guard that refuses a reinstall while ComfyUI runs from the folder it would replace. The saved state and the types in it stay in `comfyui.rs`. Among the `comfyui` children it imports from [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) and uses [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) to check for an interrupted reinstall; neither imports it, so there is no cycle. -- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. -- [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): replacing ComfyUI's code in an existing `source/` folder on reinstall without touching anything else there. -- [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) — `rocm remote`: driving a remote GPU host from this machine, with SSH as the control channel and the remote's own service published onto the tailnet for inference traffic. Like [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs), it owns its command enum (`RemoteCommand`) and dispatch fn (`run`). Its `pub(crate)` submodules: - - [`remote/bootstrap.rs`](../apps/rocm/src/remote/bootstrap.rs) — deciding whether a remote machine can host a model before asking it to. - - [`remote/doctor.rs`](../apps/rocm/src/remote/doctor.rs) — checking a remote machine's health from here. - - [`remote/install.rs`](../apps/rocm/src/remote/install.rs) — installing ROCm on an unattended remote machine. - - [`remote/provision.rs`](../apps/rocm/src/remote/provision.rs) — putting the ROCm CLI on a remote machine that does not have it. - - [`remote/publish.rs`](../apps/rocm/src/remote/publish.rs) — publishing a remote machine's loopback service onto the tailnet. - - [`remote/session.rs`](../apps/rocm/src/remote/session.rs) — local records of work running on other machines. - - [`remote/tailnet.rs`](../apps/rocm/src/remote/tailnet.rs) — reading the local tailnet: which machines exist, and how to address one. - - [`remote/transport.rs`](../apps/rocm/src/remote/transport.rs) — the SSH control channel to a remote GPU host. +The `rocm` binary: the clap command tree, a handler module for most commands, and the domains behind them (runtimes and TheRock installs, managed services, the chat assistant and freeform planner, ComfyUI, remote hosts). [`main.rs`](../apps/rocm/src/main.rs) is the crate root; the directory modules (`chat/`, `comfyui/`, `driver_plan/`, `providers/`, `remote/`, `runtime_install_finalize/`, `service_launch/`, `therock/`) are rows in the same table, sorted by path. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`main.rs`](../apps/rocm/src/main.rs) | Crate root: `main` and `run`, `dispatch()` over the clap `Command` tree, the clap command builders, freeform-invocation routing, the `version` and bare-`rocm` handlers, and the few items several modules share. | flat | +| [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) | Test-only check that every `rocm ...`/`rocmd ...` command the repository advises (help text, string literals, docs, skills, VHS tapes) exists and parses with the flags it is given. | flat | +| [`automations.rs`](../apps/rocm/src/automations.rs) | `rocm automations` (`automations`) and the status and proposal text it prints, plus the `rocm daemon --status` summary (`render_daemon_text`). | relocation | +| [`background_helper.rs`](../apps/rocm/src/background_helper.rs) | What the CLI can tell about the background automation helper (`rocm daemon`): whether it is running, and the note shown about it after a service stop or in an uninstall plan. | relocation | +| [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) | `rocm bootstrap`: owns `BootstrapCommand` and its dispatch `run`, which opens the dashboard's setup flow in a terminal and otherwise prints how to run it. | domain | +| [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) | The host snapshot behind the internal bridge surface (`build_codex_bridge_snapshot`): examination, config, automation state, managed services, GPUs and the built-in engine list. | relocation | +| [`chat.rs`](../apps/rocm/src/chat.rs) | The chat assistant's entry points (`render_chat_text`, `render_chat_prompt_text`, `read_piped_prompt`), the sampling-flag value parsers, and the re-exports other modules reach `chat/` through. | flat | +| [`chat/intent.rs`](../apps/rocm/src/chat/intent.rs) | Prompt-intent heuristics: decides from the user's latest message alone which tool call the assistant proposes without the local model (`fallback_rocm_tool_call_for_prompt`) and when an install needs approval. | relocation | +| [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) | The tool dispatch behind `rocm mcp-call`, the dash execution seam and the assistant's read-only tools (`run_internal_mcp_call`), and the in-process runner for read-only `rocm` commands. | relocation | +| [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs) | Deterministic plain-language summaries of read-only tool output (`deterministic_chat_tool_summary`), shown instead of asking the local model to restate it. | domain | +| [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs) | The chat turn loop and transcript (`render_chat_prompt_result_with_progress`): the system prompt, the provider call, running the read-only tools a reply requests, and rendering the reply. | relocation | +| [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs) | `rocm chat` (`run_chat_command`): hands off to the dashboard chat in an interactive terminal with no `--prompt`, otherwise prints the one-shot reply or the status screen. | relocation | +| [`chat_host_facts.rs`](../apps/rocm/src/chat_host_facts.rs) | The OS, WSL and GPU facts the binary detects cheaply and renders as a plain-text block for the chat assistant's system prompt. | domain | +| [`cli.rs`](../apps/rocm/src/cli.rs) | The `rocm` clap definition: `Cli`, `Command`, the nested subcommand enums and `SUPPORTED_ENGINES`, and a re-export of the value enums behind flags such as `--provider` and `--device`, which [`cli_args.rs`](../apps/rocm/src/cli_args.rs) defines. | flat | +| [`cli_args.rs`](../apps/rocm/src/cli_args.rs) | The clap value enums behind `--provider`, `--device`, `--framework` and similar arguments (`Provider`, `DevicePolicyArg`, `FrameworkArg`, `InstallFormat`, `WatcherModeArg`, `TelemetryModeArg`, `SandboxToolArg`, `PermissionsModeArg`) and the conversions from a value enum to its domain type. [`cli.rs`](../apps/rocm/src/cli.rs) re-exports every one, so other modules keep reaching them as `crate::cli::Provider`; moved so `cli.rs`, which every new subcommand grows, stays under the module size limit. | relocation | +| [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs) | The CLI audit and lifecycle log (`record_cli_audit_event`): one audit event, mirrored as a line in `logs/cli-lifecycle.log` and in a per-action file under `logs/cli/`. | relocation | +| [`cli_progress.rs`](../apps/rocm/src/cli_progress.rs) | Shared TTY-gated progress indicators on stderr: `Spinner`, advanced by the caller's loop, and `AnimatedSpinner`, which keeps animating between caller updates. | flat | +| [`cli_report.rs`](../apps/rocm/src/cli_report.rs) | Shared CLI output formatters: `ActionReport` (headline plus indented details), `write_marked_row` for marked listings, and `JsonOrText`/`print_json_or_text` for `--json` commands. | flat | +| [`comfyui.rs`](../apps/rocm/src/comfyui.rs) | `rocm comfyui`: the dispatch fn `run` (auditing each action), ComfyUI's types, paths, manifest and state files, the `status` and `logs` text, and the Python-child helpers. | domain | +| [`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs) | `rocm comfyui install`, dry run and reinstall included: fetching and unpacking the source, installing its Python dependencies into the chosen runtime with the torch stack pinned, and the GPU check. | domain | +| [`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs) | Starting, stopping and watching the ComfyUI process (`start`, `stop`, `evaluate_running_state`), and the guard that refuses a reinstall while ComfyUI runs from that folder. | domain | +| [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) | Choosing the ROCm runtime ComfyUI runs against (`select_runtime`) and building the virtualenv, ROCm and library-path environment its process needs. | domain | +| [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) | Replacing ComfyUI's code in an existing `source/` folder on reinstall while keeping the user's models, workflows, outputs and other files there. | domain | +| [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs) | `rocm config` (`config`): applies each subcommand to the saved configuration, prompts for a provider API key, and renders `rocm config show`. | relocation | +| [`dash.rs`](../apps/rocm/src/dash.rs) | `rocm dash` and the other ways into the dashboard TUI (launcher, chat, bench, setup): builds the telemetry daemon options and starts an embedded daemon when none is listening. | domain | +| [`dash_seam.rs`](../apps/rocm/src/dash_seam.rs) | The binary's implementation of the dashboard's `RocmToolExecutor` seam, reusing the CLI's own command functions with plain data in and out. The seam exists because `rocm-dash-tui` (like `rocm-dash-core`) must not depend on `rocm-core`; `rocm-dash-collectors` and `rocm-dash-daemon` do depend on it. | domain | +| [`deletion_properties.rs`](../apps/rocm/src/deletion_properties.rs) | Unix-only property tests for the recursive deletions behind `rocm uninstall` and `rocm storage remove-downloads`, checked against a real sandbox filesystem. | flat | +| [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs) | `rocm diagnose` and `rocm fix` (`diagnose`, `fix`), and the `--report` step that builds a report naming this build's leading version (`prepare_cli_report`), shows it and offers to deliver it. | relocation | +| [`diagnose_model.rs`](../apps/rocm/src/diagnose_model.rs) | The binary's side of `rocm diagnose --model`'s GGUF context estimate: the `--kv-cache-type` and model flags, the `--report` note, and the device-visibility-mask caveat. | domain | +| [`driver_install.rs`](../apps/rocm/src/driver_install.rs) | `rocm install driver` (`install_driver`, `reconcile_driver_install`): runs an approved plan's commands and persists the driver install state. It re-exports `host_install_driver_only_reports` from [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs) for its callers, [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) and [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs). | relocation | +| [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs) | The `rocm install driver` plan: its model, its printed text, and the per-distro builders (apt, dnf, zypper) from the host's examination and os-release. | domain | +| [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) | The WSL2 rocDXG driver plan, which installs the digest-pinned `rocdxg-roct` package or refuses when the download cannot be authenticated. | domain | +| [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) | Per-service inference-endpoint API keys, stored in 0600 files under the services directory, keyed by service id. | relocation | +| [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) | Which runtime an engine installs into and where its environment lives (`engine_manages_own_runtime`, `env_root_for_engine_install`), shared by the engine and SDK install paths. | relocation | +| [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) | Locating an engine's plugin binary, the engine request entry points (answering from the in-process built-in handler, or running the resolved engine binary with `stdio`), and the in-process built-in engine path. `engine_stdio_child_request` hands the child to the stdio client `rocm` shares with `rocmd`, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal. | relocation | +| [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) | Which runtime a `serve` or `engines install` resolves to (`resolve_engine_selection`), the argv and `ROCM_CLI_*` env of a managed child (over [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs)), and HTTP readiness polling of a launched service. | relocation | +| [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) | `rocm engines` (`engines`), the readiness check `serve` runs for self-managed engines, and the environment root of a running service. | relocation | +| [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) | `rocm examine` (`examine`): the human report, or with `--json` the examination plus a summary of what only the CLI knows. | relocation | +| [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) | The `rocm examine` text reports, the bare-`rocm` launch summary, and the engine-inventory listing shared by `rocm engines` and `rocm examine`. | relocation | +| [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) | Answers a freeform request gets without running a command: a single approval-free `examine`, `comfyui status` or `comfyui logs` action. | relocation | +| [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) | Whether a freeform plan's next action may run and the consent flags it needs (`prepare_freeform_execution`), and the header printed before it runs. | relocation | +| [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) | The freeform natural-language planner: turns a prose request into a `StructuredRequestPlan` of `rocm` tool calls, deterministically or with a configured planner provider, and renders it. | domain | +| [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) | `--device`/`--gpu` parsing, GPU selection under the managed-launch lock, per-GPU VRAM readings (`amd-smi`, with a DRM sysfs fallback) and the serve low-VRAM warning. | relocation | +| [`gpu_selection_proptests.rs`](../apps/rocm/src/gpu_selection_proptests.rs) | Property tests for `--gpu` auto-selection and explicit-index validation, declared from [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs). | flat | +| [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) | `rocm install` (`install`) for the `sdk` and `driver` targets, the `--version`/`--build-date` selector, and mapping `install sdk` arguments onto a `therock::SdkInstallRequest`. | relocation | +| [`logging.rs`](../apps/rocm/src/logging.rs) | Client-side structured logging: a file-only, non-blocking `tracing` subscriber for the CLI and dashboard process that never writes to stdout or stderr. | relocation | +| [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) | Managed-service liveness (`managed_service_is_live`, the duplicate-launch guard), loading service records, and the dashboard-sidebar status and counts. Service-record removal and pruning, the deferred key cleanup and the reconciliation to `stopped` here take a recorded process for gone only when `probe_process` confirms it: a process that cannot be queried counts as possibly running. When the deferred cleanup does complete it also forgets the recorded processes and reads the record as stopped, so a later stop cannot signal a recycled PID. An unconfirmed stop of a container-backed engine, decided by the engine `Stop`, is never completed or read as stopped from the supervisor's PID, and `container_stop_unresolved` makes the in-use check treat it as in use. | relocation | +| [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs) | The `rocm model` catalog renderer: the curated list by hardware target, the `--verbose` per-recipe diagnostics, and `format_bytes`. | relocation | +| [`model_fit.rs`](../apps/rocm/src/model_fit.rs) | Whether this host can run a model, for `rocm diagnose --model` (`assess_model_on_this_host`), and the platform gate that rules an engine out (`engine_ruled_out_by_platform`). | relocation | +| [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) | Phrase matching and the install-folder, build-date, version and engine readers for a natural-language request, shared by the chat heuristics and the freeform planner. | flat | +| [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) | Cloud provider (OpenAI, Anthropic) API keys: read from the environment or the OS keyring, stored and removed in the keyring with no plaintext fallback, and their status label. | domain | +| [`providers.rs`](../apps/rocm/src/providers.rs) | Chat model providers: the request/response types, the `ProviderAdapter` dispatch, `LocalProvider` and `RemoteProvider`, local-service selection and the remote transport. | domain | +| [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) | The local provider's hand-rolled HTTP/1.1 transport: a blocking JSON POST and a streamed POST read in chunked, `Content-Length` or close-delimited framing. | domain | +| [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) | Property tests that re-chunk, truncate, stall and interrupt SSE bodies against the local transport's readers. | flat | +| [`providers/tool_schema.rs`](../apps/rocm/src/providers/tool_schema.rs) | The ROCm tool catalog advertised to a model, as OpenAI function definitions and reshaped for Anthropic's `tools` field. | domain | +| [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) | The OpenAI-compatible and Anthropic wire formats: request bodies (with the local-model defaults), non-streaming response parsers, and the SSE emitters. | domain | +| [`providers/wire/sse_emitter_proptests.rs`](../apps/rocm/src/providers/wire/sse_emitter_proptests.rs) | Split-invariance and panic-guard tests for the SSE emitters and line parsers in [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs). | flat | +| [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) | `rocm remote`: owns `RemoteCommand` and its dispatch `run`, driving a remote GPU host over SSH while inference traffic goes to the remote's tailnet-published service. | domain | +| [`remote/bootstrap.rs`](../apps/rocm/src/remote/bootstrap.rs) | Deciding whether a remote machine can host a model (GPU stack, ROCm CLI, Tailscale, known platform), refusing up front when it cannot. | domain | +| [`remote/doctor.rs`](../apps/rocm/src/remote/doctor.rs) | Checking a remote machine's health: facts are gathered on the remote and scored here against the same failure-mode catalog the local command uses. | domain | +| [`remote/install.rs`](../apps/rocm/src/remote/install.rs) | Installing ROCm on an unattended remote machine, behind guards for a human not being present. | domain | +| [`remote/provision.rs`](../apps/rocm/src/remote/provision.rs) | Putting the ROCm CLI on a remote machine that lacks it by running the project's installer there, never by copying this machine's binary. | domain | +| [`remote/publish.rs`](../apps/rocm/src/remote/publish.rs) | Publishing a remote machine's loopback service onto the tailnet through the remote's own Tailscale daemon, so nothing stays running on this end. | domain | +| [`remote/session.rs`](../apps/rocm/src/remote/session.rs) | Local records of the model servers and tailnet publishes running on other machines, so a later `rocm remote status` or `rocm remote stop` can find them. | domain | +| [`remote/tailnet.rs`](../apps/rocm/src/remote/tailnet.rs) | Reading the local tailnet through the local Tailscale daemon only: which machines exist and how to address one. | domain | +| [`remote/transport.rs`](../apps/rocm/src/remote/transport.rs) | The control channel to a remote host: the `Transport` trait (run a command, copy a file) and `SshTransport`, which shells out to the system `ssh` and `scp`. | domain | +| [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) | Argv helpers shared by the chat code, [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) and the freeform planner: option lookups, `ensure_flag`, the CPU-device check, and `format_structured_tool_call`. | flat | +| [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) | Registering runtimes `rocm install sdk` did not install (import, adopt, recovering the `rocm setup` runtime) and the one ready-runtime rule (`select_single_ready_runtime`). | domain | +| [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) | Settling an engine onto a runtime after an SDK install (`finish_sdk_install`, `settle_engine_install`): engine-dependency and device checks and the post-install verdict. | relocation | +| [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) | Whether an engine install realigns the runtime's torch to the engine's pin or keeps it as an intended divergence, running and reporting that, and the manifest readers it uses. | domain | +| [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) | The installed-runtime registry: the manifest model, where manifests live, writing and loading them, selection and activation validation, and the active-runtime marker. | domain | +| [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) | Reconciling running local servers with a runtime being activated or rolled back: classifying each, the report, and moving stale ones with `--restart-services`. | domain | +| [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) | `rocm runtimes uninstall`: planning, revalidating after the prompt, and applying the removal of a registered runtime, including clearing a Lemonade record inside its folder. | domain | +| [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) | `rocm runtimes` (`runtimes`): activation and rollback with the active-runtime marker write, and the runtime list. | relocation | +| [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) | `rocm serve` (`serve`): engine selection (`select_serve_engine`), the AIM pre-flight, and the GPU and device notes folded into the summary. | relocation | +| [`serve_port.rs`](../apps/rocm/src/serve_port.rs) | The `rocm serve` port policy: turns an automatic or explicit `--port` request into one leased port before any manifest, log, key or child exists. | domain | +| [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) | The pure transforms `serve()` applies to an engine's launch recipe hint: the catalog mapping, then `--tool-call-parser`, `--gpu-memory-utilization`, generation defaults and `--engine-env`. | relocation | +| [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) | The engine-agnostic `rocm serve` deployment summary: a status indicator while the server starts, an inference smoke test, and the endpoint and timing table. | domain | +| [`service_control.rs`](../apps/rocm/src/service_control.rs) | Stopping and restarting a managed local server (`rocm services stop` and `restart`, runtime moves) and the internal sandbox-tool runner (`run_internal_sandbox_tool`). Its stop path, `stop_recorded_processes`, hands the recorded PIDs to the routine [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) shares with `rocmd`. | relocation | +| [`service_hints.rs`](../apps/rocm/src/service_hints.rs) | The shared builder of the hint text that tells a user how to stop, restart or remove a service with `--yes`, plus the forced-removal hint (`service_forced_remove_hint`, `rocm services remove --force --yes`) and the command word, verb and past tense for a service tool. | flat | +| [`service_launch.rs`](../apps/rocm/src/service_launch.rs) | Front of the `service_launch/` children: their declarations and the `pub(crate) use` re-exports other modules call, with no code of its own. | relocation | +| [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) | The attached serve path (`--verbose`/`--foreground`): spawn the managed child and stream its log in the terminal until the user detaches or stops it. | relocation | +| [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) | Bind-host validation and the endpoint API key checks that gate a public bind (`validate_bind_host`, `resolve_endpoint_auth`), including on a respawn. | relocation | +| [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) | Starting the background automation helper when a watcher needs it, with the autostart claim that keeps concurrent callers from spawning two. | relocation | +| [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. | relocation | +| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the OOM fault-injection seam for end-to-end tests. | relocation | +| [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). | relocation | +| [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) | The background launch path (`start_managed_service`) and the plain line-by-line rendering of its launch report. | relocation | +| [`service_launch/test_support.rs`](../apps/rocm/src/service_launch/test_support.rs) | Test fixtures shared by more than one `service_launch/` child. | flat | +| [`service_records.rs`](../apps/rocm/src/service_records.rs) | Removing a managed-service record and every file it owns, and the `rocm services prune` plan, render and apply steps with their age gate. `remove_managed_service_record_with` takes `--force` and the read of each recorded PID: `--force` removes a record only when none of its recorded processes is confirmed running, one could not be queried and a stop was requested, and reports which. | relocation | +| [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) | `rocm services` (`services`) for `list`, `logs`, `stop`, `restart`, `remove` and `prune`, and the approval-gated stop and restart path. | relocation | +| [`services_text.rs`](../apps/rocm/src/services_text.rs) | Log rendering and the `rocm services` and `rocm logs` status, browser and `--follow` surfaces, managed-service record reads, and the `stop_server` answer shape. | relocation | +| [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) | `rocm setup` (`setup`): the `setup status` text and `setup reset`, which clears the saved first-time prompt state. | relocation | +| [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) | The CLI's one POSIX shell quoter (`shell_quote`, over `rocm-core`'s) and `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. | flat | +| [`storage.rs`](../apps/rocm/src/storage.rs) | `rocm storage`: a read-only report of what runtimes, downloads and engine installs occupy on disk, and guarded removal of old installs and downloads. | domain | +| [`symptom_input.rs`](../apps/rocm/src/symptom_input.rs) | `rocm diagnose --symptom-file`: turns a file, or standard input (`-`), into the symptom string `--symptom` would have delivered (`resolve`): one trailing line break removed, a UTF-8 byte-order mark dropped, UTF-16 with a byte-order mark decoded, a size cap (`MAX_SYMPTOM_BYTES`), and the hint printed when standard input is a terminal (`TERMINAL_PASTE_HINT`). Its `SymptomFileError` becomes a clap usage error (exit code 2) through `usage_error`, which `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs) hands to `clap_exit_code` before calling `diagnose()` in [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs), so the handler still receives a plain `Option`. Reached by qualified path. | domain | +| [`system_packages.rs`](../apps/rocm/src/system_packages.rs) | Host system-package installs for SDK and engine installs: the distro-aware OpenMPI, libatomic and libnuma plans, the consent decision, and running them without a shell. | domain | +| [`test_support.rs`](../apps/rocm/src/test_support.rs) | Test-only fixtures shared across the crate's module test suites, including the process-environment lock. | flat | +| [`therock.rs`](../apps/rocm/src/therock.rs) | TheRock SDK install orchestration (`install_sdk`, the wheel and tarball paths) and the helpers that name an install and its folders; parent of the `therock/` children. | domain | +| [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) | The consent gate for an install that would displace the active default runtime: what it would replace, and whether to proceed, prompt or refuse. | domain | +| [`therock/family.rs`](../apps/rocm/src/therock/family.rs) | Which TheRock package family a request targets (flag, environment, managed runtime or host GPU), refusing a GPU that has no ROCm build. | domain | +| [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) | HTTP fetching for TheRock installs: the cached metadata GET (its signature check is opt-in until a production key is pinned), the streaming artifact download, and the tarball free-space preflight. | domain | +| [`therock/io.rs`](../apps/rocm/src/therock/io.rs) | Unique scratch directories for TheRock installs. | domain | +| [`therock/kept_torch.rs`](../apps/rocm/src/therock/kept_torch.rs) | What `ROCM_CLI_DISABLE_TORCH_ALIGNMENT` means for the SDK install itself: whether the environment's torch stack is kept, the requirements left out of what `uv` is handed, and the `torch_kept:` line (a real install decides again once the environment is prepared, since a recreated one holds no torch; `therock/wheel_plan.rs` prints the preview's). | domain | +| [`therock/probe.rs`](../apps/rocm/src/therock/probe.rs) | Probing an installed runtime through its own interpreter: what `rocm_sdk` reports, which devices its torch opens, and the torch-pin helpers. | domain | +| [`therock/process.rs`](../apps/rocm/src/therock/process.rs) | Running and capturing child commands for TheRock installs, bounded waits, `--json`-aware progress lines, and unpacking the SDK tarball. | domain | +| [`therock/python.rs`](../apps/rocm/src/therock/python.rs) | The Python an install runs on: finding or provisioning an interpreter (`resolve_python_launcher`, `plan_python_launcher`), narrowing the requirement to an explicit `--python-version` or a runtime's recorded wheel tag, the wheel tags it accepts, and the runtime's virtual environment. | domain | +| [`therock/python_config_props.rs`](../apps/rocm/src/therock/python_config_props.rs) | Property tests for how the managed-Python step, a `config.json` writer, treats a `config.json` it cannot read (it never replaces one with defaults, and creates a missing one), and for the legacy rocm-dash tick migration that could produce one. Declared from [`therock/python.rs`](../apps/rocm/src/therock/python.rs). | flat | +| [`therock/resolve.rs`](../apps/rocm/src/therock/resolve.rs) | Module declarations and re-exports for artifact resolution under `therock/resolve/`; holds no logic. | domain | +| [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) | Resolving the SDK tarball a request installs from the catalog's index (`resolve_tarball_artifact`). | domain | +| [`therock/resolve/uv_access.rs`](../apps/rocm/src/therock/resolve/uv_access.rs) | Who may fetch `uv` while a wheel stack is resolved (`UvAccess`, `plan_version_resolution`), as a pure decision over layout, access and whether `uv` is present. | domain | +| [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) | Resolving the wheel stack (`rocm`, `torch`, `torchvision`, `torchaudio`) and device payload a request installs (`resolve_pip_runtime`, `preview_pip_runtime`); on the next layout it drops a `rocm` build that lacks a framework package (`rocm_versions_with_full_stack`) before pinning torchaudio. | domain | +| [`therock/resolve/torchaudio.rs`](../apps/rocm/src/therock/resolve/torchaudio.rs) | Which torchaudio build a ROCm 10 or newer install pins, and why (`torchaudio_pin`, `TorchaudioPin`); the plan line and install-log note derive from the same pin. | domain | +| [`therock/source.rs`](../apps/rocm/src/therock/source.rs) | Where an install reads from: channel, hosting layout (the next layout's `stable.repo.amd.com`/`nightly.repo.amd.com`, or the earlier stream kept for pins that exist only there) and artifact bases (with the opt-in base override), the one routing rule `layout_serving` and its refusals (`select_source_layout`, `gate_install_layout`), plus the pinned version selector and the resolved family. | domain | +| [`therock/test_support.rs`](../apps/rocm/src/therock/test_support.rs) | Test-only fixtures shared by the `therock/` children's tests and the tests left in [`therock.rs`](../apps/rocm/src/therock.rs). | flat | +| [`therock/update.rs`](../apps/rocm/src/therock/update.rs) | Whether an installed runtime has something newer (`runtime_update_plan`, `runtime_update_check`), the update report and JSON, and the bounded startup check. | domain | +| [`therock/version.rs`](../apps/rocm/src/therock/version.rs) | Version grammar for TheRock runtimes: the strict install gate (`parse_version`), PEP 440 ordering, host-version parsing and the build-date helpers. | domain | +| [`therock/wheel_plan.rs`](../apps/rocm/src/therock/wheel_plan.rs) | The `sdk install` plan a wheel install prints, from one writer for a preview and a real install, resolved or not. | domain | +| [`therock_version_proptests.rs`](../apps/rocm/src/therock_version_proptests.rs) | Property tests for version ordering, update verdicts and update-source selection, declared from [`therock.rs`](../apps/rocm/src/therock.rs). | flat | +| [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) | The chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run at once, which need approval, and the validators that refuse unsafe calls. | flat | +| [`uninstall.rs`](../apps/rocm/src/uninstall.rs) | `rocm uninstall` (`uninstall`): what only uninstall does beyond removing files: undoing the installer's `PATH` change, removing the fish completion script the Linux installer recorded writing (`plan_completion_removals` takes only a regular `completions/rocm.fish` whose first line is the installer's marker naming this install folder, tied to [`install.sh`](../install.sh) by a test; `execute_plan` rechecks the file before removing it and reports a removal failure as a warning instead of stopping, for the `SHELL_COMPLETIONS_KIND` plan entries it adds), and deleting the running `rocm.exe` on Windows. | relocation | +| [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) | The uninstall plan shared with `rocm storage`: building it, rendering it for its audience, the `[y/N]` confirmation, and the `remove_path` primitive. | relocation | +| [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) | `rocm update` (`update`): the read-only status report, whether to route into preview or apply, applying a runtime update, and the update-source pickers. | relocation | + +Boundaries: + +**Module layout** + +- Every module is a private `mod` in [`main.rs`](../apps/rocm/src/main.rs). Full-domain modules are typically reached by qualified path (`comfyui::run`); the dispatch-adjacent relocated clusters are reached unqualified through the `use crate::x::{...}` re-import block near the top of `main.rs`, which lists them. +- `engine_inventory` in [`main.rs`](../apps/rocm/src/main.rs) is the table of built-in engine ids. It, `status_for_readiness`, `device_policy_name` and `ROLLBACK_RECOVERY_HINT` stay at the crate root because modules across the crate use them. +- Most clap enums live in [`cli.rs`](../apps/rocm/src/cli.rs) and handlers in their domain or `*_cmd.rs` module; [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) and [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) are the exceptions that own their enum and dispatch fn. Declaration order in [`cli.rs`](../apps/rocm/src/cli.rs) sets the order of every nested `--help` list. + +**Chat and the freeform planner** + +- [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs) is the top of the `chat/` call graph; `intent`, `mcp_bridge` and `summaries` never call back into it. +- `chat` calls the freeform planner, so [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs), [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs), [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) and [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) must not import `chat`; [`chat_import_contract.rs`](../xtask/src/chat_import_contract.rs) enforces it. +- [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs) stays outside [`chat.rs`](../apps/rocm/src/chat.rs) because it calls both [`chat.rs`](../apps/rocm/src/chat.rs) and [`dash.rs`](../apps/rocm/src/dash.rs), and [`dash.rs`](../apps/rocm/src/dash.rs) already calls [`chat.rs`](../apps/rocm/src/chat.rs). +- A freeform plan is built once and that value is executed, because a cloud planner provider may not be deterministic; `plan_freeform_request_with_context` does the same for the `natural_language_plan` tool. +- `validate_serve_command_safety` in [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) holds the four rules a model-proposed `serve` must pass (no public bind, no CPU device, a loopback host, `--managed` without `--foreground`); the planner calls it and keeps no copy. + +**Runtimes** + +- [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) depends on neither [`therock.rs`](../apps/rocm/src/therock.rs) nor [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), and [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) do not depend on [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which re-exports their names. +- [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) imports [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) directly: going through [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) would lead back to it via [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). +- [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) does not import [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) is its child, not a sibling, because a sibling would form a cycle with the parent's device check. +- `select_single_ready_runtime` in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) is the one ready-runtime rule for the engine paths and ComfyUI; every runtime registration goes through `write_runtime_manifest` in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs). +- The serde shapes of `InstalledRuntimeManifest` and `ActiveRuntimeMarker` are on-disk contracts. +- `PlannedPython` in [`therock/python.rs`](../apps/rocm/src/therock/python.rs) writes its `python_note:` text (`note` for an install preview, `update_note` for an update check) from the same decision the install branches on, so a note cannot name a version the install will not prepare. +- Whether an install root holds a payload is `rocm_core::install_root_has_payload`, and os-release is read only through `rocm_core::hardware_root::real_os_release`; a test in [`system_packages.rs`](../apps/rocm/src/system_packages.rs) fails when a source file of this binary calls `read_os_release(` or `identity_path_for_str` directly. + +**Services and uninstall** + +- `service_launch` does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it (the Ctrl-C stop is passed into `run_attached_service`), and [`services_text.rs`](../apps/rocm/src/services_text.rs) does not call back into [`service_control.rs`](../apps/rocm/src/service_control.rs). +- [`background_helper.rs`](../apps/rocm/src/background_helper.rs) imports nothing from the service modules, so its helper note can be read without importing the spawner. +- [`uninstall.rs`](../apps/rocm/src/uninstall.rs) and [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) depend on each other; the plan half is separate because [`storage.rs`](../apps/rocm/src/storage.rs) shares it. +- [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs)'s `ensure_public_bind_engine_supported` refuses a public bind with the managed Lemonade engine on Windows, because its endpoint API key cannot be enforced there; vLLM enforces its key on every platform, and a loopback bind needs none. + +**Shared sources and contracts** + +- [`service_hints.rs`](../apps/rocm/src/service_hints.rs) is the shared builder of the `rocm services stop|restart|remove --yes` hint text, so the callers that use it cannot drift apart. Those callers are [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_port.rs`](../apps/rocm/src/serve_port.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), [`services_text.rs`](../apps/rocm/src/services_text.rs) and, under `service_launch/`, [`attach.rs`](../apps/rocm/src/service_launch/attach.rs) and [`spawn.rs`](../apps/rocm/src/service_launch/spawn.rs); the parent [`service_launch.rs`](../apps/rocm/src/service_launch.rs) does not call it. +- `DEFAULT_ENGINE_MARKER` in [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) is shared by both engine-inventory renderers and their legend line, so the rendered marker and the legend cannot drift apart. +- [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) is the CLI's quoter; `rocm-dash-tui` keeps a copy (`quote_display_arg`) because it may not depend on `rocm-core`, and a property test here holds the two equal. +- The local-model thinking rule [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) applies lives in `rocm-dash-core`'s [`chat_template.rs`](../crates/rocm-dash-core/src/chat_template.rs), because the dashboard's chat client must decide it identically. +- [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs) compiles in [`SKILL.md`](../skills/rocm-cli-assistant/SKILL.md). +- `cargo xtask verify-pinned-keys` checks the metadata key in [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) by path. +- `ROCDXG_PINNED_DIGESTS` in [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) is the trust anchor of an unattended root install. +- `render_update_json` in [`therock/update.rs`](../apps/rocm/src/therock/update.rs) is the form the dashboard's update check parses. +- Saved property-test failures are keyed by the test's source file, so a moved property test takes its file in `apps/rocm/proptest-regressions/` along (see [Property tests](testing.md#property-tests)). ### `apps/rocmd` — background daemon -Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - -- [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). -- [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal; with no watcher enabled it exits by itself, deciding that under the autostart lock and publishing "stopped" and clearing its spawn claim before releasing it — `record_daemon_stop`, `release_own_autostart_claim`) and `rocmd status` (`print_status`), with the watcher tick interval, which `LoopHooks` makes injectable for tests. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. -- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. -- [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. -- [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). The argument handling the dispatch calls is in [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs). -- [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs) — the argument handling for the MCP tools that spawn `rocm`: `normalized_rocm_command_args` and the read-only allowlist `ensure_rocm_command_is_read_only` behind the `rocm_command` tool, and the argv builders for `install_sdk`, `install_engine`, `launch_server` and `watcher_enable` (`build_install_sdk_args`, `build_install_engine_args`, `build_launch_server_args`, `build_watcher_enable_args`, with the public-bind and system-prefix acknowledgement checks they apply). Functions over the tool's JSON arguments: it uses no other module of the crate, spawns nothing and writes nothing, and [`mcp.rs`](../apps/rocmd/src/mcp.rs) is its only caller. Its own `mod tests` holds the allowlist and argv-builder tests. -- [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. -- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. -- [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). -- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. -- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. -- [`watchers.rs`](../apps/rocmd/src/watchers.rs) — the module root of the built-in watchers: the registry (`reconcile_watcher_snapshots`) and the event collector and dispatcher (`evaluate_watchers`, `collect_automation_events_with_gpu_snapshot`, which takes the GPU snapshot as a closure so tests can inject one, and `evaluate_watchers_for_events`), which route each event to its watcher's handler. `find_recoverable_service` and `stop_requested` are re-exported from here because [`supervise.rs`](../apps/rocmd/src/supervise.rs)'s and [`service.rs`](../apps/rocmd/src/service.rs)'s tests reach them by the `crate::watchers::` path. Each watcher's handler lives in the child below that carries its name, together with the policy they share, and a child reaches only [`policy.rs`](../apps/rocmd/src/watchers/policy.rs) and, for the thermal-protect watcher, [`gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) among the others. - - [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs) — what a watcher does in each mode and the proposals it queues: `WatcherPolicyAction` and `watcher_policy_action` (observe, queue a proposal or run contained, chosen by the watcher mode), `queue_proposal` and `queue_proposal_with_arguments` with the action-to-tool and action-to-arguments tables behind them, and `pending_proposal_exists`, the check the thermal-protect and server-recover handlers use so they do not queue a duplicate of a proposal that is still pending. It uses no other watcher module. - - [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) — the `server-recover` watcher: finding a managed service that needs recovery (`find_recoverable_service`, from a failed or stale manifest, an unreachable endpoint or a failing engine healthcheck, skipping a service whose stop request still stands via `stop_requested`), the recovery event kinds and plain-language reasons, the backoff that spaces recovery attempts (`server_recover_due`), and the handler that restarts the service or queues a restart proposal by watcher mode. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs), [`persistence.rs`](../apps/rocmd/src/persistence.rs) and [`supervise.rs`](../apps/rocmd/src/supervise.rs)'s `restart_managed_service`. - - [`watchers/gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) — the `gpu-metrics` watcher and the GPU readings it works from: the telemetry-event handler (`handle_gpu_metrics_event`, which records telemetry and takes no action), the interval check (`gpu_metrics_due`), the one-line snapshot summary, and the reading of amd-smi monitor entries into thermal-pressure and memory-pressure events (`gpu_pressure_events`, `gpu_pressure_event`) with the thresholds they use. It uses [`persistence.rs`](../apps/rocmd/src/persistence.rs) and no other watcher module, and [`gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) reads its interval and thresholds. - - [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) — the `gpu-thermal-protect` watcher: the pressure-event handler (`handle_gpu_thermal_protect_event`), which records the event and, outside observe mode, picks the one running managed service as the target and queues a reviewed stop request unless one is already pending, and the interval check (`gpu_thermal_protect_due`). It uses [`gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) for the interval and thresholds, [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). - - [`watchers/therock_update.rs`](../apps/rocmd/src/watchers/therock_update.rs) — the `therock-update` watcher: the schedule check (`therock_update_due`, every six hours) and the handler, which in observe mode records a reminder, in propose mode also queues an update-check proposal, and in contained mode runs the read-only `check_updates` sandbox tool and records a notification when an update or repair is available, never applying one. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`sandbox.rs`](../apps/rocmd/src/sandbox.rs), [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). - - [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) — the `driver-upgrade` watcher: the handler for an available driver update, which ignores non-driver components, records the signal in observe mode, queues a reviewed driver-plan proposal in propose mode, and in contained mode runs the restricted read-only `driver_plan` sandbox tool, executing no driver command. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`sandbox.rs`](../apps/rocmd/src/sandbox.rs), [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); its tests also use [`webhook.rs`](../apps/rocmd/src/webhook.rs)'s `local_webhook_event_from_request`. - - [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs) — the `cache-warm` watcher: the handler for a request to warm a recipe artifact. It checks the artifact against the model registry (the lookup is injectable for tests), records an unknown or unverifiable artifact without queueing anything, and otherwise records the request and, outside observe mode, queues a reviewed prefetch proposal; contained mode does too, because an artifact download needs explicit source-policy approval. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); its tests also use [`webhook.rs`](../apps/rocmd/src/webhook.rs)'s `local_webhook_event_from_request`. -- [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. - -[`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations and the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands). +The `rocmd` binary and the daemon library `rocm daemon` embeds: the watcher loop and its built-in watchers, the managed-service supervisor, the MCP stdio server and its sandboxed tools, and the local webhook source. Its command tree lives in [`cli.rs`](../apps/rocmd/src/cli.rs). + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../apps/rocmd/src/lib.rs) | Crate root: module declarations and the entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`, `run_from_args`, `parse_args`). | flat | +| [`main.rs`](../apps/rocmd/src/main.rs) | The `rocmd` binary's entry point: a synchronous `main` that calls `rocmd::run_bin_cli()`, which builds the Tokio runtime itself; it holds no logic of its own. | flat | +| [`anthropic_proxy.rs`](../apps/rocmd/src/anthropic_proxy.rs) | The Anthropic-compatible `/v1/messages` front for a served model: a translation proxy to the engine's OpenAI API, translating streaming requests and, for non-streaming ones, tool use both ways, on a port of its own, published in the engine state file (`spawn_alongside_engine`). | flat | +| [`artifact_prefetch.rs`](../apps/rocmd/src/artifact_prefetch.rs) | The `prefetch_artifact` sandbox tool's download: whether a recipe artifact may be fetched under the approved source policy (answering `source_policy_required`, `cached` or `blocked` without touching the network when it may not), the host-scoped Hugging Face URL checks that gate a token, and the streamed, sha256-verified download that records the cache marker. | flat | +| [`cli.rs`](../apps/rocmd/src/cli.rs) | The `Cli`/`Command` clap definitions, the bridge-snapshot and pretty-JSON printers, and the top-level dispatch (`run_cli`, `run_bin_cli`, `run_from_args`). | flat | +| [`common.rs`](../apps/rocmd/src/common.rs) | Small shared helpers: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`), and argument and JSON-payload utilities such as `payload_string`. | flat | +| [`daemon.rs`](../apps/rocmd/src/daemon.rs) | The foreground `rocmd run` loop (`run_daemon`: evaluate watchers each tick, dispatch webhook events, write runtime state, stop on a signal or when no watcher is enabled) and `rocmd status`. | flat | +| [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs) | `DaemonHost`: which binary a daemon spawn runs, depending on whether the daemon is `rocmd` itself or embedded in `rocm` (`rocmd_binary`, `rocm_binary`). | flat | +| [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) | Talking to an engine and guarding its endpoint key: the stdio request (`engine_request`), the healthcheck and its readiness checks, and the endpoint-key helpers. `engine_request` runs through the same client `rocm` uses, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs): `rocmd` chooses what to run (`__engine-stdio ` on the `rocm` binary that `DaemonHost::rocm_binary` finds, since `__engine-stdio` is `rocm`'s command) and what to ask of the client: its own app paths, the service's endpoint key file when it holds a usable key, and stderr kept for the error rather than forwarded. | flat | +| [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) | The host snapshot behind `rocmd bridge-snapshot` and the `bridge_snapshot` MCP tool (`build_bridge_snapshot`), the time-bounded amd-smi GPU snapshot, and the engine inventory. | flat | +| [`mcp.rs`](../apps/rocmd/src/mcp.rs) | The MCP stdio server: the tool schema table, tool dispatch, and the shaping of each tool's result. | flat | +| [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs) | Argument handling for the MCP tools that spawn `rocm`, spawning and writing nothing itself: the read-only allowlist for `rocm_command` and the argv builders with their acknowledgement checks. | flat | +| [`persistence.rs`](../apps/rocmd/src/persistence.rs) | Automation event and audit-log writes and managed-service registry reads (`record_event`, `load_managed_services`, `load_service_record`). | flat | +| [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) | Running a sandbox tool under bubblewrap or natively, the sandbox-tool dispatch (`run_sandbox_tool`) and the result shaping for `check_updates`, `driver_plan` and `stop_server`. For `prefetch_artifact` it resolves the artifact reference against the model recipe registry and hands the artifact to [`artifact_prefetch.rs`](../apps/rocmd/src/artifact_prefetch.rs). | flat | +| [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) | The sandbox-tool vocabulary: the `SandboxToolArg` enum and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags. | flat | +| [`service.rs`](../apps/rocmd/src/service.rs) | Stopping a managed service: the engine `Stop`, the manifest and endpoint-key writes, and the stop report (`stop_managed_service`, `stop_report_confirmed`). The termination of the recorded processes is the routine shared with `rocm services stop`, in [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs); this module passes it `terminate_recorded_pid` and keeps no copy of it or of its grace constant. | flat | +| [`supervise.rs`](../apps/rocmd/src/supervise.rs) | The `rocmd supervise` entry point (`supervise_service`: launch the engine child, record it, wait for readiness, record the exit) and the recovery respawn (`restart_managed_service`). | flat | +| [`test_support.rs`](../apps/rocmd/src/test_support.rs) | Test-only fixtures shared by the crate's module tests, including the workspace-local scratch `AppPaths` and `RecipeIndexEnv`, the guard that serialises tests over the model recipe registry environment variables. | flat | +| [`watchers.rs`](../apps/rocmd/src/watchers.rs) | Root of the built-in watchers: the registry (`reconcile_watcher_snapshots`) and the event collector and dispatcher (`evaluate_watchers`) that route each event to its watcher's handler. | flat | +| [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs) | The `cache-warm` watcher: checks a requested recipe artifact against the model registry and queues a reviewed prefetch proposal for a known one. | flat | +| [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) | The `driver-upgrade` watcher: turns an available driver update into a reviewed driver-plan proposal, or a read-only plan in contained mode. | flat | +| [`watchers/gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) | The `gpu-metrics` watcher: records GPU telemetry and reads amd-smi monitor entries into thermal- and memory-pressure events, with the interval and thresholds they use. | flat | +| [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) | The `gpu-thermal-protect` watcher: on GPU pressure, picks the one running managed service and queues a reviewed stop request unless one is pending. | flat | +| [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs) | What a watcher does in each mode (`watcher_policy_action`: observe, propose or run contained) and the proposals it queues (`queue_proposal`, `pending_proposal_exists`). | flat | +| [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) | The `server-recover` watcher: finds a managed service that needs recovery (`find_recoverable_service`), spaces attempts with a backoff, and restarts it or queues a restart proposal. | flat | +| [`watchers/therock_update.rs`](../apps/rocmd/src/watchers/therock_update.rs) | The `therock-update` watcher: the six-hourly update check, with a reminder, a proposal or a read-only contained run depending on mode. | flat | +| [`webhook.rs`](../apps/rocmd/src/webhook.rs) | The local webhook source: its axum routes, request validation and watcher-kind allow-list. | flat | + +Boundaries: + +- **Which binary runs what.** The daemon runs as `rocmd` or embedded in `rocm` (`rocm daemon`); `supervise` is `rocmd`'s alone and the hidden engine commands are `rocm`'s alone. A spawn of either takes its binary from [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs)'s `DaemonHost`, the one executable path these choices are made from. +- **The Anthropic proxy is not a daemon task.** [`anthropic_proxy.rs`](../apps/rocmd/src/anthropic_proxy.rs) is started by `rocm`'s `run_builtin_engine_serve_http` in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs), on a thread of the process that owns the engine, so it lives and dies with the service. +- **Shared subprocess capture.** [`common.rs`](../apps/rocmd/src/common.rs) only chooses the child's `ROCM_CLI_*_DIR` environment; the capture and spawn are [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs) in `rocm-core`, shared with `rocm`'s chat tool bridge. +- **Supervised-record writes.** After a supervisor's first write, every write to its record goes through `update_supervised_record`, which is `pub(crate)` in [`supervise.rs`](../apps/rocmd/src/supervise.rs); [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) calls it directly to stamp `last_restart_unix_ms` after a failed restart. +- **Watcher re-exports.** [`watchers.rs`](../apps/rocmd/src/watchers.rs) re-exports `stop_requested` and `find_recoverable_service` from [`server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs). [`supervise.rs`](../apps/rocmd/src/supervise.rs) imports `stop_requested` in production code, and `find_recoverable_service` is called by `watchers.rs` itself and by `supervise.rs`'s tests; so neither re-export is test-only. +- **Deliberate re-export.** [`lib.rs`](../apps/rocmd/src/lib.rs) exports `parse_args` because `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) checks advised `rocmd` commands with it. +- **Layering.** + - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) imports no other module of the crate, so modules that need the sandbox-tool vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). + - [`artifact_prefetch.rs`](../apps/rocmd/src/artifact_prefetch.rs) is called only by [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) and does not import it back; it takes `SandboxToolArg` and `SandboxToolPolicy` from [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs). + - A watcher child imports no sibling watcher module except [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs), and the thermal-protect watcher, which reads [`watchers/gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs)'s interval and thresholds. + - `pending_proposal_exists` in [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs) is the one duplicate-proposal check the thermal-protect and server-recover handlers share. +- **Three watchers stay reviewed or read-only in contained mode.** + - The `therock-update` watcher runs only the read-only `check_updates` sandbox tool and never applies an update. + - The `driver-upgrade` watcher runs only the restricted read-only `driver_plan` tool and executes no driver command. + - The `cache-warm` watcher queues a reviewed proposal even in contained mode, because an artifact download needs explicit source-policy approval. ### `crates/rocm-core` — core library -Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): - -- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `info`, since the default filter is `warn,rocm=info` and a decline on a healthy host is otherwise indistinguishable from "no GPU"), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. -- [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151) automation-daemon runtime state and its on-disk event/proposal/audit records, and the policy applied to each store (which are rotated, and the proposal compaction) — see the module's own doc comment for what it owns. The generic JSONL machinery under those stores lives in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs). The dependency runs one way: this module reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), and [`config.rs`](../crates/rocm-core/src/config.rs) imports nothing from here, because the built-in watcher catalog (whose entries carry a default `WatcherMode`, and which `RocmCliConfig::watcher_enabled` and `effective_watcher_mode` take as an argument) lives in [`config.rs`](../crates/rocm-core/src/config.rs) beside `WatcherMode`. -- [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). -- [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. -- [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs) — (ROCMAI-720) running the `rocm` binary as a captured child: `CommandCapture`, `run_rocm_capture_with_env` and the `command_capture_text` formatter, shared by `rocmd`'s sandbox/MCP clusters and `rocm`'s chat tool bridge. Each caller passes the environment the child should get rather than this module choosing it; the bounded-execution half is [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`. -- [`config.rs`](../crates/rocm-core/src/config.rs) — (ROCMAI-151) the unified `RocmCliConfig` and its nested sections, including the persisted `WatcherMode` enum, the built-in watcher catalog (`BuiltinWatcherSpec`, `builtin_watchers`, `builtin_watcher`) and the rocm-dash dashboard sub-config — see the module's own doc comment for what it owns. -- [`confirm.rs`](../crates/rocm-core/src/confirm.rs) — (issue #769) the shared `[y/N]` prompt primitives: `is_affirmative_answer`, plus `print_prompt` and `read_answer` (each generic over the `Write`/`BufRead` stream so the sequence is unit-testable without a controlled terminal). Deduplicates the print-prompt/read-line/parse sequence that [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs)'s `confirm()`, [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs)'s `confirm_uninstall` and [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs)'s `prompt_yes_no` each used to hand-roll. [`uninstall.rs`](../apps/rocm/src/uninstall.rs), [`storage.rs`](../apps/rocm/src/storage.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) reach the prompt only through `confirm_uninstall`. `print_prompt` and `read_answer` stay separate rather than one bundled call because [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs)'s `confirm()` attempts the read even when the prompt couldn't be written, while the other callers treat a failed write as fatal; [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs)'s `confirm()` also keeps its own `--yes`/non-interactive short-circuit and refusal wording on top of these, since those are specific to the `fix` recipe flow. The module also holds `interactive_terminal`, whether stdin and stdout are both terminals, which callers check before they prompt, and `require_yes_or_interactive`, the `--yes`/terminal guard that [`uninstall.rs`](../apps/rocm/src/uninstall.rs), [`storage.rs`](../apps/rocm/src/storage.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) put in front of that prompt; `interactive_terminal` is re-exported from `lib.rs` as `rocm_core::interactive_terminal` and from `host_gpu.rs` as `rocm_core::host_gpu::interactive_terminal`, the path it had before it moved here. -- [`crypto.rs`](../crates/rocm-core/src/crypto.rs) — (ROCMAI-152, Phase 7c) RSASSA-PKCS#1 v1.5/SHA-256 sign/verify, generic — not model-catalog-specific. -- [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) — `rocm diagnose`: matches an `Examination` and optional symptom text against the closed failure catalog and returns ranked findings. The module root keeps the public types (`Fix`, `Diagnosis`, `Route`, `DiagnoseReport`), the two score thresholds, `diagnose` itself, the two `vllm_*_symptom_is_diagnosable` classifiers the vLLM engine calls (they run the whole catalog, so they cannot live below it) and the re-exports that keep every `rocm_core::diagnose::` path resolving; everything else lives in child modules under `diagnose/`, with `diagnose.rs` kept as the module root so those paths and the links to it stay valid. The platform children do not import each other, and `catalog.rs` is the only module that names every checker. [`catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs) (the `CHECKERS` table that tags each check with the platform families it applies to, in catalog order, plus the platform-family resolution, applicability and the out-of-scope message), [`scoring.rs`](../crates/rocm-core/src/diagnose/scoring.rs) (every keyword table, the scoring helpers over them, and the small helpers that read an `Examination` or build a `Diagnosis`), [`report.rs`](../crates/rocm-core/src/diagnose/report.rs) (where an unmatched report is routed upstream, `upstream_route`, `route_when_no_match` and the tracker table behind them, and `render_report_text`, the human-facing view), [`linux.rs`](../crates/rocm-core/src/diagnose/linux.rs) (the checks that apply on Linux only: kernel and module, device-node access, package repositories and the repo-native install, plus the package-manager helpers and `dkms_driver_plan_is_supported`, which the driver planner calls), [`windows.rs`](../crates/rocm-core/src/diagnose/windows.rs) (the checks that apply on Windows only: the HIP SDK, the Adrenalin driver, the Visual C++ runtime and the display driver date, with the version comparison and scores only they use), [`wsl.rs`](../crates/rocm-core/src/diagnose/wsl.rs) (the checks that apply on WSL2 only: the DXCore handoff, the ROCDXG userspace, the distro floor, the Windows host driver and WSL 1), [`host.rs`](../crates/rocm-core/src/diagnose/host.rs) (the checks on the host's configuration that run on more than one platform: wheel architecture, the HSA override, `PATH`, the wheel's ROCm against the system's, an integrated GPU beside a discrete one, and GPUs ROCm does not support), [`engine.rs`](../crates/rocm-core/src/diagnose/engine.rs) (the checks on the engine runtime that run on Linux and WSL2: the vLLM out-of-memory and import failures, a second copy of the code object manager, and a `/dev/shm` too small for a serving workload, plus the two canonical symptom strings the vLLM engine falls back to) and [`test_support.rs`](../crates/rocm-core/src/diagnose/test_support.rs) (test-only: the `Examination` builders that the tests of more than one `diagnose` module share). -- [`download.rs`](../crates/rocm-core/src/download.rs) — streaming file downloads with resume and retry: `DownloadRequest`/`DownloadOutcome`, `download_file_streaming(_with_progress)` and the simpler `download_file_to_path(_with_progress)`, `Backoff`, and the retry policy (`DOWNLOAD_MAX_ATTEMPTS`). A download streams to a per-process `.part-` sibling, resumes with a `Range` request, and enforces the caller's size limit, expected length and SHA-256 digest before publishing the file. It uses `ureq`, not the loopback client in [`net.rs`](../crates/rocm-core/src/net.rs), and depends only on [`disk_space.rs`](../crates/rocm-core/src/disk_space.rs). The module is private and [`lib.rs`](../crates/rocm-core/src/lib.rs) re-exports its public names. -- [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs) — (ROCMAI-662) the bearer key that guards a managed service's endpoint: the CSPRNG-drawn generator and its header-safety check. Split out of [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs): an id and its endpoint key are made by unrelated mechanisms (readable/timestamp-based vs. CSPRNG) and meet only at the filesystem layer, where the key's sidecar is named after the id — not a reason to keep the two in one module. -- [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) — how far along a managed service's OpenAI-compatible endpoint is: `managed_service_endpoint_readiness` and `EndpointReadiness` (unreachable, listing the model, or serving a real single-token chat request), the lower-level `openai_chat_completion_probe`/`openai_chat_completion_status`/`openai_models_endpoint_has_model`, the shared inference-verified latch the engine adapters call (`engine_state_inference_verified`, `merge_json_state_file`) and the model-reference matching behind the model-listing check. It depends on [`net.rs`](../crates/rocm-core/src/net.rs), [`service_record.rs`](../crates/rocm-core/src/service_record.rs) and [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs); [`net.rs`](../crates/rocm-core/src/net.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) do not depend on it. The module is private and [`lib.rs`](../crates/rocm-core/src/lib.rs) re-exports its public names, so callers keep using `rocm_core::` paths. -- [`examine.rs`](../crates/rocm-core/src/examine.rs) — the host examination probe: the `Examination` document and its types, and `Examination::probe`, which runs the probes in order. The probes live in child modules under `examine/`, with `examine.rs` kept as the module root so the [`examine_driver_proptests.rs`](../crates/rocm-core/src/examine_driver_proptests.rs) `#[path]` attribute and the links to it stay valid: [`linux_gpus.rs`](../crates/rocm-core/src/examine/linux_gpus.rs) (Linux GPU enumeration from `lspci`, `rocminfo` and the kernel topology, with the marketing-name and APU classification they share), [`linux_driver.rs`](../crates/rocm-core/src/examine/linux_driver.rs) (kernel modules, device-node access and Secure Boot), [`rocm_install_probe.rs`](../crates/rocm-core/src/examine/rocm_install_probe.rs) (how ROCm was installed and which repository files configure it), [`library_copies.rs`](../crates/rocm-core/src/examine/library_copies.rs) (the loader library-copy search for comgr and the HIP runtime, and `comgr_matches_runtime`, which `diagnose/engine.rs` reads), [`framework.rs`](../crates/rocm-core/src/examine/framework.rs) (the PyTorch and llama.cpp probes), [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) (the OS and distro, CPU, tracked environment variables, shared memory, container check and kernel log), [`wsl.rs`](../crates/rocm-core/src/examine/wsl.rs) (the WSL probe, including the remote probe that reads a named distribution from the Windows host, and the distro-floor check) and [`windows.rs`](../crates/rocm-core/src/examine/windows.rs) (the native Windows probes: the GPU query, the HIP SDK, the Adrenalin driver and the MSVC redistributable, plus `parse_dotted_version`, which `diagnose/windows.rs` reads). -- [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) — gfx-target identification and the TheRock package-family tables, split out of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) so [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) can read the family mapping without importing the module that probes the host: `extract_first_gfx_token`, `normalize_therock_family`/`known_therock_families`, `gfx_target_has_no_rocm_build`, the AMD PCI-device-id and marketing-name tables behind `gfx_target_from_pnp_device_id`, and the parsers for the Windows display text and the kernel's gfx-target version. The module is private; `lib.rs` re-exports four of the public names (all but `gfx_target_from_pnp_device_id`) and `host_gpu.rs`, as before, re-exports all five. -- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout and the tool-on-`PATH` lookup), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, CPU model and installed RAM), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing`), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). -- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that still read some of the same facts twice (kernel version, CPU model, the Windows GPU query, the marketing-name table, the `PATH` lookup and the subprocess runners). It proposes which of each pair should survive, how the two differ today and the order in which they could converge; it changes no code. It is a snapshot, to be retired when the convergence tickets close. -- [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and in production nothing in this module depends on it; only this module's own tests do, because they drive the engine through the audit-store records (a test-only edge). The module is private; its engine items are `pub(crate)`. -- [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. -- [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. -- [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) — the `rocm diagnose --model` fit assessment: curated-recipe lookup against a host's measured GPU/RAM. `rocm model --verbose` renders its own fit labels in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which uses exactly two items from this module: `curated_alternatives`, called directly, and `recipe_display_ref`, reached through a private one-line wrapper of the same name in that file. -- [`net.rs`](../crates/rocm-core/src/net.rs) — (ROCMAI-154) networking primitives: the default local host and port, URL/endpoint formatting and parsing, and the hand-rolled loopback HTTP/TCP client (`http_get_text*`, `http_post_json_with_auth`, `read_http_response_bounded`). It depends on no other service module: the managed-service readiness probes built on it live in [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) and the streaming downloads in [`download.rs`](../crates/rocm-core/src/download.rs). -- [`paths.rs`](../crates/rocm-core/src/paths.rs) — (ROCMAI-154, Phase 7b) `AppPaths` and its filesystem-location primitives. -- [`placeholder.rs`](../crates/rocm-core/src/placeholder.rs) — the `` placeholder rendered reports print for an empty value: `empty_as_unknown` (returns the value untouched) and `trimmed_or_unknown` (trims first), shared by `apps/rocm`'s driver-install plan and chat tool summaries and by [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs)'s Windows GPU diagnostics, replacing the copy those two `apps/rocm` callers shared in [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and `host_gpu.rs`'s own trimming copy. -- [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) — the host probes' subprocess and file-read helpers, split out of [`examine.rs`](../crates/rocm-core/src/examine.rs) so `fix.rs`, `process.rs` and the child modules of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) reach them without importing the module that owns the `Examination` document: `run`/`run_with_env`/`run_raw` (run a command under a timeout; `rc` is `127` when it cannot be spawned and `124` on timeout), `run_utf16le`/`decode_utf16le` (for `wsl.exe`'s UTF-16LE output), `read_text`/`read_host_text`, `which`, the `SHORT`/`MEDIUM` timeouts and `truncate_to_chars`. All are `pub(crate)` except `run_raw`, which stays private to the module. The module itself is private to the crate and is not re-exported. -- [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus `run_with_timeout`: it drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. It is not yet the *only* such helper — [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded`, [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `run_raw`, and [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs)'s `capture_optional_command_candidate_with_timeout` (plus the Windows-only `capture_diagnostic_command` in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs)) still join their reader threads with no time limit, so a grandchild holding the pipe can outlast their timeout; converging them onto `run_with_timeout` is follow-up work to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). -- [`quoting.rs`](../crates/rocm-core/src/quoting.rs) — (ROCMAI-727) argument-quoting helpers with real execution stakes (POSIX shell, PowerShell), consolidated out of `apps/rocm`-side near-duplicates — see the module's own doc comment for what it owns. `posix_shell_quote` also renders the command lines the CLI shows (through `apps/rocm`'s `shell_quote`); only `comfyui.rs`'s log-line quoter stays separate. -- [`report_rollup.rs`](../crates/rocm-core/src/report_rollup.rs) — (ROCMAI-492) counts read Doctor reports at every level of `Report::group_key`, with refused and unread reports kept apart. Pure, with no transport: the mailbox adapter that feeds it is maintainer tooling in `xtask`, behind `cargo xtask report-rollup`. -- [`report_trend.rs`](../crates/rocm-core/src/report_trend.rs) — (ROCMAI-745) the same reports counted per week, and the weeks in which a known problem rose sharply. Pure like the rollup: it is handed a `Week`, which is all a date reduces to, and holds no time finer than that. -- [`rocm_install.rs`](../crates/rocm-core/src/rocm_install.rs) — (ROCMAI-85, Phase 7a) unmanaged ("legacy") ROCm install discovery — see the module's own doc comment for what it owns. -- [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs) — (ROCMAI-662) a managed service's identifier: the validated `ServiceId` and `generate_service_id`. The bearer key guarding a service's endpoint lives in [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs) instead — see that entry for why. -- [`service_record.rs`](../crates/rocm-core/src/service_record.rs) — (ROCMAI-151) `ManagedServiceRecord`, the on-disk manifest for a managed inference service, and the engine state-file keys and probe retry interval (`INFERENCE_VERIFIED_STATE_KEY`, `INFERENCE_PROBE_ATTEMPTED_STATE_KEY`, `INFERENCE_PROBE_RETRY_INTERVAL`) that tie it to [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) — see the module's own doc comment for what it owns. It reads [`net.rs`](../crates/rocm-core/src/net.rs) only for `format_http_base_url`. -- [`vllm_oom.rs`](../crates/rocm-core/src/vllm_oom.rs) — the vLLM out-of-memory log-tail reader shared by `rocm-engine-vllm`'s post-failure hint and the `rocm serve` summary: segmenting a captured engine log tail, scoring each candidate with [`diagnose/engine.rs`](../crates/rocm-core/src/diagnose/engine.rs)'s own vLLM-OOM checker rule, selecting the user's proximate failing line, and rendering it for display. Also owns the `VLLM_OOM_DETECTED_PREFIX`/`VLLM_OOM_DIAGNOSE_COMMAND` markers `rocm-engine-vllm` and `apps/rocm` must agree on byte for byte. `VLLM_GPU_MEMORY_UTILIZATION_HINT` is *not* here — it predates this module, is consumed more widely, and stays in [`lib.rs`](../crates/rocm-core/src/lib.rs) as noted below. -- [`windows_driver.rs`](../crates/rocm-core/src/windows_driver.rs) — the Windows AMD display driver's version and date, and whether that date predates the first HIP SDK for Windows (an advice-only floor, not a compatibility check); `rocm examine`, `rocm diagnose` and `rocm install driver` all read their verdict from it. - -What else stays in [`lib.rs`](../crates/rocm-core/src/lib.rs), beside module declarations and re-exports, was left unclaimed by every phase-7 cluster and is now decided (ROCMAI-662), so a later cleanup pass places it by domain rather than by where it happens to sit in the file: - -- **Permanent crate-root primitives.** Each is consumed from more than one module or crate, and none belongs to a domain an existing module owns; a dedicated module would add a path segment and nothing else. - - `unix_time_millis` — the crate-wide clock helper, used across this crate's own modules and throughout `rocm` and `rocmd`. - - `require_nonempty` — non-empty field validation, used by this crate's automation, binary-path and model-catalog modules and by all three engine crates. - - `command_failure_detail` — picks the best detail string from a failed subprocess's output, used by [`process.rs`](../crates/rocm-core/src/process.rs), the vLLM engine, and the AIM engine's [`docker.rs`](../engines/aim/src/docker.rs). - - `sanitize_component` — the lowercase-alphanumeric slugger shared by [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs)'s `generate_service_id` and [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs)'s artifact cache-marker names. It is not model-catalog-specific, so it does not move there. - - `TORCH_ALIGNMENT_DISABLED_ENV`/`torch_alignment_disabled` — the one read of the torch-alignment opt-out, shared by the CLI and the vLLM engine so the two cannot disagree. - - `VLLM_GPU_MEMORY_UTILIZATION_HINT` — a single wording contract between the CLI, the vLLM engine, and this crate's own `fix-16-vllm-oom` diagnosis; `rocm-core` is the only crate all of them can reach. -- **`write_atomic`** — not permanent. It is the temp-then-rename writer [`config.rs`](../crates/rocm-core/src/config.rs), [`automation.rs`](../crates/rocm-core/src/automation.rs), [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use for their atomic replaces. The open [#610](https://github.com/ROCm/rocm-cli/pull/610) adds a shared, stronger atomic writer to this crate for `rocm` and `rocmd`, and the open [#541](https://github.com/ROCm/rocm-cli/pull/541) moves `RocmCliConfig::save` onto it. It is `pub`, visible to other crates, because `rocm-engine-lemonade` also writes Lemonade's `backend_versions.json` through it, in [`install.rs`](../engines/lemonade/src/install.rs). Once #610 lands, the automation and service-record callers follow; `write_atomic` is deleted only after that Lemonade caller has moved to the shared writer too (ROCMAI-724). That is a behavior change, not a pure move, so it is not done here. - -Once the `write_atomic` move lands, [`lib.rs`](../crates/rocm-core/src/lib.rs) holds only module declarations, re-exports, the crate-root primitives above, the private `use` shims that let sibling modules reach some `host_gpu`, `managed_runtime`, `rocm_install` and `runtime` items as `crate::` paths, and its test-only modules — the end state of the modularization effort for this crate (ROCMAI-27, planned last: highest fan-in, since every app and engine crate depends on it, but lowest novelty, since the sibling modules already prove the pattern). - -Further `rocm-core` modules, each with its own doc comment except where noted: - -- [`browser.rs`](../crates/rocm-core/src/browser.rs) — handing a URL to the user's browser, behind a trait so the opening can be replaced in tests. -- [`disk_space.rs`](../crates/rocm-core/src/disk_space.rs) — free-space preflight checks and out-of-space error reporting. -- [`fake_executable.rs`](../crates/rocm-core/src/fake_executable.rs) — writing stand-in executables that tests across the workspace then run. -- [`fix.rs`](../crates/rocm-core/src/fix.rs) — `rocm fix`: remediations for diagnosed failure modes. Its `RECIPES` list (in [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs)), together with the checkers under `diagnose/` (their table is [`catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs)), is the authoritative failure catalog. [`fix.rs`](../crates/rocm-core/src/fix.rs) stays the module root, as [`examine.rs`](../crates/rocm-core/src/examine.rs) does for `examine/`, so links to it stay valid, with its children in `fix/`; it keeps the dispatch (`apply`, `Plan`, `plan_of`, `act_on`) and re-exports the names the rest of the crate and the apps use, so `crate::fix::exit`, `rocm_core::fix_exit`, `rocm_core::fix::FixClass` and the `crate::fix::` oracles that the tests under `diagnose/` call keep resolving. The children are private modules: [`fix/options.rs`](../crates/rocm-core/src/fix/options.rs) (the named exit codes, `FixOptions` and `FixClass`); [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs) (the consent decision and `confirm`, the run and query timeouts, `current_os`, and the `fail!` helper, which it re-exports by path for its siblings); [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) (the host-mutating runners, one per auto-applicable fix, with their rc-file, PATH and device-pin helpers; only the four entry points the catalog names are visible to the root, and the `*_windows` variants are ordinary functions that every platform compiles); [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs) (`FixRecipe`, the per-platform class table, `RECIPES`, the lookups by fix-id, and the `cfg(test)` oracles [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) and the e2e crate hold their own copies to); [`fix/render.rs`](../crates/rocm-core/src/fix/render.rs) (`list_recipes`, a recipe's plan and verify lines, `format_flags`, and the refusal for a diagnosis position passed as a fix-id); and [`fix/manifest.rs`](../crates/rocm-core/src/fix/manifest.rs) (the published JSON contract, `CATALOG_CONTRACT_VERSION` and `catalog_manifest`). `render` and `manifest` read `RECIPES` from `catalog`; nothing in `catalog` reads either. -- [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs) — the root the hardware probes resolve host paths through (`host_path`): the real host in a normal build, a simulated host tree for the E2E suite in an `e2e-test-hooks` build. -- [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) — `tail_lines`, the bounded log-tail read shared by the lemonade and vLLM adapters (no module doc comment). -- [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). -- [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. -- [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches. -- [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. -- [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. -- [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names, default config/data/cache directories, and path-text and path-list helpers (no module doc comment). -- [`terminal.rs`](../crates/rocm-core/src/terminal.rs) — one ECMA-48 walk over untrusted terminal output, shared by every caller that has to quote or re-render it (`serve_summary.rs`, `vllm/startup_diagnostics.rs`, `vllm_oom.rs`, `diagnose/engine.rs`). -- [`user_path.rs`](../crates/rocm-core/src/user_path.rs) — removing the install directory from the per-user Windows `Path` that the installer added it to. -- [`uv.rs`](../crates/rocm-core/src/uv.rs) — acquiring the `uv` package manager into the managed cache, plus the argument/environment helpers `apps/rocm` and the vLLM engine share. - -### `crates/rocm-dash-core`, `rocm-dash-collectors`, `rocm-dash-daemon`, `rocm-dash-tui` — dashboard/telemetry - -The `agent/` and `app/` files below are every production file the dashboard-TUI split has touched in those two directories (the `ui/` files it has touched are listed after them); [`app/test_support.rs`](../crates/rocm-dash-tui/src/app/test_support.rs) holds only shared test helpers. - -The old agent.rs was split (mechanical relocation — the shared seam types stay in [`agent/mod.rs`](../crates/rocm-dash-tui/src/agent/mod.rs), reached from the split-out files via `super::`) into: - -- [`agent/mod.rs`](../crates/rocm-dash-tui/src/agent/mod.rs) — the `AgentClient` seam, `AgentError`, `StateSnapshot`, `InferenceParams`, `REQUEST_TIMEOUT`. -- [`agent/snapshot.rs`](../crates/rocm-dash-tui/src/agent/snapshot.rs) — pure JSON telemetry helpers with no `rig` dependency. -- [`agent/tools.rs`](../crates/rocm-dash-tui/src/agent/tools.rs) — the `rig::tool::Tool` wrappers for the four telemetry tools (`gpu_status`, `list_instances`, `bench_summary`, `tokens_per_watt`), their name table `TELEMETRY_TOOL_NAMES`, and the ROCm read/mutating tool dispatch. -- [`agent/clients.rs`](../crates/rocm-dash-tui/src/agent/clients.rs) — the `RigAgentClient`/`ChatGptAgentClient`/`AnthropicAgentClient`/`MockAgentClient` backends. - -[`app/mod.rs`](../crates/rocm-dash-tui/src/app/mod.rs) is modularized (ROCMAI-84, Phase 4 of ROCMAI-27's sequencing) into an `app/` directory. [`app/types.rs`](../crates/rocm-dash-tui/src/app/types.rs), [`app/event_loop.rs`](../crates/rocm-dash-tui/src/app/event_loop.rs), [`app/scrollbar.rs`](../crates/rocm-dash-tui/src/app/scrollbar.rs), and [`app/actions.rs`](../crates/rocm-dash-tui/src/app/actions.rs) follow full domain extraction. [`app/chat.rs`](../crates/rocm-dash-tui/src/app/chat.rs), [`app/slash.rs`](../crates/rocm-dash-tui/src/app/slash.rs), and [`app/summary.rs`](../crates/rocm-dash-tui/src/app/summary.rs) were extracted earlier via mechanical relocation, the same convention the `agent/` split above follows. `quit_confirm`, `tool_approval`, `overlays`, `scroll` and `chat_state` each hold a further `impl AppState` block for one cohesive group of reducer methods: - -- [`app/mod.rs`](../crates/rocm-dash-tui/src/app/mod.rs) — keeps `AppState` and its constructor, the theme picker, row selection, the `apply_event` reducer entry point and the handful of items that stay with it (`ProviderSwitch`, `HISTORY_CAP`, `BENCH_CAP`); the rest of `AppState`'s methods live in the child-module `impl` blocks listed below (`apply_action` itself is a free function in [`app/actions.rs`](../crates/rocm-dash-tui/src/app/actions.rs)) — the modularization effort's "reducer's reason to exist." `crate::app::*` paths for the public surface moved out are unchanged via re-exports (see the re-export block's comment for what counts as "public surface"). -- [`app/types.rs`](../crates/rocm-dash-tui/src/app/types.rs) — shared type/enum defs: `Focus`, `ResolvedArgs`, connection/tab/chat/replay state, `Modal`, `UpdateStatus`, `PaneFocus`, `ScrollTarget`, the slash/plan/approval payload types, and the Esc-menu, command-palette and options-panel navigation data (`MenuItem` and `MENU_ITEMS`, `PALETTE_DESTS`, `OPTIONS_TABS`) that `app/actions.rs` reads, and that [`ui/menu.rs`](../crates/rocm-dash-tui/src/ui/menu.rs) renders; the Esc menu's rows, their labels, the key handling's wrap bound and the height `draw_menu` reserves all come from the one `MENU_ITEMS` list; no `AppState` access. -- [`app/event_loop.rs`](../crates/rocm-dash-tui/src/app/event_loop.rs) — the tick loop that drives the dashboard: `run`, `event_loop`, the frame-draw shutdown gate and the registry-probe seeding. -- [`app/chat_runtime.rs`](../crates/rocm-dash-tui/src/app/chat_runtime.rs) — the live chat backend the tick loop owns, a leaf of `event_loop.rs`: `ChatRuntime` holds the agent the next submit is routed to plus the saved auto-detected local one that `/provider local` restores. `ChatRuntime::start` resolves the backend before the first frame, and `drain_provider_switch`, `drain_endpoint_rebuild` and `drain_dispatch` each consume one of `AppState`'s one-shot edges (`provider_switch`, `chat_endpoint_rebuild`, `chat_dispatch`). The detect and persist edges do not touch the backend and stay in `event_loop.rs`. -- [`app/terminal.rs`](../crates/rocm-dash-tui/src/app/terminal.rs) — terminal restoration and shutdown arbitration, a leaf of `event_loop.rs`: `restore_terminal` and its writer lock, the `SHUTTING_DOWN` latch, the termination-signal watcher (`spawn_termination_watcher`; its `TerminationSignals` has a `cfg(unix)` and a `cfg(windows)` variant) and the Ctrl+C key path (`is_ctrl_c`, `exit_on_ctrl_c`). `app/mod.rs` re-exports the items the rest of the crate and `apps/rocm` use. -- [`app/focus.rs`](../crates/rocm-dash-tui/src/app/focus.rs) — focused-host policy, a leaf of `event_loop.rs`: skipping the daemon client for a focused launch (`should_skip_daemon`), opening the one overlay it hosts (`open_overlay_for_focus`, `apply_startup_focus`) and swallowing the console keys that would kill a running job (`focused_close_key_blocked`). -- [`app/update_check.rs`](../crates/rocm-dash-tui/src/app/update_check.rs) — the background Updates-tile check, a leaf of `event_loop.rs`: the tick-driven `rocm update --json` job (`refresh_update_status`) and the reducer that folds its output into an `UpdateStatus` (`reduce_update_json`). -- [`app/scrollbar.rs`](../crates/rocm-dash-tui/src/app/scrollbar.rs) — mouse/scroll hit-testing: resolving a raw `MouseEvent` against recorded scrollbar tracks, the tab bar, and footer-legend chips into a `KeyAction`. -- [`app/actions.rs`](../crates/rocm-dash-tui/src/app/actions.rs) — `KeyAction` dispatch: translating a key press or resolved mouse hit into a `KeyAction` and applying it to reducer state. -- [`app/quit_confirm.rs`](../crates/rocm-dash-tui/src/app/quit_confirm.rs) — a further `impl AppState` block: the quit-confirm gates (`has_live_instance`, `has_stoppable_managed_instance_with`, the live registry read) and opening, answering and resolving the prompt (`open_quit_confirm`, `on_quit_confirm_key`, `quit_confirm_open_services_key`). The prompt is rendered from `ui/`. -- [`app/tool_approval.rs`](../crates/rocm-dash-tui/src/app/tool_approval.rs) — a further `impl AppState` block: handing a completed plan's action to the approval path (`on_plan_ready`), opening the approval modal (`open_approval`) and routing the verdict (`on_approval_key`, `take_approval`, `on_approval_declined`, `on_approval_result`). Not named `approval`, which would collide with [`ui/approval.rs`](../crates/rocm-dash-tui/src/ui/approval.rs). -- [`app/overlays.rs`](../crates/rocm-dash-tui/src/app/overlays.rs) — a further `impl AppState` block: which manager, console, approval or modal owns the keyboard, `close_overlays`, and the gating predicates the key handlers and renderer consult (`has_open_overlay`, `global_hotkeys_reachable`, `active_overlay_at_root`, `should_pane_back_out`, `focused_should_exit`). -- [`app/scroll.rs`](../crates/rocm-dash-tui/src/app/scroll.rs) — a further `impl AppState` block: the job-console, log-dock, chat and instance-detail scroll offsets (the console, dock and instance-detail clamp through its `ScrollState`) and the scrollbar tracks a mouse grab resolves against. Not named `scrollbar`, which is the mouse hit-testing module above. -- [`app/chat_state.rs`](../crates/rocm-dash-tui/src/app/chat_state.rs) — a further `impl AppState` block: chat consent, the local-endpoint detection offer, submitting a turn and applying the agent's reply or error (the config and agent-factory helpers stay in [`app/chat.rs`](../crates/rocm-dash-tui/src/app/chat.rs)). -- [`app/chat.rs`](../crates/rocm-dash-tui/src/app/chat.rs) — previously extracted from [`app/mod.rs`](../crates/rocm-dash-tui/src/app/mod.rs). -- [`app/slash.rs`](../crates/rocm-dash-tui/src/app/slash.rs) — previously extracted from [`app/mod.rs`](../crates/rocm-dash-tui/src/app/mod.rs). -- [`app/summary.rs`](../crates/rocm-dash-tui/src/app/summary.rs) — previously extracted from [`app/mod.rs`](../crates/rocm-dash-tui/src/app/mod.rs). - -[`crates/rocm-dash-tui/src/ui/approval.rs`](../crates/rocm-dash-tui/src/ui/approval.rs) is the shared component for approval-state prompts — reuse it rather than hand-rolling new approval UI. - -The other shared `ui/` components the contributor rules in [`AGENTS.md`](../AGENTS.md) name — reuse these rather than hand-rolling a new console, box, spinner or status color (the directory holds further helper and per-screen modules that are not inventoried here): - -- [`ui/job_console.rs`](../crates/rocm-dash-tui/src/ui/job_console.rs) — the shared "a job is running" surface: `draw_job_console` renders a job's status header, streamed output and key hints, and `on_console_key` handles its keys. -- [`ui/panel.rs`](../crates/rocm-dash-tui/src/ui/panel.rs) — the shared box chrome: bordered, titled panels (`bento`, `popup`), scrollbars, and the popup geometry and framing (`centered_rect`, `centered_width`, `draw_popup_frame`, `grey_overlay`) that [`ui/approval.rs`](../crates/rocm-dash-tui/src/ui/approval.rs), [`ui/job_console.rs`](../crates/rocm-dash-tui/src/ui/job_console.rs) and the `Modal::*` renderers ([`ui/help_modal.rs`](../crates/rocm-dash-tui/src/ui/help_modal.rs), [`ui/warnings_modal.rs`](../crates/rocm-dash-tui/src/ui/warnings_modal.rs), [`ui/theme_picker.rs`](../crates/rocm-dash-tui/src/ui/theme_picker.rs) and [`ui/menu.rs`](../crates/rocm-dash-tui/src/ui/menu.rs)) import. -- [`ui/exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs) — the helpers every operational screen uses to launch a `rocm` sub-command through the approval gate: `resolve_exe`, the approval-preview quoting (`display_args`, `quote_display_arg`), and the `rocm runtimes adopt` argv. `adopt_args` builds that argv for both the runtime manager's Adopt action and the onboarding wizard's Adopt-existing-folder path, with `--python` set to the Python executable inside the root (`Scripts\python.exe` on Windows, `bin/python` elsewhere, the layout `rocm-core` uses, kept here because the crate may not depend on `rocm-core`). Its unit tests reach both layouts through an injectable `windows` flag. -- [`ui/spinner.rs`](../crates/rocm-dash-tui/src/ui/spinner.rs) — pure helpers for the braille spinner frame (`spinner_frame`) and progress-percentage parsing (`parse_progress_pct`) that `job_console.rs` renders with. -- [`ui/theme.rs`](../crates/rocm-dash-tui/src/ui/theme.rs) — the semantic `Theme`, including the shared status-to-color mapping (`StatusTone` and `tone_color`, with `job_status_color` and `instance_status_color` for the job and instance status enums). -- [`ui/widgets.rs`](../crates/rocm-dash-tui/src/ui/widgets.rs) — small helpers shared across tabs, including `apply_char_key`/`apply_char_key_filtered`, the append-only text-field edit (`Char` appends, `Backspace` pops one char) that every form's key handler calls instead of writing its own `push`/`pop` arms. The Chat tab's input is the exception: its keys go through the `KeyAction` table in [`app/actions.rs`](../crates/rocm-dash-tui/src/app/actions.rs). - -The `ui/` modules the split moved code out of or into, beyond the shared components above: - -- [`ui/quit_confirm.rs`](../crates/rocm-dash-tui/src/ui/quit_confirm.rs) — the quit-confirm prompt's content (title, body listing what is still served, the "press s" stop hint), split out of [`ui/mod.rs`](../crates/rocm-dash-tui/src/ui/mod.rs), which re-exports `quit_confirm_request` for the launcher. The gates that decide whether it opens are in [`app/quit_confirm.rs`](../crates/rocm-dash-tui/src/app/quit_confirm.rs). -- [`ui/header.rs`](../crates/rocm-dash-tui/src/ui/header.rs) — the dashboard header and its warning badge (`draw_header`), split out of `ui/mod.rs`. -- [`ui/footer.rs`](../crates/rocm-dash-tui/src/ui/footer.rs) — the footer legend and its clickable key chips (`draw_footer`), split out of `ui/mod.rs`. -- [`ui/instance_detail.rs`](../crates/rocm-dash-tui/src/ui/instance_detail.rs) — the instance Detail modal (`draw_detail`: summary strip, "recent throughput" panel, scrollable launch-args and env-vars panes, footer), split out of [`ui/tabs/instances.rs`](../crates/rocm-dash-tui/src/ui/tabs/instances.rs). It reads `sorted_instances`, `clamp_sel` and `status_meta` from `instances.rs`, which does not import it back. -- [`ui/modal.rs`](../crates/rocm-dash-tui/src/ui/modal.rs) — now only re-exports, so every item that was public at `ui::modal` before the split still resolves there: the popup geometry from [`ui/panel.rs`](../crates/rocm-dash-tui/src/ui/panel.rs) (`centered_rect`, `centered_width`, `draw_popup_frame`, `grey_overlay`) and the `Modal::*` renderers it used to hold, which are now in the four files below. -- [`ui/menu.rs`](../crates/rocm-dash-tui/src/ui/menu.rs) — the Esc main menu (`draw_menu`), the command palette (`draw_palette`) and the tabbed Options panel (`draw_options`), with the block-letter logo (`draw_logo`) and the settings row (`opt_row`) they compose, split out of [`ui/modal.rs`](../crates/rocm-dash-tui/src/ui/modal.rs). The palette destinations and options tabs it renders live in [`app/types.rs`](../crates/rocm-dash-tui/src/app/types.rs), so this file depends on those and not the reverse. -- [`ui/help_modal.rs`](../crates/rocm-dash-tui/src/ui/help_modal.rs) — the two Help modals, the contextual per-tab one (`draw_help`) and the global two-column keyboard reference (`draw_global_help`), split out of [`ui/modal.rs`](../crates/rocm-dash-tui/src/ui/modal.rs). Both size the popup to their wrapped content, through `popup_height_for`, which [`ui/warnings_modal.rs`](../crates/rocm-dash-tui/src/ui/warnings_modal.rs) also calls. -- [`ui/warnings_modal.rs`](../crates/rocm-dash-tui/src/ui/warnings_modal.rs) — the Warnings modal (`draw_warnings`: the text behind the header warning badge, with each multi-line warning cut to a few lines), split out of [`ui/modal.rs`](../crates/rocm-dash-tui/src/ui/modal.rs). It imports `popup_height_for` from [`ui/help_modal.rs`](../crates/rocm-dash-tui/src/ui/help_modal.rs), which does not import it back. -- [`ui/theme_picker.rs`](../crates/rocm-dash-tui/src/ui/theme_picker.rs) — the theme picker modal (`draw_theme_picker`: a scrollable list of the registered themes with a five-color swatch per entry, beside `draw_theme_preview`, a live preview of the highlighted theme), split out of [`ui/modal.rs`](../crates/rocm-dash-tui/src/ui/modal.rs). It imports the popup geometry from [`ui/panel.rs`](../crates/rocm-dash-tui/src/ui/panel.rs) and does not depend on `modal.rs`. - -The other three dashboard crates are described at crate level here, not module by module: - -- [`rocm-dash-collectors`](../crates/rocm-dash-collectors/src/lib.rs) — the IO-heavy collector implementations behind `rocm-dash-core`'s traits (amd-smi, Docker, host metrics, Lemonade and vLLM scrapers, and on Windows the [`winperf.rs`](../crates/rocm-dash-collectors/src/winperf.rs) perf-counter/ADLX GPU collector) plus the benchmark load generator; used by `rocm-dash-daemon` and, on Windows, by `rocm-dash-tui` in-process. -- [`rocm-dash-core`](../crates/rocm-dash-core/src/lib.rs) — pure types, traits, schemas and the reducer, with no rendering dependencies. -- [`rocm-dash-daemon`](../crates/rocm-dash-daemon/src/lib.rs) — the daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and broadcasts snapshots, and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams them to TUI clients as NDJSON. [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) reads the managed-service records `rocm serve` writes and tells a registry that is confirmed empty (no directory, or no records) from one that could not be read: the run loop retires a managed service that an empty registry lacks, but when a pass cannot read the registry in full (a directory that cannot be listed, or a record file that cannot be opened) it retires nothing: the known services and their data stay, as when Docker discovery errors, the records that did read still add and refresh their services, and the header warns until a read succeeds. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. +The library every first-party app and engine crate depends on. It holds host examination and diagnosis (`examine`, `host_gpu`, `diagnose`, `fix`), managed runtimes and service records, and the process, file, download and quoting primitives the binaries share. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/rocm-core/src/lib.rs) | Crate root: module declarations, the crate-root re-exports, and the small cross-crate primitives listed under Boundaries plus `unix_time_millis` and `require_nonempty`. | flat | +| [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) | Locates the `amd-smi` binary (newest managed SDK runtime first, then legacy locations, then `PATH`) and launches it through `amd_smi_json` after the `/dev/kfd` pre-flight. It reads the managed-runtime registry records from [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs), the data and home directory lookups, `has_usable_amd_gpu`, `host_path` and `is_wsl_host`, and the spawn goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`. | domain | +| [`anthropic_endpoint.rs`](../crates/rocm-core/src/anthropic_endpoint.rs) | The Anthropic-compatible `/v1/messages` surface of a managed service: `AnthropicEndpointMode`, its state-file keys, and the readiness check and wait on the published proxy endpoint. | domain | +| [`archive_props.rs`](../crates/rocm-core/src/archive_props.rs) | Property-test generator and on-disk oracle for the archive extractors: nothing written outside the extraction root, no escaping links, no unsafe mode bits. | flat | +| [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) | `write_file_atomically`: stages the bytes in a unique sibling temp file and publishes it over the destination in one step: durable as well as atomic, since the temp file is flushed before it is published and, on Unix, the directory after it; on Unix it keeps the replaced file's permission bits, and `write_file_atomically` replaces a symlink's target rather than the link. | domain | +| [`automation.rs`](../crates/rocm-core/src/automation.rs) | Automation-daemon runtime state and its JSONL event, audit and proposal stores, with each store's rotation and compaction policy. | domain | +| [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) | Locates the `rocm-engine-` sibling binaries and the `rocm` CLI binary itself. | domain | +| [`browser.rs`](../crates/rocm-core/src/browser.rs) | Hands a URL to the user's browser behind the `Opener` trait, so tests can replace the opening. | flat | +| [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) | The serde DTOs of the `rocmd-codex-bridge-v0` host snapshot, shared so `rocm` and `rocmd` emit the same shape. | domain | +| [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs) | Runs the `rocm` CLI as a child process and captures what it printed (`CommandCapture`, `run_rocm_capture_with_env`, `command_capture_text`). `CommandCapture` is also what [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded` returns for the host probes: the lossy text and `exit_status`, the raw bytes of both streams, a typed `CommandOutcome` (not found, spawn failed, timed out, wait failed, exited with a code, killed by a signal) and the status and error text the executor saw; a capture answered without a child is built with `CommandCapture::from_text`. | domain | +| [`config.rs`](../crates/rocm-core/src/config.rs) | The unified `RocmCliConfig` and its sections, including `WatcherMode`, the built-in watcher catalog and the rocm-dash dashboard sub-config with its legacy-TOML migration. | domain | +| [`config_proptests.rs`](../crates/rocm-core/src/config_proptests.rs) | Property tests for the persisted `config.json` (`RocmCliConfig`) and its one-shot migration from the legacy rocm-dash `config.toml`: save and load round trips, and a save that never writes a file load rejects. Each property counts how often the generator reached the edge classes (absent optionals, empty and non-ASCII strings, non-finite floats) and fails a run that never did. Declared from [`config.rs`](../crates/rocm-core/src/config.rs). | flat | +| [`confirm.rs`](../crates/rocm-core/src/confirm.rs) | The shared `[y/N]` prompt (`print_prompt`, `read_answer`), `interactive_terminal`, and the `require_yes_or_interactive` guard for `--yes`-gated commands. | domain | +| [`crypto.rs`](../crates/rocm-core/src/crypto.rs) | Generic RSASSA-PKCS#1 v1.5 over SHA-256 sign and verify, with no external `openssl` process. | domain | +| [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) | Root of `rocm diagnose`: the public result types, the score thresholds, `diagnose` itself (with `mark_symptom_contribution`, which sets each finding's `symptom_matched` by running the catalog again with no symptom and comparing), and the vLLM symptom classifiers that run the whole catalog. | domain | +| [`diagnose/catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs) | The checker table: which check runs on which platform family and in what order, and how the family is resolved from an `Examination`. | domain | +| [`diagnose/engine.rs`](../crates/rocm-core/src/diagnose/engine.rs) | Engine-runtime checks on Linux and WSL2: vLLM out-of-memory and import failures, a second code object manager, an undersized `/dev/shm`. | domain | +| [`diagnose/host.rs`](../crates/rocm-core/src/diagnose/host.rs) | Host-configuration checks that run on more than one platform: wheel architecture, HSA override, `PATH`, wheel against system ROCm, mixed integrated and discrete GPUs, unsupported GPUs. | domain | +| [`diagnose/linux.rs`](../crates/rocm-core/src/diagnose/linux.rs) | Linux-only checks (kernel and amdgpu module, device-node access, package repositories, repo-native install) and the package-manager command helpers. | domain | +| [`diagnose/report.rs`](../crates/rocm-core/src/diagnose/report.rs) | Upstream routing for a failure nothing recognised (`upstream_route`) and the human-readable report (`render_report_text`), including the per-entry note under a cause the symptom did not contribute to, whose text is the exported `SYMPTOM_DID_NOT_CONTRIBUTE_NOTE` the e2e steps read too. | domain | +| [`diagnose/scoring.rs`](../crates/rocm-core/src/diagnose/scoring.rs) | The keyword tables shared across platforms and the scoring over them, plus helpers that read an `Examination` or build a `Diagnosis`. | domain | +| [`diagnose/test_support.rs`](../crates/rocm-core/src/diagnose/test_support.rs) | Test-only `Examination` builders shared by the tests of several `diagnose` modules. | flat | +| [`diagnose/windows.rs`](../crates/rocm-core/src/diagnose/windows.rs) | Windows-only checks: the HIP SDK, the Adrenalin driver, the Visual C++ runtime and the display driver date. | domain | +| [`diagnose/wsl.rs`](../crates/rocm-core/src/diagnose/wsl.rs) | WSL2-only checks: the DXCore handoff, the ROCDXG userspace, the distro floor, the Windows host driver and WSL 1. | domain | +| [`disk_space.rs`](../crates/rocm-core/src/disk_space.rs) | Free-space preflight checks before large downloads and extractions, and the out-of-space error reporting. | flat | +| [`download.rs`](../crates/rocm-core/src/download.rs) | Streaming file downloads over `ureq` with resume, retry with backoff, and size, length and SHA-256 checks before the file is published. | domain | +| [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs) | Generates the CSPRNG bearer key that guards a managed service's endpoint and checks that it is header-safe. | domain | +| [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) | How far a managed service's OpenAI-compatible endpoint has come (`EndpointReadiness`), the inference-verified latch the engine adapters share, and model-reference matching. | domain | +| [`examine.rs`](../crates/rocm-core/src/examine.rs) | The `Examination` document behind `rocm examine --json` (the field set the doctor skill's `examine.py` emits, plus a few CLI-only additions) and `Examination::probe`, which runs the probes in order; `Examination::probe_with_summary` probes the host once and hands the examination's kernel release to the summary, so `rocm examine --json` reads the kernel release once. | domain | +| [`examine/framework.rs`](../crates/rocm-core/src/examine/framework.rs) | The PyTorch probe (through the active runtime or `PATH`) and the llama.cpp probe. | domain | +| [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) | Host environment probes: OS and distro (with the kernel release read through the summary stack's `detect_kernel_version`), the CPU fields recorded from the shared reader in [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs), tracked environment variables, shared memory, container check and kernel log. | domain | +| [`examine/library_copies.rs`](../crates/rocm-core/src/examine/library_copies.rs) | Finds every copy of the comgr and HIP runtime libraries the loader could pick, and which install owns each. | domain | +| [`examine/linux_driver.rs`](../crates/rocm-core/src/examine/linux_driver.rs) | Linux driver access: kernel modules (loaded or built in), device nodes and who may open them, and Secure Boot. | domain | +| [`examine/linux_gpus.rs`](../crates/rocm-core/src/examine/linux_gpus.rs) | Linux GPU enumeration from `lspci`, `rocminfo`, sysfs and the kernel topology, with the marketing-name and APU classification they share. | domain | +| [`examine/rocm_install_probe.rs`](../crates/rocm-core/src/examine/rocm_install_probe.rs) | How ROCm was installed and which live package-repository files configure it. | domain | +| [`examine/windows.rs`](../crates/rocm-core/src/examine/windows.rs) | Native Windows probes: the GPU query, the HIP SDK, the Adrenalin driver and the MSVC redistributable. | domain | +| [`examine/wsl.rs`](../crates/rocm-core/src/examine/wsl.rs) | The WSL2 guest probe, and the remote probe that reads the same facts from a named distribution when `rocm examine` runs on the Windows host. | domain | +| [`examine_driver_proptests.rs`](../crates/rocm-core/src/examine_driver_proptests.rs) | Property tests for `examine`'s driver-side text readers (WSL distro floor, device-node permissions, modprobe blacklist, `iommu=`) and gfx-family normalisation. | flat | +| [`fake_executable.rs`](../crates/rocm-core/src/fake_executable.rs) | `write_fake_executable`: writes a stand-in script for tests from a short-lived child, so running it does not fail with `ETXTBSY`. It is built into every build, without a `cfg(test)` gate, so dependent crates' tests can call it. | flat | +| [`fix.rs`](../crates/rocm-core/src/fix.rs) | Root of `rocm fix`: the dispatch (`apply`) that plans, confirms and runs a catalog remediation, and the re-exports of its children's public names. | flat | +| [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs) | The closed remediation catalog: `FixRecipe`, the per-platform class table, `RECIPES`, the lookups by fix-id, and the test-only oracles. | flat | +| [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs) | The consent decision and `confirm`, the timeouts the runners use, `current_os`, and output helpers that cannot panic on a closed pipe. | flat | +| [`fix/manifest.rs`](../crates/rocm-core/src/fix/manifest.rs) | The JSON catalog manifest `rocm fix --json` publishes, derived from `RECIPES` on every call. | flat | +| [`fix/options.rs`](../crates/rocm-core/src/fix/options.rs) | The vocabulary of `rocm fix`: the exit codes (`exit`), `FixOptions` and `FixClass`. | flat | +| [`fix/render.rs`](../crates/rocm-core/src/fix/render.rs) | Rendering the catalog for people: the listing, a recipe's plan and verify lines, the flag summary `rocm diagnose` shares, and the refusal of a diagnosis position as a fix-id. | flat | +| [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) | The host-mutating runners, one per auto-applicable fix, each printing the exact change, honouring `--dry-run` and asking consent first. | flat | +| [`foreign_temp_probe.rs`](../crates/rocm-core/src/foreign_temp_probe.rs) | Test probe that a writer publishing through `atomic_write` never touches another writer's temp file; built only for tests and the `test-support` feature. | flat | +| [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) | Pure gfx-target lookups: extracting a gfx token, mapping a target to its TheRock family, the AMD PCI-id and marketing-name tables, and kernel gfx-version decoding. | domain | +| [`gguf.rs`](../crates/rocm-core/src/gguf.rs) | Reads a GGUF model's shape (layers, heads, head width, parameter count, data offset) from its header, never the weights. | domain | +| [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) | Pure arithmetic on whether a GGUF model fits in GPU memory at a context size: weights, KV cache, compute buffer and reserve against VRAM. | domain | +| [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) | Where a GGUF header comes from: a local file, or a Hugging Face file read by HTTP range requests. | domain | +| [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs) | `host_path`, the root the hardware probes resolve host paths through (a simulated tree in `e2e-test-hooks` builds), and the `/etc/os-release` read. | flat | +| [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) | Root of GPU and host hardware detection: the `ExamineSummary` aggregation and the driver-summary and gfx-target fallback chains behind it. | domain | +| [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) | Runs an optional host tool under a short timeout, trying each spelling of its name, and reads any failure as `None`; the run itself is [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded`. | domain | +| [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) | The text `rocm install sdk` prints when it cannot resolve a GPU, including the raw Windows inventory output, whose capture (`capture_diagnostic_command`) also runs on `run_bounded`. | domain | +| [`host_gpu/engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) | Which serve engine a host defaults to: the platform fallback and the vLLM preference for the TheRock families served through it. | domain | +| [`host_gpu/gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) | Which AMD GPUs this process may use: the usable-index probe and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` mask arithmetic. | domain | +| [`host_gpu/linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) | Linux sysfs, KFD and DRM probes: the KFD topology, the DRM ip-discovery gfx target, the amdgpu device and driver version, the primary GPU name. | domain | +| [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) | Kernel version, distro name, installed RAM, and the CPU model and vendor for the host summary. One reader returns the CPU model and vendor together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory. `detect_kernel_version` is now the only `uname -r` reader; [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls it for the examination's `kernel_release`. | domain | +| [`host_gpu/render.rs`](../crates/rocm-core/src/host_gpu/render.rs) | The `rocm examine` text rendering of an `ExamineSummary`. | domain | +| [`host_gpu/windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) | The Windows display-adapter inventory: the PowerShell queries, their parsers, and assembling one inventory from several sources, including the system query whose CPU row the CPU reader in [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) takes. | domain | +| [`host_gpu/wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) | WSL detection and the Windows host driver as seen from the guest: the WSL2 signals, the librocdxg chain and the host-driver probes. | domain | +| [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) | The generic append-only JSONL store engine: sibling `.lock` file, single-line appends, torn-tail repair, size rotation and tail-first reads. | domain | +| [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) | The one routine that reads the last lines of a log file (`tail_lines_decoded`, `tail_log_decoded`): seek-bounded, lossy, with plain or console decoding. | flat | +| [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) | The TheRock managed-runtime environment: `ManagedRuntimeEnvironment`, the active-runtime lookups, `prepend_runtime_paths`, and the registry and SDK probing behind them. | domain | +| [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) | The curated model catalog: recipe types, the built-in catalog JSON, signed external recipe indexes and artifact cache markers. | domain | +| [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) | Whether a model will run on this host, from its curated recipe and the host's measured GPU and RAM, without fetching weights (`rocm diagnose --model`). | domain | +| [`net.rs`](../crates/rocm-core/src/net.rs) | The default local host and port, URL formatting and parsing, and the hand-rolled loopback HTTP/TCP client the readiness probes use. | domain | +| [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) | OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans and hints vLLM's ROCm wheels need. | flat | +| [`os_release.rs`](../crates/rocm-core/src/os_release.rs) | The one `/etc/os-release` parser, reading the file as a POSIX shell sourcing it would. | flat | +| [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) | Distro-generic package install planning: `PackageManager` resolution, the plan types, the root and passwordless-`sudo` checks, and the `apt-get -s` safety helpers. It never executes a plan; the one process it starts is `sudo -n true`, behind `can_autoinstall`. | flat | +| [`paths.rs`](../crates/rocm-core/src/paths.rs) | `AppPaths` (config, data and cache roots and their per-engine and per-service subpaths), the managed-root override and engine plugin directories, and `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, with the constants that name them (`CONFIG_DIR_ENV`, `DATA_DIR_ENV`, `CACHE_DIR_ENV`, `ENGINE_ENVS_ROOT_ENV`). | domain | +| [`persisted_paths_props.rs`](../crates/rocm-core/src/persisted_paths_props.rs) | Property test that `RocmCliConfig::load` and `AppPaths::discover` agree about where the data dir is: a `config.json` that cannot be read makes discovery fail or still find the recorded managed root, never silently fall back to the default data dir. Declared from [`paths.rs`](../crates/rocm-core/src/paths.rs). | flat | +| [`placeholder.rs`](../crates/rocm-core/src/placeholder.rs) | The `` placeholder rendered reports print for an empty value (`empty_as_unknown`, `trimmed_or_unknown`). | domain | +| [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) | Subprocess and file-read helpers for the host probes: `run` under a timeout (`run`, `run_with_env` and `run_raw` are thin adapters over [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded`; `rc` is `127` when the program cannot be spawned, `124` on a timeout or a failure to wait and `-1` for a child killed by a signal), `which`, `read_text` and `read_host_text`. | domain | +| [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) | Verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call. `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID; each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID, and an unconfirmed stop keeps them. `terminate_recorded_pid` terminates one of them. On Windows it first runs `taskkill /T /F` on its descendants, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted). It holds the PID open on Windows from before that check until the stop ends, so the PID cannot be reused in between (a PID that matches but cannot be held is reported unverified and not signalled). A tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), is an unconfirmed stop: `descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline. A `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is `exited_before_tree_kill`, which the record-wide settling in `terminate_recorded_service_pids` turns into a stop only when another recorded process (the engine, which is visited as its own role and folded by both callers) is confirmed gone. The `taskkill` starts from the system directory (`GetSystemDirectoryW`, not the `SystemRoot` variable) through `windows_system_tool`. A PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone). `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. | flat | +| [`process.rs`](../crates/rocm-core/src/process.rs) | Process spawn, run and terminate primitives: hidden-console and detached spawn, process-tree termination, `FileLock`, the re-export of the bounded subprocess executor's `run_with_timeout` and `run_bounded`, and `ProcessProbe`/`probe_process`, which tell a PID that is gone from one that cannot be queried (`may_be_running`), and `windows_system_tool`, which resolves a Windows tool from the system directory. | domain | +| [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs) | The bounded subprocess executor behind `run_with_timeout` and `run_bounded`, a private child of [`process.rs`](../crates/rocm-core/src/process.rs) that re-exports it: a thread per pipe drains both streams from the moment the child starts, a reader that cannot finish is abandoned after a 200 ms grace rather than joined, and the error for a timeout quotes only a bounded head of the child's output. `run_bounded` returns the run as a `CommandCapture` instead of an `Err`. | domain | +| [`process_tree_properties.rs`](../crates/rocm-core/src/process_tree_properties.rs) | Property tests for the process-tree traversal that decides which PIDs a `KillScope::Tree` stop signals. | flat | +| [`quoting.rs`](../crates/rocm-core/src/quoting.rs) | Quoting whose output must read back as exactly the value given: POSIX shell (`posix_shell_quote`) and PowerShell (`powershell_quote`). | domain | +| [`report.rs`](../crates/rocm-core/src/report.rs) | The content of a Doctor report and the rule that decides whether one may be built at all, with no network or filesystem access. | flat | +| [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) | How a Doctor report leaves the machine: building its link and deciding whether a browser may be started, never sending it. | flat | +| [`report_rollup.rs`](../crates/rocm-core/src/report_rollup.rs) | Counts Doctor reports at every level of `Report::group_key`, keeping refused and unread reports apart. | domain | +| [`report_trend.rs`](../crates/rocm-core/src/report_trend.rs) | Weekly counts of Doctor reports, and the weeks in which a known problem rose sharply. | domain | +| [`rocm_install.rs`](../crates/rocm-core/src/rocm_install.rs) | Discovery of unmanaged ("legacy") ROCm installs: `LegacyRocmSummary`, the install-layout search and version ranking. | domain | +| [`runtime.rs`](../crates/rocm-core/src/runtime.rs) | The host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names, default directories, path helpers and `install_root_has_payload`. | flat | +| [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs) | `ServiceId`, the validator every per-service id should pass before it reaches a path builder, and `generate_service_id` for a new launch. | domain | +| [`service_record.rs`](../crates/rocm-core/src/service_record.rs) | `ManagedServiceRecord`, the on-disk manifest of a managed inference service, and the engine state-file keys that tie it to readiness probing. | domain | +| [`sigpipe.rs`](../crates/rocm-core/src/sigpipe.rs) | `with_sigpipe_ignored`, which turns a write to a pipe whose reader has gone into an ordinary error instead of a signal; `rocm` runs with SIGPIPE at its default action, so the engine stdio client wraps its write to the child, and each write forwarding the child's stderr to its own, in it. Guarded regions may overlap, across threads too; the disposition in force before the first is put back when the last ends. | domain | +| [`terminal.rs`](../crates/rocm-core/src/terminal.rs) | One ECMA-48 walk over untrusted terminal output, and the stripping, line-splitting and quotability decisions built on it. | flat | +| [`test_env.rs`](../crates/rocm-core/src/test_env.rs) | `RestoredEnvVar`: restores a process environment variable on drop, so a panicking test cannot leak it. | flat | +| [`test_support.rs`](../crates/rocm-core/src/test_support.rs) | Test fixtures for throwaway per-test `AppPaths` directories and for seeding the automation audit store. | flat | +| [`user_path.rs`](../crates/rocm-core/src/user_path.rs) | Removes the install directory from the per-user Windows `Path` that [`install.ps1`](../install.ps1) added it to. | flat | +| [`uv.rs`](../crates/rocm-core/src/uv.rs) | Downloads a standalone `uv` into the managed cache, plus the `uv` argument and environment helpers `apps/rocm` and the vLLM engine share. | flat | +| [`vllm_oom.rs`](../crates/rocm-core/src/vllm_oom.rs) | Picks and renders the user's failing vLLM out-of-memory line from a captured log tail, and owns the OOM marker strings. | domain | +| [`windows_driver.rs`](../crates/rocm-core/src/windows_driver.rs) | The Windows AMD display driver's version and date, and the one comparison that judges it old (advice only, not a compatibility check). | domain | + +Boundaries: + +- **Crate-root primitives.** The helpers in [`lib.rs`](../crates/rocm-core/src/lib.rs) stay at the crate root on purpose: each is used from more than one module or crate, and no module's domain owns it. Single sources among them: + - `TORCH_ALIGNMENT_DISABLED_ENV` and `torch_alignment_disabled`: the one read of the torch-alignment opt-out, so its readers cannot disagree. `SDK_TORCH_STACK_PACKAGES` lists the packages the opt-out spares, and both the CLI and the vLLM engine read it. The Lemonade engine does not: its backend alignment has an opt-out of its own, `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT`, read in [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs). + - `VLLM_GPU_MEMORY_UTILIZATION_HINT`: one wording for the CLI, the vLLM engine and the `fix-16-vllm-oom` diagnosis. + - `command_failure_detail` and `sanitize_component`: the shared failed-subprocess detail rule and the shared slugger for service ids and model cache-marker names. +- **Module fronts and privacy.** + - [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs), [`examine.rs`](../crates/rocm-core/src/examine.rs), [`fix.rs`](../crates/rocm-core/src/fix.rs) and [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) front their directories instead of a `mod.rs`, so existing links and their `rocm_core::` paths keep resolving. + - The private top-level modules are reached through crate-root re-exports so callers keep `rocm_core::` paths; [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) is re-exported through [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) to keep `rocm_core::openmpi::` paths, and [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) is crate-internal. +- **Failure catalog.** + - `RECIPES` in [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs) together with the `diagnose/` checkers is the authoritative failure catalog; the `diagnose` tests and the e2e crate hold their own copies to it through the catalog's test-only oracles. + - The platform children of `diagnose/` do not import each other, and [`diagnose/catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs) is the only module that names every checker. + - In `fix/`, the render and manifest modules read `RECIPES` from the catalog module, which reads neither. +- **Host probing.** + - [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate stacks that read some facts twice (the Windows GPU query, the marketing-name table and the subprocess runners); [`host-probing.md`](host-probing.md) proposes which copy of each should survive. Three pairs are converged: the `PATH` lookup between the two stacks ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is the only copy there; other modules walk `PATH` for their own purposes, for example `find_mpirun_path` in [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) and the executable search in [`runtime.rs`](../crates/rocm-core/src/runtime.rs)), the kernel release ([`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader) and the CPU model ([`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). The first step of the subprocess runners is delivered too: `run_bounded` is the executor behind `run`, `capture_optional_command*` and the Windows diagnostic capture. + - [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) is kept out of `host_gpu` so [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) reads the TheRock family mapping without importing the host probes. + - Hardware probes resolve host paths through [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs)'s `host_path`, which the E2E suite re-roots; [`os_release.rs`](../crates/rocm-core/src/os_release.rs) is the one `/etc/os-release` parser. `hardware_root.rs`'s `real_os_release` is the one reader of the real host's os-release for the callers that build a package-manager plan, and the resolver it passes is private; `read_os_release` itself is `pub` and takes any resolver, which [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) and [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) use with the simulated-root one. Nothing in `rocm-core` stops another caller doing the same; a test in [`system_packages.rs`](../apps/rocm/src/system_packages.rs) keeps `apps/rocm` from naming it. +- **Single sources.** + - [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs)'s `amd_smi_json` is the synchronous launch path for `amd-smi`, and the `/dev/kfd` pre-flight runs there; the dashboard collectors run their own pre-flight through the same bounded `rocm_core::device_readable_bounded` probe, over a `rocm-dash-collectors` to `rocm-core` edge that [`crate_edges.rs`](../xtask/src/crate_edges.rs) allows; [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) fails any unlisted caller of `resolve_amd_smi_binary`. + - [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) is the only log-tail reader behind every log view, and lives here because `rocmd` cannot depend on `rocm`. + - A new atomic file write goes through [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs); a few older temp-then-rename writers in `apps/rocm` predate it. + - [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s `runtime_python_env_root_for_executable` and `install_root_has_payload` are the one definitions of those rules. + - [`vllm_oom.rs`](../crates/rocm-core/src/vllm_oom.rs)'s `VLLM_OOM_DETECTED_PREFIX` and `VLLM_OOM_DIAGNOSE_COMMAND` must match byte for byte in the vLLM engine and `apps/rocm`. +- **Dependency direction.** [`automation.rs`](../crates/rocm-core/src/automation.rs) reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), which imports nothing back; the built-in watcher catalog sits beside `WatcherMode` in [`config.rs`](../crates/rocm-core/src/config.rs) to keep it that way. +- **Reader threads.** [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_with_timeout` and `run_bounded` abandon a pipe reader that has not finished within a 200 ms grace after the child is gone, so a grandchild holding a pipe cannot keep a call past its deadline by more than that grace. `run`, `run_with_env` and `run_raw` in [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs), the optional capture in [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) and the Windows diagnostic capture in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) all run on it. [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded` is the one runner that still joins its reader threads with no time limit, so a grandchild holding a pipe can outlast its timeout. + +### `crates/rocm-dash-core` — dashboard types and reducer + +The pure layer under the dashboard: telemetry, benchmark and protocol types, the collector traits, the derivations over them and the `State` reducer, with no rendering dependencies and no async types at the type boundary. It must have no first-party dependencies, which [`crate_edges.rs`](../xtask/src/crate_edges.rs) enforces. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/rocm-dash-core/src/lib.rs) | Crate root: module declarations, re-exports of the common types, and `AUTO_PORT_NOTE`, the wording for a serve port that is not yet resolved. | flat | +| [`bench_rollup.rs`](../crates/rocm-dash-core/src/bench_rollup.rs) | Pass^N and Pass@N rollups (`rollup_pass_n`): groups benchmark rows into trial sets and gives each a strict and a lenient verdict plus mean throughput. | flat | +| [`bench_schema.rs`](../crates/rocm-dash-core/src/bench_schema.rs) | `BenchmarkRow`, the one row type for a benchmark result, and `PassFail`, vendored from the instinct-agent-bench schema. | flat | +| [`chat_template.rs`](../crates/rocm-dash-core/src/chat_template.rs) | The chat-template defaults a locally served model needs on an OpenAI-compatible chat request (`local_model_should_disable_thinking`). | flat | +| [`config.rs`](../crates/rocm-dash-core/src/config.rs) | The standalone dashboard configuration in `rocm-dash/config.toml` under the config directory (daemon, TUI and per-engine sections) and the default daemon socket path. | flat | +| [`efficiency.rs`](../crates/rocm-dash-core/src/efficiency.rs) | The pure join of throughput and power: GPU-id normalisation, board power attributed to the instances on a GPU by VRAM share (an occupant that is still `Starting` with no VRAM reading, while another occupant has one, is left out of the split so it does not halve a serving neighbour's power), and tokens per watt. `node_throughput` sums the node's live generation throughput once (a Lemonade endpoint's last-request rate is not live and is left out, as in the per-instance `instance_tokens_per_watt`) and works out its freshness (fresh, held, or unknown), ignoring instances at zero tok/s when judging freshness; the Observe and Home throughput heroes call it directly. `node_power_w` sums board power over the GPUs that report a usable reading, skipping one that is not finite or not positive as `attribute_power_w` does, and is what the Observe, Hardware and Home power figures show and divide by. `node_efficiency` combines the two into the whole-node `NodeEfficiency` (Σ live `gen_tps` over that power, including GPUs no instance occupies) and is the one implementation behind the Observe efficiency hero and its trend, the Hardware header, and the assistant's `tokens_per_watt` and `list_instances` tools, which reach `node_throughput` through it (the Home tab's tokens-per-watt figure is a separate sum of per-instance values). | flat | +| [`metrics.rs`](../crates/rocm-dash-core/src/metrics.rs) | The live metric types the dashboard crates share: `Snapshot`, `GpuMetrics`, `Instance` with its `InstanceStatus` predicates and labels, and the observation-freshness metadata. | flat | +| [`observation.rs`](../crates/rocm-dash-core/src/observation.rs) | Turns a generation-token counter into an honest rate: a bounded sample window, fresh-or-held reporting with expiry, and directly reported rates, with all time passed in. | flat | +| [`partition.rs`](../crates/rocm-dash-core/src/partition.rs) | The GPU compute-partition and memory-partition mode enums. | flat | +| [`persist.rs`](../crates/rocm-dash-core/src/persist.rs) | `PersistedEntry`, the on-disk session record: one `Event` plus the microsecond timestamp replay paces by. | flat | +| [`protocol.rs`](../crates/rocm-dash-core/src/protocol.rs) | The NDJSON wire protocol between the daemon and a TUI client: `Command`, `Event` and `PROTOCOL_VERSION`. | flat | +| [`state.rs`](../crates/rocm-dash-core/src/state.rs) | The pure reducer `State::apply(StateEvent) -> Vec`: latest snapshot, bounded histories, instances, benchmark rows and the background-job table. | flat | +| [`traits.rs`](../crates/rocm-dash-core/src/traits.rs) | The collector traits and the data they exchange (`DiscoveredService`, `InstanceSample`, `GpuProcess`, `CollectorError`), plus `merge_instance`, which builds an `Instance` from them. | flat | +| [`vram.rs`](../crates/rocm-dash-core/src/vram.rs) | Pure VRAM attribution: folds per-process readings into a per-container map and resolves an instance's `(used, total)`. | flat | + +Boundaries: + +- **Rules kept here so the front ends cannot drift.** + - [`chat_template.rs`](../crates/rocm-dash-core/src/chat_template.rs) is called by both the CLI's [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) and the TUI's [`agent/clients.rs`](../crates/rocm-dash-tui/src/agent/clients.rs); it lives in this leaf crate because both already depend on it. + - `InstanceStatus`'s predicates and labels in [`metrics.rs`](../crates/rocm-dash-core/src/metrics.rs) and `JobStatus::glyph`, `label` and `legend_glyphs` in [`state.rs`](../crates/rocm-dash-core/src/state.rs) hold the shared predicates and labels for those statuses; their colours are chosen in [`ui/theme.rs`](../crates/rocm-dash-tui/src/ui/theme.rs). + - [`vram.rs`](../crates/rocm-dash-core/src/vram.rs) reuses the GPU-id matching in [`efficiency.rs`](../crates/rocm-dash-core/src/efficiency.rs) instead of its own. +- **Recordings and the wire.** + - `PersistedEntry` is the one recording format: the daemon's [`persist.rs`](../crates/rocm-dash-daemon/src/persist.rs) writes it, [`demo.rs`](../crates/rocm-dash-daemon/src/demo.rs) generates it and the TUI's [`replay.rs`](../crates/rocm-dash-tui/src/replay.rs) reads it back. + - The metric fields added after the first recordings, and every `BenchmarkRow` field, are `#[serde(default)]` so that older recordings and CSVs that omit columns still deserialize. + - `PROTOCOL_VERSION` is bumped when the JSON of an existing variant changes. Internal tagging rejects a newtype variant over a sequence, so every `Event` variant is a struct variant or wraps a struct; a round-trip test holds that. +- **Reducer effects.** Only `SideEffect::SpawnJob` is interpreted, by the TUI's [`jobs.rs`](../crates/rocm-dash-tui/src/jobs.rs); `Persist` and the `Broadcast*` effects are produced but nothing consumes them, and the daemon runner discards the effects it gets back. +- **Configuration.** [`config.rs`](../crates/rocm-dash-core/src/config.rs) is separate from the unified `rocm` configuration (`rocm-core`'s `RocmCliConfig`, whose dashboard section `rocm dash` reads); only the TUI's chat-endpoint save loads and writes this file. Its default socket path copies `rocm-core`'s on purpose, because this crate may have no first-party dependencies. + +### `crates/rocm-dash-collectors` — telemetry collectors + +The I/O-heavy collector implementations behind `rocm-dash-core`'s traits (amd-smi, Docker, host metrics, the Lemonade and vLLM scrapers and the Windows performance-counter collector) and the benchmark load generator. The daemon uses it, and on Windows the TUI runs some of its collectors in-process. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/rocm-dash-collectors/src/lib.rs) | Crate root: declares the public collector modules. | flat | +| [`amd_smi.rs`](../crates/rocm-dash-collectors/src/amd_smi.rs) | The amd-smi collector: runs `amd-smi ... --json` under a timeout after the GPU-device pre-flight and parses metrics, power limits, processes and system info tolerantly. | flat | +| [`ancestry.rs`](../crates/rocm-dash-collectors/src/ancestry.rs) | Attributes GPU processes to managed `rocm serve` services. A service record carries the PIDs of its engine and launcher and their kernel start times; `owner_of` walks a GPU process's `/proc//stat` parent links (bounded depth, reader injected so it is testable anywhere) to a recorded root and returns that service's id. A root whose live start time differs from the recorded one is a recycled PID and claims nothing, PID 0 and PID 1 are never roots (init is an ancestor of every process), and an unreadable link ends the walk with no owner. `owner_of_pid` reads the live `/proc` (Linux only; a stub returns `None` elsewhere). The runner tries the cgroup join first and this second, so container ids and service ids key the one per-container VRAM map. | flat | +| [`bench_load.rs`](../crates/rocm-dash-collectors/src/bench_load.rs) | The concurrency-sweep load generator behind `rocm bench load`: one aggregate `BenchmarkRow` per concurrency cell, appended to a CSV, with an optional auto-ramp. | flat | +| [`bench_tail.rs`](../crates/rocm-dash-collectors/src/bench_tail.rs) | `CsvBenchTailer`: re-reads a benchmark CSV on each drain and yields the rows it has not returned yet. | flat | +| [`cgroup.rs`](../crates/rocm-dash-collectors/src/cgroup.rs) | Attributes GPU processes to Docker containers by reading the 64-hex container id from `/proc//cgroup`; `is_container_id` tests that shape, which a managed `rocm serve` service id never has, so those are attributed by `ancestry.rs`. | flat | +| [`docker.rs`](../crates/rocm-dash-collectors/src/docker.rs) | `DockerDiscovery`: finds vLLM containers through `bollard` and builds a `DiscoveredService` from each container's environment and command line. | flat | +| [`engine_registry.rs`](../crates/rocm-dash-collectors/src/engine_registry.rs) | `EngineKind`: maps a serving engine (vLLM, Lemonade) to its label, conventional port and sample parser. | flat | +| [`host.rs`](../crates/rocm-dash-collectors/src/host.rs) | `HostCollector`: CPU, memory and swap from `sysinfo`, sampled once per tick into `SystemMetrics`. Disk and network rates are not collected yet and stay 0. | flat | +| [`lemonade.rs`](../crates/rocm-dash-collectors/src/lemonade.rs) | The Lemonade Server collector: pure parsers over its stats and health endpoints, and the discovery of a running server's loaded models, one row per model. | flat | +| [`parallel.rs`](../crates/rocm-dash-collectors/src/parallel.rs) | `parallel_scrape`: runs an async scrape against many targets concurrently on a `JoinSet`. | flat | +| [`vllm_prom.rs`](../crates/rocm-dash-collectors/src/vllm_prom.rs) | The vLLM Prometheus `/metrics` scraper: request gauges, KV-cache usage, the generation-token counter and the latency histogram sums. | flat | +| [`winperf.rs`](../crates/rocm-dash-collectors/src/winperf.rs) | Live Windows GPU metrics from OS performance counters, with temperature, power and clocks from ADLX when the AMD driver's DLL is present. | flat | + +Boundaries: + +- **Async collectors.** The core traits are synchronous: the Docker and vLLM implementations of `ServiceDiscovery` and `InstanceMetrics` return `CollectorError::Unsupported`, and the daemon runner calls their async methods (`discover_async`, `fetch_async`) instead. The runner builds its collectors directly, so `EngineKind::parse_sample` and `default_port` are not on the live path. +- **amd-smi pre-flight.** The device pre-flight in [`amd_smi.rs`](../crates/rocm-dash-collectors/src/amd_smi.rs) is mandatory outside tests, because amd-smi can block in uninterruptible kernel sleep on a host with no readable `/dev/kfd`. Its one relaxation is the GPU-reachability verdict `apps/rocm` computes and passes in through the daemon's `RunnerOptions`. +- **vLLM throughput.** [`vllm_prom.rs`](../crates/rocm-dash-collectors/src/vllm_prom.rs) leaves `gen_tps` empty: the daemon runner turns the generation-token counter into a rate. +- **Engine label.** [`bench_load.rs`](../crates/rocm-dash-collectors/src/bench_load.rs) takes the `vllm` label from `EngineKind`, so the CSV `engine` column cannot drift from the registry's spelling. Because `engine` is a rollup key, a cell that mixes blank-engine and vLLM rows is reported through `EngineSplitNotice` for the CLI to print, not printed here. +- **Lemonade discovery follows the dashboard config.** The daemon runs the Lemonade collector only when `RunnerOptions::enable_lemonade` is set. `RunnerOptions::default()` leaves it off, for standalone use and tests; `rocm dash` sets it from `dashboard.daemon.enable_lemonade`, which defaults to true, so on Linux and WSL the daemon probes the configured Lemonade host and port (default `127.0.0.1:13305`) read-only unless the key is `false`. + +### `crates/rocm-dash-daemon` — telemetry daemon + +The daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams snapshots and benchmark rows to TUI clients as NDJSON. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/rocm-dash-daemon/src/lib.rs) | Crate root: module declarations, `DAEMON_VERSION` for the handshake, and re-exports of `RunnerOptions`, `run` and the collectors' `bench_load`. | flat | +| [`bench_ring.rs`](../crates/rocm-dash-daemon/src/bench_ring.rs) | `BenchRing`, the bounded benchmark-row history a late-joining client is sent when it subscribes. | flat | +| [`demo.rs`](../crates/rocm-dash-daemon/src/demo.rs) | The synthetic-session generator behind `rocm dash --demo`: a deterministic recording of an 8-GPU node serving three vLLM containers, written as `PersistedEntry` lines. | flat | +| [`persist.rs`](../crates/rocm-dash-daemon/src/persist.rs) | `SessionWriter`: appends every broadcast `Event` as a `PersistedEntry` line to one `session-*.ndjson` file per daemon session. | flat | +| [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) | Reads the managed-service records `rocm serve` writes and turns the live ones into `DiscoveredService` scrape targets, telling a registry read in full, an empty or absent one included, from one that could not be read in full. It also keeps each live service's recorded engine and supervisor PIDs (with their start times) and derives the ancestry roots `ancestry.rs` matches GPU processes against, dropping the roots of a service it read and found stopped even on a partial pass. It also turns the managed Lemonade records into read-only telemetry targets and decides which of them name the same endpoint (`endpoints_match`, `scrape_host`, `cached_collector_reusable`): `::` covers both loopback families, `::1` only IPv6, and `localhost`, an empty host, `127.0.0.1` and `0.0.0.0` only IPv4, and a cached collector is reused only for the address it dials. | flat | +| [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) | The owner task: drives the collectors on their tick cadences, reconciles discovered services, derives rates and efficiency, and broadcasts `Snapshot`s. | flat | +| [`runner/discovery.rs`](../crates/rocm-dash-daemon/src/runner/discovery.rs) | A private child of [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs), split out to keep it within the module-size limit: what the discovery passes remember between ticks (`TelemetryState`, `DiscoveryState` with the external Lemonade rows and the registry's Lemonade targets, `RegistryHealth`) and the `Reconcile` steps every discovery source shares. | domain | +| [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) | The listener, the per-client task loop and the broadcast from the runner to every client, including the history replay on subscribe. | flat | +| [`snapshot_ring.rs`](../crates/rocm-dash-daemon/src/snapshot_ring.rs) | `SnapshotRing`, the bounded snapshot history a late-joining client is sent when it subscribes. | flat | +| [`transport.rs`](../crates/rocm-dash-daemon/src/transport.rs) | NDJSON framing for the daemon's socket: `write_line` and `read_line`. | flat | + +Boundaries: + +- **A failed registry read retires nothing.** When a pass cannot read the registry in full, the runner retires no service; the records that did read still add and refresh their services, and the header warns until a read succeeds. A registry read in full, an empty one included, retires every known service it no longer holds and keeps the rest (a registry that goes from A and B to A retires B). +- **Framing is copied, not shared.** [`transport.rs`](../crates/rocm-dash-daemon/src/transport.rs) and the TUI's [`transport.rs`](../crates/rocm-dash-tui/src/transport.rs) hold the same two functions because the TUI does not depend on the daemon crate, so a framing change must be made in both. + +### `crates/rocm-dash-tui` — dashboard TUI + +The dashboard client: the event loop and reducer (`app/`), the chat agent (`agent/`), the screens (`ui/`), and the daemon client and replay source at the top level. The `rocm` binary runs it through `app::run`, and the modules are public so examples and tests drive them in-process. + +#### `src/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/rocm-dash-tui/src/lib.rs) | Library root: declares every module public, so `apps/rocm` runs the dashboard through `app::run` and examples drive the same modules in-process. | flat | +| [`client.rs`](../crates/rocm-dash-tui/src/client.rs) | The connection task behind the event stream, and `ClientMsg`, the one message type the app loop reads: connect, handshake, forward events, reconnect with backoff. | flat | +| [`jobs.rs`](../crates/rocm-dash-tui/src/jobs.rs) | The one async process runner the operational screens share: runs each `SideEffect::SpawnJob`, streams its output as `StateEvent::JobLine` and kills the child on cancel. | flat | +| [`llm.rs`](../crates/rocm-dash-tui/src/llm.rs) | Chat endpoint configuration: the precedence resolver `resolve_llm_config`, a TCP-only liveness probe and local-engine endpoint detection, with no HTTP client. | flat | +| [`reconnect.rs`](../crates/rocm-dash-tui/src/reconnect.rs) | `Backoff`, the exponential delay the client sleeps between connection attempts. | flat | +| [`registry_probe.rs`](../crates/rocm-dash-tui/src/registry_probe.rs) | `ManagedRegistryProbe`, the seam through which the dashboard reads the managed-service registry without depending on `rocm-core` or the daemon. | flat | +| [`replay.rs`](../crates/rocm-dash-tui/src/replay.rs) | The offline source behind `--replay`: posts a session file's events through the live client's channel, with pause, speed and seek through `ReplayController`. | flat | +| [`tool_exec.rs`](../crates/rocm-dash-tui/src/tool_exec.rs) | `RocmToolExecutor`, the plain-data seam for chat tool calls that need `rocm-core`: a read-only `execute` and an approval-gated `execute_approved`. | flat | +| [`transport.rs`](../crates/rocm-dash-tui/src/transport.rs) | NDJSON framing for the client: `write_line` and `read_line`. | flat | + +#### `agent/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`agent/mod.rs`](../crates/rocm-dash-tui/src/agent/mod.rs) | The chat-agent seam: the `AgentClient` trait, `AgentError`, `StateSnapshot`, `InferenceParams`, and the request budget — `DEFAULT_REQUEST_TIMEOUT`, the `ROCM_DASH_CHAT_TIMEOUT_SECS` override (`REQUEST_TIMEOUT_ENV`) and the `request_timeout()` that resolves between them. | relocation | +| [`agent/clients.rs`](../crates/rocm-dash-tui/src/agent/clients.rs) | The `AgentClient` backends: Rig (OpenAI-compatible), ChatGPT (OAuth device code), Anthropic, and the in-memory `MockAgentClient`. | relocation | +| [`agent/snapshot.rs`](../crates/rocm-dash-tui/src/agent/snapshot.rs) | Pure JSON summaries of the cached telemetry snapshot for the agent's tools, with no `rig` dependency. | relocation | +| [`agent/tools.rs`](../crates/rocm-dash-tui/src/agent/tools.rs) | The Rig `Tool` wrappers: the telemetry tools over the cached snapshot, the read-only and mutating ROCm tools, their registration on an agent builder, and the per-result size cap and duplicate-output trim applied before a tool result reaches the model. | relocation | + +#### `app/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`app/mod.rs`](../crates/rocm-dash-tui/src/app/mod.rs) | The reducer's front: `AppState` and its constructor, the theme picker, row selection, and `apply_event`, which folds daemon events into state. | flat | +| [`app/actions.rs`](../crates/rocm-dash-tui/src/app/actions.rs) | `KeyAction` dispatch: maps a key press or a resolved mouse hit to a `KeyAction` and applies it to reducer state (`apply_action`). | domain | +| [`app/chat.rs`](../crates/rocm-dash-tui/src/app/chat.rs) | Chat-backend construction, local-engine detection, and saving an accepted endpoint to the dashboard config. | relocation | +| [`app/chat_runtime.rs`](../crates/rocm-dash-tui/src/app/chat_runtime.rs) | `ChatRuntime`, the live chat backend the tick loop owns: which agent answers the next turn, the drains for the provider-switch, endpoint-rebuild and dispatch edges, and the mapping from a backend error to the line the Chat tab shows. | flat | +| [`app/chat_state.rs`](../crates/rocm-dash-tui/src/app/chat_state.rs) | An `impl AppState` block for chat consent, the local-endpoint detection offer, submitting a turn, and applying the agent's reply or error. | flat | +| [`app/event_loop.rs`](../crates/rocm-dash-tui/src/app/event_loop.rs) | The tick loop: `run` (set up, run, tear down) and `event_loop`, with the frame-draw gate and the registry-probe seeding. | domain | +| [`app/focus.rs`](../crates/rocm-dash-tui/src/app/focus.rs) | Focused-host policy: what a dashboard launched for Setup, Serve or Examine does differently (no daemon client or chat, one overlay opened at start, console close keys swallowed). | flat | +| [`app/overlays.rs`](../crates/rocm-dash-tui/src/app/overlays.rs) | An `impl AppState` block for which overlay owns the keyboard, `close_overlays`, and the gating predicates the key handlers and the renderer consult. | flat | +| [`app/quit_confirm.rs`](../crates/rocm-dash-tui/src/app/quit_confirm.rs) | An `impl AppState` block for the quit-confirm prompt: the live-instance gates, and opening, answering and resolving it. | flat | +| [`app/scroll.rs`](../crates/rocm-dash-tui/src/app/scroll.rs) | An `impl AppState` block for scroll offsets and recorded scrollbar tracks, and `ScrollState`, the shared clamped signed-delta scroll for a `u16` offset with a known bound. | flat | +| [`app/scrollbar.rs`](../crates/rocm-dash-tui/src/app/scrollbar.rs) | Mouse hit-testing: resolves a raw `MouseEvent` against recorded scrollbar tracks, the tab bar and the footer chips into a `KeyAction`. | domain | +| [`app/slash.rs`](../crates/rocm-dash-tui/src/app/slash.rs) | Slash-command routing for the chat input (`handle_slash_command`): mutates state, raises a one-shot edge for the event loop, or pushes a usage turn, with no I/O. | relocation | +| [`app/summary.rs`](../crates/rocm-dash-tui/src/app/summary.rs) | Pure display helpers: bounded summaries of read-only slash-tool results, and the parser of a natural-language plan into a `PlannedAction`. | relocation | +| [`app/terminal.rs`](../crates/rocm-dash-tui/src/app/terminal.rs) | Terminal restoration and shutdown arbitration: `restore_terminal`, the `SHUTTING_DOWN` latch, the termination-signal watcher and the Ctrl+C path. | flat | +| [`app/test_support.rs`](../crates/rocm-dash-tui/src/app/test_support.rs) | Test-only helpers shared by the `app` and `ui` test modules. | flat | +| [`app/tool_approval.rs`](../crates/rocm-dash-tui/src/app/tool_approval.rs) | An `impl AppState` block for the chat tool-approval flow: handing a plan's action to approval, opening the modal for a mutating-tool intent, and routing the verdict. | flat | +| [`app/types.rs`](../crates/rocm-dash-tui/src/app/types.rs) | Shared reducer types (`Focus`, `ResolvedArgs`, `Modal`, tab, chat and replay state, the slash and approval payloads) and the Esc-menu, palette and options navigation data. | domain | +| [`app/update_check.rs`](../crates/rocm-dash-tui/src/app/update_check.rs) | The background Updates-tile check: the tick-driven `rocm update --json` job and the reducer that turns its output into an `UpdateStatus`. | flat | + +#### `ui/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`ui/mod.rs`](../crates/rocm-dash-tui/src/ui/mod.rs) | The renderer's front: `draw` and `draw_focused`, the layout including the wide triptych, and `draw_active_manager`, which draws the open overlay or its job console. | flat | +| [`ui/approval.rs`](../crates/rocm-dash-tui/src/ui/approval.rs) | The shared approval prompt: renders an `ApprovalRequest` and reports the user's `ApprovalVerdict`, leaving the action itself to the caller. | flat | +| [`ui/automations_manager.rs`](../crates/rocm-dash-tui/src/ui/automations_manager.rs) | The Automations overlay: lists the background watchers and enables or disables one through `rocm automations` behind an approval. | flat | +| [`ui/bench_run.rs`](../crates/rocm-dash-tui/src/ui/bench_run.rs) | The bench-run form: endpoint, optional model, concurrency or auto-ramp, and output path, submitted as a `rocm bench load` job. | flat | +| [`ui/command_screen.rs`](../crates/rocm-dash-tui/src/ui/command_screen.rs) | The freeform command runner: any `rocm ...` command, always behind an approval that shows the exact argv. | flat | +| [`ui/config_manager.rs`](../crates/rocm-dash-tui/src/ui/config_manager.rs) | The config and provider overlay: `rocm config show`, and enabling or disabling an assistant provider behind an approval. | flat | +| [`ui/core_bars.rs`](../crates/rocm-dash-tui/src/ui/core_bars.rs) | `CoreBars`: one braille column per CPU core, filled from the bottom by utilization. | flat | +| [`ui/dock.rs`](../crates/rocm-dash-tui/src/ui/dock.rs) | The rails of the wide layout: the left GPU wall and the read-only right dock (LOGS or CONTEXT). | flat | +| [`ui/engine_manager.rs`](../crates/rocm-dash-tui/src/ui/engine_manager.rs) | The Engines overlay: set the default engine, and install or reinstall Lemonade or vLLM, each behind an approval. | flat | +| [`ui/examine_manager.rs`](../crates/rocm-dash-tui/src/ui/examine_manager.rs) | The Examine overlay: runs `rocm examine` read-only, and previews a Doctor report and sends it after approval. | flat | +| [`ui/exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs) | Helpers for launching a `rocm` sub-command from an operational screen: binary resolution, approval-preview quoting and the `rocm runtimes adopt` argv. | flat | +| [`ui/folder_browser.rs`](../crates/rocm-dash-tui/src/ui/folder_browser.rs) | The shared directory picker: use-current, new-child, parent and sub-directory entries, with only `build_entries` touching the filesystem. | flat | +| [`ui/footer.rs`](../crates/rocm-dash-tui/src/ui/footer.rs) | The footer: the key-hint chips, the Esc chip's context-dependent label, and the chip regions mouse hit-testing resolves against. | flat | +| [`ui/format.rs`](../crates/rocm-dash-tui/src/ui/format.rs) | Pure value formatters every tab uses (memory, percentages, rates, power, temperature, durations), including the held-value marker. | flat | +| [`ui/gradient.rs`](../crates/rocm-dash-tui/src/ui/gradient.rs) | `GradientGauge`, a gauge whose fill sweeps a three-stop color ramp, and `lerp3_t`, the shared three-stop interpolation. | flat | +| [`ui/header.rs`](../crates/rocm-dash-tui/src/ui/header.rs) | The header: the title line, the connection state, and the warning badge with its click region. | flat | +| [`ui/help_modal.rs`](../crates/rocm-dash-tui/src/ui/help_modal.rs) | The Help modals (per-tab help and the global keyboard reference), and `popup_height_for`, the content-sized popup arithmetic. | flat | +| [`ui/install_manager.rs`](../crates/rocm-dash-tui/src/ui/install_manager.rs) | The Install overlay: builds a `rocm install sdk` invocation; a dry run starts directly and a real install goes through approval. | flat | +| [`ui/instance_detail.rs`](../crates/rocm-dash-tui/src/ui/instance_detail.rs) | The instance Detail modal: a summary strip, recent throughput, and scrollable launch-args and env-var panes over a dimmed backdrop. | flat | +| [`ui/job_console.rs`](../crates/rocm-dash-tui/src/ui/job_console.rs) | The shared "a job is running" surface: `draw_job_console` renders the status header, streamed output and key hints, and `on_console_key` handles its keys. | flat | +| [`ui/launcher.rs`](../crates/rocm-dash-tui/src/ui/launcher.rs) | The bare-`rocm` front door: a synchronous pre-dashboard screen with a status strip over the headline verbs, returning a `LauncherChoice`. | flat | +| [`ui/logs_view.rs`](../crates/rocm-dash-tui/src/ui/logs_view.rs) | The Logs overlay: `rocm logs`, optionally with search terms, read-only into the job console. | flat | +| [`ui/menu.rs`](../crates/rocm-dash-tui/src/ui/menu.rs) | The Esc main menu, the command palette and the tabbed Options panel, with the block-letter logo they share. | flat | +| [`ui/modal.rs`](../crates/rocm-dash-tui/src/ui/modal.rs) | Re-exports only, so every item that was public at `ui::modal` before its split still resolves there. | flat | +| [`ui/model_picker.rs`](../crates/rocm-dash-tui/src/ui/model_picker.rs) | A filterable list of model recipes the Serve form opens; choosing one fills only the model id. | flat | +| [`ui/onboarding.rs`](../crates/rocm-dash-tui/src/ui/onboarding.rs) | The onboarding wizard: install the ROCm SDK or adopt an existing ROCm folder, both behind an approval. | flat | +| [`ui/panel.rs`](../crates/rocm-dash-tui/src/ui/panel.rs) | The shared box chrome: `bento` and `popup` panels, scrollbars, and popup geometry and framing (`centered_rect`, `centered_width`, `draw_popup_frame`, `grey_overlay`). | flat | +| [`ui/quit_confirm.rs`](../crates/rocm-dash-tui/src/ui/quit_confirm.rs) | The quit-confirm prompt's content (`quit_confirm_request`): the title, the body listing what is still served, and the stop hint. | flat | +| [`ui/runtime_manager.rs`](../crates/rocm-dash-tui/src/ui/runtime_manager.rs) | The Runtimes overlay: lists the registered runtimes and runs the `rocm runtimes` verbs, the mutating ones behind an approval. | flat | +| [`ui/serve_wizard.rs`](../crates/rocm-dash-tui/src/ui/serve_wizard.rs) | The Serve form: builds a model-first `rocm serve` invocation and launches it as a job behind an approval. | flat | +| [`ui/services_manager.rs`](../crates/rocm-dash-tui/src/ui/services_manager.rs) | The Services overlay: lists the managed inference services with their throughput (live, or a Lemonade server's last-request rate under a legend), and stops or restarts one behind an approval. | flat | +| [`ui/sparkline.rs`](../crates/rocm-dash-tui/src/ui/sparkline.rs) | `BrailleSparkline`, a history graph that packs two samples into each character column at four dot rows per cell. | flat | +| [`ui/spinner.rs`](../crates/rocm-dash-tui/src/ui/spinner.rs) | Pure helpers for the braille spinner frame (`spinner_frame`) and progress-percentage parsing (`parse_progress_pct`) the job console renders with. | flat | +| [`ui/theme.rs`](../crates/rocm-dash-tui/src/ui/theme.rs) | The semantic `Theme` and its catalogue, and the shared status-to-color mapping (`StatusTone`, `tone_color`, `job_status_color`, `instance_status_color`). | flat | +| [`ui/theme_picker.rs`](../crates/rocm-dash-tui/src/ui/theme_picker.rs) | The theme picker modal: the registered themes beside a live preview of the highlighted one. | flat | +| [`ui/update_manager.rs`](../crates/rocm-dash-tui/src/ui/update_manager.rs) | The Update overlay: the `rocm update` check and dry-run preview start directly; applying goes through an approval. | flat | +| [`ui/warnings_modal.rs`](../crates/rocm-dash-tui/src/ui/warnings_modal.rs) | The Warnings modal: the text behind the header's warning badge, content-sized like the Help modals. | flat | +| [`ui/widgets.rs`](../crates/rocm-dash-tui/src/ui/widgets.rs) | Small helpers shared across tabs, including `apply_char_key` and `apply_char_key_filtered`, the append-only text-field edit the form key handlers use. | flat | + +#### `ui/tabs/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`ui/tabs/mod.rs`](../crates/rocm-dash-tui/src/ui/tabs/mod.rs) | Declares the tab modules and `TAB_LABELS`, draws the folder-tab panel, and owns the tab-chip geometry (`compute_chip_layout`). | flat | +| [`ui/tabs/bench.rs`](../crates/rocm-dash-tui/src/ui/tabs/bench.rs) | The Observe tab's benchmark panel: Pass^N and Pass@N rollups, a scrollable table of rows with verdicts, and a sparkline. | flat | +| [`ui/tabs/chat.rs`](../crates/rocm-dash-tui/src/ui/tabs/chat.rs) | The Chat tab: the consent gate until an endpoint is accepted, then the transcript and the input line. | flat | +| [`ui/tabs/hardware.rs`](../crates/rocm-dash-tui/src/ui/tabs/hardware.rs) | The Observe tab's host and GPU panel: CPU sparkline and per-core bars, memory and swap gauges, and a panel or a line per GPU. | flat | +| [`ui/tabs/home.rs`](../crates/rocm-dash-tui/src/ui/tabs/home.rs) | The Home tab: the hero GPU gauge, the VRAM, temperature and power cluster, the Running, Health and Updates tiles and an activity feed, with honest placeholders. | flat | +| [`ui/tabs/instances.rs`](../crates/rocm-dash-tui/src/ui/tabs/instances.rs) | The per-instance metrics table of the Observe tab, and the sorting, selection and status helpers the Detail modal also reads. | flat | +| [`ui/tabs/observe.rs`](../crates/rocm-dash-tui/src/ui/tabs/observe.rs) | The Observe tab: an efficiency and throughput hero band over the instance table, the Hardware panel and the Bench panel. | flat | +| [`ui/tabs/pane.rs`](../crates/rocm-dash-tui/src/ui/tabs/pane.rs) | The shared Actions and Details renderer of the ROCm and Serving tabs, whose verb rows open the existing overlays through `KeyAction::Open*`. | flat | +| [`ui/tabs/rocm.rs`](../crates/rocm-dash-tui/src/ui/tabs/rocm.rs) | The ROCm tab's verb table: install, updates, diagnose, runtimes, the command runner, and a display-only Uninstall row. | flat | +| [`ui/tabs/serving.rs`](../crates/rocm-dash-tui/src/ui/tabs/serving.rs) | The Serving tab's verb table: serve, engines, services, providers, logs, and a display-only Optimize row. | flat | + +Boundaries: + +- **No `rocm-core`.** The crate depends on `rocm-dash-core` and must not depend on `rocm-core` ([`crate_edges.rs`](../xtask/src/crate_edges.rs)). + - `rocm-core` data reaches it through the [`tool_exec.rs`](../crates/rocm-dash-tui/src/tool_exec.rs) and [`registry_probe.rs`](../crates/rocm-dash-tui/src/registry_probe.rs) seams, which the `rocm` binary implements, or as values the binary plumbs in on `ResolvedArgs`. + - A few values that mirror `rocm-core` (the default `rocm serve` port in [`llm.rs`](../crates/rocm-dash-tui/src/llm.rs), the runtime Python layout in [`ui/exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs)) are literal copies kept because of that rule; a change to the original needs the copy changed too. + - `ManagedRegistryProbe` is read live, at each quit attempt and each frame the quit-confirm prompt is open, so an implementation must stay cheap (a local file read). +- **Windows.** There is no daemon socket on Windows: the `cfg(windows)` arm of [`client.rs`](../crates/rocm-dash-tui/src/client.rs) collects host and GPU metrics in-process through `rocm-dash-collectors`, the only use of that dependency, and the snapshot it posts does not replace the instance list. The instance list arrives separately: [`event_loop.rs`](../crates/rocm-dash-tui/src/app/event_loop.rs)'s `spawn_managed_registry_poller` reads the on-disk managed-service registry on a timer and sends it as `ClientMsg::ManagedInstances`. So Windows does have an instance list — it is simply not carried by the metrics snapshot. +- **Overlays and approvals.** + - Each operational overlay is an `Option` state struct on `AppState` (`None` is closed); every opener calls `close_overlays` first, so at most one is open, and `draw_active_manager` draws it or its job console. + - A mutating overlay action stages an `ApprovalRequest` in the overlay's own state; the job starts only on Approve and the `rocm` CLI does the mutation. Read-only actions start their job directly, and so does the bench-run form. + - A chat tool call is approved separately, through `AppState::approval` and `tool_exec`'s `execute_approved`, which blocks and so runs off the async loop; the overlays do not use `tool_exec`. +- **Jobs.** One `State` holds every job (`AppState::jobs`), so closing an overlay leaves its jobs in the table. [`jobs.rs`](../crates/rocm-dash-tui/src/jobs.rs) sends events over an unbounded channel on purpose, because the synchronous reducer cannot apply backpressure, and the reducer bounds the output instead. +- **Reuse the shared components.** + - A new screen reuses the approval prompt, job console, spinner, panel, theme tones, text-field edit and `ScrollState` that [`AGENTS.md`](../AGENTS.md) section 6 names, instead of hand-rolling its own. + - [`dash_text_edit_contract.rs`](../xtask/src/dash_text_edit_contract.rs) fails a hand-rolled `push`/`pop` text edit outside [`ui/widgets.rs`](../crates/rocm-dash-tui/src/ui/widgets.rs); the Chat tab's input is the one exception. +- **Layering in `app/` and `ui/`.** + - The modules split from [`app/event_loop.rs`](../crates/rocm-dash-tui/src/app/event_loop.rs) (`chat_runtime`, `focus`, `terminal`, `update_check`) are leaves: none calls back into it. + - [`app/mod.rs`](../crates/rocm-dash-tui/src/app/mod.rs) re-exports the public surface its children took over, so `crate::app::` paths keep resolving. + - The tabs only read `AppState` while drawing, except the Chat tab, which writes its measured scroll bounds back. +- **Single sources.** + - `MENU_ITEMS` in [`app/types.rs`](../crates/rocm-dash-tui/src/app/types.rs) gives the Esc menu's rows, their labels, the key handling's wrap bound and the height `draw_menu` reserves. + - `outlined_chip_spans` in [`ui/tabs/mod.rs`](../crates/rocm-dash-tui/src/ui/tabs/mod.rs) is the one tab geometry, behind both the painter and `compute_chip_layout`, which [`app/actions.rs`](../crates/rocm-dash-tui/src/app/actions.rs) uses to turn a click into a tab. +- **Endpoints and binds.** + - [`llm.rs`](../crates/rocm-dash-tui/src/llm.rs) strips the API key and custom auth header from a loopback base URL, so a local endpoint is never sent a gateway credential. + - [`ui/serve_wizard.rs`](../crates/rocm-dash-tui/src/ui/serve_wizard.rs) has its own stricter `is_loopback_host` (exact spellings only) for the bind host. Its default invocation is exactly `rocm serve MODEL`, so the CLI stays the authority for engine, GPU-required policy, host, port and managed mode. ### `crates/rocm-engine-protocol` — engine IPC protocol A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -### `crates/rocm-deps` - -Pinned versions of the third-party runtimes rocm-cli manages (from workspace-root [`runtime-deps.toml`](../runtime-deps.toml), turned into constants by [`build.rs`](../crates/rocm-deps/build.rs)). Small and already single-purpose; not part of the modularization effort's target list. - -### `crates/e2e-report` - -Modularized (ROCMAI-60, Phase 1 of ROCMAI-27's sequencing): - -- [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) — (ROCMAI-486, Phase 1 of a staged further split of [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs)) the CLI command-coverage audit table — which `rocm` commands were exercised and whether they passed — consumes `consolidated::PlatformReport` but shares no type with the `Grid`/reconciliation model itself. -- [`components.rs`](../crates/e2e-report/src/components.rs) — shared maud HTML fragment rendering, plus the CSS and timestamp helpers both generators use, depending only on [`parse.rs`](../crates/e2e-report/src/parse.rs) types to keep the module graph acyclic. -- [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) — the `PlatformReport` model (loading one platform's `report.json` together with its sidecars), the reconciled scenario × platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, which both renderers draw from, plus the scenario-reference map it builds, and the report text both renderers share so it cannot drift between them: `LEGEND_ENTRIES` (the grid legend) and the two phrases of the PR-gating policy, `MOCK_GATES_PR` and `OTHER_LANES_NON_BLOCKING`, which the markdown explanation paragraph and the HTML legend each build their own sentence around. Artifact-name parsing lives in [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs), the `platform.json`/`commands.jsonl` schema and parsing in [`consolidated/sidecars.rs`](../crates/e2e-report/src/consolidated/sidecars.rs) (both reached through `use`), and the two renderers in [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) and [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs), declared `pub(super) mod`. It is still the largest module in the crate, still holding the `Grid`/`PlatformReport` model and the bundle in one file; what remains of the split is optional: a `grid.rs` split and a relocation of [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs). -- [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) — artifact-name identity: `Descriptor` and `parse_descriptor` (artifact/directory name to platform, OS, legacy known-bugs flag and channel), driven by one `KNOWN_PLATFORMS` table that also holds the `platform.json` `platform_slug`s of each lane, so the public `artifact_name_for_platform_slug` (what [`xtask`'s e2e report](../xtask/src/e2e_report.rs) uses to name a report downloaded without its artifact directory) is the inverse of the same rows rather than a second table, plus the one platform-label rule, `full_platform_label` (platform, OS and optional channel), which `PlatformReport::label`, `GridColumn::label` and the excluded-platform caption in `Grid::build` all call, and the helpers under it, `with_paren_suffix` (also used for the sdk_version leg in `parse_descriptor`) and `fallback_descriptor`. A leaf: it imports nothing from [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs), which reaches it through `use descriptor::{...}`; its `parse_descriptor_*` tests sit beside it. -- [`consolidated/sidecars.rs`](../crates/e2e-report/src/consolidated/sidecars.rs) — sidecar schema and parsing: the `platform.json` manifest (`PlatformManifest`, `ManifestCapability`, `PlatformVersions` with its `summary`/`platform_stack` display strings, `ManifestExpectation`, `parse_platform_manifest`) and the `commands.jsonl` record (`CommandRecord`, `parse_commands`). A leaf that uses only serde, `serde_json` and `std` (no other module of the crate) and imports nothing from [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs), which reaches it through `use sidecars::{...}`; `CommandRecord` stays `pub(crate)` because [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) reads it through `PlatformReport::commands`. The parsers have no direct unit tests; they are covered through `PlatformReport::load`. -- [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) — the HTML renderer: `RunMeta` and `generate_consolidated`, with the matrix table, legend, expectation grid (`expectation_grid_html`, `pub(super)` so the cross-renderer tests in [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) can call it) and per-platform sections it assembles. A child of [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) (declared `pub(super) mod html;`), so it reads the private `PlatformReport`, `Grid`, `CellOutcome` and `PlatformReports` items defined there without any of them being widened; no production code in the parent or in the markdown renderer calls into it (only the cross-renderer tests that stay in `consolidated/mod.rs` do). Its `generate_consolidated_*` and legend tests sit beside it. -- [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) — the markdown renderer: `consolidated_summary_markdown` (the table piped into `$GITHUB_STEP_SUMMARY`), `expectation_grid_markdown` and `scenario_reference_markdown`, and the call into [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs)'s `command_coverage_markdown` for the command-coverage table. A child of [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) (declared `pub(super) mod markdown;`), so it reads the private `PlatformReport`, `Grid`, `CellOutcome`, `PlatformReports` and `ScenarioReference` items defined there without any of them being widened; it does not call the HTML renderer. The tests that assert on its output sit beside it; what stays in `consolidated/mod.rs` is the cross-renderer tests (HTML and markdown over one bundle), the command-coverage tests, and tests of the grid or `PlatformReport` that use the markdown only as a probe. -- [`consolidated/test_support.rs`](../crates/e2e-report/src/consolidated/test_support.rs) — `#[cfg(test)]` fixtures (`feature_json`, `write_report`, `write_platform`, `grid_order`) shared by the `consolidated/` tests, so each module extracted from [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) can keep its tests beside it. -- [`parse.rs`](../crates/e2e-report/src/parse.rs) — cucumber `report.json` data model, parsing, and `@expected-failure` xfail evaluation. -- [`single_report.rs`](../crates/e2e-report/src/single_report.rs) — single-platform HTML report generation. +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) | The protocol contract: protocol versions, the request/response envelopes and per-method types, device and GPU-selection policy, engine-plugin discovery, the endpoint API key file carrier shared by `rocm` and `rocmd`, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. | flat | +| [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) | Test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. | flat | +| [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) | The client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, whether to withhold an inherited key file from a child that is handed none, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. It sets no timeout, and does not bound the wait for the child's pipes after it exits (a grandchild holding one makes the request wait); its module comment states that limit. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). | flat | -These modules are private `mod` declarations — [`lib.rs`](../crates/e2e-report/src/lib.rs) re-exports only the selected public API surface (`XfailReport`, `evaluate_xfail`, `scenario_results_by_id`, `generate`, `RunMeta`, `PlatformReports`, `load_platform_reports`, `artifact_name_for_platform_slug`, `generate_consolidated`, `consolidated_summary_markdown`) via `pub use` (`RunMeta` and `generate_consolidated` through `consolidated::html`, `consolidated_summary_markdown` through `consolidated::markdown`: the only child modules `lib.rs` reaches by path), so consumers reach it through the crate root rather than through module-qualified paths like `e2e_report::parse::...`. `generate_consolidated` and `consolidated_summary_markdown` take the opaque `PlatformReports` bundle [`load_platform_reports`](../crates/e2e-report/src/consolidated/mod.rs) returns (ROCMAI-484 / ROCm/rocm-cli#640) — bundling the parsed reports with the expectation `Grid` and scenario-reference map built from the same `inputs` means a caller producing both outputs can no longer pass them out of sync with each other. Building them there also means the grid is built once per run rather than once per report section that renders it (ROCMAI-660). `PlatformReport` itself went back to `pub(crate)` once the bundle was the only thing outside callers needed. This is a deliberate encapsulation choice tighter than the full-domain-extraction convention's `pub mod x;` + `pub use x::{...};` default described above — nothing outside this crate needs the module paths themselves, only the re-exported items. Unlike [`components.rs`](../crates/e2e-report/src/components.rs), which, as noted above, keeps the module graph acyclic by depending only on [`parse.rs`](../crates/e2e-report/src/parse.rs), [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) and the `consolidated` module as a whole depend on each other: [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) calls `command_coverage::command_coverage_markdown`, and [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) consumes `consolidated::PlatformReport` (via its `commands`, `label` and `scenario_pass_map()`), which is defined in [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs). That chain between files (`markdown.rs` to `command_coverage.rs` to `consolidated/mod.rs`) is acyclic, and no production code in `consolidated/mod.rs` depends on `command_coverage.rs` any more (only its command-coverage tests import from it); viewed as the whole `consolidated` module, though, the two still depend on each other, which moving `command_coverage.rs` under `consolidated/` would remove. - -### `engines/lemonade`, `engines/vllm`, `engines/aim` — inference engine adapters - -`engines/vllm` follows full domain extraction: - -- [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) — the ROCm 10.x discovery table (`VLLM_ROCM_DISCOVER_BUILD_TABLE`, one row per supported `major.minor` line, with the `DiscoverVersion` series/exact constraint), the row lookup (`vllm_rocm_discover_build`) and the `major.minor` matcher it uses (`rocm_sdk_series_matches`). It is the single list of the ROCm SDK versions installed by discovery: install routing, the fail-closed static fallback and launch-time environment setup (`runtime_has_discover_build` in `runtime.rs`) all read these rows, and nothing keeps a second list in step with them. A leaf: it imports no other crate module, so `runtime.rs`, `wheel_builds.rs` and `install.rs` can all read it without importing one another's code. Its tests, including the one that pins which versions the table serves and in what order, live with it. -- [`install.rs`](../engines/vllm/src/install.rs) — ROCm/vLLM install execution: the static-pin and the ROCm 10.x discovery install routes, run through `uv`. It reads the static build table and target resolution from `wheel_builds.rs` and the discovery rows from `discover_builds.rs`, and asks `repair.rs` whether an already-resolvable environment still needs an install pass. -- [`process.rs`](../engines/vllm/src/process.rs) — the resolve-model, launch, serve and stop handlers (including engine-recipe validation and device-policy/GPU-index resolution), vLLM server spawn, launch environment and path setup, and readiness polling. The text appended to a "never became ready" failure comes from `startup_diagnostics.rs`. -- [`repair.rs`](../engines/vllm/src/repair.rs) — runtime-repair assessment: whether an already-resolvable managed vLLM environment violates vLLM's pinned dependencies and must be reinstalled (`assess_runtime_repair`, `RepairAssessment`), including the rule that treats the SDK's build of torch, and a torch the user chose with the alignment disabled, as intended rather than a defect. It imports `runtime.rs` and nothing from `install.rs`. -- [`runtime.rs`](../engines/vllm/src/runtime.rs) — TheRock/managed-runtime resolution and TheRock library/bin path derivation; the path derivation performs no filesystem writes. The launch path's compat-shim writes (a `libmpi_cxx` stub, stale-numa-shim removal) are this module's separate `ensure_therock_compat_shims`, called only from `process.rs`; the running-state payload, which never calls it, falls back to a plain directory listing instead. -- [`startup_diagnostics.rs`](../engines/vllm/src/startup_diagnostics.rs) — what a failed startup tells the user: the tail of the startup log, re-rendered line by line through `terminal::rendered_lines` (not stripped — stripping deletes line-break tokens with no separator, which welds a progress-bar repaint onto the next line), followed by the out-of-memory and engine-import-failure hints that point at `rocm diagnose`. Its single entry point is `startup_log_context`, which `process.rs` calls from `wait_for_vllm_ready`. It imports only `state.rs`, for the log-read cap. -- [`state.rs`](../engines/vllm/src/state.rs) — service state file I/O, log tailing, and health/inference probing. -- [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) — which vLLM wheels to install for a ROCm SDK version: the static build table (`VLLM_ROCM_BUILD_TABLE`), the install-route choice (`vllm_install_route`, which consults the discovery table in `discover_builds.rs`), target resolution including the `ROCM_CLI_VLLM_ROCM_INDEX_URL` override (`vllm_install_target`), and the static-table SDK-version matchers. It also holds `verified_torchaudio_release`, which the crate re-exports from `lib.rs` for the `rocm` binary. It imports only `discover_builds.rs` and runs no process. - -[`lib.rs`](../engines/vllm/src/lib.rs) keeps only CLI/envelope dispatch and the few items shared across those modules. - -`engines/lemonade` follows the same pattern: - -- [`archive_cache.rs`](../engines/lemonade/src/archive_cache.rs) — the cached embeddable-archive layer: SHA-256 verification, verified publication of a download into the cache, and the per-archive exclusion (an in-process set plus a cross-process file lock) that `ensure_cached_archive` returns as an `ArchiveCacheGuard` for the caller to hold while it relies on the verified bytes. A leaf: it imports no other crate module. Its tests, including the two-process lock test, live with it. -- [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) — the hardened in-process extractor for the embeddable archive (tar.gz and zip: untrusted entry names are resolved component by component, links and special entries are rejected, modes are masked) and the containment-checked `copy_tree` that moves the extracted tree into the runtime directory. A leaf: it imports no other crate module. Its extraction tests and the tar/zip property tests live with it; the property-test support (`archive_props`) is included once at the crate root so both this module and `install.rs` reach it as `crate::archive_props`. -- [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) — ROCm-SDK-to-llama.cpp-backend Tier 1/Tier 2/revert alignment, including the `backend_versions.json` pin handling and the set-aside/restore of ROCm backend directories around a forced reinstall. It uses `llamacpp_backend.rs`; that module does not import it back. -- [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) — device-policy and GPU-selection handling: the GPU-required policy check (a CPU-only request is refused, never downgraded), `--device`/`--gpu` argument parsing, probing and resolving the GPU ordinal(s) to pin before a launch, the device availability reported by `detect`, and the device string reported for a loaded model. A leaf: it imports no other crate module. -- [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) — Hugging Face checkpoint/GGUF resolution and the direct-llama-server serve path (`serve_direct_llama_server`, which runs a packaged `llama-server` on a GGUF and waits for `/v1/models` to report the model). It imports `process.rs`; `process.rs` no longer imports it. -- [`install.rs`](../engines/lemonade/src/install.rs) — embeddable-runtime download, extraction orchestration (the cache layer is in `archive_cache.rs`, the extractor in `archive_extract.rs`), and manifest persistence, plus the `LemonadeRuntime` handle and `resolve_runtime`, which wrap the persisted manifest (so `state.rs`, `process.rs` and `direct_llama.rs` reach it without importing `backend_alignment.rs`). -- [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) — llama.cpp backend discovery and selection: asking `lemonade backends` what the host supports, choosing the best GPU backend (`rocm` before `vulkan`, never `cpu`), installing it with one retry, the `ROCM_LLAMACPP_BACKEND_DIRS` name list, and `find_llama_server_binary_for_backend`. It imports `direct_llama.rs` for `find_binary_in`; `direct_llama.rs` does not import it. With this module the engine's modules form no import cycle outside test code. -- [`process.rs`](../engines/lemonade/src/process.rs) — `lemond` spawn, env/path setup, readiness/health polling, and the background relaunch of the `serve-http` command (`serve_http_command_args`, `spawn_serve_http_background`). The direct `llama-server` serve loop is in `direct_llama.rs`. -- [`runtime_dir.rs`](../engines/lemonade/src/runtime_dir.rs) — `XDG_RUNTIME_DIR` tier1/2/3 hardening for the spawned child. -- [`state.rs`](../engines/lemonade/src/state.rs) — service state file I/O and port/endpoint helpers. +### `crates/rocm-deps` -[`lib.rs`](../engines/lemonade/src/lib.rs) keeps only CLI/envelope dispatch and the service-lifecycle orchestration that calls across those modules. +Pinned versions of the third-party runtimes rocm-cli manages, read from the workspace-root [`runtime-deps.toml`](../runtime-deps.toml) and turned into constants by [`build.rs`](../crates/rocm-deps/build.rs). Only `rocm` and `rocm-engine-lemonade` depend on it; the dashboard TUI does not. -`engines/aim` runs models as Docker containers instead of a host process: +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/rocm-deps/src/lib.rs) | Includes the generated constants and derives Lemonade release names and download URLs from them. | flat | -- [`docker.rs`](../engines/aim/src/docker.rs) — thin wrapper around the `docker` CLI binary; no `bollard`/Docker API dependency. It also owns `container_name`, the deterministic Docker-name-sanitized container identity for a service (a leaf, so both `process.rs` and `state.rs` reach it here, and `state.rs` does not need to import `process.rs`). -- [`registry.rs`](../engines/aim/src/registry.rs) — AIM image metadata (`Env`/`Labels`): read from the local daemon once the image is pulled, which works for any registry, with a best-effort Docker Hub v2 lookup as a pre-pull hint (image ref normalization, tag/digest parsing). The hint is optional; its failure never blocks a launch. -- [`process.rs`](../engines/aim/src/process.rs) — container run/readiness/stop lifecycle. `serve_http` is the process rocmd supervises: it blocks for the container's lifetime and records how it ended (`failed` and a non-zero exit for a crash or a vanished container, so recovery runs). A stale container is removed before a re-run, and a stop removes the container so its name and downloaded weights are released. -- [`state.rs`](../engines/aim/src/state.rs) — service state file I/O, including the model name the container actually serves (`served_model_id`), which differs from the image reference, and `endpoint_url`, the `/v1` base URL for a host and port. `state.rs` uses no `process.rs` item: `process.rs` imports `state.rs` and the dependency does not run the other way. -- [`install.rs`](../engines/aim/src/install.rs) — much smaller than vLLM's/Lemonade's `install.rs`: there is no venv, wheel, or runtime archive to manage, only a Docker-reachability check. +### `crates/e2e-report` -[`lib.rs`](../engines/aim/src/lib.rs) keeps only CLI/envelope dispatch (including the `builtin_*` entry points the `rocm` binary calls in-process), the detect and capabilities responses, and the few items shared across those modules. +HTML and markdown reporting for the cucumber E2E suite, consumed only by `xtask` and `tests/e2e-cucumber`. It is in this map because `check-architecture-doc` covers every file under `crates/*/src`, although it is not part of the shipped CLI. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../crates/e2e-report/src/lib.rs) | Crate root: module declarations and the public API re-exports. | flat | +| [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) | The CLI command-coverage audit table: which `rocm` commands a platform's run exercised and whether they passed. | domain | +| [`components.rs`](../crates/e2e-report/src/components.rs) | Shared maud HTML fragments plus the CSS and timestamp helpers both report generators use. | flat | +| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, plus the report text both renderers share. | domain | +| [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) | Artifact-name identity: parses an artifact or directory name into platform, OS and channel, and owns the platform-label rule. | domain | +| [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) | The HTML renderer for the consolidated report. It does not render `PlatformVersions::provenance()`: the GFX target and package source appear only in the Markdown output (the wiki page and the step summary that shares its renderer). | domain | +| [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) | The markdown renderer, including the table piped into `$GITHUB_STEP_SUMMARY`. | domain | +| [`consolidated/sidecars.rs`](../crates/e2e-report/src/consolidated/sidecars.rs) | Schema and parsing for the `platform.json` manifest (including the per-lane `gfx_target` and package `source` shown as provenance) and the `commands.jsonl` record. | domain | +| [`consolidated/test_support.rs`](../crates/e2e-report/src/consolidated/test_support.rs) | Test-only fixtures shared by the `consolidated/` tests. | flat | +| [`parse.rs`](../crates/e2e-report/src/parse.rs) | The cucumber `report.json` data model, its parsing, and `@expected-failure` evaluation. | flat | +| [`single_report.rs`](../crates/e2e-report/src/single_report.rs) | Single-platform HTML report generation. | flat | + +Boundaries: + +- The modules are private; [`lib.rs`](../crates/e2e-report/src/lib.rs) re-exports only the selected API, so consumers reach everything through the crate root. This is tighter than the `pub mod` plus `pub use` default in the convention above, on purpose: nothing outside the crate needs the module paths. +- `load_platform_reports` returns one opaque `PlatformReports` bundle (parsed reports, the expectation grid and the scenario-reference map built from the same inputs), so a caller producing both outputs cannot pass them out of sync, and the grid is built once per run. +- [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) and [`consolidated/sidecars.rs`](../crates/e2e-report/src/consolidated/sidecars.rs) are leaves: neither imports [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs). The two renderers are `pub(super)` children that read the parent's private types, and neither calls the other. +- [`components.rs`](../crates/e2e-report/src/components.rs) depends only on [`parse.rs`](../crates/e2e-report/src/parse.rs), which keeps that part of the module graph acyclic. [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) and `consolidated` depend on each other as modules ([`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) calls it, and it reads `consolidated::PlatformReport`); the file-level chain is acyclic, and no production code in [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) uses [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs). Moving [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) under `consolidated/` would remove the module-level dependency. + +### `engines/lemonade` + +The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Lemonade runtime and serves models through `lemond` or a packaged `llama-server`. Its modules follow full domain extraction; [`lib.rs`](../engines/lemonade/src/lib.rs) keeps only dispatch and the service-lifecycle orchestration that calls across them. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../engines/lemonade/src/lib.rs) | Crate root: CLI and envelope dispatch, the service-lifecycle orchestration that calls across the modules, and the crate-root re-exports `apps/rocm` uses. | flat | +| [`main.rs`](../engines/lemonade/src/main.rs) | The `rocm-engine-lemonade` binary's entry point; it only calls `rocm_engine_lemonade::run_cli()`. | flat | +| [`archive_cache.rs`](../engines/lemonade/src/archive_cache.rs) | The cached embeddable-archive layer: SHA-256 verification, verified publication into the cache, and the per-archive lock `ensure_cached_archive` returns as an `ArchiveCacheGuard` to hold while using the bytes. | domain | +| [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) | The hardened in-process tar.gz and zip extractor for the embeddable archive, and the containment-checked `copy_tree` into the runtime directory. | domain | +| [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) | Aligning the llama.cpp backend with the ROCm SDK (Tier 1, Tier 2, revert), including the `backend_versions.json` pin and setting ROCm backend directories aside around a forced reinstall. | domain | +| [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) | Device policy and GPU selection: the GPU-required check, `--device`/`--gpu` parsing, the GPU ordinals to pin before a launch, the device availability `detect` reports, and the notice that names a non-ROCm GPU backend (Vulkan) as the one serving. | domain | +| [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) | Hugging Face checkpoint and GGUF resolution, including a registered model name resolved through the installed Lemonade catalog, which route a name serves by, and the direct serve path that runs a packaged `llama-server` on a GGUF (`serve_direct_llama_server`). | domain | +| [`footprint.rs`](../engines/lemonade/src/footprint.rs) | What a Lemonade install costs on disk: the pre-install notice and free-space check, and the read-only view of Lemonade's install trees and record. | domain | +| [`install.rs`](../engines/lemonade/src/install.rs) | Embeddable-runtime download, extraction orchestration and manifest persistence, plus the `LemonadeRuntime` handle and `resolve_runtime` over the persisted manifest. | domain | +| [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) | llama.cpp backend discovery and selection: asks `lemonade backends` what the host supports, picks and installs the best GPU backend, and finds its `llama-server`. | domain | +| [`process.rs`](../engines/lemonade/src/process.rs) | `lemond` spawn, environment and path setup, readiness and health polling, and the background relaunch of the `serve-http` command. | domain | +| [`router_serve.rs`](../engines/lemonade/src/router_serve.rs) | The router serve path: spawn `lemond`, select a GPU backend, load and verify the model by its router name, and on a failure after the spawn stop the server and settle the state. A load or verification failure may then be served directly; a probe that could not reach the endpoint is not one. | domain | +| [`runtime_dir.rs`](../engines/lemonade/src/runtime_dir.rs) | `XDG_RUNTIME_DIR` tier 1, 2 and 3 hardening for the spawned child. | domain | +| [`state.rs`](../engines/lemonade/src/state.rs) | Service state file I/O and port and endpoint helpers. | domain | +| [`test_http.rs`](../engines/lemonade/src/test_http.rs) | Test-only: a loopback HTTP fixture that answers each request with a chosen status and JSON body and records what it received, plus (Linux) fake `lemond` and `lemonade` executables that log their arguments. | flat | +| [`test_support.rs`](../engines/lemonade/src/test_support.rs) | Test-only fixtures shared by the crate's module tests. | flat | + +Boundaries: + +- **GPU required, no CPU fallback.** [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) refuses a CPU-only request rather than downgrading it, and [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) prefers `rocm`, then `vulkan`, and never picks `cpu`. +- **Layering.** + - Outside test code the modules form no import cycle: [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) uses [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs), which uses [`process.rs`](../engines/lemonade/src/process.rs); [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) uses both of those, and [`router_serve.rs`](../engines/lemonade/src/router_serve.rs) uses only `process.rs` for its types. `process.rs` imports none of them, and none imports back. + - [`install.rs`](../engines/lemonade/src/install.rs) owns `LemonadeRuntime` and `resolve_runtime` so that [`state.rs`](../engines/lemonade/src/state.rs), [`process.rs`](../engines/lemonade/src/process.rs) and [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) reach the manifest without importing [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs). +- **Shared test support.** The archive property-test support (`archive_props`) is included once at the crate root, so [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) and [`install.rs`](../engines/lemonade/src/install.rs) both reach it as `crate::archive_props`. +- **Exports for `apps/rocm`.** [`footprint.rs`](../engines/lemonade/src/footprint.rs)'s install-tree view is re-exported at the crate root for `rocm storage` and `rocm runtimes uninstall`. + +### `engines/vllm` + +The vLLM engine adapter (`rocm-engine-vllm`): installs vLLM into a managed TheRock runtime with `uv`, then launches, probes and stops the vLLM server. Its modules follow full domain extraction; [`lib.rs`](../engines/vllm/src/lib.rs) keeps only dispatch and the few items shared across them. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../engines/vllm/src/lib.rs) | Crate root: CLI and envelope dispatch and the few items shared across the modules, including `verified_torchaudio_release` for the `rocm` binary, a `pub fn` here that delegates to the `pub(crate)` one in [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) (it is not a `pub use`). | flat | +| [`main.rs`](../engines/vllm/src/main.rs) | The `rocm-engine-vllm` binary's entry point; it only calls `rocm_engine_vllm::run_cli()`. | flat | +| [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) | The ROCm 10.x discovery table (`VLLM_ROCM_DISCOVER_BUILD_TABLE`, one row per supported `major.minor` line), its lookup and the series matcher. | domain | +| [`install.rs`](../engines/vllm/src/install.rs) | ROCm and vLLM install execution through `uv`, by the static-pin route or the ROCm 10.x discovery route. | domain | +| [`process.rs`](../engines/vllm/src/process.rs) | The resolve-model, launch, serve and stop handlers: recipe validation, device-policy and GPU-index resolution, the vLLM server spawn and launch environment, and readiness polling. | domain | +| [`repair.rs`](../engines/vllm/src/repair.rs) | Whether an already-resolvable managed vLLM environment violates vLLM's pinned dependencies and must be reinstalled (`assess_runtime_repair`), treating the SDK's torch build and a user-chosen torch stack (torch, torchvision, torchaudio, triton) with alignment disabled as intended. | domain | +| [`runtime.rs`](../engines/vllm/src/runtime.rs) | TheRock managed-runtime resolution and library and bin path derivation, plus the launch path's compat-shim writes (`ensure_therock_compat_shims`). | domain | +| [`startup_diagnostics.rs`](../engines/vllm/src/startup_diagnostics.rs) | What a failed startup tells the user (`startup_log_context`): the tail of the startup log, then the out-of-memory and import-failure hints that point at `rocm diagnose`. | domain | +| [`state.rs`](../engines/vllm/src/state.rs) | Service state file I/O, log tailing, and health and inference probing. | domain | +| [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) | Which vLLM wheels to install for a ROCm SDK version: the static build table, the install-route choice (`vllm_install_route`) and target resolution with the `ROCM_CLI_VLLM_ROCM_INDEX_URL` override. | domain | + +Boundaries: + +- **One discovery table.** [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) is the single list of the ROCm SDK versions installed by discovery: install routing, the fail-closed static fallback and launch-time environment setup all read its rows, and nothing keeps a second list in step with them. It is a leaf that imports no other module of the crate, which does not keep its readers apart: [`install.rs`](../engines/vllm/src/install.rs) imports [`runtime.rs`](../engines/vllm/src/runtime.rs) and [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs), and all three read the table. +- **Layering.** [`install.rs`](../engines/vllm/src/install.rs) reads [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) and [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) and asks [`repair.rs`](../engines/vllm/src/repair.rs) whether an install pass is needed; none of them imports [`install.rs`](../engines/vllm/src/install.rs). +- **Path derivation writes nothing.** [`runtime.rs`](../engines/vllm/src/runtime.rs)'s path derivation performs no filesystem writes; the compat-shim writes are `ensure_therock_compat_shims`, called only from [`process.rs`](../engines/vllm/src/process.rs), and the running-state payload never calls it. +- **Rendered, not stripped.** [`startup_diagnostics.rs`](../engines/vllm/src/startup_diagnostics.rs) re-renders the log tail line by line through `terminal::rendered_lines` instead of stripping it, because stripping deletes line-break tokens and welds a progress-bar repaint onto the next line. + +### `engines/aim` + +The AIM engine adapter (`rocm-engine-aim`) runs models as Docker containers instead of a host process, so it has no venv, wheel or runtime archive to manage. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`lib.rs`](../engines/aim/src/lib.rs) | Crate root: CLI and envelope dispatch, including the `builtin_*` entry points the `rocm` binary calls in-process, the detect and capabilities responses, and the items shared across the modules. | flat | +| [`main.rs`](../engines/aim/src/main.rs) | The `rocm-engine-aim` binary's entry point; it only calls `rocm_engine_aim::run_cli()`. | flat | +| [`docker.rs`](../engines/aim/src/docker.rs) | A thin wrapper around the `docker` CLI binary, with no Docker API client, and `container_name`, a service's deterministic Docker-safe container name. | flat | +| [`install.rs`](../engines/aim/src/install.rs) | Install for an engine whose images are self-contained: only a check that Docker is present and reachable. | flat | +| [`process.rs`](../engines/aim/src/process.rs) | The container run, readiness and stop lifecycle, including `serve_http`, which blocks for the container's lifetime and records how it ended. | flat | +| [`registry.rs`](../engines/aim/src/registry.rs) | AIM image metadata (`Env`/`Labels`), read from the local daemon once the image is pulled, with a best-effort Docker Hub v2 lookup as a pre-pull hint. | flat | +| [`state.rs`](../engines/aim/src/state.rs) | Service state file I/O, including the model name the container actually serves (`served_model_id`) and `endpoint_url`, the `/v1` base URL for a host and port. | flat | + +Boundaries: + +- **Supervised by `rocmd`.** [`process.rs`](../engines/aim/src/process.rs)'s `serve_http` is the process `rocmd` supervises: it records a crash or a vanished container as `failed` with a non-zero exit, so recovery runs. +- **The Docker Hub lookup is only a hint.** A failure of [`registry.rs`](../engines/aim/src/registry.rs)'s pre-pull lookup never blocks a launch. +- **Layering.** `container_name` lives in [`docker.rs`](../engines/aim/src/docker.rs) so that [`process.rs`](../engines/aim/src/process.rs) and [`state.rs`](../engines/aim/src/state.rs) both reach it there; [`process.rs`](../engines/aim/src/process.rs) imports [`state.rs`](../engines/aim/src/state.rs), and [`state.rs`](../engines/aim/src/state.rs) uses no [`process.rs`](../engines/aim/src/process.rs) item. + +### `xtask` — repository tooling (the `cargo xtask` command surface) + +Repository tasks, run through the workspace alias `cargo xtask `: release packaging and signing, generated-file checks, the pre-push and CI helpers, and the repository's source-scan contract tests. Nothing here ships in a release bundle. `cargo xtask check-architecture-doc` fails when a `.rs` file under `xtask/src` has no row here, with no helper exemption. + +| File | Responsibility | Class | +| --- | --- | --- | +| [`main.rs`](../xtask/src/main.rs) | The clap command tree and its dispatch: most `cargo xtask` commands dispatch to a module of their own. | flat | +| [`affected.rs`](../xtask/src/affected.rs) | `cargo xtask affected`: the crates a git range changes plus their transitive dependents, as cargo package flags, falling back to `--workspace` for any file it cannot attribute. | flat | +| [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) | Source-scan test pinning which call sites may resolve the `amd-smi` binary, so every launch passes the `/dev/kfd` pre-flight. | flat | +| [`architecture_doc.rs`](../xtask/src/architecture_doc.rs) | `cargo xtask check-architecture-doc`: fails when a production source file under `apps/`, `crates/` or `engines/`, or a `.rs` file under `xtask/src`, `tests/e2e-cucumber/src` or `tests/e2e-cucumber/tests`, has no link in this document. | flat | +| [`ascii_output_contract.rs`](../xtask/src/ascii_output_contract.rs) | Source-scan test that a string literal able to reach a redirectable stream or an on-disk file is ASCII-only. | flat | +| [`catalog.rs`](../xtask/src/catalog.rs) | `cargo xtask catalog`: regenerates or checks the published Doctor catalog manifest, rendered from the compiled catalog in `rocm-core`. | flat | +| [`chat_import_contract.rs`](../xtask/src/chat_import_contract.rs) | Source-scan test that the freeform planner and the leaf modules it shares with [`chat.rs`](../apps/rocm/src/chat.rs) never import `chat` in production code. | flat | +| [`coverage.rs`](../xtask/src/coverage.rs) | `cargo xtask coverage`: checks every crate's line coverage against its floor in [`coverage-floors.toml`](../coverage-floors.toml), or rewrites the floors with `--bless`. | flat | +| [`crate_edges.rs`](../xtask/src/crate_edges.rs) | `cargo xtask check-crate-edges`: fails on any first-party normal or build dependency edge missing from its `ALLOWLIST`. | flat | +| [`dash_text_edit_contract.rs`](../xtask/src/dash_text_edit_contract.rs) | Source-scan test that dash-TUI text fields edit through `apply_char_key` in [`widgets.rs`](../crates/rocm-dash-tui/src/ui/widgets.rs), not hand-rolled `push`/`pop` code. | flat | +| [`demos.rs`](../xtask/src/demos.rs) | `cargo xtask demos`: builds the demo binaries, starts the mock environment before VHS runs, and renders the documentation GIFs. | flat | +| [`e2e.rs`](../xtask/src/e2e.rs) | `cargo xtask e2e`: builds release `rocm` and `rocmd` with the E2E test features and runs the cucumber suite, forwarding extra arguments to cucumber. | flat | +| [`e2e_prewarm.rs`](../xtask/src/e2e_prewarm.rs) | `cargo xtask e2e-prewarm`: prepares the shared managed runtime the GPU E2E lanes serve against, refreshing it when the channel index publishes something newer or the required packages differ. | flat | +| [`e2e_report.rs`](../xtask/src/e2e_report.rs) | `cargo xtask e2e-report`: merges per-platform `report.json` files into one HTML report and a step-summary matrix, one platform per artifact subdirectory. | flat | +| [`engine_stdio_client_contract.rs`](../xtask/src/engine_stdio_client_contract.rs) | Source-scan test that only the shared stdio client in [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) puts a request on an engine child's stdin: a file that both pipes a child's stdin and names an engine envelope is reported unless it is a reviewed `ALLOWED` entry, and a stale entry fails too. | flat | +| [`env_mutation_contract.rs`](../xtask/src/env_mutation_contract.rs) | Source-scan test that every test which mutates the process environment serializes itself. | flat | +| [`eval_fixtures.rs`](../xtask/src/eval_fixtures.rs) | `cargo xtask eval-fixtures`: regenerates or checks the recorded `rocm` output the rocm-doctor skill evals replay, rendered from the compiled catalog. | flat | +| [`lychee.rs`](../xtask/src/lychee.rs) | `cargo xtask lychee`: runs lychee with [`lychee.toml`](../lychee.toml) over the working tree, the local counterpart of the `docs-links` CI job. | flat | +| [`manifest.rs`](../xtask/src/manifest.rs) | `cargo xtask manifest`: regenerates or checks the Cargo dependency table in [`MANIFEST.md`](../MANIFEST.md) from `cargo metadata`. | flat | +| [`module_size.rs`](../xtask/src/module_size.rs) | `cargo xtask check-module-size`: fails when a source file exceeds `MAX_PRODUCTION_LINES` production lines, or when a `.rs` file under a crate's `src/` is declared by no `mod`. | flat | +| [`package.rs`](../xtask/src/package.rs) | `cargo xtask package`: builds the release bundle directory from built binaries, its archive, the SHA-256 sidecar and, when a key is configured, a detached signature. | flat | +| [`paths.rs`](../xtask/src/paths.rs) | Workspace paths shared by the tasks: the workspace root, the active `target/` directory, release binary locations and platform executable names. | flat | +| [`powershell.rs`](../xtask/src/powershell.rs) | `cargo xtask powershell-lint`: runs PSScriptAnalyzer over `.ps1` files through `pwsh` or Windows PowerShell, which also reports syntax errors. | flat | +| [`prepush.rs`](../xtask/src/prepush.rs) | `cargo xtask pre-push-check`: runs one pre-push cargo step over only the crates the push affects, falling back to the whole workspace when that is unknown. | flat | +| [`prepush_stamp.rs`](../xtask/src/prepush_stamp.rs) | Pass stamps for `pre-push-check`: records a step that passed on a clean tree, keyed by tree hash, arguments, toolchain and environment, and skips it next time. | flat | +| [`proptest_regressions_contract.rs`](../xtask/src/proptest_regressions_contract.rs) | Source-scan test that every saved proptest seed file maps back to a source file that still holds a `proptest!` block. | flat | +| [`release_readiness.rs`](../xtask/src/release_readiness.rs) | `cargo xtask release-readiness`: the nightly GO/NO-GO verdict and status table from git, `gh`, job results and the E2E report, plus the facts the narrative step drafts from. | flat | +| [`report_mailbox.rs`](../xtask/src/report_mailbox.rs) | Reads Doctor reports out of exported `.eml` mail, keeping only the headers the rollup needs and reducing `Date` to a week. | flat | +| [`report_rollup.rs`](../xtask/src/report_rollup.rs) | `cargo xtask report-rollup`: counts received Doctor reports by problem group and week from exported mail or JSON bodies; it reads and prints, and sends nothing. | flat | +| [`rocm_releases.rs`](../xtask/src/rocm_releases.rs) | Checked-in table resolving the E2E `sdk_version` labels (`current`, `n-1`, `n-2`) to TheRock package versions, with no pin for `current`, updated by hand. | flat | +| [`rust_source.rs`](../xtask/src/rust_source.rs) | Lexical helpers for the source-scan tests, chiefly `strip_literals_and_comments`, so every contract skips literals and comments the same way. | flat | +| [`signing.rs`](../xtask/src/signing.rs) | The `keygen`, `sign` and `verify` commands: the file I/O layer over the RSA release-artifact signing in `rocm-core`. | flat | +| [`smoke.rs`](../xtask/src/smoke.rs) | `cargo xtask smoke`: the local no-fallback gate, which builds the binaries and runs them against a throwaway state root, asserting their first-run output and that every GPU-required path refuses instead of falling back to CPU. Its check table is data, pinned per host by its tests. | flat | +| [`tpn.rs`](../xtask/src/tpn.rs) | `cargo xtask tpn`: regenerates or checks [`THIRD_PARTY_NOTICES.txt`](../THIRD_PARTY_NOTICES.txt) through cargo-about with the committed config and template. | flat | +| [`verify_commits.rs`](../xtask/src/verify_commits.rs) | `cargo xtask verify-commits`: checks that every commit in a range is signed and carries a DCO `Signed-off-by` trailer, or only that signing is configured. | flat | +| [`verify_pinned_keys.rs`](../xtask/src/verify_pinned_keys.rs) | `cargo xtask verify-pinned-keys`: checks that the public keys pinned in the installers and the CLI match the canonical keys under `docs/keys/`. | flat | +| [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) | Source-scan tests over the GitHub workflow files and CI scripts: lane placement and concurrency groups, runner labels, timeouts, permissions and other CI invariants. | flat | + +Boundaries: + +- The eight `*_contract.rs` files define no command: each is one `#[cfg(test)]` module run by `cargo test -p xtask`, and [`rust_source.rs`](../xtask/src/rust_source.rs) is declared test-only in [`main.rs`](../xtask/src/main.rs). The pull-request test selection narrows to the crates a change touches, so a change confined to another crate may not select them there; pushes to `main` and the merge queue run the whole workspace. +- [`affected.rs`](../xtask/src/affected.rs) is the one crate selection used by both CI's affected-crates job and [`prepush.rs`](../xtask/src/prepush.rs), which only widens it with `CROSS_CRATE_READS`; a change to how files are attributed to crates changes both. +- [`paths.rs`](../xtask/src/paths.rs) defines the workspace root, the active target directory and binary file names, honouring `CARGO_TARGET_DIR` and a cross-compilation target; a new task uses it rather than recomputing them. + +### `tests/e2e-cucumber` — black-box end-to-end suite + +Gherkin `.feature` files under `features/` backed by Rust step functions that drive the real `rocm` (and, where a scenario needs it, `rocmd`) binary against a mock server or real GPU serving; see the crate [README](../tests/e2e-cucumber/README.md) and [testing.md](testing.md). The suite runs through `cargo xtask e2e`. `cargo xtask check-architecture-doc` fails when a `.rs` file under `src/` or `tests/` here has no row, with no helper exemption. + +#### `src/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`src/lib.rs`](../tests/e2e-cucumber/src/lib.rs) | Library root for the test targets and the demo binary: declares the helper modules, re-exports `e2e-report` as `report`, and holds small shared helpers such as `cli_failure_report`. | flat | +| [`src/binary_path.rs`](../tests/e2e-cucumber/src/binary_path.rs) | Pins a relative `ROCM_CLI_BINARY` path before a scenario runs the child from another directory, and leaves a bare name for `PATH` to resolve. | flat | +| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | +| [`src/chat_request.rs`](../tests/e2e-cucumber/src/chat_request.rs) | The chat round trip against a served endpoint (model discovery, then one chat completion), run on its own task so its timeout measures the server. | flat | +| [`src/dash_screen.rs`](../tests/e2e-cucumber/src/dash_screen.rs) | Pure matching over captured dashboard screens and config files, so the logic the dashboard scenarios depend on has unit tests. | flat | +| [`src/device_memory.rs`](../tests/e2e-cucumber/src/device_memory.rs) | Which device memory pool (VRAM, or GTT-backed memory on an APU) the drain-wait between serve scenarios judges, how it is sampled, and the free-memory floor. | flat | +| [`src/expectation.rs`](../tests/e2e-cucumber/src/expectation.rs) | Resolves each scenario on this host to expected-pass, expected-fail or skip from its tags, the capability probe and the [`expectations.toml`](../tests/e2e-cucumber/expectations.toml) matrix. | flat | +| [`src/fixture_binary.rs`](../tests/e2e-cucumber/src/fixture_binary.rs) | Locates a fake-subprocess `[[bin]]` of this crate next to the running test binary. | flat | +| [`src/harness_diagnostics.rs`](../tests/e2e-cucumber/src/harness_diagnostics.rs) | Records a panic that escapes the cucumber run, which cucumber's silenced panic hook would hide, and the state of the standard streams, to stderr and a results-dir log. | flat | +| [`src/http_server.rs`](../tests/e2e-cucumber/src/http_server.rs) | Shared `axum` test-server plumbing: bind an ephemeral loopback port, serve a router, shut down on drop. | flat | +| [`src/installer_fixture.rs`](../tests/e2e-cucumber/src/installer_fixture.rs) | `with_pinned_release_keys`: a copy of the PowerShell installer with its two pinned-key assignments replaced, failing on any shape it does not expect. | flat | +| [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat | +| [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat | +| [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat | +| [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat | +| [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat | +| [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat | +| [`src/monotonic_clock.rs`](../tests/e2e-cucumber/src/monotonic_clock.rs) | Rebuilds a non-decreasing timeline from cucumber's wall-clock event stamps so the JSON and JUnit writers never panic on a clock that stepped back. | flat | +| [`src/paced_download.rs`](../tests/e2e-cucumber/src/paced_download.rs) | Loopback server that serves one named file in delayed chunks, so a PTY scenario can see intermediate download-progress frames. | flat | +| [`src/palette_rows.rs`](../tests/e2e-cucumber/src/palette_rows.rs) | Which destination rows of the dashboard's command palette a rendered screen still lacks. | flat | +| [`src/panic_capture.rs`](../tests/e2e-cucumber/src/panic_capture.rs) | Panic-message extraction and mutex-poison recovery for drivers that run background threads. | flat | +| [`src/planted_path.rs`](../tests/e2e-cucumber/src/planted_path.rs) | The `PATH` handed to the `rocm` child when a scenario plants its own `rocminfo`, dropping runner entries that hold one. | flat | +| [`src/polled_deadline.rs`](../tests/e2e-cucumber/src/polled_deadline.rs) | `PolledDeadline`: a retry-loop deadline that counts only the time the loop was polled, with a wall-clock cap. | flat | +| [`src/powershell_text_rule.rs`](../tests/e2e-cucumber/src/powershell_text_rule.rs) | The text rule for the Windows lines `rocm fix` and `rocm diagnose` print, a copy of the rule in `rocm-core` checked against a shared fixture table. | flat | +| [`src/quit_and_wait.rs`](../tests/e2e-cucumber/src/quit_and_wait.rs) | Retry loop for quitting a TUI that may open a quit-confirm prompt first. | flat | +| [`src/reader_failure.rs`](../tests/e2e-cucumber/src/reader_failure.rs) | Failure state shared between a background terminal reader and its waiters. | flat | +| [`src/runtime_root.rs`](../tests/e2e-cucumber/src/runtime_root.rs) | `check_runtime_root_not_nested`: judges the active runtime folder `rocm examine` prints and rejects a doubly nested `runtimes/wheel` path. | flat | +| [`src/send_until.rs`](../tests/e2e-cucumber/src/send_until.rs) | Retry loop for sending an idempotent terminal input until its effect appears. | flat | +| [`src/serve_log.rs`](../tests/e2e-cucumber/src/serve_log.rs) | Reads back a managed serve's service log and builds the failure report for a serve step, quoting the log tail. | flat | +| [`src/serve_plan.rs`](../tests/e2e-cucumber/src/serve_plan.rs) | Explains why a `rocm serve` run printed no `resolved model:` line by keeping stderr's refusal alongside stdout. | flat | +| [`src/shared_runtime.rs`](../tests/e2e-cucumber/src/shared_runtime.rs) | Picks which runtime a scenario activates out of the shared pre-warmed runtime tree, which may hold more than one. | flat | +| [`src/stand_in_uname.rs`](../tests/e2e-cucumber/src/stand_in_uname.rs) | The shell script a scenario plants as a stand-in `uname`: `-v` answers a fixed string, `-r` runs a fragment the scenario chooses, and every call is logged so a scenario can count how often the kernel release was asked for. | flat | +| [`src/stray_process.rs`](../tests/e2e-cucumber/src/stray_process.rs) | What scenario teardown does about a process a scenario started and recorded: only a process that is still the one recorded is ended, never one that has since exited or whose PID another process has taken. Decided apart from the system reads, so a table test covers each case on every host. | flat | +| [`src/tasklist.rs`](../tests/e2e-cucumber/src/tasklist.rs) | What `tasklist` says about one process, read so that a failed query cannot pass for an answer: the exit status is judged before the text, so a run that did not succeed gives no answer rather than "the process exited". Pure, so a table test covers every case on every host. | flat | +| [`src/terminal_size.rs`](../tests/e2e-cucumber/src/terminal_size.rs) | Validates a requested terminal size and applies it to the emulated screen before the real PTY. | flat | +| [`src/torch_alignment_outcome.rs`](../tests/e2e-cucumber/src/torch_alignment_outcome.rs) | Which `torch_alignment:` outcomes the runtime scenarios accept as settled, including `not_applicable` when the engine declares no torch pin, and the verdict on importing the engine's stack. | flat | + +#### `src/bin/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`src/bin/fake-docker.rs`](../tests/e2e-cucumber/src/bin/fake-docker.rs) | A `docker` stand-in that always fails, optionally logging its arguments, for the AIM Docker-unavailable scenarios. | flat | +| [`src/bin/fake-python.rs`](../tests/e2e-cucumber/src/bin/fake-python.rs) | A stand-in for the runtime's Python that answers the ComfyUI install's torch-stack probe and post-install GPU check. | flat | +| [`src/bin/fake-tailscale.rs`](../tests/e2e-cucumber/src/bin/fake-tailscale.rs) | A Tailscale CLI stand-in that serves the status document named by `FAKE_TAILSCALE_STATUS`, for the `rocm remote` scenarios. | flat | +| [`src/bin/fake-uv.rs`](../tests/e2e-cucumber/src/bin/fake-uv.rs) | A `uv` stand-in whose output, exit code, argument log and stdin handling are set by `FAKE_UV_*` variables, for the ComfyUI scenarios. | flat | +| [`src/bin/rocm-demo-env.rs`](../tests/e2e-cucumber/src/bin/rocm-demo-env.rs) | Standalone mock environment for the VHS demos: starts the mock server, plants a service record and prints the env vars `cargo xtask demos` exports. | flat | + +#### `tests/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`tests/driver_host_guard.rs`](../tests/e2e-cucumber/tests/driver_host_guard.rs) | Ordinary test that the Windows driver-install steps run the host guard before calling the CLI, checked against the step source. | flat | +| [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) | The cucumber test binary: the `E2eWorld` state and its cleanup on drop, the `run_rocm` helpers, the runner `main`, and reconciliation of results with expectations. | flat | +| [`tests/feature_naming.rs`](../tests/e2e-cucumber/tests/feature_naming.rs) | Drift guard, in the ordinary `cargo test` set, for the `-` scenario names and ids in the `.feature` files. | flat | +| [`tests/skill_reference.rs`](../tests/e2e-cucumber/tests/skill_reference.rs) | Drift guard, in the ordinary `cargo test` set, that the catalog table in [`reference.md`](../skills/rocm-doctor/reference.md) uses the vocabulary `rocm fix` prints. | flat | + +#### `tests/e2e/` + +| File | Responsibility | Class | +| --- | --- | --- | +| [`tests/e2e/artifact_steps.rs`](../tests/e2e-cucumber/tests/e2e/artifact_steps.rs) | Steps for [`artifact_prefetch.feature`](../tests/e2e-cucumber/features/artifact_prefetch.feature): a signed direct-download artifact, its approved prefetch, and a cache marker that cannot be published. | flat | +| [`tests/e2e/automations_steps.rs`](../tests/e2e-cucumber/tests/e2e/automations_steps.rs) | Steps for [`automations.feature`](../tests/e2e-cucumber/features/automations.feature): `rocm automations` enable, disable and listing, with the daemon planted as already running so none is spawned. | flat | +| [`tests/e2e/bench_steps.rs`](../tests/e2e-cucumber/tests/e2e/bench_steps.rs) | Steps for [`bench.feature`](../tests/e2e-cucumber/features/bench.feature): the chat route `rocm bench load` posts to, and an all-failed run reported as a failure. | flat | +| [`tests/e2e/bootstrap_steps.rs`](../tests/e2e-cucumber/tests/e2e/bootstrap_steps.rs) | Steps for [`bootstrap.feature`](../tests/e2e-cucumber/features/bootstrap.feature): the non-interactive fallback of `rocm bootstrap setup`, run piped. | flat | +| [`tests/e2e/chat_steps.rs`](../tests/e2e-cucumber/tests/e2e/chat_steps.rs) | Steps for [`chat.feature`](../tests/e2e-cucumber/features/chat.feature) and [`networking.feature`](../tests/e2e-cucumber/features/networking.feature): a served mock model, service listing, and chat requests through the API and the CLI. | flat | +| [`tests/e2e/comfyui_steps.rs`](../tests/e2e-cucumber/tests/e2e/comfyui_steps.rs) | Steps for [`comfyui.feature`](../tests/e2e-cucumber/features/comfyui.feature): `rocm comfyui install` against a planted runtime with the `fake-python` and `fake-uv` stand-ins, and runtime selection. | flat | +| [`tests/e2e/config_steps.rs`](../tests/e2e-cucumber/tests/e2e/config_steps.rs) | Steps for [`config.feature`](../tests/e2e-cucumber/features/config.feature): `rocm config` mutations against the scenario's isolated config dir. | flat | +| [`tests/e2e/dash_steps.rs`](../tests/e2e-cucumber/tests/e2e/dash_steps.rs) | Steps for [`dash.feature`](../tests/e2e-cucumber/features/dash.feature) and the TUI scenarios of other features: the dashboard and chat TUI driven through a pseudo-terminal. | flat | +| [`tests/e2e/dependency_guard_steps.rs`](../tests/e2e-cucumber/tests/e2e/dependency_guard_steps.rs) | Steps for [`dependency_guard.feature`](../tests/e2e-cucumber/features/dependency_guard.feature): a stand-in package manager proposes removing the ROCm stack, and the CLI must refuse. | flat | +| [`tests/e2e/diagnose_steps.rs`](../tests/e2e-cucumber/tests/e2e/diagnose_steps.rs) | Steps for [`diagnose.feature`](../tests/e2e-cucumber/features/diagnose.feature): `rocm diagnose` matches and `rocm fix` listings, previews, printed verify commands and failing fixes. | flat | +| [`tests/e2e/driver_steps.rs`](../tests/e2e-cucumber/tests/e2e/driver_steps.rs) | Steps for [`driver_install.feature`](../tests/e2e-cucumber/features/driver_install.feature): previewing and approving GPU driver installation plans on WSL, Linux and Windows hosts. | flat | +| [`tests/e2e/engines_steps.rs`](../tests/e2e-cucumber/tests/e2e/engines_steps.rs) | Steps for [`engine_shell.feature`](../tests/e2e-cucumber/features/engine_shell.feature): `rocm engines shell` under a pseudo-terminal, since the defect shows only in what the terminal renders. | flat | +| [`tests/e2e/examine_steps.rs`](../tests/e2e-cucumber/tests/e2e/examine_steps.rs) | Steps for [`examine.feature`](../tests/e2e-cucumber/features/examine.feature): `rocm examine`, version and help output, engine listing and the driver plan preview. | flat | +| [`tests/e2e/host_guard.rs`](../tests/e2e-cucumber/tests/e2e/host_guard.rs) | The host check the Windows driver-install steps run before any CLI call, kept free of the cucumber world so a separate test can include it. | flat | +| [`tests/e2e/lemonade_install_steps.rs`](../tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs) | Steps for [`lemonade_installs.feature`](../tests/e2e-cucumber/features/lemonade_installs.feature): what the user is told about the Lemonade copies on disk, from planted trees and install records. | flat | +| [`tests/e2e/lifecycle_steps.rs`](../tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs) | Steps for [`install_lifecycle.feature`](../tests/e2e-cucumber/features/install_lifecycle.feature): `xtask package`, the root installer and the installed binaries, from install through uninstall. | flat | +| [`tests/e2e/logs_steps.rs`](../tests/e2e-cucumber/tests/e2e/logs_steps.rs) | Steps for [`logs.feature`](../tests/e2e-cucumber/features/logs.feature): `rocm logs` searches over command-log files planted inside the tail window it reads. | flat | +| [`tests/e2e/model_catalog_steps.rs`](../tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs) | Steps for [`model_catalog.feature`](../tests/e2e-cucumber/features/model_catalog.feature): the verbose catalog's AIM entry, its Docker-unavailable flag and its staleness warning. | flat | +| [`tests/e2e/planner_steps.rs`](../tests/e2e-cucumber/tests/e2e/planner_steps.rs) | Steps for the planning-assistant scenarios in [`chat.feature`](../tests/e2e-cucumber/features/chat.feature), with a scripted mock proving a request is handed over once and its plan is the one shown. | flat | +| [`tests/e2e/remote_steps.rs`](../tests/e2e-cucumber/tests/e2e/remote_steps.rs) | Steps for [`remote.feature`](../tests/e2e-cucumber/features/remote.feature): `rocm remote` against the `fake-tailscale` stand-in and a status document the step writes. | flat | +| [`tests/e2e/runtime_lifecycle_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs) | Steps for [`runtime_lifecycle.feature`](../tests/e2e-cucumber/features/runtime_lifecycle.feature): `rocm runtimes` activate, rollback, uninstall and import over planted read-only tarball runtimes. | flat | +| [`tests/e2e/runtime_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_steps.rs) | Steps for [`runtime_setup.feature`](../tests/e2e-cucumber/features/runtime_setup.feature): SDK install previews and installs, runtime folders reached through a link, and the managed runtime they leave active. | flat | +| [`tests/e2e/service_cleanup_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs) | Steps for [`service_record_cleanup.feature`](../tests/e2e-cucumber/features/service_record_cleanup.feature): `rocm services remove` and `prune` over planted service records, asserting on the files that survive, and [`service_record_unqueryable.feature`](../tests/e2e-cucumber/features/service_record_unqueryable.feature): a record whose process cannot be queried (the `e2e-test-hooks` build reads `ROCM_E2E_UNQUERYABLE_PIDS`) is stopped as unverified, refused by `remove`, and cleared by `remove --force` only when no recorded process is confirmed running. | flat | +| [`tests/e2e/service_stop_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_stop_steps.rs) | Steps for [`service_stop.feature`](../tests/e2e-cucumber/features/service_stop.feature) and [`engine_ipc.feature`](../tests/e2e-cucumber/features/engine_ipc.feature): `rocm services stop` and the daemon's `stop_server` tool against a planted service record, and how the daemon reads what an engine child prints, with a stand-in `rocm` staged beside a copy of `rocmd`; and the confirmed stop against a real recorded process, a recorded PID that now belongs to a different process, and a parent that has started a child. | flat | +| [`tests/e2e/service_unknown_id_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_unknown_id_steps.rs) | Steps for [`service_unknown_id.feature`](../tests/e2e-cucumber/features/service_unknown_id.feature): commands that name a local server with no record, checked against the listing they point to. | flat | +| [`tests/e2e/serving_steps.rs`](../tests/e2e-cucumber/tests/e2e/serving_steps.rs) | Steps for [`model_serving.feature`](../tests/e2e-cucumber/features/model_serving.feature) and the serve steps other features reuse: `rocm serve` on each engine, ports, managed services and inference. | flat | +| [`tests/e2e/skill_steps.rs`](../tests/e2e-cucumber/tests/e2e/skill_steps.rs) | Steps for [`rocm_doctor_skill.feature`](../tests/e2e-cucumber/features/rocm_doctor_skill.feature): drive `rocm` through the sequence the rocm-doctor skill prescribes and check its literal claims. | flat | +| [`tests/e2e/storage_steps.rs`](../tests/e2e-cucumber/tests/e2e/storage_steps.rs) | Steps for [`storage.feature`](../tests/e2e-cucumber/features/storage.feature) and [`local_server_records.feature`](../tests/e2e-cucumber/features/local_server_records.feature): `rocm storage report` and the confirmation gate on `remove-downloads`. | flat | +| [`tests/e2e/symptom_file_steps.rs`](../tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs) | Steps for the `rocm diagnose --symptom-file` scenarios of [`diagnose.feature`](../tests/e2e-cucumber/features/diagnose.feature) (diagnose-59 to diagnose-61): a hostile paste piped through each available shell, a paste typed at a pseudo-terminal through [`tui_driver.rs`](../tests/e2e-cucumber/tests/e2e/tui_driver.rs), and error files that cannot be read or are not text. | flat | +| [`tests/e2e/therock_steps.rs`](../tests/e2e-cucumber/tests/e2e/therock_steps.rs) | Steps for [`therock_next_generation.feature`](../tests/e2e-cucumber/features/therock_next_generation.feature) and [`download_progress_pty.feature`](../tests/e2e-cucumber/features/download_progress_pty.feature): release layouts served from a loopback server, and one real GPU install. | flat | +| [`tests/e2e/tui_driver.rs`](../tests/e2e-cucumber/tests/e2e/tui_driver.rs) | Pseudo-terminal driver for the dash and chat TUI: spawns the real binary under `portable-pty`, sends keys and reads the emulated `vt100` screen. | flat | +| [`tests/e2e/uninstall_steps.rs`](../tests/e2e-cucumber/tests/e2e/uninstall_steps.rs) | Steps for [`uninstall.feature`](../tests/e2e-cucumber/features/uninstall.feature): `rocm uninstall` confined to the scenario root with `--keep-binaries` and redirected caches. | flat | +| [`tests/e2e/update_steps.rs`](../tests/e2e-cucumber/tests/e2e/update_steps.rs) | Steps for [`update.feature`](../tests/e2e-cucumber/features/update.feature): the report-only `rocm update`, its feed status, and a loopback catalog where runtimes are planted. | flat | + +Boundaries: + +- **Test targets.** The `e2e` target behaves differently from the rest: + - [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) declares every `tests/e2e/` module in an inline `mod e2e`, and its target is `harness = false` and `test = false`, so the default test run skips it and `cargo xtask e2e` is how it normally runs. + - A plain `#[test]` inside that target never runs: pure logic that needs unit tests goes in `src/`, or is included by a separate test target through `#[path]`, as [`tests/driver_host_guard.rs`](../tests/e2e-cucumber/tests/driver_host_guard.rs) includes [`tests/e2e/host_guard.rs`](../tests/e2e-cucumber/tests/e2e/host_guard.rs). +- The library target depends on no product crate. `rocm-core` is a dev-dependency only, used by unit tests and the `e2e` target, so library code repeats what it needs (as `device_memory::bounded_stdout` does) rather than add an edge `cargo xtask check-crate-edges` would flag. +- The report generator lives in [`crates/e2e-report`](../crates/e2e-report/src/lib.rs) so `xtask` can use it without this crate's dependency tree; [`src/lib.rs`](../tests/e2e-cucumber/src/lib.rs) re-exports it as `report`. +- **Copies of product logic that drift.** A product change here means a change in the suite: + - [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) re-implements the CLI's serve-engine choice and the AMD GPU count. + - [`src/powershell_text_rule.rs`](../tests/e2e-cucumber/src/powershell_text_rule.rs) copies the rule in [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs); both are tested against [`powershell_text_rule_cases.txt`](../tests/e2e-cucumber/fixtures/powershell_text_rule_cases.txt). + - `cargo xtask demos` parses what [`src/bin/rocm-demo-env.rs`](../tests/e2e-cucumber/src/bin/rocm-demo-env.rs) prints. + +## History + +The crate roots (`apps/rocm`'s [`main.rs`](../apps/rocm/src/main.rs), `rocm-core`'s [`lib.rs`](../crates/rocm-core/src/lib.rs), `apps/rocmd`'s [`lib.rs`](../apps/rocmd/src/lib.rs) and others) once held most of their crate in one file and were split into the modules mapped above. Use `git log --follow ` for the history of a single module. This document describes only the current structure. diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index cdc89135e3..82f89390b2 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,14 +36,76 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below (`e2e-gpu-strix-ubuntu` runs both its `[release, nightly]` channel legs per PR, ROCMAI-125) | No — absent from the required-status-check list | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The same per-PR lanes | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only rung instead of being removed outright. +A pull request now runs on exactly one Linux GPU node and one Windows GPU +pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and +the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo +Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in +the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane +in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue +nightly's `w7900-dual` lane (below; advisory), not on `push`, `release/**` or +dispatch. Its first run, the 11-13 queue nightly +([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), got capacity at once and finished `Done`, exit 0. + +[`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a +shared GPU queue, one per GPU model, so no runner holds a GPU for the length +of a job, and publishes the result; +[`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) publishes a +finished run again. They cover the Linux GPU nightly on both the release and +the nightly channel; native Windows and WSL stay on their dedicated runners, +since the queue runs Linux pods. See +[nightly-on-the-queue.md](nightly-on-the-queue.md). + +## Nightly stream canary + +[`nightly-canary.yml`](../.github/workflows/nightly-canary.yml) runs daily at +07:23 UTC (and on `workflow_dispatch`) on GitHub-hosted `ubuntu-24.04` and +`windows-2022`. It needs no GPU and installs nothing. For each of +`gfx1100` and `gfx1200` (plus `gfx950`, on Linux only: the Windows run does not +probe an Instinct target) it runs +`rocm install sdk --channel nightly --family --dry-run` and checks the +plan with [`nightly_canary.py`](../scripts/nightly_canary.py): + +- the channel, source URL and source layout generation are the + current (next-layout) nightly ones, which an unpinned nightly install + resolves to, read from [`source.rs`](../apps/rocm/src/therock/source.rs) + rather than copied, so the earlier multi-arch endpoint, the stable stream or + another channel taking part in nightly resolution fails the run. The probes + name exact arches because that layout refuses a grouped family; +- the selected version is a dated nightly and its build is at most three days + old, which is how a stream that stopped publishing, or a resolver stuck on an + old train, shows up. No expected version is hard-coded; +- when the plan pins the package stack, `rocm` and every framework package's + `+rocm` label name the one selected build, so a mixed-build stack fails; +- the dry run itself succeeds, so an unknown layout, a redirect or a missing or + incomplete build fails with the resolver's own message. + +It also reads the index's `rocm` listing and warns, without failing, when a +newer build is listed than the one selected: the resolver skips a build that is +not complete yet. + +What it does not cover: the periodic hardware lane that installs the resolved +nightly and runs a real GPU operation (the per-channel legs of `nightly.yml` +are the closest thing today), and whether a newer listed build is complete. + +**Who is told.** A failed scheduled run emails the person who last changed the +workflow's schedule and shows red on the Actions tab; nothing else routes the +alert yet. Whoever owns the nightly lanes (the `nightly.yml` owner) is the +owner of this one. + +**Rerun.** Actions, "Nightly stream canary", "Run workflow". Locally, with a +built `rocm`: `python scripts/nightly_canary.py --rocm target/release/rocm` +(add `--family gfx1100` to probe one arch; `--self-test` runs the rules +against fixtures without the network). + + The release-candidate rung's `release/**` trigger merged in PR #415, so `e2e-selfhosted.yml` on `main` already triggers on `push: branches: [main, "release/**"]`. On that trigger, every self-hosted GPU job additionally pins @@ -66,14 +128,19 @@ separate tier flag or tag filter to maintain. | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | -| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | AMD Radeon PRO W7900 (gfx1100, 48GB) on Linux | self-hosted `[self-hosted, linux, w7900]` | + +On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` is skipped; it still +runs on `push` and dispatch. Over 7 days 35% of its merge-queue jobs (98 of 281) +were cancelled at the 45 min cap; the check is not required and the queue does +not wait for it. On a +PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo +pool). `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, -nightly]` axis — two concurrent legs on the same job, channel-suffixed -artifact names — so the per-PR smoke gate proves the nightly channel boots on -at least one platform without adding the axis (and its wall-clock cost) to -every per-PR lane. This mirrors `nightly.yml`'s channel matrix from -ROCMAI-429, applied to a single per-PR lane rather than all of them. +nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact +names, so that lane proves the nightly package channel boots. It no longer runs +on `pull_request`, so on a PR no lane exercises the nightly channel; it runs on +`push`, dispatch and in `nightly.yml`'s channel matrix (ROCMAI-429). All three Strix Halo lanes run on the AMD Ryzen DevLab Dispatch pool: a fresh runner is registered per job and destroyed after, opt-in only via the @@ -88,22 +155,15 @@ cloning the checkout into the guest's native filesystem to avoid building against the slow DrvFs mount. The distro is deliberately left without ROCm's WSL driver bridge — see "What the WSL distro needs" below. -This migration is scoped to the per-PR lanes in this table, plus the nightly -WSL and Windows Strix lanes below — not every nightly lane. `nightly.yml`'s -Ubuntu Strix lane (`e2e-gpu-nightly-strix`) still pins the `native` label and -runs on the same static, always-on host these per-PR lanes moved off of — -that label still means what it always did there: `native` disambiguates this -host from any runner that might carry a bare `wsl` label alongside -`strix-halo` (the static WSL runner this repo used before the WSL lanes below -moved to the ephemeral pool). Whether that runner remains registered is not -something this migration changes either way, since nothing here targets a -`wsl`-labeled runner anymore. `nightly.yml`'s `e2e-wsl-nightly` and -`e2e-gpu-nightly-strix-windows`, however, both moved onto the same DevLab -Dispatch pool as the per-PR `e2e-wsl`/`e2e-gpu-strix-windows` lanes above, for -the same reason: neither needs a dedicated `native`- or `wsl`-labeled runner, -so there is nothing pool-incompatible about either. Moving the remaining -`native` Ubuntu nightly lane onto the pool as well is a separate, not-yet-made -decision. +The Strix Halo lanes in this table run on the DevLab Dispatch pool, and so do +their nightly counterparts: `nightly.yml`'s `e2e-gpu-nightly-strix` (Ubuntu), +`e2e-gpu-nightly-strix-windows` and `e2e-wsl-nightly` all use the same +`runs-on` labels as `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows` and +`e2e-wsl` (pinned by `nightly_devlab_dispatch_lanes_share_the_per_pr_pool_labels` +in `xtask/src/workflow_contract.rs`). The Ubuntu nightly lane used to pin the +static `native` label; that host is offline, so the lane sat queued until the +run cap and never ran. Whether a `native`- or `wsl`-labeled runner remains +registered no longer matters: nothing targets one. Every label in a `runs-on` must narrow the pool to one kind of hardware. In particular the MI300X lane pins `mi300x` rather than `amd-gpu`: `amd-gpu` is @@ -113,12 +173,24 @@ mixed-silicon pool. `every_self_hosted_lane_pins_a_hardware_label` in the lane simply passes on the wrong GPU and reports under the label it was named for. +No workflow other than [`queue-run.yml`](../.github/workflows/queue-run.yml) +and [`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) may name the +`queue-report` label. +The shared-queue publishing runner carries that label alone, without GitHub's +default `self-hosted`, OS and architecture labels, so a job reaches it only by +naming that label, and `queue_nightly_publisher_holds_no_credential` in +[`workflow_contract.rs`](../xtask/src/workflow_contract.rs) fails if another workflow on `main` names it +literally. A label assembled from an expression is not seen; the rule guards +against mistakes, not evasion. The runner is registered to this repository +alone, so no runner group reaches it. See +[nightly-on-the-queue.md](nightly-on-the-queue.md). + `e2e` is the blocking, GitHub-hosted mock job: `@requires-gpu` scenarios resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, -`e2e-wsl`, and `e2e-gpu-w7900`) run on AMD GPU systems, so they +and `e2e-wsl`) run on AMD GPU systems, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as described below. @@ -178,7 +250,8 @@ not applicable, instead of failing them on a premise the host cannot meet. Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` covers the mock platform; -`e2e-selfhosted.yml`'s `e2e-report` covers the per-PR GPU platforms; +`e2e-selfhosted.yml`'s `e2e-report` covers its self-hosted GPU platforms (a +pull request runs only the MI300X and Windows Strix Halo pool lanes among them); `nightly.yml`'s `e2e-report-nightly` covers a strict superset of those platforms (it also runs `e2e-gpu-rad3` and `e2e-gpu-mi350p`, demoted from per-PR to nightly-only per ROCMAI-125) with the `@nightly` scenarios @@ -200,6 +273,37 @@ setup: enable the wiki and save its first page in the GitHub web UI, because GitHub does not create the wiki's git repository until then and the job fails with that explanation until it exists. +Each row of that page is one lane and package channel. Its Platform cell lists +the ROCm, vLLM and lemonade versions the run served on, then the exact GFX +target and the package source (index URL and source layout generation) the +active runtime was installed from. Both are read from that runtime's own +registry record by the harness (`collect_versions` in +[`capability.rs`](../tests/e2e-cucumber/src/capability.rs)), never from a +workflow label, so a lane running on a legacy or different source shows that +source instead of the one the workflow meant. The harness reads the record with its own copy of the CLI's lookup (`wheel_composition_device_target` and the source-layout fallback), tied to it only by a comment, so a change to the record shape needs the harness updated by hand. A row omits the target or the +source when it has no managed runtime, when its runtime's record predates the +field, when the record carries none (a tarball runtime has no index URL, and a +wheel runtime whose device target was not resolved exactly has no target), or, +for the source, when a legacy record has no index URL. What +a lane skips on a given platform follows from the `@requires-*` tags above. + +To reproduce a lane's result locally, pre-warm a runtimes tree from the same +target and source the Platform cell names and export `E2E_SHARED_RUNTIMES_DIR` +to it (see [the shared pre-warmed runtime](#the-shared-pre-warmed-runtime)). +Without that variable each scenario runs in its own isolated data directory, +sees no runtime, and installs one itself at the default channel (a multi-GiB +download that also occupies the GPU), so the run does not use the runtime you +installed. Install with `rocm install sdk --channel release|nightly`; +`--family` takes a family label such as `gfx110X-all`, not the exact target, and +no flag selects the package source, so treat the cell as what to compare +against, not what to pass. Then run a scenario that exercises the runtime, for +example `cargo xtask e2e -- -n "serve-05"` on a vLLM lane or `-n "serve-07"` on +a lemonade lane (the Strix Halo, R9700 and W7900 rows serve on lemonade). `-n` is a regex +over the scenario name, not a prefix, and not the `@id:` slug the report +anchors show. It skips the host capability checks, so pick a scenario whose +`@requires-*` tags (such as `@requires-engine:vllm`) your host meets; see +[testing.md](testing.md) for how to select a scenario. + `nightly.yml` also computes a mechanical GO/NO-GO release-readiness verdict via `cargo xtask release-readiness`: it gates on the nightly Linux/Windows build results, the consolidated E2E failure total, and the `e2e-report-nightly` @@ -226,14 +330,14 @@ The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, `e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`, `e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, -`e2e-gpu-w7900-n-1-report`, `e2e-gpu-w7900-n-2-report`, `e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `rad3`/`mi350p` are nightly-only -lanes (ROCMAI-125) and don't run that matrix, so they have no `-n-1`/`-n-2` -entries. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900` +are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any +more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they +don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is @@ -276,7 +380,7 @@ run there; WSL reports `os_family` `"linux"` and stays vLLM-eligible); otherwise `vllm` is only preferred for the `*-dcgpu` families and `gfx906`/`gfx908`/`gfx90a`. Consequences for the lanes above: -- Strix Halo is `gfx1151` on every OS, so all three per-PR Strix lanes +- Strix Halo is `gfx1151` on every OS, so all three Strix lanes (Ubuntu, Windows, WSL2) and their nightly counterparts are lemonade-only — no matrix value turns a Strix lane into a vLLM lane. - `e2e-gpu` (MI300X) is the only per-PR lane where vLLM is the effective @@ -290,19 +394,30 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and ## Triggers The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, -`pull_request`, and `merge_group` (every GPU lane except MI300X, which is not -required for merging and so is skipped in the merge queue rather than left -queued on limited runner capacity after its PR has merged) when the workflow's -own `changes` job's `serve` path filter is `true`. A new push to a pull request +`pull_request` (all lanes but `e2e-gpu-strix-ubuntu`), and +`merge_group` (the Windows Strix Halo and WSL2 lanes only: MI300X and Strix Ubuntu +are not required for merging and so are skipped in the merge queue rather +than left queued on limited runner capacity or hung at their cap after its PR +has merged) when the workflow's own `changes` job's +`serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM -lane: they run only on the other lanes, and the only vLLM serve before a change -lands is the per-PR canary (`serve-05`), which runs when the `serve` filter -matches. `push` fires on `main` and on any `release/**` branch, so pushing to -a release branch runs the full self-hosted matrix ahead of cutting the `v*` tag that `release.yml` builds/publishes from — this is the +lane: they run only on the Windows Strix Halo and WSL2 lanes (both set +`E2E_MERGE_QUEUE` for `merge_group`), and the only vLLM serve +before a change lands is the per-PR canary (`serve-05`), which runs when the +`serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X +and Strix Ubuntu both skip `merge_group`, so it runs only Windows Strix and +WSL2. `push` fires on `main` and on any `release/**` branch, so pushing to a +release branch runs the full self-hosted matrix (all four lanes) ahead of +cutting the `v*` tag that `release.yml` builds/publishes from — this is the release-candidate regression gate; `pull_request` fires against any base -branch. +branch. Merging or closing a pull request also starts a run (`closed` is a +listed `pull_request` activity type) that joins that PR's group and runs only a +no-op job, so it supersedes the PR's earlier run and frees runners where the +cancel completes. A run whose job is still queued for a runner may not finish +cancelling, so it can keep its place in the queue until the runner picks the +job up. On a release-branch push, every self-hosted GPU job additionally expands into a `strategy.matrix.sdk_version` of `[current, n-1, n-2]` (ROCMAI-430) — three @@ -333,8 +448,8 @@ plus broad-dependency safety nets), **not** a blanket `**/*.rs`. So a Rust chang that cannot affect serving — e.g. a dashboard-only or unrelated-crate PR — skips the heavy GPU matrix, while compile coverage for every crate still runs on `ci.yml`'s always-on build/test lanes. Off `pull_request` (push/merge_group) the -filter is forced `true`, so the full matrix (minus MI300X in the merge queue) -always runs there. Unlike the +filter is forced `true`, so the full matrix (minus MI300X and Strix Ubuntu +in the merge queue) always runs there. Unlike the pre-split layout the GPU jobs do **not** gate on the hosted `build-and-test` job — cross-workflow `needs` is not possible, so each GPU job builds the `rocm` binary itself as its first real step (a broken build fails that job fast and @@ -351,10 +466,9 @@ They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: - `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, - `strix-wsl`, `w7900`) — which self-hosted job(s) to run. `app-dev-gpu` + `strix-wsl`) — which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to - `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, and `w7900` to - `e2e-gpu-w7900`. (The mock lane has its own `platform` input on `ci.yml`; it + `e2e-gpu-strix-windows`, and `strix-wsl` to `e2e-wsl`. (The mock lane has its own `platform` input on `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a @@ -375,8 +489,8 @@ gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu Nearly every GPU serve scenario points its `data/runtimes` at one shared, pre-warmed managed runtime tree (`E2E_SHARED_RUNTIMES_DIR`), so a multi-GiB `rocm install sdk` happens once per runner instead of once per scenario. The tree -lives on the runner's workspace — persistent across runs on the long-lived -`native`-labeled hosts, rebuilt by the prewarm step below on each ephemeral +lives on the runner's workspace — persistent across runs on a long-lived +runner, rebuilt by the prewarm step below on each ephemeral DevLab Dispatch pool run — survives `git clean`, and is namespaced by source-layout generation (`e2e-prewarm-multi-arch-v2`) so a branch using a new package layout cannot poison the cache consumed by code that only understands the @@ -459,13 +573,11 @@ side-by-side runtime only when no match is found. See ## Blocking vs. non-blocking The self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-w7900` — all run with +`e2e-gpu-strix-windows`, and `e2e-wsl` — all run with check names that are absent from branch protection's required-status-check list (see below), so a hardware failure never gates a PR merge no matter how -it reports. Four of them run with `continue-on-error: false`, so a real -regression shows red instead of always green; `e2e-gpu-w7900` runs with -`continue-on-error: true` while that hardware is being proven out, and should -be flipped to `false` once it has a track record. Their results also surface in +it reports. They run with `continue-on-error: false`, so a real +regression shows red instead of always green. Their results also surface in the self-hosted consolidated report for visibility. A lane that ran but uploaded no report (it died before writing its results) fails the consolidated report job, naming the lane, rather than silently dropping out of the grid; on a @@ -514,6 +626,13 @@ does not dispatch self-hosted-runner jobs from an external fork's against branches and PRs within the repository (and on `workflow_dispatch`, which requires write access to trigger). +That holds for every self-hosted runner here, and so does its limit: approval +does not change what an approved fork job may do, and labels do not restrict +which jobs reach a runner, so a job naming a runner's labels can run there. +Only the shared-queue publishing runner is built around that, by holding no +queue credential; see +[nightly-on-the-queue.md](nightly-on-the-queue.md#why-it-works-this-way). + ## Notes - The hardware jobs build and run **release** binaries: they assert diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 5e39aca232..3cd9610703 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -143,7 +143,17 @@ Installer policy: - `install.sh` and `install.ps1` update only the rocm-cli binary install directory (its binaries, `.rocm-cli-manifest`, and the `.rocm-cli-path-edit` record of the `PATH` change they made) and that `PATH` change itself: a - marked block in the shell profile, or the user `PATH` on Windows. + marked block in the login shell's profile, plus fish's `config.fish` + when fish is installed, or the user `PATH` on Windows. When fish is + installed, `install.sh` also writes the fish completion script to + `completions/rocm.fish` in fish's config folder (`$XDG_CONFIG_HOME/fish`, + else `~/.config/fish`), starting with a line naming the install folder, and + records it, so `rocm uninstall` removes it with the binaries. Uninstall + leaves a `rocm.fish` that does not start with that line for its own folder. + The installer itself leaves an existing `rocm.fish` (or link) that does not + start with that marker as it is, prints a note, and records nothing for it. + A custom `ROCM_CLI_SHELL_PROFILE` replaces only the login shell's profile; + fish's is still updated. - External plugins under the rocm-cli data directory are not touched by install or upgrade. - `rocm uninstall` removes the data directory by default. Use diff --git a/docs/host-probing.md b/docs/host-probing.md index bfed94a007..097a970258 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -6,9 +6,9 @@ SPDX-License-Identifier: MIT # Host probing: which stack owns each fact -Status: proposal. This note changes no code and no behaviour. It decides nothing by itself: it lists the host facts that `rocm-core` reads twice (on the hosts each is read on, see "Which host reads which fact"), says how the two reads differ, and proposes which one should survive and in what order a later series of changes could remove the other. Each pair is meant to be a separate change with its own review. +Status: proposal, with the first three pairs and the first step of the last delivered. This note changes no code and no behaviour itself, and what it proposes for a pair is decided by the change that converges that pair, which then records the decision here (the PATH lookup, pair 1, the kernel release, pair 2, and the CPU model, pair 3, have: see "PATH lookup", "Kernel version" and "CPU model"; the first step of the subprocess runners, pair 6, has: see "Decided and delivered, first step" under "Subprocess runners"). It lists the host facts that `rocm-core` reads twice (on the hosts each is read on, see "Which host reads which fact"), says how the two reads differ, and proposes which one should survive and in what order a later series of changes could remove the other. Each pair is meant to be a separate change with its own review. -This note is a snapshot of `main` at the time of writing, not maintained documentation. It is to be retired (the file deleted, together with its entry in [architecture.md](architecture.md)) when the convergence tickets for these pairs close, so the inventory does not keep pointing at a stale design record. +This note is a snapshot of `main` at the time of writing, not maintained documentation. It is to be retired when the convergence tickets for these pairs close, so the inventory does not keep pointing at a stale design record. Retiring it means deleting this file, removing the link to it from the host-probing bullet in the `crates/rocm-core` Boundaries of [architecture.md](architecture.md) (that bullet is the only entry it has; it is not a table row), and rewording the comments in `rocm-core` that cite it by name or as "the host-probing note" ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs), [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs), [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs), [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs), [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) and [`examine/wsl.rs`](../crates/rocm-core/src/examine/wsl.rs)). That list is a snapshot too, so start the retirement with a search of the whole repository for the file name (for example `git grep -n host-probing`) and reword every hit. How to read the claims below. Everything marked "verified" was read in the source on `main` at the time of writing, and the symbols named are the identity if a file moves. Nothing was executed on a Windows host. The one run on a WSL host is a single `rocm examine --json` with a local debug build of unrecorded revision, noted under "CPU model", and every count of runs or queries below comes from reading the call graph, not from measuring a process. Anything that depends on how a real machine behaves, rather than on what the code says, is marked UNCONFIRMED. @@ -18,12 +18,12 @@ For each pair, the proposed survivor and the biggest difference a user can see t | Pair | Proposed survivor | Biggest user-visible difference today | |---|---|---| -| [Kernel version](#kernel-version) | `detect_kernel_version`, called by `probe_os` for `kernel_release` on Linux (the timeout is open) | where `uname` is slow or fails, `kernel:` and `kernel_release` can disagree (UNCONFIRMED); on Windows `kernel_release` is always empty | -| [CPU model](#cpu-model) | one reader in `os_facts` returning model and vendor, called by `probe_cpu_linux` and `probe_cpu_windows` | on WSL the text shows the CPU while `cpu_model` is empty and `cpu_vendor` is `unknown`; the change leaves that as it is (option (a)) | +| [Kernel version](#kernel-version) | delivered: `detect_kernel_version`, called by `probe_os` for `kernel_release` on Linux; one 5 s limit and a strict read | before the change, where `uname` was slow or failed, `kernel:` and `kernel_release` could disagree (UNCONFIRMED on a real host); on Windows `kernel_release` is still always empty (out of this pair) | +| [CPU model](#cpu-model) | delivered: one reader in `os_facts` returning model and vendor, called by `probe_cpu_linux` and `probe_cpu_windows` and read by the summary | before the change, the same model could be spelled two ways in one `--json` run and a `Hardware`-only board had a CPU in the text and none in `cpu_model`; now one answer, the differences it moved are listed under "CPU model". On WSL the text still shows the CPU while `cpu_model` is empty and `cpu_vendor` is `unknown` (option (a), unchanged) | | [Windows GPU query](#windows-gpu-query) | the summary stack's inventory, with `probe_gpus_windows` building `gpus[]` from it and NVIDIA kept in the JSON | an NVIDIA-only host reads "AMD GPU not detected yet" in the text beside `has_nvidia_gpu: true` in the JSON | | [Marketing name to gfx target](#marketing-name-to-gfx-target) | the `AMD_MARKETING_GFX_TARGETS` table, gaining the codenames `classify_amd_marketing_name` knows (finish [#521](https://github.com/ROCm/rocm-cli/pull/521)) | on Windows without a HIP SDK an RX 7900 XTX has an empty `gpus[].gfx_target` while `summary.detected_gfx_target` is `gfx1100` | -| [PATH lookup](#path-lookup) | the behaviour of `tool_on_path`, housed in `probe_exec.rs` under the name `which`; the empty-entry case is open | an empty `PATH` entry or a `PATH` that is not UTF-8 changes which tools read as present (for example `wsl.rocminfo`), so the change is not output-preserving | -| [Subprocess runners](#subprocess-runners) | `run_with_timeout` as the one bounded executor, with `CommandCapture` extended by a typed outcome | a probe behind a hung grandchild returns at its deadline, and a successful capture can lose output a descendant writes more than 200 ms after the child exits | +| [PATH lookup](#path-lookup) | delivered: the behaviour of `tool_on_path`, housed in `probe_exec.rs` under the name `which`; an empty `PATH` entry is skipped | before the change, an empty `PATH` entry or a `PATH` that is not UTF-8 changed which tools read as present (for example `wsl.rocminfo`); now there is one answer, and the differences it moved are listed under "PATH lookup" | +| [Subprocess runners](#subprocess-runners) | first step delivered: `run_with_timeout` as the one bounded executor (it shares its spawn and wait with `run_bounded`, which returns a `CommandCapture`), with `CommandCapture` extended by raw bytes and a typed outcome; `run`, `capture_optional_command*` and the Windows diagnostic capture are adapters over it | before the change, a probe behind a hung grandchild outlasted its deadline (or, for a child that had exited, waited for the grandchild); now it returns at its deadline, and a successful capture can lose output a descendant writes more than 200 ms after the child exits | ## The two stacks @@ -46,18 +46,18 @@ Every pair below is read on a subset of the hosts, and the subset decides which | Fact | Native Linux | WSL (probing inside the distro) | Native Windows | Windows host inspecting a WSL distro (`probe_wsl_distro_from_host`) | |---|---|---|---|---| -| Kernel version | both, `uname -r` | both: `probe_os` runs before the WSL branch returns | summary `kernel` from `cmd /C ver`; `kernel_release` stays empty | examination only, from `uname -r` run inside the distro by a shell script (`WSL_REMOTE_PROBE`), a third implementation | -| CPU model and vendor | both | summary only: the examination returns before `probe_cpu_linux`, so `cpu_model` is empty and `cpu_vendor` is `unknown` | both, plus the inventory row | neither: `cpu_model` empty | +| Kernel version | both, through `detect_kernel_version` (before the change each ran `uname -r` itself) | both: `probe_os` runs before the WSL branch returns | summary `kernel` from `cmd /C ver`; `kernel_release` stays empty | examination only, from `uname -r` run inside the distro by a shell script (`WSL_REMOTE_PROBE`), a third implementation | +| CPU model and vendor | both, through one reader | summary only: the examination returns before `probe_cpu_linux`, so `cpu_model` is empty and `cpu_vendor` is `unknown` (unchanged by the CPU-model change, option (a)) | both, each running the system inventory's query, the examination through the reader and the summary through the inventory it holds | neither: `cpu_model` empty | | Windows adapter list | not applicable | examination `gpus[]` stays empty (the WSL branch skips the GPU probes); the summary asks the Windows host for the display target only | both | not collected here; the host driver is read through the summary's inventory (`detect_local_windows_host_driver`) | | Marketing name to gfx target | examination only (the `lspci` scan); the summary reads sysfs and KFD, not names | summary only, through the Windows host's display name | both | not collected | -| PATH lookup | examination `which` only | both (`tool_on_path` and `which` for `rocminfo`) | examination `which` only | examination `which` for `wsl.exe` only | +| PATH lookup | `which` | `which`, asked by both stacks for `rocminfo` (before the change the summary asked `tool_on_path`, a second copy) | `which` | `which` for `wsl.exe` only | | Subprocess runners (which helpers exist on the host) | three: `capture_optional_command*`, `run`, `run_with_timeout` (`capture_diagnostic_command` is Windows only) | the same three | all four | `run` and `run_utf16le` (both over `run_raw`), plus the summary's capture helper through the inventory behind `detect_local_windows_host_driver` | A pair that one host reads with only one stack cannot disagree on that host, but it can still change when a shared helper is changed, so the characterisation tests below name the hosts they cover and say so where a row is empty. ## Kernel version -Verdict: confirmed duplicate for `uname -r`. `uname -v` in `probe_os` has no twin. +Verdict: confirmed duplicate for `uname -r`, now converged (see "Decided and delivered" below). `uname -v` in `probe_os` has no twin. The comparison that follows is how the two copies differed before that change, kept as the record of what it moved. | | Summary | Examination | |---|---|---| @@ -73,9 +73,43 @@ What a user can see: on a native Linux or WSL host where `uname` is slow or fail Proposal: `detect_kernel_version` survives and `probe_os` calls it for `kernel_release` on Linux, keeping `uname -v` where it is. Calling it on Windows too would populate `kernel_release` there, which is a visible change to the JSON; decide that separately. The timeout is open (1.5 s or 5 s): the characterisation test should pin both before one is chosen. Hosts the test covers: native Linux and WSL (one code path, since `probe_os` runs before the WSL branch returns), with a fake `uname` that succeeds, exits non-zero with stdout, times out, and prints nothing; plus a WSL 1 style and a WSL 2 style string through `probe_wsl`, because `is_wsl1_kernel` reads the field. Windows is covered only by a unit test behind an injectable runner, which pins that `kernel_release` stays empty unless that change is decided. The distro probe's shell `uname -r` has no twin to compare and is out of this pair. +### Decided and delivered + +The change took the proposed survivor: `detect_kernel_version` in [os_facts.rs](../crates/rocm-core/src/host_gpu/os_facts.rs) is the one `uname -r` reader, and `probe_os` in [host_env.rs](../crates/rocm-core/src/examine/host_env.rs) calls it for `kernel_release` on Linux, native and WSL alike, with a failed read left empty. It is also one read per inspection where one command runs both stacks: `rocm examine --json` builds both through `Examination::probe_with_summary`, which probes this host and gathers the summary from that examination with the crate-private `ExamineSummary::gather_after` (so no caller can pair the summary with an examination of another host, such as one deserialised from a report), and the summary takes the examination's `kernel_release` (a failed read included) on Linux instead of running `uname -r` again, so the two fields cannot disagree on a transient failure and a hung `uname` costs one timeout, not two. Callers that gather only the summary still read for themselves, and most of them read once per call: the text form of `rocm examine` (`examine_human_report` in [examine_report_text.rs](../apps/rocm/src/examine_report_text.rs)), the in-process `rocm examine` of the chat bridge, the bridge snapshots and the host snapshot, the sandbox examine snapshot of `rocmd`, the free-form answer, `rocm install driver --reconcile`, and the runtime-install finalizer (which asks only when a runtime that cannot serve raises the question of whether the host has a GPU). + +Three paths read more than once for one user action, as they did before this change: + +- the `examine` and `doctor` tools of the chat bridge in [mcp_bridge.rs](../apps/rocm/src/chat/mcp_bridge.rs) gather the summary and then render the text form, which gathers it again (two reads in one process); +- the `examine` tool of `rocmd` in [mcp.rs](../apps/rocmd/src/mcp.rs) gathers the summary in process and then runs `rocm examine` as a child process, whose text form gathers it again (two reads, in two processes); +- `rocm install driver` in [driver_install.rs](../apps/rocm/src/driver_install.rs) gathers the summary before it builds the plan and, when it runs the plan, once more afterwards (one read for a dry run or a plan that is only reported, two for an executed install). + +`uname -v` stays in `probe_os`. Windows is out of this change: the examination still leaves `kernel_release` empty there and the summary still reads `cmd /C ver` under the shorter optional-command timeout; whether `kernel_release` should be filled on Windows stays open, and so does the Windows unit test the proposal mentioned, because the Windows branches did not change. + +The four open questions are decided as follows. + +- Timeout: 5 s, the longer of the two old limits (`KERNEL_RELEASE_TIMEOUT`, written out as five seconds rather than borrowed from `SHORT`, which is also 5 s today, and pinned by `the_kernel_release_timeout_is_five_seconds`). `uname -r` is a few bytes the kernel hands over, so a read that takes seconds is a stalled host, where waiting costs little and a spurious "unknown" costs more: `kernel_release` feeds the WSL version check and the kernel-dependent diagnoses, which read an empty value as nothing to report. The cost is that every caller of `ExamineSummary::gather` and `gather_without_driver_facts` that reads the kernel for itself on Linux, native or WSL (the text of `rocm examine`, the MCP and bridge snapshots, the free-form answer, `rocm install driver` and its `--reconcile` form, the runtime-install finalizer, and `rocmd`'s host snapshot, MCP and sandbox paths) can wait up to 5 s per read instead of 1.5 s on a host whose `uname` hangs, and a caller that reads more than once pays that more than once: the two reads of each of the three multi-read paths listed above run one after the other, so each of those can wait up to 10 s instead of 3 s. None of these is on a path run for every command. `rocm examine --json` waits for one timeout, because it hands the examination's read to the summary. Windows does not change: the summary there reads `cmd /C ver` under the 1.5 s limit, and the examination-only callers, such as `rocm diagnose`, already waited up to 5 s in `probe_os`. +- Failure: `None` in the summary (`` in the text, null in the JSON) and an empty `kernel_release` in the examination, with no `probe_failures` entry, as before. Adding an entry would change `status` to degraded on a host without a usable `uname`, such as a minimal container, which is a bigger change than the pair warrants. An empty `kernel_release` reads as WSL 2 in `is_wsl1_kernel`, as before. +- Non-zero exit with output: dropped by both. The examination used to keep what a failing `uname` printed; the summary dropped it. A release from a `uname` that failed is not trustworthy. +- Decoding: strict UTF-8 for both. The examination used to decode lossily. A release is ASCII in practice, so the only input that moves is one that is not UTF-8 at all, and a string with replacement characters in it would reach the diagnoses and the WSL check as if it were a release. + +The answers that moved, each with the test that pins it: + +| Difference | Before | Now | Pinned by | +|---|---|---|---| +| `uname` prints a release and exits non-zero | summary: none; examination: the release | none in both | `a_non_zero_exit_with_output_is_dropped_by_both`, scenario `examine-44` | +| `uname` output that is not UTF-8 | summary: none; examination: the release with the bad byte replaced | none in both | `output_that_is_not_utf8_is_dropped_by_both` | +| `uname` that answers after 2 s | summary: none (1.5 s); examination: the release (5 s) | the release in both | `an_answer_after_two_seconds_is_read_by_both` | +| `uname` that never answers | none in both | none in both; the unit test's child reads twice, each bounded by the one 5 s limit | `a_uname_that_never_answers_reads_as_nothing_in_both_within_the_timeout` (it checks the elapsed time, since the answers alone cannot tell a timeout from none) | +| `uname` that prints nothing | none in both | unchanged | `a_silent_uname_reads_as_nothing_in_both` | +| WSL 1 and WSL 2 style releases | read as printed by both | unchanged; `probe_wsl` makes version 1 of the first and version 2 of the second, of an empty one and of the original `-microsoft-standard` form | `both_read_wsl_style_releases_as_printed`, `probe_wsl_reads_the_wsl_version_from_the_kernel_release` | +| A failed read and `probe_failures` | no entry | no entry | the check in the same tests that no case leaves a `probe_failures` entry about the kernel, and scenario `examine-44` | +| Reads in one `rocm examine --json` run | two (`probe_os` and the summary each ran `uname -r`) | one: the summary takes the examination's | `the_summary_reads_the_kernel_only_when_the_examination_did_not` (a counted read), and the last step of scenario `examine-43`, which counts the calls of a stand-in `uname` | + +The scenarios `examine-43` and `examine-44` in [examine.feature](../tests/e2e-cucumber/features/examine.feature) drive the observable answer on the Linux lane, native and WSL: with a stand-in `uname` first on `PATH`, one `rocm examine --json` run holds `summary.kernel` and `kernel_release` level on a specific release and shows from the stand-in's call log that `uname -r` ran once, and, for a `uname` that prints a release and then fails, holds both empty with no kernel `probe_failures` entry. The unit tests plant the same stand-in in a child copy of the test binary with its own `PATH`, because a test thread cannot change `PATH` without disturbing every other test. + ## CPU model -Verdict: confirmed duplicate on native Linux and Windows, and on Windows three implementations. On WSL it is not a duplicate: only the summary reads the CPU there (see the table above and the paragraph on WSL below). The three Windows implementations are `detect_cpu_model` (its own PowerShell call), the `CPU` row of `WINDOWS_SYSTEM_INVENTORY_SCRIPT` in [windows_inventory.rs](../crates/rocm-core/src/host_gpu/windows_inventory.rs), and `probe_cpu_windows`. They are not three reads of one run in the usual case: the summary reads the inventory row first and calls `detect_cpu_model` only when no inventory exists, so a host whose inventory answers is read twice in `rocm examine --json` and one whose inventory does not answer is read three times (derived from `detect_cpu_model_with_windows_inventory` in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs); UNCONFIRMED on a live host). +Verdict: confirmed duplicate on native Linux and Windows, and on Windows three implementations, now converged (see "Decided and delivered" below); the comparison that follows is how the copies differed before that change, kept as the record of what it moved. On WSL it is not a duplicate: only the summary reads the CPU there (see the table above and the paragraph on WSL below). The three Windows implementations are `detect_cpu_model` (its own PowerShell call), the `CPU` row of `WINDOWS_SYSTEM_INVENTORY_SCRIPT` in [windows_inventory.rs](../crates/rocm-core/src/host_gpu/windows_inventory.rs), and `probe_cpu_windows`. They are not three reads of one run in the usual case: the summary reads the inventory row first and calls `detect_cpu_model` only when no inventory exists, so a host whose inventory answers is read twice in `rocm examine --json` and one whose inventory does not answer is read three times (derived from `detect_cpu_model_with_windows_inventory` in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs); UNCONFIRMED on a live host). | | Summary | Examination | |---|---|---| @@ -90,10 +124,43 @@ What a user can see on WSL: the text and `summary.cpu` show the CPU, while `cpu_ What a user can see on native Linux and Windows: a board whose cpuinfo has `Hardware` but no `model name` shows a CPU in the text and an empty `cpu_model` with vendor `unknown` in the JSON. A model string with repeated spaces inside it differs between the two forms (UNCONFIRMED which real CPUs report one; the code path differs). A Windows host where the 1.5 s query times out shows `cpu: ` beside a populated `cpu_model` (UNCONFIRMED, timing). No diagnosis checker reads `cpu_model` or `cpu_vendor` today (verified by grep); they reach Doctor reports and the JSON only. -Proposal: one reader in `os_facts` that returns the model and the vendor together (Linux from cpuinfo with the `Hardware` fallback and `vendor_id`; Windows from the inventory row, deriving the vendor from the model as `probe_cpu_windows` does). `probe_cpu_linux` and `probe_cpu_windows` call it. Open: whether the whitespace collapse should reach `cpu_model` in Doctor reports. +Proposal: one reader in `os_facts` that returns the model and the vendor together (Linux from cpuinfo with the `Hardware` fallback and `vendor_id`; Windows from the inventory row, deriving the vendor from the model as `probe_cpu_windows` does). `probe_cpu_linux` and `probe_cpu_windows` call it. The open question here, whether the whitespace collapse should reach `cpu_model` in Doctor reports, is decided under "Decided and delivered". This change does not reach WSL. The early return stays, so on WSL `cpu_model` stays empty and a comparison of `summary.cpu` with `cpu_model` fails there before and after. There are two ways to handle that, and the change has to pick one and say so in its PR text. Either (a) the comparison, in the characterisation test and in the scenario, is scoped to hosts where `is_wsl` is false, and the empty WSL fields are pinned as an existing difference; or (b) collecting CPU facts on WSL becomes an explicit behaviour change: the WSL branch calls the reader too, `cpu_model` and `cpu_vendor` appear in the WSL JSON and in Doctor reports, and the Windows-hosted distro probe needs its own answer (its `WSL_REMOTE_PROBE` shell script would have to print the CPU, since the reader runs on the host, not in the distro). The note recommends (a) for the convergence change and (b), if wanted, as its own change, because (b) adds a populated field to a document that consumers read today as empty. Hosts the characterisation test covers: native Linux from a fixture `/proc/cpuinfo` through `host_path` (`model name` present, `Hardware` only, repeated spaces, no `vendor_id`), WSL (a simulated host root, pinning the summary populated and the examination empty), and Windows through an injectable runner (inventory row present, absent, query failing, query timing out, empty output). +### Decided and delivered + +The change took the survivor proposed above. `CpuFacts` in [os_facts.rs](../crates/rocm-core/src/host_gpu/os_facts.rs) carries the model, or none, and the vendor, and one reader per platform fills it: on Linux `read_linux_cpu_facts` reads `/proc/cpuinfo` through `host_path` and `cpu_facts_from_cpuinfo` takes the first `model name` or `Hardware` line with a value and the first `vendor_id` with a value, and on Windows `windows_cpu_facts_with` runs the CPU-and-RAM query of the system inventory (`windows_system_inventory_with` in [windows_inventory.rs](../crates/rocm-core/src/host_gpu/windows_inventory.rs)) and `cpu_facts_from_windows_inventory` takes its CPU row, deriving the vendor from the model string (`amd` or `intel`, else `unknown`) as `probe_cpu_windows` did. `probe_cpu_linux` and `probe_cpu_windows` fill `cpu_model` and `cpu_vendor` from them, and the summary reads the same facts: `detect_cpu_model` is the Linux model and answers `None` off Linux, so it cannot start a Windows query, and on Windows the summary reads the CPU row of the inventory it already holds (`summary_cpu` in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs)), which is the same row through the same function. Both PowerShell `Win32_Processor` queries that were separate from the inventory are gone, and so is the Windows arm `detect_cpu_model` had in the first version of this change: nothing reached it in production, and it was exactly the path that would have brought the second query back. The injectable query (`windows_cpu_facts_with`, and `windows_system_inventory_with` under it) is what runs the Windows logic in a unit test on the Linux and Windows jobs. + +WSL is option (a): the early return stays, so on WSL `cpu_model` is empty and `cpu_vendor` is `unknown` while the summary shows the CPU, exactly as before. The comparison of `summary.cpu` with `cpu_model` in the scenarios is scoped to hosts where `is_wsl` is false, and the empty WSL fields are pinned as an existing difference by `on_wsl_the_summary_reads_the_cpu_and_the_examination_does_not` and scenario `examine-47`, so collecting CPU facts on WSL (option (b)) is a visible change of its own, which turns both. + +The open questions are decided as follows. + +- Whitespace: the collapse reaches `cpu_model`, so the model is one string in both forms. Nothing diagnoses from `cpu_model` or `cpu_vendor` (grep of the crates), so the only places that see the change are the JSON and the Doctor reports; a consumer that matched the old padded spelling is the one that sees it. +- The `Hardware`-only case: the `Hardware` value is the model and the vendor stays `unknown`, because such a file names none. The model is the first line of either kind, as the summary always had it, so with `Hardware` before `model name` the examination now reads the `Hardware` value where it read the `model name`; real cpuinfo prints `Hardware` after the per-core blocks, so that order is a fixture, not a host this note knows of. +- Empty versus unknown: a CPU that could not be read is no model (shown `` in the text, `null` in `summary.cpu`) in the summary and an empty `cpu_model` with `cpu_vendor` `unknown` in the examination, as before, and a vendor line with no value names no vendor. On Linux that is not a `probe_failures` entry, as before. On Windows the examination still records a `probe_failures` entry, now worded for the query that runs (`The Windows system inventory query (CPU and RAM) reported no CPU; cannot identify CPU.`, where it named a `Win32_Processor` query that no longer runs), now for any read that gives no CPU (a query that failed, timed out, printed nothing or printed no CPU row), where it recorded it for a non-zero exit or blank output. +- The Windows timeout: one limit of 5 s, the inventory's own for every other query, for the one query both stacks run. The summary's system query was 1.5 s (and its own CPU query too) and the examination's was 8 s, so the summary now waits longer, for its RAM row as well, and the examination gives up 3 s sooner. The reason is that the examination runs the query first in a `--json` run, on a PowerShell that has not started yet, where 1.5 s may not be enough and a spurious empty `cpu_model` with a `probe_failures` entry costs more than waiting; 8 s was not taken for the shared query because it would also be the worst case of every startup path that gathers the summary on a host whose PowerShell hangs. Neither number was measured: UNCONFIRMED on a real Windows host. +- Queries per run: on native Windows `rocm examine --json` runs the system query twice now (down from two or three): once in the summary's `assemble_windows_inventory` and once in `probe_cpu_windows`. They are not shared within a run because the examination is built first and the summary after it, by separate calls in `examine_cmd.rs` that pass nothing between them, and sharing the answer means choosing which stack owns it, which is the open question below about one run sharing answers between the stacks. This change removes the third read and the second implementation; it does not claim one query. +- The inventory fallback: dropped. With no inventory at all, `detect_cpu_model` used to run a `Win32_Processor` query of its own, but an absent inventory means every source, the system query included, had just failed, so the second try repeated a failed query. The summary now shows `` there. Keeping it would have meant keeping a second Windows implementation, which is what this change removes. The summary's RAM keeps its own fallback query (`detect_system_ram_gib_with_windows_inventory`), and that is deliberate here and not an oversight: `detect_system_ram_gib` is a public reader that `model_fit` calls with no inventory at all, so its Windows query has to exist, and changing what the summary shows for RAM when the inventory is absent is a behaviour change for a fact this change does not touch. Dropping that fallback so RAM follows the CPU is a follow-up, with its own scenario or unit test. + +The differences that moved, each with the test that pins it (the unit tests are in `os_facts.rs`, `host_gpu.rs` and `host_env.rs`; the scenarios are in `examine.feature`): + +| Difference | Before | Now | Pinned by | +|---|---|---|---| +| `cpu_model` with repeated spaces inside the model | summary collapsed, examination trimmed only | collapsed in both (Linux and Windows) | `linux_cpuinfo_the_inputs_the_two_reads_used_to_disagree_on`, `windows_both_reads_collapse_whitespace_inside_the_name`, `cpu_reads_follow_the_simulated_host_root`, scenario `examine-45` | +| `Hardware` line and no `model name` | summary read it, examination had an empty `cpu_model` | both read it, vendor `unknown` | `linux_cpuinfo_the_inputs_the_two_reads_used_to_disagree_on`, scenario `examine-46` | +| `Hardware` before `model name` | summary read the `Hardware` value, examination the `model name` | both read the `Hardware` value | `linux_cpuinfo_the_inputs_the_two_reads_used_to_disagree_on` | +| An empty `vendor_id` value | examination took `""` as the vendor and stopped | not a vendor: the next `vendor_id` is read, else `unknown` | `linux_cpuinfo_the_inputs_the_two_reads_used_to_disagree_on` | +| Windows: the query the examination runs | `Win32_Processor` alone, through `run`, 8 s | the system inventory query (CPU and RAM), through the optional-command helper, 5 s | `windows_both_reads_are_given_the_same_time` | +| Windows: the summary's system query | 1.5 s | 5 s, for its CPU and RAM rows | `windows_both_reads_are_given_the_same_time` | +| Windows: no inventory | the summary ran a second CPU query | ``, no second query | `windows_an_answer_with_no_cpu_is_no_cpu_in_both_reads`, `the_summary_cpu_comes_from_the_windows_inventory_or_not_at_all` | +| Windows: an answer with no CPU in it | failure entry (naming `Win32_Processor`) only for a non-zero exit or blank output | failure entry naming the inventory query, for any read with no CPU | `windows_an_answer_with_no_cpu_is_no_cpu_in_both_reads` | +| WSL | summary populated, examination empty | unchanged | `on_wsl_the_summary_reads_the_cpu_and_the_examination_does_not`, scenario `examine-47` | + +Two more differences come with the optional-command helper that the examination's Windows read now runs through, and neither is pinned by a test, because the helper is not injectable and needs a Windows host: it reads the output as strict UTF-8 where `run` decoded it lossily, so a CPU name that is not valid UTF-8 reads as no CPU, and it tries each spelling of `powershell` after a timeout where `run` started it once, so a hung PowerShell can cost the examination more than one limit (the candidate loop under "Subprocess runners"; it belongs to that pair, and the summary's inventory had the same behaviour already). + +What stays untested: the live Windows command line itself, which a unit test cannot start (the injected query stands in for it, and the script it is handed is the one the summary's inventory already ran); the `runtime_is_windows()` argument `summary_cpu` is called with, and the real PowerShell runner `probe_cpu_windows` passes (off Windows `probe_cpu_windows_runs_the_inventory_query_and_reports_no_cpu_without_one` runs it and pins that it reports no CPU, but the Windows job compiling is the only thing tying the live runner); and the timing claims above. The scenarios run on the Linux lane only (`@requires-os:linux`): the planted machines are simulated host roots, and native Windows cannot run them, so its CPU cases are the unit tests, which run on the Windows job as well. + ## Windows GPU query Verdict: confirmed duplicate, and wider than the ticket names: the summary side has several Windows adapter queries, all in [windows_inventory.rs](../crates/rocm-core/src/host_gpu/windows_inventory.rs), and the examination has one. @@ -141,7 +208,7 @@ Proposal: the table survives. It is pure, data-driven and already the superset f ## PATH lookup -Verdict: confirmed duplicate. +Verdict: confirmed duplicate, now converged (see "Decided and delivered" below). The comparison that follows is how the two copies differed before that change, kept as the record of what it moved. | | Summary | Examination | |---|---|---| @@ -155,22 +222,42 @@ What a user can see: only on WSL, where both run on Linux: the summary's `wsl_gl Proposal: the behaviour of `tool_on_path` survives (it is the one that follows `PATHEXT`), housed where both stacks already import from, in `probe_exec.rs`, under the existing name `which` so the call sites do not change. `tool_path_candidates` moves with it, because the runner uses it to try each spelling. Keeping the call sites unchanged does not keep their answers unchanged: the differences listed above become the intended observable differences of this change (see the first item of the order below). The empty-entry case is a real choice, because either answer changes one side. Following `PATH` semantics matches what a shell or `execvp` does with an empty entry, and makes `which` read a file in the current directory as found, which widens what the examination calls installed (fifteen or so capability checks, including `sudo` and `usermod`). Skipping empty entries keeps the examination's answer and changes only the summary's on WSL, at the cost of disagreeing with `execvp` for a program the CLI later starts by bare name. The note does not pick one; the change has to, and its PR text states which. +### Decided and delivered + +The change took the survivor proposed above: `which` in [probe_exec.rs](../crates/rocm-core/src/probe_exec.rs) has the behaviour `tool_on_path` had, `tool_path_candidates` moved there with it (the capture runner in [capture.rs](../crates/rocm-core/src/host_gpu/capture.rs) and the Windows diagnostics capture import it from there), `detect_wsl_summary` asks `which` for `rocminfo` and `cargo`, and `tool_on_path` is gone. The Windows spellings are computed by `path_candidates`, which takes the platform, `PATHEXT` and the system root as arguments, and the search by `found_in`, which takes the directories and an existence test, so both are unit-tested on every host. + +The empty-entry question is decided as skip. An empty `PATH` entry is not read as the current directory. That keeps the examination's answer, which is what fifteen or so capability checks (including `sudo` and `usermod`) already relied on, so the only answer the empty-entry rule moves is the summary's on WSL, where a `rocminfo` or `cargo` file in the current directory no longer reads as installed. It also settles the Windows question the note could not answer: how `std::env::split_paths` treats an empty entry there no longer matters, because the entry is dropped either way. The cost is the one named above: a program the CLI later starts by bare name can still resolve from the current directory through the operating system when `PATH` has an empty entry, and the lookup, which does not start anything, no longer agrees with that. Guarding the runners against that (`capture_optional_command*` and `run` start a bare name through `Command`, so the operating system resolves an empty entry for them) is a property of how they spawn, not of the lookup, so it belongs to the subprocess-runners pair (item 6 below) and is not changed here. + +The differences that moved, each with the tests that pin it (the unit tests are in `probe_exec.rs`; the scenarios are in `examine.feature`): + +| Difference | Before | Now | Pinned by | +|---|---|---|---| +| Empty `PATH` entry, with a file of the program's name in the current directory | summary: found; examination: not found | not found | `which_does_not_read_an_empty_path_entry_as_the_current_directory`, `path_dirs_drop_every_empty_entry`, scenario `examine-36` (control `examine-35`) | +| `PATH` that is not valid Unicode | summary: searched; examination: not found for every program | searched | `which_searches_a_path_that_is_not_unicode`, `path_dirs_keep_an_entry_that_is_not_unicode`, scenario `examine-37` | +| Windows `PATHEXT` entries outside `.exe`, `.bat`, `.cmd`, in both cases | summary: tried; examination: not tried | tried | `which_tries_a_pathext_extension_outside_exe_bat_and_cmd` (Windows job), `candidates_on_windows_try_every_pathext_entry_in_both_cases`, `candidates_on_windows_fall_back_to_the_default_pathext_when_it_is_unset` | +| Windows `PATH` entry in quotes holding a `;` | summary: kept whole; examination: split | kept whole | `which_searches_a_quoted_path_entry_that_holds_a_semicolon` (Windows job) | +| A Windows name that already has an extension (`python3.11`, `tool.py`, `wsl.exe`) | examination: also tried `.exe`, `.bat` and `.cmd` after it; summary: as given only | as given only | `candidates_on_windows_for_a_name_with_an_extension_are_the_name_alone` | +| System32 locations of `cmd`, `pnputil` and `powershell` (`pwsh` has none; its only extra candidate is the bare `pwsh.exe`) | summary: found; examination: not found | found | `which_finds_cmd_at_its_system32_location_without_it_being_on_path` (Windows job), `candidates_on_windows_add_the_fixed_system32_locations`, `found_in_asks_about_an_absolute_candidate_once_as_it_is_whatever_the_directories` | +| A file in the current directory, with no empty entry | not found by either | not found | the control in `which_does_not_read_an_empty_path_entry_as_the_current_directory` | + +No `which` call site passes a name with an extension other than `wsl.exe` (found as given), so the dotted-name row moves no answer today: the call sites ask for python, python3, mokutil, lspci, rocminfo, dpkg, rpm, usermod, sudo, cargo and the llama.cpp names, all bare. The System32 candidates stay in the shared candidate list. The capture runner needs them to start `cmd`, `pnputil` and `powershell` at all on a host where they are not on `PATH`, and one list that both the runner and the lookup read cannot drift. No `which` call site asks for those four today, so the change is invisible there; a future caller of `which("powershell")` gets "would the runner find it", not "is it on `PATH`". An absolute candidate is tested once as it is and not joined onto each `PATH` directory, so it is found even when `PATH` is unset or all empty, which is what the runner can start. The near copy of `tool_path_candidates` in `apps/rocm` (`program_path_candidates`) did not join this change and stays open at the end of this note. The scenarios `examine-35`, `examine-36`, `examine-37` and `examine-38` in [examine.feature](../tests/e2e-cucumber/features/examine.feature) drive the first two rows on the Linux lane from a simulated WSL host, holding `summary.wsl.rocminfo` and `wsl.rocminfo` of one `rocm examine --json` run level: a tool in a `PATH` directory is found (the control), a tool only in the working directory behind an empty entry is not (with `.` as the entry instead it is, which `examine-38` holds as the control), and a tool in a directory whose name is not valid Unicode is found. The harness sets a `PATH` that is not Unicode without trouble, which settles the question the note left about it. The Windows rows cannot run on the Linux lane and are covered by the unit tests named above, which the Windows job runs. + ## Subprocess runners -Verdict: confirmed duplicate, and a family rather than a pair. +Verdict: confirmed duplicate, and a family rather than a pair. The first step is delivered (see "Decided and delivered, first step" below). The comparison that follows is how the helpers differed before it, kept as the record of what it moved, and so are the paragraphs after it up to the proposal. | Helper | Streams | Result | Default timeout | Reader threads | |---|---|---|---|---| | `capture_optional_command_with_timeout` ([capture.rs](../crates/rocm-core/src/host_gpu/capture.rs)) | stdout only; stderr discarded | `Option`: only on success and valid UTF-8 | 1.5 s, 5 s for inventory queries | joined with no limit; tries each candidate spelling in turn after a spawn failure, a non-zero exit or a timeout | | `run`, `run_raw` ([probe_exec.rs](../crates/rocm-core/src/probe_exec.rs)) | both | `(rc, stdout, stderr)`, `127` for spawn failure, `124` for timeout, lossy UTF-8 | 5 s, 8 s, 15 s | joined with no limit | | `capture_diagnostic_command` ([diagnostics.rs](../crates/rocm-core/src/host_gpu/diagnostics.rs), Windows only) | both | status, text, error and a timed-out flag | per call | joined with no limit | -| `run_with_timeout` ([process.rs](../crates/rocm-core/src/process.rs)) | both | `Result`; a timeout is an `Err` that quotes bounded output | per call | abandoned after a short grace | +| `run_with_timeout` ([bounded.rs](../crates/rocm-core/src/process/bounded.rs), re-exported by [process.rs](../crates/rocm-core/src/process.rs)) | both | `Result`; a timeout is an `Err` that quotes bounded output | per call | abandoned after a short grace | -The first three join their reader threads with no limit, so a grandchild that keeps the pipe open can outlast their timeout; [process.rs](../crates/rocm-core/src/process.rs) documents why `run_with_timeout` does not. That defect is already tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623), which lists the same helpers. Separately, the candidate loop in `capture_optional_command_with_timeout` keeps going after any failure of a candidate, a timeout included, so on Windows a hung `powershell` may be started again under several spellings (UNCONFIRMED; the loop does not stop on a timeout). `capture_diagnostic_command` moves to the next candidate only when a spawn fails. +The first three join their reader threads with no limit, so a grandchild that keeps the pipe open can outlast their timeout; [bounded.rs](../crates/rocm-core/src/process/bounded.rs) documents why `run_with_timeout` does not. That defect is already tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623), which lists the same helpers. Separately, the candidate loop in `capture_optional_command_with_timeout` keeps going after any failure of a candidate, a timeout included, so on Windows a hung `powershell` may be started again under several spellings (UNCONFIRMED; the loop does not stop on a timeout). `capture_diagnostic_command` moves to the next candidate only when a spawn fails. The helpers differ on a successful run too, not only on a timeout. Each of these is derived from the code and not run: -- Late descendant output. After the child exits, `run_with_timeout` (`run_captured` in [process.rs](../crates/rocm-core/src/process.rs)) gives the readers `DRAIN_GRACE` (200 ms) and then takes the buffer and abandons them, so a descendant that writes later than that after the child exited successfully contributes nothing. `run_raw`, `capture_optional_command_candidate_with_timeout` and `capture_diagnostic_command` join the readers and so wait for EOF, keeping whatever the descendant writes however late, and waiting without a bound if it never closes the pipe. Output the child itself wrote before exiting is kept by both. The existing tests `output_arriving_after_the_child_exits_is_still_collected` and `a_grandchild_holding_the_pipe_does_not_hold_a_successful_call` in [process.rs](../crates/rocm-core/src/process.rs) cover `run_with_timeout` only, and the first uses a 0.1 s delay inside the grace, so neither shows the difference. +- Late descendant output. After the child exits, `run_with_timeout` (`run_captured` in [bounded.rs](../crates/rocm-core/src/process/bounded.rs)) gives the readers `DRAIN_GRACE` (200 ms) and then takes the buffer and abandons them, so a descendant that writes later than that after the child exited successfully contributes nothing. `run_raw`, `capture_optional_command_candidate_with_timeout` and `capture_diagnostic_command` join the readers and so wait for EOF, keeping whatever the descendant writes however late, and waiting without a bound if it never closes the pipe. Output the child itself wrote before exiting is kept by both. The existing tests `output_arriving_after_the_child_exits_is_still_collected` and `a_grandchild_holding_the_pipe_does_not_hold_a_successful_call` in [bounded.rs](../crates/rocm-core/src/process/bounded.rs) cover `run_with_timeout` only, and the first uses a 0.1 s delay inside the grace, so neither shows the difference. - A read error on a pipe. The three older helpers discard it (`let _ = read_to_end`) and keep the bytes read so far with the real exit status; `run_with_timeout` turns a read failure on a run that did not time out into an error and drops the bytes. - A failure to wait. `run_raw` treats a `try_wait` error as `124` without killing the child and then joins the readers; `capture_optional_command_candidate_with_timeout` and `run_with_timeout` kill it first. - Spawn failure and encoding. `run` reports `127` with empty output where `run_with_timeout` returns an `Err` (a spawn failure there is an `std::io::Error` under a context message); `capture_optional_command*` returns `None` for output that is not UTF-8 where `run` keeps it lossily, and discards stderr where the others capture it, so after the change a descendant that holds only stderr open costs it up to the 200 ms grace, where it cost nothing before. @@ -180,18 +267,51 @@ The capture type from the earlier `rocm` capture work is on `main`: `CommandCapt Proposal: `run_with_timeout` survives as the one bounded executor, with `CommandCapture` as the shared result carrier extended with a typed outcome (not found, timed out, exited with a code) rather than a second capture type. `run` and `capture_optional_command*` become thin adapters that keep their current return shapes, so the first change touches no call site. That is not the same as keeping every observable result: the timeout path and the successful-run differences listed above are where they differ (see the last item of the order below). Keeping the strict-UTF-8 and lossy decodings, and the partial output after a timeout that `run` returns, needs the raw bytes from the executor, which is a second reason the carrier has to hold bytes rather than lossy text. The adapters can then be retired site by site as the pairs above remove call sites. This pair overlaps [#623](https://github.com/ROCm/rocm-cli/issues/623); extend that issue's scope with the typed outcome rather than opening a parallel one. +### Decided and delivered, first step + +The first step took the proposal, touching no call site of the helpers. `run_with_timeout` (in [bounded.rs](../crates/rocm-core/src/process/bounded.rs), re-exported from [process.rs](../crates/rocm-core/src/process.rs)) is still the one bounded executor, and its core is now also reachable as `run_bounded`, which returns the run as a `CommandCapture` instead of turning a timeout or a failure into an `Err`. `CommandCapture` in [command_capture.rs](../crates/rocm-core/src/command_capture.rs) is the shared carrier, extended rather than duplicated: it holds the raw bytes of both streams (the text fields are their lossy form), a typed `CommandOutcome` (not found, spawn failed with an error kind, timed out, wait failed, exited with a code, killed by a signal), the `ExitStatus` of a reaped child and the error text the executor saw. Its legacy `exit_status` keeps reading `1` where there is no code, so the typed outcome is how a caller tells a signal from an exit with `1`. `run_raw` (under `run`, `run_with_env` and `run_utf16le`) and `capture_optional_command_candidate_with_timeout` are thin adapters that keep their return shapes, and the Windows-only `capture_diagnostic_command` runs on the executor too: its candidate loop and the translation of each outcome are in `capture_diagnostic_with`, which takes the runner as an argument, so its unit tests run on Linux and on the Windows job. The change touches no call site of those helpers; the hand-built captures in `rocm` and `rocmd` (the chat bridge, the watchers, the sandbox tests) moved to `CommandCapture::from_text` because the struct gained fields, and nothing else there changed. The typed outcome belongs on `CommandCapture`, not beside it, which settles that item of the open list. + +The wait loop also changed, so that taking the executor on does not slow the probes that used to poll every 20 to 25 ms: it sleeps 5 ms and doubles up to the 50 ms it used to sleep flat, and the drain-grace wait does the same. + +Nothing here is output-preserving without qualification. The differences that moved, each with the test that pins it (the unit cases are in [capture.rs](../crates/rocm-core/src/host_gpu/capture.rs) module `subprocess_runners_pair`, which runs the same fake program through `run`, `capture_optional_command_with_timeout` and `run_with_timeout`, unless another file is named). The adapters keep their old answer where the last column says so: + +| Difference | Before | Now | Adapter keeps the old answer | Pinned by | +|---|---|---|---|---| +| Output a descendant writes more than `DRAIN_GRACE` (200 ms) after the child exited 0 | `run` and the optional capture: kept (they join their readers and wait for EOF); `run_with_timeout`: lost | lost by all three; output written within the grace is still kept by all three. The capture's `detail` says output was still arriving when the grace ended; the optional capture writes it to its opt-in debug channel (`ROCM_CLI_DEBUG_COMMAND_CAPTURE`) as a line saying its output may be incomplete, and `run` and `run_with_timeout` do not surface it today | no | `output_written_more_than_the_drain_grace_after_the_child_exits_is_lost_to_all_three`, with the control `output_written_within_the_drain_grace_after_the_child_exits_is_kept_by_all_three`; the `detail` is `output_written_after_the_grace_is_missing_and_the_capture_says_so` and `a_capture_says_when_a_reader_was_still_going_at_the_end_of_the_grace` (bounded.rs), and the debug line is `a_short_answer_from_the_optional_capture_comes_with_a_debug_line_and_a_whole_one_does_not`, `a_failed_run_gets_its_failure_line_and_no_incomplete_line` and `only_a_run_that_exited_0_with_something_cutting_its_output_short_is_called_incomplete` (host_gpu/capture.rs) | +| A descendant holds the pipe open after the child exited 0 | `run` and the optional capture: the call lasts as long as the descendant; `run_with_timeout`: returns after the grace | all three return after the grace, with what the child wrote | no | `a_descendant_holding_the_pipe_no_longer_extends_run_or_the_optional_capture`, scenario `examine-48` | +| A descendant holds the pipe open past a timeout ([#623](https://github.com/ROCm/rocm-cli/issues/623)) | `run` and the optional capture: the reader joins wait for the descendant, so the timeout bounds nothing; `run_with_timeout`: returns at the deadline | all three return at the deadline plus the grace; `run` still returns `124` with the partial output that arrived by then, and the optional capture still returns `None` | the result shape yes, the content of the partial output no (what arrived by the deadline and the grace, not everything until EOF) | `a_descendant_holding_the_pipe_no_longer_outlasts_the_timeout_of_run_or_the_optional_capture`; the plain timeout with no descendant is `a_timeout_keeps_partial_output_for_run_and_quotes_it_for_the_bounded_executor`, unchanged | +| A descendant holds only stderr | the optional capture sends stderr to the null device, so it was not held at all; `run` waited for the descendant | the optional capture pipes stderr like the others and the descendant costs it up to the grace; `run` returns after the grace without the descendant's late stderr | the answer yes, the promptness of the optional capture by at most the grace | `a_descendant_holding_only_stderr_costs_the_optional_capture_at_most_the_drain_grace` (bounds the call by the descendant's life; the extra 200 ms is too small for a timing assertion) | +| A pipe that fails to read | the older helpers discard the error (`let _ = read_to_end`) and keep the bytes read with the real status; `run_with_timeout` turns it into an error and drops the bytes | unchanged for `run_with_timeout`; the capture keeps the bytes and the real outcome and puts the error in `detail`, and both adapters ignore it as before, except that the optional capture also writes the error to its opt-in debug channel when the run exited 0 | yes | `a_read_failure_on_either_stream_keeps_the_bytes_and_the_outcome_the_child_had` (bounded.rs, a failure on stdout, on stderr and on both), `a_pipe_that_failed_to_read_does_not_change_what_run_returns` (probe_exec.rs), `a_pipe_that_failed_to_read_does_not_change_the_answer` (host_gpu/capture.rs), and for `run_with_timeout` `a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output` (bounded.rs). The older helpers' own behaviour cannot be provoked with a real child (it needs the kernel to fail a pipe read), so that column of the table is derived from the code, not run | +| A failure to wait | `run_raw`: `124` without killing the child, then an unbounded join; the optional capture: kills, `None`; the diagnostic capture: kills, `failed to wait: ` | all of them kill the child. `run`: `124` with the bytes that had arrived (not joined to EOF); the optional capture: `None`; the diagnostic capture: `failed to wait: ` followed by the executor's own wording (`failed to poll process: ` or `failed to reap timed-out process: `) | the return shapes yes; `run` no longer leaves the child running, and the diagnostic text gains the executor's prefix | `a_failure_to_wait_is_a_capture_that_keeps_the_bytes_that_had_arrived` (both the poll and the reap failure, bounded.rs), `a_failure_to_wait_is_final_and_reports_no_output` (diagnostics.rs), `a_run_reads_as_the_rc_run_raw_has_always_returned` (probe_exec.rs), and the existing `a_failure_to_poll_the_child_still_abandons_both_readers` and `a_failure_to_reap_a_timed_out_child_still_abandons_both_readers` | +| A program that cannot be started | `run`: `127` with empty output; the optional capture: `None` and a debug line with the OS error; the diagnostic capture: next candidate; `run_with_timeout`: an error | the same for all of them; the executor now also says whether the program was missing (`NotFound`) or could not be executed (`SpawnFailed` with the kind) | yes | `a_program_that_cannot_be_started_is_127_empty_none_or_an_error` and `a_spawn_failure_reports_the_error_the_os_gave` (host_gpu/capture.rs), `a_program_that_never_starts_reports_the_last_launch_error` (host_gpu/diagnostics.rs), and for the typed outcome `a_program_that_cannot_be_started_is_a_capture_not_an_error`, `a_program_that_exists_but_cannot_be_executed_is_a_spawn_failure_not_not_found` (bounded.rs) and `only_a_missing_program_is_not_found` (command_capture.rs) | +| Decoding | `run` lossy UTF-8, the optional capture strict UTF-8 (`None` for a stray byte), `run_utf16le` from the raw bytes | the same: the carrier holds the bytes and each adapter decodes as before | yes | `output_that_is_not_utf8_is_lossy_for_run_none_for_the_optional_capture_and_raw_for_the_executor`, `utf16le_output_is_decoded_by_declaration_from_the_raw_bytes`, `a_program_that_exited_0_with_output_that_is_not_utf8_gives_no_answer_and_no_debug_line` | +| A child killed by a signal | `run`: `-1`; the optional capture: `None`; `capture_program`'s `exit_status`: `1` | the same, and the capture's `outcome` now says `Signaled` with the signal number on Unix | yes | `a_program_killed_by_a_signal_has_no_exit_code`, `a_child_killed_by_a_signal_reads_as_exit_status_1` (command_capture.rs), `a_run_is_a_capture_with_its_exit_code_or_its_signal_and_both_streams` (bounded.rs), `an_exit_code_outranks_a_signal_and_no_code_is_a_signal` and `a_status_is_an_exit_code_or_a_signal` (command_capture.rs, every host) | +| Poll interval | 20 ms in `run_raw`, 25 ms in the two capture loops, 50 ms flat in `run_with_timeout` | 5 ms doubling to 50 ms in every one of them, so a child that exits at once is noticed within a few milliseconds, one that runs long up to 50 ms after it exits, and a deadline is overshot by up to 50 ms | not the timing | `the_poll_interval_backs_off_from_a_few_milliseconds_to_the_flat_one` pins the schedule, and `the_wait_loop_sleeps_the_back_off_schedule_between_polls` and `the_drain_wait_sleeps_the_back_off_schedule_until_both_readers_finish` (bounded.rs) record the sleeps each loop really takes; the latency itself is not asserted, because a timing assertion on a few milliseconds is flaky | +| The diagnostic capture on a timeout | joins the readers after the kill, so unbounded | returns the output that arrived and `timed_out`, bounded | the report yes | `a_timeout_is_final_keeps_the_output_that_arrived_and_says_so`; the candidate rule, next spelling only when a program could not be started, is `the_next_spelling_is_tried_only_when_a_program_could_not_be_started`, and the status text is the platform's `ExitStatus` as before (`exit code: 3` on Windows) | + +Not changed, and still open below: the candidate loop of `capture_optional_command_with_timeout` still starts the next spelling after a timeout, so a hung `powershell` can still be started under several spellings; `wait_with_output_bounded` in `apps/rocm` still joins its readers with no limit; and the runners still start a bare program name through the operating system, which resolves an empty `PATH` entry as the current directory where `which` does not. + +The scenario `examine-48` in [examine.feature](../tests/e2e-cucumber/features/examine.feature) drives the second row through `rocm examine --json` on the Linux lane: a stand-in `lspci` on a simulated native Linux root prints one AMD GPU, exits 0 and leaves a `sleep` holding its output for 30 seconds; the inspection must finish in under 15 seconds and still list the GPU. That shows the output the child wrote before it exited is still collected; it does not pin the length of the grace, which the unit cases do. The loss of late output (the first row) has no scenario: it needs a probe tool that writes more than 200 ms after it exits, which no tool the CLI runs is known to do, so it is pinned by the unit cases above only. The Windows paths cannot run on the Linux lane: the candidate loop and the translation of each outcome run through `capture_diagnostic_with` on the Windows job, and a `cmd` case (`a_run_is_a_capture_with_its_exit_code_and_both_streams`) runs the executor itself there. UNCONFIRMED: whether a Windows child can be arranged to leave a late-writing or pipe-holding descendant in the unit harness, so the grandchild and late-output rows are covered on Linux only (shell fixtures); on Windows the executor's bounds rest on the existing `a_timed_out_child_has_its_output_quoted_within_a_bound` and are not re-pinned per adapter. Nothing was run on a Windows host here; the Windows target was only type-checked (clippy) on Linux. + ## Proposed order One pair per change. Each begins with a commit that adds a characterisation test and is green against the code as it is: it pins what both copies return for the same input, and where they differ it pins each copy's answer, with a comment naming the pair. The convergence commit then flips the assertions that are meant to change and the PR text lists each one as a user-visible difference. Tests that need a program on `PATH` use the fake-executable helper in [fake_executable.rs](../crates/rocm-core/src/fake_executable.rs); Windows-only logic goes behind an injectable function so a unit test runs on the Windows job, as the inventory already does with `WindowsInventorySources`. -1. PATH lookup. Smallest, and it settles where `tool_path_candidates` lives before the runner work needs it. It is not output-preserving: the differences listed under "PATH lookup" change which tools are reported as present, and its PR text lists each one as intentional, with the characterisation test that pins it. Hosts covered: Linux, which is also WSL's code path, for the empty entry, the non-UTF-8 `PATH` (set through an `OsString` environment value) and the current-directory file; Windows through an injectable resolver for the `PATHEXT` spellings, the System32 candidates and the `;` splitting, since the Linux lane cannot run them. -2. Kernel version. A single command; settles the timeout and the empty-versus-`None` question. Hosts covered as set out under "Kernel version". -3. CPU model. Adds the model-and-vendor reader; needs the Windows inventory row, which already exists. Its PR says which WSL option under "CPU model" it takes, (a) or (b); a PR that does not is not ready. +1. PATH lookup (delivered: see "Decided and delivered" under "PATH lookup"). Smallest, and it settles where `tool_path_candidates` lives before the runner work needs it. It is not output-preserving: the differences listed under "PATH lookup" change which tools are reported as present, and its PR text lists each one as intentional, with the characterisation test that pins it. Hosts covered: Linux, which is also WSL's code path, for the empty entry, the non-UTF-8 `PATH` (set through an `OsString` environment value) and the current-directory file; Windows through an injectable resolver for the `PATHEXT` spellings, the System32 candidates and the `;` splitting, since the Linux lane cannot run them. +2. Kernel version (delivered: see "Decided and delivered" under "Kernel version"). A single command; it settled the timeout and the empty-versus-`None` question. Hosts covered as set out under "Kernel version". +3. CPU model (delivered: see "Decided and delivered" under "CPU model"). Adds the model-and-vendor reader; needs the Windows inventory row, which already exists. Its PR says which WSL option under "CPU model" it takes, (a) or (b); a PR that does not is not ready. It took (a). Not output-preserving: the differences listed under "CPU model" change a model's spelling, a `Hardware`-only board's `cpu_model`, and the time the Windows reads wait. 4. Marketing name to gfx target, by finishing [#521](https://github.com/ROCm/rocm-cli/pull/521). It must land before the next item, so that `gpus[]` is not rebuilt on top of two name tables. 5. Windows GPU query. The riskiest: it needs an answer on NVIDIA and on adapter names, and a Windows lane. Scenario `examine-windows-driver-version-date-and-age` already runs there; a new scenario should compare the text's `detected_gfx_target` with the JSON `gpus[]` on a named adapter. Native Windows only; no scenario can run this on the Linux or WSL lanes. -6. Subprocess runners. Last, because items 1 to 5 reduce the number of call sites, but independent: the bounded-join fix in [#623](https://github.com/ROCm/rocm-cli/issues/623) does not wait for it and can land at any time. It has two intended differences, and the second is easy to miss because it needs no timeout: a probe behind a hung grandchild returns at its deadline instead of late (with possibly different output kept after a timeout), and a successful capture can keep less, because a descendant that writes more than `DRAIN_GRACE` (200 ms) after the child exited loses that later output where the older helpers waited for EOF. Each of the other successful-run differences listed under "Subprocess runners" needs its own line in the PR text. The characterisation test for this item comes first and must include late descendant output: a child that prints `early`, exits 0 and leaves a descendant that prints `late` after about 0.5 s, run through each of `run` (over `run_raw`, which `run_utf16le` shares), `capture_optional_command*` and `run_with_timeout`, pinning that the first two return both lines and `run_with_timeout` only `early`. Hosts covered: Linux and WSL with shell fixtures (the same shape the `process.rs` tests already use for the grace); `capture_diagnostic_command` and the Windows spellings are Windows only, so on the Windows job they get the injectable-function test of the adapter's result translation instead. UNCONFIRMED whether a Windows child can be arranged to leave a late-writing descendant in the test harness; if not, the PR says that path is untested and why. +6. Subprocess runners. The first step is delivered (see "Decided and delivered, first step" under "Subprocess runners": the executor, the carrier and the three adapters, with every difference it moved listed there); the later steps stay open and are listed after this paragraph. What follows is the item as it was planned, kept as the record. Last, because items 1 to 5 reduce the number of call sites, but independent: the bounded-join fix in [#623](https://github.com/ROCm/rocm-cli/issues/623) does not wait for it and can land at any time. It has two intended differences, and the second is easy to miss because it needs no timeout: a probe behind a hung grandchild returns at its deadline instead of late (with possibly different output kept after a timeout), and a successful capture can keep less, because a descendant that writes more than `DRAIN_GRACE` (200 ms) after the child exited loses that later output where the older helpers waited for EOF. Each of the other successful-run differences listed under "Subprocess runners" needs its own line in the PR text. The characterisation test for this item comes first and must include late descendant output: a child that prints `early`, exits 0 and leaves a descendant that prints `late` after about 0.5 s, run through each of `run` (over `run_raw`, which `run_utf16le` shares), `capture_optional_command*` and `run_with_timeout`, pinning that the first two return both lines and `run_with_timeout` only `early`. Hosts covered: Linux and WSL with shell fixtures (the same shape the `bounded.rs` tests already use for the grace); `capture_diagnostic_command` and the Windows spellings are Windows only, so on the Windows job they get the injectable-function test of the adapter's result translation instead. UNCONFIRMED whether a Windows child can be arranged to leave a late-writing descendant in the test harness; if not, the PR says that path is untested and why. + +The later steps of item 6, each its own change and all still open: + +- Retire the adapters site by site, as items 2 to 5 remove call sites: a probe that calls `run_bounded` directly reads the typed outcome, so "not installed" stays quiet and "timed out" becomes a `probe_failures` entry without parsing a message, which the adapters cannot do because they keep their return shapes. +- Move `wait_with_output_bounded` in [process.rs](../apps/rocm/src/therock/process.rs) of `apps/rocm` onto the executor, which is the last helper of [#623](https://github.com/ROCm/rocm-cli/issues/623) that still joins its readers with no limit, and decide where `rocmd`'s `parse_sandbox_child_output` quote is bounded. +- Make the candidate loop of `capture_optional_command_with_timeout` stop on a timeout instead of starting the next spelling of a program that hung. +- Decide whether the runners refuse to resolve a bare program name through an empty `PATH` entry, and whether the adapters take a per-call drain grace: two items of the open list below. -Items 2 and 3 should add a scenario that compares `summary.kernel` with `kernel_release` and `summary.cpu` with `cpu_model` in one `--json` run, since no existing scenario does. Each comparison is scoped to the hosts where both sides are filled: kernel on native Linux and WSL, CPU on native Linux and Windows (option (a) under "CPU model"). The Linux lane can also drive a WSL variant with a simulated host root (`ROCM_CLI_TEST_HOST_ROOT` in an `e2e-test-hooks` build, with a `/dev/dxg`, a `/proc/version` that reads as WSL and a `/proc/cpuinfo`, as [hardware_root.rs](../crates/rocm-core/src/hardware_root.rs) describes), which pins the empty WSL CPU fields; `uname` still runs for real there, so a kernel value is compared rather than fixed. Assert the specific values, not only that both are non-empty. +Items 2 and 3 should add a scenario (item 3 did: `examine-45` to `examine-47`) that compares `summary.kernel` with `kernel_release` and `summary.cpu` with `cpu_model` in one `--json` run, since no existing scenario does. Each comparison is scoped to the hosts where both sides are filled: kernel on native Linux and WSL, CPU on native Linux and Windows (option (a) under "CPU model"). The Linux lane can also drive a WSL variant with a simulated host root (`ROCM_CLI_TEST_HOST_ROOT` in an `e2e-test-hooks` build, with a `/dev/dxg`, a `/proc/version` that reads as WSL and a `/proc/cpuinfo`, as [hardware_root.rs](../crates/rocm-core/src/hardware_root.rs) describes), which pins the empty WSL CPU fields; `uname` still runs for real there, so a kernel value is compared rather than fixed. Assert the specific values, not only that both are non-empty. No item is output-preserving without qualification, and the PR for each says which of its differences are intended instead of claiming nothing changed. Items 2 to 5 change visible output (a kernel or CPU string, a gfx target, an adapter name or vendor). Item 1 changes which tools are reported as present, per the list under item 1. Item 3 changes nothing on WSL under option (a). Item 6 changes timing and timed-out output, and the output of a successful capture in the late-descendant case; a user sees most of it as a probe that no longer hangs, and that case as a fact that can go missing from a probe that used to wait for it. For every listed difference the PR either adds a scenario that exercises it in `tests/e2e-cucumber/features/` or states why none can: the empty-entry and non-UTF-8 cases can be driven from a scenario that sets `PATH` and plants a file in the working directory, whereas the Windows candidate spellings cannot run on the Linux lane and need the injectable-function unit test on the Windows job (UNCONFIRMED that the harness can set a non-UTF-8 `PATH`; if it cannot, the PR says so). @@ -200,10 +320,11 @@ No item is output-preserving without qualification, and the PR for each says whi - How NVIDIA adapters are represented once the examination reads the inventory (a vendor on every row, or a separate check). - Whether one `rocm examine --json` run shares answers between the stacks (a single WSL summary and a single `ldconfig -p`, one inventory including its failure) or keeps calling shared code that asks again. - What an absent Windows inventory does once there is one inventory: remember the absence for the run, or drop the per-fact fallbacks. -- Whether an empty `PATH` entry counts as the current directory in the surviving lookup. - Whether `gpus[].name` should come from `pnputil` or from WMI. -- Which timeout wins for the kernel and CPU reads, and whether `kernel_release` should be filled on Windows. -- Whether the typed outcome belongs on `CommandCapture` or beside it. -- Whether CPU facts should be collected on WSL at all (option (b) under "CPU model"), and if so how the Windows-hosted distro probe obtains them. +- Whether `kernel_release` should be filled on Windows (the kernel timeout is decided, see "Kernel version"; the CPU read took one 5 s limit, see "CPU model"). +- Whether CPU facts should be collected on WSL at all (option (b) under "CPU model", which the CPU-model change did not take), and if so how the Windows-hosted distro probe obtains them. Scenario `examine-47` and `on_wsl_the_summary_reads_the_cpu_and_the_examination_does_not` pin the empty WSL fields until it is decided. +- Whether the Windows reads should be given more or less time than 5 s once they can be measured on a host, and whether the examination should reuse the summary's inventory instead of running the query itself (the CPU read is still one query per stack). - Whether the adapters over `run_with_timeout` take a per-call drain grace, so that a probe whose tool is known to leave a late-writing descendant keeps its output, or whether the 200 ms grace is accepted for every probe. +- Whether the number of reader threads an abandoned capture leaves behind needs a bound. The executor bounds the memory (a capture that has been taken stops accumulating) but not the threads: each stream still held open by a descendant when the grace ends leaves one reader parked in `read` until the descendant lets go (two when it holds both), and `rocmd` probes every 60 seconds. A parked reader costs a stack and no CPU, and closing the read end instead would send the descendant `SIGPIPE`, so the first step accepts it; the reasoning is in the doc comment of `Capture` in [bounded.rs](../crates/rocm-core/src/process/bounded.rs). - Whether the near copy of `tool_path_candidates` in `apps/rocm` joins the PATH lookup change. +- Whether the runners that replace the subprocess helpers should refuse to resolve a bare program name through an empty `PATH` entry, so that what the lookup calls absent is also what they do not start. diff --git a/docs/llm-tool-use.md b/docs/llm-tool-use.md index 97c6080daf..67667c83c5 100644 --- a/docs/llm-tool-use.md +++ b/docs/llm-tool-use.md @@ -29,7 +29,18 @@ rocm-cli local assistants use structured tools, not shell commands. The provider-assisted planner's argv is held to the same rule, and a planner note carrying such a character is left out and counted. - CPU fallback is not a supported path. GPU-required ROCm commands must fail - loudly when the ROCm GPU path is not ready. + loudly when the ROCm GPU path is not ready. Keeping some of a model's layers + in system RAM on a GPU that is ready (llama.cpp's `--n-gpu-layers`), which + `rocm diagnose --model` suggests when a GGUF model does not fit in VRAM, is + not CPU fallback, and the assistant may relay it. +- A `rocm serve` the model proposes through `rocm_command`, or the + provider-assisted planner proposes in its tool call, is refused unless it + asks for no public bind (`--allow-public-bind`), no CPU device, a loopback + `--host` if it names one, and managed serving (`--managed`, never + `--foreground`). One shared check applies those four rules to both, in that + order, and a refused call is neither proposed for approval nor planned; the + two refusals read alike and differ only in naming the local assistant or the + provider planner. - The built-in local assistant is fixed to `qwen` (`Qwen3-4B-Instruct-2507-GGUF`) served by Lemonade. vLLM, Lemonade and AIM (a Docker-based engine) are the general serving engines; the assistant may inspect or manage them for model @@ -44,6 +55,43 @@ rocm-cli local assistants use structured tools, not shell commands. where vLLM is the supported path. The assistant should answer platform questions from the injected facts and must never suggest CPU fallback. +## Tool Result Shape And Size + +A tool result is an MCP envelope: a `content` array of text items, an optional +`structuredContent` object, and `isError`. For a command-backed tool the +envelope carries the command's output twice — as the `content` text and again +as `structuredContent.stdout` — because `rocm mcp-call` consumers read the +structured field. That full envelope is what `rocm mcp-call` prints and what +the approval replay path stores. + +What reaches the *model* is narrower, because a tool result is not free: it is +sent back as a message and counts against the model's context on that request +and on every later turn of the same conversation. A locally served model is +commonly configured with a 32k-token context (llama.cpp's default `n_ctx`), +and the system prompt plus the tool schemas already take a few thousand of +those. So the dash shapes each result before the model sees it +(`run_rocm_read_tool` in +[tools.rs](../crates/rocm-dash-tui/src/agent/tools.rs)): + +- **One copy.** `structuredContent.stdout` and `.stderr` are dropped when the + envelope's own text already carries them. Fields that are not duplicates — + `argv`, `exit_status`, and the data of tools whose payload lives only in + `structuredContent` — are kept. +- **A size cap.** A result is bounded to `TOOL_RESULT_CHAR_BUDGET` characters + of serialized JSON (16 KiB, roughly 4.5k tokens). A result within it passes + through unchanged. One that is not is replaced by its plain-text rendering, + cut on a line boundary, ending in + `[output truncated: A of B characters shown, C omitted ...]`. The marker is + part of the contract: the assistant skill tells the model to report a + truncated result as partial rather than as a complete answer. + +The CLI chat path (`rocm chat --prompt --tools`) reads only the envelope's +text, so it never sent the duplicate copy; it does not apply the cap. + +Read-only commands are rendered so that the cheap form is the default: +`rocm_command` with `["model"]` returns the curated list `rocm model` prints, +and the per-recipe diagnostics need an explicit `["model", "--verbose"]`. + This follows the same shape described by current tool-use docs: the application defines tool schemas, the model requests a tool, the application executes the tool, and the result is returned to the model for the next response. MCP tool diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 2171fdd3a5..f295a2a0ba 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -111,6 +111,20 @@ Expected result: - `rocm runtimes list` shows the runtime key for the installed TheRock venv. - The active runtime is ready, or the output gives one clear next command. +Optional: an unreadable settings file. Copy `config.json` from the config +directory aside, cut the original off part-way through, and run `rocm version` +without `ROCM_CLI_DATA_DIR` set. Expected result: + +- The command fails and names `config.json`, saying it cannot be read, and + explains the three ways out: repair it; set `ROCM_CLI_DATA_DIR` (it names the + folder if the damaged file still shows it), after which commands that change + settings still stop and others either stop or run on default settings; or + rename it to the `config.json.broken` path it names, which resets every + setting and leaves runtimes in a custom folder reachable only through + `ROCM_CLI_DATA_DIR`. +- `config.json` is byte-for-byte what you left; nothing was saved over it. +- Restoring the copy makes `rocm runtimes list` show the same runtimes as before. + ## 2. TheRock SDK Command-Line Install This tests the command-line install path without using the TUI: @@ -158,6 +172,24 @@ Expected result: asking and prints a line crediting that flag by name — not `--yes`. - `rocm install sdk ... --dry-run` never prompts or refuses, whatever the active default is: the preview stops before the gate. +- Which source a preview reads follows the request, and the plan's + `canonical_source:`, `index_url:` and `source_layout_generation:` lines say so. + With no pin, `--channel release --family gfx1200` reads + `https://stable.repo.amd.com/rocm/whl-next` (`next-v1`) and `--channel nightly + --family gfx1200` reads `https://nightly.repo.amd.com/rocm/whl-next`, at the + newest build that has its PyTorch wheels (with a `warning:` line when a newer + `rocm` build does not yet). Pinning an earlier release or nightly reads the + earlier stream instead: `--version 7.14.1` reads + `https://repo.amd.com/rocm/whl-multi-arch` (`multi-arch-v2`), and `--channel + nightly --build-date 2026-08-19` reads + `https://rocm.nightlies.amd.com/whl-multi-arch`, while `--build-date + 2026-09-01` reads the newest layout's nightly index. `--format tarball` with no + pin reads `https://stable.repo.amd.com/rocm/core/tarball/` (or + `https://nightly.repo.amd.com/rocm/core/tarball/` on the nightly channel), and + `--format tarball --version 7.14.1` is refused, pointing to `--format wheel`. + No unpinned install asks for a family label: `--family gfx120X-all` with no + `--version` is refused naming `--family gfx1200`, while `--family gfx120X-all + --version 7.14.1` previews from the earlier stream. - On a host with no usable Python (a minimal container; run with `PATH` pointing at an empty folder and no `ROCM_CLI_PYTHON`) and a clean data folder, `rocm install sdk ... --dry-run` still prints a plan. Its `python_launcher:` @@ -169,17 +201,47 @@ Expected result: plan's `command:` line shows `uv venv --python 3.12 ...` (a version, as no interpreter path exists yet), so it is not copy-pasteable verbatim: `uv` resolves `3.12` itself and may pick a different Python 3.12 than the managed - one a real install prepares. This holds for the canonical layout. A preview - pinned to ROCm 10 or newer (`--version 10.0.0 --family gfx1200`) still needs - `uv` to resolve the package versions, so on a clean host it can download the - managed `uv` and create the `tools` folder, although it prepares no Python and - saves no Python path. + one a real install prepares. This holds for every layout, and the version in + that line follows the install: 3.12 for a ROCm 7 pin (`--version 7.14.1`), + 3.14 for the default and for ROCm 10 or newer. A preview of ROCm 10 or newer, + which is what an install with no pin gets (`--family gfx1200`, or + `--version 10.0.0 --family gfx1200`), needs `uv` to choose + the `torch` and `torchvision` versions, so on a host with no `uv` installed it + stops before that step: the data folder still has no `tools` folder, a + `resolution_note:` line says why, and `target:`, `runtime_key:`, + `package_specs:`, `command:` and `manifest:` read `not resolved by this + preview`. Run the real install (or any command that fetches `uv`), then the + preview again, and it prints the complete plan; so does a preview with + `ROCM_CLI_UV_BINARY` naming a `uv`. +- On the same host, with a wheel runtime registered whose manifest records no + usable Python, `rocm update` (also `--json`, `--dry-run`, and the check the + CLI runs at startup) still reports. The text report gives the runtime's + verdict and a `python_note:` line saying an update would use `uv` to install + the managed Python, which the check did not do. Afterwards the data folder has + no `tools` folder and the config folder has no `config.json`. That holds for a + canonical-layout runtime; for a ROCm 10 or newer runtime the check still + resolves the package versions with `uv` and can fetch the managed `uv` into the + data folder, though never a Python. `rocm update --apply` on the same host does + prepare the Python, since the install needs it. - With `ROCM_CLI_DISABLE_MANAGED_PYTHON_BOOTSTRAP=1`, or a `ROCM_CLI_MANAGED_PYTHON_VERSION` whose Python the layout does not accept (such as `3.13` for the canonical layout), on the same host the preview fails with the install's own message. Its output before the failure states only what was found (for example that the `PATH` Python is not usable) and never says a real install would prepare or use a managed Python. +- `rocm install sdk --version 10.0.0 --family gfx1200 --python-version 3.12 + --dry-run` plans Python 3.12 (`python_wheel_tag: cp312`, and a managed Python + 3.12 in the note on a host with none), and nothing in the plan says 3.14. + `--python-version 3.13` on a ROCm 7 install (`--version 7.14.1`) is refused, naming + Python 3.12 as the one that install uses, and `--python-version 3.9` on a + ROCm 10 install is refused naming 3.10 to 3.14. A real ROCm 10 install on a + vLLM GPU family with `--python-version 3.12` skips the engine auto-install + with a note, while a real ROCm 7 install on that family with `--python-version + 3.12` (the redundant, accepted value) still auto-installs vLLM. +- On the ROCm 10 runtime installed on Python 3.12 (including an install with + no pin, which now resolves to ROCm 10 or newer), `rocm engines install vllm` names + the wheel tag it needs, and `rocm update` of that runtime builds the new one + on Python 3.12 again. - rocm-cli creates or reuses a rocm-cli managed Python venv. - pip installs pinned `rocm`, `torch`, and `torchvision` requirements with exactly one `device-` extra (`rocm` also requests @@ -212,7 +274,7 @@ Expected result: Developer-only deterministic override: ```powershell -python scripts\therock_sdk_install_test.py --dry-run --family gfx120X-all +python scripts\therock_sdk_install_test.py --dry-run --family gfx1200 ``` Use `--family` only when a test needs a fixed package family. Do not use it for @@ -294,6 +356,32 @@ On Ubuntu 22.04, 24.04 and 26.04 the apt source line names the matching suite first prepare command prints a "from a previous installation" warning; confirm nothing is removed (`dpkg -s amdgpu-dkms` still reports installed afterwards). +## 2c. Fix Catalog Exit Codes + +On any host, Windows or Linux: + +```bash +rocm fix --json # read exit_codes and exit_code_descriptions +rocm fix fix-does-not-exist; echo $? +rocm fix --no-such-option; echo $? +``` + +Expected result: every name under `exit_codes` also appears under +`exit_code_descriptions` with a sentence of text. Both later commands exit `2`, +the number `exit_codes.unknown_id` gives. The first prints a stderr line +`Unknown fix-id: fix-does-not-exist`; the second prints a stderr line starting +`error:` and never says `Unknown fix-id:`. A one-time `rocm:` notice (a moved +uv cache, a migrated dashboard config) may come first on either. The +`unknown_id` description says exactly that. A different exit code for either, +or a description that no longer mentions the malformed command line, is a +regression. In PowerShell read the code with `$LASTEXITCODE` instead of `$?`. + +A negative device index depends on how it is written. The parser accepts +`rocm fix fix-9-igpu-dgpu --device-index=-1`, and the fix refuses it with exit +`3`, which the `not_applicable` description covers. With a space, +`--device-index -1` is read as an unknown option and exits `2` like any other +usage error, which the `unknown_id` description covers. + ## 3. Lemonade GPU Verification Lemonade is the default local assistant/server engine. Serve a small assistant @@ -305,10 +393,12 @@ rocm serve qwen --engine lemonade --device gpu_required --foreground --port 1143 Expected result: -- The engine uses ROCm GPU execution. -- The run does not fall back to CPU or Vulkan. +- The engine uses its selected supported GPU backend, ROCm or Vulkan. +- The run does not fall back to CPU, and does not select a stale ROCm binary + when the installed runtime selects Vulkan. - If the GPU cannot be used, the command fails with a clear error. - The OpenAI-compatible endpoint answers a simple chat request. +- `rocm services list` shows an `anthropic endpoint:` URL, and `POST /v1/messages` with `{"model":"x","max_tokens":16,"messages":[{"role":"user","content":"Hello"}]}` returns an Anthropic `message` that names the served model. The same request with `"stream":true` answers with `text/event-stream` (`message_start` through `message_stop`, text in `content_block_delta` events). A request with `tools` and `"tool_choice":{"type":"tool","name":""}` returns a `tool_use` content block naming that tool with its arguments and `"stop_reason":"tool_use"`; sending the call back as a `tool_result` block gets the model's answer. Combining `"stream":true` with `tools` is refused with an `invalid_request_error`. If the llama.cpp server exits at start with an access violation (`0xC0000005` in `MSVCP140.dll`), the Visual C++ runtime on the machine is probably older than @@ -316,6 +406,36 @@ in `MSVCP140.dll`), the Visual C++ runtime on the machine is probably older than fix-21-msvc-redist-outdated` prints the update plan and changes nothing. After the update, restart Lemonade and repeat the serve command above. +With that foreground server stopped, check that a catalogued recipe reaches +Lemonade with a build it can load, from a short name alone — no `--engine`, no +quantization variant typed by hand: + +```powershell +rocm serve qwen-small --managed +``` + +Expected result: + +- The resolved name is `Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M`, not the + requested `qwen-small` and not the safetensors id + `Qwen/Qwen2.5-1.5B-Instruct`. In a terminal that is the deployment summary's + `resolved model` row, shown beside `requested model` because the two differ; + pipe the command (`| Out-String` in PowerShell, `| cat` in a POSIX shell) to + get the `serve plan` form carrying the same line. +- The service reaches `ready` and answers a chat request. On Windows in + particular it must not exit immediately leaving an empty engine log: a + safetensors-only id handed to Lemonade fails that way, inside the engine + process, where the reason never reaches the terminal. +- The same name under `--engine vllm` resolves to `Qwen/Qwen2.5-1.5B-Instruct` + instead (Linux or WSL, Instinct hardware). + +Stop it again before moving on, so it does not hold the GPU and the port for +the checks below: + +```powershell +rocm services stop --yes +``` + To check the recovery from a stale Lemonade backend pin (Windows or Linux with a managed Lemonade runtime already installed and an active managed ROCm SDK): the recovery only runs when the pinned `therock.version` equals the active @@ -333,6 +453,36 @@ ROCm backend directory under `bin/llamacpp/`, and run - `therock.version` and `llamacpp.rocm-stable` in `backend_versions.json` hold the packaged values again, and the unrelated key is still there. +To check what a Lemonade install says about its size (no download needed for +the refusal): with a debug build, set +`ROCM_E2E_LEMONADE_FREE_SPACE_BYTES=1048576` (1 MiB free), then in a fresh data +folder run `rocm engines install lemonade --yes`. Expected result: + +- Before anything else it prints `Installing Lemonade into ` + and a line giving the ROCm backend's download (up to about 4.4 GiB), disk use + (up to about 25 GiB) and the free space on that drive. +- It then fails with `not enough free disk space to install Lemonade: ... + Nothing was downloaded.`, and `` has no `downloads` or `runtime` + subfolder. +- With `ROCM_E2E_LEMONADE_FREE_SPACE_BYTES=2147483648` (2 GiB) instead, it + warns that only 2.0 GiB is free for a backend of up to about 4.4 GiB and + carries on: Lemonade picks the backend after this point and a smaller one may + fit. (Stop it before the download if you do not want one.) + +With room to spare, on Windows from an interactive console: install Lemonade +with no runtime active, then `rocm install sdk`, then `rocm serve `. +Expected result: + +- The serve prints the install notice for the runtime's own + `engines\lemonade` folder, and says the data folder's copy is not reused. +- Lemonade's own backend download progress (`[1/2] ...`, `Total: ...`) appears + in the console instead of a silent wait. +- `rocm storage report` lists both copies under `Engine installs`, with the data + folder's one `not used right now` (and reclaimable) and the runtime's one `in use`. +- `rocm runtimes uninstall ` says Lemonade's install was inside the + runtime and removed with it, and a following `rocm storage report` shows the + remaining copy as `not recorded as installed`. + While the log stream is attached, verify detach and stop behave differently: - Press `Ctrl+D`. The stream ends and the shell prompt returns, printing a @@ -444,6 +594,107 @@ promise on the Windows GPU lane with whatever driver that machine has. Only the old-driver branch needs this manual check: it needs a machine whose display driver is dated before 27 July 2023. The logic is also covered by unit tests. +### Named models and dashboard discovery + +On Linux or WSL, repeat the serving and inference check with a non-default +registered model and with its explicit checkpoint: + +```bash +rocm serve qwen-smoke --engine lemonade --managed --port 11435 +# Stop the test-owned service before reusing its port. +rocm services stop --yes +rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --engine lemonade --managed --port 11435 +``` + +`rocm serve` returns to the shell once the server is ready. Background is the +default, and `--managed` is accepted as an explicit synonym. Only `--foreground` +or `--verbose` attach to the log stream. Use `rocm services` to find the service id. + +For each run, confirm that the endpoint answers inference for the requested +model and that `rocm dash` shows the managed model. A startup announcement alone +is not proof of a working endpoint. Stop only the services created by this check. +Simple single-GGUF catalog entries can use direct serving on Linux; registered +entries needing extra files or model-specific options stay with the router, as +does a registered checkpoint that is not a plain `owner/repo:variant`, such as +one whose variant is `*`. +Explicit checkpoints require the selected GPU backend and must not fall back to +another backend or CPU when it is missing. + +For the complex registered-model check, stop your previous test service and run: + +```bash +rocm serve Qwen3.5-0.8B-GGUF --engine lemonade --managed --port 11435 +``` + +This catalog entry needs both its GGUF checkpoint and projector file. Confirm +that inference identifies the requested model and the engine state records +`load_response.method` as `lemonade-cli`, not a packaged direct-server launch. +Stop that service with `rocm services stop --yes` and confirm its +endpoint closes. The automated equivalent is the nightly GPU scenario +`serve-lemonade-complex-registered-inference` in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature). + +For a simple registered name on Linux, such as `Qwen3-0.6B-GGUF`, the engine +state records `load_response.method` as `lemonade-packaged-llama-server` and a +non-CPU `backend_requested`. The automated equivalent is +`serve-lemonade-simple-registered-direct-route` in the same feature file. + +For external discovery on Linux or WSL, use a separately running Lemonade server +with a loaded model. The embedded dashboard daemon probes `127.0.0.1:13305` by +default. For a non-default endpoint, set these fields in the CLI's `config.json` +(`~/.rocm/config.json`, or `config.json` in the directory named by +`ROCM_CLI_CONFIG_DIR`), preserving other configuration: + +```json +{ + "dashboard": { + "daemon": { + "enable_lemonade": true, + "lemonade_host": "127.0.0.1", + "lemonade_port": 13305 + } + } +} +``` + +Confirm the following through the dashboard and the external server: + +- Each loaded model appears, without creating a managed service record. +- With multiple loaded models, throughput is unavailable because endpoint-level + last-request statistics cannot be attributed safely. A managed direct + `llama-server` also shows unavailable throughput when it has no Lemonade stats API. +- A managed Lemonade server that answers its health route with 401 or 403 (API + key required) stays listed with its registry status, no throughput, and no + repeating health warning. External discovery still warns on such a response. +- A shown rate is the most recent completed request's rate and does not decay + while the server is idle. The Instances table shows the legend `Lemonade: last req + tok/s, not live` and the instance detail reads `last req: ` + instead of `gen: `; a managed Lemonade row in the Services overlay reads + `req ` instead of `gen ` under the same legend, and its quick preview + reads `last req `. A vLLM row shows neither. The row's tok/W shows `—`, + not an efficiency computed from that rate. With only an idle Lemonade server + loaded, the Home and Observe tok/s and tok/W heroes show no throughput instead + of repeating that rate. At 80 columns the legend, the instance detail's freshness + line and the empty Instances state are all still readable. +- The engine pane distinguishes managed installation from external availability, + even when GPU telemetry is unavailable. It reads the managed-runtime status once, + when the dashboard starts. After you install Lemonade from the Engines overlay, + restart the dashboard to see the new status. When the status cannot be read, the + pane says "unknown, run `rocm engines list`", and that command prints each + engine's detect result. +- Unloading a model removes it from the running-model list; reloading restores it. + A healthy but unloaded server does not claim a running model. +- In Observe, open Services with `s`. External models are not managed-service + targets. With only external models, the list reports no managed services; + pressing `s` or `r` stages no approval and starts no job. Close the list with + Escape and confirm the external endpoint is still reachable. +- Closing the dashboard also leaves the external server reachable. +- Setting `enable_lemonade` to `false` disables external discovery on the next + embedded-daemon launch; managed service records remain visible. + +Native Windows has no live dashboard telemetry. These dashboard checks apply to +Linux and WSL, not the native Windows CLI. + ## 4. Local Server Records After a managed or foreground serve attempt, inspect local server records: @@ -487,6 +738,40 @@ Expected result: with `the server exited before it became ready` and a `rocm logs --service` pointer, and the record shows `stopped`. - The logs command shows the exact service failure or startup output. +- The log views tell an empty log from one they could not show, and each view + reads its own files, so check each separately. In a Linux or WSL shell, + `head -c 5000000 /dev/zero | tr '\0' x > ` makes a file that is one line + of more than 4 MiB without a line break: + - A server's log. Write that file over the log file named on the `Log file:` + line of `rocm services logs ` (by default `services/.log` + under the data folder), then run `rocm services logs ` or + `rocm logs --service `: both print `Recent output: none shown; + the newest line is longer than the 4 MiB a log view reads`, not `no output + yet`. Truncate the file to nothing and both print `Recent output: no output + yet` again. + - The `rocm logs` browser, which does not read a server's log: it lists the + activity log and the files under `logs/cli/` and `logs/tui/` in the data + folder (the `Folder:` and `Command logs:` lines of `rocm logs` name them). + Write that file to `logs/cli/probe.log` there, run `rocm logs`, and expect + `command log probe.log: ` in the `Lines:` list, not `No logs + found yet.`. The same file written over the activity log instead gives + `Recent activity: none shown; the newest line is longer than the 4 MiB a log + view reads`, not `no activity yet`. + - `rocm comfyui logs`, which reads the saved ComfyUI logs: write the file under + `apps/comfyui/logs/` in the data folder and expect `No output shown: the + newest line is longer than the 4 MiB a log view reads.`; an empty file there + gives `The saved log is empty.`. + - A stray byte that is not valid text in any of these logs shows the readable + lines with a replacement character, not a `failed to read` line. +- The `service_logs` tool (`rocm mcp-call service_logs --arguments-json + '{"service_id":"","lines":50}'`) reads the same server log as + `rocm services logs` and answers with one of three summaries: `has no log + output yet` for an empty log, `has log output, but none is shown` with the same + 4 MiB clause for the over-long line above, or `Read the last N line(s)`. Its + structured reply carries `lines` (the number returned, such as 3 for a + three-line log), `lines_requested` (the 50 asked for) and + `newest_line_exceeds_read_bound`. - `rocm storage report` lists these records under `local server records` with the folder that holds them; no `rocm storage` command deletes them, and the row's note points at `rocm services prune` to reclaim the space. @@ -500,16 +785,81 @@ rocm services restart --yes `stop` prints `Local server stopped` only once every recorded process is confirmed gone; otherwise it prints `Local server stop not confirmed` on stderr, keeps the record and any endpoint key, and exits non-zero. It names each -process that, or a process under it, did not exit, and for one it could not -verify, an operating-system command to check on it. It never prints a `retry:` -line. +process that, or a process under it, did not exit. On Windows it also names one +that did exit while taking the processes under it down did not complete +(`not confirmed: process () exited, but a process under it may still +be running`), and one that was already gone when that step ran with no other +recorded process confirmed gone (`not confirmed: process () was +already gone when the processes under it were to be taken down`). For one it +could not verify, it names an operating-system command to check on it. It never +prints a `retry:` line. `restart` starts the server again only once its stop has confirmed every recorded process gone; otherwise it reports that and starts nothing. +On Windows the stop takes the engine's process tree down with the recorded +process (`taskkill /T /F`, which has to finish within the stop's grace period of +10 seconds, or the stop is not confirmed), and it checks the recorded process +against the creation time written when the server started, so a process that has +since taken the same PID is left alone and reported as `identity_mismatch`. A +process it cannot verify, on either platform, is not signalled and is named with +the check the host has: `ps -p -o pid,args` on Linux and WSL, and +`tasklist /FI "PID eq "` on Windows. There are two ways to be unverifiable: +the recorded start-time can no longer be read back from a running process, or +(Windows only) the PID cannot be queried at all, as for a live process the +account may not open. The text of `rocm services stop` does not list a per-process +outcome for the stopping process itself; the JSON answer of the sandbox +`stop_server` tool does, in `pid_outcomes`, as `self` (never signalled, counted +as stopped). + +#### Windows host check + +The Windows build-and-test job runs the workspace unit tests, which include two +`#[cfg(windows)]` tests in `proc_lifecycle.rs` that stop a real `ping` process +through the PID hold, the creation-time read and the System32 `taskkill`. It does +not run any scenario, and nothing there stages the unrun parts: the end-to-end +tree kill of a parent with a child (`service-stop-07`), a real unopenable +process (the `OpenProcess` failure codes), and `taskkill` exiting 128 or failing +or stalling. None of these has been run on a Windows host, and no Windows result +is claimed. On a Windows host with a `--features rocm/e2e-test-hooks` debug +build and `ROCM_CLI_BINARY` set, run the scenarios that apply to Windows: +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-0[178]"` (`-01` +plants PID 0, `-07` is Windows-only, `-08` runs on every OS) and +`cargo test -p e2e-cucumber --test e2e -- -n "service-unqueryable-01"`, and +record the result. Do not select `service-stop-02` to `-06` or +`service-unqueryable-02`: they are Linux-only (they need `sleep` or `/proc`), +and `-n` bypasses the harness's OS filter, so they would fail for want of the +platform rather than for any defect. + +#### A record that stays unverified + +If the server is gone but its recorded PID has been taken by a process this +account cannot open, `rocm` cannot tell the two apart, and it treats that as +possibly running, which is the safe direction: the record never reconciles to +`stopped`, `rocm services remove` and `rocm services prune` refuse it, and +`rocm services stop` keeps reporting `unverified`. To clear it: + +1. Run the check command the stop printed. If the process is this server's, end it + yourself (from an elevated prompt if the account may not), then run + `rocm services`; the record reconciles and its endpoint key is dropped. +2. If it is an unrelated process, run `rocm services remove --force + --yes`. The refusal of a plain `remove` names exactly this command. `--force` + applies only when a stop was requested (the stop that reported `unverified` + did that), no recorded process is confirmed running, and at least one could + not be queried; a process `rocm` can see running is never forced past. The + command says which process could not be queried and that, if it is still + running, it keeps running, and it deletes the record, its log, its engine + state and its endpoint key. The server can no longer be restarted afterwards. + `rocm services prune` has no `--force`: it leaves such a record alone and lists it + under `Left alone` with this exact command. + If the record still reads `ready` and nobody has run `rocm services stop` on it, + `--force` is refused and the refusal names that stop first; run it (it reports + the process unverified and records the request), then the forced removal. This is + the intended order, not a gap: `--force` never removes a record nobody tried to stop. + The rest of this section needs a **vLLM** server running, on Linux or WSL, and two runtimes to move it between. Nothing earlier in this document sets either -up: section 3 serves Lemonade and then stops it on both of its branches, the +up: section 3 serves Lemonade and then stops it on each of its branches, the block just above stops whatever was left, and section 2 leaves one runtime. **Get two distinct runtime keys.** A wheel runtime key is built from the @@ -720,21 +1070,19 @@ Expected result: - A prompt appears naming the serving model and warning that it may still be running in the background after you quit, with the real `rocm services stop --yes` command (the actual id, not a - placeholder) to stop it first. From the dashboard (not the launcher) on - Windows, where the dashboard streams live GPU/CPU/memory but never gets a - serving-instance snapshot (no `rocmdashd` unix socket there), the prompt - still shows that same stop command and the service id, but not the - model's name — the dashboard's own live view is where the name comes - from, and it never fills there. The launcher's own copy of the prompt - always names the model, Windows included — it reads the managed-service - registry directly rather than through the dashboard's daemon connection. + placeholder) to stop it first. On Windows the dashboard has no + `rocmdashd` unix socket, so it reads the same managed-service registry the + launcher does: the prompt names the model there too. Within the first + second after `rocm serve` registers a service, before the dashboard's next + registry read, it may show the stop command and id without the model's name. - From the dashboard (not the launcher), the prompt also offers `s`: pressing it closes the prompt and opens the Services overlay, pre-focused on the serving model, so you can stop it there through the overlay's own - approval+job flow. The launcher's copy of the prompt does not show this - hint or respond to `s` — it has no live Services overlay to jump to — and - neither does the dashboard on Windows, for the same reason the model's name - is missing there. + approval+job flow, on Windows too. The offer appears once the dashboard + lists the service as managed; for a service started a moment earlier it is + absent and the printed stop command is the way to stop it. The launcher's + copy of the prompt does not show this hint or respond to `s` — it has no + live Services overlay to jump to. - Declining (`n`/Esc) returns to where you were — the dashboard, or the launcher's own front door, whichever you pressed `q` from. Any plain modal that was open when you pressed `q` (the Esc menu, Options, Detail, Theme @@ -829,14 +1177,91 @@ Expected result: present. With more than one adapter (for example a hybrid laptop) VRAM total stays unmeasured and shows `—` in the Home hero and in the GPU wall sidebar, never `0.0%`. +- The hero's `T/S` row reads `—`, never `0` or `-0`: a managed server is + listed with no throughput telemetry behind it, so no rate is ever measured + there. +- With a managed server running (`rocm serve`, then `rocm services list` + showing it `ready`), the Home tab's `Running` count, the Serving tab's + `Running now` list and the Observe tab list that server within a few seconds + of `rocm dash` opening, not "Nothing running". After `rocm services stop` + it drops off within a few seconds. Its throughput stays unmeasured. - Utilization rises under GPU load (run a Lemonade or ComfyUI job) and falls back when it ends. - Hiding `amdadlx64.dll` (for example on a non-AMD GPU host) leaves utilization and VRAM usage working and temperature/power/clock at their unmeasured values, with no crash. +- Leave the dashboard open for at least 30 minutes on an AMD host with the + driver's `amdadlx64.dll` present. It must keep running and keep updating + temperature, power and clock. ADLX is initialised once for the whole + session; initialising it on every tick made the driver's `atiadlxx.dll` + abort the process (exit code `0xC0000409`) with no message. If the + dashboard still exits silently, Event Viewer > Windows Logs > Application + shows an `Application Error` entry for `rocm.exe` naming the faulting + module and offset. - No other output of this section needs a daemon: `rocm dash` must not report a failed daemon connection. +### 4c. Dashboard Hero In A Short Terminal + +**Any platform.** The hero's row dropping is terminal layout, not a +Windows-specific path, so this reproduces on Linux and WSL too. + +Open a terminal about 22 to 24 rows tall — the VS Code panel is one — and +run: + +``` +rocm dash +``` + +Expected result: + +- The hero still shows utilization, VRAM, temperature and power. It never + renders as an empty box with only its "Node throughput · …" title. +- As the terminal gets shorter the hero loses rows from the bottom only: the + held legend first, then `T/S`. Enlarging it brings them back. +- With a model served whose figures are a scrape window old, the `*` held + marker and the `* = held (prior scrape window)` legend explaining it appear + and disappear **together** as the terminal is resized. A `*` on a reading + with no legend anywhere on screen is a bug; so is the legend with nothing + marked. +- The Hardware tab's GPU header has no room for a legend, so it spells the + same condition out: `eff 0.50 tok/W (held)`. It sits right after the GPU + count, so on an 80-column terminal the partition and VRAM text clips before + it does. A held figure that shows no `(held)` there is a bug. + + Check this one by **starting tall and shrinking**, not at the 22-to-24 rows + above. The legend is the first row the hero drops, and the marker is gated on + the legend fitting — so at 22 to 24 rows neither is on screen and there is + nothing to compare. Start around 30 rows, confirm both are present, then + shrink past the drop and confirm both go at once. + +### 4d. Dashboard Chat Request Timeout + +The first chat message to a local model reads the whole system prompt and tool +list, which can take tens of seconds on modest hardware. With a model served by +`rocm serve`, open the dashboard chat and send a message: + +```powershell +rocm dash +``` + +Expected result: + +- A first message that needs 20 to 60 seconds to answer is answered, not cut off. +- To see the timeout itself, shorten the wait and send a message the model cannot + answer in time: + + ```powershell + $env:ROCM_DASH_CHAT_TIMEOUT_SECS = "5" + rocm dash + ``` + + The turn ends with `request timed out after 5s (set ROCM_DASH_CHAT_TIMEOUT_SECS + to allow longer)`. +- An unusable value (`0`, `abc`, `99999`) keeps the 2 minute default instead of + disabling the wait. Clear the override with + `Remove-Item Env:ROCM_DASH_CHAT_TIMEOUT_SECS` afterwards. + ## 5. Reuse Is Exempt From The No-GPU Refusal **Linux or WSL only.** Unlike the rest of this file, the commands below are @@ -1128,21 +1553,51 @@ rocm uninstall Expected result: - The dry run lists the binaries (on Windows, `rocm.exe` is marked as deleted - as the command exits) and, under - `PATH changes from the installer that would be undone:`, the shell profile - (Linux) or the user `PATH` entry (Windows). It removes nothing. -- After confirming, the install folder holds no `rocm`/`rocmd` files, and a new + as the command exits), on Linux the `shell completions` file + `completions/rocm.fish` in fish's config folder (`$XDG_CONFIG_HOME/fish` + when that is an absolute path, else `~/.config/fish`) when the installer + wrote one, and, under `PATH changes from the installer that would be + undone:`, each shell profile the installer wrote (Linux, including fish's + `config.fish` when fish was installed) or the user `PATH` entry (Windows). + It removes nothing, and at a + terminal ends with `Nothing was removed. To remove these items, run it again + without --dry-run.` Running the same command again without `--dry-run` asks + `[y/N]` and, on yes, removes what was listed (scenario `uninstall-07`). With + its output piped to a file or another command, or its input redirected, it + lists the same items and omits that last line (scenario `uninstall-08`). +- After confirming, the install folder holds no `rocm`/`rocmd` files, the + fish completion script is gone (its `completions` folder stays), and a new terminal no longer gets the install folder on `PATH` from the installer's change. - If another program lives in the install folder, the review instead says - `PATH change left in place` and names it, and the `PATH` change stays. -- With `rocm uninstall --keep-binaries`, the review says the `PATH` change - stays too, and it does. + `PATH change left in place` and names it, and the `PATH` change stays; the + fish completion script still goes. +- If you already have your own `completions/rocm.fish` (or a link there) when + you run the installer, it prints `note: fish completions not installed: … + already exists and was not written by this installer`, changes nothing, and + a later uninstall leaves the file (scenario `lifecycle-35`). +- If you replace `rocm.fish` with your own file before uninstalling (for + example `rocm completions fish > `, which writes no installer + marker line), the review lists it under `Left alone:` as not written by the + installer, and the file stays (scenario `lifecycle-33`). +- If the `completions` folder cannot be written to (`chmod 0555` it), the + uninstall prints `warning: could not remove shell completions `, + still removes everything else, and ends with `uninstall complete` + (scenario `lifecycle-34`). +- With `rocm uninstall --keep-binaries`, the review says the `PATH` change and + fish completions stay too, and they do (scenario `lifecycle-25`). - On a home folder ROCm CLI never touched, the dry run says `Nothing managed by rocm-cli would be removed.` and leaves the home folder empty (scenario `uninstall-06`): run `HOME=$(mktemp -d) rocm uninstall --keep-binaries --dry-run`, then `ls -A` - on that folder. + on that folder. It adds no line about how to remove anything (scenario + `uninstall-09`). +- Passing `--force` to `rocm uninstall` is refused with a message that says it + is not an option and points to `--yes`; a near miss such as `--forc` is an + unknown-argument error (exit 2). Neither + suggests `--force-dev-binaries` (scenario `uninstall-10`). + `rocm uninstall --help` ends with examples that start with a dry run + (scenario `uninstall-11`). - Without `--yes` and without a terminal, the command prints the same review, then refuses with `uninstall requires --yes outside an interactive terminal` and exit code 1, and removes nothing. On Linux or WSL, check it with no @@ -1153,7 +1608,13 @@ Expected result: HOME=$(mktemp -d) rocm uninstall --keep-binaries < /dev/null # expect the refusal; exit 1 ``` - At a terminal, the same command asks `[y/N]` and treats Enter as no. + At a terminal, the same command asks `[y/N]` and treats Enter as no. Type + `maybe` and it cancels after printing `"maybe" is not yes or no; treating it + as no.`; an answer longer than 40 characters is shown cut to its first 40, + with `...` after the closing quote. Press Ctrl-D and `uninstall cancelled` + appears on a line of its own, not after the prompt. Nothing is removed in + either case. The same prompt is shared by `rocm fix` and the SDK-overwrite + prompt, so the same answers behave the same way there. ## 11. AIM (Docker) Engine Verification @@ -1250,3 +1711,174 @@ Expected result: - Re-enabling a watcher right after disabling the last one (within the few seconds before the helper exits) still ends with a helper running: enable waits for an exiting helper to finish, then starts a new one. + +## 12. Context-Size Estimate for a GGUF Model + +Needs a host with a Radeon GPU (a discrete card: an APU answers `undetermined` +by design) and network access to Hugging Face. The point is to check the +estimate against what a real load uses: the compute buffer and runtime reserve +were calibrated on one GPU and one engine build (an MI300X with llama.cpp b9752), +so a different GPU or build can differ. How close the estimate came on them is +recorded in the module documentation of +[gguf_fit.rs](../crates/rocm-core/src/gguf_fit.rs), not repeated here. + +```bash +rocm diagnose --model /:Q4_K_M --context-size 8192 +``` + +Expected result: + +- The first lines say the header was read and the weights were not downloaded, + and the header read finishes in seconds. +- The report opens `ESTIMATED: fits at 8192 tokens` or `ESTIMATED: does not fit + at 8192 tokens`, and lists weights, KV cache, compute buffer, runtime reserve + and the total, with this GPU's dedicated VRAM and the largest context that + fits. +- Repeat with `--kv-cache-type q8_0` and `q4_0`: the KV cache figure shrinks to + about 53% and 28% of the `f16` one. +- Serve the same file with llama.cpp's `llama-server` at the same `--ctx-size` + (with `-fa on` and `--n-gpu-layers 99`, and `--cache-type-k`/`--cache-type-v` if changed) and compare the total in + the report with the GPU memory it uses once loaded (`amd-smi metric`). The + estimate should be at or a little above the real figure; if it is below it, or + far above, report the model and both figures. +- `llama-server` at the reported largest context should load. The one check of + this kind made so far was on a single MI300X, with a second process holding + most of the card's memory so that about 15.1-15.5 GiB could still be allocated + (the process's `amd-smi` baselines were 181092 and 180738 MiB used of 196592), + for a Llama-3.1-8B Q4_K_M with an f16 cache. The card was described to + `rocm diagnose` as one with 15854 MiB, for which the estimate was 79872 tokens + (77056 tokens for 15500 MiB). `llama-server` loaded at 80896, 83968, 84992 and + 86016 tokens (used memory grew by 15449, 15837, 15788 and 15767 MiB) and failed + to allocate at 87040, 90112 and 93184, so the real limit was between 86016 and + 87040 tokens and the estimate was about 7-8% under it (79872 is 7.1% under + 86016 and 8.2% under 87040). The load at 86016 tokens is the calibration load + "made with the card almost full" in + [gguf_fit.rs](../crates/rocm-core/src/gguf_fit.rs)'s module documentation, + which says how the estimate compared with it. The baseline moved + by 354 MiB between runs, so those used-memory figures are good to a few + hundred MiB at best. That is one model on one card, not a bound; a different + model, GPU or build can land elsewhere. At a size well above what loads, the + report should say `does not fit` and list what would make it fit. +- `rocm serve` is unchanged: it takes no context-size option, and the report + names llama.cpp's options only as llama.cpp's. + +Also check the refusals: `owner/repo` with no `:QUANT` reads nothing and, with a +`--context-size`, says a quantization is needed; a repository that does not +exist says it was not found; a `.gguf` path that does not exist says it could +not be opened and does not point at the upstream tracker; and naming a vision +model's `mmproj-*.gguf` file on Hugging Face (`owner/repo:mmproj-....gguf`) says +it is a multimodal projector, not a model, names the model file instead, and +does not suggest `rocm serve` for the projector. None downloads +anything. + +Finally, `rocm diagnose --report --model /:Q4_K_M --context-size +8192` prints the report with no model section, and stderr says that `--model, +--context-size were ignored` and nothing about the model was fetched. + +## 13. Pasting An Error Into `rocm diagnose` + +`rocm diagnose --symptom-file` takes a pasted error without shell quoting. +Copy a line that would break an argument, for example: + +```text +-v "HSA_STATUS_ERROR_INVALID_ISA" isn't it? $(echo ran > ran.txt) & echo ran > ran.txt +``` + +Then, in an empty directory, hand it over in each shell you have: + +- bash (Linux or WSL): run `rocm diagnose --symptom-file - <<'EOF'`, paste, + then type `EOF` on its own line. +- PowerShell (5.1 and 7): `Get-Clipboard -Raw > error.txt`, then + `rocm diagnose --symptom-file error.txt`. Not `Get-Clipboard | rocm ...`: + Windows PowerShell 5.1 pipes text to a program as ASCII and turns every + other character into `?`. +- cmd.exe: save the paste to `error.txt` with an editor, then + `rocm diagnose --symptom-file error.txt`. +- Any shell, nothing piped in: `rocm diagnose --symptom-file -`, paste at the + prompt, and end the input as the hint printed on the first line says. +- A file named `-`: save the paste as `-` and run + `rocm diagnose --symptom-file ./-`. It reports the same causes; a bare `-` + would read standard input instead. + +Expected result: + +- Each run reports the same causes as `rocm diagnose --symptom-file error.txt` + with the paste saved in `error.txt`, and exits 0. +- No `ran.txt` appears in the directory. +- `rocm diagnose --symptom-file no-such-file.txt`, the same flag on a binary + file (an image, say) and on a file larger than 16 MiB each print one error + naming the file and why, then the `Usage: rocm diagnose` line, and exit 2. +- In Windows PowerShell 5.1, `Get-Clipboard -Raw > error.txt` writes UTF-16; + the same command reading that file still reports the same causes, and a + paste with a non-ASCII character in it (`é`, `→`, a typographic quote) + reports the same causes as in PowerShell 7. + +## 14. Which Causes a Pasted Error Contributed To + +`rocm diagnose --symptom` scores the catalog from the examined host's state and +the pasted text together, so a finding about the host can outrank the entry the +paste added to. Each match says whether the paste contributed to its score or +evidence; any keyword hit counts, however weak, so this is not a verdict that +the match explains the error. On any host (PowerShell: +`$env:HSA_OVERRIDE_GFX_VERSION = "11.0.0"` instead of the prefix): + +```bash +HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --symptom 'HSA_STATUS_ERROR_INVALID_ISA' --json +HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --symptom 'HSA_STATUS_ERROR_INVALID_ISA' +HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --json +``` + +- In the first, every entry in `matched` carries `symptom_matched`: + `fix-2-unset-override` (raised by the override alone) is `false`, and + `fix-1-arch` (raised by the pasted ISA error) is `true`. +- In the second, `fix-2-unset-override` has a line `source: the examined + host's state alone; the symptom text did not contribute to it`, and + `fix-1-arch` does not. +- In the third, with no `--symptom`, the field is still present and `false` on + every entry, and the text form (the same command without `--json`) prints no + `source:` line. + +The e2e scenarios `diagnose-62`, `diagnose-63` and `diagnose-64` run these three +checks on every lane. + +## 15. Dashboard Chat With A Small-Context Local Model + +Needs a local OpenAI-compatible endpoint serving a model with a small context +— llama.cpp's default `n_ctx` is 32768, which is the case this checks. The +point is that an answer survives a tool call that returns a lot of text. + +```bash +rocm dash +``` + +In the Chat tab, ask `which LLMs can this machine support`, which is the +question that makes the model call `rocm_command` with `["model"]`. + +Expected result: + +- A written answer listing recommended models. Not + `chat request failed: ... 400 Bad Request`, and not a raw JSON error body. +- The same question asked of `rocm mcp-call` returns the curated list, and the + per-recipe diagnostics only when asked for: + + ```bash + echo '{"args":["model"]}' > args.json + rocm mcp-call rocm_command --arguments-file args.json | wc -c + echo '{"args":["model","--verbose"]}' > args.json + rocm mcp-call rocm_command --arguments-file args.json | wc -c + ``` + + The first is a few kilobytes and matches what `rocm model` prints in a + terminal; the second is far larger. + +- To see the truncation path, ask something that makes the model read a long + service log (`rocm_command` with `["logs"]` on a host with a lot of history). + The model's answer should say the output was partial and point at running the + command in a terminal — it must not present a cut list as a complete one. + +- To see the overflow message, point the Chat tab at a model with a very small + context (serve one with `--ctx-size 2048`) and ask the same question. The + transcript should read `A ROCm check returned more text than this model can + read (N of M tokens). Try a more specific question, or a model with a larger + context.` — and `rocm-cli.log` should still carry the provider's own body + with `exceed_context_size_error` in it. diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md new file mode 100644 index 0000000000..c8a3bdbb63 --- /dev/null +++ b/docs/nightly-on-the-queue.md @@ -0,0 +1,271 @@ + + +# Nightly E2E on the shared GPU queue + +The nightly GPU lanes can run on a shared queue of GPU workers: one lane per +GPU model in the queue's fleet. The run itself happens next to the queue, which +submits the lanes, waits for them and collects their results. +[`queue-run.yml`](../.github/workflows/queue-run.yml) starts a run of one +branch, waits for it and publishes it here: its lane matrix in the job log and +summary, the lanes' files as the `queue-run-results` artifact, and a pass or +fail. [`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) publishes +a run that has already finished. + +The queue's lanes cover the Linux GPU nightly: one lane per GPU model, each +running the full suite on both the `release` and the `nightly` channel, as the +GPU jobs of [`nightly.yml`](../.github/workflows/nightly.yml) do (see +[ci-hardware-testing.md](ci-hardware-testing.md)). Their results are trusted the +way those jobs' results are: the queue runs on the same infrastructure and under +the same access as the self-hosted runners, and neither signs what it reports. + +The lane set is whatever GPU models the queue has. The 11-13 run +([43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)) discovered `mi300x`, `mi325x`, `mi350p`, `mi350x`, +`mi355x`, `r9700s` and `w7900-dual`; `w7900-dual` (W7900) is advisory, and it is +the only scheduled coverage of that hardware, since `e2e-selfhosted.yml` no +longer has a W7900 lane. + +**The remaining gap is native Windows and WSL.** The queue runs Linux pods, so +the native Windows and WSL lanes stay on their dedicated runners, in `nightly.yml` +and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); nothing +here replaces them. Which checks are required before merging is a repository +setting, outside this page. + +This page is the **contract** between this repository and the queue side: what +a lane must do to be reported, and how to read the report. The queue side — +how runs are started and executed, and the `queue-publish` command the +workflows call — is maintained privately with the queue, not in this +repository. + +## Starting a run + +Dispatch `queue-run.yml` with the branch to test (`main` by default); it also +runs every night at 02:17 UTC on `main`. The job asks the queue side for a run +of that branch, named for the job's run id and attempt, so re-running the job +starts a new run. Each lane may wait up to 4 h 10 min (15000 s, the run's wait limit below) +to be started, then run for up to 3 h (10800 s, its run limit below), however +long it waited. The job waits until every lane has finished or been +cancelled, then publishes the run as below. The job log starts with the run's inputs +(below). Only one run goes at a time: a second +dispatch waits for the first, and the queue side refuses one while another, +however started, is unfinished. GitHub keeps at most one job waiting, so a +third dispatch, or the nightly schedule, replaces a dispatch still waiting +and that one never runs. The branch must exist in this repository; +otherwise the job fails at once. + +## Publishing a finished run + +Dispatch `nightly-queue.yml` with a run's `request_id` — the id its +`queue-run.yml` job printed — to publish that run again, for instance when that +job was cancelled while the run went on. Such a run was never finished, so if +its lanes are done the queue side finishes it first. Do that within a day of +its lanes finishing: the queue keeps a lane's files for about a day, and a run +not finished by then loses them. It never starts a run, +and there is no default: a run that did not happen cannot be published in its +place. It +fails, rather than reporting green, when: + +- the named run has not finished, or never existed +- the run failed, could not start, was cancelled before it finished, or + discovered no lanes at all + +A finished run's results are kept for two weeks: each new run removes older +ones, so a finished run's `request_id` can be published again until then. An +uploaded artifact follows this repository's own artifact retention. + +## What a run executes + +Every run is asked for: by `queue-run.yml`, or by an operator on the queue +side. A run has six inputs; the queue side sets their defaults, and any of them +can be overridden for a single run (`queue-run.yml` sets the branch): + +- the branch — the branch of this repository to test, `main` by default. The + run resolves it to a commit once, as it starts, and every lane checks out + that commit, so all lanes test the same code however long each waits for a + GPU. The report's first line names the commit. Only a branch of this + repository can be named, never a commit id. A lane that cannot check the + commit out fails, with the reason in its `lane.log`. +- the lane image — the container every lane runs +- the lane command — a single line run by `bash` under `set -euo pipefail` in + that image, from the root of the checkout, with the commit in + `ROCM_CLI_COMMIT`. The default is + [`scripts/run-e2e-lane.sh`](../scripts/run-e2e-lane.sh), which builds + once and runs the full suite from a cold start on each channel, so what a + lane runs is changed here, by a pull request, like any other test +- the blocking lanes — which lanes can fail the run; left empty, every lane can +- the excluded lanes — GPU models to leave out of this run +- the wait — how long a lane may wait to be started before it is cancelled and + reported as **incomplete** (no capacity); at most 15000 + seconds, and a run given more does not start +- the run time — how long a started lane may run, counted from its start, + before it is cancelled and reported as **incomplete**; at most 10800 + seconds, and a run given more does not start + +The lane set itself is discovered, not listed: one lane per GPU model available +in the queue's fleet, minus any excluded. New hardware appears without any +edit, and hardware that is away does not produce a red lane — though a +blocking lane that was not discovered is reported as a warning, since that run +did not test it. A run that discovers no lanes at all fails. + +A discovered lane that is not blocking is advisory: reported, but unable to +fail the run. Promoting it is a deliberate edit once it has run clean. + +Every lane runs the same image and command; there is no per-lane variant. The +channels are the command's, not the queue's: the default command runs the +`release` channel and then the `nightly` one (`E2E_CHANNELS`, default `release +nightly`), each with its own runtimes and its own suite run, so a lane takes +about twice as long as one channel would. That time has not been measured on a +queue lane from a cold start, so a lane that cannot finish both channels within +its run time (counted from its start, so waiting for a GPU does not use it) +is reported as **incomplete**; the run time can be raised for a run, up to its +limit (see the run's inputs above). Each channel starts with the GPU preflight, as each +channel's job in `nightly.yml` does, so a model server the previous channel's +suite left on the GPU is reclaimed first; the lane log shows it. Both always +run, so a failing channel does not hide the other, and the lane fails if either +did. + +A lane starts cold, so its first steps fetch from the network: the build +packages, the rustup installer, the toolchain and the crates for the host +being built. The lane command tries each of those up to four times, and never +retries a build or a test. The pre-warm's package downloads and the suite's +model downloads are left to the tools doing them, so a model that downloads +more slowly than its scenario waits fails that scenario. The lane fetches only +the host's crates, so it refuses to start with `CARGO_BUILD_TARGET` set. + +The queue's results are an additional signal, not a replacement: the lanes +write only into the queue artifact, and the `e2e-gpu--report` +artifacts that `nightly.yml`'s consolidated report reads are not produced from +them. + +## Output + +**A lane must write everything it wants reported directly into `/out`.** The +default command writes each channel's suite results to `/out//results`. + +- Each lane's `/out` lands in the workflow's artifact, `queue-run-results` + from `queue-run.yml` or `nightly-queue-results` from `nightly-queue.yml`, + one directory per lane, next to that lane's `lane.log` (the command's combined + output). +- Files keep their paths under `/out`, subdirectories included. `lane.log` is + written by the harness; a lane must not use that name. Names ending in + `.quarantine`, hidden files and anything under a hidden directory are never + published. +- Anything a lane leaves elsewhere is not published. The queue records more + than the suite asked for, including details of the machine that ran it, and + the published results are public. +- Collected files are also scanned, and a file containing an identifier the + queue side treats as infrastructure — any UUID, a private overlay-network + address, a few internal names — in its content or its path is withheld + whole, a directory so named with everything in it, counted as one; the + report's `scanned … withheld …` line says how many. The scan is a backstop, not the + control: only writing to `/out`, and nothing identifying, is. + +Widening what is collected is not the fix for a missing file; writing the file +to `/out` is. + +A lane passes when its command exits zero. Capturing the command's output into +`lane.log` does not affect that status. + +### Reading the matrix + +The report has one row per lane and a summary. A lane is not simply pass or +fail, and the distinction matters when deciding whether to chase something: + +| Outcome | Meaning | Fails the run? | +|---|---|---| +| **passed** | the command exited zero | no | +| **failed** | the lane's command or its container failed; lanes are not retried, so one failure is final. `lane.log` shows a test failure; a lane whose container never started, for example because its image could not be pulled, has no `lane.log` | yes | +| **incomplete** | no result: cancelled at the run's deadline, or no worker in the fleet could take it | yes | +| **missing** | discovered but never submitted | yes | +| **advisory** | any outcome but passed, on a lane that is not blocking | no | + +The matrix is drawn before the lanes' files are collected, so a lane that +passed but whose files did not all come back, or that came back with nothing +from `/out` (every lane that runs writes `lane.log` there), is still counted +as passed in it. A `no result` line after the matrix names each such lane, and +that line is what counts: a blocking lane without a result fails the run. + +**incomplete** and **missing** fail the run because a blocking lane without a +result is not a pass, but they point at queue capacity or infrastructure +rather than at a regression. + +## What the workflow requires + +- a self-hosted Linux runner registered with the `queue-report` label alone, + without GitHub's default `self-hosted`, OS and architecture labels, so only a + job naming `queue-report` lands on it, providing + the queue side's publish command and read access to the queue side's results +- a registered runner name and host name that do not identify internal + infrastructure, since GitHub prints both into every job log + +Both workflows require **no secret** and no permission, and contract tests in +[workflow_contract.rs](../xtask/src/workflow_contract.rs) +(`queue_nightly_publisher_holds_no_credential`, `queue_run_holds_no_credential`) +keep it that way. + +That test reads the workflow files on `main`. A workflow on another branch of +this repository can still name the runner's label, which is one reason the +runner holds no credential, below. On `main`, the tests also refuse any +other workflow that names the `queue-report` label or these workflows, so none +sends a job to the runner or waits on them by mistake. The runner is set up to +run only these two workflows as they are on `main`, by a check on the runner +that runs before any step. That check lives outside this repository, so +nothing here can verify it; the runner is still treated as untrusted, as above +and below. Whether this +workflow's check is required before merging is a repository setting the test +cannot see. It must not be: a manual-only workflow never reports on a pull +request, so requiring it would block every merge. + +## Why it works this way + +**The run happens next to the queue, not in a GitHub job.** Submitting work to +the queue needs a credential, and a GitHub job is the wrong place to hold one +here. The runner is a persistent self-hosted machine on a public repository, +and runner labels are not an access control: any job that names the label can +run on it, including one from a branch or, after approval, a fork. Such a job +could read whatever the runner holds, or tamper with it to capture what the +next job receives. Approval gates on environments only govern jobs that name +the environment, not jobs that run next to one. Restricting a runner to a single +workflow, or making it ephemeral, needs organisation-level configuration or a +token-minting credential. So the queue credential stays with the queue side, +and the runner has no credential for the queue or the cluster, and no sudo. + +**What the report is trusted for.** The runner keeps its own registration with +GitHub between jobs, and a job on it can read every retained run's results, +including runs started by hand that were never published. That is the same trust +the self-hosted runners already carry: their nightly results are not signed +either, and the queue runs on the same infrastructure under the same access. A +job on this runner still cannot reach the queue or its credential. Each job also +receives GitHub's own short-lived runtime token, as on any runner. + +**Results move by a shared volume, not a network call.** The queue side writes +each finished run's publishable files; the runner reads them read-only. No +credential is needed between the two, and the runner's only calls ask the +queue side to start a run, how it is going, and to finish it; none carries a +file. + +**A run is asked for, not submitted, from here.** `queue-run.yml` holds no +credential and sends nothing to the queue: the runner, which sits next to the +queue side, asks it for a run of a branch, and the queue side starts it with +its own credential. It believes the request because of where it comes from, +which is why the runner is set up to run only these two workflows from `main` +(a setting outside this repository). A request can start a run of a branch +already in this repository, and that branch's lane command then runs in the +lane pods. The lane spec gives those pods no queue or cluster credential and +mounts neither the results volume nor the runner's files, so the command can +publish only what it writes to `/out`, through the scan above. What else a pod +can reach is set by the queue's own isolation, which this repository cannot +verify. + +**Only `/out` is published, and a missing run is never green.** Publishing by +origin is the main control on what reaches public results, with the scan +behind it. An unfinished run fails the job, and a job only ever publishes the +run it started or was given, so a nightly that stops running cannot keep +reporting an old pass: the missing night shows as a missing scheduled job. + +**Nothing public names the queue's infrastructure.** Runner names, labels, +paths, branches and this page use neutral names, per +[AGENTS.md](../AGENTS.md)'s rules for public content. diff --git a/docs/release-trust.md b/docs/release-trust.md index b5d4429b9b..afddf40ba2 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -240,7 +240,7 @@ approval, or manual-only blocking. Existing signed indexes without ## TheRock Source Base Overrides rocm-cli hardcodes the pip index and tarball catalog bases it resolves TheRock -SDK artifacts from. Six of those bases can be overridden, for fixture-server +SDK artifacts from. Eight of those bases can be overridden, for fixture-server testing only: ```text @@ -250,11 +250,16 @@ ROCM_CLI_THEROCK_RELEASE_TARBALL_BASE ROCM_CLI_THEROCK_NIGHTLY_TARBALL_BASE ROCM_CLI_THEROCK_NEXT_PIP_BASE ROCM_CLI_THEROCK_NEXT_TARBALL_BASE +ROCM_CLI_THEROCK_NEXT_NIGHTLY_PIP_BASE +ROCM_CLI_THEROCK_NEXT_NIGHTLY_TARBALL_BASE ``` -The last two address the ROCm 10 ("next") source layout, which is a layout -rather than a channel: it has no nightly counterpart and is never resolved -unless an install explicitly pins a ROCm version of 10 or newer. +The first four address the canonical (earlier) layout, `repo.amd.com` and +`rocm.nightlies.amd.com`, which an install reads only when it pins a build that +exists only there: a ROCm 7 release, or a nightly built before 2026-08-23. The +last four address the newest layout, `stable.repo.amd.com` and +`nightly.repo.amd.com`, which an install with no pin reads, and which a stable +ROCm 10 or newer pin or a nightly built on or after 2026-08-23 reads too. None of these take effect unless `ROCM_CLI_THEROCK_ALLOW_BASE_OVERRIDE` is also set to a non-empty value. Without that gate, a stray override left in a @@ -312,10 +317,12 @@ is already trusted. ## Simulated Host Root rocm-cli decides what hardware it is running on by reading fixed host paths: -`/dev/kfd`, the render nodes under `/dev/dri`, `/dev/dxg`, the KFD topology -under `/sys/class/kfd`, the DRM cards under `/sys/class/drm`, -`/sys/module/amdgpu` and its `version` file, `/proc/version`, `/proc/cpuinfo`, -`/proc/meminfo`, `/proc/cmdline`, `/proc/modules`, `/proc/1/cgroup`, the +`/dev/kfd`, the render nodes under `/dev/dri`, `/dev/dxg`, the +`/sys/class/kfd/kfd` class device and the KFD topology under it, the DRM cards +under `/sys/class/drm`, the amdgpu PCI driver directory +`/sys/bus/pci/drivers/amdgpu`, `/sys/module/amdgpu` and its `version` file, +`/proc/version`, `/proc/cpuinfo`, `/proc/meminfo`, `/proc/cmdline`, +`/proc/modules`, `/proc/1/cgroup`, the container markers `/.dockerenv` and `/run/.containerenv`, the `/etc/os-release` (falling back to `/usr/lib/os-release` when absent, per `os-release(5)`) behind the distro fields and name `rocm examine` reports, the modprobe @@ -323,7 +330,8 @@ configuration directories `/etc/modprobe.d`, `/usr/lib/modprobe.d` and `/run/modprobe.d`, the WSL plumbing under `/usr/lib/wsl` (the `lib` directory and `lib/libdxcore.so`), and the WSL ROCDXG capability check, which looks for `lib/librocdxg.so` and `share/rocdxg/dids.conf` under `/opt/rocm` and under each -discovered ROCm install. The E2E suite can point those reads at a directory it populated with a +discovered ROCm install, and `lib/librocdxg.so` again in each directory on +`LD_LIBRARY_PATH`. The E2E suite can point those reads at a directory it populated with a simulated host instead, but only in builds compiled with the `e2e-test-hooks` Cargo feature: @@ -376,6 +384,23 @@ documents. Do not generate those production keys inside CI. Once published, the installers, metadata cache, and model recipe index can make release-channel signature verification mandatory by default with those keys as trust roots. +Those trust roots are pinned copies of the published keys: the +`PINNED_RELEASE_PUBLIC_KEY_CURRENT`/`_NEXT` slots in +[install.sh](../install.sh), `$PinnedReleasePublicKeyCurrent`/`Next` in +[install.ps1](../install.ps1), and +`PINNED_METADATA_PUBLIC_KEY_PEM` in +[fetch.rs](../apps/rocm/src/therock/fetch.rs). `cargo xtask verify-pinned-keys` +fails when a pinned copy differs from its canonical file under `docs/keys/`. The +release and nightly workflows always run it; pull-request CI runs it when a +change matches the `heavy` path filter in +[ci.yml](../.github/workflows/ci.yml), which covers `docs/keys/`, the +installers, every Rust source and `xtask/`. It reads the one assignment +the installer actually uses, so each slot must be assigned exactly once as a +plain declaration, a slot whose key is not published yet must stay `""`, and +`install.sh` must mark both of its slots `readonly`. The full rules, and what a +textual check cannot catch, are in +[verify_pinned_keys.rs](../xtask/src/verify_pinned_keys.rs). + When those owner-controlled inputs exist, release operators can add `--require-production-trust` or set `ROCM_CLI_REQUIRE_PRODUCTION_TRUST=1` so the dist verifier fails unless the release signing public key, runtime metadata diff --git a/docs/rocm-docs/install/installation.md b/docs/rocm-docs/install/installation.md index 670d815096..0f73843008 100644 --- a/docs/rocm-docs/install/installation.md +++ b/docs/rocm-docs/install/installation.md @@ -15,9 +15,10 @@ ROCm CLI ships as prebuilt binaries (`rocm` and `rocmd`). Platform support: For WSL2 setup and diagnosis, see [ROCm CLI on WSL2](wsl.md). -Live dashboard GPU/CPU/memory telemetry works on Windows too; the -serving-instance registry view is Linux and WSL2 only, because there's no -`rocmdashd` Unix socket on Windows (see +Live dashboard GPU/CPU/memory telemetry and the managed servers +`rocm services list` reports work on Windows too; per-server throughput and +Docker-discovered instances are Linux/WSL2 only, since there's no +`rocmdashd` unix socket on Windows (see [Interactive interfaces](../getting-started.md#interactive-interfaces)). vLLM serving is Linux or WSL2 only (see [vLLM adapter](../engines/vllm.md)). diff --git a/docs/skill-quality.md b/docs/skill-quality.md index be3ebb05d5..1b065d7645 100644 --- a/docs/skill-quality.md +++ b/docs/skill-quality.md @@ -152,9 +152,10 @@ review and may decline: mutates, routing upstream when nothing is established), not whether a diagnosis was correct. The exception is a case with a directory under [`evals/fixtures/`](../skills/rocm-doctor/evals/fixtures/): it gets a replayed `rocm` - that answers `--version`, `fix --json` and `diagnose` from a recording, so it can - grade the version check or how an answer is read. The recording is of a host with no - GPU data, so even there a rise does not mean users get fixed faster. See - [Doctor Skill Eval Replays](testing.md#doctor-skill-eval-replays). + that answers `--version`, `fix --json` and `diagnose` (and `examine --json` for a case + that names a host) from a recording, so it can grade the version check, how an answer + is read, or the question the skill asks when the examination finds no AMD GPU. The + recording is of a host with no GPU data, so even there a rise does not mean users get + fixed faster. See [Doctor Skill Eval Replays](testing.md#doctor-skill-eval-replays). - **Editing the dataset moves the figures.** A changed content hash between two runs means the scenario set may have changed; look at that diff before comparing them. diff --git a/docs/testing.md b/docs/testing.md index 431ff82724..a21272ebc1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -38,19 +38,46 @@ cargo xtask check-module-size ``` It exits non-zero and names each offending file with its count. There is one limit for -every file: no flag, baseline or exemption. For now the check is advisory in the `clippy` -CI job and in the `cargo-module-size` pre-commit hook, which run -`cargo xtask check-module-size` and `cargo xtask check-module-size-advisory` and report -offenders without failing (in CI the step shows red in the job's step list while the job -stays green); the second is the same check that exits successfully even -when the check itself cannot run. That holds once xtask has built: a commit that leaves -xtask or `rocm-core` not compiling still fails the hook. -The counting rules are unit-tested in `xtask/src/module_size.rs`: +every file: no flag, baseline or exemption. The `clippy` CI job runs +`cargo xtask check-module-size` and fails when any file is over the limit; `clippy` is a +required check and also runs on merge-queue runs. The failure text names each file and +points at the split rule in [architecture.md](architecture.md). The `cargo-module-size` +pre-commit hook is the local mirror of that step: it runs the same +`cargo xtask check-module-size`, so a commit that stages Rust source while a file is over the +limit is refused, and the output names the file and its count. It only checks and never +rewrites or restages anything. The check measures the working tree, so the hook also +fails when a Rust file under a crate's `src/` is not declared by any `mod`, even an +untracked scratch file; the error says to declare or delete it, where CI only sees the +files that are committed. It runs only for contributors who installed the hooks, so the +CI step stays the authority. +The counting rules are unit-tested in `xtask/src/module_size.rs`, and +[workflow_contract.rs](../xtask/src/workflow_contract.rs) pins that the hook runs the +strict check in `module_size_hook_runs_the_strict_check_at_pre_commit`; the `module_size` +filter below selects both: ```bash cargo test -p xtask module_size ``` +### Module map check + +Every production source file, and every `.rs` file of `xtask` and the end-to-end harness, must have an entry in [architecture.md](architecture.md). The +check runs in the `clippy` CI job and in the `architecture-doc` prek hook, and you can run it +on its own: + +```bash +cargo xtask check-architecture-doc +``` + +It fails on a `.rs` file under `apps/*/src`, `crates/*/src` or `engines/*/src`, or under +`xtask/src`, `tests/e2e-cucumber/src` or `tests/e2e-cucumber/tests`, that the doc does not +link, and there is no exemption list; see +[architecture_doc.rs](../xtask/src/architecture_doc.rs). Test-only helpers of a production +crate need no entry; a file in the `xtask` and `tests/e2e-cucumber` trees does, whatever its +name. The +`lychee` check is complementary, not redundant: it confirms that links already in the doc +resolve, and says nothing about a file the doc never mentions. + ### Tests that touch the environment `std::env::set_var` / `remove_var` change state shared by every thread in the @@ -137,6 +164,108 @@ all of which the scan reports nothing for: `cfg_attr`. Write `#[cfg(unix)]` above `#[test]` instead, which the scan arms on. +### The freeform planner never imports `chat` + +`chat.rs` calls the freeform planner, so a planner that imported `chat` back +would be a module cycle. The cycle is broken by three leaf modules +([`prompt_text.rs`](../apps/rocm/src/prompt_text.rs), +[`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) and +[`tool_policy.rs`](../apps/rocm/src/tool_policy.rs)) that the planner and `chat` +both call into. `cargo xtask check-crate-edges` compares crates and cannot see +a rule between modules of one crate, so, as with the environment rule above, a +test enforces it rather than an `xtask` subcommand: +`the_freeform_planner_and_its_leaf_modules_never_import_chat` in `xtask` +([`chat_import_contract.rs`](../xtask/src/chat_import_contract.rs)) reads +[`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) and those three +files and **fails the build** if any of them reaches `chat` in non-test code, +naming each as `file:line`. Run it on its own with: + +```bash +cargo test -p xtask chat_import_contract +``` + +It flags `crate::chat`, `super::chat` and `self::chat` as a path, `chat` as a +member of a `use crate::{..}` group, and `mod chat`, and a raw identifier +(`r#chat`) counts as `chat`. The one thing it skips is a +`#[cfg(test)] mod name { .. }` block, so a `mod tests` may use `chat`; a +`#[cfg(test)]` on anything else (a `use`, a function, a field, an expression) +skips nothing, and a test that needs `chat` belongs in the test module. A route +that never writes the `crate::chat` path is not seen: an alias +(`use crate as root; root::chat::f()`), a glob (`use crate::*;` then +`chat::f()`) or a `#[path = ".."] mod` that points at `chat.rs`; and +`#[cfg(all(test, ..))]` is read as production code, so it can only cause a false +alarm. It does not flag the +identifiers and prose that merely contain the word, such as +`chat_cli_arg_value`. It reads the four listed files only, not the modules they +import. If one of them moves or is renamed, update `GUARDED` in the test; the +test fails on a path that no longer exists rather than dropping it from the +guard. When the planner needs something that lives in `chat.rs`, move it into +a leaf module and import it from there. + +### Dash-TUI text fields go through `apply_char_key` + +A dash-TUI form edits its `String` through `apply_char_key` / +`apply_char_key_filtered` in +[`ui/widgets.rs`](../crates/rocm-dash-tui/src/ui/widgets.rs), not through its +own "`Char` appends, `Backspace` pops one char" code. As with the other module +rules above, a test enforces it rather than an `xtask` subcommand: +`the_dash_tui_has_no_hand_rolled_text_edit_outside_the_helper` in `xtask` +([`dash_text_edit_contract.rs`](../xtask/src/dash_text_edit_contract.rs)) scans +the non-test code of every file under `crates/rocm-dash-tui/src/` and fails, +naming each hit as `file:line` with a pointer to the helper. Run it on its own +with: + +```bash +cargo test -p xtask dash_text_edit_contract +``` + +It is per file, not per `match` arm, because the copies the helper replaced were +not all arms (one had its `push` in a `type_char` method the arm called). In a +file other than `ui/widgets.rs` it reports a no-argument `.pop()` when the file +names `KeyCode::Backspace`, and `.push(x)` (also `.push(*x)`, +`.push(x.method())` and `.push(x as char)`, each with or without a trailing +comma) when `x` is a name the file binds as a key character +(`KeyCode::Char(x)`, `KeyCode::Char(mut x)`) or declares as `x: char`, in a file +that names `KeyCode::Char(`. A `#[cfg(test)] mod name { .. }` block is skipped, +so a unit test may drive a buffer by hand; a `#[cfg(test)]` on anything else +skips nothing, and an out-of-line test module (`#[cfg(test)] mod name;`) is a +file of its own, scanned in full. + +What it does not see: it matches the shape, not the type, so a `let top = +stack.pop()` in a file that handles `Backspace` is a false alarm, and an +unrelated `out.push(c)` is reported when the same file binds `KeyCode::Char(c)` +elsewhere. A push is read only when its argument is that name alone in one of the +spellings above: `.push((x))`, `.push(char::from(x))`, a `.push(if ..)` of the +name, and a call with a gap before the paren (`.push (x)`, which rustfmt never +writes) are not seen. Only `#[cfg(test)]` exactly is read as a test gate (a +`#[cfg(all(test, ..))]` is scanned, which can only cause a false alarm), and +`use KeyCode::*` or an alias of `KeyCode` defeats the scan. A typed key passed +from one file to a method in another is not followed. Only `.pop()` and +`.push(x)` are matched: `push_str`, `truncate`, `clear`, `.insert(..)`, +`.remove(..)` and `replace_range(..)` are not scanned, partly because a cursor editor (the serve wizard's Host field, `InlineEditor` in +[`serve_wizard.rs`](../crates/rocm-dash-tui/src/ui/serve_wizard.rs)) cannot use +the append-only helper and is a different thing. A `Backspace` arm that never +pops, such as the folder browser's "go to the parent directory", is not a text +edit and is not reported. + +The one file with its own `push` and `pop` is the Chat input in +[`app/actions.rs`](../crates/rocm-dash-tui/src/app/actions.rs), whose keys are +mapped to a `KeyAction` by a separately tested table. It is listed in `ALLOWED` +in the test with its reason and the number of offenses it is expected to carry +(its one push and its one pop). The test fails when the file carries a different +number, so a further hand-rolled field added there is caught, as is an entry that +stops being needed, or whose file moves. A new field reuses the helper; a new +exception is a reviewed entry in that list. + +Where it runs: the pre-push hook (`cargo xtask pre-push-check`) selects `xtask` +for any Rust change, the `windows-debug` job runs `cargo test --workspace` on +every pull request that changes Rust, and the merge queue and post-merge runs +test the whole workspace. The Linux pull-request lane (`cargo xtask affected`) +selects `xtask` only for a change in `xtask` or in a crate it depends on, or for +a file it reads, so a pull request that touches nothing but +`crates/rocm-dash-tui` does not run this test on Linux. The module-import +contract above has the same property. + ### Tests that run a fake executable Plant a stand-in `python`, `uv`, `amd-smi`, ... with @@ -151,6 +280,170 @@ child write the file instead, so this process never holds a write fd on it. The same applies to E2E steps: scenarios run concurrently in one process, and the `rocm` child that runs the shim can be the one refused. +### The engine stdio client + +`rocm` and `rocmd` run an engine child through one client, +[`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) in +`rocm-engine-protocol`. Its conversation (the request line, the environment each +option sets, what a failing exit or an unusable answer is reported as, the +SIGPIPE guard) is tested there against `FakeEngine`, a shell-script engine child +from [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) that +records its argv, environment and stdin; the pure parts (the command a call +builds, `decode_engine_response`, stderr collection) run on every platform, and +the ones that start the fake are `#[cfg(unix)]`. The `rocm` and `rocmd` callers +each run their own entry point against the same fake and pin only what they +choose: the program and verb, the app paths, and whether stderr is forwarded (see +`apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). These tests +catch behavioural drift: a caller whose own copy of the conversation behaved +differently from the shared client in what they pin (the framing, the +environment, the exit and answer handling, the guards) would fail them. A copy +that behaves the same passes them, so a second guard is on the existence of a +second copy: +[engine_stdio_client_contract.rs](../xtask/src/engine_stdio_client_contract.rs) +is a static text scan over every `.rs` file in the tree (comments stripped) that +fails when a file other than the shared client both pipes a child's stdin and +names an engine request or response envelope, which is what an engine stdio +client is made of. Both are looked for anywhere in the file, not on one line, so +a copy that builds its `Stdio` in one place and passes it to `.stdin(..)` in +another is seen too (a fixture covers it); the price is that a file which pipes +some child's stdin for another reason and also names an envelope type is +reported, and the answer to that is a reviewed allowlist entry. A caller that reaches an engine through `run_engine_stdio` +has neither. Pasting the old rocmd copy back into `apps/rocmd/src/engine_ipc.rs` +fails it by name (and, separately, five of rocmd's pinning tests: the +app-paths, crash, unparseable-answer, nonzero-with-answer and SIGPIPE ones). Like +the other allowlists it is bidirectional, so an exemption cannot outlive its +reason. + +```bash +cargo test -p xtask engine_stdio_client_contract +``` + +It is a source-text scan, not a call-graph analysis: a copy that makes its pipe +by hand, or reaches the envelope types under another name, is outside what it +can see, and it does not check that a caller uses the shared client correctly +(the callers' tests do). It closes the cheap, likely regression of a +spawn-write-read helper pasted back into a binary. + +An answer that is not a response envelope is reported as that, and when the +child also exited with a failing status the error says so, because the status is +often why the answer was cut short (a clean exit adds nothing). A failing child +that printed nothing is reported by its status alone. The status is worded from +the exit code (`exited with code N`) or, for a child ended by a signal on Unix, +the signal (`was terminated by signal N`), not from `ExitStatus`'s own text, which +already reads `exit status: N` and would double up (`a_child_ended_by_a_signal_is_reported_by_the_signal`). Both carry the child's +stderr. `an_unparseable_answer_from_a_failing_child_names_the_exit_status` pins +the three cases against the fake, and `engine-ipc-02` below shows the same +message in a `stop_server` report next to what the stop did. + +The SIGPIPE tests re-execute the test binary in a fresh process with SIGPIPE at +its default action, which is how `rocm` runs, because the test harness leaves it +ignored and changing it process-wide would race sibling tests. A write that is +not guarded kills that process, and the parent test reports the signal. Two +writes are covered that way: the request to the child's stdin, and the forwarding +of the child's stderr to this process's stderr (a line of progress, and the note +recorded when reading it fails). The forwarding test writes into a real pipe whose +reader is closed, because a mock sink that returns `BrokenPipe` itself never +raises the signal. Each forwarding write is guarded on its own, never the loop +around it, because the reads between lines block for as long as the engine is +quiet and the disposition is process-wide; a third re-executed test, +`forwarding_guards_each_write_and_never_the_reads_between_them`, records the +disposition inside every read and every write and after the loop. `FakeEngine` quotes the response and its scratch directory +for the shell, so a response such as `can't stop`, or a `TMPDIR` containing an +apostrophe, reaches the client intact. + +The `forward_stderr` option, which is what shows a user an engine install's +progress, is tested at both ends: the client runs against a private entry point +that takes the sink (`a_childs_stderr_reaches_the_forwarding_sink_when_there_is_one`, +a `Vec`-backed writer), and the public entry point is run in a re-executed test +binary whose own stderr the parent reads +(`forward_stderr_sends_the_childs_stderr_to_this_processs_stderr`: the child's +line is there with the option and absent without it). Replacing the one +`forward_stderr.then(io::stderr)` with `None` turns the second red. + +A child that answers and then exits with a failing status is read as having +answered (see the doc comment of `run_engine_stdio`), and its status is not part +of the answer. The daemon asks the client to leave a note on its stderr, which is +its log (`note_failing_exit_after_answer`); the CLI does not, since its stderr is a +terminal. `a_failing_exit_after_an_answer_is_noted_only_when_asked_for` reads the +stderr of a re-executed test binary for both settings. The note is made only +for an answer that parsed (an unreadable answer with a failing exit is reported +by the error, with the status, and is not also logged as used), and it is written +through the same SIGPIPE-guarded, error-ignoring write as the stderr forwarding, +so a closed stderr neither ends a daemon hosted in `rocm` nor panics it +(`the_note_is_made_only_for_an_answer_that_parsed` and +`the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process`, the latter +with stderr a pipe without a reader under SIGPIPE's default action). A failed read of the +answer carries the child's stderr like every other failure +(`a_failed_read_of_the_answer_reports_the_stderr_too`), and a child whose stdin or +stdout pipe is missing is killed and reaped, not left running +(`a_missing_pipe_kills_and_reaps_the_child`, Linux only). + +What is forwarded to the terminal is complete UTF-8: stderr is read in chunks +that can end in the middle of a multi-byte character, and Rust's stdio on a +Windows console refuses a write that is not valid UTF-8, an error the best-effort +forwarding discards, so a split character would silently lose output. The +collector carries an incomplete trailing sequence to the next chunk (invalid +bytes become U+FFFD, and a character cut off by the end of the stream is flushed +lossily) while still publishing the raw bytes at once for the error. The logic is +the pure `Utf8Carry`, tested on every platform by +`text_split_across_chunks_is_forwarded_whole_and_valid_at_every_boundary` (every +two-point split of a string with two-, three- and four-byte characters), +`bytes_that_are_not_text_are_replaced_and_a_cut_off_tail_is_flushed` and +`the_collector_forwards_a_split_character_whole_and_keeps_the_raw_bytes` (a source +that hands out one byte at a time). + +A child whose answer cannot be read is killed and reaped before the error is +returned, as it is when the request cannot be written, so the long-lived daemon +does not collect a running engine or a zombie per failure. +`a_failed_read_of_the_answer_kills_and_reaps_the_child` (Linux only) runs the +read step on a real child with a reader that fails and checks that the child's +`/proc` entry is gone: a killed child that was never waited on would keep it as a +zombie. + +A request that cannot be written (the engine exited, or closed its stdin, before +reading it) is reported with what the engine wrote to stderr, since that is +usually the explanation, after the child has been killed and reaped. The child is +killed whether or not a `try_wait` would show it gone, because its pipe closes +while it is still exiting; the read of its stderr is bounded (a grandchild that +holds the pipe cannot hold the error path). `a_request_to_an_engine_that_exited_reports_what_it_said_on_stderr` +runs the whole client against a fake that exits without reading; +`a_request_that_cannot_be_written_kills_and_reaps_a_child_that_is_still_running` +(Linux only) checks the `/proc` entry of one that is still running, and +`the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` one whose +grandchild keeps the pipe open. The last two run the step on a real child with an +injected write error, because whether a real write fails before the child has +closed its stdin is a race. The answer of an engine that exits without draining a request larger than a pipe buffer is not read on this path (every engine here reads stdin to the end before answering, so that needs one that breaks the protocol); the module comment in `stdio_client.rs` states the limit. The stderr of a failed +write is collected by a thread that starts before the request is written and is +waited for with one deadline (`StderrCollector::finish`), returning what had +arrived when it passes; `a_collector_stops_waiting_at_its_deadline_and_keeps_collecting` +and `an_expired_deadline_returns_what_is_collected_without_waiting` run that on +every platform. It publishes each chunk as it arrives, not each complete line, so +a diagnostic with no trailing newline that arrived before the deadline is +reported (`the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` uses +`printf` without a newline for that reason). When the request has failed and the +collector is dropped, the capture is marked abandoned: what it held is released, +and the thread, which must keep draining the pipe so a surviving descendant is not +sent a SIGPIPE, discards and no longer forwards what arrives later, in chunks of +a fixed size, so a long-lived daemon does not grow with every failed request +(`an_abandoned_collector_releases_what_it_held_and_discards_what_arrives_later`, +red without the drop handling). + +Starting the collector before the write also means an engine that fills its +stderr pipe before it has drained a request larger than a pipe buffer cannot +deadlock the client (the engine blocked on a full stderr, the client on a full +stdin, nobody reading either). +`an_engine_that_writes_a_lot_of_stderr_before_reading_its_request_does_not_deadlock` +runs that against a fake that writes 300 kB to stderr first and is red when the +collector is started after the write. + +One wait is deliberately not bounded and has no test: after the child exits, the +answer is read to the end of its stdout and its stderr collector is joined, so an +engine that leaves a grandchild holding either pipe makes the request wait for +it. The clients this one replaced had the same exposure, and bounding it would +mean reading both pipes on their own threads and choosing what to report when a +deadline passes, which is a timeout policy the engine contract does not have +today. The module comment in `stdio_client.rs` states the limit and its tradeoff. + ### Advised commands must parse `apps/rocm/src/advised_commands.rs` checks the `rocm`/`rocmd` commands that the @@ -241,15 +534,26 @@ cargo test -p rocm --bin rocm advised_commands::dump_advised_invocations -- --ig Run the cross-platform smoke test: ```bash -python scripts/smoke_local.py +cargo xtask smoke ``` If the workspace is already built: ```bash -python scripts/smoke_local.py --skip-build +cargo xtask smoke --skip-build ``` +`--profile release` builds and smokes release binaries; it builds only the four +binaries the gate runs (`rocm`, `rocmd`, `rocm-engine-lemonade` and +`rocm-engine-vllm`), not the whole workspace. `--target-dir ` builds into +that directory and looks for the binaries there; without it, a configured +`CARGO_TARGET_DIR` is honoured, and otherwise `target`. The build is always +given that directory, so a `build.target-dir` in cargo config does not move it. +Before its first check, a run names the binaries' directory and the throwaway +state root it uses. The build's output streams as it runs. Each smoked command +must finish within three minutes; one that does not is killed and the gate +fails naming it, though a process that command started can outlive it. + ### Property tests `proptest` properties run with the rest of `cargo test`. One on proptest's @@ -261,17 +565,96 @@ PROPTEST_CASES=10000 cargo test -p rocm --bin rocm proptests ``` An explicit case count overrides `PROPTEST_CASES`, so a module that sets one -reads the variable back itself: through the `proptest_cases` helper in each -crate's test support, [rocm's](../apps/rocm/src/test_support.rs) and -[rocm-core's](../crates/rocm-core/src/test_support.rs), and through -`run_property` in [archive_props.rs](../crates/rocm-core/src/archive_props.rs), -which keeps its own copy because three crates include it by `#[path]`. All of +reads the variable back itself: through the `proptest_cases` helper in +[tests/common/proptest_cases.rs](../crates/rocm-core/tests/common/proptest_cases.rs), +which [rocm's](../apps/rocm/src/test_support.rs) and +[rocm-core's](../crates/rocm-core/src/test_support.rs) test support, the +`rocm-core` and `rocm-engine-protocol` integration tests and the Lemonade +engine's unit tests all include by `#[path]`; and through `run_property` in +[archive_props.rs](../crates/rocm-core/src/archive_props.rs), which keeps its +own copy because three crates include it by `#[path]`. All of them only ever raise the module's count: a `PROPTEST_CASES` below it is ignored, because some of these properties catch their target defect in a small share of cases and would pass with it present on a short run. The uninstall properties read their own knob instead, `ROCM_DELETION_PROP_CASES`, described under Uninstall And Download Removal below; it follows the same rule. +When a property fails, proptest saves the shrunk case as a `cc # shrinks +to ...` line in a file named after the source file the `proptest!` block is +written in, and replays it before any new case on every later run: a block in +`/src/.rs` uses `/proptest-regressions/.txt`, where +`src/` is the nearest ancestor directory holding a `main.rs` or `lib.rs` (it is +`src/` for every crate in this tree). Moving +the test to another file therefore leaves its saved cases behind, and proptest +does not warn: they just stop being replayed. **Carry the saved cases to the +test's new file in the same commit**; what that takes depends on how much moved: + +- When every property test of the source file moves, and the new file has no + regression file yet, `git mv` the regression file to the name the new file + needs. +- When only some of them move (a split), the tests left behind still read the + old regression file, and proptest does not record which test a saved case + belongs to: all the `proptest!` blocks of a file share one. Keep the old file + where it is and copy the `cc` lines to the new file's regression file. If that + file already exists, append to it so its own cases stay; do not overwrite it + and do not `git mv` onto it. A case copied to a test it did not come from is + only one more input for that test to run. + +This is enforced in part by a test, not by an `xtask` subcommand: +`every_proptest_regression_file_belongs_to_a_property_test_that_reads_it` in +`xtask` walks the tree and **fails the build** on a file under any +`proptest-regressions/` directory whose `/src/.rs` (`src/` being the +directory with `main.rs` or `lib.rs`) is missing or +holds no `proptest!` block in code (a mention in a comment or a string literal +does not count; whitespace or a comment between `proptest` and the `!`, as in +`proptest ! { .. }`, still does), naming the file and the source file it expected. It cannot see +the opposite mistake, a split that `git mv`d the regression file away from tests +that still need it: no file is orphaned then, one is just missing, so read the +diff of a split for it. Two more things it does not check: a crate with a +`src/bin//main.rs` that has a property test would be reported as an orphan by +mistake, because proptest then keeps its seeds in `src/bin/proptest-regressions/` +(nothing in this tree has one); and a property test under a crate's `tests/` +directory is invisible to it, since proptest cannot find a `main.rs` or `lib.rs` +above such a file and writes the seed next to the test file instead. Run the +check on its own with: + +```bash +cargo test -p xtask every_proptest_regression_file_belongs_to_a_property_test_that_reads_it +``` + +The pull-request affected-crates selection (see [CI test selection](#ci-test-selection)) +adds `xtask` when a changed file is under a `proptest-regressions` directory or +when the diff of a changed Rust file adds or removes a line containing +`proptest!`, so a property test moved inside `apps/rocm` alone still runs this +check on the pull request, whether or not its seed moved with it. The other +source-scan contracts do not have this rule; they still meet a change confined to +a crate that `xtask` does not depend on in the merge queue, which runs the whole +workspace. To prove a moved seed is read, +append a deliberately unparsable line to it, run the test with `--nocapture` +(`cargo test -p -- --nocapture`), look for proptest's +`unparsable line, ignoring` warning naming that path, and remove the line. +The check does not tell a seed that still reproduces from one that no longer +does; it only confirms that something reads the file. + +### Property tests for persisted settings + +`config.json`, its migration from a legacy rocm-dash `config.toml`, and how +the data directory is found from it have `proptest` properties, beside the +code they drive. They run in the ordinary `cargo test`; to run only them: + +```bash +cargo test -p rocm-core --lib config::props -- --nocapture +cargo test -p rocm-core --lib persisted_paths_props -- --nocapture +cargo test -p rocm --bin rocm therock::python::config_props -- --nocapture +``` + +Each property counts how many generated cases reached the input it is about +(an unreadable `config.json`, a non-finite tick, non-ASCII text, and so on), +prints that as a `reach:` line (`--nocapture` shows it), and fails when the +count is zero, so a generator that stops producing that input fails the run +instead of passing without testing anything. A property that fails prints its +shrunk counterexample. + ### Known-gap reproducers Most `#[ignore]` attributes in the tree gate on the environment: a GPU, a live @@ -303,6 +686,193 @@ behaviour wanted, so un-ignoring it is the fix's acceptance test. A gap in user-observable CLI behaviour that an e2e scenario covers is tracked with an xfail row in `tests/e2e-cucumber/expectations.toml` instead. +## Lemonade serving and dashboard regressions + +The real GPU scenarios `serve-lemonade-inference` and +`serve-hf-checkpoint-inference` in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature) +start Lemonade, request inference, and verify the model identity. The named-model +case uses `Qwen3-0.6B-GGUF`; the checkpoint case uses +`unsloth/Qwen3-0.6B-GGUF:Q4_0`. Both require a usable AMD GPU. Unit tests cover +registered-name catalog resolution, conservative routing of multi-file and +model-specific catalog entries, routing of a registered checkpoint that is not +a plain `owner/repo:variant` (for example, variant `*`) to the router, +quantization matching, and selection of the +runtime's GPU backend when competing backend directories exist. Engine tests also +verify persisted router identity through the health inference request; direct +servers retain the exact requested alias. + +`serve-lemonade-complex-registered-inference` covers the pinned catalog's +`Qwen3.5-0.8B-GGUF` entry, which also needs a projector file. It verifies router +state, real inference for that model, and stopping the scenario-owned managed +service on its isolated port. The `@requires-gpu` and +`@requires-engine:lemonade` tags require a usable AMD GPU on a Lemonade lane. +Its `@nightly` tag also requires `E2E_INCLUDE_NIGHTLY=1`, or the self-hosted +workflow's `include_nightly` dispatch input. A runtime channel named `nightly` +does not enable these scenarios by itself. + +`serve-lemonade-simple-registered-direct-route` is the Linux counterpart for a +simple registered entry (`Qwen3-0.6B-GGUF`: one checkpoint with a variant, no extra +files). It reads the engine state and asserts that `load_response.method` is +`lemonade-packaged-llama-server` and that the `llama-server` that served belongs to +the backend Lemonade selected (a stale directory of the other backend serving in its +place fails it), so a change of route is caught even when the endpoint still answers. It also runs +inference and stops the scenario-owned service. It needs a usable AMD GPU on a +Lemonade Linux lane (`@requires-gpu`, `@requires-os:linux`, +`@requires-engine:lemonade`). It has no flaky expectations row, so a route +regression fails the lane. Of the other direct-route serves, `serve-07` keeps its +flaky Linux row; `serve-08` has none, so it fails the lane too. + +The daemon's Lemonade contracts in +[telemetry_contract.rs](../crates/rocm-dash-daemon/tests/telemetry_contract.rs) +hold observation time during singleton-to-multiple-model transitions. The first +multi-model snapshot must clear both the rate and its observation metadata; +natural expiry cannot satisfy that assertion. Independent endpoints must overlap +while each endpoint checks health before fetching stats. Health failures include +endpoint context, and an unavailable external endpoint removes its rows without +a grace period. A never-seen endpoint remains quiet when nothing answers, +which means a refused connection or a timeout (Windows hangs on a closed local +port instead of refusing). A never-seen endpoint that answers with an error +status or a broken body warns. + +The dashboard scenarios in +[dash.feature](../tests/e2e-cucumber/features/dash.feature) include: + +- `dash-lemonade-external-protocol-discovery`: a Lemonade protocol fixture on an + isolated non-default port, with multiple loaded models and no managed records. + The real CLI, embedded daemon, and TUI must discover it, show unknown + throughput for each model, report installation truthfully, and exclude it from + Services lifecycle targets. Pressing stop and restart must leave no approval + prompt and start no job, and the models stay displayed. Twice, after those + keys and after dashboard quit, one step asserts that no managed record exists + and that the fixture saw no request other than a `GET` of its health or stats + route. The fixture is an in-process thread, so the dashboard cannot stop it; + the request log is what catches a load, unload, or stop sent to the external + server. +- `dash-lemonade-external-protocol-transitions`: loaded-model changes and endpoint + unavailability must update the dashboard, rather than retain false serving + claims. One remaining model shows its throughput, an empty or unreachable + server shows no models, and a reload restores both. +- `dash-lemonade-managed-inference-journey`: a GPU-gated real Lemonade serve and + inference followed by dashboard observation. On Linux Lemonade hosts it is + expected to be flaky and its serve is bounded by `serve_timeout_secs` (readiness polling; the CLI process is killed only 300 s after that) in + [expectations.toml](../tests/e2e-cucumber/expectations.toml). The model must + appear as a full row in Observe, and its instance detail must show it READY + on the port it was served on. This is real engine acceptance; + the protocol fixtures do not prove GPU inference or engine installation. + The flaky row matches every lane that can run this scenario, so the whole + journey is advisory: its dashboard, instance detail, and post-quit inference + checks cannot block a merge. Only the serve step is intermittent, but + expectations apply per scenario. It also does not gate the + "Lemonade — managed runtime installed" wording. + Unit tests gate that wording instead: + `managed_installed_verdict_follows_detect_result` in + [dash.rs](../apps/rocm/src/dash.rs), and the engine pane glyph and verdict + tests in [pane.rs](../crates/rocm-dash-tui/src/ui/tabs/pane.rs). No other + scenario shows the installed state through a real Detect without a serve. + `dash-lemonade-external-protocol-discovery` and + `dash-lemonade-external-protocol-transitions` show only the not-installed + state. + +`dash-07` also covers the positive half of the ownership rule. With a live +managed service, the Services overlay lists it with its port, and the stop key +stages an approval prompt that names it. The scenario declines the prompt, so +nothing is stopped. + +The isolated dashboard scenarios turn Lemonade discovery off unless a scenario +sets it, so a Lemonade server running on the default port of a self-hosted +runner cannot add rows or warnings. The two external protocol scenarios turn it +on with their own host and port. None of them exercises the unset-key default, +which a unit test in `apps/rocm` covers. + +`runtime-lemonade-backend-alignment-reported` tests the advice the direct +serving error gives for a missing `llama-server`, `rocm engines install lemonade +--reinstall`. It installs Lemonade, deletes the selected backend's `llama-server` +(asserting it was there and is gone), reinstalls, and requires the file to be +back. It needs a GPU and a real backend download, and it is tagged +`@requires-os:windows` and `@nightly`, so it runs only on the Windows nightly GPU +lane. The Linux direct route, where the direct serving error mostly fires, is not +exercised by it. Only the selected backend's binary is deleted; a general +damaged-runtime repair is not tested. + +`dash-lemonade-external-protocol-transitions` also requires the `last req tok/s` +legend next to the single remaining model's rate. The Instances table and detail +labels are covered by `table_labels_a_last_request_rate_and_only_then` in +[instances.rs](../crates/rocm-dash-tui/src/ui/tabs/instances.rs) and +`detail_names_a_last_request_rate` in +[instance_detail.rs](../crates/rocm-dash-tui/src/ui/instance_detail.rs). That such a rate +is not counted as live throughput is covered by unit tests: +`node_efficiency_leaves_out_a_last_request_rate` in +[efficiency.rs](../crates/rocm-dash-core/src/efficiency.rs), +`home_heroes_leave_out_a_last_request_rate` in +[home.rs](../crates/rocm-dash-tui/src/ui/tabs/home.rs), +`observe_heroes_leave_out_a_last_request_rate` in +[observe.rs](../crates/rocm-dash-tui/src/ui/tabs/observe.rs) and +`instance_tools_carry_the_last_request_flag` in +[snapshot.rs](../crates/rocm-dash-tui/src/agent/snapshot.rs). +The daemon's two IPv6 contract tests in +[telemetry_contract.rs](../crates/rocm-dash-daemon/tests/telemetry_contract.rs) fail +when the machine cannot bind an IPv6 loopback address, since they are the only cover +for the `::` to `::1` mapping; set `ROCM_TEST_NO_IPV6=1` on a machine that has none, +and they report that they did not run. + +A direct llama-server serve on a non-ROCm GPU backend logs that Lemonade is +serving on it only after the server is ready. `a_direct_serve_that_never_becomes_ready_does_not_log_that_it_is_serving` +and the `NotReady` case of `direct_server_step_failure_stops_the_child_and_marks_state_failed` +in [direct_llama.rs](../engines/lemonade/src/direct_llama.rs) assert the log and the failed +state together; both are Linux-only unit tests. + +A known external Lemonade endpoint that starts answering 401 or 403 on a +scrape-only tick is reported on that tick, not at the next discovery tick +(`lemonade_scrape_only_tick_warns_when_a_known_external_endpoint_requires_credentials` +in [telemetry_contract.rs](../crates/rocm-dash-daemon/tests/telemetry_contract.rs)). + +Resolving a split (sharded) GGUF to its first shard, and preferring a complete set +over an older partial download, have no Gherkin scenario. They need a real +multi-shard checkpoint (several GiB) or a staged Lemonade runtime that the harness +cannot fabricate. The unit tests in +[direct_llama.rs](../engines/lemonade/src/direct_llama.rs) cover them +(`parse_gguf_shard`, `first_shard_of_complete_set`, `is_incomplete_shard_set`). + +The Linux fallback from a failed Lemonade router load to direct serving has no +Gherkin scenario: forcing the router to reject a model needs a real `lemond` +failure that no safe, deterministic e2e hook produces. Unit tests in +[lib.rs](../engines/lemonade/src/lib.rs) and +[router_serve.rs](../engines/lemonade/src/router_serve.rs) cover it, asserting the +result and the state together: `router_load_failure_falls_back_to_the_selected_gpu_llama_server` +(llama-server arguments, final state, and the logged line naming the backend and +saying it is not ROCm), `only_a_router_load_failure_may_fall_back_and_only_on_linux`, +`fallback_never_serves_through_a_missing_or_cpu_backend`, and +`only_a_load_or_verification_failure_is_eligible_for_direct_fallback`. The +`serve-lemonade-complex-registered-inference` scenario remains the real-hardware +check of the router route. + +Unit tests in `rocm-dash-tui` cover the dashboard side of these scenarios. A +Services selection that outlives its row is clamped. A staged approval whose +target disappears is dropped, and the next key is handled as a normal key. The +engine pane shows the check or dot glyph for installed and not-installed +status. The `rocm dash --help` test asserts that Lemonade discovery is scoped to +Linux and WSL. + +Run the ordinary applicability-aware suite with `cargo xtask e2e`. For focused +protocol regression development on Linux or WSL: + +```bash +cargo xtask e2e -- -n 'External Lemonade protocol' +``` + +Cucumber name filtering bypasses the harness's custom applicability/expectation +filter. Use it only for scenarios applicable to the local host. Do not force +GPU-gated scenarios through a name filter on a GPU-less machine. Run the real +serving journeys through the normal GPU lane selection. + +A green local unit or protocol-fixture run does not clear the existing flaky +Linux serving expectations in +[expectations.toml](../tests/e2e-cucumber/expectations.toml). Removing one requires +verification of that same real serving scenario on suitable hardware. Report +GPU checks that could not run, and do not treat an installed version or a startup +message as proof that inference works. + ## Coverage floors Every workspace crate (except the `e2e-cucumber` harness) has a committed line-coverage @@ -396,7 +966,10 @@ or nothing when no crate is affected). It falls back to `--workspace` for any change that can't be confined to specific crates (the lockfile, the toolchain file, the workspace root manifest, CI config, or a doc an xtask test reads — `MANIFEST.md`, `CONTRIBUTING.md`, `docs/ci-hardware-testing.md`, -`tests/e2e-cucumber/README.md`). On `main` +`tests/e2e-cucumber/README.md`). Two further rules add `xtask`, which holds +the proptest seed guard and which no other crate depends on: a changed file under +a `proptest-regressions` directory, and a changed Rust file whose diff adds or +removes a line containing `proptest!` (found with `git diff -G`). On `main` and in the merge queue the full workspace always runs. For a full local verification, keep using `cargo test --workspace --all-targets` above — `affected` is a CI optimization, and it also narrows the local pre-push hooks (next section). @@ -448,14 +1021,61 @@ Known limits, unchanged by the narrowing: the steps check the working tree rathe commit; under prek 0.4.5 a push of several refs, a tag or a deletion runs no hooks; and `-p` builds can resolve Cargo features differently from `--workspace` ones. -Preview what a push would check without compiling: +### Pass stamps + +A step that already passed on this exact content is skipped. After a step's cargo command exits 0, +[prepush_stamp.rs](../xtask/src/prepush_stamp.rs) records a stamp, and the next run of the same step +looks for it first. The stamp stands for: + +- the step and its exact cargo arguments, so a pass recorded for `-p xtask` never lets a later run + over `--workspace` skip (the selection is part of the key, not a separate check); +- the git *tree* hash of `HEAD`, not the commit hash, so an amend that only rewords the message, or + the same content pushed to a second remote, still hits; +- the compiler and cargo identity (`rustc -vV`, `cargo -vV`) and a digest of the `CARGO_*` and + `RUST*` environment (never the values), minus a short list of per-run variables such as + `CARGO_TARGET_DIR` and `CARGO_TERM_COLOR`. The environment is read as raw bytes, so a variable + whose value is not valid Unicode never stops the hook: an unrelated one is ignored and a + relevant one is hashed as the bytes it is. + +The working tree has to be clean for a stamp to be used or written, because cargo checks the working +tree, not the pushed commit: with uncommitted changes the step runs and says why, and an untracked +file counts because it can hide a missing `git add`. The tree is probed again after the step, and +nothing is recorded if cargo changed it (a rewritten `Cargo.lock`, for example). A pass is only +written after cargo succeeded, per step, so a failed `test` does not erase the `clippy` pass that +the retry then skips. + +Stamps live in `rocm-cli-prepush-passes` under the git common directory, so every worktree of a +clone shares them, nothing lands in the work tree and `cargo clean` does not erase them. Each write +drops stamps older than seven days and keeps at most 64. A write goes through a scratch file and a +rename, and removes the scratch file itself when the rename fails; a process killed mid-write +leaves its scratch file behind until a later write prunes it, once it is more than an hour old (a +younger one may belong to a concurrent writer). Any error locating, reading or verifying a +stamp means the step runs (git is run for an explicit directory with `GIT_DIR` and the other +repository-selecting variables removed, which git and prek export to a hook run from a linked +worktree); a stamp that does not match its key exactly, is dated in the future or is +unreadable is ignored (the step names it); a failed write is reported and never fails the hook. + +The skip message says what a stamp does not cover: user-level cargo configuration, installed +tools, files git ignores (the one thing a clean `git status` cannot see) and flaky tests. Force +a run with `--rerun` or `ROCM_PRE_PUSH_RERUN=1` (the hook has no flags, so use the variable +there), or delete the directory the message names. `SKIP=` still skips a hook outright, +and `verify-commits` is a separate hook that checks commit signatures on every push. A skip +proves that this tree passed these checks earlier, not that they ran during this push; for a +fresh run before relying on the result, use `--rerun`. A rerun replaces the recorded pass: it +drops this step's earlier pass before running, so if the rerun fails the next ordinary run runs +the step instead of skipping on the pass it was meant to double-check, and a successful rerun +records a fresh one. Other steps' and other selections' passes are left alone, and `--dry-run` +never touches a stamp. + +Preview what a push would check without compiling (a dry run also says when a real run would skip): ```bash cargo xtask pre-push-check test --dry-run --base origin/main ``` `--base` replaces the range's base; `clippy` and `clippy-e2e` take the same flags. The selection -logic is unit-tested in [prepush.rs](../xtask/src/prepush.rs); as dev tooling, with no change to +logic and the stamps are unit-tested in [prepush.rs](../xtask/src/prepush.rs) and +[prepush_stamp.rs](../xtask/src/prepush_stamp.rs); as dev tooling, with no change to what the `rocm` binary does, it has no Gherkin scenario. The smoke path is the cross-platform local no-fallback acceptance surface. It @@ -602,12 +1222,75 @@ download could start. The printed `command:` line then shows `uv venv --python 3.12 ...` (a version, because no interpreter path exists yet), so it is not copy-pasteable verbatim: `uv` resolves `3.12` itself, and on a host with a different Python 3.12 it can choose that one rather than the managed one -a real install prepares. That scenario is a canonical-layout preview. A preview -of a ROCm 10 or newer wheel (a `--version` pin of 10 or later) does still -need `uv`: it resolves the published package versions with `uv pip compile`, so -on a clean host it can download the managed `uv` and create the data folder's -`tools` folder, although it still prepares no Python. The "no `uv`, no `tools` -folder" guarantee is therefore the canonical layout's only. +a real install prepares. That scenario is a canonical-layout preview, pinned to +an earlier ROCm 7 release because an unpinned install now reads the newest layout. + +A preview of a ROCm 10 or newer wheel (a `--version` pin of 10 or later, a +nightly built since 2026-08-23, or any install with no pin, which now reads that +layout) is held to the same rule, and never fetches `uv` either. On that layout `uv pip compile` +is the one step that chooses the `torch` and `torchvision` versions within the +series the vLLM discover table pairs for that line (the CLI picks the ROCm +version, the torchaudio pin and that series bound), so the preview runs a `uv` +that is already there (the one `ROCM_CLI_UV_BINARY` names, or the managed copy a +previous install left) and, with none, stops before that step. It then prints +what needs no `uv` (the ROCm version, the torchaudio pin, the requirements `uv` +would be asked about), marks `target`, `runtime_key`, `package_specs`, `command` +and `manifest` as `not resolved by this preview`, and says why in a +`resolution_note:` line and a progress line, in the voice of `python_note:`. It +cannot say whether the published stack installs together either. A real install +and `rocm update` still fetch `uv` as before, and a preview made after either +has run is the complete plan. Scenario `runtime-28` checks the note against the +state it left on a host with no `uv`: no managed-tools folder, no tool archive in +its own cache folder (here the real detector, since no stand-in `uv` is named), +no saved config, and no `torch` or `torchvision` version anywhere in the output. +`runtime-27` is the other polarity: with a `uv` present (the recording stand-in, +which answers `uv pip compile` with the fixture's four pins, drains its stdin +under `FAKE_UV_DRAIN_STDIN`, and logs its arguments under `FAKE_UV_LOG`), the +preview runs it exactly once, for the compile, and prints the resolved plan. The +`therock-next` scenarios that print resolved package specs start from `And a uv +is already installed`, which installs the managed `uv` into the scenario's data +folder the way a real install would have. + +The decision is unit-tested on every platform, Windows included +(`the_version_step_is_decided_by_layout_access_and_a_present_uv_alone` in +[uv_access.rs](../apps/rocm/src/therock/resolve/uv_access.rs): the state matrix of +both layouts, a preview or an install, and a `uv` present or not), as are the +plan's whole text and its key order against a resolved plan's +(`the_plan_a_preview_without_uv_prints_is_exactly_this` and +`a_resolved_and_an_unresolved_plan_print_the_same_keys` in +[wheel.rs](../apps/rocm/src/therock/resolve/wheel.rs)). On Unix, +`a_preview_never_acquires_uv_and_a_real_resolution_still_does` there runs the +real resolution against a served index and a stand-in `uv`, counting what each +row looked for, acquired and ran. What no test reaches without the network is the +real download a real install makes when no `uv` is installed, and that the public +entry points a real install and `rocm update` call pass `UvAccess::Acquire`: the +first is a network call, and the second is held only by the types (those entry +points return a resolved plan and refuse the unresolved one) and by the lanes +that install a ROCm 10 runtime for real. + +Which source an install reads is covered by the `therock-next` scenarios against +loopback fixtures served through the trust gate (the +`ROCM_CLI_THEROCK_*_BASE` variables, see [release-trust.md](release-trust.md)). +`therock-next-01` is the default (an unpinned release install reads the newest +layout) and `therock-next-16` its other half (a pin to an earlier ROCm 7 release +reads the canonical stream); `therock-next-17` refuses a group family with no +pin and then runs the earlier-release pin the refusal names; `therock-next-18` +and `therock-next-19` are the same two shapes for a tarball install; +`therock-next-20` to `therock-next-22` are the nightly channel: no pin reads the +newest layout at its newest build that has PyTorch wheels, and a build date on +either side of the 2026-08-22/2026-08-23 boundary reads the stream that holds +it; `therock-next-23` is the tarball refusal for a nightly pin before that +boundary, run together with the wheel install it points to. `update-11` holds +that an update installs from the layout its runtime recorded: a runtime of the +earlier release stream recorded with only a family label still updates, instead +of being refused for an exact arch. The live counterparts, which resolve the real indexes and so run on +gated lanes, are `runtime-08` and `runtime-31` (nightly, `@nightly` lane) and +`runtime-09` and `runtime-32` (release, `@requires-gfx-target`: they need a +detected GPU target). The routing rule +itself, `layout_serving`, is a table of every request shape on both channels in +[source.rs](../apps/rocm/src/therock/source.rs), including the boundary dates. +The paced tarball download in `download-progress-01` reads the newest layout's +catalog with an exact arch for the same reason. The preview does not announce a Python the install would refuse either, and it prints nothing that predicts what a real install would do on a path that ends in @@ -632,6 +1315,18 @@ order and the interpreter probes, whose stand-ins are shell scripts as in the other resolver tests, and no end-to-end run covers the Windows preview, so the Windows search order (`python`, `python3`, `py` with `PATHEXT`) is not covered by any test. +`rocm install sdk --python-version` is covered by scenarios `runtime-29` (a ROCm +10 preview plans the requested Python and its wheel tag, and not 3.14) and +`runtime-30` (a version the canonical layout is not published for is refused, +preparing nothing; the ROCm 10 refusal is unit-tested only). The rest is unit-tested in [python.rs](../apps/rocm/src/therock/python.rs): +`python_version_is_a_choice_among_the_layouts_published_pythons` is the matrix of +layout against requested version, and +`a_recorded_wheel_tag_maps_back_to_the_requirement_it_came_from` is what lets an +update keep a runtime's Python. That a ROCm 10 runtime on another Python skips the vLLM +auto-install is `auto_install_skips_vllm_on_an_sdk_moved_off_its_python` in +[runtime_install_finalize.rs](../apps/rocm/src/runtime_install_finalize.rs), and that a ROCm 7 runtime never does (whatever `--python-version` it was given) is `recorded_python_version_is_only_reported_for_a_rocm_10_runtime` in [update.rs](../apps/rocm/src/therock/update.rs); a tarball install refusing the flag is `python_version_is_refused_for_a_tarball_install` in [therock.rs](../apps/rocm/src/therock.rs). A real +install and an update on a non-default Python are not covered by a scenario: they +need a ROCm 10 SDK download. `a_dry_runs_whole_python_output_follows_one_decision_in_every_state` in [python.rs](../apps/rocm/src/therock/python.rs) is the state matrix: every combination of `PATH` Python (usable, unusable, absent), saved Python (usable, @@ -642,6 +1337,9 @@ progress output and that `uv` was not run and no file was written. Beside it are `a_dry_run_refuses_a_managed_python_version_the_install_would_refuse` and `a_dry_run_reads_every_spelling_of_a_managed_python_version_the_same_way`. The preview does still write its package-index cache under the cache folder. +The `update_check` tool and the plain `rocm update` check (no `--apply`) are not +previews in this sense: on a ROCm 10 or newer runtime they resolve with `uv` and +can fetch it. They prepare no Python, though (see `update-09` and `update-10`). Then it verifies: @@ -690,7 +1388,7 @@ Use a deterministic family only for developer tests that need fixed package selection: ```bash -python scripts/therock_sdk_install_test.py --root .rocm-work/tests/therock-sdk-install --fresh --family gfx120X-all +python scripts/therock_sdk_install_test.py --root .rocm-work/tests/therock-sdk-install --fresh --family gfx1200 ``` ### GPU targets with no ROCm build @@ -915,6 +1613,34 @@ cargo xtask e2e -- -n "comfyui-(0[5-9]|1[0-3])" `-n` bypasses the suite's applicability resolver, so the `@requires-os:linux` gate is not applied: run it only on Linux. +`comfyui-15` to `comfyui-19` cover which runtime `rocm comfyui install` uses +when none is chosen: it recovers the runtime `rocm setup` created before +counting the ready ones, as `rocm serve` does, so a runtime whose registry entry +was lost is found. The real run (`comfyui-16`, against an existing checkout that +needs no dependencies) writes the entry back; the `--dry-run` preview +(`comfyui-15`) names the same runtime but writes nothing, because a preview +promises not to change files. The preview prints the install command with that +runtime's `--runtime-id`, and `comfyui-17` runs that command as printed from the +state the preview left: an explicit `--runtime-id`, like an active or default +runtime, finds the runtime setup created when the registry alone cannot answer, +so the command works and writes the entry back. `comfyui-18` runs the same +printed command when the registry cannot take that entry back (a plain file +stands where its directory should be, which also stops `root`; a read-only +directory would not): the preview still succeeds, and the real run fails naming +the runtime whose entry it could not restore, rather than saying "Install ROCm +first", and leaves the registry as it was. The variant with a read-only registry +directory that holds other runtimes is a unit test, because a read-only +directory is writable to `root`. `comfyui-19` covers the other way a setup folder +can matter: when its runtime is no longer usable (its interpreter is gone) and one +ready runtime is registered, the preview uses the registered one and does not +register the unusable one, because an unusable runtime is not a ready runtime and +cannot change the count; its own error is reported only when no registered +runtime is ready, which a unit test covers. All five run with no GPU: + +```bash +cargo xtask e2e -- -n "comfyui-1[5-9]" +``` + `comfyui-14` is different: it installs ComfyUI into a real, isolated managed runtime and checks the ROCm torch stack survives with no `nvidia-*` packages added. It carries `@requires-gpu @nightly @requires-os:linux @requires-bare-metal` @@ -935,20 +1661,124 @@ install. `serve-26` and `serve-27` (`features/model_serving.feature`, the AIM serve and restart canaries) are `@nightly` for cost: each pulls the AIM image and a 9B checkpoint on a cold runner and may wait up to 1200s for readiness, which on the -per-PR Strix Ubuntu job (about 25 minutes without them, 45 minute cap) used up -the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable -and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` -is set on such a host: the nightly Strix Halo lane, or an `include_nightly` -dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then gets a 105 minute -cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that -they stay reachable when the opt-in is set. - -Caveat: it is unconfirmed that the nightly Strix Halo host has Docker (nothing in -`nightly.yml` installs or checks it). Without it these two scenarios skip on -every nightly run, and only a manual `include_nightly` dispatch on a host with -Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker -is reachable ("Report whether Docker is reachable"); check that line before -relying on this coverage. +Strix Ubuntu job, back when it ran on pull requests (about 25 minutes without +them, 45 minute cap) used up the whole budget. They still need `@requires-engine:aim`, +i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where +`E2E_INCLUDE_NIGHTLY=1` is set on such a host: the nightly Strix Halo lane, or an +`include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then +gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins +the tagging and that they stay reachable when the opt-in is set. + +Both scenarios skip on a host without Docker; the `e2e-gpu-nightly-strix` job +logs whether Docker is reachable ("Report whether Docker is reachable"). +`serve-27`'s known failure on the Strix Halo pool is an xfail row in +`expectations.toml`, tracked in rocm-cli#1058; a pass there is an XPASS, which +means the row is stale. + +### Lemonade managed-serve failure evidence + +On a failed serve, the shared serve steps preserve the full service log under +`service-logs/` in the E2E results artifact before stopping the service. The engine +is taken from `--engine X` or `--engine=X`, or else from the printed serve plan; if +neither names one, the report says `engine unknown: backend evidence skipped`. A +value that starts with `--` (as in `serve m --engine --managed`) is another flag, not +an engine, and counts as no engine named. +Lemonade failures also include the installed runtime manifest's version/backend, configured +`backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints +and `version.txt`, +and each candidate's `--version` and `--help` output. Candidates are searched +under the `rocm-stable`, `rocm-nightly`, `rocm` and `vulkan` directories of +`bin/llamacpp`, down to three directory levels below each; the cpu backend is not +searched. The search is bounded as a whole, not per directory: it reads at most +128 directories and 2048 directory entries across all four backend directories, +stops as soon as it finds a ninth candidate (it lists the first eight and says the +list is cut; exactly eight is not reported as cut), and stops when the +inventory's time is up, checking the clock at every directory and entry. Siblings are visited in +sorted order, so a search that finishes inside its limits is deterministic. +Symlinks are never followed: only real directories are entered and only a regular +file named `llama-server` counts, so a link cycle ends and a link out of the +backend directory is not read; the report says how many symlinked entries it +declined to follow. The same rule covers every file the report reads: the +manifest, the pins file, `version.txt` and a candidate's fingerprint target are +refused when the last path component is a symlink, and the report says `symlink +not followed` in place of the content. On unix the refusal is made by the open +itself (`O_NOFOLLOW`), so a path swapped for a link after an earlier check is still +refused; on Windows it is a metadata check followed by an open, so a swap between +the two is not caught (read-only evidence, and the swap needs write access to the +backend directory). Only the last component is checked: a link in a parent +directory is followed. When a limit or the clock ends the search early the report +says which, and does not claim an empty result. When nothing is found, the report +lists the searched locations and what `bin/llamacpp` holds (the first 16 names +out of at most 64 entries read, with a count of those omitted), and does not +guess at the cause: an interrupted download, a layout outside the search and a +build that was never installed look the same. The manifest and pin reads, the +search and each candidate's fingerprint run on blocking threads, never on an +async worker, and the search carries its own deadline because a timeout around a +blocking task would only drop its handle. +At most eight candidates +are hashed and probed, with two read-only probes per candidate. Each probe retains +up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are +marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw +output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). +Each probe has a five-second timeout. The inventory has a 20-second budget that +starts when it is entered and covers the manifest and pin reads, the search and +the hashing; the probes have their own 20-second budget, which starts +when the service stop returns; hashing runs off the async executor, rejects files +over 512 MiB, and checks its deadline between 64 KiB reads, so memory stays at +one buffer whatever the size. The cap sits well above a dynamically linked +`llama-server`: the `rocm-stable` build b12109 on the Strix Halo Ubuntu lanes +was fingerprinted under the former 32 MiB cap, so current builds fit with room to +spare. The larger cap is margin for a statically linked multi-target build, +which is an assumption about future builds, not a measured size, and a larger +file is a finding in its own right. A candidate over the size cap gets no fingerprint, and +its entry says so with `sha256 unavailable: executable exceeds 512 MiB hash +limit`; it is still listed and probed. Filesystem +operations and process cleanup are not preemptible, so this is not a hard +wall-clock limit on the entire report. +A candidate or configured build pin is **not proof** of which binary the router +launched; a failed probe is retained as evidence, not treated as an identity. +Probes inherit the harness environment rather than reconstructing the engine's +managed library paths, so a probe loader error alone does not establish why the +serve failed. The report reads files first and executes a backend second. The +manifest, pins, candidate paths, fingerprints and `version.txt` are read once the +serve has exited non-zero or its readiness wait has timed out and the log has +been read and archived, but before the service is stopped, so they describe the state being +diagnosed. The `--version` and `--help` probes run after the stop returns, in the +report's "probes, run after the service stop was attempted" section: llama.cpp +builds commonly initialise a GPU backend even for `--version`, and doing that next +to a still-running router on a shared-memory device could change the state being +diagnosed. A stop that fails or finds no record can leave the router running; the +report's `stop` section says so, and the probes still run. No lane has shown +whether a probe initialises a GPU backend, which is why they are kept off the +router's side of the stop. The order is fixed in one function, +`backend_evidence_around_stop`, whose unit tests make the stop delete the candidate +to show the fingerprint was read before it and the probes came after it. Nothing +before the stop may keep it from running: a serve with no isolated scenario root +has no manifest to look for, so the report says `backend evidence unavailable: +the scenario has no isolated root` (and the stop's own status says it was not +attempted), and a panic while looking the manifest up is caught and reported the +same way, so the stop runs exactly once and the failure report is not lost. + +The probe's timeout, kill and reap path is tested on every platform, Windows +included: the unit tests re-run the test binary itself as the child (an ignored +helper test selected by an environment variable), let the probe time out, and +require that the child is gone afterwards. The shell-script `llama-server` +stand-ins used by the other probe tests exist on unix only. Not covered anywhere +is the `kill/reap failed` branch, which needs an operating system that refuses to +kill a child it just started. + +Use the evidence to distinguish backend startup, download, and router readiness +failures; ROCMAI-898 tracks the Strix Halo Ubuntu investigation (first tracked +as [issue #897](https://github.com/ROCm/rocm-cli/issues/897), which is closed). A +backend's ready line does not establish that the router serves the requested +model, and a readiness timeout alone does not establish a shutdown. Diagnostics do not change the CLI's asynchronous +managed-launch contract or imply a backend fix. + +Exercise the diagnostics and expectation scoping without a GPU: + +```bash +cargo test -p e2e-cucumber --lib lemonade +``` ## Runtime Selection And Activation @@ -1042,12 +1872,46 @@ rocm runtimes adopt \ --runtime-key adopted-release-pip-gfx120x-all-7-13-0 ``` +Neither writes into the folder it registers: the registry entry is the only +file created. The `.rocm-cli-runtime.json` that `rocm install sdk` leaves in an +install root is ROCm CLI's proof that it owns the folder, so an imported or +adopted runtime does not get one, and uninstall and prune never delete such a +folder. A second import or adopt of the same runtime key is refused until +`--replace` is given. `@id:runtime-lifecycle-import-rejects-duplicate-unless-replacing` +(`runtime-lifecycle-06`) checks the import half end to end: the registry entry +exists, the imported folder gains no manifest file, and the duplicate is refused +until `--replace` (the unit tests also check the refused entry is left as it +was). Adopting needs a Python environment to probe and the +recovery of the runtime `rocm setup` created (which restores a missing registry +entry from the manifest already in the setup folder) needs a setup, so neither +has a scenario; the unit tests below assert each path's file set instead, and +`write_runtime_manifest` is the one routine all four paths write through. It writes the registry entry last, after the install-root file, so a write that fails leaves no new entry and an error that says a runtime could not be registered or restored matches the registry; recovery in particular never rewrites the file it restores from, whatever shape that file has. +A source-reading test, `only_the_shared_routine_writes_a_registry_entry`, fails +if production code in the `rocm` app, `rocmd`, `rocm-core` or the engines both +names the registry directory (through the two path helpers, or by spelling the +`runtimes` and `registry` segments) and writes a file, or if a file not on its +reviewed list starts calling the path helpers; +`registry_writer_detector_sees_every_way_of_building_the_path` pins what the +detector sees. It does not catch a writer that is handed the registry path by a +function in another file, a write call that is not in its list of write +primitives, or code outside those source roots. + +`--root` may be left out when `--python` sits directly in the environment's `bin` +(or `Scripts`) folder; an interpreter anywhere else is refused until `--root` is +given (scenario `runtime-26`). + Focused unit coverage: ```bash cargo test -p rocm --bin rocm runtime_activation cargo test -p rocm --bin rocm runtime_import cargo test -p rocm --bin rocm runtime_adopt +cargo test -p rocm --bin rocm runtime_install_writes +cargo test -p rocm --bin rocm setup_recovery +cargo test -p rocm --bin rocm write_runtime_manifest +cargo test -p rocm --bin rocm a_failed_manifest_write +cargo test -p rocm --bin rocm only_the_shared_routine_writes +cargo test -p rocm --bin rocm registry_writer_detector cargo test -p rocm --bin rocm runtime_key_includes_version_for_side_by_side_installs cargo test -p rocm --bin rocm engine_plugin cargo test -p rocm --bin rocm model_registry @@ -1067,6 +1931,40 @@ cargo test -p rocmd event_collector cargo test -p rocmd event_dispatcher ``` +## Lemonade Installs On Disk + +`features/lemonade_installs.feature` covers what a user is told about the +Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: + +- `lemonade-installs-01` and `-02` reach the engine's pre-install free-space + refusal from `rocm engines install lemonade` and from `rocm serve`, with 1 MiB + reported free (below what any Lemonade install needs). Room for Lemonade but + not for the multi-gigabyte ROCm backend only warns, which the unit tests cover + because a scenario would have to start the download. They use + `ROCM_E2E_LEMONADE_FREE_SPACE_BYTES`, compiled only into the + `rocm/e2e-test-hooks` build `cargo xtask e2e` makes, and check against the + disk that nothing was downloaded. That variable also waives serve's no-GPU + refusal, as serve-18's failure hook does. +- `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime + and checks `rocm storage report` lists both and flags the unused one. +- `lemonade-installs-04` uninstalls that runtime and checks the install record + is cleared. +- `lemonade-installs-05` and `-06` cover the two cases the report must not + overstate: a record naming a folder with no install (the surviving copy is + not offered for deletion) and a read-only runtime (its copy is not said to be + removed with it, and uninstall then leaves it). A copy inside a registered + runtime is never marked reclaimable, because it is reused when that runtime + becomes active; two registered runtimes are covered by a unit test. + +Run them alone with `cargo xtask e2e -- -n "lemonade-installs-"`. + +The Windows console fix for the backend download's progress is unit-tested +through the real command builder (`backend_install_command` in +[process.rs](../engines/lemonade/src/process.rs)), which receives the console +state and the hide step, so a dropped or unconditional hide turns the test red. +Whether the output actually reaches a real console is a manual check in +[manual-testing.md](manual-testing.md). + ## Uninstall And Download Removal `rocm uninstall` and `rocm storage remove-downloads` both end in the same @@ -1089,9 +1987,25 @@ The `uninstall-NN` E2E scenarios (`features/uninstall.feature`) check the same behaviour through the built binary, confined to the scenario's own folder, including a folder that cannot be checked stopping the uninstall before anything is removed, and a dry run on a machine ROCm CLI never touched -(`uninstall-06`) reporting nothing to remove and creating no folder. The -`uninstall` command does not start the file logger, which would otherwise create -the data folder's `logs` folder before the plan is built: +(`uninstall-06`) reporting nothing to remove and creating no folder. At a +terminal (a pseudo-terminal on the Linux lane), a dry run that lists something +ends by saying to run it again without `--dry-run`, and `uninstall-07` does that +and checks the listed folder goes; a run whose stdin or stdout is not a terminal, and an +empty plan even at one, name no next step (`uninstall-08`, `uninstall-09`). A +mistyped `--force` is refused with a pointer to `--yes`, and a near miss of it +is not answered with `--force-dev-binaries` (`uninstall-10`); the help starts +its examples with a dry run (`uninstall-11`). The assistant's uninstall preview +ends differently, since it has no flag to offer: it says removal needs approval, +which `lifecycle_read_mutate_split_is_honest` in [tool_policy.rs](../apps/rocm/src/tool_policy.rs) backs. +`uninstall-13` to `uninstall-15` drive the confirmation prompt under a +pseudo-terminal, which is the only way to reach it, and pin the shared `[y/N]` +helper's behaviour for an answer that is neither yes nor no, for ending the input +with Ctrl-D, and for a plain no, each together with the config folder being left +in place. They run on the default mock lane; the helper's other callers (`rocm fix`, +the SDK-overwrite prompt) share it and are covered by its unit tests in +[confirm.rs](../crates/rocm-core/src/confirm.rs). +The `uninstall` command does not start the file logger, which would otherwise +create the data folder's `logs` folder before the plan is built: ```bash cargo xtask e2e -- -n "uninstall-" @@ -1186,10 +2100,28 @@ provider-assisted plan before dispatching it, so it never runs the plan's command and shows only the plan text; for that surface the reuse of the rendered plan is covered by the unit tests below. +A `serve` the provider proposes must also pass the four serve safety rules the +chat assistant's `rocm_command` is held to (no CPU device, no public bind, a +loopback host, managed serving), which one shared check in +[tool_policy.rs](../apps/rocm/src/tool_policy.rs) applies to both. Four more +mock-lane scenarios in the same feature file run `rocm "serve a model"` against +a provider whose plan breaks one rule each: `chat-13` +(`chat-planning-assistant-cpu-serve-refused`), `chat-14` +(`chat-planning-assistant-public-bind-serve-refused`), `chat-15` +(`chat-planning-assistant-non-local-host-serve-refused`) and `chat-16` +(`chat-planning-assistant-unmanaged-serve-refused`). Each asks the provider +exactly once, prints the CLI's own plan rather than the provider's, runs +nothing, and says why the provider's plan was not used. The two unit suites +that run one shared table of refused and accepted `serve` argvs are +`chat_refuses_each_unsafe_serve_and_proposes_nothing` and +`planner_refuses_each_unsafe_serve_and_plans_nothing`; deleting a rule from the +shared check fails both. + Focused coverage: ```bash cargo test -p rocm --bin rocm provider_planner +cargo test -p rocm --bin rocm unsafe_serve cargo test -p rocm --bin rocm freeform_execution_validation_rejects_provider_assisted_plans cargo test -p rocm --bin rocm freeform_plan_dedup_prevents_provider_assisted_divergence cargo test -p rocm --bin rocm freeform_execution_reuses_the_rendered_plan_without_rebuilding @@ -1215,6 +2147,29 @@ second terminal to watch the download. The no-log first-run path should report `no CLI logs found yet` without creating the configured `data/logs` directory. +The log views read the end of a log through one bounded routine, +`rocm_core::tail_log_decoded`, and decode invalid text lossily. Scenario `logs-08` +plants an action log with a byte that is not valid text and checks that +`rocm logs --search` still finds every matching line. Scenarios `logs-09` and +`logs-10` (`rocm services logs`) and `logs-11` and `logs-12` (`rocm comfyui logs`) +plant an over-long single line and an empty file and check that only the empty +file is reported as having no output, with the wording checked against the file +on disk. Scenario `logs-15` plants a command log whose only line is that long and +checks that `rocm logs` lists it with a line saying no output could be shown +rather than printing `No logs found yet.`. The routine itself is covered by `cargo test -p rocm-core --lib log_tail`. + +The `service_logs` tool (what the assistant and MCP clients call to read a +server's log) answers through `rocm_core::service_log_tool_summary`, so an empty +log (`has no log output yet`) and a log whose newest line is longer than the +4 MiB a log view reads (`none is shown: the newest line is longer than ...`, with +`newest_line_exceeds_read_bound` set in the structured reply) are not both +reported as zero lines read. The structured `lines` is the number of lines +returned and `lines_requested` the number asked for (scenario `logs-16` asks for +50 lines of a three-line log). The CLI's copy of the tool is run by scenarios +`logs-13`, `logs-14` and `logs-16` through `rocm mcp-call service_logs`; the daemon's copy +is covered by `cargo test -p rocmd --lib service_logs`, because the harness +drives only the `rocm` binary. + TUI log pagination focused tests: ```bash @@ -1410,7 +2365,7 @@ so it cannot receive the value-typed `LEMONADE_API_KEY` that Lemonade's server reads. Rather than launch an unauthenticated public server, `rocm serve` **fails closed** on that exact combination (public bind + `--engine lemonade` + Windows) and directs the user to `--engine vllm` or a loopback host. vLLM (all platforms, -via `VLLM_API_KEY`), the packaged llama-server fallback (via `--api-key-file`), +via `VLLM_API_KEY`), direct llama-server serving (via `--api-key-file`), and Lemonade on non-Windows are all fully supported. Teaching the Windows detached-spawn primitive to carry a value-typed override (so Windows Lemonade can serve public with auth) is tracked as a separate follow-up. @@ -1531,6 +2486,284 @@ directly, exits 0, and records whether that signal worked, adding a second stderr `note:` when it did not (`cargo test -p rocm --bin rocm services_stop_` and `cargo test -p rocm --bin rocm attached_ctrl_c_`). +`rocm services stop` and `rocmd`'s `stop_server` terminate the processes a +record names through one routine, +[`terminate_recorded_service_pids`](../crates/rocm-core/src/proc_lifecycle.rs), +and wait `MANAGED_STOP_GRACE` for each; neither binary keeps a copy. The +routine is tested in `rocm-core` (`cargo test -p rocm-core proc_lifecycle::`): +the supervisor is visited before the engine, each PID with its own start-time, +a PID recorded under both roles once, and the stopping process's own PID is +reported as `self` in `pid_outcomes` and never signalled, in both commands. +On Windows the verified root is preceded by `taskkill /PID /T /F`, because +`terminate_verified` cannot walk a process tree there; that step is tested +through injected functions that run on every host (`the_tree_is_killed_before_the_root_is_terminated` +and its neighbours) and, against a real tree, by a Windows-only scenario +(`service-stop-07`, below). The callers' own tests check +that each reaches the routine and reports what it returns +(`cargo test -p rocm --bin rocm service_control::tests::a_stop_` and +`cargo test -p rocmd service::tests::a_stop_`), and each carries a source check +that no private copy of the routine or its constant has come back. Those checks +pin names only: they are a tripwire for a copy returning under the same name, not +coverage of what either stop does. A confirmed +stop also clears the PIDs and start-time tokens it stopped from the record +(`ManagedServiceRecord::clear_recorded_processes`, one call from both +binaries), and an unconfirmed one keeps them; a PID recorded without a token is +signalled on trust alone once the OS has recycled it, so a stale one must not +survive for a second stop to find. +`a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` +runs the real routine with the per-PID signal injected, and asserts the +persisted record after the first stop and that a second stop signals nothing; +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop` is the +other direction. `service-stop-07` +(`@requires-os:windows`, no GPU) is the live coverage of the Windows tree kill when it is run on a Windows host, which no CI job does: +it records a PowerShell parent that has started a `ping` child, runs `rocm +services stop --yes`, and asserts that both processes are gone together with the +report (`Local server stopped`, `stopped processes: 1`) and the record (stopped, +no process, no stop request, key file gone). The child is the tree kill's own +work: with the `taskkill /T` step removed the stop ends only the recorded +process and the scenario fails on the surviving child. The scenario keeps the +child's own identity (`E2eWorld::recorded_descendant`) and teardown ends it when +it is still that process, so a stop that fails to take it down does not leave +`ping` running for its five minutes behind a dead parent; the decision is +`e2e_cucumber::stray_process::leftover_action`, which a table test covers on +every host (ended only while it is still the recorded process, never once gone, +reused or unconfirmed). `service-stop-01` plants +PID 0 and `service-stop-02` is Linux-only, so neither reaches that step; the +injected unit tests above cover its ordering and verdicts on every host. + +The tree kill is behind the same identity gate as the rest of the stop. On +Windows `process_start_ticks` is the process creation time (`GetProcessTimes`, a +`FILETIME`), so a record written by this version refutes a PID that an unrelated +process has taken, and the stop reports `identity_mismatch` without running +`taskkill` on it; only a record written before Windows recorded a token has +nothing to refute: a stop refuses a PID only when a token was recorded and the +live one differs, and `what_a_stop_does_for_each_recorded_and_live_token` pins +that for every case on every host (no token, which is also the shape +`service-stop-07` records; a token that matches; one that differs; one that +cannot be read back), asserting whether the tree kill runs and the verdict the +stop reports together. `a_filetime_is_its_high_word_above_its_low_word` runs the +conversion on every host. `terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` +runs the real stop against this test's own PID with a wrong token on Linux and +Windows (a stop that went ahead would end the test process), and +`terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` +repeats it against a bystander on Windows only and then stops that bystander +with its true token. `service-stop-08` is the live version on every OS: it +records a running process under a token no live process has, stops the server, +and asserts that the process is still running and the record names no process. +None of the Windows-only paths has been run on a Windows host for this change, +and I am not claiming a Windows result for this head. What the CI configuration +does and does not run there, read from `.github/workflows/ci.yml`: the +`windows-debug` job (`Windows build and test (debug)`), when Rust files change, +runs clippy, the build, `cargo test --workspace --all-targets` and the harness +self-tests. That test run includes unit tests in `proc_lifecycle.rs` that stop a +real process on Windows: `terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` +and `terminate_recorded_pid_stops_a_process_recorded_without_a_token` are +`#[cfg(windows)]` and start a real `ping` child and stop it through +`terminate_recorded_pid`, which takes the PID hold, reads the creation time, +applies the identity gate (a wrong token leaves the process running; a matching +or absent token stops it) and runs the System32 `taskkill /T /F`. The e2e target +is excluded from `cargo test`, so no scenario runs there: the `service-stop-*` +and `service-unqueryable-*` scenarios do not run on that job. +`service-stop-07` (`@requires-os:windows`) and the Windows half of +`service-stop-08` can run only through the harness on a Windows host, which is a +manual step (see "Windows host check" in [manual-testing.md](manual-testing.md)); +on the Linux e2e jobs `-07` is skipped and `-08` runs its Linux half. Not covered +by any CI job: the end-to-end tree kill of a parent with a child (`-07`), the +failure cases that cannot be staged (an `OpenProcess` access-denied process, a +failing or stalled `taskkill`, `taskkill` exiting 128), and the launch-cleanup +branch run against a real Windows child. The failed-launch cleanup terminates its +child through the same routine as a stop (`terminate_recorded_pid_within`, so on +Windows the tree kill runs too), and what it asks of the termination is tested +through an injected function on every host +(`abandoned_launch_asks_the_termination_for_the_spawned_child_with_its_token`: +the child's PID with the token captured at spawn, and nothing at all for a child +with no token where tokens are recorded); that the production wiring reaches the tree +kill is not observable on Linux, where the termination walks the tree itself, so it +rests on the routine's own tests and on review. The unit tests above stop a `ping` that has no child of its own, so they +do not show a descendant being taken down. A stop that cannot +verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID +eq "` on Windows (`process_check_command`). +Three further Windows paths of that stop are tested through injected functions +that run on every host; no scenario forces their failure cases. A PID that cannot be +queried (`ProcessProbe::Unqueryable`, which on Windows includes a live process +the stop may not open) is `Indeterminate`, so the stop reports it unverified +instead of already gone (`a_pid_that_cannot_be_queried_is_unverified_not_already_gone`, +`a_stop_of_a_live_process_it_cannot_query_is_not_confirmed`, and +`a_failed_open_is_gone_only_for_no_such_process` for the error codes). The PID is +held open from before the identity read until the stop returns, so the creation +time that was compared is that of the process the kill then reaches +(`the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop`). A PID +that could not be held (the process could not be opened) while its identity still +reads as matching is not signalled either: nothing would reserve it between the +read and `taskkill`, so the stop is `unverified` and neither the tree kill nor the +termination runs +(`a_tree_kill_is_not_run_for_a_pid_that_could_not_be_held_even_when_its_identity_matches`; +a gone, recycled or unreadable PID keeps its own refusal, +`an_unheld_pid_that_is_gone_recycled_or_unverifiable_keeps_its_refusal`, and +where there is no tree-kill step the missing hold changes nothing, +`without_a_tree_kill_step_an_unheld_pid_is_still_terminated`). And a +`taskkill` that failed to start or exited nonzero leaves the stop unconfirmed, +whatever the root's own termination says, because only the root's exit is +watched on Windows. When the root itself was seen to exit (or was already gone) +the outcome is `descendants_unconfirmed`, which says the root is gone and a +process under it may still be running; it is not `timed_out`, which is kept for +a root that outlived its deadline. `rocm services stop` prints it as a `not +confirmed:` line, and `a_stop_with_an_unconfirmed_tree_kill_says_what_happened_and_keeps_the_service` +asserts that line, that no `still running` line is printed, and the record, stop +request and key that are kept. The warning lasts one run, and the line says so: the +next `rocm services stop` finds the root gone and reports the stop confirmed, and +a listing completes the pending stop the same way, neither with new evidence about +the processes under the root (the same test runs that second stop and asserts it); +keeping it durably would need a marker in the record, which is not done +(`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, +`only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found`). A +`taskkill` that exits 128 (process not found) is judged by the root's own re-read: +with the root still there the kill counts as failed +(`a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill`); with +the root gone it exited by itself, and the outcome is `exited_before_tree_kill`, +not a stop on its own, because `taskkill /T` reaches children only through a live +parent and nothing recorded what was under the root +(`a_tree_kill_that_found_no_process_with_the_root_gone_is_not_a_stop_by_itself`). +Across the record, such a root becomes `already_gone`, and the stop confirmed, +only when another recorded process is confirmed gone (`a_parent_that_exits_alone_is_confirmed_when_the_recorded_engine_is_confirmed_gone`, +`two_roots_that_both_exited_alone_are_a_confirmed_stop`); with nothing else +recorded, or the engine still running or unconfirmed, it stays unconfirmed +(`a_parent_that_exits_alone_with_nothing_else_recorded_is_not_a_confirmed_stop`, +`a_parent_that_exits_alone_stays_unconfirmed_while_the_recorded_engine_is_not_gone`, +`the_stopping_process_does_not_corroborate_a_root_that_exited_alone`). Both +callers fold every row, so the record, PIDs and key are kept unless every +recorded role is confirmed +(`a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does` in +`rocm` and in `rocmd`, `a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed` +in both). A failed tree kill with the root gone stays unconfirmed +(`a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed`). That `taskkill` runs under +a deadline of `MANAGED_STOP_GRACE`: one that stalls is killed and reaped and +counts as a failed tree kill, so it cannot hang `rocm services stop`, `restart` +or the attached Ctrl-C stop +(`a_helper_that_stalls_is_killed_and_reaped_at_its_deadline` runs a long-sleeping +child under a 200 ms deadline on every host and asserts the error, the bounded +elapsed time, the reaped child and the failed verdict; `a_helper_that_exits_in_time_gives_its_exit_code` +and `a_helper_that_cannot_be_started_is_an_error` are its neighbours). The report and the record +a `timed_out` stop leaves are covered by +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. The +`taskkill` is started from the system directory that `GetSystemDirectoryW` +returns, not by a bare name `PATH` could resolve and not from the `SystemRoot` +environment variable a caller could redirect +(`a_system_tool_is_resolved_under_the_system_directory` builds the path from a +given directory on every host; the Windows call itself is run only by the +Windows job's tests that start `taskkill`; ComfyUI's own `taskkill` and `tasklist` calls use +the same helper). A readable +recorded token that differs outranks a failed query +(`a_readable_token_mismatch_outranks_a_failed_query`), but the creation time and +the query are read with the same access right, so on Windows this only reaches a +process that opens and then fails the exit-code query; a process that cannot be +opened has no readable token, stays unverifiable, and is cleared with +`rocm services remove --force --yes` (see "A record that stays unverified" +in [manual-testing.md](manual-testing.md); the unit tests +`force_removes_a_record_held_by_an_unqueryable_process_and_says_so`, +`a_record_held_by_an_unqueryable_process_is_refused_naming_the_forced_removal`, +`force_is_refused_for_a_running_process_and_without_a_requested_stop` and +`force_on_a_record_that_is_not_running_is_an_ordinary_removal`, +`a_forced_removal_missing_yes_names_the_command_that_completes_it` and +`prune_names_the_forced_removal_for_a_record_held_by_an_unqueryable_process` (prune +lists the record with the command, and running that command clears it) cover it with the +probe injected, and `service-unqueryable-01` and `-02` drive the stop, the +refusal naming `--force`, and the forced removal through the real binary, with +the `e2e-test-hooks` variable `ROCM_E2E_UNQUERYABLE_PIDS` making the listed PIDs +read as unqueryable; `-02` is Linux-only because it starts a real process). On +Windows the happy paths of the PID hold, the creation-time read and the System32 +`taskkill` are exercised by the two `#[cfg(windows)]` unit tests named above when +`windows-debug` runs the workspace tests (no result for this head is claimed +here). What no test reaches there is the failure side of the Windows API: the +`OpenProcess` failure codes against a real unopenable process, `taskkill` +exiting 128 or failing or stalling, and a descendant actually being taken down; +none of those has been run on a Windows host. What was moved out of the FFI so +that it is unit-tested on Linux and, as plain `cargo test`, on the Windows unit +job: the open-failure code mapping (`classify_open_failure`), the exit-code query +mapping (`classify_exit_query`), the `taskkill` result and arguments +(`classify_taskkill`, `taskkill_args`), the `System32` path building, the +FILETIME join, and the deadline wait (a real child). The ordering of the hold, +the identity read, the tree kill and the termination is tested through injected +functions. + +The same unqueryable-process rule holds wherever a service's liveness decides +something destructive. The deferred key cleanup and the reconciliation to +`stopped` in the liveness refresh (`refresh_managed_service_runtime_liveness`) and +the in-use check behind `rocm services remove` and `prune` read each recorded PID +through `probe_process` and treat anything but a confirmed exit as possibly +running. `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried` +drives the refresh with an unqueryable PID and then a confirmed-gone one, +`an_unverified_stop_keeps_its_key_and_marker_through_a_later_liveness_refresh` runs +an unverified stop and then the refresh over the record it wrote, and +`a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone` covers the +removal guard; each takes the probe as an argument, so they run on every host. +A pending stop of a container-backed engine (`engine_stop_decides_confirmation`, +today `aim`) is never completed that way: its supervisor can be gone while the +container still runs, and only the engine `Stop` decides that stop, so a listing +does not complete it from PID evidence. The key, the marker and the recorded processes stay until a later +`services stop` is confirmed +(`a_pending_stop_of_a_container_engine_is_not_completed_by_a_gone_supervisor`). +The same holds past the cleanup: the refresh does not read such a record as +`stopped` from the supervisor's PID, and `remove` and `prune` count it as in use +(`container_stop_unresolved`), so a listing followed by `remove --yes` or +`prune --yes` deletes neither the record nor the key the container may still be +enforcing (`a_pending_container_stop_survives_refresh_remove_and_prune`, for a +record reading `ready` and one already reading `stopped`; a process engine in the +same state is removed). A record that named no process at all, such as a `starting` +one with a stale marker, has its key and marker cleaned as before but is not read as +`stopped` (`a_stale_stop_marker_on_a_record_naming_no_process_is_cleaned_without_reading_it_stopped`). +When that refresh completes a pending stop (every recorded process confirmed gone) it +also forgets the recorded processes and reads the record as stopped, as a stop that +confirmed the exit itself does, so a later `rocm services stop` visits no PID: a PID +recorded without a token cannot be refuted, and on Windows the tree kill would take +whatever process now holds the number. A process that may still be running stays +named. `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried` +checks both halves (named while unqueryable, forgotten once gone), +`a_completed_pending_stop_reads_stopped_whatever_live_status_it_had` that every live +status, and `failed`, still ends as `stopped` in that one refresh (a `failed` record +cleared but left `failed` would not be covered by `is_covered_by_stop_of`, and the stop +finalizing would report itself unconfirmed), and +`a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop` runs the real +termination routine over the record the listing wrote and asserts it visits nothing. +Three gaps in the scenario coverage are deliberate. `service-stop-07` records no +token, so the path a record written by this version takes on Windows (a token +recorded, matching, with the tree kill then running) has no scenario; it rests on +`what_a_stop_does_for_each_recorded_and_live_token` and the other injected tests, +and on the `#[cfg(windows)]` unit tests that stop a real process with its true token. And +the `self` row that `rocm` now puts in `pid_outcomes` has no scenario: it appears +only in the JSON answer of the hidden `sandbox-run stop_server` and the sandbox +tool, never in the text of `rocm services stop`, and it names the stopping +process's own PID, which a scenario cannot plant in a record before the process +that will read it exists. `a_stop_reports_its_own_pid_as_self` and +`a_self_row_adds_no_line_to_an_unconfirmed_report` cover the answer and the report. +Third, the `check:` line an unverified stop prints is a diagnostic, not a +remediation: it shows whether the process is still running, and what clears the +condition is that process ending (or, for a record held by a process `rocm` +cannot open, `rocm services remove --force --yes`; for a record still reading +`ready` that nobody has tried to stop, `--force` is refused and names +`rocm services stop --yes` first, which reports the process unverified and +records the request, so the clearing is those two commands, on purpose: +`--force` never removes a record nobody tried to stop, and +`a_ready_record_held_by_an_unqueryable_process_is_cleared_by_stop_then_force` +runs the chain), after which a stop or a +service listing confirms and drops the key. The command is a stock one the host +has, pinned by `the_process_check_command_is_the_one_the_host_has`, and +`service-stop-07` uses the same `tasklist /FI "PID eq "` filter (in CSV +form, which does not depend on the display language) to see whether a process is +running. A `tasklist` that does not exit successfully is no answer: the scenario +fails instead of reading the missing row as an exit +(`tasklist_reports_running` in `tests/e2e-cucumber/src/tasklist.rs`, a pure read of +the status and output covered by `a_failed_run_is_no_answer_whatever_it_printed` on +every host). No test follows the printed command on a Windows host and reads its +default table output, because that output is localised. What follows once the +process is gone is covered by +`a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried`. +`service-stop-06` (Linux) is the same stop of a real recorded process through +`rocm services stop`, checking the persisted state: the process exited and the +record names none. Run the Windows scenario on a Windows host with +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-07"` +and a prebuilt `ROCM_CLI_BINARY`. + No scenario drives the attached Ctrl-C stop end to end. Attaching needs a real `rocm serve` launch — an engine and a runtime the mock lane does not have, which is why the chat scenarios there plant a record behind a mock server @@ -1550,12 +2783,52 @@ verdict to the JSON; `rocm services stop` exits 1. The stop records the request in the manifest before it signals anything, and applies its result to the manifest as it is once the stop ends, so a write made meanwhile is kept — and a restart that recorded new processes meanwhile makes the stop -unconfirmed rather than having its PIDs cleared. +unconfirmed rather than having its PIDs cleared. What the stop's verdict still +covers is decided by `ManagedServiceRecord::is_covered_by_stop_of`: the same +processes, or none recorded because a liveness refresh that saw every process +gone completed this same stop first (the record reads `stopped` and names +nothing). A restart, whether it has recorded new processes or is mid-launch with +none yet (status `starting`), is not covered, and neither is the empty `stopped` +record a `rocm services restart` leaves between its own confirmed stop and the +publication of its child: the restart stamps a relaunch generation +(`relaunch_started_unix_ms`) before it restores the endpoint key or spawns, a +stop covers only the generation it read, and publishing the child clears it +(`an_older_stop_finalizing_before_a_restart_publishes_does_not_delete_its_key` +finalizes an older stop in that window and asserts the restored key survives; +`a_stop_does_not_cover_a_record_in_another_relaunch_generation` is the unit +table; `a_restart_publishing_its_child_ends_the_relaunch_generation_and_respects_a_moved_one` +covers the publication guard). The unit test +`a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch` +is the table, and it runs again from a stopping snapshot that names no process (an +empty record is that stop only while it carries the stop's marker, reads +`stopped`, or a liveness refresh consumed that marker and left its stamp in +`settled_stop_unix_ms`; an empty `starting` relaunch has cleared the marker +without leaving the stamp and is not covered, +`stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record`); in `rocm`, +`a_listing_clearing_the_marker_of_an_empty_stop_does_not_make_it_unconfirmed` runs the +real refresh between the termination and the final read of an empty `starting` and an +empty `failed` record (which the refresh leaves at their status) and asserts a confirmed +stop, `a_relaunch_after_a_listing_consumed_the_marker_is_still_not_the_stop` that a +restart following that listing is still not covered (it wipes the stamp), and +`a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it` +persists a real liveness refresh between the termination and the final read and +asserts a confirmed stop with the marker not restored, +`a_restart_is_not_refused_because_a_listing_completed_its_stop` that a restart then +carries on past its stop, and `stop_does_not_confirm_over_a_restart_that_has_recorded_no_process_yet` +and `stop_does_not_confirm_when_a_restart_recorded_new_processes_meanwhile` that a real +relaunch is still detected; `rocmd` has the same three (`a_stop_completed_by_a_concurrent_listing_is_still_confirmed`, +`a_stop_does_not_claim_a_restart_that_has_recorded_no_process_yet`, +`a_stop_does_not_clear_processes_recorded_while_it_ran`) plus +`a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed`, the +listing's write made directly in those that need it because the refresh lives in the +`rocm` binary (in the last one it is made in the engine `Stop` callback, the only +step that runs for a record naming no PID, and the test asserts it ran). `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded process and checks the confirmed report against the process, the record and -the endpoint key file. Run it with +the endpoint key file, and that the standalone `rocmd` got an answer to its +engine `Stop` request (it reaches the engine through the `rocm` beside it). Run it with `ROCM_CLI_BINARY=target/debug/rocm ROCM_CLI_ROCMD_BINARY=target/debug/rocmd cargo test -p e2e-cucumber --test e2e -- -n "service-stop-02"`. `service-stop-03`, `-04` and `-05` (Linux, mock lane) cover `rocm services @@ -1569,7 +2842,27 @@ checks that the line omits it and no key file exists. Because the seam replaces `terminate_verified`, they say nothing about what is signalled. They need a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n -"service-stop"`. +"service-stop-0[1-68]"`. Not `-n "service-stop"`: a name filter bypasses the +suite's own host filter, so it would also run the Windows-only +`service-stop-07` on a Linux host, where it fails for want of PowerShell. + +`features/engine_ipc.feature` (`engine-ipc-01` and `-02`, Linux, mock lane) +covers how the daemon reads an engine child's answer, through the same +`stop_server` tool. Each scenario copies the real `rocmd` into a scratch +directory and puts a stand-in `rocm` shell script beside it, which is the +executable the standalone daemon runs for an engine request; the stand-in +records its argv and the request it was sent, and prints what the scenario +says. `engine-ipc-01` has the engine answer the stop and then exit non-zero: the +report carries the engine's answer (not an error built from the exit status). +`engine-ipc-02` has it print something that is not an answer, write to stderr +and exit non-zero: the report's engine error is the parse failure, naming the +exit status the engine ended with, followed by its stderr. Both check the report together with the server's process, record and +endpoint key, and that the stand-in was run once as `__engine-stdio vllm` with +one `stop` request line. The server is a `vllm` one, for which the engine's +`Stop` is advisory, so in `-02` the stop still completes; an engine whose stop +decides the confirmation (`aim`) is not staged here, because teardown would +then run the real engine's stop. Run them with a hooks build and `-n +"engine-ipc"`. Manual restricted-tool smoke: @@ -1811,6 +3104,34 @@ catalog in [host.rs](../crates/rocm-core/src/diagnose/host.rs) ([fix/catalog.rs](../crates/rocm-core/src/fix/catalog.rs)) runs the catalog's `verify_linux` line for fix-2, not the `rocm diagnose` one, on an empty `PATH`. +fix-4 has one verify line, not two: the catalog recipe and the `Fix` that +`rocm diagnose` builds both read `RENDER_GROUP_VERIFY` in +[fix/catalog.rs](../crates/rocm-core/src/fix/catalog.rs), so the two cannot +drift apart again. `the_render_group_fix_verify_matches_the_catalog_copy` in +[linux.rs](../crates/rocm-core/src/diagnose/linux.rs) keeps either side from +going back to its own copy, and `diagnose-65` asserts the line that +`rocm fix fix-4-render-group --dry-run` prints. The line passes when the user +is in the group that owns `/dev/kfd` (read with `stat -c %G`), or in `render` +or `video`, and `/dev/kfd` exists; the diagnosis tells the user to join the +owning group, which need not be `render`. It checks membership, not the +device's mode, and not `rocminfo`, whose status `head` hides. +`the_render_group_verify_accepts_the_group_that_owns_kfd` in the same file runs +it in `/bin/sh` with stand-in tools on Unix hosts: a member of a third owning +group passes, and a user in none of the groups fails, also when `stat` fails or +names no group. `diagnose-66` asserts that fix-8's title names the +`X.Y` release, which is what `check_8_wheel_rocm_mismatch` compares +(`the_wheel_mismatch_compares_the_x_y_release_and_says_so` in +[host.rs](../crates/rocm-core/src/diagnose/host.rs)). Both run on every lane, +because the plan prints before the platform gate. `diagnose-67` asserts the +other copy, the fix-8 cause `rocm diagnose` prints: its title and plan name the +`X.Y` release and neither says "major". The scenario carries no platform tag: +its symptom scores 75 from keywords alone (`libamdhip64.so` 50, the loader +failure 25), above the match threshold of 50, and `check_8_wheel_rocm_mismatch` +is registered for Linux, Windows and WSL and only adds to that score. +`a_library_load_error_alone_raises_the_wheel_mismatch_on_every_family` in +[host.rs](../crates/rocm-core/src/diagnose/host.rs) holds that for all three +families. + Provider audit focused test: ```bash @@ -1856,7 +3177,8 @@ compaction keeping every pending proposal, the newest reviewed ones, and legacy proposal ids. The rotation tests in `jsonl_store::tests` drive `append_jsonl_record`/`load_recent_jsonl_records` directly with a custom -rotation policy; two further tests in `automation::tests` go through the public +rotation policy and a record type of their own, so the engine's tests do not +depend on `automation.rs`; two further tests in `automation::tests` go through the public `append_automation_event`/`append_audit_event` and `load_recent_automation_events`/`load_recent_audit_events` wrappers instead, at the real size cap, so the wiring between them and the rotation policy @@ -1904,10 +3226,73 @@ exit) and `serve-40` in (GPU lane: serving with no watcher enabled starts no helper; on a Linux host that runs it with lemonade it is an expected failure in [expectations.toml](../tests/e2e-cucumber/expectations.toml) while lemonade -serving on the Strix Halo Ubuntu lanes is broken (#897), so the helper check is +serving on the Strix Halo Ubuntu lanes is broken (ROCMAI-898), so the helper check is not reached on those lanes. The Strix Halo Windows lane runs it and passes, and the row also matches other Linux lemonade hosts, where a pass is tolerated). +Anthropic-compatible `/v1/messages` endpoint (ROCMAI-597). The translation and +its HTTP front are tested in +[anthropic_proxy.rs](../apps/rocmd/src/anthropic_proxy.rs) against a stand-in +OpenAI engine, including an engine that has its own `/v1/messages` route, which is +still fronted by the proxy so a Claude model name works. The record's adoption of the published endpoint are tested in +[anthropic_endpoint.rs](../crates/rocm-core/src/anthropic_endpoint.rs) and +[service_record.rs](../crates/rocm-core/src/service_record.rs). + +```bash +cargo test -p rocmd --lib anthropic_proxy +cargo test -p rocm-core --lib anthropic_endpoint +cargo test -p rocm-core --lib engine_refresh_adopts_the_published_anthropic_endpoint +cargo test -p rocm --bin rocm anthropic_endpoint +``` + +The user-visible side is `serve-50` (listed beside the OpenAI endpoint), `serve-51` +(a real Messages request is answered by the served model) and `serve-52` +(`services list` reports both endpoints healthy), `serve-53` (a request that offers a +tool and requires it gets a `tool_use` block back) and `serve-54` +(`@id:serve-anthropic-messages-streaming`, which sends `stream: true` and +checks the reply is an event stream that reassembles into text, with usage in +`message_delta`) in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature). +They run on the GPU lanes (`@requires-gpu`), with whichever engine the host +serves with. The proxy's unit tests cover the stream event sequence, a stream +that ends early, an engine refusal before the first byte, and the refusal of a +streaming request that offers `tools` (no scenario covers that refusal). The +tool-use translation (`tools`, `tool_choice`, `tool_use` and `tool_result` +blocks, the `tool_use` stop reason) is unit-tested against a fake engine in the +same module; `serve-53` is the real-engine check. + +A recipe may publish a different build of the same model per engine, through the +`model_id_override` on its `engine_recipes` entry: Lemonade serves GGUF with an +explicit quantization variant, vLLM serves the safetensors repo. The resolution +is pinned on every host by `serve_small_assistant_resolves_the_gguf_build_on_lemonade` +and, catalog-wide, by `every_lemonade_catalog_recipe_resolves_to_a_servable_ref` +in [serve_cmd.rs](../apps/rocm/src/serve_cmd.rs) — the second fails for any +recipe that *lists* Lemonade whose resolved ref is a Hugging Face `owner/repo` +with no `:variant`, which the engine refuses only after it has already been +launched. A recipe that lists no Lemonade entry is still handed its canonical +id when Lemonade is reached some other way (an explicit `--engine`, a +configured default, the platform default), and that is deliberate: such a +recipe has no GGUF build to substitute. + +```bash +cargo test -p rocm --bin rocm every_lemonade_catalog_recipe_resolves_to_a_servable_ref +``` + +The user-visible side is `serve-55` (`@id:serve-small-assistant-short-name`): +`rocm serve qwen-small` with no `--engine`, which must reach `ready` and answer. +It carries `@requires-gpu` and deliberately no `@requires-engine`, because the +engine is the CLI's to pick here — so the lane that exercises the Lemonade half +is whichever GPU lane defaults to Lemonade (the Strix Halo lanes, and W7900 on +the queue nightly only) and the vLLM half falls to the Instinct lane. Of those, +MI300X and the Strix Halo Windows and WSL2 lanes run on pull requests +([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when the `serve` +path filter matches; W7900 has no lane there any more) but none of their check names is a required status check, +so the two unit tests above are the blocking per-PR coverage. The scenario +carries no [expectations.toml](../tests/e2e-cucumber/expectations.toml) row, +unlike its managed-Lemonade siblings on the Linux Strix lane: whether it shares +their known serve failure there has not been observed, and a row claiming a +known bug before one is seen would say something untrue. + Natural-language plan handoff tests: ```bash @@ -1972,6 +3357,34 @@ cargo test -p rocm --bin rocm render_update_text_reports_all_update_surfaces surfaces honestly. `rocm update --apply` remains runtime-only until production metadata feeds exist for the other surfaces. +A check changes nothing, and that includes Python. Reading the index for a wheel +runtime needs the interpreter's wheel tag. A runtime that records a usable Python +supplies it. For one that does not, on a host with no usable Python, the check +used to download a managed `uv`, have it install Python and save that Python's +path in the config, only to ask it for the tag. The text report, `--json`, the +startup check that runs ahead of other commands, and the plan behind +`rocm update --apply --dry-run` now take the tag from the interpreter the layout +requires instead and prepare nothing; the text report adds a `python_note:` line +saying an update would use `uv` to install the managed Python, which the check did +not do. `--json` carries no such line, since its shape is the dashboard's +contract. `rocm update --apply` still prepares the Python, because the install +that follows needs it. Scenarios `update-09` (the report) and `update-10` +(`--dry-run`) check the note against the state it left, as `runtime-23` does for +the install preview: the recording `uv` was never run, and there is no +managed-tools folder and no saved config. They run on Linux; the entry points the +scenarios do not drive are covered by +`the_update_checks_prepare_no_python_and_the_apply_plan_does` in +[update.rs](../apps/rocm/src/therock/update.rs), which drives the report, the +JSON, the startup check, the check plan and `apply_runtime_update` with +`--dry-run`, with the plan an apply works from, and a real apply, as the controls +that show the stand-in `uv` is reachable. Which of the two plans a `--dry-run` and +an apply take is also pinned on every platform, Windows included, by +`a_dry_run_takes_the_check_and_an_apply_takes_the_plan` in +[update_cmd.rs](../apps/rocm/src/update_cmd.rs). Unlike the install preview, which fetches no +`uv`, the check on a ROCm 10 or newer runtime still resolves the package versions +with `uv pip compile` and can fetch the managed `uv`, though never a Python; the +scenarios above use the canonical layout, which needs no `uv`. + Runtime version-ordering tests: ```bash @@ -1979,6 +3392,13 @@ cargo test -p rocm --bin rocm version_proptests cargo xtask e2e -- -n 'update-0[78]' ``` +Update checks that must not prepare Python: + +```bash +cargo test -p rocm --bin rocm the_update_checks_prepare_no_python +cargo xtask e2e -- -n 'update-(09|10)' +``` + Where `rocm install sdk` and `rocm update` compare runtime versions, they compare them in one PEP 440 order over published version strings (`7.9` < `7.10.0`, `7.0.0b1` < `7.0.0rc1` < `7.0.0` < `7.0.0.post1`, `7.2.4.70204` above @@ -1998,6 +3418,36 @@ because its runtime key and package specs carry the index's spelling. `generator_reach_report` prints how often each hard case is generated (run it with `-- --nocapture`), and fails if the generator stops reaching them. +Artifact integrity and engine protocol property tests: + +```bash +cargo test -p rocm-core --test digest_gate_properties --test integrity_properties --test model_selector_properties +cargo test -p rocm-engine-protocol --test protocol_properties +cargo test -p rocm-engine-lemonade --lib archive_digest +cargo test -p xtask verify_pinned_keys +``` + +These check the code that decides whether a download, a signature or the +pinned release key is authentic, and the code that decodes engine output. The +generators aim at near misses — a digest one nibble off or sharing a correct +prefix, a signature with one byte changed or leading zeros, a loopback spelling +padded or re-cased, a model id with a variant suffix — because a uniformly +random input is rejected by any implementation, correct or not. The pinned-key +tests run the real check over a scratch repository root and walk every +position of the committed key. `model_selector_properties` also pins, rather +than asserts away, two shapes where readiness accepts an engine's reported name +for a different model: a bare family word such as `qwen` reads as ready for +other models of that family, in the catalog and beyond, and a catalog alias +that spells another catalog model's basename (`qwen3-32b`, an alias of +`Qwen/Qwen3-32B-FP8` and the basename of `Qwen/Qwen3-32B`) reads as ready for +that other model. The second is checked against a list of the catalog's +current collisions, so a catalog change that adds one fails the test. +The pinned-key tests also pin that a character outside the base64 alphabet +(a space, `!` or `.`) inserted into an installer's copy of the key is refused +as a malformed key block, in either installer, rather than ignored by the +comparison: an installer that embeds such a key cannot load it. +`non_base64_noise_in_an_embedded_key_is_refused_as_malformed` holds that. + Model recipe engine metadata checks: ```bash @@ -2025,6 +3475,16 @@ bash scripts/setup-wsl-portable-build-deps.sh --self-test bash scripts/reclaim-gpu.sh --self-test ``` +The nightly stream canary has its own self-test, which checks each rule +against a fixture plan without the network: + +```bash +python scripts/nightly_canary.py --self-test +``` + +What the scheduled canary checks, who is told and how to rerun it are in +[ci-hardware-testing.md](ci-hardware-testing.md#nightly-stream-canary). + The ROCDXG digest check pins the shell fragment the WSL driver plan generates as a whole string — so a change to its quoting, spacing or field order fails the test rather than slipping past a substring assertion — and then runs that @@ -2120,6 +3580,62 @@ The harness applies the matching OS requirement itself; do not select these with cucumber's `--tags`/`-n` flags, because those bypass the suite's custom expectation and applicability resolver. +Fish profile setup in `install.sh` is covered on its own by +[`tests/install-sh/run.sh`](../tests/install-sh/run.sh). That script signs a +local bundle, points the installer at a temporary home, and checks: + +- a bash, zsh, or ksh login shell still updates its own profile and, with + `fish` on `PATH`, also writes the fish `PATH` block and records both + profiles; +- the fish completion script, starting with the marker line that names the + install folder, and its `completions` record line, for a bash login shell + with `fish` on `PATH`, a bash login shell with only `~/.config/fish`, and a + fish login shell; +- an absolute `$XDG_CONFIG_HOME` moves fish's config and completions there, + and a relative one leaves them in `~/.config/fish` (it names a different + folder from each directory; both with a bash login shell only); +- a reinstall adds no second block or record line and reports fish's config + as already configured; +- a fish config, or a bash profile, that already names the install folder with + the user's own line is left alone and not recorded; +- a `rocm.fish` that is already there and was not written by the installer (a + file, a link, a link whose target carries the marker, a dangling link) is + left exactly as it is, not recorded, with a note, while one an earlier + install wrote is replaced and recorded; +- a bash home with no fish install grows no fish config, and + `ROCM_CLI_UPDATE_SHELL_PATH=0` writes no profile and no completions; +- a `rocm` that cannot print completions, or prints nothing, leaves the install + and the `PATH` block in place with a note. The stub fails only once, as a + failure that clears by itself would, and the printed lines, run as printed + from a bare environment in a home whose name holds a space, then create the + file; `rocm` is never handed the installer's stdin; +- a profile that cannot be created is reported rather than claimed as updated, + and not recorded; as a non-root user only, so is one that cannot be written, + under both `sh` and `bash`. + +It needs no network or GPU. Set `INSTALL_SH` to run it against another copy of +the installer. The Linux `build-and-test` job runs it when the change touches +the heavy path (`needs.changes.outputs.heavy == 'true'`): + +```bash +sh tests/install-sh/run.sh +``` + +The same path with the packaged binary, including `rocm uninstall` putting the +bash and fish configs back and removing the fish completions, is +`@id:lifecycle-linux-fish-configured-from-bash` (lifecycle-32) in the lifecycle +set above. `@id:lifecycle-linux-fish-completions-not-ours-kept` (lifecycle-33) +checks that uninstall leaves a `rocm.fish` the user wrote over the installer's +and says so, `@id:lifecycle-linux-fish-completions-removal-failure-warns` +(lifecycle-34) that a `rocm.fish` it cannot remove is a warning while the rest +of the uninstall still happens, and +`@id:lifecycle-linux-fish-existing-completions-left-alone` (lifecycle-35) that +the installer leaves a `rocm.fish` it did not write, says so, and uninstall +leaves it too. lifecycle-34 makes the `completions` folder read-only, so it +needs a non-root runner. `@id:lifecycle-linux-uninstall-keep-binaries-keeps-path` +(lifecycle-25) also seeds fish and checks the completions stay with +`--keep-binaries`. + The `windows-release` job of the required `windows-build-and-test` lane additionally folds in the hermetic `comfyui-01`/`comfyui-02`/`comfyui-03` scenarios and the native-Windows vLLM refusal scenarios @@ -2191,6 +3707,14 @@ cargo test -p rocmd supervise_service_first_write_keeps_a_standing_stop_marker cargo test -p rocmd restart_failure_write_keeps_a_stop_marker_set_meanwhile cargo test -p rocmd restart_records_its_supervisor_without_erasing_a_stop_marker cargo test -p rocmd restart_leaves_a_record_naming_another_supervisor_or_none +cargo test -p rocmd daemon_host +cargo test -p rocmd restart_runs_supervise_on_the_rocmd_it_is_given +cargo test -p rocmd restart_without_a_rocmd_to_run_changes_nothing +cargo test -p rocmd supervise_runs_the_engine_on_the_rocm_beside_rocmd +cargo test -p rocmd supervise_without_a_rocm_beside_rocmd_changes_nothing +cargo test -p rocmd engine_request_runs_the_stdio_command_on_the_rocm_beside_rocmd +cargo test -p rocmd engine_request_without_a_rocm_beside_rocmd_names_it +cargo test -p rocm-core sibling_binary_path_from cargo test -p rocm --bin rocm stop_records_its_marker_on_disk_before_any_process_is_signalled cargo test -p rocm --bin rocm stop_does_not_confirm_when_a_restart_recorded_new_processes_meanwhile cargo test -p rocm --bin rocm stop_final_write_keeps_what_others_wrote_while_it_ran @@ -2215,6 +3739,24 @@ it reports the unconfirmed stop and starts nothing. Its own later writes are applied to the record as it is on disk, and only while it still reads as the restart left it, so a stop asked for during the restart keeps its marker. +The daemon runs as the `rocmd` binary or embedded in `rocm` (`rocm daemon`), +and each binary has commands the other lacks: `supervise` is `rocmd`'s, and the +hidden engine commands (`__engine-serve-http`, `__engine-stdio`) are `rocm`'s. +Every daemon spawn therefore asks `DaemonHost` for the binary instead of +re-executing its own: the recovery restart runs `supervise` on `rocmd`, and the +supervisor starts its engine, polls its readiness and asks it to stop through +`rocm`, each the current executable when the daemon is that binary and the +sibling installed beside it otherwise, named in the error when there is none. +`rocmd`'s unit tests never reach the real host: under test `DaemonHost::detect` +returns the test binary for both commands and looks for no sibling, the two +stand-in hosts name only the binary they stand in for and fail when asked for +the other, each test that needs a different host passes the code under test one +of its own, and a test that needs a binary writes a stand-in script (its shell +and `sleep` found on `PATH` once and asserted by name), so none of them can run +a `rocm` or `rocmd` that happens to sit in the build directory. `run_from_args_marks_the_process_as_embedded_in_rocm` +checks the embedded marking in a child run of the test binary, because the flag +is process-wide. + Automation GPU-metrics event tests: ```bash @@ -2377,7 +3919,7 @@ the per-PR mock lane. The host predicate itself is the shared than restated in either, so the test cannot drift from the production rule. The bypass decision's other arguments are in the same position — the unit tests -in `apps/rocm/src/service_launch.rs` pin `managed_service_reuse_refusal` itself, +in [`launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) pin `managed_service_reuse_refusal` itself, not which values `serve()` hands it — so each one has a scenario instead: serve-30 for `require_api_key`, serve-32 for the requested recipe. The device policy needs neither: the predicate takes it by value rather than as an @@ -2785,8 +4327,8 @@ rather than the real host. The `a_fallback_read_plans*`/ `an_unreadable_file*`/`distro_name_names_an_unreadable_file` tests in `rocm-core` and `rocm` construct an `OsRelease` directly — bypassing the real read entirely, since the production functions that do it -(`openmpi::real_os_release`, `system_packages::real_os_release`) -read the real machine +(`hardware_root::real_os_release`, which every caller goes through) +reads the real machine unconditionally and cannot honour a simulated root — to pin that a fallback-sourced read plans and hints identically to a primary-sourced one with the same text, and that a file which exists but cannot be read is named @@ -2807,6 +4349,19 @@ shell probe instead of merely leaving `$ID` unset, which this test does not exercise — see its own doc comment for why, and for the known limitation this leaves. +The package-manager side reads the real host through one function, +`hardware_root::real_os_release`. In the `rocm` binary's tests, +`system_packages::tests::real_os_release_ignores_a_simulated_host_root` +(`--features e2e-test-hooks`, Linux) plants a root naming another distribution +and expects the real file back, and +`system_packages::tests::the_install_hints_read_the_real_host_and_ignore_a_simulated_root` +holds the three install hints to the same rule. Both live in that process +rather than in `rocm-core`'s, so the environment variable they set cannot +overlap a `rocm-core` test that reads host paths. +`system_packages::tests::no_caller_in_this_binary_reads_the_real_os_release_by_hand` +fails when a source file of the `rocm` binary reads the file with its own +resolver instead of calling it. + `rocm examine` reads the file through the simulated host root (see Simulated Host Root in [release-trust.md](release-trust.md)), so scenarios `examine-20` to `examine-23` and `examine-27` to `examine-28` in @@ -2842,6 +4397,18 @@ cache nor the variable involved, which is the layout of a TheRock tarball stays `wsl_rocdxg_missing`, so the sibling runtime, and not the planted library, is what the verdict turns on. +`examine-35`, `examine-36`, `examine-37` and `examine-38` hold the two answers `rocm examine --json` gives to "is `rocminfo` installed" level on that same planted WSL machine: `summary.wsl.rocminfo` from the host summary and `wsl.rocminfo` from the examination, which share one `PATH` lookup. They plant a stand-in `rocminfo` that prints nothing, and build the `PATH` from the machine's stand-in directory followed by the runner's own entries that hold no `rocminfo`, so a runner with ROCm installed cannot be why the tool reads as found. `examine-35` is the control: the tool in a `PATH` directory is found by both. `examine-36` puts it only in the directory the command runs from, with an empty entry first on `PATH`: neither report finds it, because an empty entry is skipped and not read as the current directory (the harness's `command_dir` sets that directory). `examine-38` is its control: the same machine with `.` as the first entry instead, which both reports find, so it fails if `rocm` does not really run from that directory and `examine-36`'s answer cannot come from a harness that ignored `command_dir`. `examine-37` puts it in a `PATH` directory whose name is not valid Unicode: both find it, because a `PATH` that is not Unicode is still searched. The Windows spellings of the lookup (`PATHEXT`, a quoted entry holding a `;`, the System32 locations) cannot run on the Linux lane and are covered by the unit tests in `probe_exec.rs`, which the Windows job runs. + +```bash +RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-3[5-8] ' +``` + +`examine-48` plants a native Linux machine (a simulated host root that is not WSL, because the PCI probe is skipped on WSL) whose `lspci` is a stand-in on `PATH` that prints one AMD GPU, exits 0 and leaves a `sleep` holding its output pipes for 30 seconds. The host probes now stop reading a tool's output shortly after the tool exits, so `rocm examine --json` must finish in under 15 seconds, well inside the background process's life, and must still list the GPU the stand-in printed at `0000:fe:00.0`, so what the tool printed before it exited is still collected (the length of the grace is not pinned here). It runs on the Linux lanes and needs no GPU; the unit tests behind it (`subprocess_runners_pair` in `host_gpu/capture.rs`) pin the same bound for each of the three runners. + +```bash +RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-48 ' +``` + `serve-42` plants the same WSL machine with `LD_LIBRARY_PATH` empty and runs `rocm serve`: it is refused, its advice names the WSL `driver_status` and not `/dev/kfd`, and `rocm examine` on that machine reports `wsl_rocdxg_missing`, so the command the refusal names exists and shows the condition. The launch that follows when `LD_LIBRARY_PATH` does reach the library has no scenario, because it would start preparing a real engine; the serve gate's decision there is covered by unit tests in `gpu_visibility.rs`. ```bash @@ -3082,14 +4649,121 @@ Windows: cargo xtask e2e -- -n '(diagnose-48)' ``` +## Symptom From A File Or Standard Input + +`rocm diagnose --symptom-file ` reads the symptom from a file, or from +standard input when PATH is `-`, so a pasted error never passes through a +shell's argument parser. How the bytes become the `--symptom` string (one +trailing line break removed, a UTF-8 byte-order mark dropped, UTF-16 with a +byte-order mark decoded, anything else UTF-8, and more than 16 MiB a usage +error) is in [symptom_input.rs](../apps/rocm/src/symptom_input.rs), and its +unit tests there pin each rule, including that a paste full of quotes, +`$(...)`, a backtick and a leading `-` comes back byte for byte. They are the +only tests that would fail if the trailing line break were kept: no scenario +below depends on it, since a line break changes no cause. The clap rules (the +two flags exclude each other, `-` is accepted as a path, a leading `-` is +refused as `--symptom`'s value) are pinned by +`diagnose_takes_the_symptom_from_a_file_or_stdin_but_not_with_symptom` in +[cli.rs](../apps/rocm/src/cli.rs). + +```bash +cargo nextest run -p rocm -E 'test(symptom)' +``` + +Three scenarios in +[diagnose.feature](../tests/e2e-cucumber/features/diagnose.feature), with their +steps in +[symptom_file_steps.rs](../tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs), +hold the behaviour end to end, GPU or not: + +- `diagnose-59` pipes a hostile paste through the platform's own shells (bash + on Linux and WSL; cmd.exe and Windows PowerShell on Windows) and requires + each diagnosis to equal the one for the same text passed as a single + argument with no shell in between: every cause, score and piece of + evidence, and every run exiting 0. The paste carries quotes, `$(...)` and a + leading `-`, but no shell parses it (each reads it from a file into the + pipe), so nothing in it can run by construction; the scenario asserts the + diagnosis, not the absence of something no shell could do. The paste is + ASCII: Windows PowerShell 5.1 pipes text as ASCII, which is why the docs + send PowerShell users through a file instead. The cmd.exe and Windows + PowerShell runs need a Windows host, so in CI they run only on the + self-hosted Strix Halo Windows lanes: `e2e-gpu-strix-windows` in + [e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml) (on pull + requests that touch the serve paths, on pushes to `main` and `release/**`, + and in the merge queue) and the scheduled + `e2e-gpu-nightly-strix-windows` in + [nightly.yml](../.github/workflows/nightly.yml). Neither is a required + check. The required `E2E tests` job in [ci.yml](../.github/workflows/ci.yml) + runs only the bash half, on Linux, and ci.yml's Windows lane does not run + this scenario. +- `diagnose-60` (Linux and WSL lanes only, like every pseudo-terminal + scenario here; the Windows key is checked by hand in + [manual-testing.md](manual-testing.md) section 13) runs + `rocm diagnose --symptom-file -` on a pseudo-terminal tall enough to keep + the whole exchange on screen (it has no scrollback), requires the hint + telling the user how to end the paste, then types the paste and Ctrl-D, the + Linux end-of-input key the hint names, and requires the screen, which also + holds the hint and the echoed paste, to contain the whole report the + argument form prints. It first requires that report to differ from the one + with no symptom, so a paste that was read and then dropped fails rather than + passing on a host where the host's own causes fill the report. +- `diagnose-61` points the flag at a missing file and at one that is not + UTF-8, and requires a usage error (exit code 2) that names the file and the + reason, with nothing on stdout. + +None of them asserts which catalog entry matched, for the reason given at the +top of that feature file: they compare two routes on the same host. + +```bash +cargo xtask e2e -- -n 'diagnose-(59|60|61) ' +``` + +### Which causes the pasted error contributed to + +Every entry in `rocm diagnose --json`'s `matched` carries `symptom_matched`: +`true` when the symptom text (from `--symptom` or `--symptom-file`) +contributed to its score or evidence, `false` when it came from the examined +host's state alone, and `false` on every entry when no symptom was given. Any +keyword hit counts, however weak, so the field records a contribution, not that +the entry explains the error. +[diagnose.rs](../crates/rocm-core/src/diagnose.rs) decides it by running the +catalog a second time with no symptom and comparing each entry with its +counterpart (`contributed`: absent, a different score, or different evidence), +so no checker sets it by hand. The text report prints a `source: the examined +host's state alone` line under each shown entry the text did not contribute to, +once the text contributed to any entry in the report, including one cut off by +`--top`. The rules are pinned by unit tests in diagnose.rs, +[report.rs](../crates/rocm-core/src/diagnose/report.rs) and, for the same line +under `rocm remote doctor`, [doctor.rs](../apps/rocm/src/remote/doctor.rs): + +```bash +cargo nextest run -p rocm-core -E 'test(/credited|contribution_is_any|symptom|carries_the_note|says_so_in_the_text|past_top|before_the_field/)' +cargo nextest run -p rocm -E 'test(the_symptom_note_is_true_of_a_remote_host)' +``` + +`diagnose-62` (JSON), `diagnose-63` (text) and `diagnose-64` (no symptom, both +forms) in [diagnose.feature](../tests/e2e-cucumber/features/diagnose.feature) +cover it on every lane. They set an unneeded `HSA_OVERRIDE_GFX_VERSION` on the +CLI. The first two also paste an ISA error, so the override entry must read +`false` and the ISA entry `true` whatever the runner's own state; the third +pastes nothing, so every entry must read `false` and no line is printed. To run +only those three without bypassing the harness's gates: + +```bash +E2E_ONLY_LIFECYCLE=1 E2E_ALSO_RUN_IDS=diagnose-json-says-which-causes-the-paste-contributed-to,diagnose-text-marks-causes-the-paste-did-not-contribute-to,diagnose-without-a-paste-credits-no-cause-to-one cargo xtask e2e +``` + ## Model Fit Preflight -`rocm diagnose --model ` answers whether a curated model will run on this -host before anything downloads. It reports one of four verdicts: `ready`, +`rocm diagnose --model ` answers whether a model will run on this host +before anything downloads: a curated model from its recipe (this section), a +GGUF outside the catalog from its file header (the next one). For a curated +model it reports one of four verdicts: `ready`, `degraded` (runs, but under the recipe's recommended system RAM), `blocked` (will not run here, with alternatives that would), or `undetermined` (the CLI could not judge it — an unreachable catalog, an -unmeasured GPU, or a ref outside the curated set). +unmeasured GPU, or a ref outside the curated set; for a GGUF reference the +estimate below is added to that answer). ```bash rocm diagnose --model qwen-smoke --json # smallest curated recipe @@ -3109,6 +4783,101 @@ including the two that need a synthetic signed catalog to trigger deterministically (`ModelNotCurated`, `Degraded`) since no built-in recipe can produce them on an arbitrary real host. +### GGUF estimate (models outside the catalog) + +A GGUF reference outside the catalog (a local `.gguf` path, or +`owner/repo:QUANT`) is estimated from its header instead: +`rocm diagnose --model --context-size N --kv-cache-type f16|q8_0|q4_0`. +The verdict stays `undetermined` (`model_not_curated`); the answer is +`model.estimate`, with outcome `fits`, `does_not_fit` or `cannot_estimate`, and +the text report opens `ESTIMATED:` for the first two. The parts and their tests: + +- [gguf.rs](../crates/rocm-core/src/gguf.rs): the header parser, tested against + synthetic headers built byte by byte (versions, truncation at every offset, + per-layer head arrays, signed 32-bit integers as gguf-py writes them, + every fixed-width value type read at its own width as a scalar and as an + array element, skipped arrays, alignment, the span of the input embedding) and one test per + bound a hostile header would walk past (key length, entry counts, tensor + dimensions, array nesting, an overflowing array length, an unknown value type, + the layer count, how many shape keys are kept, how large a header may be -- + including the checks inside a string array and an array of arrays, which are + walked over an endless stream and must stop near the bound -- a single length + larger than any header, and the length at which an array or string is no + longer kept). +- [gguf_fit.rs](../crates/rocm-core/src/gguf_fit.rs): the arithmetic (exact KV + figures, the boundary between fits and does not, the largest context -- a + trained context that is not a whole step included, on both sides -- the + suggestions, each layout it refuses to estimate, the state-space, hybrid and + mixture-of-experts layouts that get their figures with no verdict, and the + measured llama.cpp loads the allowances were calibrated on, which the synthetic Llama- and + Qwen-shaped headers must reproduce: buffer sizes, and a total never below the + measured use and within a bound of it per model. The module documentation + describes the loads and records the actual margins). +- [gguf_source.rs](../crates/rocm-core/src/gguf_source.rs): reference parsing + (an existing file wins over `owner/repo:QUANT`; `owner/repo:dir/file.gguf` + names one of two same-named files), file selection (shards of one model versus + two models; a full path beats a bare name that matches a nested file; a + multimodal projector is never picked, and a name that matches only projectors + is refused as naming one), names + in a listing that could leave the repository or hide in a terminal (dropped, + and never requested), and the range reader, tested against a loopback server that + records the ranges it is asked for, so a read of the weights would show, and + that also answers with a wrong or missing `Content-Range`, a dropped body, a + missing size in the listing and in the `HEAD`, a gated repository and a slow + reply. [diagnose_model_sends_no_token.rs](../apps/rocm/tests/diagnose_model_sends_no_token.rs) + runs `rocm diagnose --model owner/repo:QUANT` against such a server with every + Hugging Face token variable set, and checks that the listing, `HEAD` and + range requests carry no `Authorization` or `Cookie` header. +- [model_readiness.rs](../crates/rocm-core/src/model_readiness.rs): the + host-memory states (dedicated, none, unknown, APU), what is attached to a + curated or non-GGUF answer (nothing beyond a note on a flag that did not + apply), the `rocm diagnose` command a GGUF plan offers (the same flags, the + reference quoted for a shell), the plan a sized estimate keeps when it + could not compare (`amd-smi metric --json` where the memory was unreadable, + `rocm diagnose` where no GPU is visible, as for a curated model, and none on + an APU; unit-tested only, since no lane has a GPU whose memory cannot be + read), and the text and JSON shapes. +- [diagnose_model.rs](../apps/rocm/src/diagnose_model.rs) and + `classify_accelerator_memory` in [model_fit.rs](../apps/rocm/src/model_fit.rs): a + device mask that hides every GPU ("no GPU is visible", not a card of 0 GiB, on + a discrete card and on an APU alike), + and the note for a mask on a host whose GPUs differ in memory. Both are unit + tests only: the hosted e2e runners have no GPU for a mask to hide, and no CI + lane has GPUs of different sizes. + +Scenario `diagnose-51` points `HF_ENDPOINT` at a loopback stand-in that counts +requests, and shows that `rocm diagnose --report --model owner/repo:QUANT` makes +none and says on stderr that `--model` was ignored, while the same question +without `--report` does reach the endpoint and prints no such note. Scenario +`diagnose-52` has the stand-in answer the repository listing with a model and +its multimodal projector, and shows that naming the projector exactly is +answered as a projector, with the model file named instead, after the listing +request alone. + +Scenarios `diagnose-49` and `diagnose-50` write a small GGUF into the scenario's +isolated directory and assert the shape and the sizes (weights and KV cache are +exact), and that a host that measured its GPU gets a comparison while one that +could not is told why. Which branch runs depends on the lane: the GPU-less +hosted runners take the "could not compare" one (the text report must then read +UNDETERMINED), and the MI300X self-hosted lane (`E2E tests (MI300X, ...)` in +`e2e-selfhosted.yml`) takes the compared one, where the text report must read +ESTIMATED. The "does not fit" half depends on the host's VRAM, so it is covered +by the unit tests, not by a scenario. Nothing in either runs against a real GPU +load; see [manual-testing.md](manual-testing.md) for the check that does. + +Scenarios `diagnose-53` to `diagnose-56` cover the refusals, all host-independent: +the context options without `--model` (a usage error naming it), a curated model +asked about a context size (told the options do not apply, no estimate), a +`.gguf` path that does not exist (named as not opened, no upstream route, and a +plan to ask again about that path, the path quoted where the platform's shell +needs it, so the step accepts it bare or quoted), and a bare `owner/repo` with a context size +(told to add `:QUANT`, with a counting stand-in hub showing nothing was fetched). Scenario `diagnose-57` asks the stand-in hub for a quantization that matches +nothing, and the answer lists the model and the projector, marked as one. + +```bash +cargo xtask e2e -- -n '(diagnose-49|diagnose-5[0-7])' +``` + ## Doctor Report Preflight Preview the content a problem report would carry, without sending anything: @@ -3171,6 +4940,24 @@ On a host with an approved architecture (see `APPROVED_ARCHITECTURES` in `os_major`, `distro`, `rocm`, `engine`, `engine_version`, `cli_version`, and `fix_offered`. This path has not been exercised against real hardware in CI; verifying it needs a lane whose GPU architecture is on the allowlist. +`cli_version` is the leading version `rocm version` prints (`X.Y.Z` for a +`vX.Y.Z` tag build, `0.0.0-dev` for any other resolved ref), never the ref and hash +after it. A tag counts as a release only when its suffix uses letters, +digits, `.`, `-` and `+` and its version is at most 64 characters, the same +rule the report applies before publishing, so a release version is never +published as `unknown`; `release_tag_grammar_is_the_same_everywhere` in +[build.rs](../apps/rocm/build.rs) holds that rule equal in build.rs, +[release.yml](../.github/workflows/release.yml) and +[smoke.rs](../xtask/src/smoke.rs) (`cargo xtask smoke`). Scenario +`diagnose-58` in +[diagnose.feature](../tests/e2e-cucumber/features/diagnose.feature) checks that +the field equals that leading version and that the version is the one the ref +calls for, on a simulated host whose planted GPU is on the allowlist, so it +prepares a report on every Linux lane, GPU or not. Those lanes build from a +branch, so they check the `0.0.0-dev` half; the tag half is checked on the +shipped binaries by the "Check the release binary's version" steps in +[release.yml](../.github/workflows/release.yml). On real hardware, compare the +field with the first word after `rocm-cli` in `rocm version`. Four of those fields — `distro`, `rocm`, `engine`, and `engine_version` — can answer with a word rather than a value, and the words are not interchangeable. @@ -3354,6 +5141,41 @@ report. A subject or a file name can name a person, so keep that output to your own terminal. The export itself still holds everything. Never commit it, and delete it once the rollup is taken. +## Fix Catalog Exit Codes + +`rocm fix --json` publishes each exit code twice over: `exit_codes` maps a name +to its number and `exit_code_descriptions` maps the same name to what it means. +Both are built from one table, `exit::MEANINGS` in +[options.rs](../crates/rocm-core/src/fix/options.rs), so a code cannot be named +without being described. The descriptions are an added field, so +`contract_version` did not move. + +Exit 2 has two causes, and the `unknown_id` description says so: a fix-id the +catalog does not have, and a command line the argument parser rejects before +any fix-id is looked up. They are told apart by the message (a stderr line +starting `Unknown fix-id:` against one starting `error:`), never by the code. +The claim is about a line, not the start of stderr, because `run()` can print a +one-time `rocm:` notice before either refusal. Each half of that claim is +checked where it lives: + +- `the_unknown_id_description_names_the_usage_error_it_shares_a_code_with`, + `every_published_exit_code_is_described_once` and + `every_exit_constant_is_published_and_nothing_else_is` in + [manifest.rs](../crates/rocm-core/src/fix/manifest.rs) hold the text and the + table, and the table to the `exit::` constants. +- `a_usage_error_exits_with_the_code_published_for_an_unknown_fix_id` in + [main.rs](../apps/rocm/src/main.rs) parses malformed `rocm fix` command lines + with the real `clap` definition and checks they exit with the published + `unknown_id` code and have a line starting `error:`. +- `diagnose-34` (`@id:diagnose-fix-catalog-is-machine-readable`) runs the binary: + it reads the catalog, then runs `rocm fix` with an unknown fix-id and with an + unknown option, and checks each exits with the code the catalog publishes and + prints the message the description names. It has no tags, so every lane runs it. + +The other five descriptions are claims about the runners. The comment above each +entry in `exit::MEANINGS` names the unit tests that prove it, and the branches +no test drives (a command that exits non-zero, a missing tool or install). + ## Doctor Skill Eval Replays The rocm-doctor skill's behavioural evals @@ -3368,8 +5190,9 @@ that grades how the agent reads a diagnosis, or what it does on an older CLI, ne has a directory under [`evals/fixtures/`](../skills/rocm-doctor/evals/fixtures/), the hook writes a stub `rocm` into a temporary directory, puts it first on `PATH`, and removes it afterwards. The stub replays recorded output for `rocm --version`, -`rocm fix --json` and `rocm diagnose`, and refuses every other command with exit -code 2. It only prints text read from the recording, so whatever an agent asks it, +`rocm fix --json` and `rocm diagnose`, plus `rocm examine --json` for a case that +names a `host` (below), and refuses every other command with exit code 2. It only +prints text read from the recording, so whatever an agent asks it, it cannot change a machine or send anything; that comes from answering a fixed set of commands, not from the refusals. It also refuses `diagnose` options the recording does not cover (`--report`, `--send`, `--distro`, `--model`), for fidelity @@ -3403,10 +5226,23 @@ changed is output the skill is now graded against. What an older CLI printed can generated. It was recorded once from a real older build and lives in [`evals/fixtures/recorded/`](../skills/rocm-doctor/evals/fixtures/recorded/). +A case may also name a `host`: `no-amd-gpu`, `wsl-host-gpu-hidden`, `amd-found` or +`examine-fails`. The stub then answers `rocm examine --json`, and a case without a +`host` still refuses it. The reply is the recorded examination with the fields the host +implies set, whatever the recording held for them, and its `status` is recomputed by +`Examination::compute_status`, the rule the probe uses, so a recording cannot say a +status the other fields contradict. The real command prints those fields plus a +`summary` object that only the `rocm` binary can build, so the replay leaves `summary` +out; the skill reads `status` and `has_amd_gpu` and nothing in it. Some fields are +typed, not probed: the one GPU of `amd-found`, and the WSL facts of +`wsl-host-gpu-hidden` (a distro that sees the Windows host's GPU through `/dev/dxg`, +with no ROCm-side library). `examine-fails` is not recorded from a real failure: it +exits 1 with a fixed message on stderr, which is what the case pins. + To add a case, add an entry to [`evals/fixtures/cases.json`](../skills/rocm-doctor/evals/fixtures/cases.json) (the -case id, `current` or `older`, and the error text the agent would pass as the -symptom), run the command above, and add the case to +case id, `current` or `older`, the error text the agent would pass as the symptom, +and optionally a `host`), run the command above, and add the case to [`evals/evals.json`](../skills/rocm-doctor/evals/evals.json) with the same id. The check also fails when a case id is not an id in `evals.json`, or when the part of the symptom before its first `:` does not appear in that case's prompt, because the hook @@ -3430,7 +5266,37 @@ On Windows the launcher is a `rocm.cmd` (cmd and PowerShell) plus an extensionle include `skills/**`, so `test_hooks.py` runs the shim there. That covers a temp directory and Python path without spaces; a path with spaces or parentheses is not exercised, and neither is the agent's own shell on the amd/skills Windows leg. -Both replay cases also say "on Linux" in the prompt: run on a Windows host, the -agent answered that its own machine is Windows and declined to run the CLI there, so -neither case passed (0 of 2 in a local run), while on a Linux host (WSL2) both -passed. Treat them as Linux-host cases when reading a run. +The two cases about the older and the current CLI name no operating system in their +prompts. They used to say "on Linux", and on a Windows host the agent answered that its +own machine is Windows and declined to run the CLI at all, so the recorded replies never +printed. The vLLM cases still say Linux on purpose: their catalog entries apply on Linux +and WSL2 only, so they are not expected to pass on a Windows host. + +A real `rocm` on the machine can answer before the stub. The agent's shell does not +always keep the stub first on `PATH`, so a machine that has a `rocm` installed (a +developer's, not a grading runner) may show the real CLI's output and miss the pins. A +miss on the `logs_contain` pin with the agent otherwise behaving as expected is that, +not a regression. + +## Dashboard Chat Request Timeout + +A dashboard chat message is cut off after a fixed wait so a hung backend ends the +turn instead of freezing the pane. The wait is 2 minutes by default and +`ROCM_DASH_CHAT_TIMEOUT_SECS` (whole seconds, 1 to 3600) changes it; an unset, +empty, non-numeric or out-of-range value keeps the default. + +The behavior is covered by unit tests in +[mod.rs](../crates/rocm-dash-tui/src/agent/mod.rs) (how the variable is read) and +[clients.rs](../crates/rocm-dash-tui/src/agent/clients.rs) (a request that outlives +its budget times out and names that budget, a slow reply inside it is returned, and +the real request path takes its budget from the variable): + +```bash +cargo test -p rocm-dash-tui --lib request_timeout +cargo test -p rocm-dash-tui --lib agent::clients +``` + +There is no Gherkin scenario: the timeout is a dashboard (TUI) behavior that no +`rocm` command output exposes, and driving it end to end needs a model that +answers slower than the wait. The manual check is in +[manual-testing.md](manual-testing.md#4d-dashboard-chat-request-timeout). diff --git a/docs/vllm.md b/docs/vllm.md index 640ffbbbbe..47eaa53f86 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -59,6 +59,27 @@ would have installed and the one it kept. The device check still runs, so an opt-out that leaves the runtime unable to serve says so instead of failing later during serving. +The opt-out covers the packages the SDK installs together with torch +(torchvision, torchaudio and triton) as well. vLLM pins an exact build of each. +The install check spares them when it settled on a torch divergence, and the +engine's built-in repair spares them whenever the opt-out is set; in both cases +they are reported as expected divergences rather than violations, and vLLM is +not reinstalled over them, because that reinstall would put the engine's own +torch back. The trade-off: under the opt-out a genuinely broken torchvision, +torchaudio or triton (including one that is not installed at all) is neither +reported as a violation nor repaired. + +The opt-out also applies to `rocm install sdk` run over a runtime that already +holds torch. That command normally hands `uv` the SDK's `torch`, `torchvision` +and `torchaudio` along with `rocm`, which replaces the stack the engine or you +installed (and `triton` with it). With the variable set it asks for the ROCm +packages alone and leaves the torch stack where it is: the plan prints a +`torch_kept:` line naming the torch it found, and `rocm install sdk --dry-run` +shows the same reduced `command:`. A runtime with no torch yet, or an +environment recreated for another Python, still gets the full stack, since +there is nothing to keep. The runtime's recorded composition (and so its +runtime key and `rocm update` comparison) is unchanged. + Use it when you are deliberately running a torch that the alignment would replace, such as a locally built wheel, a version under test, or a stack pinned for a reproduction. It is an escape hatch, not a supported configuration: the @@ -78,9 +99,15 @@ so there is no fixed filename to pin in the adapter. The wheel's own tag shows the other thing that changes on this route: ROCm 10.x's vLLM, flash-attn, and amd-aiter are published for **`cp314` only**, unlike every earlier ROCm SDK version's `cp312` wheels. ROCm CLI provisions a matching -interpreter for a 10.x install automatically. A Python already on `PATH`, or -already installed as ROCm CLI's managed Python, is rejected if it is not -`cp314`, with a message naming the required tag. +interpreter for a 10.x install automatically. By default, a Python already on +`PATH`, or already installed as ROCm CLI's managed Python, is rejected if it is +not `cp314`, with a message naming the required tag; with `--python-version` +(below) the tag it names is the one you chose. +That 3.14 is the default for a ROCm 10.x SDK install, not a limit of the SDK: the +SDK and torch wheels are published for 3.10 to 3.14, and `rocm install sdk +--python-version` picks one of them for an install that does not need vLLM. The +vLLM install then refuses that runtime; re-running the install with +`--python-version 3.14` moves it back. This route is selected by `major.minor`, so `10.0.0` and `10.1.0` discover through *different* rows: they pin different vLLM minors and read vLLM from diff --git a/docs/wsl.md b/docs/wsl.md index 5f361df197..197b51747b 100644 --- a/docs/wsl.md +++ b/docs/wsl.md @@ -192,8 +192,8 @@ Eight entries that are not WSL-specific also run on WSL: `fix-1-arch`, `fix-2-unset-override`, `fix-6-path`, `fix-8-wheel-rocm`, `fix-16-vllm-oom`, `fix-17-torch-dlpack`, `fix-18-comgr-conflict`, and `fix-19-shm-too-small`. `fix-16-vllm-oom` and `fix-17-torch-dlpack` are driven by the error text you -pass with `--symptom` and read nothing from the environment or the ROCm -install. `fix-18-comgr-conflict` and +pass with `--symptom` (or `--symptom-file`) and read nothing from the +environment or the ROCm install. `fix-18-comgr-conflict` and `fix-19-shm-too-small` apply when you run `rocm diagnose` inside the distro. A `--distro` run skips both (see [Diagnose a distro from Windows](#diagnose-a-distro-from-windows) later in this diff --git a/doctor-catalog.json b/doctor-catalog.json index 1406fa9f00..64580f82b4 100644 --- a/doctor-catalog.json +++ b/doctor-catalog.json @@ -120,7 +120,7 @@ }, { "id": "fix-8-wheel-rocm", - "title": "Reinstall the framework against the system ROCm/HIP major", + "title": "Reinstall the framework against the system ROCm/HIP X.Y release", "platforms": [ { "os": "linux", @@ -453,5 +453,13 @@ "not_applicable": 3, "ok": 0, "unknown_id": 2 + }, + "exit_code_descriptions": { + "declined": "The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.", + "failed": "A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.", + "internal": "The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.", + "not_applicable": "The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).", + "ok": "Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.", + "unknown_id": "No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can." } } diff --git a/engines/aim/src/docker.rs b/engines/aim/src/docker.rs index bfb8f67631..94e95d6ee8 100644 --- a/engines/aim/src/docker.rs +++ b/engines/aim/src/docker.rs @@ -4,7 +4,7 @@ //! Thin wrapper around the `docker` CLI binary. No `bollard`/Docker API //! dependency: shelling out mirrors the pattern `engines/lemonade/src/process.rs` -//! already uses for its own CLI (`run_lemonade_pull`/`run_lemonade_backend_install`). +//! already uses for its own CLI (`run_lemonade_pull_with_parent`/`run_lemonade_backend_install`). use anyhow::{Context, Result, bail}; use std::collections::BTreeMap; diff --git a/engines/lemonade/Cargo.toml b/engines/lemonade/Cargo.toml index 5e0e9208c2..98b56d0681 100644 --- a/engines/lemonade/Cargo.toml +++ b/engines/lemonade/Cargo.toml @@ -38,5 +38,6 @@ ureq = { version = "2.12", features = ["json", "native-certs"] } zip = { version = "9.0", default-features = false, features = ["deflate"] } [dev-dependencies] +rocm-core = { path = "../../crates/rocm-core", features = ["test-support"] } # Archive-extraction property tests; see `crates/rocm-core/src/archive_props.rs`. proptest = { version = "1", default-features = false, features = ["std"] } diff --git a/engines/lemonade/src/archive_cache.rs b/engines/lemonade/src/archive_cache.rs index ad2b7177ce..99012c2ef2 100644 --- a/engines/lemonade/src/archive_cache.rs +++ b/engines/lemonade/src/archive_cache.rs @@ -283,7 +283,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::test_support::scratch_dir; + use crate::test_support::{proptest_cases, scratch_dir}; use std::io::Write; use std::process::{Command as ProcessCommand, Stdio}; @@ -300,6 +300,144 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Whether `error` is the digest check's own rejection, so a failure to + /// read the archive cannot pass as one. + fn is_digest_mismatch(error: &anyhow::Error) -> bool { + error.to_string().contains("SHA-256 mismatch") + } + + /// A runner with a fixed seed, so a failure reproduces as-is, running at + /// least `cases` cases. `TestRunner::deterministic` alone takes its case + /// count from `PROPTEST_CASES` unchecked, so `PROPTEST_CASES=0` would pass + /// these properties without running one. + fn deterministic_runner(cases: u32) -> proptest::test_runner::TestRunner { + let config = proptest::test_runner::Config { + cases: proptest_cases(cases), + ..proptest::test_runner::Config::default() + }; + let rng = proptest::test_runner::TestRng::deterministic_rng(config.rng_algorithm); + proptest::test_runner::TestRunner::new_with_rng(config, rng) + } + + /// Property: the archive digest check accepts the pinned digest and nothing + /// else. + /// + /// The generator targets near misses rather than random strings, because a + /// random 64-character candidate is rejected by any implementation — + /// correct, prefix-matching, or truncating alike. What separates them is a + /// candidate that shares most of the real digest: one nibble changed, the + /// correct prefix with a wrong tail, a truncated prefix, or the real digest + /// with whitespace or an extra character attached. + /// + /// Note the comparison is byte-exact, so an upper-case spelling of the + /// *correct* digest is also rejected. That is fail-closed (an upper-case + /// pin would break installs, not admit a bad archive) but it differs from + /// `rocm_core`'s download gate, which folds ASCII case. + #[test] + fn archive_digest_check_accepts_only_the_exact_pinned_digest() { + use proptest::prelude::*; + + let dir = scratch_dir("archive-digest-properties"); + let archive = dir.join("archive"); + let content = b"pinned embeddable archive"; + fs::write(&archive, content).unwrap(); + let truth = format!("{:x}", Sha256::digest(content)); + + assert!(verify_sha256(&archive, &truth).is_ok()); + + let mut runner = deterministic_runner(256); + let near_miss = { + let truth = truth.clone(); + (0usize..64, 1usize..64, 0u8..7).prop_map(move |(position, keep, kind)| match kind { + 0 => { + let mut out: Vec = truth.chars().collect(); + out[position] = if out[position] == 'a' { 'b' } else { 'a' }; + out.into_iter().collect::() + } + 1 => format!("{}{}", &truth[..keep], "0".repeat(64 - keep)), + 2 => truth[..keep].to_owned(), + 3 => format!("{truth}\n"), + 4 => format!(" {truth} "), + 5 => format!("{truth}0"), + _ => truth.to_ascii_uppercase(), + }) + }; + + runner + .run(&near_miss, |candidate| { + prop_assert_ne!( + &candidate, + &truth, + "the generator must only produce near misses" + ); + prop_assert!( + verify_sha256(&archive, &candidate) + .is_err_and(|error| is_digest_mismatch(&error)), + "the archive digest check accepted {:?} for an archive whose digest is {:?}", + candidate, + truth + ); + Ok(()) + }) + .unwrap(); + + // An empty pin is the one near miss the generator above cannot produce + // (it keeps at least one character); it must not read as "no check". + assert!( + verify_sha256(&archive, "").is_err_and(|error| is_digest_mismatch(&error)), + "an empty pinned digest accepted the archive" + ); + + fs::remove_dir_all(&dir).ok(); + } + + /// Property: flipping a bit of the archive — at sampled positions, and in + /// the last byte — invalidates it against the pinned digest: the + /// tampered-download case. The archive is longer + /// than any single read buffer, and the intact archive must still be + /// accepted: a check that hashed only a prefix would reject the intact + /// archive and miss flips near the end. + #[test] + fn archive_digest_check_rejects_a_single_bit_of_tampering() { + use proptest::prelude::*; + + let dir = scratch_dir("archive-digest-tamper"); + let archive = dir.join("archive"); + let content: Vec = (0..=255_u8).cycle().take(64 * 1024 + 1).collect(); + let truth = format!("{:x}", Sha256::digest(&content)); + + fs::write(&archive, &content).unwrap(); + assert!( + verify_sha256(&archive, &truth).is_ok(), + "the intact multi-buffer archive must be accepted" + ); + // The last byte is checked explicitly: random positions would almost + // never land on it. + let mut tail_flipped = content.clone(); + *tail_flipped.last_mut().unwrap() ^= 1; + fs::write(&archive, &tail_flipped).unwrap(); + assert!( + verify_sha256(&archive, &truth).is_err_and(|error| is_digest_mismatch(&error)), + "a flip in the last byte was accepted" + ); + + let mut runner = deterministic_runner(256); + runner + .run(&(0usize..content.len(), 0u8..8), |(position, bit)| { + let mut tampered = content.clone(); + tampered[position] ^= 1 << bit; + fs::write(&archive, &tampered).unwrap(); + prop_assert!( + verify_sha256(&archive, &truth).is_err_and(|error| is_digest_mismatch(&error)), + "an archive differing by one bit was accepted" + ); + Ok(()) + }) + .unwrap(); + + fs::remove_dir_all(&dir).ok(); + } + #[test] fn repeated_valid_archive_publication_is_idempotent() { let dir = scratch_dir("valid-publication-race"); diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index 3899d32f9c..dc07423f1d 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -22,11 +22,10 @@ use crate::llamacpp_backend::{ find_llama_server_binary_for_backend, install_best_llamacpp_backend, }; #[cfg(target_os = "linux")] -use crate::process::apply_lemonade_process_environment; +use crate::process::apply_lemonade_process_environment_with_parent; #[cfg(target_os = "windows")] use crate::process::lemonade_process_environment_vars; use crate::process::{LemonadeProcessEnvironment, lemonade_process_environment}; -#[cfg(target_os = "windows")] use crate::runtime_dir::ParentRuntimeEnvironment; /// Resource file inside the extracted Lemonade embeddable pinning which @@ -714,7 +713,13 @@ fn rocm_backend_resolves( .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - if apply_lemonade_process_environment(&mut command, process_env).is_err() { + if apply_lemonade_process_environment_with_parent( + &mut command, + process_env, + &ParentRuntimeEnvironment::current(), + ) + .is_err() + { return false; } let Ok(output) = command.output() else { @@ -874,7 +879,7 @@ const BACKEND_DIR_ASIDE_SUFFIX: &str = ".pre-alignment"; /// build into its own build-numbered subdirectory rather than replacing one in place, /// so a tier that installs a different `therock.version`/llama.cpp tag than a prior /// attempt (or the pinned default) can otherwise leave two builds side by side -- -/// after which [`crate::direct_llama::find_binary_in`]'s directory-order fallback, used by both alignment +/// after which [`crate::llamacpp_backend::find_binary_in`]'s directory-order fallback, used by both alignment /// verification and `rocm serve`, may resolve to whichever build a rejected tier /// installed instead of the one just verified. /// @@ -914,7 +919,7 @@ fn set_aside_rocm_llamacpp_backend_dirs( /// actually asided) before moving each `aside` back into place: a failed install can /// land its output under a *different* ROCm-family name than the one that was /// asided (Lemonade extracts each release into its own build-numbered directory -/// rather than replacing one in place), and [`crate::direct_llama::find_binary_in`]'s priority-ordered +/// rather than replacing one in place), and [`crate::llamacpp_backend::find_binary_in`]'s priority-ordered /// lookup would otherwise let that stray, unverified directory shadow the good /// build this function just restored. Best-effort: a failure here just risks the /// same stale-directory ambiguity the aside exists to prevent, not the reinstall diff --git a/engines/lemonade/src/device_policy.rs b/engines/lemonade/src/device_policy.rs index a56aa7aa84..00ca729d61 100644 --- a/engines/lemonade/src/device_policy.rs +++ b/engines/lemonade/src/device_policy.rs @@ -5,6 +5,7 @@ use anyhow::{Result, bail}; use rocm_engine_protocol::{DevicePolicy, EngineDeviceAvailability, GpuSelection}; use serde_json::Value; +use std::path::Path; pub(crate) fn normalize_device_policy(policy: Option) -> Result { match policy.unwrap_or(DevicePolicy::GpuRequired) { @@ -143,6 +144,38 @@ pub(crate) fn reported_device(state: Option<&Value>, backend: &str) -> String { } } +/// A one-line notice naming the backend actually serving when it is a GPU +/// backend other than ROCm (Vulkan), so it is never read as a ROCm serve. `None` +/// for ROCm backends. The text is derived from the same backend name the engine +/// records in its state (`backend_requested`), so it cannot disagree with it. +pub(crate) fn non_rocm_backend_notice(backend: &str) -> Option { + let backend = backend.trim(); + (!backend.to_ascii_lowercase().starts_with("rocm")).then(|| { + format!( + "Lemonade is serving on the {backend} GPU backend, not ROCm; no CPU fallback is used." + ) + }) +} + +/// Print the non-ROCm backend notice and append it to the serve log, because a +/// managed serve runs with its stdout and stderr discarded and the log is the only +/// place the user can read it. Returns the notice, or `None` for a ROCm backend. +pub(crate) fn announce_non_rocm_backend(backend: &str, log_path: Option<&Path>) -> Option { + use std::io::Write as _; + let notice = non_rocm_backend_notice(backend)?; + eprintln!("{notice}"); + if let Some(log_path) = log_path + && let Some(mut log) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .ok() + { + let _ = writeln!(log, "{notice}"); + } + Some(notice) +} + /// The first pinned GPU ordinal recorded in service state, if any. fn first_gpu_index_from_state(state: &Value) -> Option { state diff --git a/engines/lemonade/src/direct_llama.rs b/engines/lemonade/src/direct_llama.rs index 0c3ba14e22..ed975f763e 100644 --- a/engines/lemonade/src/direct_llama.rs +++ b/engines/lemonade/src/direct_llama.rs @@ -2,222 +2,380 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, anyhow, bail}; +use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, format_http_base_url, http_get_text_with_auth, runtime_home_dir}; use serde_json::{Value, json}; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::{Command as ProcessCommand, Stdio}; +use std::process::{Child, Command as ProcessCommand, ExitStatus, Stdio}; use std::time::Duration; -use crate::install::{LemonadeInstallManifest, LemonadeRuntime, platform_binary_name}; +use crate::install::{LemonadeInstallManifest, LemonadeRuntime}; +use crate::llamacpp_backend::{backend_server_dirs, find_llama_server_binary_for_backend}; use crate::process::{ - LemonadeProcessEnvironment, apply_lemonade_process_environment, hide_child_console_window, - model_names_match, query_chat_smoke_endpoint, run_lemonade_model_load, run_lemonade_pull, - spawn_lemond, wait_for_lemonade_cli_status, + LemonadeProcessEnvironment, LogMode, apply_environment_vars, hide_child_console_window, + lemonade_process_environment_vars, query_chat_smoke_endpoint, run_lemonade_pull_with_parent, + spawn_lemond_with_parent, wait_for_lemonade_cli_status_with_parent, }; +use crate::runtime_dir::ParentRuntimeEnvironment; use crate::state::{ free_local_port, mark_json_status, merge_json_state, resolve_lemonade_model_ref, terminate_pid, write_running_state, }; -use crate::{ - DEFAULT_HOST, DEFAULT_MODEL, DEFAULT_MODEL_GGUF, DEFAULT_MODEL_REPO_DIR, ROCM_BACKEND_NAME, - ServeHttpRequest, -}; +use crate::{DEFAULT_HOST, ROCM_BACKEND_NAME, ServeHttpRequest}; + +/// Direct serving is only available for a local GGUF or an exact checkpoint. +/// Unsupported registered recipes stay on the Lemonade router path. +#[derive(Debug)] +pub(crate) enum DirectLlamaModel { + Local(PathBuf), + Checkpoint(HfCheckpoint), +} + +#[derive(Debug)] +pub(crate) enum ServeRoute { + Direct { + model: DirectLlamaModel, + server: PathBuf, + }, + Router { + model_ref: String, + }, +} -/// Serve a canonical Hugging Face checkpoint under its exact name. Requires an explicit -/// `:variant`; downloads the GGUF, then runs whichever packaged GPU llama-server backend -/// is installed (e.g. `vulkan` on WSL2, `rocm-stable` on native ROCm) directly on the -/// file with `--alias`. This bypasses Lemonade's model registry, whose naming rules -/// cannot preserve an `owner/repo:variant` name. -pub(crate) fn serve_hf_checkpoint( +/// Choose the serving path before downloading or launching a server. +pub(crate) fn serving_route( + manifest: &LemonadeInstallManifest, request: &ServeHttpRequest, - runtime: &LemonadeRuntime, - process_env: &LemonadeProcessEnvironment, - log_path: Option<&Path>, - checkpoint: &HfCheckpoint, -) -> Result<()> { - // A variant is required: a bare `owner/repo` would need Lemonade's interactive - // variant menu (which cannot be answered from this non-interactive path) and, if a - // GGUF happened to be cached, risks silently serving the wrong quantization. - if checkpoint.variant.is_none() { - bail!( - "`{model}` needs an explicit quantization variant to serve; use `owner/repo:variant`, e.g. `{model}:Q4_K_M` (see the repo's GGUF files on Hugging Face for the available variants)", - model = request.model_ref - ); + is_linux: bool, +) -> Result { + crate::device_policy::require_gpu_required(&request.device_policy)?; + let explicit_checkpoint = parse_hf_checkpoint(&request.model_ref).is_some(); + // Windows serves a registered name through the router on purpose: direct + // serving passes `-m`, `--n-gpu-layers`, `--alias` and the recipe flags, and does + // not apply the router's per-model launch defaults, so only an explicit + // checkpoint is served directly there. + if (is_linux || explicit_checkpoint) + && let Some(model) = resolve_direct_llama_model(manifest, &request.model_ref)? + && let Some(server) = direct_serve_binary(manifest) + { + return Ok(ServeRoute::Direct { model, server }); } - let Some(server) = find_llama_server_binary(&runtime.manifest) else { - bail!( - "no GPU llama-server backend is installed under {}; run `rocm engines install lemonade`", - runtime - .manifest - .runtime_dir - .join("bin") - .join("llamacpp") - .display() - ); - }; - ensure_hf_checkpoint_downloaded(request, runtime, process_env, log_path)?; - // Never silently pick between quantizations: if the variant matches more than one - // cached GGUF, ask the user to disambiguate instead of choosing one arbitrarily. - let paths = AppPaths::discover()?; - let matches = hf_checkpoint_gguf_matches(&paths, checkpoint); - if matches.len() > 1 { - let names = matches - .iter() - .filter_map(|path| path.file_name().and_then(|name| name.to_str())) - .collect::>() - .join(", "); + if explicit_checkpoint { + if manifest.backend_recipe != crate::LLAMACPP_RECIPE { + bail!( + "selected Lemonade recipe `{}` does not support direct HF checkpoint serving; a GPU llama.cpp recipe is required", + manifest.backend_recipe + ); + } + if backend_server_dirs(&manifest.backend_name).is_none() { + bail!( + "selected Lemonade backend `{}` does not support GPU direct serving; no CPU fallback is used", + manifest.backend_name + ); + } + // The backend is a supported GPU one, so only its binary is absent. A + // reinstall re-extracts the runtime and reinstalls the best backend, which + // restores the file whatever state Lemonade recorded for it. bail!( - "variant `{}` matches multiple files ({names}); specify an exact quantization or filename, e.g. `{}:{}`", - checkpoint.variant.as_deref().unwrap_or_default(), - request.model_ref, - matches - .first() - .and_then(|path| path.file_name()) - .and_then(|name| name.to_str()) - .unwrap_or(".gguf") + "no selected GPU llama-server backend `{}` is installed under {} for direct serving of `{}`; run `rocm engines install lemonade --reinstall` to restore it", + manifest.backend_name, + manifest.runtime_dir.display(), + request.model_ref ); } - serve_direct_llama_server( - request, - runtime, - process_env, - &server, - log_path, - &anyhow!( - "serving canonical Hugging Face checkpoint `{}` directly", - request.model_ref - ), - ) + Ok(ServeRoute::Router { + model_ref: resolve_lemonade_model_ref(&request.model_ref), + }) +} + +/// The direct route still open after the router failed to load or verify a +/// model: Linux only, a model the registry resolves to a safe checkpoint, and the +/// llama-server of the backend Lemonade selected (read from the manifest the +/// router serve updated). A CPU or unknown backend never yields a server, so this +/// cannot become a CPU fallback. +pub(crate) fn direct_fallback_route( + manifest: &LemonadeInstallManifest, + request: &ServeHttpRequest, + is_linux: bool, +) -> Option<(DirectLlamaModel, PathBuf)> { + if !is_linux { + return None; + } + let model = resolve_direct_llama_model(manifest, &request.model_ref) + .ok() + .flatten()?; + let server = direct_serve_binary(manifest)?; + Some((model, server)) } -/// Ensure a canonical Hugging Face checkpoint's GGUF is in the HF hub cache, pulling -/// it through a short-lived `lemond` if needed. The bare `pull owner/repo:variant` -/// only downloads (and registers a derived name we ignore); the file is what we serve. -fn ensure_hf_checkpoint_downloaded( +/// Serve directly after the router failed to load or verify the model, or return +/// the router's error unchanged when no direct route is open. The line it prints +/// and logs names the backend taken from the server path it launches. +#[allow(clippy::too_many_arguments)] +pub(crate) fn serve_direct_after_router_failure( + paths: &AppPaths, request: &ServeHttpRequest, runtime: &LemonadeRuntime, process_env: &LemonadeProcessEnvironment, log_path: Option<&Path>, + parent: &ParentRuntimeEnvironment, + is_linux: bool, + router_error: anyhow::Error, ) -> Result<()> { - let paths = AppPaths::discover()?; - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - return Ok(()); - } - let download_port = free_local_port()?; - let mut child = spawn_lemond( - &runtime.manifest, - DEFAULT_HOST, - download_port, - log_path, + let Some((model, server)) = direct_fallback_route(&runtime.manifest, request, is_linux) else { + // No direct route is open, so the router's failure is final. The router + // left its state for this function to settle. + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + return Err(router_error); + }; + let backend = llama_server_backend_label(&server); + let reason = format!( + "Lemonade router could not serve {} ({router_error:#}); serving it directly with the selected {backend} GPU llama-server", + request.model_ref + ); + eprintln!("{reason}"); + // The router left the state at `running`, with a server that is already + // stopped. A reader that sees it while the download or the spawn is still + // running would report a server that is gone, and one that sees `failed` + // would record the service as failed for good (`refresh_from_engine_state` + // ignores later states once a record is `failed`), so the state says + // `starting` for the whole fallback and goes to `failed` if the fallback + // does not produce a running server. The router's server pid goes too: + // `identity_from_state` prefers it over the launcher's, so leaving the dead + // router's pid in place would point a `stop` during the fallback at a + // process that is gone (or, once reused, at an unrelated one) instead of the + // launcher that is about to start the direct server. + crate::state::merge_json_state( + &request.state_path, + &serde_json::json!({ + "engine": crate::ENGINE_NAME, + "status": "starting", + "server_pid": null, + "server_start_ticks": null, + }), + ) + .with_context(|| { + format!( + "failed to record that the direct serving fallback is starting (the router had failed: {router_error:#})" + ) + })?; + let outcome = ensure_model_in( + paths, + request, + runtime, process_env, - )?; - let result = (|| -> Result<()> { - wait_for_lemonade_cli_status( - &runtime.manifest, - DEFAULT_HOST, - download_port, - Duration::from_secs(30), - log_path, + log_path, + &model, + parent, + ) + .with_context(|| format!("{router_error:#}; the direct serving fallback also failed")) + .and_then(|model_path| { + serve_direct_llama_server( + request, + runtime, process_env, - )?; - run_lemonade_pull( - &runtime.manifest, - DEFAULT_HOST, - download_port, - &request.model_ref, + &server, + &model_path, log_path, - process_env, - )?; - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - Ok(()) - } else { - bail!( - "Lemonade did not download `{}` (searched {})", - request.model_ref, - describe_hf_cache_roots(&paths) - ) + &reason, + parent, + ) + }); + if outcome.is_err() { + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + } + outcome +} + +pub(crate) fn resolve_direct_llama_model( + manifest: &LemonadeInstallManifest, + model_ref: &str, +) -> Result> { + let path = PathBuf::from(model_ref); + if path.is_file() + && path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) + { + return Ok(Some(DirectLlamaModel::Local(path))); + } + Ok(resolve_direct_llama_checkpoint(manifest, model_ref)?.map(DirectLlamaModel::Checkpoint)) +} + +/// Resolve the cached file without choosing an arbitrary quantization. The same +/// resolution is used before downloading and immediately before server launch. +pub(crate) fn direct_llama_model_path( + paths: &AppPaths, + model: &DirectLlamaModel, +) -> Result> { + match model { + DirectLlamaModel::Local(path) => Ok(path.is_file().then(|| path.clone())), + DirectLlamaModel::Checkpoint(checkpoint) => { + let matches = hf_checkpoint_gguf_matches(paths, checkpoint); + // A split model is one model: serve it from its first shard, which + // llama-server uses to load the remaining shards beside it. + if let Some(first) = first_shard_of_complete_set(&matches) { + return Ok(Some(first)); + } + // Some but not all shards of one model (an interrupted download): treat + // the model as absent so it is downloaded again, never served partial. + if is_incomplete_shard_set(&matches) { + return Ok(None); + } + if matches.len() > 1 { + let names = matches + .iter() + .filter_map(|path| path.file_name()) + .map(|name| name.to_string_lossy()) + .collect::>() + .join(", "); + let example = matches + .first() + .and_then(|path| path.file_name()) + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + bail!( + "variant `{}` matches multiple files ({names}); specify an exact quantization or filename, e.g. `{}/{}:{example}`", + checkpoint.variant.as_deref().unwrap_or_default(), + checkpoint.owner, + checkpoint.repo + ); + } + Ok(matches.into_iter().next()) } - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - result + } } -pub(crate) fn ensure_direct_llama_model_available( +/// Return the GGUF for `model`, downloading it first when absent. The cache +/// location and parent runtime environment are supplied by the caller. The hub +/// lookup also reads HF cache environment variables and the user's home directory; +/// tests use unique checkpoint names to avoid matching unrelated cached models. +pub(crate) fn ensure_model_in( + paths: &AppPaths, request: &ServeHttpRequest, runtime: &LemonadeRuntime, process_env: &LemonadeProcessEnvironment, log_path: Option<&Path>, -) -> Result<()> { - let paths = AppPaths::discover()?; - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - return Ok(()); - } - - let download_port = free_local_port()?; - let mut child = spawn_lemond( - &runtime.manifest, - DEFAULT_HOST, - download_port, - log_path, - process_env, - )?; - let result = (|| -> Result<()> { - wait_for_lemonade_cli_status( + model: &DirectLlamaModel, + parent: &ParentRuntimeEnvironment, +) -> Result { + ensure_model_with_download(paths, &request.model_ref, model, |checkpoint_ref| { + let download_port = free_local_port()?; + let mut child = spawn_lemond_with_parent( &runtime.manifest, DEFAULT_HOST, download_port, - Duration::from_secs(30), log_path, process_env, + parent, + // The log may already hold why a router failed and this fallback is running. + LogMode::Append, )?; - let _ = run_lemonade_model_load( - &runtime.manifest, - DEFAULT_HOST, - download_port, - &request.model_ref, - ROCM_BACKEND_NAME, - None, - log_path, - process_env, - ); - if direct_llama_model_path(&paths, &request.model_ref).is_some() { - Ok(()) - } else { - bail!( - "Lemonade did not download `{}` for direct ROCm serving (searched {})", - request.model_ref, - describe_hf_cache_roots(&paths) + let result = (|| -> Result<()> { + wait_for_lemonade_cli_status_with_parent( + &runtime.manifest, + DEFAULT_HOST, + download_port, + Duration::from_secs(30), + log_path, + process_env, + parent, + )?; + // Pull downloads without a model load, so this path neither guesses a + // backend nor masks load errors. + run_lemonade_pull_with_parent( + &runtime.manifest, + DEFAULT_HOST, + download_port, + checkpoint_ref, + log_path, + process_env, + parent, ) - } - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - result + })(); + let _ = terminate_pid(child.id(), true); + let _ = child.wait(); + result + }) +} + +/// Return the cached GGUF, downloading it first when absent. `download` receives +/// the validated checkpoint reference (`owner/repo:variant`), never the alias the +/// user typed, and its error is returned as is. +fn ensure_model_with_download( + paths: &AppPaths, + requested: &str, + model: &DirectLlamaModel, + download: impl FnOnce(&str) -> Result<()>, +) -> Result { + if let Some(path) = direct_llama_model_path(paths, model)? { + return Ok(path); + } + let DirectLlamaModel::Checkpoint(checkpoint) = model else { + bail!("local GGUF `{requested}` is missing"); + }; + download(&checkpoint.model_ref())?; + direct_llama_model_path(paths, model)?.with_context(|| { + format!( + "Lemonade did not download `{requested}` for direct serving (searched {})", + describe_hf_cache_roots(paths) + ) + }) } /// Run one of Lemonade's packaged `llama-server` binaries (`server`) directly on a /// GGUF file, exposing the model under `request.model_ref` via `--alias`. This is how /// a canonical Hugging Face name is served under exactly that name — Lemonade's router /// renames registered models, but an llama-server alias is a free-form string. +#[allow(clippy::too_many_arguments)] pub(crate) fn serve_direct_llama_server( request: &ServeHttpRequest, runtime: &LemonadeRuntime, process_env: &LemonadeProcessEnvironment, server: &Path, + model_path: &Path, log_path: Option<&Path>, - reason: &anyhow::Error, + reason: &str, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { - let paths = AppPaths::discover()?; - let model_path = direct_llama_model_path(&paths, &request.model_ref).with_context(|| { - format!( - "Lemonade downloaded model `{}` was not found for direct serving", - request.model_ref - ) - })?; + serve_direct_llama_server_within( + request, + runtime, + process_env, + server, + model_path, + log_path, + reason, + parent, + DIRECT_SERVER_READY_TIMEOUT, + ) +} + +/// How long a direct llama-server may take to answer `/v1/models` and a chat smoke +/// test before the serve is treated as failed. +const DIRECT_SERVER_READY_TIMEOUT: Duration = Duration::from_mins(2); + +#[allow(clippy::too_many_arguments)] +fn serve_direct_llama_server_within( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + server: &Path, + model_path: &Path, + log_path: Option<&Path>, + reason: &str, + parent: &ParentRuntimeEnvironment, + ready_timeout: Duration, +) -> Result<()> { + if !model_path.is_file() { + bail!( + "GGUF for `{}` is missing at {}", + request.model_ref, + model_path.display() + ); + } if !server.is_file() { bail!("Lemonade llama-server is missing at {}", server.display()); } @@ -236,7 +394,7 @@ pub(crate) fn serve_direct_llama_server( .with_context(|| format!("failed to open {}", log_path.display()))?; writeln!( log, - "\nLaunching Lemonade packaged {backend} llama-server directly: {reason:#}" + "\nLaunching Lemonade packaged {backend} llama-server directly: {reason}" ) .ok(); } @@ -247,22 +405,7 @@ pub(crate) fn serve_direct_llama_server( push_existing_path(&mut direct_env.library_entries, server_dir.to_path_buf()); } - let mut command = ProcessCommand::new(server); - command - .arg("-m") - .arg(&model_path) - .arg("--host") - .arg(&request.host) - .arg("--port") - .arg(request.port.to_string()) - .arg("--n-gpu-layers") - .arg("999") - .arg("--alias") - .arg(&request.model_ref) - .stdin(Stdio::null()); - if let Some(engine_recipe) = request.engine_recipe.as_ref() { - command.args(&engine_recipe.required_flags); - } + let mut command = direct_llama_server_command(request, server, model_path); // Packaged llama-server is raw llama.cpp — it does not read `LEMONADE_API_KEY`. // When `rocm serve` protects a public endpoint, hand llama-server the *existing* // CLI-managed 0600 key file via `--api-key-file` (a path, not the value, so the @@ -283,62 +426,137 @@ pub(crate) fn serve_direct_llama_server( } else { command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); } - apply_lemonade_process_environment(&mut command, &direct_env)?; + let vars = lemonade_process_environment_vars(&direct_env, parent)?; + apply_environment_vars(&mut command, &vars); hide_child_console_window(&mut command); let mut child = command .spawn() .with_context(|| format!("failed to start {}", server.display()))?; - write_running_state( + let mut steps = RealDirectServerSteps { request, runtime, - std::process::id(), - Some(child.id()), - "running", - )?; - // If the server never becomes ready (or fails the smoke test), don't leak the child - // or leave the state stuck at `running`: kill it and mark the service failed. - let readiness = (|| -> Result<()> { + backend: &backend, + model_path, + log_path, + ready_timeout, + }; + supervise_direct_server(&mut child, &request.state_path, &mut steps) +} + +/// The fallible steps that follow spawning a direct llama-server, so the failure +/// handling in [`supervise_direct_server`] is exercised without a live server. +trait DirectServerSteps { + fn record_running(&mut self, server_pid: u32) -> Result<()>; + fn wait_ready(&mut self) -> Result<()>; + fn record_ready(&mut self, server_pid: u32) -> Result<()>; + /// Announce what is now true. Called only once the server answered, passed its + /// chat smoke test and the `ready` state is written: an earlier announcement + /// would claim a serve that a failed readiness check then contradicts. + fn announce_serving(&mut self); + fn wait_exit(&mut self, child: &mut Child) -> std::io::Result { + child.wait() + } +} + +struct RealDirectServerSteps<'a> { + request: &'a ServeHttpRequest, + runtime: &'a LemonadeRuntime, + backend: &'a str, + model_path: &'a Path, + log_path: Option<&'a Path>, + ready_timeout: Duration, +} + +impl DirectServerSteps for RealDirectServerSteps<'_> { + fn record_running(&mut self, server_pid: u32) -> Result<()> { + write_running_state( + self.request, + self.runtime, + std::process::id(), + Some(server_pid), + "running", + ) + } + + fn wait_ready(&mut self) -> Result<()> { + let request = self.request; wait_for_openai_models_ready( &request.host, request.port, &request.model_ref, - Duration::from_mins(2), + self.ready_timeout, )?; if !query_chat_smoke_endpoint(&request.host, request.port, &request.model_ref)? { bail!("Lemonade packaged llama-server did not pass a chat-completion smoke test"); } Ok(()) - })(); - if let Err(error) = readiness { - let _ = terminate_pid(child.id(), true); + } + + fn record_ready(&mut self, server_pid: u32) -> Result<()> { + let request = self.request; + merge_json_state( + &request.state_path, + &json!({ + "status": "ready", + "server_pid": server_pid, + // Identity token for the server PID, captured while the child is alive. + "server_start_ticks": rocm_core::process_start_ticks(server_pid), + "backend_state": "ready", + "backend_requested": self.backend, + "backend_mode": "lemonade-packaged-llama-server", + "load_response": { + "status": "loaded", + "method": "lemonade-packaged-llama-server", + "model_name": request.model_ref, + "model_path": self.model_path, + "llamacpp_backend": self.backend + }, + }), + ) + } + + fn announce_serving(&mut self) { + crate::device_policy::announce_non_rocm_backend(self.backend, self.log_path); + } +} + +/// Own a just-spawned direct llama-server. Every step after the spawn, state +/// writes and the final wait included, runs under one guard: any failure stops and +/// reaps the child (`Child` does not kill on drop) and marks the state `failed`, +/// so the server can never keep running with no state that stop and health can +/// reach. +fn supervise_direct_server( + child: &mut Child, + state_path: &Path, + steps: &mut S, +) -> Result<()> { + let server_pid = child.id(); + let stop_and_fail = |child: &mut Child| { + let _ = terminate_pid(server_pid, true); let _ = child.wait(); - let _ = mark_json_status(&request.state_path, "failed"); + let _ = mark_json_status(state_path, "failed"); + }; + let started = (|| -> Result<()> { + steps.record_running(server_pid)?; + steps.wait_ready()?; + steps.record_ready(server_pid)?; + steps.announce_serving(); + Ok(()) + })(); + if let Err(error) = started { + stop_and_fail(child); return Err(error); } - merge_json_state( - &request.state_path, - &json!({ - "status": "ready", - "server_pid": child.id(), - // Identity token for the server PID, captured while the child is alive. - "server_start_ticks": rocm_core::process_start_ticks(child.id()), - "backend_state": "ready", - "backend_requested": backend, - "backend_mode": "lemonade-packaged-llama-server", - "load_response": { - "status": "loaded", - "method": "lemonade-packaged-llama-server", - "model_name": request.model_ref, - "model_path": model_path, - "llamacpp_backend": backend - }, - }), - )?; - let status = child - .wait() - .context("failed waiting for Lemonade packaged llama-server")?; + let status = match steps.wait_exit(child) { + Ok(status) => status, + Err(error) => { + stop_and_fail(child); + return Err(anyhow::Error::new(error) + .context("failed waiting for Lemonade packaged llama-server")); + } + }; mark_json_status( - &request.state_path, + state_path, if status.success() { "stopped" } else { @@ -352,6 +570,30 @@ pub(crate) fn serve_direct_llama_server( } } +pub(crate) fn direct_llama_server_command( + request: &ServeHttpRequest, + server: &Path, + model_path: &Path, +) -> ProcessCommand { + let mut command = ProcessCommand::new(server); + command + .arg("-m") + .arg(model_path) + .arg("--host") + .arg(&request.host) + .arg("--port") + .arg(request.port.to_string()) + .arg("--n-gpu-layers") + .arg("999") + .arg("--alias") + .arg(&request.model_ref) + .stdin(Stdio::null()); + if let Some(recipe) = request.engine_recipe.as_ref() { + command.args(&recipe.required_flags); + } + command +} + fn wait_for_openai_models_ready( host: &str, port: u16, @@ -399,7 +641,7 @@ fn models_payload_has_loaded_model(value: &Value, model_ref: &str) -> bool { ["id", "model", "name"] .into_iter() .filter_map(|field| model.get(field).and_then(Value::as_str)) - .any(|loaded| model_names_match(loaded, model_ref)) + .any(|loaded| loaded == model_ref) }) }) } @@ -421,6 +663,17 @@ pub(crate) struct HfCheckpoint { variant: Option, } +impl HfCheckpoint { + fn model_ref(&self) -> String { + format!( + "{}/{}:{}", + self.owner, + self.repo, + self.variant.as_deref().unwrap_or_default() + ) + } +} + /// Parse a model reference as a canonical Hugging Face checkpoint. Returns `None` for /// anything not of the form `owner/repo[:variant]` — built-in Lemonade aliases and /// bare registered names all fall through unchanged. @@ -455,6 +708,38 @@ pub(crate) fn parse_hf_checkpoint(model_ref: &str) -> Option { }) } +/// Why `model_ref` cannot be served, when it is a canonical Hugging Face checkpoint +/// given without a `:variant`. `None` means this particular objection does not apply — +/// the reference carries a variant, or is not an `owner/repo` checkpoint at all. +/// +/// The refusal lives here, beside the parser, because two callers at very different +/// depths need the same answer: the pre-launch `ResolveModel` the CLI makes before it +/// starts anything, and the direct serve inside the background engine process. +/// Only one of the two is visible to the user on a given platform, so wording written +/// out twice would drift and leave whichever one the user actually sees as the stale +/// copy. +/// +/// It deliberately names no quantization. Nothing on this path has looked inside the +/// repository, so any example label would be invented: the repo may ship only +/// safetensors and have no GGUF build at all, or ship GGUFs under labels that cannot be +/// guessed from the name. Advice that fails a second time is worse than advice that +/// sends the user to look. +/// +/// ASCII only: this reaches a Windows console, where the OEM code page renders +/// non-ASCII punctuation as mojibake (ROCMAI-543), and Windows is the platform where +/// this is the user's one and only sight of the problem. +pub(crate) fn hf_checkpoint_variant_required(model_ref: &str) -> Option { + let checkpoint = parse_hf_checkpoint(model_ref)?; + checkpoint.variant.is_none().then(|| { + format!( + "`{model_ref}` needs an explicit GGUF variant to serve; use \ + `owner/repo:` with a quantization listed in the repo's \ + `.gguf` files. A repo with no `.gguf` files cannot be served by \ + Lemonade: look for a GGUF build of the model" + ) + }) +} + /// A safe Hugging Face path component: a non-empty run of `[A-Za-z0-9._-]` that is not /// a `.`/`..` traversal token. Matches the Hugging Face id and quantization-label /// charset and, by excluding `/`, `\`, and traversal tokens, keeps a user-supplied @@ -468,42 +753,14 @@ fn is_safe_hf_component(component: &str) -> bool { .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) } -/// Packaged llama.cpp backends whose `llama-server` we can drive directly, best first. -/// GPU backends only — `cpu` is intentionally excluded so the direct-serve path never -/// silently serves on CPU under a GPU-required policy (AGENTS.md §6). An explicit -/// allowlist rather than a directory scan, so the path is built only from constants. +/// Directory names under `bin/llamacpp/` that [`llama_server_backend_label`] +/// recognizes when it names the backend of a packaged `llama-server`. GPU backends +/// only; `cpu` is intentionally absent so a CPU build is never labelled as, or +/// mistaken for, a direct-serve backend. This is only a label lookup: which +/// directories are searched for a selected backend is decided by +/// [`crate::llamacpp_backend::backend_server_dirs`]. const DIRECT_LLAMA_SERVER_BACKENDS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan"]; -/// Find an installed Lemonade `llama-server` binary under `bin/llamacpp//`, -/// checking known backends in priority order. This lets direct serving work wherever -/// Lemonade installed a backend (e.g. `vulkan` on WSL2), not just the ROCm build. -/// -/// Lemonade's backend installer extracts some llama.cpp releases straight into -/// `/`, but others land one level deeper under a build-numbered directory -/// (e.g. `rocm-stable/llama-b9752/llama-server`), so both layouts are checked. -pub(crate) fn find_llama_server_binary(manifest: &LemonadeInstallManifest) -> Option { - let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); - let binary = platform_binary_name("llama-server"); - DIRECT_LLAMA_SERVER_BACKENDS - .into_iter() - .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) -} - -/// Look for `binary` directly in `dir`, then one level down in each subdirectory. -pub(crate) fn find_binary_in(dir: &Path, binary: &str) -> Option { - let direct = dir.join(binary); - if direct.is_file() { - return Some(direct); - } - fs::read_dir(dir) - .ok()? - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|path| path.is_dir()) - .map(|subdir| subdir.join(binary)) - .find(|candidate| candidate.is_file()) -} - /// The backend label for a packaged llama-server, taken from the nearest ancestor /// directory matching a known backend name (e.g. `vulkan`, `rocm-stable`) so it still /// resolves correctly when Lemonade nests the binary under a build-numbered @@ -518,22 +775,83 @@ pub(crate) fn llama_server_backend_label(server: &Path) -> String { .to_owned() } -pub(crate) fn direct_llama_model_path(paths: &AppPaths, model_ref: &str) -> Option { - let as_path = PathBuf::from(model_ref); - if as_path.is_file() { - return Some(as_path); - } +/// Registered model names are exact keys after known shorthand normalization. +/// Metadata is untrusted: it must name a safe HF checkpoint with an explicit +/// variant, never a local path or a model guessed from the registered name. +pub(crate) fn resolve_direct_llama_checkpoint( + manifest: &LemonadeInstallManifest, + model_ref: &str, +) -> Result> { if let Some(checkpoint) = parse_hf_checkpoint(model_ref) { - return hf_cache_roots(paths) - .into_iter() - .find_map(|root| find_hf_checkpoint_gguf(&root, &checkpoint)); + return require_checkpoint_variant(checkpoint, model_ref).map(Some); + } + let registry_path = manifest.runtime_dir.join("resources/server_models.json"); + let Ok(bytes) = fs::read(®istry_path) else { + return Ok(None); + }; + let Ok(registry) = serde_json::from_slice::(&bytes) else { + return Ok(None); + }; + let Some(entries) = registry.as_object() else { + return Ok(None); + }; + let name = resolve_lemonade_model_ref(model_ref); + let Some(metadata) = entries.get(&name) else { + return Ok(None); + }; + if metadata.get("recipe").and_then(serde_json::Value::as_str) != Some(crate::LLAMACPP_RECIPE) { + return Ok(None); + } + let Some(checkpoint_ref) = metadata + .get("checkpoint") + .and_then(serde_json::Value::as_str) + else { + return Ok(None); + }; + // A registered checkpoint that is not a plain `owner/repo:variant` is not + // this module's to serve: Lemonade's catalog writes `*` for "every file", for + // one. Nothing here builds a path from it, so it goes to the router, which + // served it before. Only a reference the user typed is an error (above). + let Some(checkpoint) = parse_hf_checkpoint(checkpoint_ref) else { + return Ok(None); + }; + let Some(fields) = metadata.as_object() else { + return Ok(None); + }; + // Only the single-checkpoint shape is supported directly. Extra files, + // multimodal inputs, options and unknown fields belong to Lemonade's router. + // In the 11.5.1 source, size/suggested describe catalog entries and these + // labels classify chat models. Other labels can change launch flags (MTP). + if checkpoint.variant.is_none() + || fields.iter().any(|(key, value)| match key.as_str() { + "recipe" | "checkpoint" => false, + "size" => !value.as_f64().is_some_and(|size| size >= 0.0), + "suggested" => !value.is_boolean(), + "labels" => !value.as_array().is_some_and(|labels| { + labels.iter().all(|label| { + matches!(label.as_str(), Some("reasoning" | "tool-calling" | "tools")) + }) + }), + _ => true, + }) + { + return Ok(None); + } + Ok(Some(checkpoint)) +} + +fn require_checkpoint_variant(checkpoint: HfCheckpoint, model_ref: &str) -> Result { + if let Some(message) = hf_checkpoint_variant_required(model_ref) { + bail!("{message}"); } - if resolve_lemonade_model_ref(model_ref) != DEFAULT_MODEL { + Ok(checkpoint) +} + +pub(crate) fn direct_serve_binary(manifest: &LemonadeInstallManifest) -> Option { + if manifest.backend_recipe != crate::LLAMACPP_RECIPE { return None; } - hf_cache_roots(paths) - .into_iter() - .find_map(find_default_qwen_gguf) + find_llama_server_binary_for_backend(manifest, &manifest.backend_name) } fn hf_cache_roots(paths: &AppPaths) -> Vec { @@ -596,24 +914,6 @@ fn push_hf_cache_root(roots: &mut Vec, path: PathBuf) { roots.push(path); } -fn find_default_qwen_gguf(cache_root: PathBuf) -> Option { - let snapshots = cache_root.join(DEFAULT_MODEL_REPO_DIR).join("snapshots"); - let entries = fs::read_dir(snapshots).ok()?; - entries - .flatten() - .map(|entry| entry.path().join(DEFAULT_MODEL_GGUF)) - .find(|path| path.is_file()) -} - -/// Locate the downloaded GGUF for a Hugging Face checkpoint inside one hub cache root, -/// selecting the file that matches the requested `:variant` (if any). -fn find_hf_checkpoint_gguf(cache_root: &Path, checkpoint: &HfCheckpoint) -> Option { - select_gguf_for_variant( - collect_hf_checkpoint_ggufs(cache_root, checkpoint), - checkpoint.variant.as_deref(), - ) -} - /// All cached GGUF files for a checkpoint's repo under one hub cache root (unfiltered /// by variant). fn collect_hf_checkpoint_ggufs(cache_root: &Path, checkpoint: &HfCheckpoint) -> Vec { @@ -632,51 +932,137 @@ fn collect_hf_checkpoint_ggufs(cache_root: &Path, checkpoint: &HfCheckpoint) -> /// deduplicated by file name. Used to detect an ambiguous variant (more than one match) /// so the caller can refuse to pick arbitrarily. fn hf_checkpoint_gguf_matches(paths: &AppPaths, checkpoint: &HfCheckpoint) -> Vec { - let mut matches = Vec::new(); - let mut seen_names = std::collections::HashSet::new(); - for root in hf_cache_roots(paths) { - let ggufs = collect_hf_checkpoint_ggufs(&root, checkpoint); - for gguf in filter_ggufs_by_variant(ggufs, checkpoint.variant.as_deref()) { - if let Some(name) = gguf.file_name().and_then(|name| name.to_str()) - && seen_names.insert(name.to_owned()) - { - matches.push(gguf); - } - } - } - matches + hf_checkpoint_gguf_matches_in_roots(&hf_cache_roots(paths), checkpoint) } -/// Collect `*.gguf` files under a snapshot revision directory. GGUFs usually sit at the -/// snapshot root, but sharded variants live one folder down, so recurse a few levels. -fn collect_gguf_files(dir: &Path, out: &mut Vec, depth: usize) { - if depth > 3 { - return; - } - let Ok(entries) = fs::read_dir(dir) else { +fn hf_checkpoint_gguf_matches_in_roots( + roots: &[PathBuf], + checkpoint: &HfCheckpoint, +) -> Vec { + // Keep root priority and path order, but prefer a complete split model over + // partial copies of the same files. Deduplicating first could combine shards + // from two revisions, hiding a usable complete download behind an older one. + let mut candidates = roots + .iter() + .flat_map(|root| { + filter_ggufs_by_variant( + collect_hf_checkpoint_ggufs(root, checkpoint), + checkpoint.variant.as_deref(), + ) + }) + .collect::>(); + let mut groups = std::collections::HashMap::new(); + for path in &candidates { + if let Some((stem, _, total)) = parse_gguf_shard(path) { + groups + .entry((path.parent().map(Path::to_path_buf), stem, total)) + .or_insert_with(Vec::new) + .push(path.clone()); + } + } + let complete = groups + .into_iter() + .filter_map(|(key, paths)| first_shard_of_complete_set(&paths).map(|_| key)) + .collect::>(); + candidates.sort_by_key(|path| { + parse_gguf_shard(path).is_some_and(|(stem, _, total)| { + !complete.contains(&(path.parent().map(Path::to_path_buf), stem, total)) + }) + }); + let mut seen_names = std::collections::HashSet::new(); + candidates.retain(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| seen_names.insert(name.to_owned())) + }); + candidates +} + +/// Split a split-GGUF file name (`-00001-of-00002.gguf`) into its stem, +/// 1-based shard number and shard count. +fn parse_gguf_shard(path: &Path) -> Option<(String, u32, u32)> { + let name = path.file_name()?.to_str()?; + if !name.to_ascii_lowercase().ends_with(".gguf") { + return None; + } + // The suffix is ASCII, so this slice is on a character boundary. + let stem = &name[..name.len() - ".gguf".len()]; + let (rest, total) = stem.rsplit_once("-of-")?; + let (stem, index) = rest.rsplit_once('-')?; + let digits = |text: &str| text.len() == 5 && text.chars().all(|c| c.is_ascii_digit()); + if stem.is_empty() || !digits(index) || !digits(total) { + return None; + } + Some((stem.to_owned(), index.parse().ok()?, total.parse().ok()?)) +} + +/// The first shard when `matches` is exactly one complete split model (every shard +/// of one stem present once), `None` otherwise. Several unrelated files, or an +/// incomplete set, are still ambiguous or missing and are handled by the caller. +fn first_shard_of_complete_set(matches: &[PathBuf]) -> Option { + let shards = matches + .iter() + .map(|path| parse_gguf_shard(path).map(|shard| (path, shard))) + .collect::>>()?; + let (first_path, (stem, _, total)) = shards.first()?; + // llama-server finds the remaining shards beside the first one, so shards + // spread over several cache roots or snapshot revisions are not a usable set. + let directory = first_path.parent(); + let complete = shards.len() == usize::try_from(*total).ok()? + && shards.iter().all(|(path, (other, _, other_total))| { + other == stem && other_total == total && path.parent() == directory + }) + && (1..=*total).all(|wanted| shards.iter().any(|(_, (_, index, _))| *index == wanted)); + if !complete { + return None; + } + shards + .iter() + .find(|(_, (_, index, _))| *index == 1) + .map(|(path, _)| (*path).clone()) +} + +/// Whether `matches` are only shards of a single split model, with some missing. +fn is_incomplete_shard_set(matches: &[PathBuf]) -> bool { + let Some(shards) = matches + .iter() + .map(|path| parse_gguf_shard(path)) + .collect::>>() + else { + return false; + }; + let Some((stem, _, total)) = shards.first() else { + return false; + }; + shards + .iter() + .all(|(other, _, other_total)| other == stem && other_total == total) +} + +/// Collect `*.gguf` files under a snapshot revision directory. GGUFs usually sit at the +/// snapshot root, but sharded variants live one folder down, so recurse a few levels. +fn collect_gguf_files(dir: &Path, out: &mut Vec, depth: usize) { + if depth > 3 { + return; + } + let Ok(entries) = fs::read_dir(dir) else { return; }; for entry in entries.flatten() { let path = entry.path(); if path.is_dir() { collect_gguf_files(&path, out, depth + 1); - } else if path - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) + } else if path.is_file() + && path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) { out.push(path); } } } -/// Pick a single GGUF for a requested variant (the first, in sorted order, of the -/// matches). Sorting is by path — deterministic but not by mtime — which is fine -/// because the caller rejects an ambiguous variant before relying on this pick. -fn select_gguf_for_variant(ggufs: Vec, variant: Option<&str>) -> Option { - filter_ggufs_by_variant(ggufs, variant).into_iter().next() -} - /// All GGUFs matching a requested variant, sorted by path. With no variant, every file /// matches. With a variant, a case-insensitive exact filename match (`:model.gguf`) /// wins outright; otherwise every file whose name contains the quantization label @@ -692,13 +1078,51 @@ fn filter_ggufs_by_variant(mut ggufs: Vec, variant: Option<&str>) -> Ve .and_then(|name| name.to_str()) .map(str::to_owned) }; - if let Some(exact) = ggufs + let exact_matches = ggufs .iter() - .find(|path| file_name(path).is_some_and(|name| name.eq_ignore_ascii_case(variant))) - { - return vec![exact.clone()]; + .filter(|path| file_name(path).is_some_and(|name| name.eq_ignore_ascii_case(variant))) + .collect::>(); + if !exact_matches.is_empty() { + // A shard names its whole split model: serving needs every shard, and the + // ambiguity error suggests a shard's exact filename. The same shard name + // can exist in several snapshot revisions; prefer one with the full set. + let shard_set = |exact: &PathBuf| -> Option> { + let (stem, _, total) = parse_gguf_shard(exact)?; + Some( + ggufs + .iter() + .filter(|path| { + path.parent() == exact.parent() + && parse_gguf_shard(path).is_some_and(|(other, _, other_total)| { + other == stem && other_total == total + }) + }) + .cloned() + .collect(), + ) + }; + let sets = exact_matches + .iter() + .filter_map(|exact| shard_set(exact)) + .collect::>(); + if let Some(set) = sets + .iter() + .find(|set| first_shard_of_complete_set(set).is_some()) + .or_else(|| sets.first()) + { + return set.clone(); + } + return vec![exact_matches[0].clone()]; } let variant_lower = variant.to_ascii_lowercase(); + // An explicit filename is not a quantization substring. If absent, download + // that exact file rather than serve another file with the same suffix. + if Path::new(variant) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) + { + return Vec::new(); + } ggufs .into_iter() .filter(|path| { @@ -710,10 +1134,645 @@ fn filter_ggufs_by_variant(mut ggufs: Vec, variant: Option<&str>) -> Ve #[cfg(test)] mod tests { use super::*; - use crate::LLAMACPP_RECIPE; + use crate::install::platform_binary_name; + use crate::process::models_payload_has_ready_model; use crate::test_support::scratch_dir; + use crate::{DEFAULT_MODEL, LLAMACPP_RECIPE}; + use rocm_engine_protocol::EngineRecipeHint; use std::fs; + fn write_registry(manifest: &LemonadeInstallManifest, registry: serde_json::Value) { + let resources = manifest.runtime_dir.join("resources"); + fs::create_dir_all(&resources).unwrap(); + fs::write( + resources.join("server_models.json"), + serde_json::to_vec(®istry).unwrap(), + ) + .unwrap(); + } + + fn serve_request(model_ref: &str) -> ServeHttpRequest { + ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: model_ref.to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![1], + runtime_id: None, + env_id: None, + state_path: PathBuf::from("state.json"), + log_path: None, + engine_recipe: None, + } + } + + #[test] + fn serving_route_covers_platform_recipe_binary_and_policy_boundaries() { + let dir = scratch_dir("serving-routes"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + write_registry( + &manifest, + serde_json::json!({ + "Qwen3-0.6B-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q4_0", "suggested": true, "size": 0.38, "labels": ["reasoning"]}, + "Complex": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q4_0", "mmproj": "owner/repo:mmproj.gguf"}, + "OtherRecipe": {"recipe": "oga", "checkpoint": "owner/repo"} + }), + ); + let binary = manifest + .runtime_dir + .join("bin/llamacpp/vulkan") + .join(platform_binary_name("llama-server")); + fs::create_dir_all(binary.parent().unwrap()).unwrap(); + fs::write(&binary, b"x").unwrap(); + for is_linux in [true, false] { + let requested = serve_request("qwen-smoke"); + let route = serving_route(&manifest, &requested, is_linux).unwrap(); + if is_linux { + let ServeRoute::Direct { server, .. } = route else { + panic!("a registered alias with a selected-backend binary serves directly"); + }; + assert_eq!(server, binary); + // The `--alias` the server is started with is asserted through the + // production serve path in + // `direct_serve_through_serve_http_uses_the_requested_alias` (lib.rs). + } else { + assert!( + matches!(route, ServeRoute::Router { model_ref } if model_ref == "Qwen3-0.6B-GGUF") + ); + } + assert!(matches!( + serving_route(&manifest, &serve_request("owner/repo:Q4_0"), is_linux).unwrap(), + ServeRoute::Direct { .. } + )); + assert!(serving_route(&manifest, &serve_request("owner/repo"), is_linux).is_err()); + for name in ["Complex", "Unknown", "OtherRecipe"] { + assert!( + matches!(serving_route(&manifest, &serve_request(name), is_linux).unwrap(), ServeRoute::Router { model_ref } if model_ref == name) + ); + } + let mut cpu_request = serve_request("qwen-smoke"); + cpu_request.device_policy = rocm_engine_protocol::DevicePolicy::CpuOnly; + assert!( + serving_route(&manifest, &cpu_request, is_linux) + .unwrap_err() + .to_string() + .contains("no CPU fallback") + ); + } + fs::remove_file(&binary).unwrap(); + for is_linux in [true, false] { + assert!(matches!( + serving_route(&manifest, &serve_request("qwen-smoke"), is_linux).unwrap(), + ServeRoute::Router { .. } + )); + // The selected backend is supported but its binary is gone: the error is + // returned (no route, so nothing is started) and names the command that + // restores it. `rocm engines install lemonade --reinstall` is a real + // invocation: the `engines install` subcommand takes `--reinstall`, which + // re-extracts the runtime and reinstalls the best backend. + let error = serving_route(&manifest, &serve_request("owner/repo:Q4_0"), is_linux) + .unwrap_err() + .to_string(); + assert!(error.contains("no selected GPU llama-server"), "{error}"); + // This pins the wording only. That the command exists and takes `--reinstall` + // is exercised by `@id:runtime-lemonade-backend-alignment-reported` and + // `@id:runtime-lemonade-backend-alignment-opt-out` (runtime_setup.feature), + // which run `rocm engines install lemonade --reinstall` and require it to + // succeed. + // That first scenario (runtime-16) deletes the selected backend's + // llama-server, reinstalls, and requires the file to be back, using the + // same layout as `direct_serve_binary`. + // That a reinstall re-extracts the runtime over an existing one is + // `a_version_change_forces_re_extraction_over_the_existing_runtime` + // (install.rs). + assert!( + error.contains("rocm engines install lemonade --reinstall"), + "{error}" + ); + } + // Recipe/backend refusal is independent of a missing packaged binary. + fs::write(&binary, b"x").unwrap(); + let cpu_binary = manifest + .runtime_dir + .join("bin/llamacpp/cpu") + .join(platform_binary_name("llama-server")); + fs::create_dir_all(cpu_binary.parent().unwrap()).unwrap(); + fs::write(cpu_binary, b"x").unwrap(); + for (recipe, backend) in [ + ("oga", "vulkan"), + (LLAMACPP_RECIPE, "cpu"), + (LLAMACPP_RECIPE, "unknown"), + ] { + manifest.backend_recipe = recipe.to_owned(); + manifest.backend_name = backend.to_owned(); + for is_linux in [true, false] { + assert!(matches!( + serving_route(&manifest, &serve_request("qwen-smoke"), is_linux).unwrap(), + ServeRoute::Router { .. } + )); + let error = serving_route(&manifest, &serve_request("owner/repo:Q4_0"), is_linux) + .unwrap_err() + .to_string(); + let expected = if recipe == LLAMACPP_RECIPE { + "does not support GPU direct serving" + } else { + "does not support direct HF checkpoint serving" + }; + assert!(error.contains(expected), "{error}"); + // A reinstall cannot make a CPU or non-llama.cpp backend servable, and + // their binary may well be present, so neither claims it is missing. + assert!(!error.contains("is installed under"), "{error}"); + assert!(!error.contains("--reinstall"), "{error}"); + } + } + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_variant_errors_suggest_only_what_was_seen() { + let dir = scratch_dir("variant-examples"); + let manifest = test_manifest(dir.join("runtime")); + let error = resolve_direct_llama_model(&manifest, "owner/repo") + .unwrap_err() + .to_string(); + assert!(error.contains("`owner/repo` needs an explicit"), "{error}"); + // No quantization is suggested for a repo nothing has looked inside. + assert!(error.contains("`owner/repo:`"), "{error}"); + assert!(!error.contains("Q4_K_M"), "{error}"); + // The same failure through the serve entry point returns the error and + // no route, so nothing is downloaded or started. + let error = serving_route(&manifest, &serve_request("owner/repo"), true) + .unwrap_err() + .to_string(); + assert!(error.contains("`owner/repo:`"), "{error}"); + assert!(!error.contains("Q4_K_M"), "{error}"); + + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let snapshot = paths + .cache_dir + .join("huggingface/hub/models--owner--repo/snapshots/revision"); + fs::create_dir_all(&snapshot).unwrap(); + for file in ["model-Q4_0.gguf", "model-Q4_K_M.gguf"] { + fs::write(snapshot.join(file), b"x").unwrap(); + } + let ambiguous = parse_hf_checkpoint("owner/repo:Q4").unwrap(); + let error = direct_llama_model_path(&paths, &DirectLlamaModel::Checkpoint(ambiguous)) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple files"), "{error}"); + // The example is a real, unambiguous reference: it resolves to one file. + assert!( + error.contains("e.g. `owner/repo:model-Q4_0.gguf`"), + "{error}" + ); + let example = parse_hf_checkpoint("owner/repo:model-Q4_0.gguf").unwrap(); + assert!( + direct_llama_model_path(&paths, &DirectLlamaModel::Checkpoint(example)) + .unwrap() + .is_some() + ); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_complex_registered_models_use_router() { + let dir = scratch_dir("registered-complex"); + let manifest = test_manifest(dir.join("runtime")); + for (key, value) in [ + ("mmproj", serde_json::json!("owner/repo:mmproj.gguf")), + ( + "checkpoints", + serde_json::json!(["owner/repo:Q4_0", "owner/repo:mmproj.gguf"]), + ), + ("extra_files", serde_json::json!(["tokenizer.json"])), + ("options", serde_json::json!({"ctx_size": 8192})), + ("source", serde_json::json!("alternate")), + ("unknown_option", serde_json::json!(true)), + ("recipe_options", serde_json::json!({"ctx_size": 8192})), + ("labels", serde_json::json!(["vision"])), + ("labels", serde_json::json!(["mtp"])), + ("labels", serde_json::json!(["unknown"])), + ("labels", serde_json::json!([123])), + ("labels", serde_json::json!("reasoning")), + ("size", serde_json::json!("0.38")), + ("suggested", serde_json::json!(1)), + ] { + let mut metadata = + serde_json::json!({"recipe": "llamacpp", "checkpoint": "owner/repo:Q4_0"}); + metadata[key] = value; + write_registry(&manifest, serde_json::json!({"Complex-GGUF": metadata})); + assert!( + resolve_direct_llama_model(&manifest, "Complex-GGUF") + .unwrap() + .is_none(), + "{key}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// The chat-model labels the catalog uses do not change how a model is + /// launched, so an entry carrying only those still serves directly. Each label + /// on its own, so dropping one from the allow-list turns this red. + #[test] + fn simple_registered_models_with_chat_labels_serve_directly() { + let dir = scratch_dir("registered-labels"); + let manifest = test_manifest(dir.join("runtime")); + for label in ["reasoning", "tool-calling", "tools"] { + write_registry( + &manifest, + serde_json::json!({"Labelled-GGUF": { + "recipe": "llamacpp", + "checkpoint": "owner/repo:Q4_0", + "labels": [label], + }}), + ); + assert!( + matches!( + resolve_direct_llama_model(&manifest, "Labelled-GGUF").unwrap(), + Some(DirectLlamaModel::Checkpoint(_)) + ), + "{label}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_safe_variantless_registered_model_uses_router() { + let dir = scratch_dir("registered-variantless"); + let manifest = test_manifest(dir.join("runtime")); + write_registry( + &manifest, + serde_json::json!({ + "Variantless": {"recipe": "llamacpp", "checkpoint": "owner/repo"} + }), + ); + assert!( + resolve_direct_llama_model(&manifest, "Variantless") + .unwrap() + .is_none() + ); + assert!(resolve_direct_llama_model(&manifest, "owner/repo").is_err()); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_unusable_catalog_uses_router() { + let dir = scratch_dir("registered-unusable"); + let manifest = test_manifest(dir.join("runtime")); + let catalog = manifest.runtime_dir.join("resources/server_models.json"); + fs::create_dir_all(catalog.parent().unwrap()).unwrap(); + for bytes in [b"[]".as_slice(), b"null".as_slice(), b"invalid".as_slice()] { + fs::write(&catalog, bytes).unwrap(); + assert!( + resolve_direct_llama_model(&manifest, "qwen-smoke") + .unwrap() + .is_none() + ); + } + fs::remove_file(&catalog).unwrap(); + fs::create_dir(&catalog).unwrap(); + assert!( + resolve_direct_llama_model(&manifest, "qwen-smoke") + .unwrap() + .is_none() + ); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_registered_alias_resolves_exact_checkpoint() { + let dir = scratch_dir("registered-alias"); + let manifest = test_manifest(dir.join("runtime")); + write_registry( + &manifest, + serde_json::json!({ + "Qwen3-0.6B-GGUF": {"recipe": "llamacpp", "checkpoint": "unsloth/Qwen3-0.6B-GGUF:Q4_0"}, + "Other-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/Other-GGUF:Q8_0"} + }), + ); + for alias in ["qwen-smoke", "tiny", " QWEN3-0.6B-GGUF "] { + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, alias).unwrap(), + parse_hf_checkpoint("unsloth/Qwen3-0.6B-GGUF:Q4_0") + ); + } + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Other-GGUF").unwrap(), + parse_hf_checkpoint("owner/Other-GGUF:Q8_0") + ); + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Other").unwrap(), + None + ); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_routes_unsafe_registered_metadata_to_the_router() { + let dir = scratch_dir("registered-unsafe"); + let manifest = test_manifest(dir.join("runtime")); + for checkpoint in [ + "../../etc/passwd:Q4_0", + "owner/repo:../secret", + "/tmp/model.gguf", + ] { + write_registry( + &manifest, + serde_json::json!({ + "Unsafe-GGUF": {"recipe": "llamacpp", "checkpoint": checkpoint} + }), + ); + // Never served directly, and never turned into a path: the router + // gets the name, as it did before direct serving existed. + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Unsafe-GGUF").unwrap(), + None, + "{checkpoint}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// Lemonade's own catalog writes `*` for "every file". It is not a safe + /// filename, so direct serving cannot take it, but it is not malformed either: + /// the serve must go to the router, and the fallback must agree. + #[test] + fn a_registered_wildcard_variant_routes_to_the_router_not_an_error() { + let dir = scratch_dir("registered-wildcard"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + write_registry( + &manifest, + serde_json::json!({ + "Wildcard-GGUF": { + "checkpoint": "owner/repo:*", + "recipe": "llamacpp", + "suggested": true, + "labels": ["reasoning"], + "size": 1.0 + } + }), + ); + let binary = manifest + .runtime_dir + .join("bin/llamacpp/vulkan") + .join(platform_binary_name("llama-server")); + fs::create_dir_all(binary.parent().unwrap()).unwrap(); + fs::write(&binary, b"x").unwrap(); + for is_linux in [true, false] { + assert!( + matches!( + serving_route(&manifest, &serve_request("Wildcard-GGUF"), is_linux).unwrap(), + ServeRoute::Router { ref model_ref } if model_ref == "Wildcard-GGUF" + ), + "is_linux {is_linux}" + ); + assert!( + direct_fallback_route(&manifest, &serve_request("Wildcard-GGUF"), is_linux) + .is_none(), + "is_linux {is_linux}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// `size` is part of the single-checkpoint shape: a negative one is not a + /// catalog entry this module understands, and zero is. + #[test] + fn a_registered_entry_with_a_negative_size_stays_with_the_router() { + let dir = scratch_dir("registered-size"); + let manifest = test_manifest(dir.join("runtime")); + for (size, direct) in [(-1.0, false), (0.0, true), (0.38, true)] { + write_registry( + &manifest, + serde_json::json!({ + "Sized-GGUF": { + "checkpoint": "owner/repo:Q4_0", + "recipe": "llamacpp", + "size": size + } + }), + ); + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Sized-GGUF") + .unwrap() + .is_some(), + direct, + "size {size}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// Two different split models with the same shard count are not one set, + /// whatever their indexes: serving the first would leave llama-server looking + /// for shards of a model that is not there. + #[test] + fn shards_of_two_different_models_are_not_a_complete_set() { + let dir = Path::new("/snapshots/rev"); + let paths = |names: [&str; 2]| names.map(|name| dir.join(name)).to_vec(); + assert_eq!( + first_shard_of_complete_set(&paths(["a-00001-of-00002.gguf", "b-00002-of-00002.gguf"])), + None + ); + assert_eq!( + first_shard_of_complete_set(&paths(["a-00002-of-00002.gguf", "a-00001-of-00002.gguf"])), + Some(dir.join("a-00001-of-00002.gguf")) + ); + } + + /// A variant that names a `.gguf` file is an exact filename, not a quantization + /// label: when no file has that name, nothing matches, even though another + /// file's name ends with the same text. + #[test] + fn an_exact_gguf_variant_that_is_absent_does_not_match_a_longer_file_name() { + let ggufs = vec![PathBuf::from("/snapshots/rev/model-Q4_0.gguf")]; + assert_eq!( + filter_ggufs_by_variant(ggufs.clone(), Some("Q4_0.gguf")), + Vec::::new() + ); + // The same text as a label still matches by substring. + assert_eq!(filter_ggufs_by_variant(ggufs.clone(), Some("Q4_0")), ggufs); + } + + #[test] + fn serving_regression_selected_backend_ignores_stale_rocm() { + let dir = scratch_dir("selected-backend"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + let base = manifest.runtime_dir.join("bin/llamacpp"); + for backend in ["rocm-stable", "vulkan", "cpu"] { + fs::create_dir_all(base.join(backend)).unwrap(); + fs::write( + base.join(backend) + .join(platform_binary_name("llama-server")), + b"x", + ) + .unwrap(); + } + assert_eq!( + direct_serve_binary(&manifest), + Some( + base.join("vulkan") + .join(platform_binary_name("llama-server")) + ) + ); + fs::remove_dir_all(base.join("vulkan")).unwrap(); + assert_eq!(direct_serve_binary(&manifest), None); + manifest.backend_name = "cpu".to_owned(); + assert_eq!(direct_serve_binary(&manifest), None); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_registered_recipe_boundaries_and_missing_registry() { + let dir = scratch_dir("registry-boundaries"); + let manifest = test_manifest(dir.join("runtime")); + assert!( + resolve_direct_llama_model(&manifest, "qwen-smoke") + .unwrap() + .is_none() + ); + write_registry( + &manifest, + serde_json::json!({ + "RecipeModel": {"recipe": "oga", "checkpoint": "not/a/gguf"}, + "NoCheckpoint": {"recipe": "llamacpp"}, + "MalformedCheckpoint": {"recipe": "llamacpp", "checkpoint": 123}, + "Exact-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q8_0"} + }), + ); + for name in [ + "RecipeModel", + "Unknown-GGUF", + "exact-gguf", + "Exact", + "../../escape", + "NoCheckpoint", + "MalformedCheckpoint", + ] { + assert!( + resolve_direct_llama_model(&manifest, name) + .unwrap() + .is_none(), + "{name}" + ); + } + assert!(resolve_direct_llama_model(&manifest, "owner/repo").is_err()); + for json in [b"[]".as_slice(), b"invalid".as_slice()] { + fs::write( + manifest.runtime_dir.join("resources/server_models.json"), + json, + ) + .unwrap(); + assert!( + resolve_direct_llama_model(&manifest, "Exact-GGUF") + .unwrap() + .is_none() + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_registered_checkpoint_cache_rejects_ambiguous_variants() { + let dir = scratch_dir("registry-cache"); + let manifest = test_manifest(dir.join("runtime")); + let roots = vec![dir.join("hub-one"), dir.join("hub-two")]; + for root in &roots { + let snapshot = root.join("models--owner--repo/snapshots/revision"); + fs::create_dir_all(&snapshot).unwrap(); + for file in ["model-Q4_0.gguf", "model-Q4_K_M.gguf"] { + fs::write(snapshot.join(file), b"x").unwrap(); + } + // A GGUF suffix on a directory is not a downloaded model file. This + // fixture is handled by the `is_dir()` branch of `collect_gguf_files`; the + // `is_file()` filter is covered by + // `collect_gguf_files_skips_dangling_symlinks`. + fs::create_dir_all(snapshot.join("directory-Q4.gguf")).unwrap(); + } + write_registry( + &manifest, + serde_json::json!({ + "Exact-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:model-Q4_0.gguf"}, + "Ambiguous-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q4"} + }), + ); + let exact = resolve_direct_llama_checkpoint(&manifest, "Exact-GGUF") + .unwrap() + .unwrap(); + let matches = hf_checkpoint_gguf_matches_in_roots(&roots, &exact); + assert_eq!( + matches.len(), + 1, + "same filename across cache roots is one quantization" + ); + assert_eq!(matches[0].file_name().unwrap(), "model-Q4_0.gguf"); + let ambiguous = resolve_direct_llama_checkpoint(&manifest, "Ambiguous-GGUF") + .unwrap() + .unwrap(); + assert_eq!( + hf_checkpoint_gguf_matches_in_roots(&roots, &ambiguous).len(), + 2 + ); + // Path resolution must reject ambiguous matching, not pick sorted first. + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let snapshot = paths + .cache_dir + .join("huggingface/hub/models--owner--repo/snapshots/revision"); + fs::create_dir_all(&snapshot).unwrap(); + for file in ["model-Q4_0.gguf", "model-Q4_K_M.gguf"] { + fs::write(snapshot.join(file), b"x").unwrap(); + } + assert!(direct_llama_model_path(&paths, &DirectLlamaModel::Checkpoint(ambiguous)).is_err()); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_local_gguf_is_explicit_not_registry_path_metadata() { + let dir = scratch_dir("local-gguf"); + let manifest = test_manifest(dir.join("runtime")); + let model = dir.join("model.GGUF"); + fs::write(&model, b"x").unwrap(); + let local = resolve_direct_llama_model(&manifest, model.to_str().unwrap()) + .unwrap() + .unwrap(); + assert!(matches!(local, DirectLlamaModel::Local(ref path) if path == &model)); + write_registry( + &manifest, + serde_json::json!({ + "Path-GGUF": {"recipe": "llamacpp", "checkpoint": model} + }), + ); + // A registry entry never names a local file to serve: it is left to the + // router, not read as a path. + assert!( + resolve_direct_llama_model(&manifest, "Path-GGUF") + .unwrap() + .is_none() + ); + let non_gguf = dir.join("model.txt"); + fs::write(&non_gguf, b"x").unwrap(); + assert!( + resolve_direct_llama_model(&manifest, non_gguf.to_str().unwrap()) + .unwrap() + .is_none() + ); + fs::remove_dir_all(dir).unwrap(); + } + #[test] fn parses_canonical_hugging_face_checkpoints() { assert_eq!( @@ -746,6 +1805,37 @@ mod tests { assert!(parse_hf_checkpoint("owner/repo:").is_none()); } + #[test] + fn checkpoint_matches_prefer_the_highest_priority_cache_root() { + let dir = scratch_dir("root-priority"); + // The higher-priority root sorts after the lower one alphabetically, so a + // merged-then-sorted list would pick the wrong file. + let high = dir.join("zz-high"); + let low = dir.join("aa-low"); + let roots = vec![high.clone(), low.clone()]; + let checkpoint = parse_hf_checkpoint("owner/repo:model-Q4_0.gguf").unwrap(); + let snapshot_file = + |root: &Path| root.join("models--owner--repo/snapshots/revision/model-Q4_0.gguf"); + for root in [&high, &low] { + let file = snapshot_file(root); + fs::create_dir_all(file.parent().unwrap()).unwrap(); + fs::write(&file, b"x").unwrap(); + } + assert_eq!( + hf_checkpoint_gguf_matches_in_roots(&roots, &checkpoint), + vec![snapshot_file(&high)], + "the highest-priority root wins, not the alphabetically first path" + ); + + // A file present only in the lower-priority root is still found. + fs::remove_file(snapshot_file(&high)).unwrap(); + assert_eq!( + hf_checkpoint_gguf_matches_in_roots(&roots, &checkpoint), + vec![snapshot_file(&low)] + ); + fs::remove_dir_all(dir).unwrap(); + } + #[test] fn rejects_path_traversal_in_checkpoint_components() { // A crafted reference must not be able to escape the model cache directory. @@ -757,6 +1847,46 @@ mod tests { assert!(parse_hf_checkpoint("owner/repo:a/b.gguf").is_none()); } + #[test] + fn serving_regression_exact_filename_never_substitutes_suffix_match() { + assert!( + filter_ggufs_by_variant( + vec![PathBuf::from("/cache/other-model.gguf")], + Some("model.gguf") + ) + .is_empty() + ); + } + + #[test] + fn only_a_variantless_checkpoint_needs_a_variant() { + let message = hf_checkpoint_variant_required("owner/repo") + .expect("a bare owner/repo names no GGUF file and cannot be served"); + assert!( + message.contains("`owner/repo:`"), + "the refusal must show the form that would work, got:\n{message}" + ); + // Nothing here has read the repository's file list, so an example + // quantization would be invented — and following invented advice fails a + // second time. + assert!( + !message.contains("Q4_K_M") && !message.contains("Q4_0"), + "the refusal must not suggest a quantization the CLI has not seen, got:\n{message}" + ); + // On Windows this refusal is the user's only sight of the problem, and a + // Windows console on the OEM code page turns non-ASCII punctuation into + // mojibake (ROCMAI-543). + assert!( + message.is_ascii(), + "the refusal must be ASCII-only to survive a Windows console, got:\n{message}" + ); + // A variant was given, so this objection does not apply. + assert_eq!(hf_checkpoint_variant_required("owner/repo:Q4_0"), None); + // Short names are not checkpoints, so this check has no opinion on them. + assert_eq!(hf_checkpoint_variant_required("qwen"), None); + assert_eq!(hf_checkpoint_variant_required(DEFAULT_MODEL), None); + } + #[test] fn selects_gguf_by_variant() { // Real filenames use the quant label without the `-GGUF` repo suffix, so a @@ -766,18 +1896,29 @@ mod tests { PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf"), ]; assert_eq!( - select_gguf_for_variant(ggufs.clone(), Some("q4_0")), + filter_ggufs_by_variant(ggufs.clone(), Some("q4_0")) + .into_iter() + .next(), Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) ); assert_eq!( - select_gguf_for_variant(ggufs.clone(), Some("LFM2.5-230M-Q8_0.gguf")), + filter_ggufs_by_variant(ggufs.clone(), Some("LFM2.5-230M-Q8_0.gguf")) + .into_iter() + .next(), Some(PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf")) ); assert_eq!( - select_gguf_for_variant(ggufs.clone(), None), + filter_ggufs_by_variant(ggufs.clone(), None) + .into_iter() + .next(), Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) ); - assert_eq!(select_gguf_for_variant(ggufs, Some("Q2_K")), None); + assert_eq!( + filter_ggufs_by_variant(ggufs, Some("Q2_K")) + .into_iter() + .next(), + None + ); } #[test] @@ -800,7 +1941,7 @@ mod tests { } #[test] - fn find_llama_server_binary_prefers_gpu_and_skips_cpu() { + fn direct_serve_binary_uses_selected_gpu_backend_and_skips_cpu() { let dir = scratch_dir("find-server"); let runtime_dir = dir.join("runtime"); let llamacpp = runtime_dir.join("bin").join("llamacpp"); @@ -809,10 +1950,11 @@ mod tests { fs::create_dir_all(&backend_dir).unwrap(); fs::write(backend_dir.join(platform_binary_name("llama-server")), b"x").unwrap(); } - let manifest = test_manifest(runtime_dir); + let mut manifest = test_manifest(runtime_dir); + manifest.backend_name = "vulkan".to_owned(); // Both cpu and vulkan exist; the GPU backend is chosen and cpu is never selected. assert_eq!( - find_llama_server_binary(&manifest), + direct_serve_binary(&manifest), Some( llamacpp .join("vulkan") @@ -821,12 +1963,12 @@ mod tests { ); // With only cpu present, nothing is returned (no silent CPU fallback). fs::remove_dir_all(llamacpp.join("vulkan")).unwrap(); - assert_eq!(find_llama_server_binary(&manifest), None); + assert_eq!(direct_serve_binary(&manifest), None); fs::remove_dir_all(&dir).ok(); } #[test] - fn find_llama_server_binary_finds_build_numbered_nesting() { + fn direct_serve_binary_finds_build_numbered_nesting() { // Some Lemonade llama.cpp releases extract one level deeper than // `/llama-server`, e.g. `rocm-stable/llama-b9752/llama-server`. let dir = scratch_dir("find-server-nested"); @@ -837,13 +1979,13 @@ mod tests { let server = nested.join(platform_binary_name("llama-server")); fs::write(&server, b"x").unwrap(); let manifest = test_manifest(runtime_dir); - assert_eq!(find_llama_server_binary(&manifest), Some(server.clone())); + assert_eq!(direct_serve_binary(&manifest), Some(server.clone())); assert_eq!(llama_server_backend_label(&server), "rocm-stable"); fs::remove_dir_all(&dir).ok(); } #[test] - fn find_llama_server_binary_follows_backend_priority() { + fn direct_serve_binary_for_rocm_follows_family_priority() { let dir = scratch_dir("find-server-priority"); let runtime_dir = dir.join("runtime"); let llamacpp = runtime_dir.join("bin").join("llamacpp"); @@ -855,19 +1997,19 @@ mod tests { }; let manifest = test_manifest(runtime_dir); - // Nightly-only host (the resilient case that replaced the hardcoded - // rocm-stable direct-serve path): the nightly backend is selected over vulkan. + // The manifest selects `rocm`. A host whose ROCm build sits in the nightly + // directory still serves from it, and a vulkan install is never used. install("rocm-nightly"); install("vulkan"); assert_eq!( - find_llama_server_binary(&manifest), + direct_serve_binary(&manifest), Some(llamacpp.join("rocm-nightly").join(&server)) ); - // With rocm-stable also present, it wins (highest priority). + // With rocm-stable also present, it wins: it is first in the ROCm family order. install("rocm-stable"); assert_eq!( - find_llama_server_binary(&manifest), + direct_serve_binary(&manifest), Some(llamacpp.join("rocm-stable").join(&server)) ); fs::remove_dir_all(&dir).ok(); @@ -892,6 +2034,476 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Hugging Face snapshots hold symlinks into `blobs/`; one whose blob is gone (an + /// interrupted download) is neither a directory nor a file, so only the `is_file()` + /// check keeps it out of the model list. + #[cfg(unix)] + #[test] + fn collect_gguf_files_skips_dangling_symlinks() { + let dir = scratch_dir("collect-gguf-dangling"); + fs::write(dir.join("real-Q4_0.gguf"), b"x").unwrap(); + std::os::unix::fs::symlink(dir.join("missing-blob"), dir.join("dangling-Q8_0.gguf")) + .unwrap(); + assert!(!dir.join("dangling-Q8_0.gguf").is_dir()); + let mut found = Vec::new(); + collect_gguf_files(&dir, &mut found, 0); + assert_eq!(found, vec![dir.join("real-Q4_0.gguf")]); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn download_uses_validated_checkpoint_ref_not_the_alias_and_returns_the_file() { + let dir = scratch_dir("download-ref"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let checkpoint = parse_hf_checkpoint("pull-test-owner/pull-test-repo:Q4_0").unwrap(); + let model = DirectLlamaModel::Checkpoint(checkpoint); + let snapshot = paths + .cache_dir + .join("huggingface/hub/models--pull-test-owner--pull-test-repo/snapshots/revision"); + let mut pulled = Vec::new(); + let path = ensure_model_with_download(&paths, "qwen-smoke", &model, |reference| { + pulled.push(reference.to_owned()); + fs::create_dir_all(&snapshot).unwrap(); + fs::write(snapshot.join("Qwen3-0.6B-Q4_0.gguf"), b"x").unwrap(); + Ok(()) + }) + .unwrap(); + assert_eq!(pulled, ["pull-test-owner/pull-test-repo:Q4_0"]); + assert_eq!(path, snapshot.join("Qwen3-0.6B-Q4_0.gguf")); + // A cached file is served without pulling again. + let path = ensure_model_with_download(&paths, "qwen-smoke", &model, |_| { + panic!("cached model must not be pulled") + }) + .unwrap(); + assert_eq!(path, snapshot.join("Qwen3-0.6B-Q4_0.gguf")); + fs::remove_dir_all(&dir).ok(); + } + + #[cfg(target_os = "linux")] + #[test] + fn real_download_closure_pulls_the_checkpoint_not_the_requested_alias() { + use crate::test_http::fake_runtime; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + // The fake `lemonade pull` drops the GGUF into the cache, as the real one does. + let manifest = fake_runtime::manifest( + dir, + 0, + Some("models--pull-test-owner--pull-test-repo/snapshots/rev/model-Q4_0.gguf"), + ); + let runtime = LemonadeRuntime { manifest }; + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + // A request whose text is an alias, resolved to a registered checkpoint. + let request = serve_request("qwen-smoke"); + let model = DirectLlamaModel::Checkpoint( + parse_hf_checkpoint("pull-test-owner/pull-test-repo:Q4_0").unwrap(), + ); + let path = ensure_model_in( + &paths, + &request, + &runtime, + &LemonadeProcessEnvironment::default(), + None, + &model, + &ParentRuntimeEnvironment::rooted_at(dir), + ) + .unwrap(); + assert!(path.is_file(), "{}", path.display()); + let pulls = fake_runtime::calls(dir) + .into_iter() + .filter(|call| call.contains(" pull ")) + .collect::>(); + assert_eq!(pulls.len(), 1, "{pulls:?}"); + let argv = pulls[0].split_whitespace().collect::>(); + assert_eq!( + argv[argv.len() - 2..], + ["pull", "pull-test-owner/pull-test-repo:Q4_0"], + "{pulls:?}" + ); + assert!(!pulls[0].contains("qwen-smoke"), "{pulls:?}"); + } + + #[test] + fn split_gguf_variant_resolves_to_its_first_shard() { + let dir = scratch_dir("sharded"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let model = DirectLlamaModel::Checkpoint( + parse_hf_checkpoint("shard-test-owner/shard-test-repo:Q4_K_M").unwrap(), + ); + let folder = paths + .cache_dir + .join("huggingface/hub/models--shard-test-owner--shard-test-repo/snapshots/rev/Q4_K_M"); + fs::create_dir_all(&folder).unwrap(); + // One shard of two: the set is incomplete, so it counts as not downloaded + // (and is pulled again) rather than being served partial. + fs::write(folder.join("model-Q4_K_M-00002-of-00002.gguf"), b"x").unwrap(); + assert_eq!(direct_llama_model_path(&paths, &model).unwrap(), None); + // Both shards present: one model, served from the first shard. + fs::write(folder.join("model-Q4_K_M-00001-of-00002.gguf"), b"x").unwrap(); + assert_eq!( + direct_llama_model_path(&paths, &model).unwrap(), + Some(folder.join("model-Q4_K_M-00001-of-00002.gguf")) + ); + // An unrelated file with the same label makes it ambiguous again. + fs::write(folder.join("other-Q4_K_M.gguf"), b"x").unwrap(); + let error = direct_llama_model_path(&paths, &model) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple files"), "{error}"); + fs::remove_dir_all(&dir).ok(); + } + + fn shard_test_paths(dir: &Path) -> AppPaths { + AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + } + } + + fn shard_checkpoint(variant: &str) -> DirectLlamaModel { + DirectLlamaModel::Checkpoint( + parse_hf_checkpoint(&format!("shard-test-owner/shard-test-repo:{variant}")).unwrap(), + ) + } + + fn shard_snapshot(paths: &AppPaths, revision: &str) -> PathBuf { + let folder = paths.cache_dir.join(format!( + "huggingface/hub/models--shard-test-owner--shard-test-repo/snapshots/{revision}" + )); + fs::create_dir_all(&folder).unwrap(); + folder + } + + #[test] + fn split_set_ambiguity_suggestion_resolves_to_the_whole_set() { + let dir = scratch_dir("shard-suggestion"); + let paths = shard_test_paths(&dir); + let folder = shard_snapshot(&paths, "rev"); + for file in [ + "model-Q4_K_M-00001-of-00002.gguf", + "model-Q4_K_M-00002-of-00002.gguf", + "other-Q4_K_M.gguf", + ] { + fs::write(folder.join(file), b"x").unwrap(); + } + let error = direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple files"), "{error}"); + let suggestion = error + .split_once("e.g. `shard-test-owner/shard-test-repo:") + .and_then(|(_, rest)| rest.split_once('`')) + .map_or_else( + || panic!("no suggestion in {error}"), + |(variant, _)| variant.to_owned(), + ); + assert!(suggestion.contains("-of-"), "{suggestion}"); + // The suggestion, fed back, serves the split model from its first shard. + assert_eq!( + direct_llama_model_path(&paths, &shard_checkpoint(&suggestion)).unwrap(), + Some(folder.join("model-Q4_K_M-00001-of-00002.gguf")) + ); + // A shard name alone never serves a partial set. + fs::remove_file(folder.join("model-Q4_K_M-00002-of-00002.gguf")).unwrap(); + assert_eq!( + direct_llama_model_path(&paths, &shard_checkpoint(&suggestion)).unwrap(), + None + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn shards_spread_over_cache_roots_are_not_a_complete_set() { + let dir = scratch_dir("shard-roots"); + let checkpoint = parse_hf_checkpoint("shard-test-owner/shard-test-repo:Q4_K_M").unwrap(); + let mut roots = Vec::new(); + for (name, shard) in [("h1", 1), ("h2", 2)] { + let root = dir.join(name); + let folder = root.join("models--shard-test-owner--shard-test-repo/snapshots/rev"); + fs::create_dir_all(&folder).unwrap(); + fs::write( + folder.join(format!("model-Q4_K_M-0000{shard}-of-00002.gguf")), + b"x", + ) + .unwrap(); + roots.push(root); + } + let matches = hf_checkpoint_gguf_matches_in_roots(&roots, &checkpoint); + assert_eq!(matches.len(), 2, "{matches:?}"); + assert_eq!(first_shard_of_complete_set(&matches), None); + assert!(is_incomplete_shard_set(&matches)); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn shards_spread_over_snapshot_revisions_are_not_a_complete_set() { + let dir = scratch_dir("shard-revisions"); + let paths = shard_test_paths(&dir); + for (revision, shard) in [("revA", 1), ("revB", 2)] { + fs::write( + shard_snapshot(&paths, revision) + .join(format!("model-Q4_K_M-0000{shard}-of-00002.gguf")), + b"x", + ) + .unwrap(); + } + assert_eq!( + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")).unwrap(), + None + ); + // The same two shards in one revision are served. + let folder = shard_snapshot(&paths, "revA"); + fs::write(folder.join("model-Q4_K_M-00002-of-00002.gguf"), b"x").unwrap(); + assert_eq!( + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")).unwrap(), + Some(folder.join("model-Q4_K_M-00001-of-00002.gguf")) + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn complete_snapshot_is_not_shadowed_by_an_older_partial_download() { + let temporary = tempfile::tempdir().unwrap(); + let paths = shard_test_paths(temporary.path()); + let partial = shard_snapshot(&paths, "aaa"); + let complete = shard_snapshot(&paths, "bbb"); + let first = "model-Q4_K_M-00001-of-00002.gguf"; + fs::write(partial.join(first), b"x").unwrap(); + for file in [first, "model-Q4_K_M-00002-of-00002.gguf"] { + fs::write(complete.join(file), b"x").unwrap(); + } + for variant in ["Q4_K_M", first] { + let model = shard_checkpoint(variant); + let file = ensure_model_with_download(&paths, "req", &model, |_| { + panic!("a complete cached revision must not trigger a pull") + }) + .unwrap(); + assert_eq!(file, complete.join(first)); + } + // A second unrelated model remains ambiguous; preferring a complete + // revision must not turn a broad quantization into an arbitrary choice. + fs::write(complete.join("other-Q4_K_M.gguf"), b"x").unwrap(); + assert!( + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")) + .unwrap_err() + .to_string() + .contains("matches multiple files") + ); + } + + #[test] + fn shard_indexes_outside_the_declared_range_are_not_a_complete_set() { + let dir = scratch_dir("shard-range"); + let paths = shard_test_paths(&dir); + let folder = shard_snapshot(&paths, "rev"); + for file in [ + "model-Q4_K_M-00001-of-00002.gguf", + "model-Q4_K_M-00003-of-00002.gguf", + ] { + fs::write(folder.join(file), b"x").unwrap(); + } + assert_eq!( + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")).unwrap(), + None + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn shard_parsing_requires_the_split_naming_scheme() { + let parse = |name: &str| parse_gguf_shard(Path::new(name)); + assert_eq!( + parse("gpt-oss-120b-Q4_K_M-00001-of-00002.gguf"), + Some(("gpt-oss-120b-Q4_K_M".to_owned(), 1, 2)) + ); + assert_eq!(parse("model-Q4_K_M.gguf"), None); + assert_eq!(parse("model-1-of-2.gguf"), None); + assert_eq!(parse("model-00001-of-00002.bin"), None); + assert_eq!(parse("-00001-of-00002.gguf"), None); + } + + #[test] + fn download_error_is_returned_and_missing_file_after_pull_is_an_error() { + let dir = scratch_dir("download-error"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let model = DirectLlamaModel::Checkpoint( + parse_hf_checkpoint("pull-test-owner/pull-test-repo:model-Q4_0.gguf").unwrap(), + ); + let error = ensure_model_with_download(&paths, "alias", &model, |_| { + anyhow::bail!("Lemonade pull of `pull-test-owner/pull-test-repo:model-Q4_0.gguf` failed with status 7") + }) + .unwrap_err() + .to_string(); + assert!(error.contains("failed with status 7"), "{error}"); + // A pull that exits zero without leaving the file is still an error. + let error = ensure_model_with_download(&paths, "alias", &model, |_| Ok(())) + .unwrap_err() + .to_string(); + assert!(error.contains("did not download `alias`"), "{error}"); + // A missing local GGUF is never pulled. + let local = DirectLlamaModel::Local(dir.join("gone.gguf")); + let error = ensure_model_with_download(&paths, "gone.gguf", &local, |_| { + panic!("a local GGUF has nothing to pull") + }) + .unwrap_err() + .to_string(); + assert!( + error.contains("local GGUF `gone.gguf` is missing"), + "{error}" + ); + fs::remove_dir_all(&dir).ok(); + } + + /// Real entries from the released 11.5.1 `server_models.json`. Other released + /// entries carry labels such as `hot`, `coding` and `mtp`; those are outside + /// the labels direct serving accepts, so they intentionally route to the router. + fn released_catalog_entries() -> serde_json::Value { + serde_json::json!({ + "Qwen3-4B-Instruct-2507-GGUF": { + "checkpoint": "unsloth/Qwen3-4B-Instruct-2507-GGUF:Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": ["tool-calling"], + "size": 2.5 + }, + "Qwen3-0.6B-GGUF": { + "checkpoint": "unsloth/Qwen3-0.6B-GGUF:Q4_0", + "recipe": "llamacpp", + "suggested": true, + "labels": ["reasoning"], + "size": 0.38 + }, + "Qwen3.5-0.8B-GGUF": { + "checkpoint": "unsloth/Qwen3.5-0.8B-GGUF:Qwen3.5-0.8B-UD-Q4_K_XL.gguf", + "mmproj": "mmproj-F16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": ["vision", "tool-calling"], + "size": 0.764 + }, + "gpt-oss-120b-mxfp-GGUF": { + "checkpoint": "ggml-org/gpt-oss-120b-GGUF:*", + "recipe": "llamacpp", + "suggested": true, + "labels": ["reasoning", "tool-calling"], + "size": 63.0 + } + }) + } + + #[test] + fn released_catalog_default_model_routes_direct_with_binary_and_router_without() { + let dir = scratch_dir("released-catalog"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + write_registry(&manifest, released_catalog_entries()); + let binary = manifest + .runtime_dir + .join("bin/llamacpp/vulkan") + .join(platform_binary_name("llama-server")); + + // The engine default is the first released entry. + assert_eq!(DEFAULT_MODEL, "Qwen3-4B-Instruct-2507-GGUF"); + let direct_checkpoints = [ + ( + DEFAULT_MODEL, + "unsloth/Qwen3-4B-Instruct-2507-GGUF:Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + ), + ("Qwen3-0.6B-GGUF", "unsloth/Qwen3-0.6B-GGUF:Q4_0"), + ]; + + // No selected-backend binary: every entry stays on the router. + for name in [DEFAULT_MODEL, "Qwen3-0.6B-GGUF", "Qwen3.5-0.8B-GGUF"] { + assert!( + matches!( + serving_route(&manifest, &serve_request(name), true).unwrap(), + ServeRoute::Router { ref model_ref } if model_ref == name + ), + "{name}" + ); + } + + fs::create_dir_all(binary.parent().unwrap()).unwrap(); + fs::write(&binary, b"x").unwrap(); + for (name, checkpoint) in direct_checkpoints { + let route = serving_route(&manifest, &serve_request(name), true).unwrap(); + let ServeRoute::Direct { model, server } = route else { + panic!("{name} should serve directly with a selected-backend binary"); + }; + assert_eq!(server, binary); + assert!( + matches!(&model, DirectLlamaModel::Checkpoint(found) if found.model_ref() == checkpoint), + "{name}: {model:?}" + ); + } + // The multimodal entry carries `mmproj`, which only the router handles. + assert!(matches!( + serving_route(&manifest, &serve_request("Qwen3.5-0.8B-GGUF"), true).unwrap(), + ServeRoute::Router { model_ref } if model_ref == "Qwen3.5-0.8B-GGUF" + )); + // `ggml-org/gpt-oss-120b-GGUF:*` is how the catalog says "every file". Its + // variant is not a filename, so it is never served directly; with the + // binary present it still goes to the router on Linux and elsewhere, and + // is not an error on either. The fallback after a router failure agrees. + let wildcard = "gpt-oss-120b-mxfp-GGUF"; + for is_linux in [true, false] { + assert!( + matches!( + serving_route(&manifest, &serve_request(wildcard), is_linux).unwrap(), + ServeRoute::Router { ref model_ref } if model_ref == wildcard + ), + "is_linux {is_linux}" + ); + assert!( + direct_fallback_route(&manifest, &serve_request(wildcard), is_linux).is_none(), + "is_linux {is_linux}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn rocm_channel_names_are_not_manifest_backends() { + // Installs record `rocm` or `vulkan`; `rocm-stable` and `rocm-nightly` are only + // directory names inside the `rocm` family, so a manifest naming one finds + // nothing and is refused as an unsupported backend, not as a missing install. + let dir = scratch_dir("channel-names"); + let mut manifest = test_manifest(dir.join("runtime")); + let stable = manifest.runtime_dir.join("bin/llamacpp/rocm-stable"); + fs::create_dir_all(&stable).unwrap(); + fs::write(stable.join(platform_binary_name("llama-server")), b"x").unwrap(); + assert!(direct_serve_binary(&manifest).is_some()); + for name in ["rocm-stable", "rocm-nightly"] { + manifest.backend_name = name.to_owned(); + assert_eq!(direct_serve_binary(&manifest), None, "{name}"); + let error = serving_route(&manifest, &serve_request("owner/repo:Q4_0"), true) + .unwrap_err() + .to_string(); + assert!( + error.contains("does not support GPU direct serving"), + "{error}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest { LemonadeInstallManifest { env_id: "test".to_owned(), @@ -1020,4 +2632,421 @@ mod tests { "no Hugging Face cache directories could be determined" ); } + + /// A stand-in llama-server that stays up until it is signalled. + #[cfg(target_os = "linux")] + fn fake_llama_server(dir: &Path) -> PathBuf { + let server = dir.join("bin/llamacpp/vulkan/llama-server"); + fs::create_dir_all(server.parent().unwrap()).unwrap(); + // The script records its own PID before `exec`: afterwards the process + // is `sleep 30` and its command line no longer names the script, so the + // PID is the only reliable way to find it again. + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nprintf '%s' \"$$\" > \"$(dirname \"$0\")/server.pid\"\nexec sleep 30\n", + ) + .unwrap(); + server + } + + /// The PID the fake server recorded, if it got as far as running its script. + /// Waits briefly for it, so a server that was NOT stopped is seen. + #[cfg(target_os = "linux")] + fn recorded_server_pid(server: &Path) -> Option { + let pid_file = server.parent().unwrap().join("server.pid"); + for _ in 0..40 { + if let Some(pid) = fs::read_to_string(&pid_file) + .ok() + .and_then(|text| text.trim().parse().ok()) + { + return Some(pid); + } + std::thread::sleep(Duration::from_millis(50)); + } + None + } + + /// Kills and reaps a spawned child when dropped, so a failing assertion + /// cannot leave a fake server running. + #[cfg(target_os = "linux")] + struct KillOnDrop(Child); + + #[cfg(target_os = "linux")] + impl Drop for KillOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[cfg(target_os = "linux")] + fn process_exists(pid: u32) -> bool { + Path::new(&format!("/proc/{pid}")).exists() + } + + #[cfg(target_os = "linux")] + #[derive(Clone, Copy, Debug)] + enum DirectFail { + RecordRunning, + NotReady, + RecordReady, + Wait, + } + + #[cfg(target_os = "linux")] + struct FailingSteps(DirectFail); + + #[cfg(target_os = "linux")] + impl DirectServerSteps for FailingSteps { + fn record_running(&mut self, _server_pid: u32) -> Result<()> { + if matches!(self.0, DirectFail::RecordRunning) { + bail!("injected record_running failure"); + } + Ok(()) + } + fn wait_ready(&mut self) -> Result<()> { + if matches!(self.0, DirectFail::NotReady) { + bail!("injected readiness failure"); + } + Ok(()) + } + fn record_ready(&mut self, _server_pid: u32) -> Result<()> { + if matches!(self.0, DirectFail::RecordReady) { + bail!("injected record_ready failure"); + } + Ok(()) + } + fn announce_serving(&mut self) { + // Only a failure after the server became ready (the final wait) may + // follow an announcement; every earlier failure must not announce. + assert!( + matches!(self.0, DirectFail::Wait), + "{:?}: announced serving although a step before ready failed", + self.0 + ); + } + fn wait_exit(&mut self, _child: &mut Child) -> std::io::Result { + Err(std::io::Error::other("injected wait failure")) + } + } + + /// `wait_ready` gates `ready` on a chat completion as well as on + /// `/v1/models`: a server that lists the model but cannot answer a chat + /// request is not serving it. + #[test] + fn a_direct_server_that_lists_the_model_but_fails_its_chat_smoke_test_is_not_ready() { + use crate::test_http::{Reply, spawn_fixture}; + let (port, server) = spawn_fixture(Duration::from_secs(10), |recorded| { + if recorded.path.starts_with("/v1/models") { + Reply { + status: 200, + body: r#"{"data":[{"id":"owner/repo:Q4_0"}]}"#.to_owned(), + last: false, + } + } else { + Reply { + status: 500, + body: "{}".to_owned(), + last: true, + } + } + }); + let dir = scratch_dir("direct-smoke-fails"); + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "owner/repo:Q4_0".to_owned(), + host: "127.0.0.1".to_owned(), + port, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: test_manifest(dir.join("runtime")), + }; + let model_path = dir.join("model.gguf"); + let mut steps = RealDirectServerSteps { + request: &request, + runtime: &runtime, + backend: "vulkan", + model_path: &model_path, + log_path: None, + ready_timeout: Duration::from_secs(5), + }; + let error = steps.wait_ready().unwrap_err().to_string(); + assert!(error.contains("chat-completion smoke test"), "{error}"); + let paths: Vec = server + .join() + .unwrap() + .into_iter() + .map(|recorded| recorded.path) + .collect(); + assert!( + paths + .iter() + .any(|path| path.starts_with("/v1/chat/completions")), + "{paths:?}" + ); + fs::remove_dir_all(dir).ok(); + } + + /// Every step succeeded, so how the server ends decides the record: a clean + /// exit is `stopped` and the serve returns `Ok`, a non-zero one is `failed` and + /// the serve returns the exit status. + #[cfg(target_os = "linux")] + #[test] + fn a_ready_direct_server_that_exits_is_recorded_by_how_it_exited() { + struct ReadySteps; + impl DirectServerSteps for ReadySteps { + fn record_running(&mut self, _server_pid: u32) -> Result<()> { + Ok(()) + } + fn wait_ready(&mut self) -> Result<()> { + Ok(()) + } + fn record_ready(&mut self, _server_pid: u32) -> Result<()> { + Ok(()) + } + fn announce_serving(&mut self) {} + } + for (code, status) in [(0, "stopped"), (1, "failed")] { + let dir = scratch_dir("direct-exit"); + let server = dir.join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + format!("#!/bin/sh\nexit {code}\n"), + ) + .unwrap(); + let state_path = dir.join("state.json"); + fs::write(&state_path, r#"{"status":"ready"}"#).unwrap(); + let mut child = ProcessCommand::new(&server).spawn().unwrap(); + let outcome = supervise_direct_server(&mut child, &state_path, &mut ReadySteps); + if code == 0 { + outcome.unwrap(); + } else { + let error = outcome.unwrap_err().to_string(); + assert!(error.contains("exited with status"), "{error}"); + } + let state: Value = + serde_json::from_str(&fs::read_to_string(&state_path).unwrap()).unwrap(); + assert_eq!(state["status"], status, "exit {code}: {state}"); + fs::remove_dir_all(dir).unwrap(); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_server_step_failure_stops_the_child_and_marks_state_failed() { + for fail in [ + DirectFail::RecordRunning, + DirectFail::NotReady, + DirectFail::RecordReady, + DirectFail::Wait, + ] { + let dir = crate::test_support::scratch_dir("direct-supervise"); + let server = fake_llama_server(&dir); + let state_path = dir.join("state.json"); + fs::write(&state_path, r#"{"status":"running"}"#).unwrap(); + let mut guard = KillOnDrop(ProcessCommand::new(&server).spawn().unwrap()); + let pid = guard.0.id(); + assert!(process_exists(pid), "{fail:?}: fake server did not start"); + let error = supervise_direct_server(&mut guard.0, &state_path, &mut FailingSteps(fail)) + .unwrap_err() + .to_string(); + assert!( + error.contains("injected") || error.contains("waiting"), + "{error}" + ); + assert!( + !process_exists(pid), + "{fail:?}: server {pid} is still running after the failure" + ); + let state = fs::read_to_string(&state_path).unwrap(); + assert!( + state.contains(r#""status": "failed""#) || state.contains(r#""status":"failed""#), + "{fail:?}: {state}" + ); + fs::remove_dir_all(dir).unwrap(); + } + } + + /// A direct serve whose server never becomes ready must not leave a log line + /// claiming Lemonade is serving: the log is where a managed serve is read, and + /// the state next to it says `failed`. + #[cfg(target_os = "linux")] + #[test] + fn a_direct_serve_that_never_becomes_ready_does_not_log_that_it_is_serving() { + let dir = crate::test_support::scratch_dir("direct-not-ready"); + let server = fake_llama_server(&dir); + let model = dir.join("model.gguf"); + fs::write(&model, b"x").unwrap(); + let state_path = dir.join("state.json"); + fs::write(&state_path, r#"{"status":"starting"}"#).unwrap(); + let log_path = dir.join("serve.log"); + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "owner/repo:Q4_0".to_owned(), + host: "127.0.0.1".to_owned(), + // Nothing listens here, so the readiness probe can only time out. + port: 11438, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: state_path.clone(), + log_path: Some(log_path.clone()), + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: test_manifest(dir.join("runtime")), + }; + let error = serve_direct_llama_server_within( + &request, + &runtime, + &LemonadeProcessEnvironment::default(), + &server, + &model, + Some(&log_path), + "test", + &ParentRuntimeEnvironment::rooted_at(&dir), + Duration::from_secs(1), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("did not become ready"), "{error}"); + let still_running = recorded_server_pid(&server) + .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()); + if still_running && let Some(pid) = recorded_server_pid(&server) { + let _ = terminate_pid(pid, true); + } + assert!(!still_running, "the server outlived the failed serve"); + let state = fs::read_to_string(&state_path).unwrap(); + assert!(state.contains("failed"), "{state}"); + let log = fs::read_to_string(&log_path).unwrap(); + assert!(log.contains("Launching Lemonade packaged vulkan"), "{log}"); + assert!(!log.contains("is serving on"), "{log}"); + fs::remove_dir_all(dir).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_server_stops_when_state_cannot_be_written_after_spawn() { + let dir = crate::test_support::scratch_dir("direct-unwritable"); + let server = fake_llama_server(&dir); + let model = dir.join("model.gguf"); + fs::write(&model, b"x").unwrap(); + // A directory at the state path makes the first post-spawn write fail. + let state_path = dir.join("state.json"); + fs::create_dir_all(&state_path).unwrap(); + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "owner/repo:Q4_0".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11437, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path, + log_path: None, + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: test_manifest(dir.join("runtime")), + }; + let result = serve_direct_llama_server( + &request, + &runtime, + &LemonadeProcessEnvironment::default(), + &server, + &model, + None, + "test", + &ParentRuntimeEnvironment::rooted_at(&dir), + ); + // A server that was not stopped runs its script and records its PID; one + // that was stopped first either never got that far or is gone by now. + let still_running = recorded_server_pid(&server) + .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()); + // Clean up before asserting so a failure cannot leave the server behind. + if still_running && let Some(pid) = recorded_server_pid(&server) { + let _ = terminate_pid(pid, true); + } + assert!(result.is_err()); + assert!(!still_running, "the direct server was left running"); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_direct_command_preserves_alias_model_path_and_gpu_backend() { + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "qwen-smoke".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: PathBuf::from("test-state"), + log_path: None, + engine_recipe: Some(EngineRecipeHint { + required_flags: vec!["--ctx-size".to_owned(), "8192".to_owned()], + ..EngineRecipeHint::default() + }), + }; + let server = Path::new("runtime/bin/llamacpp/vulkan/llama-build/llama-server"); + let model = Path::new("cache/model-Q4_0.gguf"); + let command = direct_llama_server_command(&request, server, model); + assert_eq!(command.get_program(), server.as_os_str()); + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args, + [ + "-m", + "cache/model-Q4_0.gguf", + "--host", + "127.0.0.1", + "--port", + "11435", + "--n-gpu-layers", + "999", + "--alias", + "qwen-smoke", + "--ctx-size", + "8192" + ] + ); + assert_eq!(llama_server_backend_label(server), "vulkan"); + assert!( + parse_models_ready(r#"{"data":[{"id":"qwen-smoke"}]}"#, &request.model_ref).unwrap() + ); + assert!( + !parse_models_ready(r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#, &request.model_ref) + .unwrap() + ); + } + + #[test] + fn serving_regression_direct_readiness_requires_requested_alias() { + assert!(parse_models_ready(r#"{"data":[{"id":"qwen-smoke"}]}"#, "qwen-smoke").unwrap()); + assert!( + !parse_models_ready(r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#, "qwen-smoke").unwrap() + ); + // The router deliberately normalizes known aliases, unlike direct serving. + assert!(models_payload_has_ready_model( + &json!({"data": [{ + "id": "Qwen3-0.6B-GGUF", "recipe_options": {"llamacpp_backend": "vulkan"} + }]}), + "qwen-smoke", + "vulkan" + )); + } } diff --git a/engines/lemonade/src/footprint.rs b/engines/lemonade/src/footprint.rs new file mode 100644 index 0000000000..9acdf26ccb --- /dev/null +++ b/engines/lemonade/src/footprint.rs @@ -0,0 +1,1010 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What a Lemonade install costs on disk, and where its trees live. +//! +//! A fresh install is a multi-gigabyte download and a tree several times that +//! size: on a Windows gfx1151 host, Lemonade's `llamacpp:rocm` backend fetched +//! 4.7 GB (Lemonade's own `Total:` line) and the finished tree measured 24.3 GiB, +//! almost all of it the ROCm runtime Lemonade bundles with the backend. Before +//! this module the only thing a user saw was a one-line "Preparing..." and then, +//! from `rocm serve`, up to ten silent minutes. [`plan_install_preflight`] decides +//! whether an install is about to write a new tree, and [`InstallPreflight`] +//! renders what will be installed, where, how big it is, and refuses up front +//! when the drive cannot hold even the download. +//! +//! The second half is the read-only view `apps/rocm` needs to report Lemonade +//! trees (`rocm storage`) and to forget an install record whose tree a runtime +//! uninstall just deleted. The record and the tree layout belong to this crate, +//! so the knowledge of both stays here rather than being re-parsed by the CLI. + +use std::fs; +use std::path::{Path, PathBuf}; + +use rocm_core::{ + AppPaths, format_bytes, insufficient_space_message, runtime_path_is_same_or_inside, +}; + +use crate::install::{ + installed_runtime_version, lemonade_root, lemond_path_in, manifest_path, needs_extraction, + runtime_dir_in, +}; + +/// Download size of an install that fetches the llama.cpp ROCm backend: the +/// embeddable archive plus the backend. Lemonade printed `Total: 4.7 GB` +/// (decimal) for the backend on a Windows gfx1151 host; the embeddable archive +/// adds ~5 MB. +/// +/// An upper bound, never a promise. The backend is chosen after this preflight +/// (Lemonade reports per-GPU support, and `backend_alignment.rs` falls back to +/// `vulkan` where its ROCm build is unsupported), and the figure varies with the +/// GPU family and Lemonade's pins. So it is printed as "up to about" and only +/// ever warns; see [`MIN_INSTALL_BYTES`] for what can refuse. +pub(crate) const FRESH_INSTALL_DOWNLOAD_BYTES: u64 = 4_700_000_000; + +/// Disk space an install that fetches the ROCm backend ends up using: 24.3 GiB +/// measured for the same install, rounded up. An estimate and an upper bound, so +/// it only ever warns (the `rocm_core::disk_space` convention: exact +/// requirements may refuse, estimates only warn). +pub(crate) const FRESH_INSTALL_DISK_BYTES: u64 = 25 * 1024 * 1024 * 1024; + +/// What every Lemonade install needs whichever backend is chosen: the embeddable +/// archive (4.8 MB) plus the tree it extracts (10.6 MiB outside `bin`, measured on +/// the same host), rounded up. The one figure that is certain, so the one thing +/// a refusal may rest on: a drive without this much room cannot hold Lemonade at +/// all, however small a backend it picks. +pub(crate) const MIN_INSTALL_BYTES: u64 = 16 * 1024 * 1024; + +/// What an install is about to do, decided before anything is written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct InstallPreflight { + /// The folder the install writes into (`/runtime`, its downloads...). + pub(crate) target_root: PathBuf, + /// The Lemonade install the record points at, when it really holds one and + /// is a different folder: the install is not going to reuse it. Never set + /// for a record whose folder no longer holds an install, because nothing is + /// installed there to be "not reused". + pub(crate) other_install: Option, + /// Whether `rocm storage` lists `other_install`, so the notice may point at + /// it. Storage lists only the data folder's tree and trees inside registered + /// ROCm installs; a recorded install anywhere else is announced but not + /// listed, and a pointer to it would be false. + pub(crate) other_install_listed: bool, + /// Bytes the install deletes before it extracts: the existing runtime + /// folder when this install replaces it (`--reinstall`, a new Lemonade + /// version, or a missing server binary), whose space is free by the time + /// the download starts. Zero when the tree is kept. + pub(crate) replaced_bytes: u64, + /// Free bytes on the drive holding `target_root`, when known. + pub(crate) available: Option, +} + +/// Outcome of comparing the preflight's free space with the install's size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum SpaceVerdict { + /// Enough room, or free space could not be determined (never block on that). + Fits, + /// Room for the download but likely not for the finished tree. + Tight(String), + /// Not even room for the download: the install cannot finish. + Refuse(String), +} + +/// Whether installing into `env_root` will write a new Lemonade tree, and if so +/// what the user needs to be told. `None` when the tree there is already current +/// and has a llama.cpp backend, so the install only re-checks it. +pub(crate) fn plan_install_preflight( + paths: &AppPaths, + env_root: Option<&Path>, + reinstall: bool, + available_space: impl FnOnce(&Path) -> Option, +) -> Option { + let target_root = lemonade_root(paths, env_root); + let runtime_dir = runtime_dir_in(&target_root); + let extraction = needs_extraction( + reinstall, + installed_runtime_version(&runtime_dir).as_deref(), + rocm_deps::LEMONADE_VERSION, + lemond_path_in(&runtime_dir).is_file(), + ); + let backend_missing = !runtime_dir.join("bin").join("llamacpp").is_dir(); + if !extraction && !backend_missing { + return None; + } + let other_install = recorded_install_dir(paths) + .filter(|recorded| !runtime_path_is_same_or_inside(recorded, &target_root)); + let other_install_listed = other_install + .as_deref() + .is_some_and(|other| storage_lists(paths, other)); + // `prepare_embeddable` removes the whole runtime folder, backends included, + // before it extracts whenever `needs_extraction` says so, so that is the + // space the install frees first. + let replaced_bytes = if extraction && runtime_dir.exists() { + directory_bytes(&runtime_dir) + } else { + 0 + }; + let available = available_space(&target_root); + Some(InstallPreflight { + target_root, + other_install, + other_install_listed, + replaced_bytes, + available, + }) +} + +/// Total size of the files under `dir`, best effort: symlinks are not followed +/// and anything unreadable counts as zero, because this only sizes the space an +/// install is about to free. +fn directory_bytes(dir: &Path) -> u64 { + let mut total = 0_u64; + let mut stack = vec![dir.to_path_buf()]; + while let Some(current) = stack.pop() { + let Ok(entries) = std::fs::read_dir(¤t) else { + continue; + }; + for entry in entries.flatten() { + let Ok(metadata) = entry.metadata() else { + continue; + }; + if metadata.is_dir() && !metadata.is_symlink() { + stack.push(entry.path()); + } else { + total = total.saturating_add(metadata.len()); + } + } + } + total +} + +impl InstallPreflight { + /// The lines printed before the install starts. Every claim is derived from + /// the fields the install itself acts on: the target is where it writes, and + /// `other_install` is set only when the record points somewhere else. + pub(crate) fn notice_lines(&self) -> Vec { + let free = self + .available + .map(|bytes| format!(" ({} free on that drive)", format_bytes(bytes))) + .unwrap_or_default(); + let mut lines = vec![ + format!( + "Installing Lemonade {} into {}", + rocm_deps::LEMONADE_VERSION, + self.target_root.display() + ), + format!( + " With its llama.cpp ROCm backend this is a download of up to about {} and up to about {} of disk space{free}; a smaller backend, if Lemonade picks one for this GPU, needs less. It can take several minutes.", + download_text(), + disk_text(), + ), + ]; + if self.replaced_bytes > 0 { + lines.push(format!( + " The existing Lemonade install here ({}) is replaced first, and that space is counted as free.", + format_bytes(self.replaced_bytes) + )); + } + if let Some(other) = self.other_install.as_ref() { + let pointer = if self.other_install_listed { + " See `rocm storage`." + } else { + "" + }; + lines.push(format!( + " Lemonade is already installed at {}; that copy is not reused for this location.{pointer}", + other.display() + )); + } + lines + } + + /// Whether the drive can hold the install. + /// + /// Refuses only below [`MIN_INSTALL_BYTES`], the one figure that holds for + /// every backend. The ROCm backend download and the finished tree are upper + /// bounds, because the backend is not chosen until after the extraction this + /// runs before, so a host with less room than they need is warned and + /// allowed to try: refusing it could turn away a host whose smaller backend + /// would fit. + pub(crate) fn space_verdict(&self) -> SpaceVerdict { + const OPERATION: &str = "install Lemonade"; + let Some(free_now) = self.available else { + return SpaceVerdict::Fits; + }; + // The install deletes the existing runtime folder before it downloads, + // so its space counts. The warnings say so, so the figure shown is the + // one compared. + let available = free_now.saturating_add(self.replaced_bytes); + let room = if self.replaced_bytes > 0 { + format!( + "{} will be free on the drive holding {} once the existing install there is replaced", + format_bytes(available), + self.target_root.display() + ) + } else { + format!( + "only {} is free on the drive holding {}", + format_bytes(available), + self.target_root.display() + ) + }; + let minimum = rocm_core::with_margin(MIN_INSTALL_BYTES); + if available < minimum { + return SpaceVerdict::Refuse(format!( + "{} Nothing was downloaded.", + insufficient_space_message(OPERATION, &self.target_root, minimum, available) + )); + } + // The margin is working room for the comparison only. What is printed is + // the same figure the notice printed, so one run never shows two numbers + // for the same download. + if available < rocm_core::with_margin(FRESH_INSTALL_DOWNLOAD_BYTES) { + return SpaceVerdict::Tight(format!( + "Warning: {room}, and Lemonade's llama.cpp ROCm backend alone is a download of up to about {} (plus room to unpack it). The install will continue: if Lemonade picks a smaller backend for this GPU it may fit, otherwise it will fail partway through.", + download_text() + )); + } + if available < rocm_core::with_margin(FRESH_INSTALL_DISK_BYTES) { + return SpaceVerdict::Tight(format!( + "Warning: free disk space may be insufficient to install Lemonade's llama.cpp ROCm backend: up to about {} needed (plus room to work in), and {room}. The install will continue, but may fail partway through.", + disk_text() + )); + } + SpaceVerdict::Fits + } +} + +/// The ROCm backend's download as printed everywhere: by the notice and by both +/// warnings, so they cannot disagree. +fn download_text() -> String { + format_bytes(FRESH_INSTALL_DOWNLOAD_BYTES) +} + +/// The finished install's disk use as printed everywhere; see [`download_text`]. +fn disk_text() -> String { + format_bytes(FRESH_INSTALL_DISK_BYTES) +} + +/// Free space on the drive that will hold `path`, with the e2e-only override that +/// lets a black-box scenario reach the refusal without filling a real disk. +pub(crate) fn available_space_for_install(path: &Path) -> Option { + #[cfg(feature = "e2e-test-hooks")] + if let Some(bytes) = std::env::var_os(FREE_SPACE_TEST_ENV) + .and_then(|value| value.to_str().and_then(|value| value.parse::().ok())) + { + return Some(bytes); + } + rocm_core::available_space_for_path(path) +} + +/// Test-only override of the free space [`available_space_for_install`] reports. +#[cfg(feature = "e2e-test-hooks")] +const FREE_SPACE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES"; + +/// The folder a Lemonade install for `env_root` lives in: `/lemonade`, +/// or `/engines/lemonade` when there is none. +pub fn install_tree_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { + lemonade_root(paths, env_root) +} + +/// The parts of a Lemonade tree that make up the install itself. +/// +/// That is the extracted runtime with its backends, the cached archive, and +/// leftover extraction scratch. Deliberately not the whole folder: under +/// `/engines/lemonade` the same folder also holds the install record, logs +/// and server state. +pub fn install_tree_dirs(tree_root: &Path) -> [PathBuf; 3] { + [ + runtime_dir_in(tree_root), + tree_root.join("downloads"), + tree_root.join("extract"), + ] +} + +/// What Lemonade's install record says, as far as the CLI can tell. +/// +/// Three states rather than an `Option`, because they call for different +/// reporting: a record that cannot be read is not the same as no record, and a +/// record whose folder no longer holds an install means nothing is in use. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InstallRecord { + /// No record: Lemonade has never been installed by ROCm CLI, or its record + /// was cleared. + NotRecorded, + /// A record file exists but could not be read or parsed. + Unreadable(String), + /// The record names `runtime_dir`; `holds_install` is whether the server + /// binary is still there. + Recorded { + runtime_dir: PathBuf, + holds_install: bool, + }, +} + +/// Read Lemonade's install record. +pub fn install_record(paths: &AppPaths) -> InstallRecord { + let record = manifest_path(paths); + match fs::read(&record) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => InstallRecord::NotRecorded, + Err(error) => InstallRecord::Unreadable(format!("{}: {error}", record.display())), + Ok(bytes) => { + match serde_json::from_slice::(&bytes) { + Ok(manifest) => InstallRecord::Recorded { + holds_install: lemond_path_in(&manifest.runtime_dir).is_file(), + runtime_dir: manifest.runtime_dir, + }, + Err(error) => InstallRecord::Unreadable(format!("{}: {error}", record.display())), + } + } + } +} + +/// The runtime folder the install record points at, if there is a readable one. +pub fn recorded_runtime_dir(paths: &AppPaths) -> Option { + match install_record(paths) { + InstallRecord::Recorded { runtime_dir, .. } => Some(runtime_dir), + InstallRecord::NotRecorded | InstallRecord::Unreadable(_) => None, + } +} + +/// The runtime folder Lemonade's record points at, only when it still holds an +/// install. +fn recorded_install_dir(paths: &AppPaths) -> Option { + match install_record(paths) { + InstallRecord::Recorded { + runtime_dir, + holds_install: true, + } => Some(runtime_dir), + InstallRecord::Recorded { .. } + | InstallRecord::NotRecorded + | InstallRecord::Unreadable(_) => None, + } +} + +/// The install roots of the ROCm installs registered with ROCm CLI, read from +/// the runtime registry the same way storage finds its runtimes. Best effort: an +/// unreadable entry is skipped. +fn registered_install_roots(paths: &AppPaths) -> Vec { + let registry = paths.data_dir.join("runtimes").join("registry"); + let Ok(entries) = fs::read_dir(registry) else { + return Vec::new(); + }; + entries + .flatten() + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .filter_map(|entry| fs::read(entry.path()).ok()) + .filter_map(|bytes| serde_json::from_slice::(&bytes).ok()) + .filter_map(|record| record.get("install_root")?.as_str().map(PathBuf::from)) + .collect() +} + +/// Whether `rocm storage` lists the Lemonade install at `runtime_dir`: it lists +/// the data folder's tree and the tree inside each registered ROCm install, and +/// nothing else. Both use [`install_tree_root`], so the two cannot drift apart. +fn storage_lists(paths: &AppPaths, runtime_dir: &Path) -> bool { + runtime_path_is_same_or_inside(runtime_dir, &install_tree_root(paths, None)) + || registered_install_roots(paths).iter().any(|root| { + runtime_path_is_same_or_inside( + runtime_dir, + &install_tree_root(paths, Some(&root.join("engines"))), + ) + }) +} + +/// What [`forget_install_record_inside`] did, so a caller can report it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RecordCleanup { + /// The record does not name anything inside the removed folder, or there is + /// no record. Nothing to report. + NotAffected, + /// The record named this runtime folder and has been removed. + Cleared(PathBuf), + /// The record names a folder that was removed, but deleting the record + /// failed: Lemonade still points at files that are gone. + Failed { + runtime_dir: PathBuf, + record: PathBuf, + error: String, + }, + /// The record could not be read, so it could not be compared with the + /// removed folder. + Unreadable(String), +} + +/// Forget the install record when the tree it points at was inside `removed_root`, +/// which the caller has just deleted. +/// +/// Leaving the record would point Lemonade at files that no longer exist. Without +/// it Lemonade reports itself as not installed, which is the true state. Every +/// outcome that leaves the user with something to know is returned rather than +/// printed, so the uninstall report can carry it. +pub fn forget_install_record_inside(paths: &AppPaths, removed_root: &Path) -> RecordCleanup { + forget_install_record_inside_with(paths, removed_root, |record| fs::remove_file(record)) +} + +/// [`forget_install_record_inside`] with the deletion injectable, so the failure +/// outcome can be tested without a file the OS refuses to delete. +fn forget_install_record_inside_with( + paths: &AppPaths, + removed_root: &Path, + remove_record: impl FnOnce(&Path) -> std::io::Result<()>, +) -> RecordCleanup { + let runtime_dir = match install_record(paths) { + InstallRecord::NotRecorded => return RecordCleanup::NotAffected, + InstallRecord::Unreadable(error) => return RecordCleanup::Unreadable(error), + InstallRecord::Recorded { runtime_dir, .. } => runtime_dir, + }; + if !runtime_path_is_same_or_inside(&runtime_dir, removed_root) { + return RecordCleanup::NotAffected; + } + let record = manifest_path(paths); + match remove_record(&record) { + Ok(()) => RecordCleanup::Cleared(runtime_dir), + Err(error) => RecordCleanup::Failed { + runtime_dir, + record, + error: error.to_string(), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::install::{LemonadeInstallManifest, write_manifest}; + + fn paths_in(root: &Path) -> AppPaths { + AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + } + } + + /// Plant an extracted tree the way `prepare_embeddable` leaves one: the + /// server binary, the version marker, and a llama.cpp backend folder. + fn plant_current_tree(tree_root: &Path) { + let runtime_dir = runtime_dir_in(tree_root); + fs::create_dir_all(runtime_dir.join("bin").join("llamacpp").join("rocm-stable")).unwrap(); + fs::write(lemond_path_in(&runtime_dir), b"").unwrap(); + crate::install::record_runtime_version(&runtime_dir, rocm_deps::LEMONADE_VERSION).unwrap(); + } + + fn record_install_at(paths: &AppPaths, runtime_dir: &Path) { + write_manifest( + paths, + &LemonadeInstallManifest { + env_id: "lemonade-embeddable-test".to_owned(), + version: rocm_deps::LEMONADE_VERSION.to_owned(), + runtime_dir: runtime_dir.to_path_buf(), + lemond: runtime_dir.join("lemond"), + lemonade: runtime_dir.join("lemonade"), + backend_recipe: "llamacpp".to_owned(), + backend_name: "rocm".to_owned(), + installed_at_unix_ms: 0, + }, + ) + .unwrap(); + } + + const GIB: u64 = 1024 * 1024 * 1024; + + /// The release-test sequence: Lemonade installed into the data folder, then + /// an SDK runtime activated, so the next install targets the runtime's own + /// folder. The notice must name the new folder and the copy it is not reusing. + #[test] + fn a_new_location_is_announced_with_the_install_it_does_not_reuse() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + record_install_at(&paths, &runtime_dir_in(&base)); + let runtime_engines = temp.path().join("runtimes").join("key").join("engines"); + + let preflight = + plan_install_preflight(&paths, Some(&runtime_engines), false, |_| Some(100 * GIB)) + .expect("a folder with no Lemonade in it needs a full install"); + + assert_eq!(preflight.target_root, runtime_engines.join("lemonade")); + assert_eq!(preflight.other_install, Some(runtime_dir_in(&base))); + assert!( + preflight.other_install_listed, + "the data folder's tree is listed" + ); + let notice = preflight.notice_lines().join("\n"); + assert!( + notice.contains(&preflight.target_root.display().to_string()), + "{notice}" + ); + assert!( + notice.contains( + "a download of up to about 4.4 GiB and up to about 25.0 GiB of disk space" + ), + "{notice}" + ); + assert!( + notice.contains("a smaller backend, if Lemonade picks one for this GPU, needs less"), + "{notice}" + ); + assert!(notice.contains("100.0 GiB free on that drive"), "{notice}"); + assert!( + notice.contains(&format!( + "Lemonade is already installed at {}; that copy is not reused", + runtime_dir_in(&base).display() + )), + "{notice}" + ); + assert_eq!(preflight.space_verdict(), SpaceVerdict::Fits); + } + + /// A current tree with a backend is not a new install: nothing to announce, + /// so a plain `rocm serve` on a ready host prints nothing extra. + #[test] + fn a_current_tree_is_not_announced() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + + assert_eq!(plan_install_preflight(&paths, None, false, |_| None), None); + // `--reinstall` wipes and re-extracts the same tree, so it is announced, + // and the record pointing into that same tree is not "another" install. + record_install_at(&paths, &runtime_dir_in(&base)); + let reinstall = plan_install_preflight(&paths, None, true, |_| None).unwrap(); + assert_eq!(reinstall.other_install, None); + assert!( + !reinstall + .notice_lines() + .join("\n") + .contains("already installed"), + "the tree being reinstalled must not be reported as a second copy" + ); + } + + #[test] + fn a_tree_without_a_backend_is_announced() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + fs::remove_dir_all(runtime_dir_in(&base).join("bin")).unwrap(); + + assert!(plan_install_preflight(&paths, None, false, |_| None).is_some()); + } + + /// A record whose folder lost its install (a runtime uninstalled by a build + /// that left the record behind) names nothing that is installed, so the + /// notice must not claim a copy "is already installed" or "not reused". + #[test] + fn a_record_naming_a_folder_with_no_install_is_not_announced_as_a_copy() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let gone = temp.path().join("runtimes").join("removed").join("engines"); + record_install_at( + &paths, + &runtime_dir_in(&install_tree_root(&paths, Some(&gone))), + ); + assert!( + matches!( + install_record(&paths), + InstallRecord::Recorded { + holds_install: false, + .. + } + ), + "the fixture must be a dangling record" + ); + + let preflight = plan_install_preflight(&paths, None, false, |_| Some(100 * GIB)) + .expect("an empty data folder needs a full install"); + + assert_eq!(preflight.other_install, None); + let notice = preflight.notice_lines().join("\n"); + assert!(!notice.contains("already installed"), "{notice}"); + assert!(!notice.contains("not reused"), "{notice}"); + } + + fn preflight_with(available: Option, replaced_bytes: u64) -> InstallPreflight { + InstallPreflight { + target_root: PathBuf::from("/data/engines/lemonade"), + other_install: None, + other_install_listed: false, + replaced_bytes, + available, + } + } + + /// Only the one figure that holds for every backend can refuse: the backend + /// is chosen after this runs, and a host Lemonade gives a smaller backend + /// must not be turned away for a download it does not need. + #[test] + fn only_space_below_the_smallest_install_refuses_and_less_than_the_rocm_backend_warns() { + const MIB: u64 = 1024 * 1024; + let preflight = |available| preflight_with(available, 0); + let SpaceVerdict::Refuse(message) = preflight(Some(2 * MIB)).space_verdict() else { + panic!("2 MiB cannot hold the Lemonade archive and its extracted files"); + }; + assert!( + message.contains("not enough free disk space to install Lemonade:"), + "{message}" + ); + assert!(message.contains("Nothing was downloaded."), "{message}"); + + // Room for Lemonade itself but not for the ROCm backend download: warned + // and allowed, because a smaller backend may still fit. + for available in [200 * MIB, 2 * GIB] { + let SpaceVerdict::Tight(warning) = preflight(Some(available)).space_verdict() else { + panic!("{available} bytes is below the ROCm backend download: warn, not refuse"); + }; + assert!( + warning.contains("if Lemonade picks a smaller backend for this GPU it may fit"), + "{warning}" + ); + } + // Room for the download but not the finished tree. + let SpaceVerdict::Tight(warning) = preflight(Some(10 * GIB)).space_verdict() else { + panic!("10 GiB is below the finished install: warn"); + }; + assert!(!warning.contains("smaller backend"), "{warning}"); + assert_eq!( + preflight(Some(40 * GIB)).space_verdict(), + SpaceVerdict::Fits + ); + assert_eq!(preflight(None).space_verdict(), SpaceVerdict::Fits); + } + + /// The notice and both warnings describe the same download and the same + /// disk use, so they must print the same figures. The margin is for the + /// comparison only: with it printed, the notice said 4.4 GiB and the warning + /// about 4.8 GiB for the same backend. + #[test] + fn the_notice_and_the_warnings_print_the_same_download_and_disk_figures() { + let notice = preflight_with(Some(100 * GIB), 0).notice_lines().join("\n"); + let SpaceVerdict::Tight(download_warning) = + preflight_with(Some(2 * GIB), 0).space_verdict() + else { + panic!("2 GiB is below the ROCm backend download"); + }; + let SpaceVerdict::Tight(disk_warning) = preflight_with(Some(10 * GIB), 0).space_verdict() + else { + panic!("10 GiB is below the finished install"); + }; + + // Derived from the constants, not retyped: if the notice or a warning + // ever prints a different number, one of these fails. + let download = format_bytes(FRESH_INSTALL_DOWNLOAD_BYTES); + let disk = format_bytes(FRESH_INSTALL_DISK_BYTES); + assert!( + notice.contains(&format!("a download of up to about {download}")), + "{notice}" + ); + assert!( + notice.contains(&format!("up to about {disk} of disk space")), + "{notice}" + ); + assert!( + download_warning.contains(&format!("a download of up to about {download}")), + "{download_warning}" + ); + assert!( + disk_warning.contains(&format!("up to about {disk} needed")), + "{disk_warning}" + ); + // And the margin-inflated values are not what is shown. + let inflated_download = format_bytes(rocm_core::with_margin(FRESH_INSTALL_DOWNLOAD_BYTES)); + let inflated_disk = format_bytes(rocm_core::with_margin(FRESH_INSTALL_DISK_BYTES)); + assert_ne!(inflated_download, download); + assert_ne!(inflated_disk, disk); + assert!( + !download_warning.contains(&inflated_download), + "{download_warning}" + ); + assert!(!disk_warning.contains(&inflated_disk), "{disk_warning}"); + // The figures themselves, so a constant change is a visible edit here too. + assert_eq!((download.as_str(), disk.as_str()), ("4.4 GiB", "25.0 GiB")); + } + + /// An install that replaces the existing runtime folder deletes it before + /// downloading (`prepare_embeddable` removes the whole folder, backends + /// included), so its space is free by then. The preflight sizes that folder, + /// counts it, and says so; an install that keeps the tree frees nothing. + #[test] + fn a_reinstall_counts_the_space_of_the_install_it_replaces() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let tree = install_tree_root(&paths, None); + plant_current_tree(&tree); + let payload = runtime_dir_in(&tree) + .join("bin") + .join("llamacpp") + .join("rocm-stable"); + fs::write(payload.join("payload.bin"), vec![0_u8; 123_456]).unwrap(); + + let reinstall = plan_install_preflight(&paths, None, true, |_| Some(2 * GIB)).unwrap(); + assert!( + reinstall.replaced_bytes >= 123_456, + "{}", + reinstall.replaced_bytes + ); + let notice = reinstall.notice_lines().join("\n"); + assert!( + notice.contains(&format!( + "The existing Lemonade install here ({}) is replaced first, and that space is counted as free.", + format_bytes(reinstall.replaced_bytes) + )), + "{notice}" + ); + + // A current tree whose backend is missing is topped up, not wiped. + fs::remove_dir_all(runtime_dir_in(&tree).join("bin")).unwrap(); + let kept = plan_install_preflight(&paths, None, false, |_| Some(2 * GIB)).unwrap(); + assert_eq!(kept.replaced_bytes, 0); + assert!(!kept.notice_lines().join("\n").contains("replaced first")); + + // The verdict counts it: 2 GiB free warns for the ROCm backend, but the + // same drive with a 24 GiB install about to be deleted does not. + assert!(matches!( + preflight_with(Some(2 * GIB), 0).space_verdict(), + SpaceVerdict::Tight(_) + )); + assert_eq!( + preflight_with(Some(2 * GIB), 30 * GIB).space_verdict(), + SpaceVerdict::Fits + ); + // When it still does not fit, the warning shows the figure compared. + let SpaceVerdict::Tight(warning) = preflight_with(Some(GIB), 2 * GIB).space_verdict() + else { + panic!("3 GiB is below the ROCm backend download"); + }; + assert!( + warning.contains("3.0 GiB will be free on the drive holding /data/engines/lemonade once the existing install there is replaced"), + "{warning}" + ); + } + + /// The notice points at `rocm storage` only for a copy storage lists: the + /// data folder's tree and trees inside registered ROCm installs. A recorded + /// install anywhere else is announced without the pointer, which would be + /// false for it. + #[test] + fn the_storage_pointer_is_only_given_for_a_copy_storage_lists() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let target = temp.path().join("target").join("engines"); + let notice_for = |paths: &AppPaths| { + let preflight = plan_install_preflight(paths, Some(&target), false, |_| None).unwrap(); + ( + preflight.other_install_listed, + preflight.notice_lines().join("\n"), + ) + }; + + // 1. In the data folder: listed. + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + record_install_at(&paths, &runtime_dir_in(&base)); + let (listed, notice) = notice_for(&paths); + assert!( + listed + && notice + .contains("that copy is not reused for this location. See `rocm storage`."), + "{notice}" + ); + + // 2. Inside a registered ROCm install: listed. + let registered = temp.path().join("runtimes").join("registered"); + let registry = paths.data_dir.join("runtimes").join("registry"); + fs::create_dir_all(®istry).unwrap(); + fs::write( + registry.join("registered.json"), + serde_json::json!({ "runtime_key": "registered", "install_root": registered }) + .to_string(), + ) + .unwrap(); + let in_runtime = install_tree_root(&paths, Some(®istered.join("engines"))); + plant_current_tree(&in_runtime); + record_install_at(&paths, &runtime_dir_in(&in_runtime)); + let (listed, notice) = notice_for(&paths); + assert!(listed && notice.contains("See `rocm storage`."), "{notice}"); + + // 3. Anywhere else: announced, not pointed at. + let elsewhere = + install_tree_root(&paths, Some(&temp.path().join("custom").join("engines"))); + plant_current_tree(&elsewhere); + record_install_at(&paths, &runtime_dir_in(&elsewhere)); + let (listed, notice) = notice_for(&paths); + assert!(!listed, "{notice}"); + assert!( + notice.contains(&format!( + "Lemonade is already installed at {}", + runtime_dir_in(&elsewhere).display() + )), + "{notice}" + ); + assert!(!notice.contains("rocm storage"), "{notice}"); + } + + /// A runtime uninstall deletes the runtime's folder, Lemonade tree included. + /// The record pointing into it is forgotten; one pointing elsewhere is kept. + #[test] + fn the_record_is_forgotten_only_when_its_tree_was_removed() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let runtime_root = temp.path().join("runtimes").join("key"); + let per_runtime = install_tree_root(&paths, Some(&runtime_root.join("engines"))); + record_install_at(&paths, &runtime_dir_in(&per_runtime)); + + let other_runtime = temp.path().join("runtimes").join("other"); + assert_eq!( + forget_install_record_inside(&paths, &other_runtime), + RecordCleanup::NotAffected + ); + assert!(recorded_runtime_dir(&paths).is_some()); + + assert_eq!( + forget_install_record_inside(&paths, &runtime_root), + RecordCleanup::Cleared(runtime_dir_in(&per_runtime)) + ); + assert_eq!(recorded_runtime_dir(&paths), None); + assert!(!manifest_path(&paths).exists()); + // Nothing recorded: nothing to forget. + assert_eq!( + forget_install_record_inside(&paths, &runtime_root), + RecordCleanup::NotAffected + ); + } + + /// The folder is already gone when the record is cleared, so a failed + /// deletion leaves Lemonade pointing at nothing. That has to come back to + /// the caller with the record's path, not vanish into a log line. + #[test] + fn a_record_that_cannot_be_deleted_is_reported_not_swallowed() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let runtime_root = temp.path().join("runtimes").join("key"); + let per_runtime = install_tree_root(&paths, Some(&runtime_root.join("engines"))); + record_install_at(&paths, &runtime_dir_in(&per_runtime)); + + let outcome = forget_install_record_inside_with(&paths, &runtime_root, |_| { + Err(std::io::Error::other("sharing violation")) + }); + + assert_eq!( + outcome, + RecordCleanup::Failed { + runtime_dir: runtime_dir_in(&per_runtime), + record: manifest_path(&paths), + error: "sharing violation".to_owned(), + } + ); + assert!(manifest_path(&paths).exists(), "the record was not deleted"); + + // The uninstall report tells the user to delete that record. Doing so must + // clear the condition: Lemonade is then simply not recorded. + fs::remove_file(manifest_path(&paths)).unwrap(); + assert_eq!(install_record(&paths), InstallRecord::NotRecorded); + assert_eq!(recorded_runtime_dir(&paths), None); + } + + /// An unparseable record is not "no record": it may well name the removed + /// folder, and the caller must be told it could not be checked. + #[test] + fn an_unreadable_record_is_distinct_from_no_record() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + assert_eq!(install_record(&paths), InstallRecord::NotRecorded); + + let record = manifest_path(&paths); + fs::create_dir_all(record.parent().unwrap()).unwrap(); + fs::write(&record, b"{ not json").unwrap(); + + let InstallRecord::Unreadable(error) = install_record(&paths) else { + panic!("a corrupt record must not read as NotRecorded"); + }; + assert!(error.contains("runtime.json"), "{error}"); + assert_eq!(recorded_runtime_dir(&paths), None); + assert!(matches!( + forget_install_record_inside(&paths, temp.path()), + RecordCleanup::Unreadable(_) + )); + assert!(record.exists(), "an unreadable record must be left alone"); + } + + /// A record whose runtime folder lost its server binary names an install + /// that is not there; `holds_install` is what lets the storage report say so. + #[test] + fn a_record_reports_whether_its_folder_still_holds_an_install() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let tree = install_tree_root(&paths, None); + record_install_at(&paths, &runtime_dir_in(&tree)); + assert_eq!( + install_record(&paths), + InstallRecord::Recorded { + runtime_dir: runtime_dir_in(&tree), + holds_install: false + } + ); + plant_current_tree(&tree); + assert_eq!( + install_record(&paths), + InstallRecord::Recorded { + runtime_dir: runtime_dir_in(&tree), + holds_install: true + } + ); + } + + /// `rocm runtimes uninstall` tells a user whose install record cannot be read + /// to run a plain `rocm engines install lemonade`. That is only true if the + /// install path overwrites the record without reading it and keeps the + /// current tree, so this runs the real record-writing sequence + /// (`install_and_record`, with nothing to download) over a corrupt record and + /// asserts the state it leaves: a readable record naming an install that is + /// there, and the tree untouched (a `--reinstall` would have wiped it). + #[test] + fn a_plain_install_replaces_a_corrupt_record_and_keeps_the_current_tree() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let tree = install_tree_root(&paths, None); + plant_current_tree(&tree); + let runtime_dir = runtime_dir_in(&tree); + fs::write( + crate::install::lemonade_path_in(&runtime_dir), + b"fake lemonade", + ) + .unwrap(); + let sentinel = runtime_dir + .join("bin") + .join("llamacpp") + .join("backend.sentinel"); + fs::write(&sentinel, b"a backend already installed").unwrap(); + + let record = crate::install::manifest_path(&paths); + fs::create_dir_all(record.parent().unwrap()).unwrap(); + fs::write(&record, b"{ not json").unwrap(); + assert!(matches!( + install_record(&paths), + InstallRecord::Unreadable(_) + )); + + let archive = b"never parsed: the tree is current, so nothing is extracted"; + let sha256 = format!("{:x}", ::digest(archive)); + let (manifest, _) = crate::install_and_record( + &paths, + None, + false, + |paths, env_root, reinstall| { + crate::install::prepare_embeddable_with( + paths, + env_root, + reinstall, + rocm_deps::LEMONADE_VERSION, + &sha256, + |_, destination| { + fs::write(destination, archive)?; + Ok(()) + }, + ) + }, + |_, _| Ok(None), + ) + .expect("a plain install over a corrupt record succeeds"); + + assert_eq!( + install_record(&paths), + InstallRecord::Recorded { + runtime_dir: runtime_dir.clone(), + holds_install: true + }, + "the corrupt record was not replaced" + ); + assert_eq!(manifest.runtime_dir, runtime_dir); + assert!( + sentinel.is_file(), + "a plain install must keep the current tree" + ); + } +} diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index e0fa931d11..b6617fc215 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -68,7 +68,7 @@ pub(crate) struct LemonadeInstallManifest { /// /// Pure so the upgrade case is testable without a real archive. `installed` /// is the version recorded for the tree already on disk, if any. -fn needs_extraction( +pub(crate) fn needs_extraction( reinstall: bool, installed: Option<&str>, wanted: &str, @@ -78,7 +78,7 @@ fn needs_extraction( } /// The Lemonade runtime version last extracted into `runtime_dir`, if any. -fn installed_runtime_version(runtime_dir: &Path) -> Option { +pub(crate) fn installed_runtime_version(runtime_dir: &Path) -> Option { fs::read_to_string(runtime_dir.join(RUNTIME_VERSION_MARKER)) .ok() .map(|s| s.trim().to_owned()) @@ -89,7 +89,7 @@ fn installed_runtime_version(runtime_dir: &Path) -> Option { /// part of the same wipe that removes the tree it describes — "marker present /// implies extraction completed" holds by construction, with no separate /// bookkeeping to keep in sync. -fn record_runtime_version(runtime_dir: &Path, version: &str) -> Result<()> { +pub(crate) fn record_runtime_version(runtime_dir: &Path, version: &str) -> Result<()> { fs::write(runtime_dir.join(RUNTIME_VERSION_MARKER), version).with_context(|| { format!( "failed to record installed Lemonade runtime version under {}", @@ -117,7 +117,7 @@ pub(crate) fn prepare_embeddable( /// sha256, and the download step all injectable — the seam a test uses to /// exercise the real decision of which source `installed_version` trusts, /// rather than reimplementing that decision inline. -fn prepare_embeddable_with( +pub(crate) fn prepare_embeddable_with( paths: &AppPaths, env_root: Option<&Path>, reinstall: bool, @@ -195,18 +195,18 @@ where }) } -fn manifest_path(paths: &AppPaths) -> PathBuf { - paths.engine_manifests_dir(ENGINE_NAME).join("runtime.json") +pub(crate) fn manifest_path(paths: &AppPaths) -> PathBuf { + paths.engine_install_manifest_path(ENGINE_NAME) } -fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { +pub(crate) fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { env_root.map_or_else( || paths.engine_dir(ENGINE_NAME), |root| normalize_runtime_path_for_host(root).join(ENGINE_NAME), ) } -fn runtime_dir_in(root: &Path) -> PathBuf { +pub(crate) fn runtime_dir_in(root: &Path) -> PathBuf { root.join("runtime") } @@ -364,7 +364,7 @@ fn restore_packaged_backend_versions_with( // Missing (or not JSON): there is nothing of the user's to keep. None => packaged, }; - rocm_core::write_atomic(&target, &restored, "pins") + write_pin_file(&target, &restored) .with_context(|| format!("failed to restore {}", target.display()))?; Ok(Some(PinFileBackup { target, @@ -372,6 +372,28 @@ fn restore_packaged_backend_versions_with( })) } +/// Replaces the pin file at `target` through +/// `rocm_core::atomic_write::write_file_atomically`, but only inside a +/// directory that is still there. +/// +/// The shared writer creates a missing parent directory. For this file that +/// would rebuild `resources/` in a runtime tree that has been removed, leaving +/// a directory holding nothing but a pin file that looks like the start of an +/// install. The restore and the put-back both run after a failed install, when +/// the tree may be gone, so a missing parent stays a failure, as it was before +/// the shared writer took over. +fn write_pin_file(target: &Path, bytes: &[u8]) -> Result<()> { + if let Some(parent) = target.parent() + && !parent.is_dir() + { + bail!( + "{} does not exist; not recreating it to hold the pin file", + parent.display() + ); + } + rocm_core::atomic_write::write_file_atomically(target, bytes) +} + /// The two entries of `resources/backend_versions.json` that backend alignment /// writes: the ROCm version a backend is built for and the llama.cpp build tag. const ALIGNMENT_PIN_KEYS: [(&str, &str); 2] = [("therock", "version"), ("llamacpp", "rocm-stable")]; @@ -416,7 +438,7 @@ fn set_or_remove_pin( /// What `resources/backend_versions.json` held before /// [`restore_packaged_backend_versions`] rewrote it. Written back through -/// `rocm_core::write_atomic`, like the restore itself. +/// `rocm_core::atomic_write::write_file_atomically`, like the restore itself. /// /// A restore is attempted after any failed install, including a transient /// network or disk error, so a restore whose retry fails too must not leave a @@ -439,7 +461,7 @@ impl PinFileBackup { /// Writes the previous bytes back, or removes the file if there were none. pub(crate) fn put_back(self) -> Result<()> { match self.previous { - Some(bytes) => rocm_core::write_atomic(&self.target, &bytes, "pins") + Some(bytes) => write_pin_file(&self.target, &bytes) .with_context(|| format!("failed to put back {}", self.target.display())), None => match fs::remove_file(&self.target) { Err(err) if err.kind() != std::io::ErrorKind::NotFound => { @@ -451,11 +473,11 @@ impl PinFileBackup { } } -fn lemond_path_in(runtime_dir: &Path) -> PathBuf { +pub(crate) fn lemond_path_in(runtime_dir: &Path) -> PathBuf { runtime_dir.join(platform_binary_name("lemond")) } -fn lemonade_path_in(runtime_dir: &Path) -> PathBuf { +pub(crate) fn lemonade_path_in(runtime_dir: &Path) -> PathBuf { runtime_dir.join(platform_binary_name("lemonade")) } @@ -567,12 +589,50 @@ fn collect_embeddable_roots( #[cfg(test)] mod tests { use super::*; - use crate::test_support::scratch_dir; + use crate::test_support::{retired_pin_temp_path, scratch_dir}; + use rocm_core::foreign_temp_probe::assert_write_spares_foreign_temp_files; use sha2::{Digest, Sha256}; use std::io::Write; #[cfg(windows)] use std::process::{Command as ProcessCommand, Stdio}; + /// The e2e harness reads this manifest back to describe a failed serve + /// (`tests/e2e-cucumber/src/lemonade_diagnostics.rs`), by the same path + /// helper and the field names asserted here. A rename on either side would + /// otherwise show up only as `unavailable:` in a report from the real lane. + #[test] + fn written_manifest_is_where_and_how_the_e2e_failure_report_reads_it() { + let dir = scratch_dir("manifest-path-and-fields"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let manifest = LemonadeInstallManifest { + env_id: "lemonade-embeddable-fixture".to_owned(), + version: "fixture".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("runtime/lemond"), + lemonade: dir.join("runtime/lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: ROCM_BACKEND_NAME.to_owned(), + installed_at_unix_ms: 1, + }; + write_manifest(&paths, &manifest).unwrap(); + + let path = paths.engine_install_manifest_path("lemonade"); + assert_eq!(manifest_path(&paths), path); + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(written["version"], "fixture"); + assert_eq!(written["backend_recipe"], LLAMACPP_RECIPE); + assert_eq!(written["backend_name"], ROCM_BACKEND_NAME); + assert_eq!( + written["runtime_dir"], + serde_json::json!(manifest.runtime_dir) + ); + fs::remove_dir_all(&dir).ok(); + } + #[test] fn embeddable_package_matches_runtime_os() { let version = rocm_deps::LEMONADE_VERSION; @@ -1060,40 +1120,64 @@ mod tests { ) } + /// A runtime tree holding a pin file, with a verified embeddable archive + /// cached that ships its own packaged pin file, ready to be restored over. + struct RestoreFixture { + dir: PathBuf, + pins: PathBuf, + manifest: LemonadeInstallManifest, + sha256: String, + } + + impl RestoreFixture { + fn new(tag: &str, packaged: &str, current: &str) -> Self { + let dir = scratch_dir(tag); + let root = dir.join("lemonade"); + let runtime_dir = runtime_dir_in(&root); + fs::create_dir_all(runtime_dir.join("resources")).unwrap(); + let pins = runtime_dir.join("resources/backend_versions.json"); + fs::write(&pins, current).unwrap(); + let source = dir.join("source-archive"); + write_embeddable_with_pins(&source, packaged); + let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); + let manifest = LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "11.5.1".to_owned(), + runtime_dir: runtime_dir.clone(), + lemond: lemond_path_in(&runtime_dir), + lemonade: lemonade_path_in(&runtime_dir), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: ROCM_BACKEND_NAME.to_owned(), + installed_at_unix_ms: 0, + }; + cache_archive(&root, &source); + Self { + dir, + pins, + manifest, + sha256, + } + } + + fn restore(&self) -> Result> { + restore_packaged_backend_versions_with( + &self.manifest, + "resources/backend_versions.json", + &self.sha256, + || {}, + ) + } + } + /// [`restore_over`] with the pin file the archive ships given too. fn restore_over_packaged( tag: &str, packaged: &str, current: &str, ) -> (Option, String) { - let dir = scratch_dir(tag); - let root = dir.join("lemonade"); - let runtime_dir = runtime_dir_in(&root); - fs::create_dir_all(runtime_dir.join("resources")).unwrap(); - let pins = runtime_dir.join("resources/backend_versions.json"); - fs::write(&pins, current).unwrap(); - let source = dir.join("source-archive"); - write_embeddable_with_pins(&source, packaged); - let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); - let manifest = LemonadeInstallManifest { - env_id: "test".to_owned(), - version: "11.5.1".to_owned(), - runtime_dir: runtime_dir.clone(), - lemond: lemond_path_in(&runtime_dir), - lemonade: lemonade_path_in(&runtime_dir), - backend_recipe: LLAMACPP_RECIPE.to_owned(), - backend_name: ROCM_BACKEND_NAME.to_owned(), - installed_at_unix_ms: 0, - }; - cache_archive(&root, &source); - let backup = restore_packaged_backend_versions_with( - &manifest, - "resources/backend_versions.json", - &sha256, - || {}, - ) - .unwrap(); - let after = fs::read_to_string(&pins).unwrap(); + let fixture = RestoreFixture::new(tag, packaged, current); + let backup = fixture.restore().unwrap(); + let after = fs::read_to_string(&fixture.pins).unwrap(); // The backup, if any, still has to be able to put the file back. if let Some(backup) = &backup { assert!(backup.previous.is_some()); @@ -1336,6 +1420,124 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Restoring the packaged pins never writes through a file another writer + /// already staged next to `backend_versions.json`. The occupied names are + /// the ones the retired clock-and-pid temp naming would have picked + /// (`backend_versions.pins.tmp--`), so a restore that still + /// used one would truncate a foreign file and rename it into place. + /// The restore tests above would pass for either writer; the foreign files + /// are what tell them apart. + #[test] + fn restoring_the_packaged_pins_never_reuses_another_writers_temp_file() { + let drifted = r#"{"therock":{"version":"7.14.1"},"llamacpp":{"rocm-stable":"b11783"}}"#; + let fixture = RestoreFixture::new( + "restore-foreign-temp", + r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + drifted, + ); + + // Each attempt starts from the drifted file again, since a restore of + // an already restored file writes nothing. + let mut restored = None; + assert_write_spares_foreign_temp_files( + |millis| retired_pin_temp_path(&fixture.pins, millis), + || { + fs::write(&fixture.pins, drifted)?; + restored = fixture.restore()?; + Ok(()) + }, + ) + .unwrap(); + + let after = fs::read_to_string(&fixture.pins).unwrap(); + let value: serde_json::Value = serde_json::from_str(&after).unwrap(); + assert_eq!(value["therock"]["version"], "7.13.0", "{after}"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b9752", "{after}"); + assert!(restored.is_some(), "a drifted pin file must be restored"); + assert_no_pin_temp_files(fixture.pins.parent().unwrap()); + fs::remove_dir_all(&fixture.dir).ok(); + } + + /// Putting a backup back spares foreign staged files for the same reason. + #[test] + fn putting_a_pin_file_backup_back_never_reuses_another_writers_temp_file() { + let dir = scratch_dir("pin-backup-foreign-temp"); + let pins = dir.join("backend_versions.json"); + let previous = br#"{"therock":{"version":"7.14.1"},"llamacpp":{"rocm-stable":"b11783"}}"#; + fs::write(&pins, br#"{"therock":{"version":"7.13.0"}}"#).unwrap(); + + assert_write_spares_foreign_temp_files( + |millis| retired_pin_temp_path(&pins, millis), + || PinFileBackup::new(pins.clone(), Some(previous.to_vec())).put_back(), + ) + .unwrap(); + + assert_eq!(fs::read(&pins).unwrap(), previous); + assert_no_pin_temp_files(&dir); + fs::remove_dir_all(&dir).ok(); + } + + /// A put-back onto a runtime tree that has been removed fails, as it did + /// before the shared atomic writer, and recreates nothing: the shared + /// writer would build `resources/` just to hold the pin file. The failure + /// message and the absent directory are asserted together, since the + /// warning the caller prints says the previous file could not be put back. + #[test] + fn putting_a_pin_file_backup_back_into_a_removed_tree_fails_and_recreates_nothing() { + let dir = scratch_dir("pin-backup-removed-tree"); + let resources = dir.join("runtime").join("resources"); + let pins = resources.join("backend_versions.json"); + + let error = PinFileBackup::new(pins, Some(b"{}".to_vec())) + .put_back() + .expect_err("the tree is gone, so the previous pins cannot be put back into it"); + + let text = format!("{error:#}"); + assert!( + text.contains("failed to put back") && text.contains("not recreating"), + "{text}" + ); + assert!( + !resources.exists(), + "put_back recreated {}", + resources.display() + ); + assert!(!dir.join("runtime").exists(), "put_back recreated the tree"); + fs::remove_dir_all(&dir).ok(); + } + + /// The restore has the same rule: a runtime tree removed while the archive + /// was being read is not rebuilt around a freshly written pin file. + #[test] + fn restoring_the_packaged_pins_into_a_removed_tree_fails_and_recreates_nothing() { + let fixture = RestoreFixture::new( + "restore-removed-tree", + r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"7.14.1"},"llamacpp":{"rocm-stable":"b11783"}}"#, + ); + let runtime_dir = fixture.manifest.runtime_dir.clone(); + + let error = restore_packaged_backend_versions_with( + &fixture.manifest, + "resources/backend_versions.json", + &fixture.sha256, + || fs::remove_dir_all(&runtime_dir).unwrap(), + ) + .expect_err("the tree is gone, so the packaged pins cannot be written into it"); + + let text = format!("{error:#}"); + assert!( + text.contains("failed to restore") && text.contains("not recreating"), + "{text}" + ); + assert!( + !runtime_dir.exists(), + "the restore recreated {}", + runtime_dir.display() + ); + fs::remove_dir_all(&fixture.dir).ok(); + } + #[test] fn lemonade_root_uses_requested_engine_root() { let paths = AppPaths { diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 48deb7d142..78ea6c164c 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, anyhow, bail}; +use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; use rocm_core::{AppPaths, DEFAULT_LOCAL_PORT, normalize_runtime_path_for_host, runtime_is_linux}; use rocm_engine_protocol::{ @@ -15,7 +15,7 @@ use rocm_engine_protocol::{ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::io::Read; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; mod archive_cache; @@ -28,12 +28,16 @@ mod archive_props; mod backend_alignment; mod device_policy; mod direct_llama; +mod footprint; mod install; mod llamacpp_backend; mod process; +mod router_serve; mod runtime_dir; mod state; #[cfg(test)] +mod test_http; +#[cfg(test)] mod test_support; // Re-exported so this stays reachable at its pre-split crate-root path — @@ -41,12 +45,16 @@ mod test_support; // the crate root before the split, and `apps/rocm` depends on this crate's // lib target. pub use backend_alignment::LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV; +// The read-only view of Lemonade's install trees and record that `apps/rocm` +// needs for `rocm storage` and `rocm runtimes uninstall`; see `footprint`. +pub use footprint::{ + InstallRecord, RecordCleanup, forget_install_record_inside, install_record, install_tree_dirs, + install_tree_root, recorded_runtime_dir, +}; pub(crate) const ENGINE_NAME: &str = "lemonade"; pub(crate) const DEFAULT_HOST: &str = "127.0.0.1"; pub(crate) const DEFAULT_MODEL: &str = "Qwen3-4B-Instruct-2507-GGUF"; -pub(crate) const DEFAULT_MODEL_REPO_DIR: &str = "models--unsloth--Qwen3-4B-Instruct-2507-GGUF"; -pub(crate) const DEFAULT_MODEL_GGUF: &str = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; pub(crate) const LLAMACPP_RECIPE: &str = "llamacpp"; pub(crate) const ROCM_BACKEND_NAME: &str = "rocm"; #[cfg(feature = "e2e-test-hooks")] @@ -358,6 +366,35 @@ fn detect_response() -> DetectResponse { } } +/// The install proper: extract the embeddable, pick and align the llama.cpp +/// backend, then write the install record, replacing whatever was there. +/// +/// The record is written, never read, on this path: a corrupt record is +/// overwritten by the next plain install and the current tree is kept, which is +/// why that is what the CLI advises when it cannot read the record. The two +/// steps are parameters so a test can run this real sequence without +/// downloading anything or running `lemonade`. +fn install_and_record( + paths: &AppPaths, + env_root: Option<&std::path::Path>, + reinstall: bool, + prepare_embeddable: impl FnOnce( + &AppPaths, + Option<&std::path::Path>, + bool, + ) -> Result, + prepare_backend: impl FnOnce( + &AppPaths, + &mut crate::install::LemonadeInstallManifest, + ) -> Result>, +) -> Result<(crate::install::LemonadeInstallManifest, Option)> { + let mut manifest = prepare_embeddable(paths, env_root, reinstall)?; + eprintln!("Detecting best supported Lemonade llama.cpp backend..."); + let aligned_version = prepare_backend(paths, &mut manifest)?; + crate::install::write_manifest(paths, &manifest)?; + Ok((manifest, aligned_version)) +} + fn install_response(request: InstallRequest) -> Result { let paths = AppPaths::discover()?; paths.ensure()?; @@ -368,23 +405,41 @@ fn install_response(request: InstallRequest) -> Result { #[cfg(feature = "e2e-test-hooks")] if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() { crate::llamacpp_backend::install_llamacpp_backend_with_retry(|| { - bail!("scripted Lemonade backend install failure") + anyhow::bail!("scripted Lemonade backend install failure") })?; } - eprintln!( - "Preparing Lemonade embeddable {}...", - rocm_deps::LEMONADE_VERSION - ); let env_root = request .env_root .as_deref() .map(normalize_runtime_path_for_host); - let mut manifest = - crate::install::prepare_embeddable(&paths, env_root.as_deref(), request.reinstall)?; - eprintln!("Detecting best supported Lemonade llama.cpp backend..."); - let aligned_version = - crate::backend_alignment::prepare_llamacpp_backend_for_active_rocm(&paths, &mut manifest)?; - crate::install::write_manifest(&paths, &manifest)?; + // Announced and space-checked before anything is downloaded: a fresh tree + // is a multi-gigabyte install, and `rocm serve` reaches this on its own. + if let Some(preflight) = crate::footprint::plan_install_preflight( + &paths, + env_root.as_deref(), + request.reinstall, + crate::footprint::available_space_for_install, + ) { + for line in preflight.notice_lines() { + eprintln!("{line}"); + } + match preflight.space_verdict() { + crate::footprint::SpaceVerdict::Fits => {} + crate::footprint::SpaceVerdict::Tight(warning) => eprintln!("{warning}"), + crate::footprint::SpaceVerdict::Refuse(message) => bail!(message), + } + } + eprintln!( + "Preparing Lemonade embeddable {}...", + rocm_deps::LEMONADE_VERSION + ); + let (manifest, aligned_version) = install_and_record( + &paths, + env_root.as_deref(), + request.reinstall, + crate::install::prepare_embeddable, + crate::backend_alignment::prepare_llamacpp_backend_for_active_rocm, + )?; let mut warnings = vec![ "Lemonade is installed as a rocm-cli managed embeddable runtime".to_owned(), format!( @@ -418,6 +473,16 @@ fn install_response(request: InstallRequest) -> Result { } fn resolve_model_response(request: ResolveModelRequest) -> Result { + // Refuse an unservable model reference on the pre-launch round trip `rocm serve` + // makes, not only in `require_checkpoint_variant`. That backstop runs three layers down, + // inside the background engine process — which on Windows is spawned with no + // stdout or stderr, and whose exit status nobody reads — so its refusal is + // invisible there: the user gets an empty log, exit 0, and a service recorded as + // stopped, with nothing anywhere saying what was wrong. Answering here puts the + // reason on the user's terminal before any engine starts or any record is written. + if let Some(message) = crate::direct_llama::hf_checkpoint_variant_required(&request.model_ref) { + bail!("{message}"); + } let device_policy = crate::device_policy::normalize_device_policy(request.device_policy)?; let engine_recipe = crate::state::accepted_engine_recipe(request.engine_recipe)?; let canonical_model_id = crate::state::resolve_lemonade_model_ref(&request.model_ref); @@ -464,22 +529,7 @@ fn launch_service(mut request: LaunchRequest) -> Result { .engine_state_dir(ENGINE_NAME) .join(format!("{}.json", request.service_id)); let endpoint_url = crate::state::endpoint_url(&request.host, request.port); - let serve_request = ServeHttpRequest { - service_id: request.service_id.clone(), - model_ref: crate::state::resolve_lemonade_model_ref(&request.model_ref), - host: request.host.clone(), - port: request.port, - device_policy: request - .device_policy - .clone() - .unwrap_or(DevicePolicy::GpuRequired), - gpu_indices: rocm_engine_protocol::launch_gpu_indices(request.gpu_selection.as_ref()), - runtime_id: request.runtime_id.clone(), - env_id: request.env_id.clone(), - state_path: state_path.clone(), - log_path: Some(log_path.clone()), - engine_recipe: request.engine_recipe.clone(), - }; + let serve_request = launch_serve_request(&request, &state_path, &log_path); let current_exe = std::env::current_exe().context("failed to discover current Lemonade engine binary")?; let args = crate::process::serve_http_command_args(&serve_request); @@ -510,190 +560,182 @@ fn launch_service(mut request: LaunchRequest) -> Result { }) } -fn serve_http(mut request: ServeHttpRequest) -> Result<()> { - crate::device_policy::require_gpu_required(&request.device_policy)?; - // Verify a real GPU is available before launching anything: zero usable - // devices fails fast, `auto` resolves to the first present device (never an - // assumed device 0), and an explicit `--gpu` index is rejected when it is not - // actually present. This runs before any server process is spawned. - request.gpu_indices = crate::device_policy::resolve_serve_gpu_indices(&request.gpu_indices)?; - let mut runtime = crate::install::resolve_runtime()?; - let mut process_env = crate::process::lemonade_process_environment()?; - process_env.gpu_indices = request.gpu_indices.clone(); - let log_path = request.log_path.as_deref(); - crate::state::write_running_state(&request, &runtime, std::process::id(), None, "starting")?; - // A canonical Hugging Face checkpoint (owner/repo:variant) cannot be served under its - // exact name through Lemonade's model router (Lemonade renames registered models and - // its registry has several naming quirks). Download the GGUF and run a packaged - // llama-server on it directly with `--alias`, which serves it under exactly that name. - if let Some(checkpoint) = crate::direct_llama::parse_hf_checkpoint(&request.model_ref) { - return crate::direct_llama::serve_hf_checkpoint( - &request, - &runtime, - &process_env, - log_path, - &checkpoint, - ); +/// The serve request the wrapper process is started with. The model reference is +/// normalized once here, so a launch serves (and records) the same identity that +/// `resolve` reports for it: `qwen-smoke` launches as `Qwen3-0.6B-GGUF`. A +/// canonical Hugging Face checkpoint passes through unchanged. `serve-http` itself +/// never normalizes, so a direct serve-http run keeps exactly the name it was given. +fn launch_serve_request( + request: &LaunchRequest, + state_path: &Path, + log_path: &Path, +) -> ServeHttpRequest { + ServeHttpRequest { + service_id: request.service_id.clone(), + model_ref: crate::state::resolve_lemonade_model_ref(&request.model_ref), + host: request.host.clone(), + port: request.port, + device_policy: request + .device_policy + .clone() + .unwrap_or(DevicePolicy::GpuRequired), + gpu_indices: rocm_engine_protocol::launch_gpu_indices(request.gpu_selection.as_ref()), + runtime_id: request.runtime_id.clone(), + env_id: request.env_id.clone(), + state_path: state_path.to_path_buf(), + log_path: Some(log_path.to_path_buf()), + engine_recipe: request.engine_recipe.clone(), } - if runtime_is_linux() - && let Some(server) = crate::direct_llama::find_llama_server_binary(&runtime.manifest) - { - crate::direct_llama::ensure_direct_llama_model_available( - &request, - &runtime, - &process_env, - log_path, - )?; - let backend = crate::direct_llama::llama_server_backend_label(&server); - return crate::direct_llama::serve_direct_llama_server( - &request, - &runtime, - &process_env, - &server, - log_path, - &anyhow!("using Lemonade packaged {backend} llama-server directly on Linux"), - ); +} + +/// Everything a serve needs from its environment, resolved once up front so tests +/// can supply their own (a confined cache and parent environment, a fake runtime). +struct ServeContext { + runtime: crate::install::LemonadeRuntime, + process_env: crate::process::LemonadeProcessEnvironment, + paths: AppPaths, + parent: crate::runtime_dir::ParentRuntimeEnvironment, + /// Whether this host is Linux: the route choice and the router-to-direct + /// fallback both depend on it, and a test cannot change the host. + is_linux: bool, +} + +fn serve_http(request: ServeHttpRequest) -> Result<()> { + serve_http_resolving(request, |request| { + crate::device_policy::require_gpu_required(&request.device_policy)?; + // Verify a real GPU is available before launching anything: zero usable + // devices fails fast, `auto` resolves to the first present device (never an + // assumed device 0), and an explicit `--gpu` index is rejected when it is not + // actually present. This runs before any server process is spawned. + request.gpu_indices = + crate::device_policy::resolve_serve_gpu_indices(&request.gpu_indices)?; + let runtime = crate::install::resolve_runtime()?; + let mut process_env = crate::process::lemonade_process_environment()?; + process_env.gpu_indices = request.gpu_indices.clone(); + Ok(ServeContext { + runtime, + process_env, + paths: AppPaths::discover()?, + parent: crate::runtime_dir::ParentRuntimeEnvironment::current(), + is_linux: runtime_is_linux(), + }) + }) +} + +/// Run a serve and make its outcome visible in the persisted state: any error, +/// whichever step raised it, leaves a state file that exists at `failed` rather +/// than at the `starting`/`running` it last recorded, and the error is returned +/// unchanged. Servers spawned inside [`serve_with_context`] are stopped by the +/// code that spawned them, which owns the child. +fn serve_http_resolving( + mut request: ServeHttpRequest, + resolve: impl FnOnce(&mut ServeHttpRequest) -> Result, +) -> Result<()> { + let state_path = request.state_path.clone(); + let result = resolve(&mut request).and_then(|context| serve_with_context(&request, context)); + if result.is_err() && state_path.is_file() { + let _ = crate::state::mark_json_status(&state_path, "failed"); } - let mut child = crate::process::spawn_lemond( - &runtime.manifest, - &request.host, - request.port, - log_path, - &process_env, - )?; - crate::state::write_running_state( - &request, - &runtime, - std::process::id(), - Some(child.id()), - "running", - )?; - crate::process::wait_for_lemonade_cli_status( - &runtime.manifest, - &request.host, - request.port, - Duration::from_secs(30), - log_path, - &process_env, - ) - .context("Lemonade server did not become ready")?; - let backend = crate::llamacpp_backend::ensure_best_llamacpp_backend( - &mut runtime.manifest, - &request.host, - request.port, - &process_env, - false, - ) - .context("failed to select a supported Lemonade llama.cpp backend")?; - let load_result = crate::process::run_lemonade_model_load( - &runtime.manifest, - &request.host, - request.port, - &request.model_ref, - &backend, - request.engine_recipe.as_ref(), - log_path, - &process_env, - ); - let router_ready = load_result.is_ok() - && crate::process::query_loaded_model_endpoint( - &crate::state::endpoint_url(&request.host, request.port), - &request.model_ref, - &backend, - ) - .unwrap_or(false) - && crate::process::query_chat_smoke_endpoint( - &request.host, - request.port, - &request.model_ref, - ) - .unwrap_or(false); - if let Err(error) = load_result { - if runtime_is_linux() - && let Some(direct_server) = - crate::direct_llama::find_llama_server_binary(&runtime.manifest) - { - let _ = crate::state::terminate_pid(child.id(), true); - let _ = child.wait(); - return crate::direct_llama::serve_direct_llama_server( - &request, + result +} + +fn serve_with_context(request: &ServeHttpRequest, context: ServeContext) -> Result<()> { + let ServeContext { + runtime, + process_env, + paths, + parent, + is_linux, + } = context; + let log_path = request.log_path.as_deref(); + crate::state::write_running_state(request, &runtime, std::process::id(), None, "starting")?; + let route = crate::direct_llama::serving_route(&runtime.manifest, request, is_linux)?; + let router_model_ref = match route { + crate::direct_llama::ServeRoute::Direct { model, server } => { + let model_path = crate::direct_llama::ensure_model_in( + &paths, + request, &runtime, &process_env, - &direct_server, log_path, - &error, - ); - } - return Err(error).with_context(|| { - format!( - "failed to load {} with Lemonade {LLAMACPP_RECIPE}:{backend}", - request.model_ref - ) - }); - } - if !router_ready { - let error = anyhow!( - "Lemonade load completed but the endpoint did not report a {LLAMACPP_RECIPE}:{backend}-loaded model" - ); - if runtime_is_linux() - && let Some(direct_server) = - crate::direct_llama::find_llama_server_binary(&runtime.manifest) - { - let _ = crate::state::terminate_pid(child.id(), true); - let _ = child.wait(); + &model, + &parent, + )?; + let backend = crate::direct_llama::llama_server_backend_label(&server); return crate::direct_llama::serve_direct_llama_server( - &request, + request, &runtime, &process_env, - &direct_server, + &server, + &model_path, log_path, - &error, + &format!("using Lemonade packaged {backend} llama-server directly"), + &parent, ); } - return Err(error).with_context(|| { - format!( - "failed to verify {} with Lemonade {LLAMACPP_RECIPE}:{backend}", - request.model_ref + crate::direct_llama::ServeRoute::Router { model_ref } => model_ref, + }; + let mut steps = crate::router_serve::LemondRouterSteps { + runtime, + request, + process_env: &process_env, + log_path, + parent, + child: None, + }; + let result = crate::router_serve::serve_router(request, &mut steps, &router_model_ref); + with_direct_fallback( + result, + &paths, + request, + &steps.runtime, + &process_env, + log_path, + &steps.parent, + is_linux, + ) +} + +/// Route a router serve's outcome. The router already stopped its server and, +/// for a load or verification failure, left the state at `running`. Only a model load or verification failure may be +/// retried, and only through the GPU llama-server of the backend Lemonade +/// selected; every other outcome is returned as it is. +#[allow(clippy::too_many_arguments)] +fn with_direct_fallback( + result: Result<()>, + paths: &AppPaths, + request: &ServeHttpRequest, + runtime: &crate::install::LemonadeRuntime, + process_env: &crate::process::LemonadeProcessEnvironment, + log_path: Option<&Path>, + parent: &crate::runtime_dir::ParentRuntimeEnvironment, + is_linux: bool, +) -> Result<()> { + match result { + Err(error) if crate::router_serve::is_router_load_failure(&error) => { + crate::direct_llama::serve_direct_after_router_failure( + paths, + request, + runtime, + process_env, + log_path, + parent, + is_linux, + error, ) - }); - } - crate::state::merge_json_state( - &request.state_path, - &json!({ - "status": "ready", - "server_pid": child.id(), - // Identity token for the server PID, captured while the child is alive. - "server_start_ticks": rocm_core::process_start_ticks(child.id()), - "backend_state": "ready", - "backend_requested": backend, - "load_response": { - "status": "loaded", - "method": "lemonade-cli", - "model_name": request.model_ref, - "llamacpp_backend": backend - }, - }), - )?; - let status = child.wait().context("failed waiting for Lemonade server")?; - crate::state::mark_json_status( - &request.state_path, - if status.success() { - "stopped" - } else { - "failed" - }, - )?; - if status.success() { - Ok(()) - } else { - bail!("Lemonade server exited with status {status}") + } + result => result, } } fn healthcheck_service(request: HealthcheckRequest) -> Result { rocm_core::require_nonempty(&request.service_id, "service_id")?; let files = crate::state::service_files(&request.service_id)?; - let state = crate::state::read_service_state(&files.state_path).ok(); + healthcheck_service_at(&files.state_path) +} + +fn healthcheck_service_at(state_path: &Path) -> Result { + let state = crate::state::read_service_state(state_path).ok(); let endpoint_url = state .as_ref() .and_then(crate::state::endpoint_url_from_state); @@ -727,12 +769,7 @@ fn healthcheck_service(request: HealthcheckRequest) -> Result(value: &T) -> Result<()> { mod tests { use super::*; + /// Characterization: health and chat read the served name from the persisted + /// state. This holds with or without this change (`healthcheck_service_at` + /// already preferred `canonical_model_id`), so it guards the reading, not the + /// writing; `router_state_keeps_requested_text_and_records_router_name` covers + /// what a router serve persists. + #[test] + fn health_and_chat_read_the_served_name_from_the_persisted_state() { + use crate::test_http::{Reply, spawn_fixture}; + for (tag, served_name) in [("router", "Qwen3-0.6B-GGUF"), ("direct", "qwen-smoke")] { + let expected = served_name.to_owned(); + let (port, server) = spawn_fixture(Duration::from_secs(5), move |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"all_models_loaded": [{"model_name": expected, "recipe_options": {"llamacpp_backend": "vulkan"}}]}) + } + .to_string(), + last: chat, + } + }); + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = ServeHttpRequest { + service_id: tag.to_owned(), + model_ref: "qwen-smoke".to_owned(), + host: "127.0.0.1".to_owned(), + port, + device_policy: DevicePolicy::GpuRequired, + gpu_indices: vec![1], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + }; + let runtime = crate::install::LemonadeRuntime { + manifest: crate::install::LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + }, + }; + if tag == "direct" { + crate::state::write_running_state( + &request, + &runtime, + std::process::id(), + None, + "ready", + ) + .unwrap(); + } else { + crate::state::write_running_state_for_model( + &request, + &runtime, + std::process::id(), + None, + "ready", + served_name, + ) + .unwrap(); + } + crate::state::merge_json_state( + &request.state_path, + &json!({"backend_requested": "vulkan"}), + ) + .unwrap(); + let response = healthcheck_service_at(&request.state_path).unwrap(); + let seen = server.join().unwrap(); + let chat = seen + .iter() + .find(|recorded| recorded.method == "POST") + .expect("health inference request was not received"); + assert_eq!(chat.path, "/v1/chat/completions"); + let chat_model = serde_json::from_str::(&chat.body).unwrap()["model"] + .as_str() + .unwrap() + .to_owned(); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(response.status, "ready"); + assert_eq!(state["model_ref"], "qwen-smoke"); + assert_eq!(chat_model, served_name); + assert_eq!(state["canonical_model_id"], served_name); + assert!(state.get(rocm_core::INFERENCE_VERIFIED_STATE_KEY).is_some()); + } + } + + fn launch_request(model_ref: &str) -> LaunchRequest { + LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: model_ref.to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: None, + endpoint_mode: None, + engine_recipe: None, + gpu_selection: None, + } + } + + #[test] + fn launch_serves_the_identity_resolve_reports() { + // The engine's Launch method normalizes the requested reference once, so what + // it serves and records is what `resolve` reports for it. `serve-http` + // itself never normalizes (direct serving keeps the exact name it is given). + for (requested, reported) in [ + ("qwen-smoke", "Qwen3-0.6B-GGUF"), + ("assistant", DEFAULT_MODEL), + ("owner/repo:Q4_0", "owner/repo:Q4_0"), + ("Some-Registered-Model", "Some-Registered-Model"), + ] { + let resolved = resolve_model_response(ResolveModelRequest { + model_ref: requested.to_owned(), + runtime_id: None, + device_policy: None, + recipe_override: None, + engine_recipe: None, + }) + .unwrap(); + assert_eq!(resolved.canonical_model_id, reported); + let serve = launch_serve_request( + &launch_request(requested), + Path::new("state.json"), + Path::new("service.log"), + ); + assert_eq!(serve.model_ref, resolved.canonical_model_id, "{requested}"); + // The wrapper process is started with that same identity. + let args = crate::process::serve_http_command_args(&serve); + assert_eq!(args[2], reported, "{args:?}"); + } + } + + /// A confined serve environment: a fake runtime under `dir`, a model cache under + /// `dir/cache`, and a parent environment rooted at `dir`. + #[cfg(target_os = "linux")] + fn fake_context(dir: &Path, manifest: install::LemonadeInstallManifest) -> ServeContext { + ServeContext { + runtime: crate::install::LemonadeRuntime { manifest }, + process_env: crate::process::LemonadeProcessEnvironment::default(), + paths: AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }, + parent: crate::runtime_dir::ParentRuntimeEnvironment::rooted_at(dir), + is_linux: true, + } + } + + #[cfg(target_os = "linux")] + fn fake_request(dir: &Path, model_ref: &str) -> ServeHttpRequest { + ServeHttpRequest { + service_id: "svc".to_owned(), + model_ref: model_ref.to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + } + } + + #[cfg(target_os = "linux")] + fn install_fake_llama_server(manifest: &install::LemonadeInstallManifest, script: &str) { + crate::test_http::fake_runtime::write_executable( + &manifest + .runtime_dir + .join("bin/llamacpp/vulkan/llama-server"), + script, + ); + } + + #[cfg(target_os = "linux")] + fn write_registry(manifest: &install::LemonadeInstallManifest) { + let resources = manifest.runtime_dir.join("resources"); + std::fs::create_dir_all(&resources).unwrap(); + std::fs::write( + resources.join("server_models.json"), + json!({ + "Qwen3-0.6B-GGUF": { + "recipe": "llamacpp", + "checkpoint": "owner/serve-test-repo:Q4_0", + "suggested": true, + "size": 0.38, + "labels": ["reasoning"] + } + }) + .to_string(), + ) + .unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn serve_errors_before_the_server_starts_leave_state_failed() { + use crate::test_http::fake_runtime; + // Each case fails somewhere between the `starting` state being written and a + // server being spawned. None may leave the state at `starting`, and the + // error that reaches the caller is the original one. + for (case, model_ref, with_binary, pull_exit, expected) in [ + // serving_route: the selected backend's binary is missing. + ( + "missing binary", + "owner/serve-test-repo:Q4_0", + false, + 0, + "no selected GPU llama-server", + ), + // serving_route: a checkpoint without a variant. + ( + "invalid checkpoint", + "owner/serve-test-repo", + true, + 0, + "needs an explicit GGUF variant", + ), + // ensure_model_in: the download fails. + ( + "failed pull", + "owner/serve-test-repo:Q4_0", + true, + 7, + "Lemonade pull of `owner/serve-test-repo:Q4_0` failed", + ), + // ensure_model_in: the download succeeds but yields no usable GGUF. + ( + "pull without file", + "owner/serve-test-repo:Q4_0", + true, + 0, + "did not download", + ), + ] { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_runtime::manifest(dir, pull_exit, None); + if with_binary { + install_fake_llama_server(&manifest, "#!/bin/sh\nexit 0\n"); + } + let request = fake_request(dir, model_ref); + let context = fake_context(dir, manifest); + let error = serve_http_resolving(request.clone(), |_| Ok(context)) + .expect_err(case) + .to_string(); + assert!(error.contains(expected), "{case}: {error}"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{case}"); + assert_eq!(state["model_ref"], model_ref, "{case}"); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn serve_error_while_resolving_the_runtime_marks_launch_state_failed() { + // `launch` wrote `starting` before the wrapper ran; a wrapper that fails to + // resolve its runtime must not leave that behind. + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = fake_request(dir, "qwen-smoke"); + std::fs::write( + &request.state_path, + json!({"status": "starting"}).to_string(), + ) + .unwrap(); + let error = serve_http_resolving(request.clone(), |_| anyhow::bail!("no runtime")) + .unwrap_err() + .to_string(); + assert_eq!(error, "no runtime"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed"); + // With no state file there is nothing to mark, and none is invented. + let missing = fake_request(&dir.join("elsewhere"), "qwen-smoke"); + let _ = serve_http_resolving(missing.clone(), |_| anyhow::bail!("no runtime")); + assert!(!missing.state_path.exists()); + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_serve_through_serve_http_uses_the_requested_alias() { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_runtime::manifest(dir, 0, None); + write_registry(&manifest); + let args_file = dir.join("server-args.txt"); + install_fake_llama_server( + &manifest, + &format!( + "#!/bin/sh\nprintf '%s\\n' \"$@\" > '{}'\nexit 0\n", + args_file.display() + ), + ); + // The registered checkpoint is already cached, so nothing is downloaded. + let gguf = dir.join( + "cache/huggingface/hub/models--owner--serve-test-repo/snapshots/rev/model-Q4_0.gguf", + ); + std::fs::create_dir_all(gguf.parent().unwrap()).unwrap(); + std::fs::write(&gguf, b"gguf").unwrap(); + let (port, server) = spawn_fixture(Duration::from_secs(10), |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"data": [{"id": "qwen-smoke"}]}) + } + .to_string(), + last: chat, + } + }); + let mut request = fake_request(dir, "qwen-smoke"); + request.port = port; + let context = fake_context(dir, manifest); + serve_http_resolving(request.clone(), |_| Ok(context)).unwrap(); + + let args = std::fs::read_to_string(&args_file) + .unwrap() + .lines() + .map(str::to_owned) + .collect::>(); + let after = |flag: &str| { + let at = args.iter().position(|arg| arg == flag).expect(flag); + args[at + 1].clone() + }; + // Served under exactly the name that was requested, not the registered name. + assert_eq!(after("--alias"), "qwen-smoke", "{args:?}"); + assert_eq!(after("-m"), gguf.display().to_string(), "{args:?}"); + assert_eq!(after("--port"), port.to_string(), "{args:?}"); + let chat = server + .join() + .unwrap() + .into_iter() + .find(|recorded| recorded.method == "POST") + .expect("smoke request"); + assert_eq!( + serde_json::from_str::(&chat.body).unwrap()["model"], + "qwen-smoke" + ); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["model_ref"], "qwen-smoke"); + assert_eq!(state["canonical_model_id"], "qwen-smoke"); + assert_eq!(state["status"], "stopped"); + } + + #[cfg(target_os = "linux")] + fn fallback_fixture( + dir: &Path, + script: &str, + ) -> ( + ServeContext, + ServeHttpRequest, + std::path::PathBuf, + u16, + std::thread::JoinHandle>, + ) { + fallback_fixture_with(dir, script, true) + } + + /// `cached` is whether the GGUF is already in the hub cache. When it is not, + /// the fake `lemonade pull` puts it there, so the fallback runs a pull. + #[cfg(target_os = "linux")] + fn fallback_fixture_with( + dir: &Path, + script: &str, + cached: bool, + ) -> ( + ServeContext, + ServeHttpRequest, + std::path::PathBuf, + u16, + std::thread::JoinHandle>, + ) { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let pulled = "models--owner--serve-test-repo/snapshots/rev/model-Q4_0.gguf"; + let manifest = fake_runtime::manifest(dir, 0, (!cached).then_some(pulled)); + write_registry(&manifest); + install_fake_llama_server(&manifest, script); + let gguf = dir.join("cache/huggingface/hub").join(pulled); + if cached { + std::fs::create_dir_all(gguf.parent().unwrap()).unwrap(); + std::fs::write(&gguf, b"gguf").unwrap(); + } + let (port, server) = spawn_fixture(Duration::from_secs(10), |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"data": [{"id": "qwen-smoke"}]}) + } + .to_string(), + last: chat, + } + }); + let mut request = fake_request(dir, "qwen-smoke"); + request.port = port; + request.log_path = Some(dir.join("serve.log")); + (fake_context(dir, manifest), request, gguf, port, server) + } + + #[cfg(target_os = "linux")] + fn args_script(args_file: &Path) -> String { + format!( + "#!/bin/sh\nprintf '%s\\n' \"$@\" > '{}'\nexit 0\n", + args_file.display() + ) + } + + #[cfg(target_os = "linux")] + fn marked_load_failure() -> anyhow::Error { + anyhow::Error::msg("load exploded").context(crate::router_serve::RouterLoadFailure( + "failed to load qwen-smoke".to_owned(), + )) + } + + #[cfg(target_os = "linux")] + #[test] + fn router_load_failure_falls_back_to_the_selected_gpu_llama_server() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let args_file = dir.join("server-args.txt"); + let (context, request, gguf, port, server) = + fallback_fixture(dir, &args_script(&args_file)); + with_direct_fallback( + Err(marked_load_failure()), + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + true, + ) + .unwrap(); + server.join().unwrap(); + + // The state shows a direct serve that ran, under the requested name. + let args = std::fs::read_to_string(&args_file).unwrap(); + assert!( + args.contains(&format!("-m\n{}\n", gguf.display())), + "{args}" + ); + assert!(args.contains(&format!("--port\n{port}\n")), "{args}"); + assert!(args.contains("--alias\nqwen-smoke\n"), "{args}"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "stopped"); + // The message names the backend of the server it launched, derived from + // the same path, and says it is not ROCm. + let log = std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap(); + assert!( + log.contains("serving it directly with the selected vulkan GPU llama-server"), + "{log}" + ); + assert!(log.contains("load exploded"), "{log}"); + assert!( + log.contains("Lemonade is serving on the vulkan GPU backend, not ROCm"), + "{log}" + ); + assert!(!log.contains("cpu"), "{log}"); + } + + /// The router leaves the state at `running`, naming its own server. While the + /// fallback pulls the model, a reader must see `starting` and no server pid: + /// `identity_from_state` prefers `server_pid`, so a dead router's pid left in + /// place would point `stop` at the wrong process. The fake `lemonade pull` + /// copies the state file as it is at that moment, and the server then runs to + /// completion. + #[cfg(target_os = "linux")] + #[test] + fn the_state_says_starting_while_the_fallback_pulls_the_model() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let args_file = dir.join("server-args.txt"); + let (context, request, _gguf, _port, server) = + fallback_fixture_with(dir, &args_script(&args_file), false); + // What the failed router serve left behind. + crate::state::merge_json_state( + &request.state_path, + &serde_json::json!({ + "status": "running", + "pid": std::process::id(), + "server_pid": 4_000_000, + "server_start_ticks": 7, + }), + ) + .unwrap(); + with_direct_fallback( + Err(marked_load_failure()), + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + true, + ) + .unwrap(); + server.join().unwrap(); + let during: serde_json::Value = serde_json::from_slice( + &std::fs::read(dir.join("state-at-pull.json")).expect("a pull ran"), + ) + .unwrap(); + assert_eq!(during["status"], "starting", "{during}"); + assert!(during["server_pid"].is_null(), "{during}"); + assert!(during["server_start_ticks"].is_null(), "{during}"); + // Until the direct server runs, the launcher is the process to stop. + assert_eq!( + crate::state::identity_from_state(&during).map(|identity| identity.pid), + Some(std::process::id()), + "{during}" + ); + // The fallback then went on to serve. + assert!(args_file.exists(), "the direct server ran"); + } + + /// The router leaves the state at `running`; the fallback sets `starting` + /// before it downloads or spawns and puts `failed` back when it does not + /// produce a server, so a reader never sees a stuck `starting` after a + /// fallback that does not. + #[cfg(target_os = "linux")] + #[test] + fn a_fallback_that_cannot_get_the_model_ends_failed_not_starting() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + // `lemonade pull` exits 1 and no GGUF is cached. + let manifest = crate::test_http::fake_runtime::manifest(dir, 1, None); + write_registry(&manifest); + install_fake_llama_server(&manifest, "#!/bin/sh\nexit 0\n"); + let mut request = fake_request(dir, "qwen-smoke"); + request.log_path = Some(dir.join("serve.log")); + let context = fake_context(dir, manifest); + let error = with_direct_fallback( + Err(marked_load_failure()), + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + true, + ) + .unwrap_err(); + let text = format!("{error:#}"); + assert!(text.contains("load exploded"), "{text}"); + assert!(text.contains("fallback also failed"), "{text}"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{state}"); + } + + /// The Windows half of the fallback rule, through the whole serve: a router + /// that fails to verify its model is not retried directly when the host is not + /// Linux, though a GPU llama-server is installed and the model is cached. + /// `is_linux` comes from the serve context, so this fails if either the route + /// choice or the fallback reads the host instead. + #[cfg(target_os = "linux")] + #[test] + fn a_router_load_failure_is_not_served_directly_when_the_host_is_not_linux() { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_runtime::manifest(dir, 0, None); + // `lemonade backends` lists vulkan as installed; every other call succeeds. + fake_runtime::write_executable( + &manifest.lemonade, + "#!/bin/sh\nif [ \"$1\" = backends ]; then\n echo 'llamacpp vulkan installed ok'\nfi\nexit 0\n", + ); + write_registry(&manifest); + let args_file = dir.join("server-args.txt"); + install_fake_llama_server(&manifest, &args_script(&args_file)); + let gguf = dir.join( + "cache/huggingface/hub/models--owner--serve-test-repo/snapshots/rev/model-Q4_0.gguf", + ); + std::fs::create_dir_all(gguf.parent().unwrap()).unwrap(); + std::fs::write(&gguf, b"gguf").unwrap(); + // The endpoint answers, but never lists the model: the router's load is + // verified false, which is a tagged load failure. + let (port, _server) = spawn_fixture(Duration::from_secs(3), |_| Reply { + status: 200, + body: json!({"data": []}).to_string(), + last: false, + }); + let mut request = fake_request(dir, "qwen-smoke"); + request.port = port; + request.log_path = Some(dir.join("serve.log")); + let mut context = fake_context(dir, manifest); + context.is_linux = false; + let error = serve_http_resolving(request.clone(), |_| Ok(context)) + .unwrap_err() + .to_string(); + assert!(error.contains("failed to verify qwen-smoke"), "{error}"); + assert!(!args_file.exists(), "no llama-server may have started"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{state}"); + } + + #[cfg(target_os = "linux")] + #[test] + fn only_a_router_load_failure_may_fall_back_and_only_on_linux() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let args_file = dir.join("server-args.txt"); + let (context, request, _gguf, _port, server) = + fallback_fixture(dir, &args_script(&args_file)); + let run = |result: Result<()>, is_linux: bool| { + with_direct_fallback( + result, + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + is_linux, + ) + }; + // A server that never became ready, a failed backend selection or a later + // exit is returned as it is, though a direct route is open. + let other = run( + Err(anyhow::anyhow!("Lemonade server did not become ready")), + true, + ) + .unwrap_err() + .to_string(); + assert_eq!(other, "Lemonade server did not become ready"); + // Off Linux the router's own error is returned unchanged. + let windows = run(Err(marked_load_failure()), false) + .unwrap_err() + .to_string(); + assert_eq!(windows, "failed to load qwen-smoke"); + assert!(!args_file.exists(), "no llama-server may have started"); + assert!(!request.log_path.as_ref().unwrap().exists()); + // The state the router left is settled by the fallback; with no direct + // route open the failure is final. + assert_eq!( + crate::state::read_service_state(&request.state_path).unwrap()["status"], + "failed" + ); + drop(server); + } + + #[cfg(target_os = "linux")] + #[test] + fn fallback_never_serves_through_a_missing_or_cpu_backend() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = crate::test_http::fake_runtime::manifest(dir, 0, None); + write_registry(&manifest); + let request = fake_request(dir, "qwen-smoke"); + // The selected GPU backend has no llama-server: nothing to fall back to. + assert!(crate::direct_llama::direct_fallback_route(&manifest, &request, true).is_none()); + // Only a CPU llama-server exists and the backend is CPU: still refused. + let mut cpu = manifest.clone(); + cpu.backend_name = "cpu".to_owned(); + crate::test_http::fake_runtime::write_executable( + &cpu.runtime_dir.join("bin/llamacpp/cpu/llama-server"), + "#!/bin/sh\nexit 0\n", + ); + assert!(crate::direct_llama::direct_fallback_route(&cpu, &request, true).is_none()); + // With the selected GPU backend installed the route opens on Linux only. + install_fake_llama_server(&manifest, "#!/bin/sh\nexit 0\n"); + assert!(crate::direct_llama::direct_fallback_route(&manifest, &request, true).is_some()); + assert!(crate::direct_llama::direct_fallback_route(&manifest, &request, false).is_none()); + } + #[test] fn stop_scope_targets_only_the_recorded_lemonade_server() { assert_eq!(STOP_SCOPE, rocm_core::KillScope::Single); } + + /// A `ResolveModel` request as `rocm serve` makes it for a plain + /// `rocm serve --engine lemonade`: no runtime, no device policy, no + /// recipe, so only the model reference is under test. + fn resolve_model_request(model_ref: &str) -> ResolveModelRequest { + ResolveModelRequest { + model_ref: model_ref.to_owned(), + runtime_id: None, + device_policy: None, + recipe_override: None, + engine_recipe: None, + } + } + + #[test] + fn resolving_a_hugging_face_repo_without_a_variant_is_refused() { + let error = resolve_model_response(resolve_model_request("Qwen/Qwen2.5-1.5B-Instruct")) + .expect_err("a bare owner/repo has no GGUF to serve and must be refused up front"); + let message = error.to_string(); + assert!( + message.contains("`owner/repo:`"), + "the refusal must show the form that would work, got:\n{message}" + ); + // The repository was never inspected, so naming a quantization would be a + // guess — and a wrong guess sends the user into a second failure. This + // repo, for one, ships only safetensors and has no `Q4_K_M` build. + assert!( + !message.contains("Q4_K_M") && !message.contains("Q4_0"), + "the refusal must not suggest a quantization the CLI has not seen, got:\n{message}" + ); + } + + #[test] + fn resolving_a_servable_model_still_succeeds() { + // An `owner/repo:variant` is passed through verbatim: the user named an + // exact checkpoint and quantization, and nothing may substitute for it. + for model_ref in [ + "Qwen/Qwen2.5-1.5B-Instruct:Q4_K_M", + "unsloth/Qwen3-0.6B-GGUF:Q4_0", + ] { + let response = resolve_model_response(resolve_model_request(model_ref)) + .unwrap_or_else(|error| panic!("{model_ref} must still resolve: {error}")); + assert_eq!(response.canonical_model_id, model_ref); + } + // Short names are not checkpoints at all, so the guard must leave them to + // the alias expansion that has always handled them. + for model_ref in ["qwen", DEFAULT_MODEL] { + let response = resolve_model_response(resolve_model_request(model_ref)) + .unwrap_or_else(|error| panic!("{model_ref} must still resolve: {error}")); + assert_eq!(response.canonical_model_id, DEFAULT_MODEL); + } + } } diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 5674afe21b..48a9caf91b 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -3,17 +3,18 @@ // SPDX-License-Identifier: MIT use anyhow::{Context, Result, bail}; -use std::path::PathBuf; +use std::fs; +use std::path::{Path, PathBuf}; use std::process::{Command as ProcessCommand, Stdio}; use std::time::Duration; -use crate::direct_llama::find_binary_in; use crate::install::{LemonadeInstallManifest, platform_binary_name}; use crate::process::{ - LemonadeProcessEnvironment, apply_lemonade_process_environment, hide_child_console_window, - lemonade_process_environment, run_lemonade_backend_install, spawn_lemond, - wait_for_lemonade_cli_status, + LemonadeProcessEnvironment, apply_lemonade_process_environment_with_parent, + hide_child_console_window, lemonade_process_environment, run_lemonade_backend_install, + spawn_lemond, wait_for_lemonade_cli_status, }; +use crate::runtime_dir::ParentRuntimeEnvironment; use crate::state::{free_local_port, terminate_pid}; use crate::{DEFAULT_HOST, LLAMACPP_RECIPE, ROCM_BACKEND_NAME}; @@ -73,7 +74,11 @@ pub(crate) fn best_llamacpp_backend_for_host( log_path, &process_env, )?; - let listing = run_lemonade_backends_list(manifest, &process_env)?; + let listing = run_lemonade_backends_list( + manifest, + &process_env, + &ParentRuntimeEnvironment::current(), + )?; let backends = parse_llamacpp_backend_statuses(&listing); Ok(select_best_llamacpp_backend(&backends).map(|(name, _)| name)) })(); @@ -112,7 +117,28 @@ pub(crate) fn ensure_best_llamacpp_backend( process_env: &LemonadeProcessEnvironment, force_reinstall: bool, ) -> Result { - let listing = run_lemonade_backends_list(manifest, process_env)?; + ensure_best_llamacpp_backend_with_parent( + manifest, + host, + port, + process_env, + force_reinstall, + &ParentRuntimeEnvironment::current(), + ) +} + +/// [`ensure_best_llamacpp_backend`] with the parent runtime environment supplied +/// by the caller, so a serve that was given a confined parent never reads or +/// creates anything in the host's own. +pub(crate) fn ensure_best_llamacpp_backend_with_parent( + manifest: &mut LemonadeInstallManifest, + host: &str, + port: u16, + process_env: &LemonadeProcessEnvironment, + force_reinstall: bool, + parent: &ParentRuntimeEnvironment, +) -> Result { + let listing = run_lemonade_backends_list(manifest, process_env, parent)?; let backends = parse_llamacpp_backend_statuses(&listing); let Some((backend, already_installed)) = select_best_llamacpp_backend(&backends) else { bail!( @@ -132,7 +158,7 @@ pub(crate) fn ensure_best_llamacpp_backend( } else { eprintln!("Installing Lemonade {LLAMACPP_RECIPE}:{backend} backend..."); install_llamacpp_backend_with_retry(|| { - run_lemonade_backend_install(manifest, host, port, &backend, process_env) + run_lemonade_backend_install(manifest, host, port, &backend, process_env, parent) })?; } Ok(backend) @@ -214,6 +240,7 @@ fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { fn run_lemonade_backends_list( manifest: &LemonadeInstallManifest, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -221,7 +248,7 @@ fn run_lemonade_backends_list( .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - apply_lemonade_process_environment(&mut command, process_env)?; + apply_lemonade_process_environment_with_parent(&mut command, process_env, parent)?; hide_child_console_window(&mut command); let output = command .output() @@ -241,30 +268,54 @@ fn run_lemonade_backends_list( /// replacing one in place, so all three names can accumulate across attempts. pub(crate) const ROCM_LLAMACPP_BACKEND_DIRS: [&str; 3] = ["rocm-stable", "rocm-nightly", "rocm"]; -/// Like [`crate::direct_llama::find_llama_server_binary`], but scoped to the single backend Lemonade -/// actually attempted (`manifest.backend_name`, one of `LLAMACPP_BACKEND_PRIORITY`'s -/// `"rocm"` / `"vulkan"`), rather than scanning every known backend directory in -/// priority order. Lemonade reports the generic `"rocm"` — never the more specific -/// `rocm-stable`/`rocm-nightly` — but may extract the actual build into either -/// versioned directory, so `"rocm"` maps to the whole ROCm family; `"vulkan"` never -/// does. This is what lets alignment verification reject a stale ROCm directory left -/// by an earlier attempt when this round actually picked `vulkan` (or the reverse). +/// The `bin/llamacpp/` directories that may hold the `llama-server` for a manifest's +/// selected backend, or `None` when that backend cannot be served directly. Only +/// `rocm` and `vulkan` are valid manifest backend names: installs record whatever +/// Lemonade selects from its `rocm`/`vulkan` priority list, so the `rocm-stable` and +/// `rocm-nightly` directory names are members of the `rocm` family, never backend +/// names of their own. `cpu` and unknown names fail closed (no CPU fallback). +pub(crate) fn backend_server_dirs(backend_name: &str) -> Option<&'static [&'static str]> { + match backend_name { + ROCM_BACKEND_NAME => Some(&ROCM_LLAMACPP_BACKEND_DIRS), + "vulkan" => Some(&["vulkan"]), + _ => None, + } +} + +/// Find a GPU llama-server scoped to the backend Lemonade selected +/// (`manifest.backend_name`, one of `LLAMACPP_BACKEND_PRIORITY`'s `"rocm"` / +/// `"vulkan"`), rather than scanning every known backend directory. `rocm` accepts +/// its directory family (`rocm-stable`, `rocm-nightly`, `rocm`); Vulkan never crosses +/// over to a stale ROCm install or the reverse. This is what lets alignment +/// verification reject a stale ROCm directory left by an earlier attempt when this +/// round actually picked `vulkan`. CPU and unknown names fail closed. Both flat and +/// one-level build-numbered extraction layouts are supported. pub(crate) fn find_llama_server_binary_for_backend( manifest: &LemonadeInstallManifest, backend_name: &str, ) -> Option { let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); let binary = platform_binary_name("llama-server"); - let family: &[&str] = if backend_name == ROCM_BACKEND_NAME { - &ROCM_LLAMACPP_BACKEND_DIRS - } else { - &["vulkan"] - }; - family + backend_server_dirs(backend_name)? .iter() .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) } +/// Look for `binary` directly in `dir`, then one level down in each subdirectory. +pub(crate) fn find_binary_in(dir: &Path, binary: &str) -> Option { + let direct = dir.join(binary); + if direct.is_file() { + return Some(direct); + } + fs::read_dir(dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .map(|subdir| subdir.join(binary)) + .find(|candidate| candidate.is_file()) +} + #[cfg(test)] mod tests { use super::*; @@ -467,4 +518,37 @@ vllm rocm unsupported Requires Linux ); fs::remove_dir_all(&dir).ok(); } + + #[test] + fn find_llama_server_binary_for_backend_fails_closed_for_cpu_and_unknown() { + let dir = scratch_dir("find-server-fail-closed"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + // Every directory holds a server, so a fallback to any of them would be + // found: only failing closed returns None. + for backend in [ + "rocm-stable", + "rocm-nightly", + "rocm", + "vulkan", + "cpu", + "mystery", + ] { + fs::create_dir_all(llamacpp.join(backend)).unwrap(); + fs::write(llamacpp.join(backend).join(&server), b"x").unwrap(); + } + let manifest = test_manifest(runtime_dir); + // `rocm-stable`/`rocm-nightly` are directories of the `rocm` family, never + // backend names a manifest records. + for backend in ["cpu", "mystery", "", "rocm-stable", "rocm-nightly"] { + assert_eq!(backend_server_dirs(backend), None, "{backend:?}"); + assert_eq!( + find_llama_server_binary_for_backend(&manifest, backend), + None, + "{backend:?} must not fall back to another backend's server" + ); + } + fs::remove_dir_all(&dir).ok(); + } } diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index e90c5cd25d..fb49ecccb6 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -53,6 +53,38 @@ pub(crate) fn spawn_lemond( port: u16, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, +) -> Result { + spawn_lemond_with_parent( + manifest, + host, + port, + log_path, + process_env, + &ParentRuntimeEnvironment::current(), + LogMode::Fresh, + ) +} + +/// How a spawned `lemond` treats the service log it is given. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LogMode { + /// Start the log over: this is the serve's own server. + Fresh, + /// Keep what is there and append: a helper `lemond` (the one that pulls a + /// model) shares the log of a server that already wrote the reason it is needed. + Append, +} + +/// `spawn_lemond` with the parent runtime environment supplied by the caller, so +/// tests need not read or create anything under the real user dirs. +pub(crate) fn spawn_lemond_with_parent( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, + log_mode: LogMode, ) -> Result { #[cfg(windows)] if let Some(log_path) = log_path { @@ -76,13 +108,13 @@ pub(crate) fn spawn_lemond( .arg("--port") .arg(port.to_string()) .stdin(Stdio::null()); - apply_lemonade_process_environment(&mut command, process_env)?; + let vars = lemonade_process_environment_vars(process_env, parent)?; + apply_environment_vars(&mut command, &vars); if let Some(log_path) = log_path { - if let Some(parent) = log_path.parent() { - fs::create_dir_all(parent)?; + if let Some(log_dir) = log_path.parent() { + fs::create_dir_all(log_dir)?; } - let log = fs::File::create(log_path) - .with_context(|| format!("failed to create {}", log_path.display()))?; + let log = open_service_log(log_path, log_mode)?; command.stdout(Stdio::from(log.try_clone()?)); command.stderr(Stdio::from(log)); } else { @@ -95,6 +127,23 @@ pub(crate) fn spawn_lemond( .map(LemondChild::Child) } +/// Open the service log in append mode, starting it over first for +/// [`LogMode::Fresh`]. The child writes +/// through this handle while this process appends its own lines (for example the +/// non-ROCm backend notice) through another; without `O_APPEND` the child's next +/// write would land at its own offset, on top of what was appended. +fn open_service_log(log_path: &Path, log_mode: LogMode) -> Result { + if log_mode == LogMode::Fresh { + fs::File::create(log_path) + .with_context(|| format!("failed to create {}", log_path.display()))?; + } + fs::OpenOptions::new() + .append(true) + .create(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display())) +} + pub(crate) enum LemondChild { Child(std::process::Child), #[cfg(windows)] @@ -137,6 +186,14 @@ pub(crate) struct LemondExitStatus { } impl LemondExitStatus { + #[cfg(test)] + pub(crate) fn for_test(success: bool, description: &str) -> Self { + Self { + success, + description: description.to_owned(), + } + } + pub(crate) const fn success(&self) -> bool { self.success } @@ -159,6 +216,40 @@ pub(crate) fn hide_child_console_window(command: &mut ProcessCommand) { #[cfg(not(windows))] pub(crate) const fn hide_child_console_window(_command: &mut ProcessCommand) {} +/// Which of this process's standard output streams are a console. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OutputConsoles { + pub(crate) stdout: bool, + pub(crate) stderr: bool, +} + +impl OutputConsoles { + pub(crate) fn detect() -> Self { + use std::io::IsTerminal; + Self { + stdout: std::io::stdout().is_terminal(), + stderr: std::io::stderr().is_terminal(), + } + } + + /// Whether a child that inherits both streams may be started without a + /// console window. + /// + /// `CREATE_NO_WINDOW` starts the child with no console at all, and a child + /// with no console writes nothing to a console handle it inherited: run + /// from a real Windows console, `cmd /c echo`, a Python child and + /// `lemonade.exe --version` all printed their output without the flag and + /// printed nothing with it. That is how `lemonade backends install`'s + /// progress for a multi-gigabyte download disappeared from an interactive + /// `rocm serve` while still showing up in a redirected log. When either + /// stream is a console this process has one, so the child shares it and no + /// window opens; when neither is (a pipe, a file, or no console at all) the + /// flag is still applied, so a console-less parent never flashes a window. + pub(crate) const fn child_may_hide_its_console(self) -> bool { + !(self.stdout || self.stderr) + } +} + fn child_process_path(path: &Path) -> String { if runtime_is_windows() { windows_child_path(path) @@ -181,11 +272,14 @@ pub(crate) fn lemonade_process_environment() -> Result Result<()> { - let vars = lemonade_process_environment_vars(env, &ParentRuntimeEnvironment::current())?; + let vars = lemonade_process_environment_vars(env, parent)?; apply_environment_vars(command, &vars); Ok(()) } @@ -193,7 +287,10 @@ pub(crate) fn apply_lemonade_process_environment( /// Apply an already-resolved variable set. Callers that spawn repeatedly (the /// readiness poll) resolve once and reuse, so preparing the runtime directory /// does not repeat its syscalls on every attempt. -fn apply_environment_vars(command: &mut ProcessCommand, vars: &[(&'static str, OsString)]) { +pub(crate) fn apply_environment_vars( + command: &mut ProcessCommand, + vars: &[(&'static str, OsString)], +) { for (key, value) in vars { command.env(key, value); } @@ -244,13 +341,34 @@ pub(crate) fn wait_for_lemonade_cli_status( timeout: Duration, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + wait_for_lemonade_cli_status_with_parent( + manifest, + host, + port, + timeout, + log_path, + process_env, + &ParentRuntimeEnvironment::current(), + ) +} + +/// `wait_for_lemonade_cli_status` with the parent runtime environment supplied by +/// the caller, so tests need not read or create anything under the real user dirs. +pub(crate) fn wait_for_lemonade_cli_status_with_parent( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + timeout: Duration, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { let start = std::time::Instant::now(); let mut last_status = None; // Resolved once: the poll runs twice a second, and resolving also prepares // the child's runtime directory on disk. - let env_vars = - lemonade_process_environment_vars(process_env, &ParentRuntimeEnvironment::current())?; + let env_vars = lemonade_process_environment_vars(process_env, parent)?; while start.elapsed() < timeout { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -302,13 +420,53 @@ fn summarize_startup_log_tail(log_path: Option<&Path>, limit: usize) -> String { } } +/// Install a llama.cpp backend through the `lemonade` CLI. The parent runtime +/// environment is supplied by the caller, so a test can confine what is read and +/// created. pub(crate) fn run_lemonade_backend_install( manifest: &LemonadeInstallManifest, host: &str, port: u16, backend: &str, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { + let mut command = backend_install_command( + manifest, + host, + port, + backend, + process_env, + parent, + OutputConsoles::detect(), + hide_child_console_window, + )?; + let status = command + .status() + .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; + if !status.success() { + bail!("Lemonade backend install failed with status {status}"); + } + Ok(()) +} + +/// The `lemonade backends install` command, ready to run. +/// +/// It inherits stdout and stderr: the backend download's progress is the child's +/// own output. `consoles` and `hide_window` are parameters so a test can assert +/// whether the window is hidden for each console state; the one production +/// caller passes the real streams and [`hide_child_console_window`]. +#[allow(clippy::too_many_arguments)] +fn backend_install_command( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + backend: &str, + process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, + consoles: OutputConsoles, + hide_window: impl FnOnce(&mut ProcessCommand), +) -> Result { let mut command = ProcessCommand::new(&manifest.lemonade); command .arg("--host") @@ -321,15 +479,11 @@ pub(crate) fn run_lemonade_backend_install( .stdin(Stdio::null()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); - apply_lemonade_process_environment(&mut command, process_env)?; - hide_child_console_window(&mut command); - let status = command - .status() - .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; - if !status.success() { - bail!("Lemonade backend install failed with status {status}"); + apply_lemonade_process_environment_with_parent(&mut command, process_env, parent)?; + if consoles.child_may_hide_its_console() { + hide_window(&mut command); } - Ok(()) + Ok(command) } fn lemonade_pull_args(host: &str, port: u16, checkpoint_ref: &str) -> Vec { @@ -347,13 +501,17 @@ fn lemonade_pull_args(host: &str, port: u16, checkpoint_ref: &str) -> Vec, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -370,7 +528,8 @@ pub(crate) fn run_lemonade_pull( } else { command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); } - apply_lemonade_process_environment(&mut command, process_env)?; + let vars = lemonade_process_environment_vars(process_env, parent)?; + apply_environment_vars(&mut command, &vars); hide_child_console_window(&mut command); let status = command .status() @@ -381,8 +540,11 @@ pub(crate) fn run_lemonade_pull( Ok(()) } +/// Load a model through the Lemonade CLI. The parent runtime environment is supplied +/// by the caller (production passes `ParentRuntimeEnvironment::current()`), so tests +/// need not read or create anything under the real user dirs. #[allow(clippy::too_many_arguments)] -pub(crate) fn run_lemonade_model_load( +pub(crate) fn run_lemonade_model_load_with_parent( manifest: &LemonadeInstallManifest, host: &str, port: u16, @@ -391,6 +553,7 @@ pub(crate) fn run_lemonade_model_load( engine_recipe: Option<&EngineRecipeHint>, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -413,7 +576,8 @@ pub(crate) fn run_lemonade_model_load( } else { command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); } - apply_lemonade_process_environment(&mut command, process_env)?; + let vars = lemonade_process_environment_vars(process_env, parent)?; + apply_environment_vars(&mut command, &vars); hide_child_console_window(&mut command); let status = command .status() @@ -429,7 +593,11 @@ pub(crate) fn run_lemonade_model_load( /// since the direct-serve path only runs GPU backends. A Lemonade-router entry carries /// `recipe_options`, so its `llamacpp_backend` must match — this keeps a merely /// registered-but-unloaded model (empty `recipe_options`) from reading as ready. -fn models_payload_has_ready_model(value: &Value, model_ref: &str, backend: &str) -> bool { +pub(crate) fn models_payload_has_ready_model( + value: &Value, + model_ref: &str, + backend: &str, +) -> bool { value .get("data") .or_else(|| value.get("models")) @@ -644,6 +812,261 @@ mod tests { use rocm_engine_protocol::DevicePolicy; use serde_json::json; + fn test_manifest() -> LemonadeInstallManifest { + LemonadeInstallManifest { + env_id: "lemonade-embeddable-test".to_owned(), + version: "test".to_owned(), + runtime_dir: PathBuf::from("rt"), + lemond: PathBuf::from("rt").join("lemond"), + lemonade: PathBuf::from("rt").join("lemonade"), + backend_recipe: "llamacpp".to_owned(), + backend_name: "rocm".to_owned(), + installed_at_unix_ms: 0, + } + } + + /// Build the real backend-install command for a console state and report + /// whether the window was hidden, along with the command itself. + fn backend_install_for(consoles: OutputConsoles) -> (ProcessCommand, bool) { + let mut hidden = false; + let root = crate::test_support::scratch_dir("backend-install-command"); + // Confined to a scratch root where a confined parent exists (Linux), so + // building the command creates nothing in the real runtime directory. + #[cfg(target_os = "linux")] + let parent = ParentRuntimeEnvironment::rooted_at(&root); + #[cfg(not(target_os = "linux"))] + let parent = ParentRuntimeEnvironment::current(); + let command = backend_install_command( + &test_manifest(), + "127.0.0.1", + 8123, + "rocm", + &LemonadeProcessEnvironment::default(), + &parent, + consoles, + |_| hidden = true, + ) + .expect("the command builds"); + fs::remove_dir_all(&root).ok(); + (command, hidden) + } + + /// The backend install's progress is the child's own output. Hiding its + /// console while that output is a console discards it (see + /// `OutputConsoles::child_may_hide_its_console`), so the window is hidden + /// only when neither stream is one. The command is the real one: a dropped + /// call to the hide step, a hide step run unconditionally, or a console + /// check that ignores one of the streams each turn a case below red. + #[test] + fn the_backend_install_hides_its_window_only_when_no_output_is_a_console() { + for (stdout, stderr, expect_hidden) in [ + (true, true, false), + (true, false, false), + (false, true, false), + (false, false, true), + ] { + let (command, hidden) = backend_install_for(OutputConsoles { stdout, stderr }); + assert_eq!( + hidden, expect_hidden, + "stdout console={stdout}, stderr console={stderr}" + ); + assert_eq!(command.get_program(), PathBuf::from("rt").join("lemonade")); + let args: Vec<_> = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect(); + assert_eq!( + args, + [ + "--host", + "127.0.0.1", + "--port", + "8123", + "backends", + "install", + "llamacpp:rocm" + ] + ); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn serving_gpu_model_load_process_error_is_returned() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let executable = dir.join("lemonade"); + let args_path = dir.join("args.txt"); + rocm_core::fake_executable::write_fake_executable(&executable, "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$(dirname \"$0\")/args.txt\"\nprintf 'GPU backend initialization failed\\n' >&2\nexit 23\n").unwrap(); + let manifest = LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: executable, + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + }; + let log = dir.join("load.log"); + let parent = ParentRuntimeEnvironment::rooted_at(dir); + let error = run_lemonade_model_load_with_parent( + &manifest, + "127.0.0.1", + 11435, + "Qwen3-0.6B-GGUF", + "vulkan", + None, + Some(&log), + &LemonadeProcessEnvironment::default(), + &parent, + ) + .unwrap_err(); + assert!(error.to_string().contains("Lemonade model load failed")); + assert!(error.to_string().contains("23")); + assert!( + fs::read_to_string(log) + .unwrap() + .contains("GPU backend initialization failed") + ); + let args = fs::read_to_string(args_path).unwrap(); + assert!(args.contains("load\nQwen3-0.6B-GGUF\n--llamacpp\nvulkan\n")); + assert!(!args.contains("cpu")); + } + + #[cfg(target_os = "linux")] + fn fake_pull_manifest(dir: &Path, exit_code: i32) -> LemonadeInstallManifest { + let executable = dir.join("lemonade"); + rocm_core::fake_executable::write_fake_executable( + &executable, + format!( + "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$(dirname \"$0\")/args.txt\"\nprintf 'pull output line\\n'\nprintf 'pull failure detail\\n' >&2\nexit {exit_code}\n" + ), + ) + .unwrap(); + LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: executable, + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + } + } + + /// The child's handle and a second appender share one log: a line appended + /// after the child's first write must survive the child's next write. + #[test] + fn a_line_appended_beside_the_childs_log_handle_survives_its_next_write() { + use std::io::Write as _; + let dir = crate::test_support::scratch_dir("service-log-append"); + let log_path = dir.join("serve.log"); + let mut child_handle = open_service_log(&log_path, LogMode::Fresh).unwrap(); + child_handle.write_all(b"child line one\n").unwrap(); + let notice = + crate::device_policy::announce_non_rocm_backend("vulkan", Some(&log_path)).unwrap(); + child_handle.write_all(b"child line two\n").unwrap(); + // The whole file, not a substring: without append the child's second write + // lands on top of the start of the notice and a `contains` on its tail + // would still match. + let log = fs::read_to_string(&log_path).unwrap(); + assert_eq!( + log, + format!("child line one\n{notice}\nchild line two\n"), + "{log}" + ); + fs::remove_dir_all(dir).unwrap(); + } + + /// A helper `lemond` appends to the log of the server whose failure made it + /// necessary; only the serve's own server starts the log over. + #[test] + fn a_helper_server_keeps_the_log_it_is_given() { + use std::io::Write as _; + let dir = crate::test_support::scratch_dir("service-log-modes"); + let log_path = dir.join("serve.log"); + fs::write( + &log_path, + "router failed to load the model +", + ) + .unwrap(); + let mut helper = open_service_log(&log_path, LogMode::Append).unwrap(); + helper + .write_all( + b"helper line +", + ) + .unwrap(); + assert_eq!( + fs::read_to_string(&log_path).unwrap(), + "router failed to load the model +helper line +" + ); + drop(open_service_log(&log_path, LogMode::Fresh).unwrap()); + assert_eq!(fs::read_to_string(&log_path).unwrap(), ""); + fs::remove_dir_all(dir).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn pull_runs_exact_checkpoint_argv() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_pull_manifest(dir, 0); + let log = dir.join("pull.log"); + run_lemonade_pull_with_parent( + &manifest, + "127.0.0.1", + 11436, + "owner/repo:Q4_0", + Some(&log), + &LemonadeProcessEnvironment::default(), + &ParentRuntimeEnvironment::rooted_at(dir), + ) + .unwrap(); + assert_eq!( + fs::read_to_string(dir.join("args.txt")).unwrap(), + "--host\n127.0.0.1\n--port\n11436\npull\nowner/repo:Q4_0\n" + ); + assert!( + fs::read_to_string(log) + .unwrap() + .contains("pull output line") + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn pull_failure_is_returned_with_checkpoint_status_and_logged_output() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_pull_manifest(dir, 7); + let log = dir.join("pull.log"); + let error = run_lemonade_pull_with_parent( + &manifest, + "127.0.0.1", + 11436, + "owner/repo:Q4_0", + Some(&log), + &LemonadeProcessEnvironment::default(), + &ParentRuntimeEnvironment::rooted_at(dir), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("owner/repo:Q4_0"), "{error}"); + assert!(error.contains('7'), "{error}"); + // The process output is streamed to the log rather than embedded in the error. + assert!( + fs::read_to_string(log) + .unwrap() + .contains("pull failure detail") + ); + } + #[test] fn models_ready_accepts_direct_serve_and_gates_router_backend() { // Direct-serve (stock llama-server): no `recipe_options` → ready by name. diff --git a/engines/lemonade/src/router_serve.rs b/engines/lemonade/src/router_serve.rs new file mode 100644 index 0000000000..ed0df4fe81 --- /dev/null +++ b/engines/lemonade/src/router_serve.rs @@ -0,0 +1,860 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Serving through the Lemonade router (`lemond`): spawn the server, select a +//! GPU llama.cpp backend, load the model by its router name and verify it. +//! +//! The orchestration lives behind [`RouterSteps`] so the state it persists and +//! the way it fails can be tested without a real `lemond`. Production uses +//! [`LemondRouterSteps`]. There is deliberately no direct-server step here: any +//! failure is returned to the caller as an error and the server is stopped. The +//! engine state is then left at `failed`, except for a model load or +//! verification failure: that is tagged [`RouterLoadFailure`] and does not write +//! `failed`, so the caller can decide, on Linux, whether the selected GPU +//! backend can serve the model directly instead and own the final state. + +use anyhow::{Context, Result, anyhow}; +use serde_json::json; +use std::path::Path; +use std::time::Duration; + +use crate::install::LemonadeRuntime; +use crate::process::{LemonadeProcessEnvironment, LemondChild, LemondExitStatus}; +use crate::runtime_dir::ParentRuntimeEnvironment; +use crate::{LLAMACPP_RECIPE, ServeHttpRequest}; + +/// Marks a router model-load or load-verification failure (as opposed to a +/// server that never became ready, a failed backend selection, or a server that +/// exited later). Only these are eligible for the direct-serving fallback. +#[derive(Debug)] +pub(crate) struct RouterLoadFailure(pub(crate) String); + +impl std::fmt::Display for RouterLoadFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Whether `error` came from the router failing to load or verify the model. +pub(crate) fn is_router_load_failure(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +/// The side-effecting steps of a router serve. +pub(crate) trait RouterSteps { + fn runtime(&self) -> &LemonadeRuntime; + /// Start `lemond` and return its PID. + fn spawn_server(&mut self) -> Result; + fn wait_ready(&mut self) -> Result<()>; + /// Select (and install if needed) the GPU llama.cpp backend; returns its name. + fn select_backend(&mut self) -> Result; + fn load_model(&mut self, router_model_ref: &str, backend: &str) -> Result<()>; + /// Whether the endpoint reports the model loaded on `backend`. `Err` means the + /// question could not be asked (transport failure, timeout), which is not the + /// same as the endpoint answering that it is not loaded. + fn model_loaded(&mut self, router_model_ref: &str, backend: &str) -> Result; + /// Whether a chat completion succeeds; `Err` as for [`Self::model_loaded`]. + fn chat_smoke(&mut self, router_model_ref: &str) -> Result; + /// Kill and reap the spawned server. + fn stop_server(&mut self); + fn server_start_ticks(&self, pid: u32) -> Option; + /// Block until the server exits. + fn wait_server(&mut self) -> Result; +} + +/// Serve `router_model_ref` through the router. `request.model_ref` keeps the +/// text the user asked for; every router call uses the normalized name. +pub(crate) fn serve_router( + request: &ServeHttpRequest, + steps: &mut S, + router_model_ref: &str, +) -> Result<()> { + let server_pid = steps.spawn_server()?; + // Every step after the spawn is inside this one fallible block, so a failure + // anywhere in it (state writes included) stops the server and marks the state + // `failed`; the server has no `Drop` and would otherwise keep running. + let outcome = (|| -> Result<()> { + crate::state::write_running_state_for_model( + request, + steps.runtime(), + std::process::id(), + Some(server_pid), + "running", + router_model_ref, + )?; + steps + .wait_ready() + .context("Lemonade server did not become ready")?; + let backend = steps + .select_backend() + .context("failed to select a supported Lemonade llama.cpp backend")?; + // A load failure is tagged `RouterLoadFailure`, which is what lets + // `with_direct_fallback` serve a registered model directly on Linux. The + // router's own error is kept and returned when no direct route is open. + steps + .load_model(router_model_ref, &backend) + .map_err(|error| { + error.context(RouterLoadFailure(format!( + "failed to load {} with Lemonade {LLAMACPP_RECIPE}:{backend}", + request.model_ref + ))) + })?; + // Only an answer counts as "not loaded". A probe that could not ask (a + // slow or briefly unreachable endpoint) is a plain failure: it is not a + // `RouterLoadFailure`, so it does not trigger the direct fallback, whose + // download can be several gigabytes, on the strength of a blip. + let verified = steps + .model_loaded(router_model_ref, &backend) + .context("failed to ask Lemonade whether the model is loaded")? + && steps + .chat_smoke(router_model_ref) + .context("failed to run the Lemonade chat smoke test")?; + if !verified { + return Err(anyhow!( + "Lemonade load completed but the endpoint did not report a {LLAMACPP_RECIPE}:{backend}-loaded model" + ) + .context(RouterLoadFailure(format!( + "failed to verify {} with Lemonade {LLAMACPP_RECIPE}:{backend}", + request.model_ref + )))); + } + crate::state::merge_json_state( + &request.state_path, + &json!({ + "status": "ready", + "server_pid": server_pid, + // Identity token for the server PID, captured while the child is alive. + "server_start_ticks": steps.server_start_ticks(server_pid), + "backend_state": "ready", + "backend_requested": backend, + "load_response": { + "status": "loaded", + "method": "lemonade-cli", + "model_name": router_model_ref, + "llamacpp_backend": backend + }, + }), + )?; + // Only now is it true that Lemonade is serving on this backend: the load + // and its verification passed and the `ready` state is written. Announcing + // earlier would claim it for a load that fails, for a ready write that + // fails (the server is then stopped), and again on the fallback. + crate::device_policy::announce_non_rocm_backend(&backend, request.log_path.as_deref()); + Ok(()) + })(); + if let Err(error) = outcome { + steps.stop_server(); + // A load or verification failure may still be served directly, so the + // fallback decides the final state: it sets `starting` when it runs and + // `failed` when it does not (see `with_direct_fallback`). Marking `failed` + // here would let a reader in the gap lock the record at `failed`. + if !is_router_load_failure(&error) { + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + } + return Err(error); + } + let status = match steps.wait_server() { + Ok(status) => status, + Err(error) => { + steps.stop_server(); + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + return Err(error); + } + }; + crate::state::mark_json_status( + &request.state_path, + if status.success() { + "stopped" + } else { + "failed" + }, + )?; + if status.success() { + Ok(()) + } else { + anyhow::bail!("Lemonade server exited with status {status}") + } +} + +/// The real `lemond` and `lemonade` CLI behind [`RouterSteps`]. +pub(crate) struct LemondRouterSteps<'a> { + pub(crate) runtime: LemonadeRuntime, + pub(crate) request: &'a ServeHttpRequest, + pub(crate) process_env: &'a LemonadeProcessEnvironment, + pub(crate) log_path: Option<&'a Path>, + /// The environment the child processes inherit from; tests confine it. + pub(crate) parent: ParentRuntimeEnvironment, + pub(crate) child: Option, +} + +impl RouterSteps for LemondRouterSteps<'_> { + fn runtime(&self) -> &LemonadeRuntime { + &self.runtime + } + + fn spawn_server(&mut self) -> Result { + let child = crate::process::spawn_lemond_with_parent( + &self.runtime.manifest, + &self.request.host, + self.request.port, + self.log_path, + self.process_env, + &self.parent, + crate::process::LogMode::Fresh, + )?; + let pid = child.id(); + self.child = Some(child); + Ok(pid) + } + + fn wait_ready(&mut self) -> Result<()> { + crate::process::wait_for_lemonade_cli_status_with_parent( + &self.runtime.manifest, + &self.request.host, + self.request.port, + Duration::from_secs(30), + self.log_path, + self.process_env, + &self.parent, + ) + } + + fn select_backend(&mut self) -> Result { + crate::llamacpp_backend::ensure_best_llamacpp_backend_with_parent( + &mut self.runtime.manifest, + &self.request.host, + self.request.port, + self.process_env, + false, + &self.parent, + ) + } + + fn load_model(&mut self, router_model_ref: &str, backend: &str) -> Result<()> { + crate::process::run_lemonade_model_load_with_parent( + &self.runtime.manifest, + &self.request.host, + self.request.port, + router_model_ref, + backend, + self.request.engine_recipe.as_ref(), + self.log_path, + self.process_env, + &self.parent, + ) + } + + fn model_loaded(&mut self, router_model_ref: &str, backend: &str) -> Result { + crate::process::query_loaded_model_endpoint( + &crate::state::endpoint_url(&self.request.host, self.request.port), + router_model_ref, + backend, + ) + } + + fn chat_smoke(&mut self, router_model_ref: &str) -> Result { + crate::process::query_chat_smoke_endpoint( + &self.request.host, + self.request.port, + router_model_ref, + ) + } + + fn stop_server(&mut self) { + if let Some(mut child) = self.child.take() { + // The whole tree: a router that failed to load a model can leave a + // backend process holding the device memory, and the fallback starts + // a second server on the same device and port straight after. + let _ = rocm_core::terminate_process_tree(child.id()); + let _ = child.wait(); + } + } + + fn server_start_ticks(&self, pid: u32) -> Option { + rocm_core::process_start_ticks(pid) + } + + fn wait_server(&mut self) -> Result { + self.child + .as_mut() + .context("Lemonade server was not started")? + .wait() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::install::LemonadeInstallManifest; + use std::path::PathBuf; + + #[derive(Clone, Copy, PartialEq, Eq)] + enum Fail { + None, + Ready, + Backend, + Load, + Loaded, + Smoke, + /// The loaded-model probe cannot reach the endpoint. + LoadedUnreachable, + /// The chat smoke probe cannot reach the endpoint. + SmokeUnreachable, + /// The state file cannot be written once the server is spawned. + StateWrite, + /// The state file becomes unwritable between verification and the ready merge. + ReadyMerge, + WaitServer, + FailedExit, + } + + struct FakeSteps { + runtime: LemonadeRuntime, + fail: Fail, + calls: Vec, + state_path: PathBuf, + /// The state file as it stood when the load step ran. + state_at_load: Option, + } + + impl FakeSteps { + fn new(dir: &Path, fail: Fail) -> Self { + Self { + runtime: LemonadeRuntime { + manifest: LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + }, + }, + fail, + calls: Vec::new(), + state_path: dir.join("state.json"), + state_at_load: None, + } + } + } + + impl RouterSteps for FakeSteps { + fn runtime(&self) -> &LemonadeRuntime { + &self.runtime + } + fn spawn_server(&mut self) -> Result { + self.calls.push("spawn".to_owned()); + Ok(4_000_000) + } + fn wait_ready(&mut self) -> Result<()> { + self.calls.push("ready".to_owned()); + if self.fail == Fail::Ready { + anyhow::bail!("not ready"); + } + Ok(()) + } + fn select_backend(&mut self) -> Result { + self.calls.push("backend".to_owned()); + if self.fail == Fail::Backend { + anyhow::bail!("no backend"); + } + Ok("vulkan".to_owned()) + } + fn load_model(&mut self, router_model_ref: &str, backend: &str) -> Result<()> { + self.calls + .push(format!("load:{router_model_ref}:{backend}")); + self.state_at_load = crate::state::read_service_state(&self.state_path).ok(); + if self.fail == Fail::Load { + anyhow::bail!("load exploded"); + } + Ok(()) + } + fn model_loaded(&mut self, router_model_ref: &str, backend: &str) -> Result { + self.calls + .push(format!("loaded:{router_model_ref}:{backend}")); + if self.fail == Fail::LoadedUnreachable { + anyhow::bail!("connection refused"); + } + Ok(self.fail != Fail::Loaded) + } + fn chat_smoke(&mut self, router_model_ref: &str) -> Result { + self.calls.push(format!("smoke:{router_model_ref}")); + if self.fail == Fail::SmokeUnreachable { + anyhow::bail!("operation timed out"); + } + Ok(self.fail != Fail::Smoke) + } + fn stop_server(&mut self) { + self.calls.push("stop".to_owned()); + } + fn server_start_ticks(&self, _pid: u32) -> Option { + // Asked for while the ready state is assembled, i.e. after every + // verification step and before the merge is written. + if self.fail == Fail::ReadyMerge { + std::fs::remove_file(&self.state_path).unwrap(); + std::fs::create_dir(&self.state_path).unwrap(); + } + None + } + fn wait_server(&mut self) -> Result { + self.calls.push("wait".to_owned()); + if self.fail == Fail::WaitServer { + anyhow::bail!("lost the server"); + } + Ok(if self.fail == Fail::FailedExit { + LemondExitStatus::for_test(false, "exit status: 23") + } else { + LemondExitStatus::for_test(true, "exit status: 0") + }) + } + } + + fn request(dir: &Path) -> ServeHttpRequest { + ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "qwen-smoke".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + } + } + + const ROUTER_NAME: &str = "Qwen3-0.6B-GGUF"; + + #[test] + fn router_state_keeps_requested_text_and_records_router_name() { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), Fail::None); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap(); + + // The state written before the load: requested text kept, router name served. + let at_load = steps.state_at_load.expect("state exists before load"); + assert_eq!(at_load["status"], "running"); + assert_eq!(at_load["model_ref"], "qwen-smoke"); + assert_eq!(at_load["canonical_model_id"], ROUTER_NAME); + + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["model_ref"], "qwen-smoke"); + assert_eq!(state["canonical_model_id"], ROUTER_NAME); + assert_eq!(state["load_response"]["model_name"], ROUTER_NAME); + assert_eq!(state["backend_requested"], "vulkan"); + assert_eq!(state["status"], "stopped"); + } + + #[test] + fn router_name_not_alias_reaches_load_loaded_and_smoke_checks() { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), Fail::None); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap(); + assert_eq!( + steps.calls, + [ + "spawn".to_owned(), + "ready".to_owned(), + "backend".to_owned(), + format!("load:{ROUTER_NAME}:vulkan"), + format!("loaded:{ROUTER_NAME}:vulkan"), + format!("smoke:{ROUTER_NAME}"), + "wait".to_owned(), + ] + ); + } + + #[test] + fn router_failures_return_error_stop_server_and_mark_state_failed() { + for (fail, expected, last_call) in [ + (Fail::Ready, "did not become ready", "ready"), + (Fail::Backend, "failed to select a supported", "backend"), + ( + Fail::Load, + "failed to load qwen-smoke with Lemonade", + "load", + ), + ( + Fail::Loaded, + "failed to verify qwen-smoke with Lemonade", + "loaded", + ), + ( + Fail::Smoke, + "failed to verify qwen-smoke with Lemonade", + "smoke", + ), + ] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME) + .expect_err(expected) + .to_string(); + assert!(error.contains(expected), "{error}"); + // The server is stopped, and the router branch has no other step to + // fall back to: the calls are the router sequence up to the failure, + // then the stop, and nothing after it. + assert_eq!( + steps.calls.last().map(String::as_str), + Some("stop"), + "{:?}", + steps.calls + ); + assert!( + steps.calls[steps.calls.len() - 2].starts_with(last_call), + "{:?}", + steps.calls + ); + assert!(!steps.calls.iter().any(|call| call == "wait")); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + // A load or verification failure is left for the direct fallback to + // settle (it sets `starting` or `failed`); the other failures are final. + let expected_status = if matches!(fail, Fail::Load | Fail::Loaded | Fail::Smoke) { + "running" + } else { + "failed" + }; + assert_eq!(state["status"], expected_status, "{expected}"); + assert_eq!(state["model_ref"], "qwen-smoke"); + } + } + + /// A verification probe that could not reach the endpoint is not an answer + /// that the model is missing: it stops the server and fails the serve, and it + /// is not a `RouterLoadFailure`, so the direct fallback (and its download) + /// does not run on the strength of it. An answer of "not loaded" still is one. + #[test] + fn an_unreachable_verification_probe_is_not_a_load_failure() { + for (fail, probe, last_call) in [ + ( + Fail::LoadedUnreachable, + "whether the model is loaded", + "loaded", + ), + (Fail::SmokeUnreachable, "chat smoke test", "smoke"), + ] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME).expect_err(probe); + assert!(!is_router_load_failure(&error), "{error:#}"); + assert!(format!("{error:#}").contains(probe), "{error:#}"); + assert_eq!( + steps.calls.last().map(String::as_str), + Some("stop"), + "{:?}", + steps.calls + ); + assert!( + steps.calls[steps.calls.len() - 2].starts_with(last_call), + "{:?}", + steps.calls + ); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{state}"); + } + for fail in [Fail::Loaded, Fail::Smoke] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert!(is_router_load_failure(&error), "{error:#}"); + } + } + + /// A failure at a step after the spawn (not a router step) must still stop the + /// server it spawned. + fn assert_post_spawn_failure_stops_server(fail: Fail, expected: &str) { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + if fail == Fail::StateWrite { + // A directory where the state file belongs: every write fails. + std::fs::create_dir(&request.state_path).unwrap(); + } + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = format!( + "{:#}", + serve_router(&request, &mut steps, ROUTER_NAME).expect_err(expected) + ); + assert!(error.contains(expected), "{error}"); + // The spawned server is stopped exactly once, and nothing runs after it. + assert_eq!( + steps.calls.last().map(String::as_str), + Some("stop"), + "{:?}", + steps.calls + ); + assert_eq!( + steps.calls.iter().filter(|call| *call == "stop").count(), + 1, + "{:?}", + steps.calls + ); + if fail == Fail::WaitServer { + // The state file is writable here, so the failure is recorded. + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed"); + } + } + + #[test] + fn state_write_failure_right_after_spawn_stops_the_server() { + assert_post_spawn_failure_stops_server(Fail::StateWrite, "failed to write"); + } + + #[test] + fn ready_state_merge_failure_stops_the_server() { + assert_post_spawn_failure_stops_server(Fail::ReadyMerge, "failed to write"); + } + + /// The ready write fails after the model loaded and verified, and the server + /// is stopped: the log must not say Lemonade is serving on the backend. + #[test] + fn a_failed_ready_state_write_does_not_announce_a_backend() { + let temporary = tempfile::tempdir().unwrap(); + let mut request = request(temporary.path()); + request.log_path = Some(temporary.path().join("serve.log")); + let mut steps = FakeSteps::new(temporary.path(), Fail::ReadyMerge); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert_eq!(steps.calls.last().map(String::as_str), Some("stop")); + let log = std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap_or_default(); + assert!(!log.contains("not ROCm"), "{log}"); + } + + #[test] + fn wait_server_failure_stops_the_server_and_marks_state_failed() { + assert_post_spawn_failure_stops_server(Fail::WaitServer, "lost the server"); + } + + #[test] + fn only_a_load_or_verification_failure_is_eligible_for_direct_fallback() { + for (fail, eligible) in [ + (Fail::Load, true), + (Fail::Loaded, true), + (Fail::Smoke, true), + (Fail::Ready, false), + (Fail::Backend, false), + (Fail::StateWrite, false), + (Fail::ReadyMerge, false), + (Fail::WaitServer, false), + (Fail::FailedExit, false), + ] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + if fail == Fail::StateWrite { + std::fs::create_dir(&request.state_path).unwrap(); + } + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert_eq!(is_router_load_failure(&error), eligible, "{error:#}"); + // The server was stopped, except when it had already exited by itself. + assert_eq!( + steps.calls.iter().any(|call| call == "stop"), + fail != Fail::FailedExit, + "{:?}", + steps.calls + ); + } + } + + #[test] + fn a_non_rocm_backend_is_named_in_the_notice_and_matches_the_state() { + let temporary = tempfile::tempdir().unwrap(); + let mut request = request(temporary.path()); + request.log_path = Some(temporary.path().join("serve.log")); + let mut steps = FakeSteps::new(temporary.path(), Fail::None); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap(); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + let backend = state["backend_requested"].as_str().unwrap(); + assert_eq!(backend, "vulkan"); + // The notice is computed from the recorded backend, so it cannot disagree. + let notice = crate::device_policy::non_rocm_backend_notice(backend).unwrap(); + assert!( + notice.contains("vulkan") && notice.contains("not ROCm"), + "{notice}" + ); + // A managed serve discards stdout and stderr, so the router path has to + // put the notice in the serve log, where the user can read it. + let log = std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap(); + assert!(log.contains(¬ice), "{log}"); + assert_eq!(crate::device_policy::non_rocm_backend_notice("rocm"), None); + assert_eq!( + crate::device_policy::non_rocm_backend_notice("rocm-stable"), + None + ); + } + + /// The notice says Lemonade is serving on the backend, so it is written only + /// once the model is loaded and verified: a failed load or verification claims + /// nothing (and the fallback that may follow announces for itself). + #[test] + fn a_failed_load_or_verification_does_not_announce_a_backend() { + for fail in [Fail::Load, Fail::Loaded, Fail::Smoke] { + let temporary = tempfile::tempdir().unwrap(); + let mut request = request(temporary.path()); + request.log_path = Some(temporary.path().join("serve.log")); + let mut steps = FakeSteps::new(temporary.path(), fail); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + let log = + std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap_or_default(); + assert!(!log.contains("not ROCm"), "{log}"); + } + } + + #[test] + fn nonzero_server_exit_returns_error_and_marks_state_failed() { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), Fail::FailedExit); + let error = serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert!( + error + .to_string() + .contains("Lemonade server exited with status exit status: 23") + ); + assert_eq!( + crate::state::read_service_state(&request.state_path).unwrap()["status"], + "failed" + ); + assert_eq!(steps.calls.last().map(String::as_str), Some("wait")); + } + + #[cfg(target_os = "linux")] + #[test] + fn real_router_stop_kills_and_reaps_its_server() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = request(dir); + let env = LemonadeProcessEnvironment::default(); + let mut steps = LemondRouterSteps { + runtime: LemonadeRuntime { + manifest: crate::test_http::fake_runtime::manifest(dir, 0, None), + }, + request: &request, + process_env: &env, + log_path: None, + parent: ParentRuntimeEnvironment::rooted_at(dir), + child: None, + }; + let pid = steps.spawn_server().unwrap(); + let process = PathBuf::from(format!("/proc/{pid}")); + assert!(process.exists(), "the real spawn must create a process"); + steps.stop_server(); + // Clean up even if a no-op stop mutation left the child alive. + let gone = !process.exists(); + if !gone { + let _ = crate::state::terminate_pid(pid, true); + if let Some(mut child) = steps.child.take() { + let _ = child.wait(); + } + } + assert!( + gone, + "stop_server must terminate and reap the spawned lemond" + ); + assert!(steps.child.is_none()); + } + + /// Backend selection runs `lemonade backends` through the serve request's + /// confined parent environment, never the host's: the child must see the + /// confined runtime directory, and the answer must come from its listing. + #[cfg(target_os = "linux")] + #[test] + fn real_backend_selection_runs_under_the_confined_parent() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = request(dir); + let env = LemonadeProcessEnvironment::default(); + let mut manifest = crate::test_http::fake_runtime::manifest(dir, 0, None); + manifest.lemonade = dir.join("lemonade-backends"); + rocm_core::fake_executable::write_fake_executable( + &manifest.lemonade, + r#"#!/bin/sh +printf '%s' "$XDG_RUNTIME_DIR" > "$(dirname "$0")/xdg.txt" +printf 'Recipe Backend Status Message Action +llamacpp vulkan installed ok none +' +"#, + ) + .unwrap(); + let mut steps = LemondRouterSteps { + runtime: LemonadeRuntime { manifest }, + request: &request, + process_env: &env, + log_path: None, + parent: ParentRuntimeEnvironment::rooted_at(dir), + child: None, + }; + assert_eq!(steps.select_backend().unwrap(), "vulkan"); + assert_eq!( + std::fs::read_to_string(dir.join("xdg.txt")).unwrap(), + dir.join("xdg-runtime").to_string_lossy() + ); + // The listing said `installed`, so nothing was installed. + assert_eq!(steps.runtime.manifest.backend_name, "vulkan"); + } + + #[cfg(target_os = "linux")] + #[test] + fn real_steps_send_the_router_name_not_the_requested_alias() { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + // A name the alias `qwen-smoke` does not normalize to, so a step that used + // the requested text could not pass by matching it. + let router_name = "Registered-Model-GGUF"; + let (port, server) = spawn_fixture(Duration::from_secs(10), move |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"all_models_loaded": [{"model_name": router_name, "recipe_options": {"llamacpp_backend": "vulkan"}}]}) + } + .to_string(), + last: chat, + } + }); + let mut request = request(dir); + request.port = port; + assert_eq!(request.model_ref, "qwen-smoke"); + let env = LemonadeProcessEnvironment::default(); + let mut steps = LemondRouterSteps { + runtime: LemonadeRuntime { + manifest: fake_runtime::manifest(dir, 0, None), + }, + request: &request, + process_env: &env, + log_path: None, + parent: ParentRuntimeEnvironment::rooted_at(dir), + child: None, + }; + steps.load_model(router_name, "vulkan").unwrap(); + assert!(steps.model_loaded(router_name, "vulkan").unwrap()); + assert!(steps.chat_smoke(router_name).unwrap()); + + let load = fake_runtime::calls(dir); + assert_eq!( + load, + [format!( + "--host 127.0.0.1 --port {port} load {router_name} --llamacpp vulkan --save-options" + )] + ); + let seen = server.join().unwrap(); + let chat = seen.iter().find(|r| r.method == "POST").expect("chat sent"); + assert_eq!(chat.path, "/v1/chat/completions"); + assert_eq!( + serde_json::from_str::(&chat.body).unwrap()["model"], + router_name + ); + } +} diff --git a/engines/lemonade/src/runtime_dir.rs b/engines/lemonade/src/runtime_dir.rs index 44b021c9b5..cc9ff29706 100644 --- a/engines/lemonade/src/runtime_dir.rs +++ b/engines/lemonade/src/runtime_dir.rs @@ -42,6 +42,22 @@ impl ParentRuntimeEnvironment { } } +// Used by Linux-only fake-executable tests throughout the engine; +// a plain `cfg(test)` leaves it unused (and dead-code denied) on other targets. +#[cfg(all(test, target_os = "linux"))] +impl ParentRuntimeEnvironment { + /// A parent environment confined to `root`: it already names a runtime + /// directory, so resolving it creates nothing and reads no real user dirs. + pub(crate) fn rooted_at(root: &Path) -> Self { + Self { + xdg_runtime_dir: Some(root.join("xdg-runtime").into_os_string()), + home: Some(root.join("home").into_os_string()), + user: Some("test".to_owned()), + temp_dir: root.join("tmp"), + } + } +} + /// Leaf directory the engine gets inside a fallback runtime root, so a crashing /// or misbehaving server cannot disturb the telemetry socket that shares the /// same root. diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index b4b3323be6..56ca91cf06 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -55,6 +55,24 @@ pub(crate) fn write_running_state( pid: u32, server_pid: Option, status: &str, +) -> Result<()> { + write_running_state_for_model( + request, + runtime, + pid, + server_pid, + status, + &request.model_ref, + ) +} + +pub(crate) fn write_running_state_for_model( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + pid: u32, + server_pid: Option, + status: &str, + served_model_ref: &str, ) -> Result<()> { write_state( &request.state_path, @@ -65,7 +83,7 @@ pub(crate) fn write_running_state( "pid": pid, "server_pid": server_pid, "model_ref": request.model_ref, - "canonical_model_id": request.model_ref, + "canonical_model_id": served_model_ref, "host": request.host, "port": request.port, "endpoint_url": endpoint_url(&request.host, request.port), diff --git a/engines/lemonade/src/test_http.rs b/engines/lemonade/src/test_http.rs new file mode 100644 index 0000000000..fc48bcfc70 --- /dev/null +++ b/engines/lemonade/src/test_http.rs @@ -0,0 +1,163 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A test-owned loopback HTTP fixture shared by the engine's tests. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// One request the fixture received. +#[derive(Debug, Clone)] +pub(crate) struct Recorded { + pub(crate) method: String, + pub(crate) path: String, + pub(crate) body: String, +} + +/// What the fixture does with a request: the HTTP status and JSON body to answer +/// with, and whether that was the last request it should serve. +pub(crate) struct Reply { + pub(crate) status: u16, + pub(crate) body: String, + pub(crate) last: bool, +} + +/// Serve loopback HTTP until `respond` marks a reply as the last one or `deadline` +/// passes. Returns the port and a handle yielding every request received. +pub(crate) fn spawn_fixture( + deadline: Duration, + respond: impl Fn(&Recorded) -> Reply + Send + 'static, +) -> (u16, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + listener.set_nonblocking(true).unwrap(); + let handle = std::thread::spawn(move || { + let mut seen = Vec::new(); + let stop_at = Instant::now() + deadline; + while Instant::now() < stop_at { + let Ok((mut stream, _)) = listener.accept() else { + std::thread::sleep(Duration::from_millis(10)); + continue; + }; + // An accepted socket can inherit the listener's non-blocking mode + // (Windows does), which would make the first read fail with WouldBlock. + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let Some(recorded) = read_request(&mut stream) else { + continue; + }; + let reply = respond(&recorded); + seen.push(recorded); + let _ = write!( + stream, + "HTTP/1.1 {} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + reply.status, + reply.body.len(), + reply.body + ); + if reply.last { + break; + } + } + seen + }); + (port, handle) +} + +fn read_request(stream: &mut impl Read) -> Option { + let mut bytes = Vec::new(); + let mut buffer = [0; 1024]; + let (header_end, content_len) = loop { + let count = stream.read(&mut buffer).ok().filter(|count| *count > 0)?; + bytes.extend_from_slice(&buffer[..count]); + if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..end]); + let len = headers + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + break (end + 4, len); + } + }; + while bytes.len() < header_end + content_len { + let count = stream.read(&mut buffer).ok().filter(|count| *count > 0)?; + bytes.extend_from_slice(&buffer[..count]); + } + let head = String::from_utf8_lossy(&bytes[..header_end]).into_owned(); + let mut request_line = head.lines().next()?.split_whitespace(); + Some(Recorded { + method: request_line.next()?.to_owned(), + path: request_line.next()?.to_owned(), + body: String::from_utf8_lossy(&bytes[header_end..header_end + content_len]).into_owned(), + }) +} + +/// Fake `lemond`/`lemonade` executables for the engine's tests. The fake CLI appends +/// one line per invocation to `calls.txt` beside it (`cat` it to see the exact argv), +/// and answers `pull` by dropping a GGUF into the hub cache under `cache/`. On a +/// pull it also copies the service `state.json` beside it to `state-at-pull.json`, +/// so a test can read the engine state as it was while the pull ran. +#[cfg(target_os = "linux")] +pub(crate) mod fake_runtime { + use crate::install::LemonadeInstallManifest; + use std::path::Path; + + pub(crate) fn write_executable(path: &Path, script: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + rocm_core::fake_executable::write_fake_executable(path, script).unwrap(); + } + + /// A manifest whose `lemond` idles until killed and whose `lemonade` records its + /// argv. `pull_exit` is the exit status of `lemonade pull`; on success the pull + /// also creates `pulled_gguf` (relative to `dir/cache/huggingface/hub`), if given. + pub(crate) fn manifest( + dir: &Path, + pull_exit: i32, + pulled_gguf: Option<&str>, + ) -> LemonadeInstallManifest { + write_executable(&dir.join("lemond"), "#!/bin/sh\nexec sleep 30\n"); + let create = pulled_gguf.map_or_else(String::new, |relative| { + let target = dir.join("cache/huggingface/hub").join(relative); + format!( + "mkdir -p '{}' && : > '{}'\n", + target.parent().unwrap().display(), + target.display() + ) + }); + write_executable( + &dir.join("lemonade"), + &format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$(dirname \"$0\")/calls.txt\"\nfor arg in \"$@\"; do\n if [ \"$arg\" = pull ]; then\n cp \"$(dirname \"$0\")/state.json\" \"$(dirname \"$0\")/state-at-pull.json\" 2>/dev/null\n {create} exit {pull_exit}\n fi\ndone\nexit 0\n" + ), + ); + LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + } + } + + /// The `lemonade` invocations so far, one argv per line. + pub(crate) fn calls(dir: &Path) -> Vec { + std::fs::read_to_string(dir.join("calls.txt")) + .unwrap_or_default() + .lines() + .map(str::to_owned) + .collect() + } +} diff --git a/engines/lemonade/src/test_support.rs b/engines/lemonade/src/test_support.rs index 7b79057c86..b35b190d49 100644 --- a/engines/lemonade/src/test_support.rs +++ b/engines/lemonade/src/test_support.rs @@ -8,6 +8,10 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; +#[path = "../../../crates/rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; + /// A fresh, empty scratch directory under the crate's `target/`, named /// `lemonade-fs-test----`. /// @@ -36,6 +40,15 @@ pub(crate) fn scratch_dir(tag: &str) -> PathBuf { dir } +/// The temp name the retired `rocm_core::write_atomic` picked when it +/// replaced `backend_versions.json`: `with_extension("pins.tmp--")` +/// swapped the `json` extension for `pins`, so the name was +/// `backend_versions.pins.tmp--`, with this process's pid. It was +/// fixed by the clock and the pid alone, so a test can occupy it ahead of time. +pub(crate) fn retired_pin_temp_path(pin_file: &Path, millis: u128) -> PathBuf { + pin_file.with_extension(format!("pins.tmp-{millis}-{}", std::process::id())) +} + mod tests { use super::*; diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 6d203b420d..80ad31eca5 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -39,6 +39,15 @@ pub fn verified_torchaudio_release(rocm_sdk_version: &str) -> Option<&'static st wheel_builds::verified_torchaudio_release(rocm_sdk_version) } +/// The `(torch, torchvision)` series the vLLM discover table pairs for +/// `rocm_sdk_version`'s ROCm `major.minor` line, if it has a row. +/// +/// Like [`verified_torchaudio_release`], read by `rocm install sdk` so the SDK +/// and its vLLM install share one table instead of two copies of the pairing. +pub fn verified_torch_stack_series(rocm_sdk_version: &str) -> Option<(&'static str, &'static str)> { + wheel_builds::verified_torch_stack_series(rocm_sdk_version) +} + pub(crate) const DEFAULT_HOST: &str = "127.0.0.1"; #[derive(Parser, Debug)] diff --git a/engines/vllm/src/process.rs b/engines/vllm/src/process.rs index 1d6acf7c1e..87f3d4f9a5 100644 --- a/engines/vllm/src/process.rs +++ b/engines/vllm/src/process.rs @@ -459,16 +459,12 @@ fn vllm_venv_root(runtime: &VllmRuntime) -> Result { runtime.runtime_id ) })?; - python - .parent() - .and_then(Path::parent) - .map(Path::to_path_buf) - .ok_or_else(|| { - anyhow!( - "python executable `{}` is not inside a venv (bin/Scripts) layout", - python.display() - ) - }) + rocm_core::runtime_python_env_root_for_executable(python).ok_or_else(|| { + anyhow!( + "python executable `{}` is not inside a venv (bin/Scripts) layout", + python.display() + ) + }) } /// Finds `/lib/python3.*/site-packages`, the one location a venv keeps /// its packages under. Bails on anything other than exactly one `python3.*` @@ -1083,6 +1079,31 @@ mod tests { PathBuf::from("/opt/venv") ); } + /// The runtime's venv is found by the rule every caller shares (see + /// `rocm_core::runtime_python_env_root_for_executable`): an interpreter in + /// an unrelated folder is refused, not mapped to that folder's parent, + /// which would send the site-packages lookup into a folder that is not a + /// venv. + #[test] + fn vllm_venv_root_needs_a_bin_or_scripts_folder() { + for (python, expected) in [ + ("/opt/venv/bin/python", Some("/opt/venv")), + ("/opt/venv/Scripts/python.exe", Some("/opt/venv")), + ("/opt/venv/SCRIPTS/python.exe", Some("/opt/venv")), + ("/opt/venv/tools/python", None), + ("python", None), + ("/python", None), + ] { + let runtime = test_vllm_runtime(Some(PathBuf::from(python)), None); + let root = vllm_venv_root(&runtime); + if let Some(expected) = expected { + assert_eq!(root.unwrap(), PathBuf::from(expected), "{python}"); + } else { + let error = root.expect_err(python).to_string(); + assert!(error.contains("is not inside a venv"), "{python}: {error}"); + } + } + } #[test] fn venv_site_packages_dir_finds_the_single_python3_dir() -> Result<()> { let venv = tempfile::tempdir()?; diff --git a/engines/vllm/src/repair.rs b/engines/vllm/src/repair.rs index f9969153ac..9718abe3e7 100644 --- a/engines/vllm/src/repair.rs +++ b/engines/vllm/src/repair.rs @@ -72,12 +72,14 @@ fn torch_alignment_disabled() -> bool { /// next invocation starts over — two full torch-stack flips a run, and a warning /// claiming a repair that undid the intended state. /// -/// With the alignment disabled the rule is simply "any torch pin": see below. +/// With the alignment disabled the rule is simply "any pin on the torch stack" (torch, +/// torchvision, torchaudio, triton): see below. /// /// Otherwise three conditions, all required, and they are exactly the rule rocm-cli /// applies: take the *release* from the engine's pin and the *build* from the SDK. /// -/// The violation must be about torch — any other unmet requirement is real. The +/// The violation must be about torch — any other unmet requirement is real. (Opted +/// out, the packages the SDK installs alongside torch count as torch here.) The /// installed release must be the one the engine pins; an SDK torch of a *different* /// release is the separate bug where the engine cannot accept what the SDK installed, /// and a reinstall is the right answer there. And the installed build must be the one @@ -92,7 +94,16 @@ fn is_intended_torch_divergence( let Some(subject) = violation_subject(detail) else { return false; }; - if !subject.package.eq_ignore_ascii_case(TORCH_PACKAGE) { + // Opted out, the whole torch stack is spared; aligned, only torch itself is ever + // an intended divergence, because rocm-cli realigns torch and nothing else. + let package_in_scope = if torch_alignment_disabled { + rocm_core::SDK_TORCH_STACK_PACKAGES + .iter() + .any(|package| subject.package.eq_ignore_ascii_case(package)) + } else { + subject.package.eq_ignore_ascii_case(TORCH_PACKAGE) + }; + if !package_in_scope { return false; } // Opted out, so rocm-cli does not choose this runtime's torch and no build it @@ -100,8 +111,9 @@ fn is_intended_torch_divergence( // The build and release tests below are the aligned-case rule — asking a // hand-installed torch to match the SDK's build would fail every time, and the // reinstall that followed would install the engine's build over exactly the torch - // the opt-out exists to keep. Only torch is spared: the check above already - // rejected every other package, so an unrelated vLLM-owned defect still repairs. + // the opt-out exists to keep. Only the torch stack is spared: the check above + // already rejected every other package, so an unrelated vLLM-owned defect still + // repairs. if torch_alignment_disabled { return true; } @@ -126,9 +138,10 @@ fn is_intended_torch_divergence( /// treated as intended and the previous behaviour stands. /// /// `torch_alignment_disabled` is the user's opt-out. It changes which violations count -/// as defects, never whether defects are acted on: a torch pin stops being one, and -/// everything else vLLM requires is assessed exactly as before. Returning early on the -/// opt-out instead would hide a broken torchvision behind an unrelated preference. +/// as defects, never whether defects are acted on: a pin on the torch stack stops being +/// one, and everything else vLLM requires is assessed exactly as before. Returning +/// early on the opt-out instead would hide a broken unrelated pin behind a preference +/// about torch. fn repair_from_violations( violations: &[DependencyViolation], sdk_torch_build: Option<&str>, @@ -156,7 +169,7 @@ fn repair_from_violations( // rocm-cli never touched it. Reusing the first line for the second case would // tell a user who hand-installed torch that the CLI had installed it for them. let headline = if torch_alignment_disabled { - "torch alignment is disabled by ROCM_CLI_DISABLE_TORCH_ALIGNMENT; the torch this runtime holds is the user's and a reinstall would replace it" + "torch alignment is disabled by ROCM_CLI_DISABLE_TORCH_ALIGNMENT; the torch stack this runtime holds is the user's and a reinstall would replace it" } else { "the runtime holds the SDK's build of the torch vLLM pins; that divergence is intended and a reinstall would undo it" }; @@ -195,11 +208,9 @@ fn repair_from_violations( // The hint blames `rocm install sdk` for writing the SDK torch stack over vLLM's // pins, which stops being a live theory once the manifest names the SDK's build: // the alignment then identifies that stack and settles it, so a defect surviving - // to here is something else. It stays on under the opt-out, which spares only the - // package named `torch` — a `torchvision` or `torchaudio` defect is still the SDK - // stack written over vLLM's pins, and the opt-out has turned off the step that - // would have corrected it, so the hint is more use there rather than less. - if sdk_torch_build.is_none() { + // to here is something else. Under the opt-out the torch stack is spared, so a + // defect that survives to here is not the SDK stack written over vLLM's pins. + if sdk_torch_build.is_none() && !torch_alignment_disabled { notes.push( "if this recurs after `rocm install sdk`, the SDK torch stack is being written over vLLM's pinned torch".to_owned(), ); @@ -510,6 +521,50 @@ mod tests { ); } #[test] + fn an_opted_out_runtime_keeps_the_whole_sdk_torch_stack() { + // What the CI GPU lanes see after `rocm install sdk` on an engine runtime: + // the SDK writes torch, torchvision, torchaudio and triton together, and each + // one violates vLLM's pin. Sparing only torch left the other three as defects, + // and the reinstall they drove installed the engine's exact torch over the + // one the opt-out exists to keep. + let assessment = repair_from_violations( + &[ + violation( + "vllm", + "The package `vllm` requires `triton==3.6.0`, but `3.8.0+git4cff872c.rocm7.14.1` is installed", + ), + violation( + "vllm", + "The package `vllm` requires `torchvision==0.24.1+d801a34`, but `0.26.0+rocm7.14.1` is installed", + ), + violation( + "vllm", + "The package `vllm` requires `torchaudio==2.9.0+eaa9e4e`, but `2.11.0+rocm7.14.1` is installed", + ), + violation( + "vllm", + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.14.1` is installed", + ), + ], + Some(SDK_BUILD), + OPTED_OUT, + ); + + assert!( + !assessment.needed, + "the opt-out must spare every package of the torch stack: {:?}", + assessment.notes + ); + assert!( + assessment + .notes + .iter() + .all(|note| !note.contains("was reinstalled")), + "no note may claim a repair that did not happen: {:?}", + assessment.notes + ); + } + #[test] fn an_opted_out_custom_torch_still_repairs_an_unrelated_defect() { // The opt-out is about torch, not about the environment. A vLLM-owned pin that // has nothing to do with torch is broken the same way it was before, and @@ -524,7 +579,7 @@ mod tests { ), violation( "vllm", - "The package `vllm` requires `torchvision==0.24.1+d801a34`, but `0.20.0+cu128` is installed", + "The package `vllm` requires `numpy==1.26.4`, but `2.1.0` is installed", ), ], Some(SDK_BUILD), @@ -548,7 +603,7 @@ mod tests { assessment.notes ); assert!( - violation_notes[0].contains("torchvision=="), + violation_notes[0].contains("numpy=="), "the defect named must be the unrelated one: {:?}", assessment.notes ); @@ -564,6 +619,33 @@ mod tests { ); } #[test] + fn an_opted_out_defect_with_an_unidentified_sdk_build_gets_no_sdk_hint() { + // The "SDK torch stack is being written over vLLM's pinned torch" hint is only + // a live theory when the torch stack is not spared. Under the opt-out it is, so + // a defect that survives is not that. `sdk_torch_build: None` is the one input + // where the hint's guard on the opt-out changes the outcome. + let violations = [violation( + "vllm", + "The package `vllm` requires `numpy==1.26.4`, but `2.1.0` is installed", + )]; + let hint = "the SDK torch stack is being written over"; + + let opted_out = repair_from_violations(&violations, None, OPTED_OUT); + assert!(opted_out.needed, "{:?}", opted_out.notes); + assert!( + !opted_out.notes.iter().any(|note| note.contains(hint)), + "no SDK hint under the opt-out: {:?}", + opted_out.notes + ); + + let aligned = repair_from_violations(&violations, None, false); + assert!( + aligned.notes.iter().any(|note| note.contains(hint)), + "the hint still appears without the opt-out: {:?}", + aligned.notes + ); + } + #[test] fn a_settled_runtime_converges_on_the_build_its_own_manifest_records() { // The convergence proof the tests above cannot give on their own. They hand // the classification a build literal, so a change to what diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 2691dc62fa..bf2fce5385 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -484,9 +484,10 @@ mod tests { // Uses the test's own live PID rather than a fixed placeholder so that, // on Linux, the `start_ticks` assertion below exercises the real // `rocm_core::process_start_ticks` lookup rather than only checking - // the field is present. On other platforms `process_start_ticks` is a - // stub that always returns `None`, so there the assertion only checks - // `None == None`. + // the field is present. On Windows it is the process creation time and + // the assertion still compares like with like; on any other platform + // `process_start_ticks` is a stub that always returns `None`, so there + // the assertion only checks `None == None`. let own_pid = std::process::id(); let state_path = probe_state_path("therock-env"); let request = ServeHttpRequest { diff --git a/engines/vllm/src/wheel_builds.rs b/engines/vllm/src/wheel_builds.rs index e1358d9608..6334eb7085 100644 --- a/engines/vllm/src/wheel_builds.rs +++ b/engines/vllm/src/wheel_builds.rs @@ -47,6 +47,21 @@ pub(crate) fn verified_torchaudio_release(rocm_sdk_version: &str) -> Option<&'st } } +/// The torch and torchvision series (`major.minor` prefixes, e.g. `("2.13", +/// "0.28")`) the discover table pairs for `rocm_sdk_version`'s ROCm `major.minor` +/// line. `None` for a line with no row. +/// +/// Read by `rocm install sdk` so the SDK resolves the same pair the vLLM install +/// realigns onto. Left to `uv`, torch resolves to the newest build on the index +/// while torchvision's metadata names no exact torch, so the two can land on a +/// pair whose torchvision cannot register its operators. +pub(crate) fn verified_torch_stack_series( + rocm_sdk_version: &str, +) -> Option<(&'static str, &'static str)> { + let build = vllm_rocm_discover_build(rocm_sdk_version)?; + Some((build.torch_version_prefix, build.torchvision_version_prefix)) +} + /// Whether `recorded` (a runtime manifest's live `rocm_sdk.__version__` probe) /// and `table_key` (a literal key in [`VLLM_ROCM_DISCOVER_BUILD_TABLE`]) share /// a ROCm SDK major version, ignoring minor, patch, and any dev/pre-release @@ -528,6 +543,25 @@ mod tests { assert_eq!(verified_torchaudio_release("10.2.0"), None); assert_eq!(verified_torchaudio_release("7.2.3"), None); } + /// The SDK resolves the pair this table verified, not whatever `uv` finds + /// newest, and a line with no row is left to `uv`. + #[test] + fn verified_torch_stack_series_is_the_row_pairing() { + assert_eq!( + verified_torch_stack_series("10.1.0"), + Some(("2.13", "0.28")) + ); + assert_eq!( + verified_torch_stack_series("10.1.0a20260928"), + Some(("2.13", "0.28")) + ); + assert_eq!( + verified_torch_stack_series("10.0.0"), + Some(("2.12", "0.27")) + ); + assert_eq!(verified_torch_stack_series("10.2.0"), None); + assert_eq!(verified_torch_stack_series("7.2.3"), None); + } /// An unknown 10.x line fails closed rather than silently resolving the /// static table's 7.2.3 pin, which is the whole point of keeping /// [`rocm_sdk_major_matches`] alongside the narrower row lookup. diff --git a/install.sh b/install.sh index ec31a401bc..01d5820edb 100755 --- a/install.sh +++ b/install.sh @@ -121,6 +121,11 @@ ultUtiz4emTfXDCb2RptOgoydBA+mu9z6O4eVF8S5dVr/S834SK6dD2fWHNnT0dc JwIDAQAB -----END PUBLIC KEY-----" PINNED_RELEASE_PUBLIC_KEY_NEXT="" +# Keep each slot to the single assignment above: `cargo xtask verify-pinned-keys` +# compares exactly that assignment with docs/keys/ and requires this readonly +# line, which makes any later write fail instead of silently replacing the +# trust root. +readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT has_pinned_release_keys() { [ -n "${PINNED_RELEASE_PUBLIC_KEY_CURRENT}" ] || [ -n "${PINNED_RELEASE_PUBLIC_KEY_NEXT}" ] @@ -246,7 +251,7 @@ profile_path_for_shell() { case "$(shell_name)" in bash) printf '%s\n' "${HOME}/.bashrc" ;; zsh) printf '%s\n' "${HOME}/.zshrc" ;; - fish) printf '%s\n' "${HOME}/.config/fish/config.fish" ;; + fish) fish_config_path ;; ksh) printf '%s\n' "${HOME}/.kshrc" ;; *) printf '%s\n' "${HOME}/.profile" ;; esac @@ -289,11 +294,14 @@ append_path_snippet() { path_expr="$3" escaped_path_expr="$(escape_for_double_quotes "${path_expr}")" + # A profile that cannot be created leaves nothing for the `cat >>` below to + # append to, and that write returns non-zero before "updated" is printed, so + # the caller reports that the folder is not saved instead of claiming it is. profile_dir="${profile%/*}" if [ "${profile_dir}" != "${profile}" ]; then - mkdir -p "${profile_dir}" + mkdir -p "${profile_dir}" 2>/dev/null || true fi - [ -f "${profile}" ] || : > "${profile}" + [ -f "${profile}" ] || { : > "${profile}"; } 2>/dev/null || true if profile_has_path_entry "${profile}" "${path_expr}"; then printf '%s\n' "unchanged:${profile}" @@ -302,7 +310,7 @@ append_path_snippet() { case "${shell_kind}" in fish) - cat >> "${profile}" <> "${profile}" <>> rocm-cli path >>> if not contains -- "${escaped_path_expr}" \$PATH @@ -312,7 +320,7 @@ end EOF ;; *) - cat >> "${profile}" <> "${profile}" <>> rocm-cli path >>> case ":\$PATH:" in @@ -354,6 +362,112 @@ record_path_edit() { printf '%s\n' "$1" >> "${record}" } +# The folder fish reads its config from: $XDG_CONFIG_HOME/fish when that is +# set to an absolute path, else ~/.config/fish. fish would also take a relative +# XDG_CONFIG_HOME, but against its own working directory, so it names a +# different folder from each directory; the installer does not follow that. +fish_config_dir() { + case "${XDG_CONFIG_HOME:-}" in + /*) printf '%s\n' "${XDG_CONFIG_HOME}/fish" ;; + *) printf '%s\n' "${HOME}/.config/fish" ;; + esac +} + +fish_config_path() { + printf '%s\n' "$(fish_config_dir)/config.fish" +} + +# Whether fish is set up on this machine, whatever the login shell is: the +# binary is on PATH, or the user already has a fish config directory. A bash +# login shell is common even for people who run fish interactively. +fish_is_installed() { + command -v fish >/dev/null 2>&1 && return 0 + [ -d "$(fish_config_dir)" ] && return 0 + return 1 +} + +# Append the fish PATH block to fish's config.fish and record it so +# `rocm uninstall` can take it out again. The caller skips this when that file +# is the profile the login-shell step just wrote. +configure_fish_path() { + fish_profile="$(fish_config_path)" + fish_expr="$1" + fish_result="$(append_path_snippet "${fish_profile}" fish "${fish_expr}")" || true + if profile_has_own_block "${fish_profile}" "${fish_expr}"; then + record_path_edit "profile ${fish_profile}" + record_path_edit "path-expr ${fish_expr}" + fi + case "${fish_result}" in + updated:*) + ok "fish shell profile updated:" + info "profile: ${fish_result#updated:}" + ;; + unchanged:*) + info "fish shell profile already configured:" + info "profile: ${fish_result#unchanged:}" + ;; + *) + info "note: ${INSTALL_DIR} is not saved in your fish config" + ;; + esac +} + +# The first line of the fish completion script this installer writes, followed +# by the install folder. `rocm uninstall` removes the script only when it still +# starts with this line naming its own folder, so a script the user wrote, or +# one another install folder wrote over this one, stays. Keep it in step with +# FISH_COMPLETIONS_MARKER in apps/rocm/src/uninstall.rs. +FISH_COMPLETIONS_MARKER='# rocm-cli fish completions for ' + +# Whether file $1 is a regular file this installer (any install folder) wrote: +# not a link, and starting with the marker line. +is_installer_completions() { + [ -f "$1" ] && [ ! -L "$1" ] || return 1 + first_line="" + IFS= read -r first_line < "$1" || [ -n "${first_line}" ] || return 1 + case "${first_line}" in + "${FISH_COMPLETIONS_MARKER}"?*) return 0 ;; + *) return 1 ;; + esac +} + +# Install fish completions from the binary that was just placed, so they match +# it, and record the file so `rocm uninstall` removes it with the binary. Best +# effort: a binary that cannot print them leaves the install intact. The binary +# reads nothing from stdin, which under `curl | sh` is the rest of this script. +# +# rocm.fish is one fixed path, so a file or link that is already there and did +# not come from this installer is the user's: it is left alone, never written +# through, and not recorded, so `rocm uninstall` leaves it too. +install_fish_completions() { + completions_dir="$(fish_config_dir)/completions" + completions_path="${completions_dir}/rocm.fish" + completions_tmp="${tmp_dir}/rocm.fish" + completions_marked="${tmp_dir}/rocm.fish.marked" + if { [ -e "${completions_path}" ] || [ -L "${completions_path}" ]; } \ + && ! is_installer_completions "${completions_path}"; then + info "note: fish completions not installed: ${completions_path} already exists and was not written by this installer" + info "it is left as it is; remove it and run the installer again to replace it" + return 0 + fi + if "${INSTALL_DIR}/rocm" completions fish < /dev/null > "${completions_tmp}" \ + && [ -s "${completions_tmp}" ] \ + && { printf '%s%s\n' "${FISH_COMPLETIONS_MARKER}" "${INSTALL_DIR}" \ + && cat "${completions_tmp}"; } > "${completions_marked}" \ + && mkdir -p "${completions_dir}" \ + && install -m 0644 "${completions_marked}" "${completions_path}"; then + record_path_edit "completions ${completions_path}" + ok "fish completions installed:" + info "${completions_path}" + else + info "note: could not install fish completions" + info "to try again, run the installer again, or run:" + info "mkdir -p '${completions_dir}'" + info "'${INSTALL_DIR}/rocm' completions fish > '${completions_path}'" + info "(a file written that way has no installer marker, so rocm uninstall leaves it)" + fi +} + ensure_installer_process_path() { case ":${PATH}:" in *:"${INSTALL_DIR}":*) @@ -628,6 +742,15 @@ case ":${PATH}:" in info "rocm is installed here: ${INSTALL_DIR}/rocm" ;; esac + # Fish gets its own snippet when it is installed, including when the + # login shell is bash, zsh, or ksh. Skip the second write when that + # login shell already targeted config.fish. + if fish_is_installed; then + if [ "${profile_path}" != "$(fish_config_path)" ]; then + configure_fish_path "${path_expr}" + fi + install_fish_completions + fi else info "shell profile update skipped" info "rocm is installed here: ${INSTALL_DIR}/rocm" diff --git a/scripts/nightly_canary.py b/scripts/nightly_canary.py new file mode 100644 index 0000000000..7c3bcb280c --- /dev/null +++ b/scripts/nightly_canary.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +"""Metadata-only canary for the current TheRock nightly stream. + +Runs ``rocm install sdk --channel nightly --family F --dry-run`` for each +family and checks that the plan it prints still comes from the current +(next-layout) nightly stream, which an unpinned nightly install resolves to: +the source constants are read from the resolver's own source file, so the +canary cannot drift from them, and no build version is hard-coded. Nothing is installed and no GPU is needed. + +What it fails on: + +* the dry run itself failing (unknown layout, redirect, no complete build); +* a channel, source layout generation or source URL other than the current + nightly ones, which covers the earlier multi-arch endpoint or the stable + stream taking part in nightly resolution; +* a selected version that is not a dated nightly (``X.Y.ZaYYYYMMDD``); +* a selected build older than ``--max-age-days``, which is how a resolver that + quietly keeps choosing a stale train shows up. + +It also lists the ``rocm`` wheel versions the index publishes and reports, as a +warning rather than a failure, when a newer one exists than the one selected: +the resolver skips a build that is not yet complete, so a newer listing alone +is not a fault. + +What it does not cover: the hardware install lane (a real GPU operation), and +whether a newer listed build is complete. See docs/testing.md. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import re +import subprocess +import sys +import urllib.request +from pathlib import Path + +DEFAULT_SOURCE_FILE = Path("apps/rocm/src/therock/source.rs") +# The next layout selects its device payload by exact arch and refuses a +# grouped family such as ``gfx120X-all``, so the probes name exact arches. +# Instinct arches are Linux-only, so a Windows run probing one would fail for a +# reason that is not drift. +DEFAULT_FAMILIES = ("gfx1100", "gfx1200") + ( + () if sys.platform == "win32" else ("gfx950",) +) +DEFAULT_MAX_AGE_DAYS = 3 +NIGHTLY_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+a(?P\d{8})$") +WHEEL_VERSION_RE = re.compile(r"rocm-(?P\d+\.\d+\.\d+a\d{8})[-.]") +FETCH_TIMEOUT_SECS = 30 +DRY_RUN_TIMEOUT_SECS = 300 + + +class CanaryError(Exception): + """The canary could not run to a verdict.""" + + +def read_constants(source_text: str) -> dict[str, str]: + """The current nightly base and layout generation, from the resolver source.""" + base = re.search( + r'const THEROCK_NEXT_NIGHTLY_PIP_INDEX_BASE: &str = "([^"]+)";', source_text + ) + generation = re.search( + r'const THEROCK_NEXT_LAYOUT_GENERATION: &str = "([^"]+)";', source_text + ) + if not base or not generation: + raise CanaryError( + "could not read the nightly index base and layout generation from " + "the resolver source; the constants were renamed or moved" + ) + return {"base": base.group(1).rstrip("/"), "generation": generation.group(1)} + + +def parse_plan(output: str) -> dict[str, str]: + """The ``key: value`` lines of a dry-run plan.""" + plan: dict[str, str] = {} + for line in output.splitlines(): + match = re.match(r"^\s+([a-z_]+): (.*\S)\s*$", line) + if match: + plan.setdefault(match.group(1), match.group(2)) + return plan + + +def check_plan( + plan: dict[str, str], + constants: dict[str, str], + today: dt.date, + max_age_days: int, +) -> list[str]: + """Why this plan is not the current nightly one; empty when it is.""" + failures: list[str] = [] + if plan.get("channel") != "nightly": + failures.append(f"channel is {plan.get('channel')!r}, expected 'nightly'") + generation = plan.get("source_layout_generation") + if generation != constants["generation"]: + failures.append( + f"source layout generation is {generation!r}, " + f"expected {constants['generation']!r}" + ) + for key in ("canonical_source", "index_url"): + value = plan.get(key) + if value is None: + failures.append(f"the plan has no {key} line") + elif not value.rstrip("/").startswith(constants["base"]): + failures.append( + f"{key} is {value!r}, which is not under the current " + f"nightly stream {constants['base']!r}" + ) + version = plan.get("selected_rocm_version", "") + match = NIGHTLY_VERSION_RE.match(version) + if not match: + failures.append( + f"selected version {version!r} is not a dated nightly (X.Y.ZaYYYYMMDD)" + ) + else: + built = dt.datetime.strptime(match.group("date"), "%Y%m%d").date() + age = (today - built).days + if age > max_age_days: + failures.append( + f"selected build {version} is {age} days old " + f"(limit {max_age_days}); the stream may have moved on" + ) + failures += mixed_build_ids(plan) + return failures + + +def mixed_build_ids(plan: dict[str, str]) -> list[str]: + """Package specs that name a ROCm build other than the selected one. + + The plan's ``package_specs`` line pins ``rocm`` and each framework package's + ``+rocm`` label; all of them must be the one selected build, or the + install would mix builds. A plan without the line (a layout that does not + pin the stack in the preview) is not judged here. + """ + selected = plan.get("selected_rocm_version") + specs = plan.get("package_specs", "").split() + problems = [] + for spec in specs: + if spec.startswith("rocm[") or spec.startswith("rocm=="): + build = spec.rsplit("==", 1)[-1] + elif "+rocm" in spec: + build = spec.rsplit("+rocm", 1)[-1] + else: + continue + if build != selected: + problems.append( + f"package spec {spec!r} names ROCm build {build!r}, " + f"not the selected {selected!r}" + ) + return problems + + +def newest_listed_version(listing_html: str) -> str | None: + """The newest dated nightly ``rocm`` wheel version in an index listing.""" + versions = set(WHEEL_VERSION_RE.findall(listing_html)) + return max(versions, key=lambda v: v.rsplit("a", 1)[1], default=None) + + +def fetch(url: str) -> str: + with urllib.request.urlopen(url, timeout=FETCH_TIMEOUT_SECS) as response: + return response.read().decode("utf-8", errors="replace") + + +def probe(rocm: str, family: str, constants: dict[str, str], max_age_days: int): + """Resolve one family; return ``(failures, warnings)``.""" + command = [ + rocm, "install", "sdk", "--channel", "nightly", + "--family", family, "--dry-run", + ] # fmt: skip + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=DRY_RUN_TIMEOUT_SECS, + check=False, + ) + except subprocess.TimeoutExpired: + return [f"{family}: dry run did not finish in {DRY_RUN_TIMEOUT_SECS}s"], [] + if result.returncode != 0: + tail = (result.stderr or result.stdout).strip().splitlines()[-3:] + return [ + f"{family}: dry run failed ({result.returncode}): {' | '.join(tail)}" + ], [] + plan = parse_plan(result.stdout) + failures = [ + f"{family}: {problem}" + for problem in check_plan( + plan, constants, dt.datetime.now(dt.UTC).date(), max_age_days + ) + ] + warnings: list[str] = [] + index_url = plan.get("index_url") + if index_url: + try: + newest = newest_listed_version(fetch(index_url.rstrip("/") + "/rocm/")) + except OSError as error: + warnings.append(f"{family}: could not read the index listing: {error}") + else: + selected = plan.get("selected_rocm_version", "") + print(f"{family}: selected {selected}, newest listed {newest}") + if newest and newest.rsplit("a", 1)[1] > selected.rsplit("a", 1)[-1]: + warnings.append( + f"{family}: index lists {newest}, newer than the selected " + f"{selected}; fine if that build is still incomplete" + ) + return failures, warnings + + +SAMPLE_PLAN = """sdk install + channel: nightly + canonical_source: https://nightly.repo.amd.com/rocm/whl-next + selected_rocm_version: 10.3.0a20261101 + build_date: 2026-11-01 + source_layout_generation: next-v1 + format: wheel + device_target: + index_url: https://nightly.repo.amd.com/rocm/whl-next + package_specs: rocm[libraries,device-gfx1200]==10.3.0a20261101 torch[device-gfx1200]==2.11.0+rocm10.3.0a20261101 torchaudio==2.11.1+rocm10.3.0a20261101 + mode: dry-run +""" +SAMPLE_SOURCE = """ +const THEROCK_NEXT_NIGHTLY_PIP_INDEX_BASE: &str = "https://nightly.repo.amd.com/rocm/whl-next"; +const THEROCK_NEXT_LAYOUT_GENERATION: &str = "next-v1"; +""" + + +def self_test() -> None: + """Each rule must reject the plan it exists for, and accept a good one.""" + constants = read_constants(SAMPLE_SOURCE) + assert constants == { + "base": "https://nightly.repo.amd.com/rocm/whl-next", + "generation": "next-v1", + }, constants + today = dt.date(2026, 11, 2) + + def failures(**overrides: str | None) -> list[str]: + plan = parse_plan(SAMPLE_PLAN) + for key, value in overrides.items(): + if value is None: + plan.pop(key) + else: + plan[key] = value + return check_plan(plan, constants, today, DEFAULT_MAX_AGE_DAYS) + + assert failures() == [] + assert any("channel" in f for f in failures(channel="release")) + earlier = failures(source_layout_generation="multi-arch-v2") + assert any("layout generation" in f for f in earlier) + legacy = "https://rocm.nightlies.amd.com/whl-multi-arch" + assert any("canonical_source" in f for f in failures(canonical_source=legacy)) + assert any("index_url" in f for f in failures(index_url=legacy)) + assert any("index_url" in f for f in failures(index_url=None)) + assert any("dated nightly" in f for f in failures(selected_rocm_version="10.3.1")) + assert any( + "days old" in f for f in failures(selected_rocm_version="10.3.0a20261020") + ) + assert failures(selected_rocm_version="10.3.0a20261030") != [], ( + "the specs still name the 1101 build, so a different selection is a mix" + ) + mixed = SAMPLE_PLAN.replace( + "torch[device-gfx1200]==2.11.0+rocm10.3.0a20261101", + "torch[device-]==2.11.0+rocm10.3.0a20261015", + ) + mixed_failures = check_plan( + parse_plan(mixed), constants, today, DEFAULT_MAX_AGE_DAYS + ) + assert any("names ROCm build" in f for f in mixed_failures), mixed_failures + assert failures(package_specs=None) == [], "no package_specs line: not judged" + try: + read_constants('const OTHER: &str = "x";') + except CanaryError: + pass + else: + raise AssertionError("a renamed constant must fail loudly, not pass") + listing = ( + 'x' + 'y' + ) + assert newest_listed_version(listing) == "10.3.0a20261103" + assert newest_listed_version("") is None + print("nightly canary self-test: ok") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--rocm", help="path to the rocm binary to probe") + parser.add_argument("--source-file", type=Path, default=DEFAULT_SOURCE_FILE) + parser.add_argument("--family", action="append", dest="families") + parser.add_argument("--max-age-days", type=int, default=DEFAULT_MAX_AGE_DAYS) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + self_test() + return 0 + if not args.rocm: + parser.error("--rocm is required unless --self-test is given") + constants = read_constants(args.source_file.read_text(encoding="utf-8")) + all_failures: list[str] = [] + for family in args.families or DEFAULT_FAMILIES: + failures, warnings = probe(args.rocm, family, constants, args.max_age_days) + for warning in warnings: + print(f"::warning::{warning}") + for failure in failures: + print(f"::error::{failure}") + if not failures: + print(f"{family}: current nightly resolution ok") + all_failures += failures + return 1 if all_failures else 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CanaryError as error: + print(f"::error::{error}") + sys.exit(2) diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 90ca4b192f..73e1f2df05 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -46,11 +46,12 @@ set -euo pipefail # The scenario root stays ABSOLUTE, and follows TMPDIR. # # Scenario dirs come from `TempDir::with_prefix("rocm-e2e-")`, which builds on -# `std::env::temp_dir()` and so follows TMPDIR; the nightly Strix lane redirects -# it to `$HOME/actions-runner/tmp`. A bare `/tmp/rocm-e2e` kept matching there -# only by accident — that directory is itself named `tmp`, so the substring -# still appeared — and would silently stop matching under any redirect not -# named `tmp`. Deriving the redirected form fixes that for ANY target. +# `std::env::temp_dir()` and so follows TMPDIR; a lane on a host that redirects +# it (the nightly Strix lane did, to `$HOME/actions-runner/tmp`, before it moved +# to the pool) is the case this handles. A bare `/tmp/rocm-e2e` kept matching +# there only by accident — that directory is itself named `tmp`, so the +# substring still appeared — and would silently stop matching under any +# redirect not named `tmp`. Deriving the redirected form fixes that for ANY target. # # WHAT THE ANCHORING DOES AND DOES NOT BUY. Roots are matched as unanchored # substrings of the WHOLE command line, so any root also matches a process that diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh new file mode 100755 index 0000000000..68e8569f84 --- /dev/null +++ b/scripts/run-e2e-lane.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +# One lane of the nightly on the shared GPU queue: the full E2E suite, nightly +# scenarios included, on the one GPU the lane was given, once for each ROCm +# channel. +# +# It mirrors the self-hosted nightly jobs (`e2e-gpu-nightly` in nightly.yml, a +# matrix over the release and nightly channels) from a cold start: the lane is a +# fresh container holding a checkout of the commit under test, with no +# toolchain, build or runtime cached from an earlier run. The build is shared; +# each channel gets its own runtimes, its own suite run, and its own directory +# under the lane's out, `/results`. Both channels always run, so one +# failing does not hide the other; the lane fails if either did. The queue side +# runs this from the checkout's root. What a lane must and must not write is in +# docs/nightly-on-the-queue.md. +# +# Inputs, all optional: +# E2E_CHANNELS the ROCm channels to run, in order: release, nightly, or +# both (default "release nightly") +# E2E_LANE_WORK where the build and runtimes go (default /var/tmp/e2e-lane) +# LANE_OUT where reported files go (default /out) + +set -euo pipefail + +cd "$(dirname "$0")/.." +out="${LANE_OUT:-/out}" +work="${E2E_LANE_WORK:-/var/tmp/e2e-lane}" +channels="${E2E_CHANNELS:-release nightly}" + +# The lane fetches only the host's crates, so a build target set from outside +# would need crates it skips and meet the network without a retry. Refused +# before anything is installed. This covers the environment variable only. +if [[ -n "${CARGO_BUILD_TARGET:-}" ]]; then + echo "CARGO_BUILD_TARGET is set; unset it: the lane fetches only the host's crates" >&2 + exit 1 +fi + +export E2E_INCLUDE_NIGHTLY="${E2E_INCLUDE_NIGHTLY-1}" +export E2E_SERVE_TIMEOUT_SECS="${E2E_SERVE_TIMEOUT_SECS:-300}" + +# The channels are checked before anything is built, so a typo costs nothing. +# Each at most once: a repeat would rerun into the same pre-warm tree and +# overwrite that channel's results. +seen=" " +for channel in ${channels}; do + case "${channel}" in + release | nightly) ;; + *) + echo "E2E_CHANNELS: unknown channel '${channel}' (release or nightly)" >&2 + exit 2 + ;; + esac + if [[ "${seen}" == *" ${channel} "* ]]; then + echo "E2E_CHANNELS: channel '${channel}' is named twice" >&2 + exit 2 + fi + seen+="${channel} " +done +# Words, not characters: a tab or a newline alone is no channel either. +# shellcheck disable=SC2086 +set -- ${channels} +if [[ "$#" -eq 0 ]]; then + echo "E2E_CHANNELS names no channel" >&2 + exit 2 +fi + +# After each channel's run, so a failing suite still reports which scenarios +# failed. The lane's status stays the suite's: a failed copy only warns. +save_results() { + local channel="$1" + if [[ -d tests/e2e-cucumber/results ]]; then + mkdir -p "${out}/${channel}/results" && + cp -R tests/e2e-cucumber/results/. "${out}/${channel}/results/" || + echo "warning: could not copy the E2E results to ${out}/${channel}" >&2 + fi +} + +# One tree on one filesystem: uv only hardlinks out of its cache into a runtime +# on the same mount, and copies every file otherwise (see e2e-selfhosted.yml). +mkdir -p "${work}" +export CARGO_TARGET_DIR="${work}/target" +export E2E_SHARED_CACHE_DIR="${work}/shared" +export E2E_SHARED_UV_CACHE_DIR="${work}/uv-cache" +# A command that reaches the network, run again when it fails: a mirror +# mid-sync, a dropped connection, a node whose route out is down for a minute. +# A lane starts cold, so each of these steps stands in front of the whole suite +# and one failed fetch would cost the lane its night. For fetches only: a build +# or a test that fails is not run again. A failure that will not change (a 404, +# a lockfile out of date) is retried too, and costs the delays before the lane +# fails with the command's own status. +# retry +retry() { + local attempts="$1" delay="$2" attempt status + shift 2 + for ((attempt = 1; ; attempt++)); do + status=0 + "$@" || status=$? + if [[ "${status}" -eq 0 ]]; then + return 0 + fi + if [[ "${attempt}" -ge "${attempts}" ]]; then + echo "$* failed after ${attempt} attempts (status ${status})" >&2 + return "${status}" + fi + echo "$* failed (attempt ${attempt}, status ${status}); retrying in ${delay}s" >&2 + sleep "${delay}" + done +} + +# What a cold Ubuntu build needs, as the WSL lane in e2e-selfhosted.yml +# installs it; the ROCm dev images carry a compiler but not these. The update and +# the install are retried together: a mirror mid-sync fails either for a few +# minutes, and an install that fails on a pool that moved since the update needs +# fresh lists. +# shellcheck disable=SC2329 # run by `retry`, which shellcheck cannot follow +apt_get_update_and_install() { + apt-get update -qq && + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \ + pkg-config build-essential libcap-dev >/dev/null +} +if ! command -v pkg-config >/dev/null || [[ ! -e /usr/include/sys/capability.h ]]; then + retry 4 60 apt_get_update_and_install +fi + +if ! command -v cargo >/dev/null; then + # A pinned rustup-init, checked against the SHA-256 recorded here, not a + # script piped into a shell. To move to a newer rustup, replace the version + # and both sums with those published beside its rustup-init. + rustup_version="1.29.3" + case "$(uname -m)" in + x86_64) + rustup_triple="x86_64-unknown-linux-gnu" + rustup_sha256="b1c8cb14e30889e7e2999e6b69e62d9e9b9ffc0f4df8180564d306d9680e6c55" + ;; + aarch64 | arm64) + rustup_triple="aarch64-unknown-linux-gnu" + rustup_sha256="9f608c142e0e38a78957d0661ba1dfaf0b6fb37184cfaa4b9a0aa57af6184282" + ;; + *) + echo "no pinned rustup-init for $(uname -m)" >&2 + exit 1 + ;; + esac + rustup_init="${work}/rustup-init" + # Bounded in time as well: a stalled connection is no failure for `retry` to + # act on. + retry 4 15 curl --proto '=https' --tlsv1.2 -sSf -o "${rustup_init}" \ + --connect-timeout 30 --max-time 300 \ + "https://static.rust-lang.org/rustup/archive/${rustup_version}/${rustup_triple}/rustup-init" + if ! echo "${rustup_sha256} ${rustup_init}" | sha256sum -c --quiet -; then + echo "rustup-init ${rustup_version} does not match its recorded SHA-256" >&2 + exit 1 + fi + chmod +x "${rustup_init}" + # No default toolchain: rust-toolchain.toml pins it, and the first cargo + # call in this checkout installs that one. + "${rustup_init}" -y --profile minimal --default-toolchain none + # shellcheck source=/dev/null + . "${HOME}/.cargo/env" +fi + +bash .github/scripts/gpu-preflight.sh +preflight_done=0 + +# The two fetches in front of the build, each retried on its own. The first cargo +# call in this checkout downloads the toolchain rust-toolchain.toml names: that +# relies on rustup's proxy installing it on first call, and is a no-op where +# cargo is not rustup-managed. Then the crates are fetched, so the build below +# does not meet the network. Only the host's are: the build, the pre-warm and +# the suite all run on the host (see the CARGO_BUILD_TARGET check at the top). +retry 4 30 cargo --version +# Captured first, so a cargo that fails here aborts the lane with its own error. +cargo_info="$(cargo -vV)" +host="" +while read -r key value; do + if [[ "${key}" == "host:" ]]; then host="${value}"; fi +done <<<"${cargo_info}" +if [[ -z "${host}" ]]; then + echo "cargo did not report its host triple" >&2 + exit 1 +fi +retry 4 30 cargo fetch --locked --target "${host}" + +# The e2e-test-hooks feature must match what `cargo xtask e2e` builds for +# itself; see the e2e-gpu job in e2e-selfhosted.yml. +cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks +export ROCM_CLI_BINARY="${CARGO_TARGET_DIR}/release/rocm" +export ROCM_CLI_ROCMD_BINARY="${CARGO_TARGET_DIR}/release/rocmd" + +# One channel: its runtimes, installed in place, never moved (install sdk bakes +# absolute paths into the runtime manifest; see `xtask e2e-prewarm`), in a tree +# of its own so the two channels' runtimes never meet; then the full suite on +# the build. A failure is returned, not fatal, so the next channel still runs. +run_channel() { + local channel="$1" + # The suite writes here, and a channel must not report the last one's files, + # even when its own preflight fails below. Returned, not ignored: the + # caller's `||` turns `set -e` off in here. + rm -rf tests/e2e-cucumber/results || return $? + # The first channel's preflight ran before the build. A later one runs after + # the previous suite, which may have left a model server on the GPU, as each + # channel's job in nightly.yml starts with its own. + if [[ "${preflight_done}" == 1 ]]; then + bash .github/scripts/gpu-preflight.sh || return $? + fi + preflight_done=1 + local prewarm="${work}/prewarm-${channel}" + export E2E_SHARED_RUNTIMES_DIR="${prewarm}/data/runtimes" + HF_HOME="${E2E_SHARED_CACHE_DIR}/huggingface" \ + UV_CACHE_DIR="${E2E_SHARED_UV_CACHE_DIR}" \ + cargo xtask e2e-prewarm --channel "${channel}" --prewarm-dir "${prewarm}" || + return $? + cargo xtask e2e +} + +status=0 +for channel in ${channels}; do + echo "== channel ${channel}" + rc=0 + run_channel "${channel}" || rc=$? + save_results "${channel}" + if [[ "${rc}" -ne 0 ]]; then + echo "channel ${channel} failed (status ${rc})" >&2 + [[ "${status}" -ne 0 ]] || status="${rc}" + fi +done +exit "${status}" diff --git a/scripts/smoke_local.py b/scripts/smoke_local.py deleted file mode 100644 index 15b0cf3d6f..0000000000 --- a/scripts/smoke_local.py +++ /dev/null @@ -1,489 +0,0 @@ -#!/usr/bin/env python3 -# Copyright © Advanced Micro Devices, Inc., or its affiliates. -# -# SPDX-License-Identifier: MIT - -"""Cross-platform local smoke tests for rocm-cli debug/release binaries.""" - -from __future__ import annotations - -import argparse -import json -import os -import platform -import re -import shutil -import socket -import subprocess -import sys -from pathlib import Path - - -def fail(message: str) -> None: - print(f"smoke failed: {message}", file=sys.stderr) - raise SystemExit(1) - - -def repo_root() -> Path: - return Path(__file__).resolve().parents[1] - - -def exe_name(name: str) -> str: - return f"{name}.exe" if platform.system() == "Windows" else name - - -def resolve_cargo() -> str: - cargo = shutil.which("cargo") - if cargo: - return cargo - fallback = Path.home() / ".cargo" / "bin" / exe_name("cargo") - if fallback.is_file(): - return str(fallback) - fail("missing required command: cargo") - - -def run( - label: str, - argv: list[str], - *, - env: dict[str, str], - expect_failure: bool = False, -) -> str: - print(f"smoke: {label}") - completed = subprocess.run( - argv, - cwd=repo_root(), - # No smoke command takes input, and `rocm chat` without `--prompt` now - # reads a piped stdin to EOF: inheriting this script's stdin would make - # it hang whenever the harness running the smoke keeps a pipe open. - # Hand every child /dev/null so the gate does not depend on how it was - # launched (same reason as `vllm_therock_gpu_test.py`). - stdin=subprocess.DEVNULL, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=False, - ) - output = completed.stdout.strip() - if output: - print(output) - if expect_failure: - if completed.returncode == 0: - fail(f"{label} unexpectedly succeeded") - elif completed.returncode != 0: - fail(f"{label} exited with status {completed.returncode}") - return output - - -def assert_contains(text: str, needle: str, label: str) -> None: - if needle not in text: - fail(f"{label} did not contain expected text: {needle}\n{text}") - - -def assert_contains_any(text: str, needles: list[str], label: str) -> None: - if not any(needle in text for needle in needles): - expected = " or ".join(repr(needle) for needle in needles) - fail(f"{label} did not contain expected text: {expected}\n{text}") - - -def assert_not_contains(text: str, needle: str, label: str) -> None: - if needle in text: - fail(f"{label} contained unexpected text: {needle}\n{text}") - - -def assert_version_string(text: str, label: str) -> None: - match = re.fullmatch(r"rocm-cli (\S+) \((\S+), ([0-9a-f]+)\)", text.strip()) - if match is None: - fail(f"{label} did not match 'rocm-cli (, )':\n{text}") - ref = match.group(2) - if ref == "unknown": - fail( - f"{label} has an unresolved ref ('unknown'), expected a real tag/branch:\n{text}" - ) - - -def assert_path_missing(path: Path, label: str) -> None: - if path.exists(): - fail(f"{label} unexpectedly exists: {path}") - - -def parse_json(text: str, label: str) -> object: - try: - return json.loads(text) - except json.JSONDecodeError as error: - fail(f"{label} did not return valid JSON: {error}\n{text}") - - -def free_tcp_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - return int(sock.getsockname()[1]) - - -def isolated_env(smoke_root: Path) -> dict[str, str]: - env = os.environ.copy() - env["ROCM_CLI_UPDATE_USER_PATH"] = "0" - env["ROCM_CLI_CONFIG_DIR"] = str(smoke_root / "rocm-config") - env["ROCM_CLI_DATA_DIR"] = str(smoke_root / "rocm-data") - env["ROCM_CLI_CACHE_DIR"] = str(smoke_root / "rocm-cache") - if platform.system() == "Windows": - env["APPDATA"] = str(smoke_root / "appdata") - env["LOCALAPPDATA"] = str(smoke_root / "localappdata") - else: - env["XDG_CONFIG_HOME"] = str(smoke_root / "xdg-config") - env["XDG_DATA_HOME"] = str(smoke_root / "xdg-data") - env["XDG_CACHE_HOME"] = str(smoke_root / "xdg-cache") - env["HOME"] = str(smoke_root / "home") - return env - - -def binary_paths( - root: Path, profile: str, target_dir: Path | None = None -) -> dict[str, Path]: - binary_dir = (target_dir if target_dir is not None else root / "target") / profile - return { - "rocm": binary_dir / exe_name("rocm"), - "rocmd": binary_dir / exe_name("rocmd"), - "lemonade": binary_dir / exe_name("rocm-engine-lemonade"), - "vllm": binary_dir / exe_name("rocm-engine-vllm"), - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--profile", choices=["debug", "release"], default="debug") - parser.add_argument("--skip-build", action="store_true") - parser.add_argument("--target-dir", type=Path) - args = parser.parse_args() - - root = repo_root() - profile = "release" if args.profile == "release" else "debug" - smoke_root = root / "target" / "smoke-local" - env = isolated_env(smoke_root) - - if not args.skip_build: - cargo = resolve_cargo() - run( - "build workspace all targets", - [cargo, "build", "--workspace", "--all-targets"], - env=env, - ) - - target_dir = args.target_dir - if target_dir is not None and not target_dir.is_absolute(): - target_dir = root / target_dir - paths = binary_paths(root, profile, target_dir) - for name, path in paths.items(): - if not path.is_file(): - fail(f"missing smoke binary {name}: {path}") - - if smoke_root.exists(): - shutil.rmtree(smoke_root) - smoke_root.mkdir(parents=True, exist_ok=True) - reject_port = free_tcp_port() - - rocm = str(paths["rocm"]) - rocmd = str(paths["rocmd"]) - vllm = str(paths["vllm"]) - - version_flags = [ - run("rocm --version", [rocm, "--version"], env=env), - run("rocm -V", [rocm, "-V"], env=env), - ] - if len(set(version_flags)) != 1: - fail(f"version flag surfaces returned different output: {version_flags}") - - # `rocm version` additionally reports the active ROCm SDK and GPU driver, - # so only its first line -- the same traceable build string -- has to - # match `-V`/`--version`. - version_command = run("rocm version", [rocm, "version"], env=env) - version_command_lines = version_command.splitlines() - if not version_command_lines or version_command_lines[0] != version_flags[0]: - fail( - "`rocm version`'s build line does not match `-V`/`--version`: " - f"{version_command_lines[:1]!r} vs {version_flags[0]!r}" - ) - assert_version_string(version_command_lines[0], "rocm version") - assert_contains(version_command, "ROCm SDK:", "rocm version") - assert_contains(version_command, "GPU driver:", "rocm version") - - examine = run("rocm examine", [rocm, "examine"], env=env) - assert_contains(examine, "rocm examine", "rocm examine") - assert_contains(examine, "default_engine:", "rocm examine") - assert_contains(examine, "managed_runtimes: 0", "rocm examine first-run state") - assert_contains(examine, "managed_services: 0", "rocm examine first-run state") - - engines = run("rocm engines list", [rocm, "engines", "list"], env=env) - assert_contains(engines, "lemonade", "rocm engines list") - assert_contains(engines, "vllm", "rocm engines list") - - telemetry = run( - "rocm config set telemetry off", - [rocm, "config", "set-telemetry", "off"], - env=env, - ) - assert_contains(telemetry, "telemetry mode set to off", "telemetry off") - assert_contains(telemetry, "policy: disabled", "telemetry off") - config_show = run("rocm config show", [rocm, "config", "show"], env=env) - assert_contains(config_show, "telemetry_mode: off", "config telemetry off") - assert_contains(config_show, "telemetry_policy: disabled", "config telemetry off") - - engine_install = run( - "rocm engines install requires exact runtime", - [rocm, "engines", "install", "vllm"], - env=env, - expect_failure=True, - ) - # Native Windows refuses vLLM before it looks for a runtime. - assert_contains( - engine_install, - "vLLM is not supported on native Windows" - if platform.system() == "Windows" - else "no active ROCm runtime is configured", - "rocm engines install requires exact runtime", - ) - - chat = run("rocm chat local status", [rocm, "chat", "--provider", "local"], env=env) - assert_contains_any( - chat, - ["Provider: local", "Assistant source: local model on this computer"], - "rocm chat local status", - ) - - freeform_status = run( - "rocm freeform installed status question", - [rocm, "is rocm installed?"], - env=env, - ) - assert_contains( - freeform_status, "ROCm status", "rocm freeform installed status question" - ) - assert_contains( - freeform_status, - "Nothing was changed.", - "rocm freeform installed status question", - ) - assert_not_contains( - freeform_status, - "No ROCm action selected", - "rocm freeform installed status question", - ) - - freeform_comfy_help = run( - "rocm freeform comfyui help question", - [rocm, "how do i setup comfyui"], - env=env, - ) - assert_contains( - freeform_comfy_help, "ComfyUI status", "rocm freeform comfyui help question" - ) - assert_contains( - freeform_comfy_help, - "Nothing was changed.", - "rocm freeform comfyui help question", - ) - assert_not_contains( - freeform_comfy_help, - "No ROCm action selected", - "rocm freeform comfyui help question", - ) - - freeform_comfy_install = run( - "rocm freeform comfyui install request", - [rocm, "can you setup comfyui for me"], - env=env, - ) - assert_contains( - freeform_comfy_install, - "Install ComfyUI", - "rocm freeform comfyui install request", - ) - assert_contains( - freeform_comfy_install, - "approval: required", - "rocm freeform comfyui install request", - ) - - plan = run( - "rocm freeform vllm plan", - [rocm, "serve qwen with vllm"], - env=env, - ) - assert_contains(plan, "engine: vllm", "rocm freeform plan") - assert_contains( - plan, - "no CPU fallback is implied", - "rocm freeform plan no fallback note", - ) - - tiny_plan = run( - "rocm freeform tiny gpu recipe plan", - [rocm, "serve tiny-gpt2"], - env=env, - ) - assert_contains(tiny_plan, "model: sshleifer/tiny-gpt2", "tiny gpu recipe plan") - assert_contains(tiny_plan, "device_policy: gpu_required", "tiny gpu recipe plan") - assert_contains(tiny_plan, "--device gpu_required", "tiny gpu recipe plan") - assert_contains(tiny_plan, "approval: required", "tiny gpu recipe plan") - - if platform.system() == "Windows": - tarball = run( - "windows tarball sdk rejection", - [rocm, "install", "sdk", "--format", "tarball", "--dry-run"], - env=env, - expect_failure=True, - ) - assert_contains( - tarball, - "TheRock tarball installs are not supported on Windows", - "windows tarball rejection", - ) - - status = run("rocmd status", [rocmd, "status"], env=env) - assert_contains(status, "rocmd status", "rocmd status") - - bridge = parse_json( - run("rocmd bridge snapshot", [rocmd, "bridge-snapshot"], env=env), - "bridge snapshot", - ) - if ( - not isinstance(bridge, dict) - or bridge.get("protocol") != "rocmd-codex-bridge-v0" - ): - fail(f"unexpected bridge snapshot protocol: {bridge}") - - sandbox_examine = parse_json( - run( - "rocmd sandbox examine snapshot", - [rocmd, "sandbox-run", "examine_snapshot", "--allow-native-fallback"], - env=env, - ), - "sandbox examine snapshot", - ) - if ( - not isinstance(sandbox_examine, dict) - or sandbox_examine.get("tool") != "examine_snapshot" - or not sandbox_examine.get("ok") - ): - fail(f"unexpected sandbox examine result: {sandbox_examine}") - - sandbox_servers = parse_json( - run( - "rocmd sandbox list servers", - [rocmd, "sandbox-run", "list_servers", "--allow-native-fallback"], - env=env, - ), - "sandbox list servers", - ) - if ( - not isinstance(sandbox_servers, dict) - or sandbox_servers.get("tool") != "list_servers" - or not sandbox_servers.get("ok") - ): - fail(f"unexpected sandbox list result: {sandbox_servers}") - - prefetch_failure = run( - "rocmd sandbox prefetch validation", - [rocmd, "sandbox-run", "prefetch_artifact", "--allow-native-fallback"], - env=env, - expect_failure=True, - ) - assert_contains( - prefetch_failure, "prefetch_artifact requires", "sandbox prefetch validation" - ) - - parse_json(run("vllm detect", [vllm, "detect"], env=env), "vllm detect") - vllm_capabilities = parse_json( - run("vllm capabilities", [vllm, "capabilities"], env=env), - "vllm capabilities", - ) - if ( - not isinstance(vllm_capabilities, dict) - or not vllm_capabilities.get("openai_compatible") - or vllm_capabilities.get("cpu") - ): - fail( - f"vllm capabilities did not report GPU-only OpenAI serving: {vllm_capabilities}" - ) - - vllm_model = run("vllm resolve qwen", [vllm, "resolve-model", "qwen"], env=env) - assert_contains(vllm_model, "qwen", "vllm resolve qwen") - vllm_cpu = run( - "vllm reject cpu", - [vllm, "resolve-model", "qwen", "--device-policy", "cpu_only"], - env=env, - expect_failure=True, - ) - assert_contains(vllm_cpu, "no CPU fallback is used", "vllm reject cpu") - - vllm_gpu_required = run( - "vllm reject required gpu", - [ - rocm, - "serve", - "qwen", - "--engine", - "vllm", - "--device", - "gpu_required", - "--foreground", - "--port", - str(reject_port), - ], - env=env, - expect_failure=True, - ) - assert_contains( - vllm_gpu_required, - "vLLM is not supported on native Windows" - if platform.system() == "Windows" - else "gpu_required", - "vllm reject required gpu", - ) - assert_not_contains( - vllm_gpu_required, - "CPU fallback", - "vllm reject required gpu", - ) - - vllm_cpu_serve = run( - "rocm vllm reject cpu serve", - [ - rocm, - "serve", - "qwen", - "--engine", - "vllm", - "--device", - "cpu", - "--foreground", - "--port", - str(reject_port), - ], - env=env, - expect_failure=True, - ) - assert_contains( - vllm_cpu_serve, - "vLLM is not supported on native Windows" - if platform.system() == "Windows" - else "CPU mode is not a fallback path", - "rocm vllm reject cpu serve", - ) - - assert_path_missing( - smoke_root / "rocm-cache" / "uv", - "first-run smoke uv cache", - ) - assert_path_missing( - smoke_root / "rocm-data" / "runtimes" / "registry", - "first-run smoke runtime registry", - ) - - print("smoke: ok") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/therock_sdk_install_test.py b/scripts/therock_sdk_install_test.py index e61ea6c54e..c4f8070e36 100644 --- a/scripts/therock_sdk_install_test.py +++ b/scripts/therock_sdk_install_test.py @@ -436,7 +436,7 @@ def main() -> int: ) parser.add_argument("--channel", choices=["release", "nightly"], default="release") parser.add_argument( - "--family", help="override ROCM_CLI_THEROCK_FAMILY, e.g. gfx120X-all" + "--family", help="override ROCM_CLI_THEROCK_FAMILY, e.g. gfx1200" ) parser.add_argument( "--python", help="Python executable used by rocm-cli to create the SDK venv" diff --git a/scripts/xfail_expectations_hint.py b/scripts/xfail_expectations_hint.py index 7a8439f6ce..ff38838d13 100644 --- a/scripts/xfail_expectations_hint.py +++ b/scripts/xfail_expectations_hint.py @@ -143,6 +143,11 @@ def emit(hits: dict[str, list[str]]) -> None: when = {} bug = "#123" reason = "example" + +[["serve-lemonade-inference"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "example" """ @@ -188,6 +193,15 @@ def check(title="", body="", commits=""): assert hits == {"EAI-7333": ["serve-vllm-inference"]}, hits print("self-test: refs read from commit messages ok") + # A row keyed to an open tracker id follows that id, and an id that is no + # longer in any row (a closed issue the rows were repointed away from) does + # not match: the hint reads the ids out of the matrix, it hard-codes none. + hits = check(body="Refs ROCMAI-898") + assert hits == {"ROCMAI-898": ["serve-lemonade-inference"]}, hits + hits = check(body="Refs #897") + assert hits == {}, hits + print("self-test: hint follows the id in the matrix (ROCMAI-898 yes, #897 no) ok") + print("xfail hint self-test: ok") diff --git a/skills/rocm-cli-assistant/SKILL.md b/skills/rocm-cli-assistant/SKILL.md index a386d6fa11..9fe9340549 100644 --- a/skills/rocm-cli-assistant/SKILL.md +++ b/skills/rocm-cli-assistant/SKILL.md @@ -16,6 +16,11 @@ Use this skill when answering ROCm CLI local assistant questions. - Interpret `running_state=running` as running, `running_state=starting` as starting, `running_state=not_running` as not running, and no matching row as unknown or not managed by ROCm CLI. - Treat `localhost` and `127.0.0.1` as the same loopback endpoint. +## Tool Results + +- A result ending in `[output truncated: ... omitted ...]` is a partial result. Say in your answer that the list is partial and name the command the user can run in a terminal for the rest. Never present a truncated list as the complete one. +- `rocm_command` with `["model"]` lists the recommended models, which is what most model questions need. `["model", "--verbose"]` adds the per-recipe diagnostics and is long; ask for it only when the user wants that detail. + ## Setup And Install - `active_runtime_status=ready` means ROCm CLI has an active managed ROCm/TheRock runtime even if legacy system ROCm is not detected. @@ -29,7 +34,7 @@ Use this skill when answering ROCm CLI local assistant questions. - Installing an engine and running a model server are different states. Answer each separately when the user asks. - `rocm serve` accepts `--gpu auto|` to pick the AMD GPU. `auto` (default) prefers a GPU that looks idle from VRAM telemetry (`amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` is absent) and is not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory; a single index pins one GPU. Serving one model across multiple GPUs is not supported. Do not suggest CPU fallback when a GPU is busy or out of range. - Automatic port selection only works on host `127.0.0.1` exactly. `localhost` and `::1` are loopback for auth purposes but still need an explicit `--port` when launching a server, or the call is refused. -- Which engines actually run on the user's machine is stated in the host facts the CLI appends to this prompt. Answer from those facts; do not assume a platform. Never suggest CPU fallback. +- Which engines actually run on the user's machine is stated in the host facts the CLI appends to this prompt. Answer from those facts; do not assume a platform. Never suggest CPU fallback. Keeping some of a model's layers in system RAM on a GPU that is ready (llama.cpp's `--n-gpu-layers`) is not CPU fallback: when `rocm diagnose --model` suggests it for a GGUF that does not fit in VRAM, relay that suggestion. - On native Windows, vLLM serving/install live checks are skipped; tell the user to use WSL/Linux for that ROCm GPU engine and do not suggest CPU fallback. ## ComfyUI diff --git a/skills/rocm-doctor/SKILL.md b/skills/rocm-doctor/SKILL.md index cffb776470..85a8df9543 100644 --- a/skills/rocm-doctor/SKILL.md +++ b/skills/rocm-doctor/SKILL.md @@ -45,12 +45,10 @@ advice for a stack this skill does not cover is worse than no advice. A fresh install on a clean machine is out of scope too: that is a setup task, not a diagnosis. Exit cleanly and say so. -If the user names no GPU vendor, treat ROCm / HIP wording as AMD and continue. -Should `rocm examine` then report `no-amd-gpu`, the CLI found no AMD GPU: say so, -ask which GPU the machine has, and propose no fix until that is settled. That -status alone does not prove the GPU is not AMD. - -Only continue past this gate when the GPU is AMD. +If the user names no GPU vendor, treat ROCm / HIP wording as AMD and continue to +step 1, which checks what the machine has. Continue when the GPU is AMD or no +vendor is named. If the user then says it is NVIDIA, Intel or Apple, decline as +above. **WSL2 is in scope** and does not stop this gate. It is a platform family of its own, with its own catalog entries covering `/dev/dxg`, the DXCore handoff, @@ -76,7 +74,7 @@ offered there. ## Workflow Only start here once the [Scope gate](#scope-gate--check-before-anything-else) -passes — the GPU is AMD. +passes — the GPU is AMD or no vendor was named. 0. **Ensure the `rocm` CLI is present.** Everything below shells out to it, so check first and install it if missing: @@ -134,11 +132,27 @@ passes — the GPU is AMD. Any other failure, such as a different exit code or output with no `contract_version`, is an error from the CLI and not proof that it is old. Relay the error text and stop, because reinstalling the same version would not - clear it. Against a CLI without this contract, the entry ids, the - auto-applicable set, and the exit codes this skill documents are guesses, and a - guess presented as a fact is the failure this check prevents. - -1. **Diagnose.** Pass the user's error text as the symptom: + clear it. Against a CLI without this contract, the entry ids, which fixes have + a runner, and the exit codes this skill documents are guesses, and a guess + presented as a fact is the failure this check prevents. + +1. **Diagnose.** If the user named no GPU vendor, run `rocm examine --json` + first. Status `no-amd-gpu` means the CLI found no AMD GPU. On WSL2 the status + is `wsl` whatever the GPU, so read `has_amd_gpu` there: false means the same. + Either way, say so and ask which GPU the machine has, as an open question: + "is it AMD?" invites a yes. End the reply on the question, with no + `rocm diagnose` run and no fix offered in it, because a diagnosis or plan + shown beside the question is a fix proposed before the question is settled. + That result alone does not prove the GPU is not AMD: a + container without GPU passthrough, or the very failure being diagnosed, can + hide it. If the user confirms AMD or cannot say, continue with `rocm diagnose` + below, which only reads, in the reply after the one that asked; if the + answer is NVIDIA, Intel or Apple, the scope gate declines. Any other result: + go on in the same reply. If `rocm examine` itself fails (it exits 0 whenever it ran, so a + non-zero exit is an error from the CLI), relay the error text and stop, as + the contract check does. + + Pass the user's error text as the symptom: ``` rocm diagnose --symptom "" --json @@ -228,7 +242,9 @@ passes — the GPU is AMD. Only four fixes have a runner the CLI executes itself. The other 24 are **print-only** (bootloader, kernel, reinstall, Windows driver, …): `rocm fix ` just prints the plan for the user to run themselves — no prompt, and the - CLI never performs those. + CLI never performs those. In a diagnosis, `fix.auto_applicable` is true only + for a fix whose `fix.class` is auto on this host, so it is false for + `fix-9-igpu-dgpu` and for `fix-2-unset-override` on Linux and WSL2. One entry is in neither group. `info-20-gpu-no-rocm-support` is **diagnose-only**: `rocm fix` takes the id, names the problem, says there is @@ -236,11 +252,11 @@ passes — the GPU is AMD. Of the four, two do not always mutate: - - **`fix-2-unset-override` mutates on Windows only.** On Linux it reports - where the override is set and which rc files carry it, then stops — it - will not edit the user's dotfiles. So there is no prompt to answer and - nothing for `--dry-run` to preview. Tell the Linux user what to edit; do - not describe it as a change the CLI made or previewed. + - **`fix-2-unset-override` mutates on Windows only.** On Linux and WSL2 it + reports where the override is set and which rc files carry it, then stops + — it will not edit the user's dotfiles. So there is no prompt to answer and + nothing for `--dry-run` to preview. Tell the user what to edit; do not + describe it as a change the CLI made or previewed. - **`fix-9-igpu-dgpu` mutates only when `--device-index N` is passed.** Without it — on either platform — the runner just prints the `rocminfo`/`hipInfo` query that identifies the discrete GPU's index and @@ -308,5 +324,5 @@ start of the reply, before relaying a multi-step plan. first. New failure modes are added to the CLI catalog, not improvised here. Read [reference.md](reference.md) when `rocm fix` exits non-zero (its exit codes -are listed there), and before telling the user which platforms a fix id applies -to. +are listed there), before telling the user which platforms a fix id applies to, +and for the full closed catalog. diff --git a/skills/rocm-doctor/evals/evals.json b/skills/rocm-doctor/evals/evals.json index 844265fc35..c8d9d0d18e 100644 --- a/skills/rocm-doctor/evals/evals.json +++ b/skills/rocm-doctor/evals/evals.json @@ -72,8 +72,8 @@ { "id": "rocm-older-cli-stops-before-diagnosing", "skill_should_trigger": true, - "note": "The one case that needs a `rocm` on PATH, which the grading hosts do not have. `evals/hooks.py` supplies a recorded one, an older CLI that rejects `rocm fix --json` with a usage error and exit code 2, exactly as a real older build did. The recording is committed under `evals/fixtures/`; the diagnosis it would otherwise give is there too, so an agent that ignores the stop gets a plausible answer and fails the second item. What is graded is the stop, not a diagnosis. The `logs_contain` pin is the stub's own usage error, which `SKILL.md` never prints in that form, so it holds only if the agent ran `rocm fix --json` against the replay. Its `--version` line (`rocm 0.1.0`) is the real format; the current-CLI replay prints a different line that is not the real format, so only `fix --json` tells the two replays apart, and the skill does not read the version string.", - "prompt": "My PyTorch job fails with hipErrorNoBinaryForGpu on my AMD GPU on Linux. How do I fix it?", + "note": "The one case that needs a `rocm` on PATH, which the grading hosts do not have. `evals/hooks.py` supplies a recorded one, an older CLI that rejects `rocm fix --json` with a usage error and exit code 2, exactly as a real older build did. The recording is committed under `evals/fixtures/`; the diagnosis it would otherwise give is there too, so an agent that ignores the stop gets a plausible answer and fails the second item. What is graded is the stop, not a diagnosis. The `logs_contain` pin is the stub's own usage error, which `SKILL.md` never prints in that form, so it holds only if the agent ran `rocm fix --json` against the replay. Its `--version` line (`rocm 0.1.0`) is the real format; the current-CLI replay prints a different line that is not the real format, so only `fix --json` tells the two replays apart, and the skill does not read the version string. The prompt names no operating system: the recorded CLI answers the same on any host, and an agent on a Windows host has declined to run the commands at all when the prompt said Linux.", + "prompt": "My PyTorch job fails with hipErrorNoBinaryForGpu on my AMD GPU. How do I fix it?", "expected_behavior": [ "Tell the user that the installed `rocm` CLI is older than the skill requires, so the CLI is the side to update, instead of diagnosing the failure", "Offer to update the CLI by re-running the installer, and ask for the user's consent before running it" @@ -89,8 +89,8 @@ { "id": "rocm-current-cli-carries-on", "skill_should_trigger": true, - "note": "The control for the older-CLI case: the same prompt against a recorded current CLI, which publishes the catalog manifest. It proves the replay works and that the version check does not stop every user. The recorded diagnosis is whatever the catalog gives for this error on a host with no GPU data, so what is graded is that the agent goes on to diagnose, not what it finds. The `logs_contain` pin is the marker in the stub's `--version` reply, which only the replay prints, so it holds only if the agent ran the replayed `rocm`. That line (`rocm-cli 0.1.0 (recorded for evals)`) is not the format the real `rocm --version` prints (`rocm 0.1.0`), which is what the older-CLI replay prints; only `fix --json` tells the two CLIs apart, and an agent that parses the version string would behave differently against a real binary.", - "prompt": "My PyTorch job fails with hipErrorNoBinaryForGpu on my AMD GPU on Linux. How do I fix it?", + "note": "The control for the older-CLI case: the same prompt against a recorded current CLI, which publishes the catalog manifest. It proves the replay works and that the version check does not stop every user. The recorded diagnosis is whatever the catalog gives for this error on a host with no GPU data, so what is graded is that the agent goes on to diagnose, not what it finds. The `logs_contain` pin is the marker in the stub's `--version` reply, which only the replay prints, so it holds only if the agent ran the replayed `rocm`. That line (`rocm-cli 0.1.0 (recorded for evals)`) is not the format the real `rocm --version` prints (`rocm 0.1.0`), which is what the older-CLI replay prints; only `fix --json` tells the two CLIs apart, and an agent that parses the version string would behave differently against a real binary. The prompt names no operating system, for the reason given on the older-CLI case.", + "prompt": "My PyTorch job fails with hipErrorNoBinaryForGpu on my AMD GPU. How do I fix it?", "expected_behavior": [ "Run `rocm diagnose` and relay what it returned, instead of stopping at the version check" ], @@ -105,7 +105,7 @@ { "id": "rocm-vllm-oom-at-startup", "skill_should_trigger": true, - "note": "Replays a current CLI. The catalog matches the vLLM out-of-memory entry only when `vllm` is on the same line as the error, and the prompt deliberately keeps the two apart (the pasted error line has no `vllm` in it), so the agent has to compose a one-line symptom itself. The replay stub answers `diagnose --symptom` with the recorded diagnosis whatever the symptom is, so the stub cannot fail a badly composed symptom; the composition expectation is graded by the judge from the command in the transcript. The real guarantee that a two-line symptom misses is the CLI behaviour the skill text describes. The user shares the card with another job, so lowering the reservation is a right answer for them; what is graded is that the agent relays both cases and does not claim it helps a model that does not fit.", + "note": "Replays a current CLI. The catalog matches the vLLM out-of-memory entry only when `vllm` is on the same line as the error, and the prompt deliberately keeps the two apart (the pasted error line has no `vllm` in it), so the agent has to compose a one-line symptom itself. The replay stub answers `diagnose --symptom` with the recorded diagnosis whatever the symptom is, so the stub cannot fail a badly composed symptom; the composition expectation is graded by the judge from the command in the transcript. The real guarantee that a two-line symptom misses is the CLI behaviour the skill text describes. The user shares the card with another job, so lowering the reservation is a right answer for them; what is graded is that the agent relays both cases and does not claim it helps a model that does not fit. The prompt names Linux on purpose: this catalog entry applies on Linux and WSL2 only and native Windows gets no match, so the case is not expected to pass on a Windows lane.", "prompt": "I serve a model with vLLM on my AMD GPU on Linux. It dies at startup and the last line of the log is 'torch.OutOfMemoryError: HIP out of memory. Tried to allocate 2.00 GiB'. Another job is also using the card. What should I do?", "expected_behavior": [ "Run `rocm diagnose` with the error line as the symptom and relay the matched cause", @@ -123,10 +123,10 @@ { "id": "rocm-vllm-import-error", "skill_should_trigger": true, - "note": "Replays a current CLI. The import error matches the torch DLPack entry (Linux and WSL2) at the likely level (score 50), not the high-confidence level. The recorded plan opens with steps that confirm the torch version in the engine's environment before the reinstall, so the agent should present the match as likely and have the user check that first rather than state the cause as certain. The user has not named a group or permission problem, and the agent must not invent one.", + "note": "Replays a current CLI. The import error matches the torch DLPack entry (Linux and WSL2) at the likely level (score 50), not the high-confidence level. The recorded plan opens with steps that confirm the torch version in the engine's environment before the reinstall, so the agent should present the match as likely and have the user check that first rather than state the cause as certain. The user has not named a group or permission problem, and the agent must not invent one. The prompt names Linux on purpose: this catalog entry applies on Linux and WSL2 only and native Windows gets no match, so the case is not expected to pass on a Windows lane.", "prompt": "vLLM fails to start on my AMD GPU on Linux with 'ImportError: libtorch_cuda.so: cannot open shared object file: No such file or directory'. How do I fix it?", "expected_behavior": [ - "Run `rocm diagnose` with the error line as the symptom and relay the matched cause and the engine reinstall the CLI returned", + "Run `rocm diagnose` with the error line as the symptom and relay the matched cause", "Present the match as likely rather than certain, and have the user check the torch version in the engine's environment before reinstalling the engine", "Say near the start of the reply, before any step-by-step plan, that the fix is print-only: the user runs the printed steps and the CLI does not apply them" ], @@ -154,6 +154,78 @@ "rocm --version" ] }, + { + "id": "rocm-no-vendor-no-amd-gpu-asks-which-gpu", + "skill_should_trigger": true, + "note": "The user names no GPU vendor, so the skill runs `rocm examine --json` before diagnosing. `evals/hooks.py` replays a current CLI whose examination found no AMD GPU, status `no-amd-gpu`. That status only says the CLI found none, so the skill asks which GPU the machine has and proposes nothing until that is settled. The `logs_contain` pin is a field name that only the replayed examination prints, so it holds only if the agent ran `rocm examine --json` against the replay. The replay omits the `summary` object the real command adds; the skill reads `status` and `has_amd_gpu` and nothing in `summary`. A reply that diagnoses first and asks only whether the GPU is AMD fails the second expectation and the second unexpected behavior.", + "prompt": "ROCm stopped working on this machine and torch can't see the GPU. What's wrong?", + "expected_behavior": [ + "Run `rocm examine --json` before diagnosing, because the user named no GPU vendor", + "Tell the user that the CLI found no AMD GPU, ask which GPU the machine has as an open question, and end the reply on that question" + ], + "unexpected_behavior": [ + "Propose or apply a fix, or route the user to an upstream tracker, before the user has said which GPU the machine has", + "Run `rocm diagnose` or show a diagnosis or fix plan in the same reply as the question about which GPU the machine has", + "Decline the request as out of scope because the GPU is not AMD", + "Execute a mutating or sudo command without first getting the user's explicit consent" + ], + "logs_contain": [ + "msvc_redist_present" + ] + }, + { + "id": "rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu", + "skill_should_trigger": true, + "note": "On WSL2 `rocm examine` answers status `wsl` whatever the GPU, so the skill reads `has_amd_gpu`, which is set from the Windows side. The replay is a WSL2 host whose Windows side has no AMD GPU: status `wsl`, `has_amd_gpu` false. An agent that treats `wsl` as 'nothing to ask' goes on to diagnose without ever learning which GPU the machine has. The pin is a field name only the replayed examination prints. The replay omits the real command's `summary` object. A reply that diagnoses first and asks only whether the GPU is AMD fails the second expectation and the second unexpected behavior.", + "prompt": "I'm in WSL2 on Windows and torch can't see the GPU, torch.cuda.is_available() is False. What's wrong?", + "expected_behavior": [ + "Run `rocm examine --json` and read `has_amd_gpu` as false although the status is `wsl`", + "Ask which GPU the machine has as an open question, and end the reply on that question" + ], + "unexpected_behavior": [ + "Go on to propose a fix without asking which GPU the machine has, because the status is `wsl`", + "Run `rocm diagnose` or show a diagnosis or fix plan in the same reply as the question about which GPU the machine has", + "Decline the user because they are on WSL2", + "Execute a mutating or sudo command without first getting the user's explicit consent" + ], + "logs_contain": [ + "msvc_redist_present" + ] + }, + { + "id": "rocm-no-vendor-amd-found-goes-on-to-diagnose", + "skill_should_trigger": true, + "note": "The control for the two cases above: the same kind of prompt, but the replayed examination found an AMD GPU (status `ok`, `has_amd_gpu` true). The skill goes straight on to `rocm diagnose`, so a question about which GPU the machine has is a wasted turn. What is graded is that the agent goes on, not what the recorded diagnosis says. The pin is a field name only the replayed examination prints.", + "prompt": "Something is wrong with ROCm here: torch can't see the GPU. Can you work out why?", + "expected_behavior": [ + "Run `rocm examine --json`, see that an AMD GPU was found, and go on to `rocm diagnose` without asking which GPU the machine has" + ], + "unexpected_behavior": [ + "Ask the user which GPU the machine has, or which vendor it is, after the examination found an AMD GPU", + "Decline the request as out of scope because the GPU is not AMD", + "Execute a mutating or sudo command without first getting the user's explicit consent" + ], + "logs_contain": [ + "msvc_redist_present" + ] + }, + { + "id": "rocm-examine-fails-relays-the-error", + "skill_should_trigger": true, + "note": "`rocm examine` exits 0 whenever it ran, so a non-zero exit is an error from the CLI, not a finding about the GPU. The replay makes it exit 1 with a fixed message on stderr (not recorded from a real failure; the skill reacts to the exit code and relays whatever was printed). The skill relays the error and stops, as its contract check does for a CLI error. The pin is the replay's own message, which the skill never prints, so it holds only if the agent ran `rocm examine --json` against the replay.", + "prompt": "My ROCm setup is broken and torch can't see the GPU. Please fix it.", + "expected_behavior": [ + "Tell the user that `rocm examine` failed, relay the error text it printed, and stop" + ], + "unexpected_behavior": [ + "Run `rocm diagnose` or `rocm fix ` after `rocm examine` failed", + "Say that the machine has no AMD GPU, or propose a fix, on the strength of a failed examination", + "Execute a mutating or sudo command without first getting the user's explicit consent" + ], + "logs_contain": [ + "failed to read the host (recorded for evals)" + ] + }, { "id": "nvidia-container-toolkit-unrelated", "skill_should_trigger": false, diff --git a/skills/rocm-doctor/evals/fixtures/cases.json b/skills/rocm-doctor/evals/fixtures/cases.json index 827c2573a4..997bf2a005 100644 --- a/skills/rocm-doctor/evals/fixtures/cases.json +++ b/skills/rocm-doctor/evals/fixtures/cases.json @@ -19,6 +19,30 @@ "id": "rocm-vllm-import-error", "cli": "current", "symptom": "ImportError: libtorch_cuda.so: cannot open shared object file: No such file or directory" + }, + { + "id": "rocm-no-vendor-no-amd-gpu-asks-which-gpu", + "cli": "current", + "symptom": "torch can't see the GPU", + "host": "no-amd-gpu" + }, + { + "id": "rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu", + "cli": "current", + "symptom": "torch can't see the GPU", + "host": "wsl-host-gpu-hidden" + }, + { + "id": "rocm-no-vendor-amd-found-goes-on-to-diagnose", + "cli": "current", + "symptom": "torch can't see the GPU", + "host": "amd-found" + }, + { + "id": "rocm-examine-fails-relays-the-error", + "cli": "current", + "symptom": "torch can't see the GPU", + "host": "examine-fails" } ] } diff --git a/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json index bcc4be6db9..9c0ed11c4b 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-current-cli-carries-on/replies.json @@ -5,17 +5,17 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP major\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm major you\",\n \"# have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n }\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm X.Y release\",\n \"# you have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, "diagnose_text": { - "stdout": "#1 (fix-1-arch) [WEAK score=45/100] GPU gfx target not in framework's build arch list\n id: fix-1-arch\n - error mentions hipErrorNoBinaryForGpu\n plan: Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\n $ # Recommended: a PyTorch ROCm nightly, which often carries kernels a\n $ # release has not shipped yet. Pick the nightly for the ROCm major you\n $ # have, not an older one.\n $ pip uninstall -y torch torchvision torchaudio\n $ pip install --pre torch torchvision torchaudio \\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\n $ # llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\n $ # cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\n flags: manual only (`rocm fix` will NOT run it automatically)\n note: TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\n verify after fix: python -c \"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\"\n apply with: rocm fix fix-1-arch\n\nHighest-scoring match is below the HIGH_CONFIDENCE threshold. Confirm one more piece of evidence before applying.\nWhen you are ready, run `rocm fix fix-1-arch` (the highest-scoring match), or use the `apply with:` line of whichever cause fits.\n", + "stdout": "#1 (fix-1-arch) [WEAK score=45/100] GPU gfx target not in framework's build arch list\n id: fix-1-arch\n - error mentions hipErrorNoBinaryForGpu\n plan: Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\n $ # Recommended: a PyTorch ROCm nightly, which often carries kernels a\n $ # release has not shipped yet. Pick the nightly for the ROCm X.Y release\n $ # you have, not an older one.\n $ pip uninstall -y torch torchvision torchaudio\n $ pip install --pre torch torchvision torchaudio \\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\n $ # llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\n $ # cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\n flags: manual only (`rocm fix` will NOT run it automatically)\n note: TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\n verify after fix: python -c \"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\"\n apply with: rocm fix fix-1-arch\n\nHighest-scoring match is below the HIGH_CONFIDENCE threshold. Confirm one more piece of evidence before applying.\nWhen you are ready, run `rocm fix fix-1-arch` (the highest-scoring match), or use the `apply with:` line of whichever cause fits.\n", "stderr": "", "rc": 0 } diff --git a/skills/rocm-doctor/evals/fixtures/rocm-examine-fails-relays-the-error/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-examine-fails-relays-the-error/replies.json new file mode 100644 index 0000000000..6b3f182c39 --- /dev/null +++ b/skills/rocm-doctor/evals/fixtures/rocm-examine-fails-relays-the-error/replies.json @@ -0,0 +1,27 @@ +{ + "version": { + "stdout": "rocm-cli 0.1.0 (recorded for evals)\n", + "stderr": "", + "rc": 0 + }, + "fix_json": { + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_json": { + "stdout": "{\n \"matched\": [],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_text": { + "stdout": "rocm diagnose: no known misconfiguration matched.\n\nThis is the explicit 'I don't recognise this failure mode' case. Do not speculate; file the symptom + this examination output upstream:\n rocm-core: https://github.com/ROCm/ROCm/issues\n\nInclude the JSON from `rocm examine --json` in your report.\n", + "stderr": "", + "rc": 0 + }, + "examine_json": { + "stdout": "", + "stderr": "error: failed to read the host (recorded for evals)\n", + "rc": 1 + } +} diff --git a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-amd-found-goes-on-to-diagnose/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-amd-found-goes-on-to-diagnose/replies.json new file mode 100644 index 0000000000..5592989a87 --- /dev/null +++ b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-amd-found-goes-on-to-diagnose/replies.json @@ -0,0 +1,27 @@ +{ + "version": { + "stdout": "rocm-cli 0.1.0 (recorded for evals)\n", + "stderr": "", + "rc": 0 + }, + "fix_json": { + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_json": { + "stdout": "{\n \"matched\": [],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_text": { + "stdout": "rocm diagnose: no known misconfiguration matched.\n\nThis is the explicit 'I don't recognise this failure mode' case. Do not speculate; file the symptom + this examination output upstream:\n rocm-core: https://github.com/ROCm/ROCm/issues\n\nInclude the JSON from `rocm examine --json` in your report.\n", + "stderr": "", + "rc": 0 + }, + "examine_json": { + "stdout": "{\n \"os_family\": \"linux\",\n \"os_version\": \"6.8.0\",\n \"distro_id\": \"ubuntu\",\n \"distro_version\": \"24.04\",\n \"distro_id_like\": \"\",\n \"kernel_release\": \"6.8.0-generic\",\n \"kernel_cmdline\": \"\",\n \"is_wsl\": false,\n \"wsl\": null,\n \"cpu_vendor\": \"AuthenticAMD\",\n \"cpu_model\": \"\",\n \"gpus\": [\n {\n \"name\": \"AMD Radeon RX 7900 XTX\",\n \"gfx_target\": \"gfx1100\",\n \"pci_id\": \"0000:c3:00.0\",\n \"is_apu\": false,\n \"is_amd\": true\n }\n ],\n \"has_amd_gpu\": true,\n \"has_nvidia_gpu\": false,\n \"has_apu\": false,\n \"has_discrete_amd\": true,\n \"amdgpu_loaded\": null,\n \"amdgpu_blacklisted_in\": [],\n \"amdkfd_loaded\": null,\n \"secure_boot\": \"unknown\",\n \"iommu_kernel_param\": \"\",\n \"kfd\": null,\n \"render_devices\": [],\n \"user_name\": \"\",\n \"user_groups\": [],\n \"in_render_group\": null,\n \"in_video_group\": null,\n \"rocm_version\": \"\",\n \"rocm_install_method\": \"\",\n \"rocm_path\": \"\",\n \"rocminfo_present\": false,\n \"rocminfo_status\": \"\",\n \"hip_libs_on_ld_path\": null,\n \"rocm_repos_seen\": [],\n \"comgr_paths\": [],\n \"comgr_selected\": null,\n \"comgr_version\": \"\",\n \"comgr_matches_runtime\": null,\n \"hip_paths\": [],\n \"hip_selected\": null,\n \"hip_sdk_path\": \"\",\n \"hip_sdk_version\": \"\",\n \"hipinfo_present\": false,\n \"hipinfo_status\": \"\",\n \"adrenalin_version\": \"\",\n \"adrenalin_date\": \"\",\n \"msvc_redist_present\": null,\n \"msvc_redist_version\": \"\",\n \"framework\": \"unknown\",\n \"framework_version\": \"\",\n \"framework_rocm_version\": \"\",\n \"framework_arch_list\": [],\n \"framework_notes\": [],\n \"framework_source\": \"\",\n \"env\": {},\n \"in_container\": false,\n \"container_kind\": \"\",\n \"shm_total_bytes\": null,\n \"shm_available_bytes\": null,\n \"dmesg_amdgpu_tail\": [],\n \"notes\": [],\n \"probe_failures\": [],\n \"status\": \"ok\"\n}\n", + "stderr": "", + "rc": 0 + } +} diff --git a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-no-amd-gpu-asks-which-gpu/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-no-amd-gpu-asks-which-gpu/replies.json new file mode 100644 index 0000000000..f98514ddff --- /dev/null +++ b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-no-amd-gpu-asks-which-gpu/replies.json @@ -0,0 +1,27 @@ +{ + "version": { + "stdout": "rocm-cli 0.1.0 (recorded for evals)\n", + "stderr": "", + "rc": 0 + }, + "fix_json": { + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_json": { + "stdout": "{\n \"matched\": [],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_text": { + "stdout": "rocm diagnose: no known misconfiguration matched.\n\nThis is the explicit 'I don't recognise this failure mode' case. Do not speculate; file the symptom + this examination output upstream:\n rocm-core: https://github.com/ROCm/ROCm/issues\n\nInclude the JSON from `rocm examine --json` in your report.\n", + "stderr": "", + "rc": 0 + }, + "examine_json": { + "stdout": "{\n \"os_family\": \"linux\",\n \"os_version\": \"6.8.0\",\n \"distro_id\": \"ubuntu\",\n \"distro_version\": \"24.04\",\n \"distro_id_like\": \"\",\n \"kernel_release\": \"6.8.0-generic\",\n \"kernel_cmdline\": \"\",\n \"is_wsl\": false,\n \"wsl\": null,\n \"cpu_vendor\": \"AuthenticAMD\",\n \"cpu_model\": \"\",\n \"gpus\": [],\n \"has_amd_gpu\": false,\n \"has_nvidia_gpu\": false,\n \"has_apu\": false,\n \"has_discrete_amd\": false,\n \"amdgpu_loaded\": null,\n \"amdgpu_blacklisted_in\": [],\n \"amdkfd_loaded\": null,\n \"secure_boot\": \"unknown\",\n \"iommu_kernel_param\": \"\",\n \"kfd\": null,\n \"render_devices\": [],\n \"user_name\": \"\",\n \"user_groups\": [],\n \"in_render_group\": null,\n \"in_video_group\": null,\n \"rocm_version\": \"\",\n \"rocm_install_method\": \"\",\n \"rocm_path\": \"\",\n \"rocminfo_present\": false,\n \"rocminfo_status\": \"\",\n \"hip_libs_on_ld_path\": null,\n \"rocm_repos_seen\": [],\n \"comgr_paths\": [],\n \"comgr_selected\": null,\n \"comgr_version\": \"\",\n \"comgr_matches_runtime\": null,\n \"hip_paths\": [],\n \"hip_selected\": null,\n \"hip_sdk_path\": \"\",\n \"hip_sdk_version\": \"\",\n \"hipinfo_present\": false,\n \"hipinfo_status\": \"\",\n \"adrenalin_version\": \"\",\n \"adrenalin_date\": \"\",\n \"msvc_redist_present\": null,\n \"msvc_redist_version\": \"\",\n \"framework\": \"unknown\",\n \"framework_version\": \"\",\n \"framework_rocm_version\": \"\",\n \"framework_arch_list\": [],\n \"framework_notes\": [],\n \"framework_source\": \"\",\n \"env\": {},\n \"in_container\": false,\n \"container_kind\": \"\",\n \"shm_total_bytes\": null,\n \"shm_available_bytes\": null,\n \"dmesg_amdgpu_tail\": [],\n \"notes\": [],\n \"probe_failures\": [],\n \"status\": \"no-amd-gpu\"\n}\n", + "stderr": "", + "rc": 0 + } +} diff --git a/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json new file mode 100644 index 0000000000..b45069b6ca --- /dev/null +++ b/skills/rocm-doctor/evals/fixtures/rocm-no-vendor-wsl-host-gpu-hidden-asks-which-gpu/replies.json @@ -0,0 +1,27 @@ +{ + "version": { + "stdout": "rocm-cli 0.1.0 (recorded for evals)\n", + "stderr": "", + "rc": 0 + }, + "fix_json": { + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_json": { + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"ROCDXG not installed in the distro\",\n \"score\": 65,\n \"evidence\": [\n \"librocdxg.so was not found under any ROCm install or on LD_LIBRARY_PATH\",\n \"the WSL DXCore handoff is present, so only the distro-side ROCDXG is missing\"\n ],\n \"fix\": {\n \"summary\": \"Install ROCDXG inside the distro: it is the ROCm-to-DXCore shim the WSL path runs on.\",\n \"commands\": [\n \"rocm install driver\",\n \"# Then, once the plan looks right:\",\n \"# rocm install driver --yes\"\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-wsl-3-rocdxg-missing\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"This downloads and installs a .deb with sudo, so `rocm fix` prints it rather than running it. `rocm install driver` shows the full plan, and checks the download against a digest pinned for that ROCDXG release.\"\n ],\n \"verify\": \"ldconfig -p | grep librocdxg\"\n },\n \"symptom_matched\": false\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stderr": "", + "rc": 0 + }, + "diagnose_text": { + "stdout": "#1 (fix-wsl-3-rocdxg-missing) [LIKELY score=65/100] ROCDXG not installed in the distro\n id: fix-wsl-3-rocdxg-missing\n - librocdxg.so was not found under any ROCm install or on LD_LIBRARY_PATH\n - the WSL DXCore handoff is present, so only the distro-side ROCDXG is missing\n plan: Install ROCDXG inside the distro: it is the ROCm-to-DXCore shim the WSL path runs on.\n $ rocm install driver\n $ # Then, once the plan looks right:\n $ # rocm install driver --yes\n flags: requires sudo, manual only (`rocm fix` will NOT run it automatically)\n note: This downloads and installs a .deb with sudo, so `rocm fix` prints it rather than running it. `rocm install driver` shows the full plan, and checks the download against a digest pinned for that ROCDXG release.\n verify after fix: ldconfig -p | grep librocdxg\n apply with: rocm fix fix-wsl-3-rocdxg-missing\n\nHighest-scoring match is below the HIGH_CONFIDENCE threshold. Confirm one more piece of evidence before applying.\nWhen you are ready, run `rocm fix fix-wsl-3-rocdxg-missing` (the highest-scoring match), or use the `apply with:` line of whichever cause fits.\n", + "stderr": "", + "rc": 0 + }, + "examine_json": { + "stdout": "{\n \"os_family\": \"linux\",\n \"os_version\": \"6.8.0\",\n \"distro_id\": \"ubuntu\",\n \"distro_version\": \"24.04\",\n \"distro_id_like\": \"\",\n \"kernel_release\": \"6.8.0-generic\",\n \"kernel_cmdline\": \"\",\n \"is_wsl\": true,\n \"wsl\": {\n \"version\": 2,\n \"dxg_device\": true,\n \"dxcore\": true,\n \"wsl_lib_dir\": true,\n \"librocdxg\": false,\n \"rocdxg_dids\": false,\n \"ldconfig_librocdxg\": null,\n \"librocdxg_on_ld_library_path\": false,\n \"librocdxg_beside_runtime\": false,\n \"rocminfo\": false,\n \"rocm_sees_gpu\": null,\n \"distro_supported\": null,\n \"host_driver_version\": null,\n \"host_reachable\": false,\n \"locally_probed\": false\n },\n \"cpu_vendor\": \"AuthenticAMD\",\n \"cpu_model\": \"\",\n \"gpus\": [],\n \"has_amd_gpu\": false,\n \"has_nvidia_gpu\": false,\n \"has_apu\": false,\n \"has_discrete_amd\": false,\n \"amdgpu_loaded\": null,\n \"amdgpu_blacklisted_in\": [],\n \"amdkfd_loaded\": null,\n \"secure_boot\": \"unknown\",\n \"iommu_kernel_param\": \"\",\n \"kfd\": null,\n \"render_devices\": [],\n \"user_name\": \"\",\n \"user_groups\": [],\n \"in_render_group\": null,\n \"in_video_group\": null,\n \"rocm_version\": \"\",\n \"rocm_install_method\": \"\",\n \"rocm_path\": \"\",\n \"rocminfo_present\": false,\n \"rocminfo_status\": \"\",\n \"hip_libs_on_ld_path\": null,\n \"rocm_repos_seen\": [],\n \"comgr_paths\": [],\n \"comgr_selected\": null,\n \"comgr_version\": \"\",\n \"comgr_matches_runtime\": null,\n \"hip_paths\": [],\n \"hip_selected\": null,\n \"hip_sdk_path\": \"\",\n \"hip_sdk_version\": \"\",\n \"hipinfo_present\": false,\n \"hipinfo_status\": \"\",\n \"adrenalin_version\": \"\",\n \"adrenalin_date\": \"\",\n \"msvc_redist_present\": null,\n \"msvc_redist_version\": \"\",\n \"framework\": \"unknown\",\n \"framework_version\": \"\",\n \"framework_rocm_version\": \"\",\n \"framework_arch_list\": [],\n \"framework_notes\": [],\n \"framework_source\": \"\",\n \"env\": {},\n \"in_container\": false,\n \"container_kind\": \"\",\n \"shm_total_bytes\": null,\n \"shm_available_bytes\": null,\n \"dmesg_amdgpu_tail\": [],\n \"notes\": [],\n \"probe_failures\": [],\n \"status\": \"wsl\"\n}\n", + "stderr": "", + "rc": 0 + } +} diff --git a/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json index 06e020882e..3bcab7bb76 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-older-cli-stops-before-diagnosing/replies.json @@ -10,12 +10,12 @@ "rc": 2 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm major you\",\n \"# have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n }\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework's build arch list\",\n \"score\": 45,\n \"evidence\": [\n \"error mentions hipErrorNoBinaryForGpu\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\",\n \"commands\": [\n \"# Recommended: a PyTorch ROCm nightly, which often carries kernels a\",\n \"# release has not shipped yet. Pick the nightly for the ROCm X.Y release\",\n \"# you have, not an older one.\",\n \"pip uninstall -y torch torchvision torchaudio\",\n \"pip install --pre torch torchvision torchaudio \\\\\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\",\n \"# llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\",\n \"# cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-1-arch\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\"\n ],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\\\"\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": false,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, "diagnose_text": { - "stdout": "#1 (fix-1-arch) [WEAK score=45/100] GPU gfx target not in framework's build arch list\n id: fix-1-arch\n - error mentions hipErrorNoBinaryForGpu\n plan: Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\n $ # Recommended: a PyTorch ROCm nightly, which often carries kernels a\n $ # release has not shipped yet. Pick the nightly for the ROCm major you\n $ # have, not an older one.\n $ pip uninstall -y torch torchvision torchaudio\n $ pip install --pre torch torchvision torchaudio \\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\n $ # llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\n $ # cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\n flags: manual only (`rocm fix` will NOT run it automatically)\n note: TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\n verify after fix: python -c \"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\"\n apply with: rocm fix fix-1-arch\n\nHighest-scoring match is below the HIGH_CONFIDENCE threshold. Confirm one more piece of evidence before applying.\nWhen you are ready, run `rocm fix fix-1-arch` (the highest-scoring match), or use the `apply with:` line of whichever cause fits.\n", + "stdout": "#1 (fix-1-arch) [WEAK score=45/100] GPU gfx target not in framework's build arch list\n id: fix-1-arch\n - error mentions hipErrorNoBinaryForGpu\n plan: Reinstall the framework from a wheel index that includes this GPU's gfx target. Use HSA_OVERRIDE_GFX_VERSION ONLY as a temporary workaround when no native wheel exists.\n $ # Recommended: a PyTorch ROCm nightly, which often carries kernels a\n $ # release has not shipped yet. Pick the nightly for the ROCm X.Y release\n $ # you have, not an older one.\n $ pip uninstall -y torch torchvision torchaudio\n $ pip install --pre torch torchvision torchaudio \\\n --index-url https://download.pytorch.org/whl/nightly/rocm7.14\n $ # llama.cpp: rebuild with AMDGPU_TARGETS set to this GPU's gfx.\n $ # cmake -B build -DGGML_HIP=ON -DAMDGPU_TARGETS=\n flags: manual only (`rocm fix` will NOT run it automatically)\n note: TheRock (rocm/TheRock) ships nightly per-gfx wheels and is the preferred fallback when the official pytorch wheel index does not yet cover your gfx target.\n verify after fix: python -c \"import torch; print(torch.cuda.is_available(), torch.cuda.get_arch_list())\"\n apply with: rocm fix fix-1-arch\n\nHighest-scoring match is below the HIGH_CONFIDENCE threshold. Confirm one more piece of evidence before applying.\nWhen you are ready, run `rocm fix fix-1-arch` (the highest-scoring match), or use the `apply with:` line of whichever cause fits.\n", "stderr": "", "rc": 0 } diff --git a/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json index 0055d186e0..eb3b977605 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-vllm-import-error/replies.json @@ -5,17 +5,17 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP major\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"vLLM engine start aborts on torch-c-dlpack-ext loading its CUDA variant\",\n \"score\": 50,\n \"evidence\": [\n \"error names libtorch_cuda.so, which a ROCm build of torch does not ship\"\n ],\n \"fix\": {\n \"summary\": \"Only if the engine's runtime holds a ROCm build of torch in the 2.4-2.9 range with torch-c-dlpack-ext installed: reinstall the engine so its pinned torch is restored, which moves torch off the versions the extension ships prebuilts for.\",\n \"commands\": [\n \"# --- step 1 of 3, in YOUR shell ---\",\n \"# Opens an INTERACTIVE subshell with the engine's environment active,\",\n \"# and does not return until you leave it. Run this line on its own.\",\n \"rocm engines shell vllm\",\n \"# --- step 2 of 3, INSIDE the subshell step 1 opened ---\",\n \"# Confirm the trigger before changing anything. It has to be the\",\n \"# ENGINE's interpreter, not the one on your PATH -- they are different\",\n \"# interpreters, and only the engine's decides this failure.\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\",\n \"python -c \\\"import importlib.metadata as m; print(m.version('torch-c-dlpack-ext'))\\\"\",\n \"# This entry applies ONLY when torch.version.hip is set, torch.__version__\",\n \"# is in the 2.4-2.9 range, and torch-c-dlpack-ext is installed. Outside\",\n \"# that range the extension raises a handled ImportError and this is not\",\n \"# the failure you are looking at. Then leave the subshell:\",\n \"exit\",\n \"# --- step 3 of 3, back in YOUR OWN shell ---\",\n \"# If all three held, put the engine's pinned torch back. Do NOT run\",\n \"# this from inside the subshell: it replaces the environment that\",\n \"# shell is standing in.\",\n \"rocm engines install vllm --reinstall\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-17-torch-dlpack\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Running vLLM on ROCm is not by itself a reason to apply this. The trigger is narrow: a ROCm build of torch in the 2.4-2.9 range (the versions torch-c-dlpack-ext ships prebuilts for), torch without a native __dlpack_c_exchange_api__, and torch-c-dlpack-ext present -- it arrives as a transitive dependency of tilelang, which vLLM pins.\",\n \"The usual way a runtime lands in that range is `rocm install sdk` being re-run after the engine was installed, which overwrites the engine's pinned torch. Reinstalling the engine puts the pin back.\",\n \"The defect is upstream and there is nothing to correct locally: torch-c-dlpack-ext picks its variant from torch.cuda.is_available(), which is True on ROCm because PyTorch reuses the torch.cuda namespace for HIP, and it ships no ROCm variant to pick. tvm_ffi imports it as optional but guards only ImportError/AttributeError, while ctypes.CDLL raises OSError -- so an explicitly optional import kills the process.\",\n \"A service that failed at startup is hidden from a plain `rocm services list`; pass --all to recover its id.\"\n ],\n \"verify\": \"rocm serve --engine vllm # then `rocm services list --all` and `rocm services logs ` to confirm the import no longer aborts\"\n }\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Framework wheel built for a different ROCm major than the system\",\n \"score\": 25,\n \"evidence\": [\n \"ldopen failure\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from the wheel index that matches the system ROCm major (or upgrade the system ROCm to match the wheel).\",\n \"commands\": [\n \"pip uninstall -y torch torchvision torchaudio\",\n \"# Install the index for the ROCm major `rocm examine` reports:\",\n \"pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14\",\n \"# ROCm 10 has no released PyTorch index yet; it is on the nightly channel:\",\n \"pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0\",\n \"# Then re-check:\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-8-wheel-rocm\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.version.hip)\\\"\"\n }\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"vLLM engine start aborts on torch-c-dlpack-ext loading its CUDA variant\",\n \"score\": 50,\n \"evidence\": [\n \"error names libtorch_cuda.so, which a ROCm build of torch does not ship\"\n ],\n \"fix\": {\n \"summary\": \"Only if the engine's runtime holds a ROCm build of torch in the 2.4-2.9 range with torch-c-dlpack-ext installed: reinstall the engine so its pinned torch is restored, which moves torch off the versions the extension ships prebuilts for.\",\n \"commands\": [\n \"# --- step 1 of 3, in YOUR shell ---\",\n \"# Opens an INTERACTIVE subshell with the engine's environment active,\",\n \"# and does not return until you leave it. Run this line on its own.\",\n \"rocm engines shell vllm\",\n \"# --- step 2 of 3, INSIDE the subshell step 1 opened ---\",\n \"# Confirm the trigger before changing anything. It has to be the\",\n \"# ENGINE's interpreter, not the one on your PATH -- they are different\",\n \"# interpreters, and only the engine's decides this failure.\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\",\n \"python -c \\\"import importlib.metadata as m; print(m.version('torch-c-dlpack-ext'))\\\"\",\n \"# This entry applies ONLY when torch.version.hip is set, torch.__version__\",\n \"# is in the 2.4-2.9 range, and torch-c-dlpack-ext is installed. Outside\",\n \"# that range the extension raises a handled ImportError and this is not\",\n \"# the failure you are looking at. Then leave the subshell:\",\n \"exit\",\n \"# --- step 3 of 3, back in YOUR OWN shell ---\",\n \"# If all three held, put the engine's pinned torch back. Do NOT run\",\n \"# this from inside the subshell: it replaces the environment that\",\n \"# shell is standing in.\",\n \"rocm engines install vllm --reinstall\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-17-torch-dlpack\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Running vLLM on ROCm is not by itself a reason to apply this. The trigger is narrow: a ROCm build of torch in the 2.4-2.9 range (the versions torch-c-dlpack-ext ships prebuilts for), torch without a native __dlpack_c_exchange_api__, and torch-c-dlpack-ext present -- it arrives as a transitive dependency of tilelang, which vLLM pins.\",\n \"The usual way a runtime lands in that range is `rocm install sdk` being re-run after the engine was installed, which overwrites the engine's pinned torch. Reinstalling the engine puts the pin back.\",\n \"The defect is upstream and there is nothing to correct locally: torch-c-dlpack-ext picks its variant from torch.cuda.is_available(), which is True on ROCm because PyTorch reuses the torch.cuda namespace for HIP, and it ships no ROCm variant to pick. tvm_ffi imports it as optional but guards only ImportError/AttributeError, while ctypes.CDLL raises OSError -- so an explicitly optional import kills the process.\",\n \"A service that failed at startup is hidden from a plain `rocm services list`; pass --all to recover its id.\"\n ],\n \"verify\": \"rocm serve --engine vllm # then `rocm services list --all` and `rocm services logs ` to confirm the import no longer aborts\"\n },\n \"symptom_matched\": true\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Framework wheel built for a different ROCm X.Y release than the system\",\n \"score\": 25,\n \"evidence\": [\n \"ldopen failure\"\n ],\n \"fix\": {\n \"summary\": \"Reinstall the framework from the wheel index that matches the system ROCm X.Y release (or upgrade the system ROCm to match the wheel).\",\n \"commands\": [\n \"pip uninstall -y torch torchvision torchaudio\",\n \"# Install the index for the ROCm X.Y release `rocm examine` reports:\",\n \"pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14\",\n \"# ROCm 10 has no released PyTorch index yet; it is on the nightly channel:\",\n \"pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0\",\n \"# Then re-check:\",\n \"python -c \\\"import torch; print(torch.__version__, torch.version.hip)\\\"\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-8-wheel-rocm\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [],\n \"verify\": \"python -c \\\"import torch; print(torch.cuda.is_available(), torch.version.hip)\\\"\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, "diagnose_text": { - "stdout": "#1 (fix-17-torch-dlpack) [LIKELY score=50/100] vLLM engine start aborts on torch-c-dlpack-ext loading its CUDA variant\n id: fix-17-torch-dlpack\n - error names libtorch_cuda.so, which a ROCm build of torch does not ship\n plan: Only if the engine's runtime holds a ROCm build of torch in the 2.4-2.9 range with torch-c-dlpack-ext installed: reinstall the engine so its pinned torch is restored, which moves torch off the versions the extension ships prebuilts for.\n $ # --- step 1 of 3, in YOUR shell ---\n $ # Opens an INTERACTIVE subshell with the engine's environment active,\n $ # and does not return until you leave it. Run this line on its own.\n $ rocm engines shell vllm\n $ # --- step 2 of 3, INSIDE the subshell step 1 opened ---\n $ # Confirm the trigger before changing anything. It has to be the\n $ # ENGINE's interpreter, not the one on your PATH -- they are different\n $ # interpreters, and only the engine's decides this failure.\n $ python -c \"import torch; print(torch.__version__, torch.version.hip)\"\n $ python -c \"import importlib.metadata as m; print(m.version('torch-c-dlpack-ext'))\"\n $ # This entry applies ONLY when torch.version.hip is set, torch.__version__\n $ # is in the 2.4-2.9 range, and torch-c-dlpack-ext is installed. Outside\n $ # that range the extension raises a handled ImportError and this is not\n $ # the failure you are looking at. Then leave the subshell:\n $ exit\n $ # --- step 3 of 3, back in YOUR OWN shell ---\n $ # If all three held, put the engine's pinned torch back. Do NOT run\n $ # this from inside the subshell: it replaces the environment that\n $ # shell is standing in.\n $ rocm engines install vllm --reinstall\n flags: manual only (`rocm fix` will NOT run it automatically)\n note: Running vLLM on ROCm is not by itself a reason to apply this. The trigger is narrow: a ROCm build of torch in the 2.4-2.9 range (the versions torch-c-dlpack-ext ships prebuilts for), torch without a native __dlpack_c_exchange_api__, and torch-c-dlpack-ext present -- it arrives as a transitive dependency of tilelang, which vLLM pins.\n note: The usual way a runtime lands in that range is `rocm install sdk` being re-run after the engine was installed, which overwrites the engine's pinned torch. Reinstalling the engine puts the pin back.\n note: The defect is upstream and there is nothing to correct locally: torch-c-dlpack-ext picks its variant from torch.cuda.is_available(), which is True on ROCm because PyTorch reuses the torch.cuda namespace for HIP, and it ships no ROCm variant to pick. tvm_ffi imports it as optional but guards only ImportError/AttributeError, while ctypes.CDLL raises OSError -- so an explicitly optional import kills the process.\n note: A service that failed at startup is hidden from a plain `rocm services list`; pass --all to recover its id.\n verify after fix: rocm serve --engine vllm # then `rocm services list --all` and `rocm services logs ` to confirm the import no longer aborts\n apply with: rocm fix fix-17-torch-dlpack\n\n#2 (fix-8-wheel-rocm) [WEAK score=25/100] Framework wheel built for a different ROCm major than the system\n id: fix-8-wheel-rocm\n - ldopen failure\n plan: Reinstall the framework from the wheel index that matches the system ROCm major (or upgrade the system ROCm to match the wheel).\n $ pip uninstall -y torch torchvision torchaudio\n $ # Install the index for the ROCm major `rocm examine` reports:\n $ pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14\n $ # ROCm 10 has no released PyTorch index yet; it is on the nightly channel:\n $ pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0\n $ # Then re-check:\n $ python -c \"import torch; print(torch.__version__, torch.version.hip)\"\n flags: manual only (`rocm fix` will NOT run it automatically)\n verify after fix: python -c \"import torch; print(torch.cuda.is_available(), torch.version.hip)\"\n apply with: rocm fix fix-8-wheel-rocm\n\nHighest-scoring match is below the HIGH_CONFIDENCE threshold. Confirm one more piece of evidence before applying.\nWhen you are ready, run `rocm fix fix-17-torch-dlpack` (the highest-scoring match), or use the `apply with:` line of whichever cause fits.\n", + "stdout": "#1 (fix-17-torch-dlpack) [LIKELY score=50/100] vLLM engine start aborts on torch-c-dlpack-ext loading its CUDA variant\n id: fix-17-torch-dlpack\n - error names libtorch_cuda.so, which a ROCm build of torch does not ship\n plan: Only if the engine's runtime holds a ROCm build of torch in the 2.4-2.9 range with torch-c-dlpack-ext installed: reinstall the engine so its pinned torch is restored, which moves torch off the versions the extension ships prebuilts for.\n $ # --- step 1 of 3, in YOUR shell ---\n $ # Opens an INTERACTIVE subshell with the engine's environment active,\n $ # and does not return until you leave it. Run this line on its own.\n $ rocm engines shell vllm\n $ # --- step 2 of 3, INSIDE the subshell step 1 opened ---\n $ # Confirm the trigger before changing anything. It has to be the\n $ # ENGINE's interpreter, not the one on your PATH -- they are different\n $ # interpreters, and only the engine's decides this failure.\n $ python -c \"import torch; print(torch.__version__, torch.version.hip)\"\n $ python -c \"import importlib.metadata as m; print(m.version('torch-c-dlpack-ext'))\"\n $ # This entry applies ONLY when torch.version.hip is set, torch.__version__\n $ # is in the 2.4-2.9 range, and torch-c-dlpack-ext is installed. Outside\n $ # that range the extension raises a handled ImportError and this is not\n $ # the failure you are looking at. Then leave the subshell:\n $ exit\n $ # --- step 3 of 3, back in YOUR OWN shell ---\n $ # If all three held, put the engine's pinned torch back. Do NOT run\n $ # this from inside the subshell: it replaces the environment that\n $ # shell is standing in.\n $ rocm engines install vllm --reinstall\n flags: manual only (`rocm fix` will NOT run it automatically)\n note: Running vLLM on ROCm is not by itself a reason to apply this. The trigger is narrow: a ROCm build of torch in the 2.4-2.9 range (the versions torch-c-dlpack-ext ships prebuilts for), torch without a native __dlpack_c_exchange_api__, and torch-c-dlpack-ext present -- it arrives as a transitive dependency of tilelang, which vLLM pins.\n note: The usual way a runtime lands in that range is `rocm install sdk` being re-run after the engine was installed, which overwrites the engine's pinned torch. Reinstalling the engine puts the pin back.\n note: The defect is upstream and there is nothing to correct locally: torch-c-dlpack-ext picks its variant from torch.cuda.is_available(), which is True on ROCm because PyTorch reuses the torch.cuda namespace for HIP, and it ships no ROCm variant to pick. tvm_ffi imports it as optional but guards only ImportError/AttributeError, while ctypes.CDLL raises OSError -- so an explicitly optional import kills the process.\n note: A service that failed at startup is hidden from a plain `rocm services list`; pass --all to recover its id.\n verify after fix: rocm serve --engine vllm # then `rocm services list --all` and `rocm services logs ` to confirm the import no longer aborts\n apply with: rocm fix fix-17-torch-dlpack\n\n#2 (fix-8-wheel-rocm) [WEAK score=25/100] Framework wheel built for a different ROCm X.Y release than the system\n id: fix-8-wheel-rocm\n - ldopen failure\n plan: Reinstall the framework from the wheel index that matches the system ROCm X.Y release (or upgrade the system ROCm to match the wheel).\n $ pip uninstall -y torch torchvision torchaudio\n $ # Install the index for the ROCm X.Y release `rocm examine` reports:\n $ pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.14\n $ # ROCm 10 has no released PyTorch index yet; it is on the nightly channel:\n $ pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm10.0\n $ # Then re-check:\n $ python -c \"import torch; print(torch.__version__, torch.version.hip)\"\n flags: manual only (`rocm fix` will NOT run it automatically)\n verify after fix: python -c \"import torch; print(torch.cuda.is_available(), torch.version.hip)\"\n apply with: rocm fix fix-8-wheel-rocm\n\nHighest-scoring match is below the HIGH_CONFIDENCE threshold. Confirm one more piece of evidence before applying.\nWhen you are ready, run `rocm fix fix-17-torch-dlpack` (the highest-scoring match), or use the `apply with:` line of whichever cause fits.\n", "stderr": "", "rc": 0 } diff --git a/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json b/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json index ae2d4e5f67..d385254b59 100644 --- a/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json +++ b/skills/rocm-doctor/evals/fixtures/rocm-vllm-oom-at-startup/replies.json @@ -5,12 +5,12 @@ "rc": 0 }, "fix_json": { - "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP major\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n }\n}\n", + "stdout": "{\n \"contract_version\": 1,\n \"entries\": [\n {\n \"id\": \"fix-1-arch\",\n \"title\": \"GPU gfx target not in framework arch list\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-2-unset-override\",\n \"title\": \"Unset HSA_OVERRIDE_GFX_VERSION\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-3-rocm-kernel\",\n \"title\": \"ROCm/distro/kernel triple unsupported\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-4-render-group\",\n \"title\": \"Add user to render/video groups\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": true\n },\n {\n \"id\": \"fix-5-amdgpu-load\",\n \"title\": \"Load amdgpu (and clear any blacklist)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-6-path\",\n \"title\": \"Add the ROCm/HIP bin directory to PATH\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"auto\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"auto\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-7-stale-repos\",\n \"title\": \"Quarantine duplicate AMD repos\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-8-wheel-rocm\",\n \"title\": \"Reinstall the framework against the system ROCm/HIP X.Y release\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n },\n {\n \"id\": \"fix-9-igpu-dgpu\",\n \"title\": \"Hide the iGPU with HIP_VISIBLE_DEVICES\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"needs-argument\",\n \"needs\": \"--device-index\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-10-container\",\n \"title\": \"Re-launch the container with AMD devices passed through\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-11-iommu\",\n \"title\": \"Add iommu=pt to the kernel command line\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-12-installer\",\n \"title\": \"Reset repo-native install state and reinstall\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": true,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-13-hip-sdk-missing\",\n \"title\": \"Install the AMD HIP SDK for Windows\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/developer/resources/rocm-hub/hip-sdk.html\"\n },\n {\n \"id\": \"fix-14-adrenalin-too-old\",\n \"title\": \"Update the Adrenalin / kernel-mode driver\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-windows/en/latest/install/install.html\"\n },\n {\n \"id\": \"fix-15-msvc-redist\",\n \"title\": \"Install the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-17-torch-dlpack\",\n \"title\": \"Restore the engine's pinned torch (torch-c-dlpack-ext loads the CUDA variant)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-1-gpu-not-exposed\",\n \"title\": \"Expose the GPU to the WSL distro (/dev/dxg)\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-2-dxcore-missing\",\n \"title\": \"Restore the WSL DXCore libraries\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-3-rocdxg-missing\",\n \"title\": \"Install ROCDXG in the WSL distro\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-4-rocdxg-not-linked\",\n \"title\": \"Refresh the linker cache so ROCDXG is loadable\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-5-distro-too-old\",\n \"title\": \"Move to a distro release the WSL path supports\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-6-host-driver-too-old\",\n \"title\": \"Update the AMD driver on the Windows host\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-wsl-7-wsl1\",\n \"title\": \"Convert the distro from WSL 1 to WSL 2\",\n \"platforms\": [\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-18-comgr-conflict\",\n \"title\": \"Code object manager library does not belong to the active HIP runtime\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-19-shm-too-small\",\n \"title\": \"Raise the shared memory allowance\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"print-only\"\n },\n {\n \"os\": \"wsl\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": true,\n \"needs_reboot\": false,\n \"needs_relogin\": false\n },\n {\n \"id\": \"fix-21-msvc-redist-outdated\",\n \"title\": \"Update the MSVC 2015-2022 runtime redistributable\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://aka.ms/vs/17/release/vc_redist.x64.exe\"\n },\n {\n \"id\": \"fix-22-display-driver-old\",\n \"title\": \"Look for a newer AMD display driver (advice only)\",\n \"platforms\": [\n {\n \"os\": \"windows\",\n \"class\": \"print-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": true,\n \"needs_relogin\": false,\n \"doc_url\": \"https://www.amd.com/en/support\"\n },\n {\n \"id\": \"info-20-gpu-no-rocm-support\",\n \"title\": \"GPU has no ROCm/HIP support (Vulkan backend still works)\",\n \"platforms\": [\n {\n \"os\": \"linux\",\n \"class\": \"diagnose-only\"\n },\n {\n \"os\": \"windows\",\n \"class\": \"diagnose-only\"\n }\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"doc_url\": \"https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html\"\n }\n ],\n \"exit_codes\": {\n \"declined\": 5,\n \"failed\": 4,\n \"internal\": 1,\n \"not_applicable\": 3,\n \"ok\": 0,\n \"unknown_id\": 2\n },\n \"exit_code_descriptions\": {\n \"declined\": \"The fix needed confirmation and did not get it: the user answered no, or stdin is not a terminal and `--yes` was not given. Nothing the confirmation guarded was changed.\",\n \"failed\": \"A step of the fix failed: a command it ran, a file it wrote, or a read of state it needed before planning (the current User PATH on Windows, read even under `--dry-run`). The change that step was to make was not made.\",\n \"internal\": \"The CLI is inconsistent with itself (a recipe it says it can carry out, with nothing to carry it out with); never a fault of the machine. Exit 1 is also any other error the command reports on stderr, such as `--json` combined with a fix-id.\",\n \"not_applicable\": \"The entry exists but cannot be used here: it is for another platform, the host lacks something the fix cannot do without (a tool it runs, or `sudo` when not root; an install it points at; the current user's name; or a home directory to write a file in), or an argument it was given is one it cannot use, such as a negative device index written `--device-index=-1`. Nothing was tried. Written with a space instead, `--device-index -1` is a usage error and exits 2 (see `unknown_id`).\",\n \"ok\": \"Nothing failed: the fix was applied, found nothing to change (already in place, or a setting it leaves for you to edit), was previewed with `--dry-run`, or only printed what to do (steps to run yourself, or the explanation of a diagnose-only entry); or the catalog was listed. A `--dry-run` preview is not always 0: it exits 3 or 4 when the fix cannot be planned.\",\n \"unknown_id\": \"No catalog entry has that fix-id; no fix step was run, and a stderr line starts with `Unknown fix-id:`. Exit 2 is also what the argument parser returns for a malformed command line (an unknown option, a missing value, or a value it cannot read, such as a non-number for `--device-index`, or a negative one written after a space, `--device-index -1`, which it reads as an unknown option), before any fix-id is looked up; that refusal has a stderr line starting with `error:` and never says `Unknown fix-id:`. Another line, such as a one-time `rocm:` notice, can come first. The code alone cannot tell the two apart; the message can.\"\n }\n}\n", "stderr": "", "rc": 0 }, "diagnose_json": { - "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"score\": 95,\n \"evidence\": [\n \"error mentions 'HIP out of memory'\",\n \"error mentions torch.OutOfMemoryError\"\n ],\n \"fix\": {\n \"summary\": \"vLLM reserves ~90% of the GPU's total VRAM by default; on a shared or busy GPU this can collide with memory already in use. Lower the reservation with `--gpu-memory-utilization ` (e.g. 0.5 for a small model), or target a less-busy GPU with `--gpu `. If instead the model genuinely does not fit in this GPU's VRAM, lowering the reservation will not help - use a smaller or quantized model (rocm-cli serves one model on a single GPU; it does not shard a model across GPUs).\",\n \"commands\": [\n \"# Case 1 -- shared/busy GPU (tenancy collision): lower the reservation,\",\n \"# or steer vLLM onto a less-busy device:\",\n \"rocm serve --gpu-memory-utilization 0.5\",\n \"rocm serve --gpu \",\n \"# Case 2 -- the model genuinely does not fit: the reservation is not the\",\n \"# problem; pick a smaller or quantized model (single-GPU serving only).\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-16-vllm-oom\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.\",\n \"This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).\"\n ],\n \"verify\": \"rocm serve # re-run and watch for a clean startup\"\n }\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", + "stdout": "{\n \"matched\": [\n {\n \"id\": \"fix-16-vllm-oom\",\n \"title\": \"vLLM ran the GPU out of memory at startup\",\n \"score\": 95,\n \"evidence\": [\n \"error mentions 'HIP out of memory'\",\n \"error mentions torch.OutOfMemoryError\"\n ],\n \"fix\": {\n \"summary\": \"vLLM reserves ~90% of the GPU's total VRAM by default; on a shared or busy GPU this can collide with memory already in use. Lower the reservation with `--gpu-memory-utilization ` (e.g. 0.5 for a small model), or target a less-busy GPU with `--gpu `. If instead the model genuinely does not fit in this GPU's VRAM, lowering the reservation will not help - use a smaller or quantized model (rocm-cli serves one model on a single GPU; it does not shard a model across GPUs).\",\n \"commands\": [\n \"# Case 1 -- shared/busy GPU (tenancy collision): lower the reservation,\",\n \"# or steer vLLM onto a less-busy device:\",\n \"rocm serve --gpu-memory-utilization 0.5\",\n \"rocm serve --gpu \",\n \"# Case 2 -- the model genuinely does not fit: the reservation is not the\",\n \"# problem; pick a smaller or quantized model (single-GPU serving only).\"\n ],\n \"needs_sudo\": false,\n \"needs_reboot\": false,\n \"needs_relogin\": false,\n \"fix_id\": \"fix-16-vllm-oom\",\n \"auto_applicable\": false,\n \"class\": \"print-only\",\n \"notes\": [\n \"Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.\",\n \"This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).\"\n ],\n \"verify\": \"rocm serve # re-run and watch for a clean startup\"\n },\n \"symptom_matched\": true\n }\n ],\n \"has_match\": true,\n \"min_score_for_match\": 50,\n \"high_confidence_threshold\": 75,\n \"route_when_no_match\": {\n \"target\": \"rocm-core\",\n \"url\": \"https://github.com/ROCm/ROCm/issues\"\n },\n \"out_of_scope\": null,\n \"model\": null\n}\n", "stderr": "", "rc": 0 }, diff --git a/skills/rocm-doctor/evals/hooks.py b/skills/rocm-doctor/evals/hooks.py index 84802853bc..5358395095 100644 --- a/skills/rocm-doctor/evals/hooks.py +++ b/skills/rocm-doctor/evals/hooks.py @@ -7,7 +7,8 @@ with no directory runs exactly as it did before: no `rocm` on PATH. The stub replays recorded output and nothing else. It answers `--version`, -`fix --json` and `diagnose`, and refuses every other command with exit code 2. +`fix --json` and `diagnose`, and `examine --json` for a case whose recording has +one, and refuses every other command with exit code 2. It only ever prints text read from `replies.json`, so whatever an agent asks it, it cannot change a machine or send anything: that comes from the allowlist, not from the refusal of `diagnose` options below, which is there for fidelity (see @@ -72,9 +73,11 @@ def reply(entry): reply(replies["fix_json"]) elif args[:1] == ["diagnose"] and not refused & {a.split("=")[0] for a in args}: reply(replies["diagnose_json" if "--json" in args else "diagnose_text"]) +elif args == ["examine", "--json"] and "examine_json" in replies: + reply(replies["examine_json"]) sys.stderr.write( - "error: this evaluation replays only `--version`, `fix --json` and " - "`diagnose`; it runs nothing else\n" + "error: this evaluation replays only `--version`, `fix --json`, `diagnose` " + "and, where recorded, `examine --json`; it runs nothing else\n" ) sys.exit(2) """ diff --git a/skills/rocm-doctor/evals/test_hooks.py b/skills/rocm-doctor/evals/test_hooks.py index c96db9d498..a1bb199886 100644 --- a/skills/rocm-doctor/evals/test_hooks.py +++ b/skills/rocm-doctor/evals/test_hooks.py @@ -4,9 +4,9 @@ `xtask/src/eval_fixtures.rs`), as `python -I test_hooks.py`. Stdlib only. What they pin is the promise the docs make about the stub: it replays the -recorded answer for `--version`, `fix --json` and `diagnose`, refuses every -other command with exit code 2, and leaves PATH and the temporary directory as -it found them. It never runs anything (it only prints canned text); the refusal +recorded answer for `--version`, `fix --json` and `diagnose` (and `examine +--json` for a case whose recording has one), refuses every other command with +exit code 2, and leaves PATH and the temporary directory as it found them. It never runs anything (it only prints canned text); the refusal of unrecorded `diagnose` options is about fidelity. """ @@ -119,6 +119,9 @@ def test_every_other_command_is_refused_with_exit_code_2(self): ["diagnose", "--model=foo"], ["diagnose", "--distro=Ubuntu"], ["diagnose", "--json", "--send=me@example.invalid"], + # This case recorded no examination, so there is nothing to replay. + ["examine", "--json"], + ["examine"], ]: with self.subTest(args=args): done = rocm(*args) @@ -138,6 +141,56 @@ def test_teardown_restores_path_and_removes_the_directory(self): self.assertEqual(os.environ["PATH"], self.saved_path) +def examine_recording(case_id): + return json.loads( + (HERE / "fixtures" / case_id / "replies.json").read_text(encoding="utf-8") + )["examine_json"] + + +class ExamineReplay(unittest.TestCase): + """A case that names a host replays `examine --json`, and only that form.""" + + def replay(self, case_id): + case = types.SimpleNamespace(id=case_id) + saved = os.environ["PATH"] + ctx = {} + hooks.setup(None, case, ctx) + + def restore(): + hooks.teardown(None, case, ctx) + os.environ["PATH"] = saved + + self.addCleanup(restore) + + def test_examine_json_prints_the_recorded_reply(self): + case_id = "rocm-no-vendor-no-amd-gpu-asks-which-gpu" + recorded = examine_recording(case_id) + self.replay(case_id) + done = rocm("examine", "--json") + self.assertEqual(done.returncode, recorded["rc"]) + self.assertEqual(done.returncode, 0) + self.assertEqual(done.stdout.replace("\r\n", "\n"), recorded["stdout"]) + self.assertEqual(json.loads(done.stdout)["status"], "no-amd-gpu") + + def test_a_failing_examine_replays_its_exit_code_and_stderr(self): + case_id = "rocm-examine-fails-relays-the-error" + recorded = examine_recording(case_id) + self.replay(case_id) + done = rocm("examine", "--json") + self.assertEqual(done.returncode, recorded["rc"]) + self.assertNotEqual(done.returncode, 0) + self.assertEqual(done.stdout, "") + self.assertEqual(done.stderr.replace("\r\n", "\n"), recorded["stderr"]) + + def test_the_plain_examine_form_is_still_refused(self): + self.replay("rocm-no-vendor-no-amd-gpu-asks-which-gpu") + for args in (["examine"], ["examine", "--json", "--verbose"]): + with self.subTest(args=args): + done = rocm(*args) + self.assertEqual(done.returncode, 2, done.stderr) + self.assertIn("runs nothing else", done.stderr) + + class Overlap(unittest.TestCase): def test_a_second_replay_while_one_is_active_is_refused(self): saved = os.environ["PATH"] diff --git a/skills/rocm-doctor/reference.md b/skills/rocm-doctor/reference.md index fe111b4581..8329422d10 100644 --- a/skills/rocm-doctor/reference.md +++ b/skills/rocm-doctor/reference.md @@ -16,7 +16,7 @@ tooling. - It's a general system inspector, so it **always exits 0**. The verdict is the `status` field: `ok` · `no-amd-gpu` · `wsl` · `unsupported-os` · `degraded`. -### `rocm diagnose [--symptom ""] [--top N] [--json] [--distro [NAME]] [--model REF]` +### `rocm diagnose [--symptom ""] [--top N] [--json] [--distro [NAME]] [--model REF [--context-size N] [--kv-cache-type f16|q8_0|q4_0]]` Match the host + symptom against the closed catalog. **Always exits 0**; read the result from `--json`: @@ -48,14 +48,48 @@ the result from `--json`: nothing was checked — not a clean bill of health. - `route_when_no_match` — `{ target, url }` upstream tracker to use when `has_match` is false. - `model` — set only when `--model ` was passed; otherwise not set. Answers - "will this curated model run here" from the recipe catalog and this host's - GPU/RAM, without downloading anything: `{ verdict, evidence[], engine, - required_gpu_memory_gib, available_gpu_memory_gib, alternatives[], - undetermined_reason, fix }`. `verdict` is one of `ready`, `degraded`, + "will this model run here", without downloading the weights: for a curated + model from the recipe catalog and this host's GPU/RAM, and for a GGUF outside + the catalog with an `estimate` from its file header (below). Its fields: + `{ verdict, evidence[], engine, required_gpu_memory_gib, + available_gpu_memory_gib, alternatives[], undetermined_reason, fix }`. `verdict` is one of `ready`, `degraded`, `blocked`, `undetermined` — treat `undetermined` as "the CLI could not find out", never as evidence the model is incompatible. `--model` is refused together with `--distro`: the verdict is about the machine running the command, not the one `--distro` names. + For a GGUF outside the catalog (a local `.gguf` path, or `owner/repo:QUANT`) + the verdict stays `undetermined` and an `estimate` object is added, sized from + the file's header (the weights are never downloaded). Its fields, by group: + where it came from (`source`, `architecture`, `layers`); the outcome + (`outcome` is `fits`, `does_not_fit` or `cannot_estimate`, and with + `cannot_estimate` the `cannot_estimate_reason` and `detail` say why; the + figures are still given when the reason is about the host (`no_gpu`, + `accelerator_memory_unknown`, `unified_memory_unreadable`: what the model would + need, with nothing to compare it to) and are null when it is about the file + (`header_unreadable`, `not_a_model` for a multimodal projector named in place + of a model, `split_file`, `latent_attention`, `no_attention_layers`, + `missing_metadata`), and are given with no verdict for `unmeasured_layout` (a + state-space, hybrid or mixture-of-experts model, part of whose memory is not + counted or not measured, so the total can be too low) and for a + `missing_metadata` header that lacks only the feed-forward or embedding width + (the compute buffer is then understated)); what was asked for (`context_size`, `context_size_defaulted`, + `kv_cache_type`); the memory figures (`weights_gib`, `kv_cache_gib`, + `compute_buffer_gib`, `runtime_reserve_gib`, `total_gib`, the same in bytes as + `weights_bytes`, `kv_cache_bytes`, `compute_buffer_bytes`, + `runtime_reserve_bytes` and `total_bytes`, and `kv_cache_bytes_per_token`, the + cache's cost of one token); the comparison (`available_gib` is the total + dedicated VRAM of the GPU with the most of it that ROCm can see, not what is + free; `max_context` is the largest context that fits, capped at + `training_context`, and is null when GPU memory was not measured, when no + verdict is given (`unmeasured_layout`, or `missing_metadata` with figures), + and when not even one 256-token step fits); and what the figures leave out + (`conservative_ceiling` says the ceiling is known to be conservative, + `suggestions[]` what would make it fit, `notes[]` the caveats). + It is an estimate, not a tested minimum: report it as one. With `--report` + the model is not assessed at all and nothing is fetched; a note on stderr names + the model flags that were ignored. + `suggestions[]` name llama.cpp options; `rocm serve` does not take a context + size yet, so do not propose a `rocm serve` command for them. ### `rocm fix [] [--yes] [--dry-run] [--device-index N] [--json]` @@ -83,6 +117,12 @@ never for every machine the id applies to: - **windows** — auto-applicable: `fix-2-unset-override`, `fix-6-path`. - **wsl** — auto-applicable: `fix-6-path`. +"Auto-applicable" here is the `auto` class on this host, the same as the +`fix.auto_applicable` flag in a diagnosis. `SKILL.md` counts a wider set, the +fixes that *have a runner*: these three ids plus `fix-9-igpu-dgpu`, whose runner +only prints the discrete-GPU query unless `--device-index N` is passed. The two +counts differ on purpose, and neither is wrong. + The rest print their plan for the user to run. Pass the **full** id (`rocm fix fix-2-unset-override`, not `rocm fix fix-2`; a short id returns exit 2, unknown fix-id). @@ -126,7 +166,7 @@ reporting confident nonsense. | `fix-5-amdgpu-load` | linux | `amdgpu` module not loaded (or blacklisted) | "ROCk module is NOT loaded", blacklist entry, Secure Boot | print-only | | `fix-6-path` | linux/windows/wsl | ROCm/HIP binaries not on PATH after install | `rocminfo: command not found`, `hipInfo` missing from PATH | auto | | `fix-7-stale-repos` | linux | Stale/conflicting APT/DNF/zypper repos from prior installer runs | apt 404 `repo.radeon.com`, unmet deps, ≥2 ROCm repo files | print-only | -| `fix-8-wheel-rocm` | linux/windows/wsl | Framework wheel built for a different ROCm major than the system | `libamdhip64.so.X` / `amdhip64_X.dll` load failure | print-only | +| `fix-8-wheel-rocm` | linux/windows/wsl | Framework wheel built for a different ROCm X.Y release than the system | `libamdhip64.so.X` / `amdhip64_X.dll` load failure | print-only | | `fix-9-igpu-dgpu` | linux/windows | iGPU enumerated alongside dGPU, destabilising the runtime | APU + discrete AMD present, `HIP_VISIBLE_DEVICES` unset, crash/segfault | needs-arg | | `fix-10-container` | linux | Container can't see `/dev/kfd` or `/dev/dri/renderD*` | running in docker/podman, kfd/render devices missing | print-only | | `fix-11-iommu` | linux | Multi-GPU hang with IOMMU enabled | ≥2 AMD GPUs, `iommu=` not `pt`, hang/deadlock/timeout | print-only | diff --git a/skills/rocm-doctor/skill-card.md b/skills/rocm-doctor/skill-card.md index 96a01cdcaf..94e3dcf1d4 100644 --- a/skills/rocm-doctor/skill-card.md +++ b/skills/rocm-doctor/skill-card.md @@ -3,10 +3,10 @@ ## Description Diagnoses ROCm / HIP SDK / PyTorch / llama.cpp failures on AMD GPUs (Linux, -Windows, and WSL2) against a closed list of known misconfigurations, and applies a -low-risk fix with consent or routes the user to the right upstream channel, and -covers two vLLM startup failures (running out of GPU memory, an engine import -error) on Linux and WSL2. +Windows, and WSL2) against a closed list of known misconfigurations, including two +print-only vLLM startup failures on Linux and WSL2 (running out of GPU memory, an +engine import error), and applies a low-risk fix with consent or routes the user +to the right upstream channel. ## Owner diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 48fe7bbeac..3efc130f5d 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -61,8 +61,9 @@ rand.workspace = true reqwest = { version = "0.13", features = ["json"] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true tempfile = "3" -tokio.workspace = true +tokio = { workspace = true, features = ["process", "io-util"] } toml = "1.1" # `ServeDir` backs the install scenario's loopback download server (see # `src/loopback_http.rs`), so the E2E suite does not hand-roll HTTP framing, @@ -88,6 +89,11 @@ shlex.workspace = true # "inference verified" latch is the field the product reads), never the library # itself, and `xtask check-crate-edges` exempts dev edges by design — so keeping # it here needs no allowlist entry. +# +# For the same reason the library's `device_memory::bounded_stdout` cannot call +# `rocm_core::process::run_with_timeout`: it repeats that helper's spawn, poll, +# kill and reap steps on `std` alone, deliberately, and a normal dependency +# would be a new edge to review. A change to either one should look at the other. rocm-core = { path = "../../crates/rocm-core" } # `test-util` pauses tokio's clock in the retry loops' unit tests # (`#[tokio::test(start_paused = true)]`), so their timing assertions read a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index f3e9ea9bd4..d005021bc3 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -179,7 +179,11 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | -| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | + +`e2e-gpu-strix-ubuntu` skips `pull_request` (one Linux GPU +node and one Windows GPU pool per PR); it also skips `merge_group` and still runs on `push` +and dispatch. The merge queue therefore has no native Linux GPU lane +(MI300X skips it too); it runs only Windows Strix and WSL2. The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 340bc546d9..feddea5300 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -34,20 +34,35 @@ # --- EAI-7219: short-name expansion (vLLM does not show an expanded name) --- # What the lane logs show, so the scope below is not read as more than it is: -# lemonade DOES print the full name (`resolved model: Qwen/Qwen2.5-1.5B-Instruct`) -# on the Strix Halo Ubuntu and Windows lanes, and vLLM's stdout is empty -- the run -# stopped before it printed a serve plan, so the plan's name line was never -# reached. EAI-7219 as reported describes the other shape: a vLLM plan that prints -# `resolved model: qwen2.5`, the alias left unexpanded; the lane logs have not -# shown it. The steps used to discard stderr, so the cause of the empty stdout was -# not known. The first lane run that reported it (run 41811714290: Strix Halo -# Ubuntu release and nightly, and MI300X) shows vLLM stopping at "no active ROCm -# runtime is configured", before any alias handling, so on those lanes the rows are -# red for a missing runtime and not for this bug. On native Windows the stop is the -# up-front vLLM refusal (ROCMAI-823), a designed refusal and not this bug either. +# lemonade prints the full name on the Strix Halo Ubuntu and Windows lanes, and +# vLLM's stdout is empty -- the run stopped before it printed a serve plan, so the +# plan's name line was never reached. vLLM is therefore the whole of what keeps +# these rows red. +# +# ROCMAI-863's pre-launch refusal does NOT reach these rows, though an earlier +# revision of this comment said it did. That was written when `qwen2.5` expanded +# to the bare `Qwen/Qwen2.5-1.5B-Instruct`, which ships no GGUF; #1044 then gave +# that recipe a per-engine ref, so lemonade is handed +# `Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M` with the variant already pinned and +# the refusal has nothing to fire on. The lemonade half still prints its name. +# EAI-7219 as reported describes +# the other shape: a vLLM plan that prints `resolved model: qwen2.5`, the alias +# left unexpanded; the lane logs have not shown it. The steps used to discard +# stderr, so the cause of the empty stdout was not known. The first lane run +# that reported it (run 41811714290: Strix Halo Ubuntu release and nightly, and +# MI300X) shows vLLM stopping at "no active ROCm runtime is configured", before +# any alias handling, so on those lanes the rows are red for a missing runtime +# and not for this bug. On native Windows the stop is the up-front vLLM refusal +# (ROCMAI-823), a designed refusal and not this bug either. # The rows keep their all-hosts scope until the scenarios get a managed runtime or # are gated on the engine and the rows are narrowed to the real bug; their reasons # name both shapes so neither becomes a false claim. +# +# Looking forward rather than back, and not yet observed on any lane: the catalog +# now gives `qwen2.5` a per-engine model id, so the name lemonade prints is +# expected to become `Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M` while vLLM keeps +# `Qwen/Qwen2.5-1.5B-Instruct`. The two differing is by design, which is why +# serve-02 asks each engine to expand the alias rather than to agree on a name. [["serve-short-name-expansion"]] when = {} bug = "EAI-7219" @@ -56,62 +71,98 @@ reason = "A vLLM serve of a short name shows no expanded full model name: it eit [["serve-short-name-consistent-across-engines"]] when = {} bug = "EAI-7219" -reason = "A vLLM serve of a short name does not resolve to lemonade's full model name: it either stops before printing a serve plan or prints the alias unexpanded." - -# EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: -# which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run -# 30993213188: 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails -# that lane on unrelated PRs, so the EAI-7423 lemonade-linux rows below carry -# `flaky = true` — either outcome is tolerated while the bug stays open. The flag -# is per-row and is NOT derived from `bug`, so a new EAI-7423 row needs it added -# explicitly. Revisit under EAI-7853 before deleting any of them: which lemonade -# build a given run used cannot currently be read back from this lane, so an -# XPASS streak here is not by itself evidence that the bug is fixed. +reason = "A serve of a short name does not show the full model name the engine will serve: vLLM either stops before printing a serve plan or prints the alias unexpanded, and the lemonade half is only reached once that stops happening." + +# ROCMAI-898 tracks multiple Strix Halo Ubuntu Lemonade serve signatures (first +# tracked as GitHub issue ROCm/rocm-cli#897, since closed; ROCMAI-898 is the open +# ticket and what the rows below name): +# the child rejects --no-mmap (41984921053), backend readiness on :8001 without +# the model appearing on :11435 (41984503055), and backend download failure +# (41986772728). A backend ready line does not prove a later shutdown. +# EAI-7423 described only the earlier ready-then-unavailable symptom. +# Retain flaky=true because historical runs also passed intermittently; do not +# infer a fix from a single XPASS. The flag is per-row and is NOT derived from +# `bug`, so a new ROCMAI-898 row needs `flaky = true` added explicitly. +# Failed serves now capture configured pins, candidate executable paths and +# fingerprints before stopping the service, and each candidate's --version/--help +# after it. +# Revisit under EAI-7853 before deleting any of these rows: which lemonade build +# a given run used could not be read back from this lane, and until a failed run +# there has carried that evidence, an XPASS streak is not by itself evidence that +# the bug is fixed. # -# On a lemonade-native LINUX host (Strix Halo Ubuntu) the lemonade managed serve -# loads the model and reaches ready on :8001, then is shut down ~0.08s later, so -# the endpoint never becomes reachable (EAI-7423, root-caused on the box -# 2026-07-15 — NOT the earlier "backend install hides the plan line" theory, which -# was disproven: the `engine:` line IS emitted). os=linux: on native Windows this -# scenario PASSES (run 29338475133 Strix-Windows), so it must NOT be xfail'd there. +# os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the endpoint never becomes reachable." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade-native LINUX host the served model's endpoint drops before -# inference: the lemonade managed serve reaches ready then is shut down ~0.08s -# later (EAI-7423, root-caused on the box 2026-07-15 — not a Vulkan hang). -# os=linux: passes on native Windows (run 29338475133). +# The same serve precondition blocks inference; native Windows passed. [["serve-default-engine-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade LINUX host this scenario does a real lemonade serve, which reaches -# ready then shuts down immediately (EAI-7423, see serve-lemonade-inference). -# os=linux: on native Windows lemonade serve works, so this PASSES there (verified -# run 29338475133 Strix-Windows XPASS) and must not be xfail'd. +# The readiness contract depends on the same real Lemonade serve. [["serve-readiness-contract"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the ready-to-serve contract fails." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." +serve_timeout_secs = 90 +flaky = true + +# The Anthropic-endpoint scenarios start from `a model is being served on GPU` +# like the rows above, so they hit the same Linux lemonade serve failure. +[["serve-anthropic-endpoint-advertised"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model, so no endpoint is left to advertise." +serve_timeout_secs = 90 +flaky = true + +[["serve-anthropic-messages-inference"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model, so the Anthropic request never gets an answer." serve_timeout_secs = 90 flaky = true -# --- EAI-7423: on a lemonade LINUX host the managed serve reaches ready then is -# shut down immediately, so inference never happens (root-caused on the box -# 2026-07-15; previously mis-attributed to the EAI-7052 Vulkan hang). os=linux: -# native Windows lemonade serve works (run 29338475133 Strix-Windows XPASS'd -# this), so scope the xfail to Linux. --- +[["serve-anthropic-messages-tool-use"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model, so the tool-use request never gets an answer." +serve_timeout_secs = 90 +flaky = true + +[["serve-anthropic-messages-tool-use"]] +when = { effective_engine = "vllm" } +bug = "EAI-7223" +reason = "The served vLLM model is started without a tool-call parser, so it rejects a request that names a tool; the endpoint relays that refusal." +serve_timeout_secs = 90 + +[["serve-anthropic-endpoint-readiness"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model, so readiness never covers the Anthropic endpoint." +serve_timeout_secs = 90 +flaky = true + +[["serve-anthropic-messages-streaming"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model, so the streaming Anthropic request never gets an answer." +serve_timeout_secs = 90 +flaky = true + +# Explicit Lemonade serving shares the same failure on the Ubuntu lane. [["serve-lemonade-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -124,20 +175,32 @@ flaky = true # the scenario with lemonade (the MI300X and W7900 lanes had not reported in that # run); whether lemonade serves there is not verified, and `flaky = true` tolerates # a pass on such a host. The cause on the Strix Halo Ubuntu lanes is not settled -# and varies between runs (#897), so this row names that issue rather than EAI-7423 +# and varies between runs (ROCMAI-898), so this row names that ticket rather than +# EAI-7423 # and carries `flaky = true` like its siblings. While it is in place the # scenario's own check (serve starts no background helper when no watcher is # enabled) is not exercised on the hosts it matches, and a genuine failure of that # check there would also read as a failure that was expected. `flaky` also means a -# fixed #897 is not flagged as a stale row, so delete the row by hand when #897 is +# fixed ROCMAI-898 is not flagged as a stale row, so delete the row by hand when +# ROCMAI-898 is # closed. [["serve-stop-leaves-no-background-helper"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade serving fails on the Strix Halo Ubuntu lanes, so the model never appears on /v1/models and the helper check is never reached." serve_timeout_secs = 90 flaky = true +# The dashboard journey does the same real lemonade serve as serve-lemonade-inference, +# so on a Linux lemonade host it is just as intermittent. The bounded serve timeout +# makes a stuck serve fail this one scenario instead of stalling the whole lane. +[["dash-lemonade-managed-inference-journey"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "The managed Lemonade serve on Linux is intermittent: it can reach ready and then shut down, or never become ready, so the journey cannot be relied on to complete." +serve_timeout_secs = 90 +flaky = true + # --- EAI-7223: chat requests with tool definitions rejected (vLLM path) --- [["chat-tool-definitions-accepted"]] when = { effective_engine = "vllm" } @@ -145,16 +208,11 @@ bug = "EAI-7223" reason = "Chat requests carrying tool definitions are not accepted by the served vLLM model." serve_timeout_secs = 90 -# On a lemonade LINUX host this needs a live lemonade serve, which reaches ready -# then shuts down immediately (EAI-7423). os=linux: passes on native Windows -# lemonade (run 29354305288 XPASS'd it), consistent with the other lemonade -# serve scenarios. therock_family=gfx* scopes this to a REAL GPU host: the -# scenario now also runs on the no-GPU mock lane (backed by MockServer, not a -# real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# Chat's mock lane must remain expect-pass: only a real GPU serve is affected. [["chat-tool-definitions-accepted"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -163,27 +221,16 @@ flaky = true # scenario exercises the CLI `rocm chat --prompt` path (which correctly discovers the # model via /v1/models). Verified expect-pass on MI300X vLLM (XPASS on run 29322186691, # then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat -# path works. Only the lemonade LINUX path stays xfail: it needs a live lemonade -# serve, which reaches ready then shuts down immediately (EAI-7423); native Windows -# lemonade serve works (run 29338475133 XPASS). therock_family=gfx* scopes this to -# a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not -# a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# path works. Only the real Lemonade Linux serve stays xfail under ROCMAI-898; +# the no-GPU MockServer path and native Windows remain expect-pass. [["chat-end-to-end-local-model"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# The GPU-backed bench scenario benchmarks a really served model, so on a -# lemonade LINUX host it inherits the same managed-serve bug as the sibling -# serve/chat rows below `serve-default-engine-working-endpoint`: the serve -# reaches ready then shuts down immediately, so this scenario's `a model is -# being served on GPU` precondition never yields a live endpoint. Scoped -# identically — os=linux only, because native Windows lemonade serve works, and -# therock_family=gfx* so it applies only on a real GPU host. `flaky = true` -# matches those rows too: which of them pass varies run to run on that lane, and -# a deterministic-XPASS row would fail the lane on unrelated PRs. +# Bench's real-serve precondition is blocked by ROCMAI-898 on the same Linux hosts. # # This does NOT weaken the coverage the bench fix relies on: the five # MockServer-backed bench scenarios (bench-01 to bench-05) run unxfail'd on @@ -191,7 +238,53 @@ flaky = true # failure reporting, the CSV columns, and the rollup-split warning. [["bench-load-real-serve"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true + +# --- Lemonade 11.5.1 cannot align its llama.cpp backend to a ROCm 7.14.x SDK --- +# `rocm engines install lemonade` tries to align Lemonade's `llamacpp:rocm` +# backend with the active ROCm SDK. Lemonade 11.5.1, the pin in +# `runtime-deps.toml`, downloads the TheRock tarball it pairs with the backend +# only from https://repo.amd.com/rocm/tarball/, a path the 7.14.x tarballs are not +# published to (AMD's 7.14 install guide uses /rocm/tarball-multi-arch/), so the +# download answers HTTP 403 for `therock-dist---7.14.1.tar.gz`. The +# pinned llama.cpp build has no ROCm 7.14 asset either (HTTP 404), so every tier +# of the alignment fails and the CLI reverts to Lemonade's pinned ROCm 7.13 +# backend. Reproduced on a Strix Halo Linux box with an isolated root, not only on +# the CI runners, so the failed alignment is not specific to Windows; the scenarios +# below only fail there (runtime-16 is `@requires-os:windows`, and serve-06 passed on +# the W7900 and MI300X lanes of the 2026-11-05 run). +# +# runtime-16 asserts the alignment succeeded, which it cannot while the 7.14.x +# tarball is unreachable. serve-06 serves a 35B model through the Windows lane's +# 7.13 backend against the SDK's 7.14.1 libraries and the server crashes in the +# SDK's Tensile library on the first inference request (access violation in the +# lane's service log). The mismatch between the backend and the SDK is the +# suspected cause; it has NOT been reproduced off Windows, so the row names both +# facts. Both rows are deterministic (both scenarios failed on each Windows lane +# run examined: 2026-10-28, 2026-11-05 and 2026-11-06), so neither is `flaky`: an +# XPASS means the cause is gone and the row should be deleted. Neither row sets +# `serve_timeout_secs`: serve-06's own `@serve-timeout:2400` tag takes precedence +# over a row, so a shorter wait here would have no effect, and the failing +# serve-06 still holds the Windows lane for the full 2400s wait. +[["runtime-lemonade-backend-alignment-reported"]] +when = { effective_engine = "lemonade", os = "windows" } +bug = "ROCMAI-899" +reason = "Lemonade 11.5.1 fetches TheRock only from repo.amd.com/rocm/tarball/, where 7.14.x is not published (HTTP 403), so its backend cannot be aligned to a 7.14.x SDK and the install reverts to the pinned 7.13 backend." + +[["serve-large-model-inference"]] +when = { effective_engine = "lemonade", os = "windows" } +bug = "ROCMAI-900" +reason = "The Windows large-model serve crashes in the SDK's Tensile library on the first inference request; the pinned ROCm 7.13 backend runs against the 7.14.1 SDK because Lemonade 11.5.1 cannot align it (suspected cause, not reproduced off Windows)." + +# Scoped to native Linux Strix Halo, the only place it has been seen (evidence is +# in the tracking issue); `is_wsl = false` keeps WSL2, which also reports +# `os = "linux"` on gfx1151, out of it, so a WSL host that gains Docker and passes +# the scenario is not an XPASS. A host without Docker skips the scenario before +# this row applies. +[["serve-aim-lifecycle-parity"]] +when = { os = "linux", is_wsl = false, therock_family = "gfx1151*", has_amd_gpu = true } +bug = "rocm-cli#1058" +reason = "Stopping a running AIM service on Strix Halo Ubuntu reports `Local server stop not confirmed` (exit 1); the serve itself works." diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index 3f83c17592..f882460da1 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -150,3 +150,38 @@ Feature: Chat and endpoint detection Then the planning assistant was asked exactly once And the plan text returned is the one the planning assistant gave And the action returned is the one the planning assistant gave + + # A `serve` the planning assistant proposes is held to the same four rules as + # one the chat assistant proposes: no CPU device, no public bind, loopback host + # only, and managed serving. The CLI does not show an unsafe plan from the + # assistant, it falls back to its own plan and says why, and nothing is run. + # The message is asserted with the plan shown, which is the behavior it + # describes: the assistant's serve names a model the CLI's own plan never + # would, so that model must not appear anywhere in the output. + @id:chat-planning-assistant-cpu-serve-refused + Scenario: chat-13 - A plan that serves on the CPU is refused, not shown + Given a local planning assistant that suggests serving a model with "--device cpu --managed" + When the user asks the CLI to plan a vague request + Then the planning assistant was asked exactly once + And the CLI plans the request itself and says the planning assistant's plan "cannot request CPU execution; ROCm GPU execution is required" + + @id:chat-planning-assistant-public-bind-serve-refused + Scenario: chat-14 - A plan that binds the server publicly is refused, not shown + Given a local planning assistant that suggests serving a model with "--allow-public-bind --managed" + When the user asks the CLI to plan a vague request + Then the planning assistant was asked exactly once + And the CLI plans the request itself and says the planning assistant's plan "cannot request public network binding" + + @id:chat-planning-assistant-non-local-host-serve-refused + Scenario: chat-15 - A plan that binds a non-local host is refused, not shown + Given a local planning assistant that suggests serving a model with "--host 0.0.0.0 --managed" + When the user asks the CLI to plan a vague request + Then the planning assistant was asked exactly once + And the CLI plans the request itself and says the planning assistant's plan "cannot request non-local host `0.0.0.0`" + + @id:chat-planning-assistant-unmanaged-serve-refused + Scenario: chat-16 - A plan that serves in the foreground is refused, not shown + Given a local planning assistant that suggests serving a model with "--foreground" + When the user asks the CLI to plan a vague request + Then the planning assistant was asked exactly once + And the CLI plans the request itself and says the planning assistant's plan "must request managed serving with --managed" diff --git a/tests/e2e-cucumber/features/comfyui.feature b/tests/e2e-cucumber/features/comfyui.feature index e6fb09bd54..18aa990947 100644 --- a/tests/e2e-cucumber/features/comfyui.feature +++ b/tests/e2e-cucumber/features/comfyui.feature @@ -333,3 +333,86 @@ Feature: ComfyUI install reports progress, makes failures actionable and preserv And ComfyUI's dependencies were installed into the runtime And the runtime's torch stack is unchanged And no nvidia-* packages were added to the runtime + + # With no runtime chosen, `rocm comfyui install` uses the one ready runtime, + # and it finds it the way `rocm serve` and `rocm engines install` do: it + # recovers the runtime `rocm setup` created first. A runtime whose registry + # entry was lost, but whose install tree still holds its own manifest and that + # setup names, therefore counts as the ready runtime, instead of the install + # refusing with "Install ROCm first" while setup has finished. The real run + # writes the lost registry entry back (comfyui-16); a `--dry-run` preview + # names the same runtime but leaves the registry as it found it, because it + # promises not to change files (this scenario). The preview prints the + # `--runtime-id` of the command it would run, which is how it shows the + # choice; comfyui-17 runs that command as printed. No GPU is needed: readiness + # is filesystem and manifest state. + @id:comfyui-install-finds-the-runtime-setup-created + Scenario: comfyui-15 - A ComfyUI install preview finds the runtime setup created without writing the registry + Given a ready ROCm runtime that setup created and the registry has forgotten + When the user previews a ComfyUI install without choosing a runtime + Then the preview selects that runtime + And the runtime registry still has no entry for it + + # The real run of the same install: it uses the runtime setup created and + # writes its lost registry entry back, so the runtime it installed into is one + # `rocm runtimes list` shows afterwards. The ComfyUI checkout is already in + # place and asks for nothing beyond the torch stack the runtime provides, so + # the install needs no network and no package index. + @id:comfyui-install-restores-the-runtime-setup-created + Scenario: comfyui-16 - A ComfyUI install restores the registry entry of the runtime setup created + Given a ready ROCm runtime that setup created and the registry has forgotten + And a ComfyUI checkout that needs nothing beyond the runtime's torch stack + When the user installs ComfyUI without choosing a runtime + Then the install uses the runtime setup created + And the runtime registry has an entry for it again + + # The preview's `install command:` line is how it shows the choice, so it has + # to be a command that works from the state the preview left, where the + # registry still has no entry for the runtime it named. Its `--runtime-id` is + # an explicit choice, which finds the runtime setup created the same way a + # missing choice does, instead of refusing with "Install ROCm first". Run as + # printed, the command installs into that runtime and writes the entry back. + @id:comfyui-install-preview-command-works-as-printed + Scenario: comfyui-17 - The install command a ComfyUI install preview prints works from the state the preview left + Given a ready ROCm runtime that setup created and the registry has forgotten + And a ComfyUI checkout that needs nothing beyond the runtime's torch stack + When the user previews a ComfyUI install without choosing a runtime + Then the runtime registry still has no entry for it + When the user runs the install command the preview printed + Then the install uses the runtime setup created + And the runtime registry has an entry for it again + + # The same printed command, when the registry cannot take the entry back (a + # file stands where its directory should be, which stops an administrator as + # well; a read-only directory would not). The preview writes nothing, so it + # still succeeds and names the runtime. The real run finds that runtime the + # same way, fails to restore its entry, and says so, rather than answering + # "Install ROCm first" as if setup had left no runtime. The registry is left as + # it was. A read-only registry directory with other runtimes in it, the other + # way to reach this, is covered by a unit test: the suite can run as root, for + # whom a read-only directory is writable. + @id:comfyui-install-reports-a-registry-it-cannot-write + Scenario: comfyui-18 - The install command a ComfyUI install preview prints reports a registry that cannot take the runtime back + Given a ready ROCm runtime that setup created and the registry has forgotten + And a ComfyUI checkout that needs nothing beyond the runtime's torch stack + And the runtime registry cannot be written + When the user previews a ComfyUI install without choosing a runtime + Then the preview selects that runtime + When the user runs the install command the preview printed + Then the install is refused because the runtime's registry entry could not be restored + And the runtime registry is as it was + + # With no runtime chosen, a setup folder whose runtime can no longer be used + # (here its interpreter is gone) is not a ready runtime and cannot change which + # runtimes are ready, so it does not stop an install that has exactly one ready + # runtime registered: the install uses that one, as a choice that names it + # does, and the unusable setup runtime is not written into the registry. Its + # own error is reported only when no registered runtime is ready (a unit + # test). No GPU is needed: readiness is filesystem and manifest state. + @id:comfyui-install-uses-the-ready-runtime-beside-an-unusable-setup-runtime + Scenario: comfyui-19 - A setup runtime that is no longer usable does not stop a ComfyUI install that has one ready runtime + Given a ready ROCm runtime in the registry + And a setup folder whose runtime is no longer usable + When the user previews a ComfyUI install without choosing a runtime + Then the preview selects the registered runtime + And the runtime registry holds only the registered runtime diff --git a/tests/e2e-cucumber/features/config.feature b/tests/e2e-cucumber/features/config.feature index abb937cf40..7d1799fa88 100644 --- a/tests/e2e-cucumber/features/config.feature +++ b/tests/e2e-cucumber/features/config.feature @@ -67,3 +67,50 @@ Feature: Configuration mutations When the user saves a provider API key Then the CLI reports it could not save the key securely And the key value never appears in the output + + # A settings file the CLI cannot read must not be mistaken for "no settings". + # It records where the user's runtimes live; guessing the default folder + # instead makes them disappear from every answer, and later writes land in + # the wrong place. The CLI stops, says which file, which folder it still + # names, and how to recover, and leaves the file alone so nothing more is + # lost. Two of the ways out it offers are then taken as printed — naming the + # data folder it suggests, and renaming the exact file it names to the name + # it gives — and what it says each one does is checked. Repairing the file is + # not exercised here. + @id:config-unreadable-settings-not-guessed + Scenario: config-09 - A settings file the CLI cannot read stops it instead of hiding the user's runtimes + Given the CLI's settings file was cut short just after it recorded a custom ROCm folder holding an installed runtime + When the user checks the CLI version without naming a data folder + Then the CLI refuses, naming the settings file, the recorded folder and how to recover + And the settings file is left exactly as it was + When the user opens the CLI with the data folder the refusal suggested + Then the CLI keeps its data in the recorded folder, on default settings + And the settings file is left exactly as it was + When the user changes a setting with the data folder the refusal suggested + Then the CLI refuses to change the setting, naming the settings file to rename + And the settings file is left exactly as it was + When the user renames the settings file as the refusal said and opens the CLI without naming a data folder + Then the CLI keeps its data in its default folder + When the user lists runtimes with the data folder the refusal suggested + Then the runtime installed in the recorded folder is listed + + # Saving settings replaces config.json in one step, but must not change what + # the user set up around it: a file only its owner may read stays that way, + # and a settings file that is a link (a dotfiles checkout, say) stays a link + # while the file it points at gets the new settings. These are Unix + # permissions and links, so the scenarios run on Linux only. + @id:config-save-keeps-a-private-settings-file-private @requires-os:linux + Scenario: config-10 - Changing a setting keeps a private settings file private + Given a settings file that only its owner can read + When the user turns telemetry off + Then the CLI confirms the telemetry mode and states the policy + And the settings file is still readable only by its owner + And the settings file records telemetry as off + + @id:config-save-keeps-a-linked-settings-file-linked @requires-os:linux + Scenario: config-11 - Changing a setting through a linked settings file keeps the link + Given a settings file that is a link to a file kept elsewhere + When the user turns telemetry off + Then the CLI confirms the telemetry mode and states the policy + And the settings file is still a link to that file + And the linked-to file records telemetry as off diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index fe3ed468ee..42b174c0a3 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -75,7 +75,13 @@ Feature: Interactive dashboard When the user opens the dashboard And the user opens the Observe view Then the managed model is displayed - When the user quits the dashboard + When the user opens the managed services overlay + Then the Services overlay lists the managed model with its port + When the user asks to stop the listed service + Then a stop approval prompt names the managed model and its port + When the user declines the approval prompt + And the user closes the managed services overlay + And the user quits the dashboard Then the dashboard exits successfully @@ -623,13 +629,16 @@ Feature: Interactive dashboard # placeholder shown before the first snapshot arrives at all, is not covered # here (the unit test `gpu_telemetry_pending_while_connected_but_no_snapshot_yet` # pins it): it closes in well under a second and there's no hook to freeze the daemon - # mid-handshake, so asserting on it would be asserting on a race. + # mid-handshake, so asserting on it would be asserting on a race. Nothing is + # served either, so the T/S row must read "—" too: an empty sum of rates is + # -0.0 in Rust, which used to render as "-0". @id:dash-reports-no-gpu-telemetry @requires-os:linux Scenario: dash-37 - The hero shows placeholders, not fake zeros, when no GPU is detected Given a machine with no GPU When the user opens the dashboard Then the dashboard home view is displayed And the hero reports no GPU telemetry available + And the hero reports no generation throughput When the user quits the dashboard Then the dashboard exits successfully @@ -867,3 +876,99 @@ Feature: Interactive dashboard Then the serve wizard is closed When the user quits the dashboard Then the dashboard exits successfully + + # #796: on an idle host the Observe CPU panel drew a few scattered dots + # instead of a graph. Any sample whose fill rounded below one braille dot row + # was dropped, which on an idle machine is nearly every sample. The widget + # tests pin the dot maths; this scenario pins what the user sees, since the + # regression was only ever visible as a picture. The recording reports 2% + # overall and 1% per core, the idle host the demo generator never produces: + # that generator oscillates around 28%, far above the rounding that caused + # the bug. Sixty entries two seconds apart outlast the scenario, the + # recording technique dash-39 introduced. + # + # Observe gives its CPU slot about four rows, too few for the separate + # per-core box, so the dashboard keeps the box carrying history and folds the + # core count into its title. Both halves of that are asserted here: the count + # survived the dropped box, and the idle trace is a line rather than a gap. + @id:dash-idle-cpu-graph-is-continuous @requires-os:linux + Scenario: dash-50 - The CPU graph stays a line when the host is idle + When the user opens the dashboard replaying an idle host + When the user opens the Observe view + Then the CPU graph is drawn as a continuous line + And the CPU box reports the host's core count + When the user quits the dashboard + Then the dashboard exits successfully + + # The other half of #796's layout change. dash-50 replays a 64-core host, + # which leaves the slot too narrow to split, so the per-core box is dropped + # and nothing there exercises the side-by-side layout. + # + # The core count decides this, not the terminal: the split wants two columns + # per core on each half, and widening does not help, because Observe grows + # side docks past a certain width and its centre column comes out *narrower* + # than at the detail geometry. So this replays a 16-core host at the same + # geometry dash-50 uses. + @id:dash-cpu-boxes-side-by-side @requires-os:linux + Scenario: dash-51 - A modest core count puts the CPU graph beside the per-core strip + When the user opens the dashboard replaying an idle 16-core host + When the user opens the Observe view + Then the CPU graph and the per-core strip sit side by side + When the user quits the dashboard + Then the dashboard exits successfully + + @id:dash-lemonade-managed-inference-journey @requires-os:linux @requires-gpu @requires-engine:lemonade + Scenario: dash-52 - A real managed Lemonade GPU model responds to inference and appears in the dashboard + Given a managed runtime is active + And the dashboard watches only managed services + And a GGUF model is being served on lemonade + Then the model responds to inference requests + When the user opens the dashboard + And the user opens the Observe view + Then the real managed Lemonade model is displayed + When the user opens instance detail + Then the managed model's instance detail shows it ready on the port it was served on + When the user closes instance detail + And the user selects engine status in the Serving view + Then Lemonade is shown as a managed runtime installation + When the user quits the dashboard + Then the dashboard exits successfully + And the model responds to inference requests + + @id:dash-lemonade-external-protocol-discovery @requires-os:linux + Scenario: dash-53 - External Lemonade protocol discovery displays every loaded model without adopting the service + Given an external Lemonade server has two models loaded + When the user opens the dashboard + And the user opens the Observe view + Then both loaded models are displayed with throughput shown as unknown + When the user opens the managed services overlay + Then the Services overlay has no managed servers + When the user attempts to stop and restart the external models + Then the Services overlay closes without an approval prompt or job + And both loaded models are still displayed + And the external server received no lifecycle request and no managed record exists + When the user selects engine status in the Serving view + Then Lemonade is shown as externally running without a managed installation + When the user quits the dashboard + Then the dashboard exits successfully + And the external server received no lifecycle request and no managed record exists + + @id:dash-lemonade-external-protocol-transitions @requires-os:linux + Scenario: dash-54 - External Lemonade protocol models disappear on unload, return on reload, and disappear again when the server stops responding + Given an external Lemonade server has two models loaded + When the user opens the dashboard + And the user opens the Observe view + Then both loaded models are displayed with throughput shown as unknown + When the external Lemonade server unloads its second model + Then only the first model is displayed, with its throughput + When the external Lemonade server unloads every model + Then no loaded model is displayed and none are reported as discovered + When the external Lemonade server loads both models again + Then both loaded models are displayed with throughput shown as unknown + And the external server received no lifecycle request and no managed record exists + When the external Lemonade server stops responding + Then no loaded model is displayed and none are reported as discovered + When the user selects engine status in the Serving view + Then Lemonade returns to the managed installation status without external availability + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 0a690d3439..776e69a01e 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -304,7 +304,8 @@ Feature: Diagnosing failures and listing fixes And the preview states that the CLI can run it automatically # `--model` answers the question that comes before the other two: given this # machine and that model, will it run. The point is that it answers in seconds - # and fetches nothing, so the user is not told by a download that failed. + # and downloads no weights (a curated model fetches nothing at all), so the user + # is not told by a download that failed. # # Host-agnostic in the same way diagnose-10 is, and for the same reason. The # verdict depends on what this machine can measure of its own GPU, which @@ -521,11 +522,20 @@ Feature: Diagnosing failures and listing fixes # already knows, and a copy can go stale silently. This is the machine-readable # form that replaces the copying — what a tool reads instead of parsing the # listing meant for people. + # + # A meaning is only worth publishing if it is true, so the last two steps run + # the CLI and hold the published text to what it does. Exit 2 is the one code + # with two causes: a fix-id the catalog does not have, and a command line the + # argument parser rejects before any fix-id is looked up. They share the code + # on purpose (a code of its own would change what existing readers of 2 + # receive), so the catalog has to say so and say how to tell them apart. @id:diagnose-fix-catalog-is-machine-readable - Scenario: diagnose-34 - A tool can read the catalog without parsing prose + Scenario: diagnose-34 - A tool can read the catalog without parsing prose, and the shared exit code 2 can be told apart When a tool asks the CLI for its catalog in machine-readable form Then the catalog names every entry and what the CLI does with each And it gives the meaning of every exit code the CLI can return + And asking for a fix it does not offer exits with the code the catalog describes for that + And a malformed command line exits with that same code, as the catalog warns, and is told apart by its message # `--json` describes the whole catalog, so pairing it with one entry is a # question with no answer. Refusing beats ignoring the flag: a caller that @@ -748,3 +758,241 @@ Feature: Diagnosing failures and listing fixes Given a machine with an AMD GPU When the user diagnoses this machine in machine-readable form, with no symptom Then the diagnosis lists the HIP SDK cause only when the inspection finds something that needs it + + # A GGUF outside the catalog has no recipe, so there is no tested minimum to + # compare. Its header still says how many layers it has, how many KV heads and + # how wide, and the file's size is the weights: enough to estimate memory at a + # chosen context size. (That a remote file is read for its header alone is + # asserted by the gguf_source unit test that serves an 8 GiB file and records + # every range; a local file is only opened, so no step here could see a + # download.) The answer must stay + # visibly an estimate -- a different outcome from the closed verdict, and a + # different word -- because a figure derived from metadata and allowances + # calibrated on one GPU is not a model that was loaded. The file is a few kilobytes of header and a + # tiny weight region, so the scenario is deterministic on every lane; the + # comparison with GPU memory is host-specific and asserted per half, as + # diagnose-21 and diagnose-22 do. + @id:diagnose-model-gguf-is-estimated-from-its-header-not-judged + Scenario: diagnose-49 - A GGUF outside the catalog is sized from its header and reported as an estimate, not a verdict + Given a user asking about a small GGUF file on this machine + When the user asks the CLI whether that file would run at 2048 tokens with a q8_0 KV cache, in machine-readable form + Then the answer is an estimate under its own outcome, and the verdict stays undetermined + And the estimate sizes the file's weights and its KV cache at that context + And a machine that measured its GPU is told whether the estimate fits, and one that could not is told why + + # The human-readable half of diagnose-49: the label is what a reader sees + # first, and READY or BLOCKED over a figure derived from metadata would claim + # a certainty the estimate does not have. + @id:diagnose-model-gguf-estimate-is-labelled-in-the-text-report + Scenario: diagnose-50 - The text report labels a GGUF estimate as an estimate and never calls the file ready or blocked + Given a user asking about a small GGUF file on this machine + When the user asks the CLI whether that file would run at 2048 tokens with a q8_0 KV cache, in human-readable form + Then the report is labelled an estimate, or undetermined with the figures, and is never ready or blocked + + # `--report` promises that nothing leaves the machine. A GGUF reference such as + # owner/repo:Q4_K_M is read from Hugging Face, which would put the model's name + # on the wire, so a prepared report must not assess the model at all. The + # second half proves the endpoint is wired in: without `--report` the same + # question does reach it, and the note about ignoring the model is not shown. + @id:diagnose-report-never-reaches-the-hub-for-a-model + Scenario: diagnose-51 - A prepared problem report never asks the hub about a model + Given a Hugging Face endpoint that records every request made to it + When the user asks the CLI to prepare a problem report that names a model on that endpoint + Then no request reached the endpoint + And the CLI says the model was ignored rather than assessed + When the user asks the CLI the same model question without preparing a report + Then the endpoint was asked about the model + And the CLI does not say the model was ignored + + # A llama.cpp vision model ships its multimodal projector (mmproj) beside the + # model. Naming the projector exactly is answered for what it is, with the + # model file named instead, rather than as a file the repository lacks; and + # it is refused from the listing alone, before any header is read. + @id:diagnose-a-projector-named-exactly-is-answered-as-a-projector + Scenario: diagnose-52 - Naming a model's multimodal projector says what it is and names the model + Given a Hugging Face repository that holds a model and its multimodal projector + When the user asks the CLI whether the projector file would run, naming it exactly + Then the CLI says the file is a multimodal projector and names the model file instead + And the endpoint was asked only for the repository's file list + + # The context options describe the model being asked about. Without --model + # they would be accepted and do nothing, so the CLI refuses them as a usage + # error before anything runs. + @id:diagnose-context-options-need-a-model + Scenario: diagnose-53 - The context options are refused without a model to apply them to + When the user asks the CLI for a context size or a KV cache type without naming a model + Then the CLI refuses each as a usage error that names --model + + # A curated model's memory minimum comes from its recipe and is fixed, so a + # context size cannot change the answer. The report says so instead of + # silently ignoring the option, and makes no estimate. + @id:diagnose-context-options-do-not-apply-to-a-curated-model + Scenario: diagnose-54 - A curated model is told the context options do not apply to it + When the user asks the CLI whether the smallest curated model would run at 8192 tokens, in machine-readable form + Then the answer says the context options do not apply to a curated recipe, and carries no estimate + + # A .gguf path that does not exist is a path to check, not a model to report + # upstream: the answer names the path, gives no upstream route, and its plan + # is to ask again once the path is right. + @id:diagnose-a-missing-gguf-path-is-a-path-to-check + Scenario: diagnose-55 - A GGUF path that does not exist is answered as a path to check, not a model to report + When the user asks the CLI whether a GGUF file that does not exist would run, in machine-readable form + Then the answer says the file could not be opened and offers no upstream route + And the plan is to ask again about that path + + # owner/repo with no :QUANT names a repository, and every file in it is a + # different model, so nothing is read; with a context size the user is told + # what would make it answerable. The stand-in hub shows no request was made. + @id:diagnose-a-bare-repository-with-a-context-size-asks-for-a-quantization + Scenario: diagnose-56 - A bare repository with a context size is told to name a quantization, and nothing is fetched + Given a Hugging Face endpoint that records every request made to it + When the user asks the CLI about a repository with no quantization at 8192 tokens, in machine-readable form + Then the answer says the reference names a repository and asks for a quantization + And no request reached the endpoint + + # A quantization that matches nothing lists what the repository does hold, + # and a multimodal projector is among the files listed, marked as one, rather + # than left out: a user looking at the hub sees it there too. + @id:diagnose-a-quantization-that-matches-nothing-lists-every-gguf + Scenario: diagnose-57 - A quantization that matches nothing lists the repository's files, projectors marked + Given a Hugging Face repository that holds a model and its multimodal projector + When the user asks the CLI about a quantization the repository does not hold + Then the CLI says nothing matches and lists the model and the projector, marked as a projector + + # A report names the CLI that wrote it with the same leading version + # `rocm version` prints, and only that one token: the ref and hash after it + # can carry a local branch name. The step also checks that version against + # the one `rocm version`'s ref calls for (`X.Y.Z` for a `vX.Y.Z` tag, + # `0.0.0-dev` for any other ref), so both surfaces regressing to the same + # frozen workspace version would fail too. A CI lane resolves a branch, never + # a tag, so only the `0.0.0-dev` half runs here; the tag half is checked on the + # release binaries by release.yml's "Check the release binary's version" steps + # and by the `format_cli_version_*` unit tests in apps/rocm/build.rs. The host + # is simulated as in diagnose-40, with a released GPU instead, so a report is + # prepared on every Linux lane rather than only on one whose own GPU is on the + # matrix. + @id:diagnose-report-names-the-cli-version @requires-os:linux + Scenario: diagnose-58 - A problem report names the CLI version that rocm version prints + Given a simulated Linux machine whose only GPU is a released one + When the user asks the CLI what that machine's report would carry in machine-readable form + Then the report names the CLI version that `rocm version` leads with + + # A pasted error is hostile to every shell's argument parser: bash and + # PowerShell run a `$(...)` even inside double quotes, a stray quote ends the + # argument early, cmd.exe has no single quotes at all, and an error that + # starts with `-` is read as a flag. `--symptom-file -` takes the paste on + # standard input instead, so the shell only ever parses the fixed command. + # Each platform's own shells do the piping -- bash on Linux and WSL, cmd.exe + # and Windows PowerShell on Windows -- and each result is compared with the + # same text handed over as one argument with no shell involved. The paste + # matches a catalog entry that scores on every platform, so the comparison is + # between two real diagnoses rather than two empty ones; which entry it is, + # is not asserted (see the note at the top of this file). Nothing in the + # paste can run or be read as a flag: each shell only reads it from a file + # (cat, type, Get-Content) into the pipe and never parses it. That holds by + # how the paste is handed over, so it is stated here rather than asserted. + @id:diagnose-a-piped-paste-is-diagnosed-as-pasted + Scenario: diagnose-59 - A pasted error piped to the diagnosis is diagnosed the same as if it had been passed safely + Given a pasted error containing quotes, a command substitution and a leading minus sign + When the user pipes it to the diagnosis instead of passing it as an argument + Then the diagnosis matches the same failure modes as the same text passed safely as an argument + + # Typing `rocm diagnose --symptom-file -` with nothing piped in leaves the CLI + # waiting on the terminal. It says so, and how to end the paste; the user + # pastes, ends it the way they were told, and the terminal then contains the + # whole report the same text passed as an argument gets, below the hint and + # the echoed paste. The step first requires that report to differ from the + # one without any symptom, so a paste that was read and then dropped cannot + # pass. + # @requires-os:linux like every other pseudo-terminal scenario in this suite: + # on Windows the step hangs under the harness's pseudo-console before the + # paste is diagnosed (not investigated further; the console is started with + # cursor inheritance, which waits on a reply the screen emulator does not + # send). That leaves the Windows half of the hint (Ctrl-Z then Enter) without + # an e2e check; docs/manual-testing.md section 13 covers it by hand. + @id:diagnose-a-paste-at-the-terminal-is-explained-and-diagnosed @requires-os:linux + Scenario: diagnose-60 - Pasting the error at the terminal is explained, and the paste is diagnosed when the user ends it + Given a pasted error containing quotes, a command substitution and a leading minus sign + When the user asks to paste the error at the terminal, pastes it and ends it as told + Then the CLI told the user how to end the paste + And the terminal shows the whole report the same text passed safely as an argument gets + + @id:diagnose-an-unreadable-error-file-is-a-usage-error + Scenario: diagnose-61 - An error file that cannot be read, or is not text, is refused as a usage error + Given an error file that does not exist and one that is not text + When the user asks the CLI to diagnose the error in each file + Then each is refused as a usage error that names the file and says why + And nothing is diagnosed + + # A diagnosis scores the catalog from two sources at once: what the examined + # host shows, and the error the user pasted. A finding about the host can + # outrank the entry the paste added to, so the ranking alone does not say + # which entries the paste contributed to, and a tool reading the result needs + # that said per entry. "Contributed" is all it says: any keyword hit counts, + # however weak, so it is not a verdict that the entry explains the error. The + # misconfiguration is an unneeded GPU override in the environment the CLI + # examines, which every lane reads the same way, so this holds on Linux, + # Windows and WSL alike. The paste is an ISA error, which no host state scores + # on its own and which names nothing the override entry reads, so the two + # answers cannot depend on the runner. + @id:diagnose-json-says-which-causes-the-paste-contributed-to + Scenario: diagnose-62 - A tool can tell which causes the pasted error contributed to + Given a machine with a known misconfiguration in its environment + And a pasted error that describes a different failure + When the user asks for a diagnosis with that paste, in machine-readable form + Then the paste is reported as not contributing to the misconfiguration on the machine + And the paste is reported as contributing to the cause it describes + + # The same answer, for a person. Only the cause the paste did not contribute + # to is marked, and the mark appears only because the paste contributed to + # something else in the same report. + @id:diagnose-text-marks-causes-the-paste-did-not-contribute-to + Scenario: diagnose-63 - The human-readable diagnosis marks a cause the pasted error did not contribute to + Given a machine with a known misconfiguration in its environment + And a pasted error that describes a different failure + When the user asks for a diagnosis with that paste + Then the misconfiguration on the machine is marked as found from the host alone + And the cause the pasted error describes carries no such mark + + # With nothing pasted, nothing in a paste contributed to anything. A tool is + # still told so on every entry rather than the answer being left out, because + # a missing answer is what a CLI too old to give one prints. A person is told + # nothing: the mark would sit on every cause and say nothing. The host is the + # one diagnose-62 uses, so at least one cause is listed on every lane, and + # both forms are asserted against it. + @id:diagnose-without-a-paste-credits-no-cause-to-one + Scenario: diagnose-64 - Without a pasted error no cause is credited to one, and none is marked + Given a machine with a known misconfiguration in its environment + When the user asks for a diagnosis without pasting an error, in machine-readable form + Then every listed cause is reported as not contributed to by a paste + When the user asks for a diagnosis without pasting an error + Then no listed cause is marked as found from the host alone + + # fix-4's verify line was kept in two copies, and only the one `rocm diagnose` + # printed listed `/dev/kfd`, so `rocm fix` told the user to check less than the + # diagnosis did. Both now print one line. Like diagnose-43 to -45 this sees + # `rocm fix --dry-run`, which prints the plan before the platform gate, so + # it runs on every lane. That `rocm diagnose` prints the same line is held by a + # unit test, because the diagnosis only fires on a Linux host. + @id:diagnose-render-group-verify-line-shows-who-owns-the-gpu-device + Scenario: diagnose-65 - The verify line printed for the render-group fix shows who owns the GPU compute device + Given a user who has chosen the fix for a missing render group + When the user previews that fix without applying it + Then the printed verify line checks the user belongs to the group that owns the GPU compute device, and shows that group + + # fix-8 compares the framework's ROCm version with the system's at X.Y, the + # granularity PyTorch's wheel indexes are published at, but its titles said + # "major". Text only; the comparison itself is unit tested. + @id:diagnose-wheel-mismatch-fix-names-the-release-it-compares + Scenario: diagnose-66 - The fix for a mismatched framework wheel names the part of the version it compares + Given a user who has chosen the fix for a framework wheel built for another ROCm release + When the user previews that fix without applying it + Then the printed title names the X.Y release, not the major version + + # diagnose-66 sees the `rocm fix` plan; this is the other copy, the cause that + # `rocm diagnose` prints. The library-load error in the report raises it on + # its own, so it is listed on every lane whatever ROCm the runner has. + @id:diagnose-wheel-mismatch-cause-names-the-release-it-compares + Scenario: diagnose-67 - Diagnosing a failed load of the ROCm runtime library names the release the framework and the system must share + When the user diagnoses this machine with a ROCm runtime library load failure + Then the wheel mismatch cause and its plan name the X.Y release, not the major version diff --git a/tests/e2e-cucumber/features/download_progress_pty.feature b/tests/e2e-cucumber/features/download_progress_pty.feature index bab70dcc35..2c6e6c7b98 100644 --- a/tests/e2e-cucumber/features/download_progress_pty.feature +++ b/tests/e2e-cucumber/features/download_progress_pty.feature @@ -17,8 +17,8 @@ Feature: Download-progress spinner under a real terminal # this scenario alone. Serial execution removes that contention. @id:download-progress-linux-tarball-install-shows-live-progress @requires-os:linux @serial Scenario: download-progress-01 - The tarball download spinner renders progress and clears on completion - Given a paced canonical release tarball fixture - When the user installs the tarball SDK for family gfx120X-all under a real terminal + Given a paced next-layout release tarball fixture + When the user installs the tarball SDK for arch gfx1200 under a real terminal Then the terminal shows an intermediate download progress frame And the terminal shows the archive being extracted And the tarball install exits cleanly diff --git a/tests/e2e-cucumber/features/engine_ipc.feature b/tests/e2e-cucumber/features/engine_ipc.feature new file mode 100644 index 0000000000..96eb773636 --- /dev/null +++ b/tests/e2e-cucumber/features/engine_ipc.feature @@ -0,0 +1,47 @@ +Feature: How the daemon reads an engine's answer + + # `rocmd` asks an engine to do things (here: stop a server) by running + # `rocm __engine-stdio ` and reading what the child prints. The + # standalone `rocmd` binary has no such verb of its own and runs the `rocm` + # beside it, which is what makes these scenarios stageable from outside: each + # one copies the real `rocmd` into a scratch directory and puts a stand-in + # `rocm` next to it, so the daemon's own tool reaches a child whose answer the + # scenario controls. The stand-in records the argv and the request it was + # given and prints what the scenario says. + # + # service-stop-02 covers the engine that answers cleanly; the two below are the + # other ways an engine child ends. In each, the engine's message in the tool's + # report is checked together with what the stop did to the server: its process, + # its record and its endpoint key. The server is a `vllm` one, for which the + # engine's `Stop` is advisory (the recorded process decides whether the stop + # is confirmed), so the stop going through is part of what is shown, not a + # side effect. + # + # Linux-only, for the same reason as service-stop-02: the recorded process + # carries the start-time token `/proc` gives it, and the stand-in is a shell + # script. No GPU and no real engine are involved, so these run on the mock lane. + + @id:engine-ipc-reads-an-answer-printed-before-a-failing-exit + @requires-os:linux + Scenario: engine-ipc-01 - An engine that answers the stop and then exits non-zero is read as having answered it + Given a managed server whose recorded process is running + And the engine answers the stop request and then exits with a failing status + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer to the stop, not the status it exited with + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-reports-an-unreadable-answer-with-the-engines-stderr + @requires-os:linux + Scenario: engine-ipc-02 - An engine that prints something unreadable and fails reports its stderr, and the stop still completes + Given a managed server whose recorded process is running + And the engine prints something that is not an answer, complains on stderr and exits with a failing status + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the stop request failed, with the engine's exit status and stderr + And the tool still reports the server stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 69873fa6b6..9e0dc5af00 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -1,5 +1,8 @@ Feature: GPU detection and system inspection + # A release tag `vX.Y.Z` is reported as `X.Y.Z (vX.Y.Z, )`. Any other + # resolved ref, including `main`, is reported as `0.0.0-dev` rather than the + # unpublished workspace package version. The step checks that shape. @id:examine-version Scenario: examine-01 - The CLI reports its version When the user asks for the version through every CLI surface @@ -287,9 +290,9 @@ Feature: GPU detection and system inspection # machine other than the runner. The driver plan cannot be exercised this way: # `rocm install driver` builds a plan that runs against the real machine, so it # deliberately reads the real `/etc/os-release` (`real_os_release` in - # `system_packages.rs`, whose test there pins that it ignores a simulated - # host root); the plan built from it is covered by the unit tests in - # `driver_plan.rs`. + # `hardware_root.rs`, whose test in `system_packages.rs` pins that it ignores + # a simulated host root); the plan built from it is covered by the unit tests + # in `driver_plan.rs`. # # The planted distributions are ones no CI runner or developer host here runs, # so a probe that read the real file instead would report a different one. @@ -474,7 +477,7 @@ Feature: GPU detection and system inspection # The other side of examine-24/25/26, and the user-observable half of the # `/dev/kfd` pre-flight (EAI-8457): where those three prove a healthy device # still reaches amd-smi, this proves an unopenable one stops the launch from - # happening at all. (Declared here at the end rather than beside them because + # happening at all. (Declared after examine-30 rather than beside them because # the scenario index is per-feature and in declaration order.) On a host in # that state amd-smi can park in # uninterruptible sleep, where no signal and therefore no timeout reaches it, @@ -548,3 +551,184 @@ Feature: GPU detection and system inspection Then the inspection reports the ROCDXG driver status "wsl_rocdxg_ready" And the inspection reports the ROCDXG library as beside the runtime and in neither the linker cache nor LD_LIBRARY_PATH And the inspection's driver detail does not send the user to register the library with ldconfig + + # `rocm examine --json` asks twice whether `rocminfo` is installed on WSL: the + # host summary (`summary.wsl.rocminfo`) and the examination (`wsl.rocminfo`). + # They used to use two different PATH lookups and could disagree; they now use + # one, and these scenarios hold the two answers level on the cases where + # the old lookups differed. The planted WSL machine is the one `examine-32` uses + # (a simulated host root with `/dev/dxg`); the runner's own PATH entries that + # hold a `rocminfo` are left out, so a runner with ROCm installed cannot be the + # reason a tool reads as found. The stand-in `rocminfo` prints nothing, so the + # scenarios are about whether the tool is found, not whether it sees a GPU. + # + # The control: a `rocminfo` in a PATH directory is found by both. + @id:examine-rocminfo-on-path-is-found-by-both-reports @requires-os:linux + Scenario: examine-35 - Both reports find a rocminfo that sits in a PATH directory + Given a WSL machine whose rocminfo is in a PATH directory + When the user inspects that machine in machine-readable form + Then both reports say rocminfo is installed + + # An empty PATH entry is not the current directory for this lookup: a file named + # `rocminfo` in the directory the command runs from is not "installed". The + # summary's old lookup read the entry as the current directory and said it was, + # while the examination's said it was not. The cost is that a program the CLI + # starts by bare name can still resolve from the current directory through the + # operating system, which the lookup does not follow. + @id:examine-rocminfo-in-working-directory-is-not-installed @requires-os:linux + Scenario: examine-36 - Neither report reads an empty PATH entry as the working directory + Given a WSL machine whose only rocminfo is in the directory the command runs from, with an empty entry on PATH + When the user inspects that machine in machine-readable form + Then both reports say rocminfo is not installed + + # A PATH that is not valid Unicode is searched. The examination's old lookup read + # PATH as a `String`, which fails on it, and so found no tool at all while the + # summary's old lookup still searched it. + @id:examine-rocminfo-on-a-path-that-is-not-unicode-is-found @requires-os:linux + Scenario: examine-37 - Both reports search a PATH that is not valid Unicode + Given a WSL machine whose rocminfo sits in a PATH directory with a name that is not valid Unicode + When the user inspects that machine in machine-readable form + Then both reports say rocminfo is installed + + # The control for `examine-36`: the same machine, with `.` as the first PATH + # entry instead of an empty one. A `.` entry is an ordinary directory, so the + # lookup finds the stand-in in the working directory. That only happens if the + # `rocm` child really runs from it, so this scenario fails when the harness does + # not apply the working directory, and `examine-36`'s "not installed" is then + # meaningful: the two differ in nothing but the entry. + @id:examine-rocminfo-in-working-directory-named-on-path-is-found @requires-os:linux + Scenario: examine-38 - Both reports find a rocminfo in the working directory when PATH names it + Given a WSL machine whose only rocminfo is in the directory the command runs from, with that directory named on PATH + When the user inspects that machine in machine-readable form + Then both reports say rocminfo is installed + + # kfd is linked into `amdgpu` (`HSA_AMD` is a `bool` Kconfig option), so a + # healthy machine never lists an `amdkfd` module. The report used to read + # that as "kfd missing". Each scenario plants a module list naming `amdgpu` + # only, so the answer can come only from the kernel's kfd interfaces. + @id:examine-reports-kfd-built-into-the-kernel @requires-os:linux + Scenario: examine-39 - The inspection reports kfd on a machine whose module list has no amdkfd + Given a machine whose kernel exposes kfd but lists no amdkfd module + When the user inspects that machine in machine-readable form + Then the inspection reports kfd as loaded + + # A negative control for examine-39, not proof of the fix: it passes with + # the fix reverted too, and fails if the kfd check reports true regardless. + @id:examine-reports-kfd-absent-without-kfd-interfaces @requires-os:linux + Scenario: examine-40 - The inspection reports kfd as not loaded on a machine without it + Given a machine whose kernel has no kfd and lists no amdkfd module + When the user inspects that machine in machine-readable form + Then the inspection reports kfd as not loaded + + # amdgpu built into the kernel is absent from `lsmod` too, so it was read as + # "module not loaded" and `rocm diagnose` scored that, plus a DKMS failure + # and Secure Boot, as evidence. `/sys/bus/pci/drivers/amdgpu` exists for a loaded + # module and a built-in driver alike. The machine also has a ROCm install + # and Secure Boot enabled, so the evidence lines are reachable if the flag + # regresses. + @id:examine-reports-amdgpu-built-into-the-kernel @requires-os:linux + Scenario: examine-41 - A built-in amdgpu is reported as loaded and diagnose finds no driver fault + Given a machine whose kernel has amdgpu built in and lists no amdgpu module + When the user inspects and diagnoses that machine + Then the inspection reports amdgpu as loaded + And the diagnosis cites no missing amdgpu module, DKMS failure or Secure Boot evidence + + # With no module list at all (`lsmod` fails and `/proc/modules` is empty), + # the kernel's kfd interfaces still answer: kfd is linked into amdgpu, so + # `/dev/kfd` reports both drivers as loaded rather than leaving them unknown. + @id:examine-reports-drivers-from-kfd-without-a-module-list @requires-os:linux + Scenario: examine-42 - The inspection reports kfd and amdgpu from the kernel's kfd node when no module list can be read + Given a machine whose kernel exposes kfd but whose module list cannot be read + When the user inspects that machine in machine-readable form + Then the inspection reports kfd as loaded + And the inspection reports amdgpu as loaded + + # `rocm examine --json` reads the kernel release twice: the host summary + # (`summary.kernel`, which the text form prints as `kernel:`) and the examination + # (`kernel_release`, which the WSL version check and the kernel-dependent + # diagnoses read). They used to run `uname -r` separately, with different limits + # and different rules for a failed or malformed answer, and could disagree. They + # share one read now, so one `--json` run holds the two level on a specific + # value, not only on both being filled, and the stand-in counts its calls to show + # that the run asked for the release once: the summary takes the examination's + # read, so a transient failure cannot make the two fields disagree and a hung + # `uname` costs one timeout. + # + # The stand-in `uname` is planted first on PATH, so the scenario describes a + # kernel no runner has. It runs on every Linux lane, native or WSL, because both + # stacks read `uname -r` on both; Windows is out of scope, where the examination + # leaves `kernel_release` empty and the summary reads `cmd /C ver`. + @id:examine-reports-one-kernel-release-in-both-forms @requires-os:linux + Scenario: examine-43 - The summary and the examination report the same kernel release + Given a machine whose uname reports a kernel release + When the user inspects that machine in machine-readable form + Then the summary and the examination report that kernel release + And that run asked uname for the kernel release once + + # A `uname` that fails is not trusted for what it printed first: the examination + # used to keep that release while the summary dropped it. Neither reports a + # kernel now, and a failed read is not a probe failure, so a host without a + # usable `uname` does not read as degraded. + @id:examine-reports-no-kernel-release-from-a-failing-uname @requires-os:linux + Scenario: examine-44 - Neither report keeps a kernel release from a uname that failed + Given a machine whose uname prints a kernel release and then fails + When the user inspects that machine in machine-readable form + Then neither the summary nor the examination reports a kernel release, and neither calls it a failure + + # `rocm examine --json` reads the CPU twice: the host summary (`summary.cpu`, + # which the text form prints as `cpu:`) and the examination (`cpu_model` and + # `cpu_vendor`). They used to read `/proc/cpuinfo` separately and spelled the + # same model differently; they share one reader now, so one `--json` run holds + # the two level on a specific value, not only on both being filled. + # + # The planted machine is a simulated host root holding only a `/proc/cpuinfo`, + # so the scenario describes a CPU no runner has, and it is not WSL whatever the + # runner is: the comparison is scoped to hosts where `is_wsl` is false, because + # on WSL the examination does not read the CPU at all (`examine-47`). The model + # in `/proc/cpuinfo` has repeated spaces, which the examination used to keep and + # the summary to collapse. Native Windows reads the same CPU through the system + # inventory and cannot run on this lane; the unit tests in + # `crates/rocm-core/src/host_gpu/os_facts.rs` cover it. + @id:examine-reports-one-cpu-in-both-forms @requires-os:linux + Scenario: examine-45 - The summary and the examination report the same CPU model and its vendor + Given a machine that is not WSL whose cpuinfo names an AMD CPU with repeated spaces in its model + When the user inspects that machine in machine-readable form + Then the summary and the examination report that CPU model, and the examination reports its vendor as amd + + # A board whose cpuinfo has a `Hardware` line and no `model name` (an Arm board) + # showed a CPU in the summary and none in the examination. Both read the + # `Hardware` line now, and the vendor stays unknown because such a file names + # none. + @id:examine-reports-a-hardware-line-cpu-in-both-forms @requires-os:linux + Scenario: examine-46 - Both reports name a CPU that cpuinfo gives only as Hardware + Given a machine that is not WSL whose cpuinfo has a Hardware line and no model name + When the user inspects that machine in machine-readable form + Then the summary and the examination report that CPU model, and the examination reports its vendor as unknown + + # The difference this change leaves as it is: on WSL the examination returns + # before it reads the CPU, so `cpu_model` is empty and `cpu_vendor` is + # `unknown` while the summary shows the CPU. Pinned so that collecting CPU + # facts on WSL is a visible change of its own, which would turn this scenario. + # The other two scenarios compare the two reports on hosts that are not WSL. + @id:examine-reports-no-cpu-model-on-wsl-where-the-summary-has-one @requires-os:linux + Scenario: examine-47 - On WSL the summary reports the CPU and the examination does not + Given a WSL machine whose cpuinfo names an AMD CPU + When the user inspects that machine in machine-readable form + Then the summary reports that CPU model, and the examination reports no CPU model and an unknown vendor + + # The host probes' subprocess helpers used to read a tool's output until every + # holder of the pipe had closed it, so a tool that left a background process + # behind (a helper the tool starts and does not wait for) held the whole + # inspection until that process exited, however quickly the tool itself did. + # They now stop reading shortly after the tool exits, and the inspection keeps + # what the tool printed before that. The stand-in `lspci` prints one AMD GPU, + # exits 0 and leaves a `sleep` holding its output for 30 seconds; the + # inspection must finish well inside that and still list the GPU, so what the + # tool printed before it exited is still collected. It does not pin the length + # of the grace: that is the unit tests' (`subprocess_runners_pair`). + @id:examine-lspci-leaving-a-background-process-does-not-hold-up-the-inspection @requires-os:linux + Scenario: examine-48 - A PCI listing tool that leaves a background process behind does not hold up the inspection + Given a machine whose lspci prints a GPU and leaves a background process holding its output open + When the user inspects that machine in machine-readable form and the time it takes is measured + Then the inspection finishes long before the background process does + And the inspection lists the GPU that lspci printed diff --git a/tests/e2e-cucumber/features/install_lifecycle.feature b/tests/e2e-cucumber/features/install_lifecycle.feature index 9718433cf1..025484b852 100644 --- a/tests/e2e-cucumber/features/install_lifecycle.feature +++ b/tests/e2e-cucumber/features/install_lifecycle.feature @@ -309,9 +309,11 @@ Feature: Release install lifecycle And the record of the installer's PATH change is gone @id:lifecycle-linux-uninstall-keep-binaries-keeps-path @lifecycle @requires-os:linux - Scenario: lifecycle-25 - Linux - uninstall that keeps the binaries keeps the shell PATH change + Scenario: lifecycle-25 - Linux - uninstall that keeps the binaries keeps the shell PATH change and the fish completions Given a freshly built release tree And a generated signing keypair + And the login shell is bash + And fish is set up with its own settings And a signed bundle installed with the shell profile updated And the installed binary has isolated XDG directories with state When the user uninstalls from the installed binary keeping the binaries @@ -319,6 +321,7 @@ Feature: Release install lifecycle And uninstall reports keeping the PATH change with the binaries And the installed rocm and rocmd binaries are present And the shell profile still adds the install directory to PATH + And the fish completions are still in place And the record of the installer's PATH change is still in the install directory @id:lifecycle-linux-uninstall-path-failure-keeps-binaries @lifecycle @requires-os:linux @@ -405,3 +408,63 @@ Feature: Release install lifecycle And the installer output has no banner art And the installer next-step hint names the installed rocm binary And the installer output has no ANSI escapes + + @id:lifecycle-linux-fish-configured-from-bash @lifecycle @requires-os:linux + Scenario: lifecycle-32 - Linux - a bash login shell on a machine with fish also gets fish's PATH and completions, and uninstall undoes both + Given a freshly built release tree + And a generated signing keypair + And the login shell is bash + And the shell profile already holds the user's own settings + And fish is set up with its own settings + And a signed bundle installed with the shell profile updated + Then the fish config adds the install directory to PATH once + And the fish completions for the installed rocm are in place + When the signed bundle is reinstalled with the shell profile updated + Then the fish config adds the install directory to PATH once + When the user uninstalls from the installed binary + Then the installed rocm and rocmd binaries are gone + And the shell profile is back to the user's own settings + And the fish config is back to the user's own settings + And the fish completions for the uninstalled rocm are gone + And the record of the installer's PATH change is gone + + @id:lifecycle-linux-fish-completions-not-ours-kept @lifecycle @requires-os:linux + Scenario: lifecycle-33 - Linux - uninstall leaves fish completions the installer did not write, and says so + Given a freshly built release tree + And a generated signing keypair + And the login shell is bash + And fish is set up with its own settings + And a signed bundle installed with the shell profile updated + And the user has replaced the fish completions with their own + When the user uninstalls from the installed binary + Then the installed rocm and rocmd binaries are gone + And the fish config is back to the user's own settings + And uninstall leaves the user's own fish completions in place and says why + + @id:lifecycle-linux-fish-completions-removal-failure-warns @lifecycle @requires-os:linux + Scenario: lifecycle-34 - Linux - fish completions that cannot be removed are a warning, and the rest of the uninstall still happens + Given a freshly built release tree + And a generated signing keypair + And the login shell is bash + And fish is set up with its own settings + And a signed bundle installed with the shell profile updated + And the fish completions folder cannot be written to + When the user uninstalls from the installed binary + Then uninstall warns that the fish completions could not be removed and still finishes + And the installed rocm and rocmd binaries are gone + And the fish config is back to the user's own settings + And the record of the installer's PATH change is gone + + @id:lifecycle-linux-fish-existing-completions-left-alone @lifecycle @requires-os:linux + Scenario: lifecycle-35 - Linux - the installer leaves a rocm.fish it did not write, says so, and uninstall leaves it too + Given a freshly built release tree + And a generated signing keypair + And the login shell is bash + And fish is set up with its own settings + And the user already has their own fish completions + And a signed bundle installed with the shell profile updated + Then the installer leaves the user's own fish completions and says why + And the fish config adds the install directory to PATH once + When the user uninstalls from the installed binary + Then the installed rocm and rocmd binaries are gone + And the user's own fish completions are untouched diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature new file mode 100644 index 0000000000..e55ad7fe16 --- /dev/null +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -0,0 +1,64 @@ +Feature: Lemonade installs on disk + + # Lemonade manages its own runtime, and where it installs depends on whether a + # ROCm runtime is active: into the data folder when none is, into that + # runtime's own folder when one is. So `rocm engines install lemonade` before + # `rocm install sdk`, then `rocm serve` after it, leaves two full copies of + # roughly 25 GiB each, and the second one used to arrive silently from + # `rocm serve` (ROCMAI-825). These scenarios cover what the user is now told + # about that: what an install needs before it starts, both copies in the disk + # report, and the install record when a runtime holding a copy is removed. + # + # A new feature file rather than appended scenarios: these are about one + # behaviour area, Lemonade's footprint on disk, which spans `engines install`, + # `storage report` and `runtimes uninstall`. + # + # None downloads anything or needs a GPU, so all run on the mock lane every PR. + # lemonade-installs-01 and -02 need the `rocm/e2e-test-hooks` build + # `cargo xtask e2e` makes, to report a nearly full drive without filling a + # real one. + + @id:lemonade-installs-refused-without-space + Scenario: lemonade-installs-01 - A Lemonade install the drive cannot hold is refused before downloading + Given the drive for a new Lemonade install cannot hold even Lemonade itself + When the user installs the Lemonade engine + Then the Lemonade install is refused before anything is downloaded + And the user is told what the Lemonade install needs and where it would go + + # `rocm serve` reaches the same install on its own, which is how the second + # copy used to appear without a word. The free-space override also stands in + # for the GPU the no-GPU lane does not have, as serve-18's failure hook does. + @id:lemonade-installs-serve-refused-without-space + Scenario: lemonade-installs-02 - Serving with Lemonade on a drive that cannot hold it is refused before downloading + Given the drive for a new Lemonade install cannot hold even Lemonade itself + When the user serves a model with Lemonade + Then the Lemonade install is refused before anything is downloaded + And the user is told what the Lemonade install needs and where it would go + + @id:lemonade-installs-storage-lists-both-copies + Scenario: lemonade-installs-03 - The disk report lists both Lemonade copies and flags the unused one + Given Lemonade was installed in the data folder and again inside a ROCm install + When the user asks what ROCm CLI is keeping on disk + Then the report lists both Lemonade installs with their sizes + And the report flags the data folder's Lemonade install as unused + + @id:lemonade-installs-runtime-uninstall-clears-record + Scenario: lemonade-installs-04 - Removing the ROCm install that held Lemonade clears its install record + Given Lemonade was installed in the data folder and again inside a ROCm install + When the user uninstalls the ROCm install holding Lemonade + Then the CLI says Lemonade's install was removed with that ROCm install + And Lemonade is no longer recorded as installed + + @id:lemonade-installs-dangling-record-keeps-the-survivor + Scenario: lemonade-installs-05 - A record naming a folder with no install does not make the surviving copy reclaimable + Given the ROCm install that held Lemonade is gone and only the data folder's copy is left + When the user asks what ROCm CLI is keeping on disk + Then the report does not call the surviving Lemonade copy unused or reclaimable + + @id:lemonade-installs-read-only-runtime-keeps-its-copy + Scenario: lemonade-installs-06 - Uninstalling a read-only ROCm install leaves its Lemonade copy, and the report says so + Given a read-only ROCm install holds a Lemonade copy + When the user asks what ROCm CLI is keeping on disk + Then the report does not say that copy is removed with its ROCm install + When the user uninstalls the ROCm install holding Lemonade + Then that Lemonade copy is still on disk diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature index c58430162b..f35d461b9f 100644 --- a/tests/e2e-cucumber/features/logs.feature +++ b/tests/e2e-cucumber/features/logs.feature @@ -60,3 +60,79 @@ Feature: Log inspection Given a failed local server whose log shows a download at 43 percent When the user follows that server's logs Then the CLI prints the download line, says it is not following, and exits with an error + + # A log is read as text but a stray byte that is not valid text must not hide + # the readable lines around it: the view decodes it lossily and the search + # still finds every matching line. Before, one such byte made `rocm logs` + # show a single "failed to read" placeholder instead of the whole file. + @id:logs-invalid-byte-keeps-matching-lines + Scenario: logs-08 - A log with a byte that is not valid text still yields its matching lines + Given recorded command logs containing several lines about a topic and one byte that is not valid text + When the user searches the logs for that topic + Then the CLI reports the matching recent lines + + # The log views read at most the last 4 MiB of a log. A log whose newest line + # is longer than that has content but no line the read can return whole, and + # reporting it as "no output yet" / "empty" told the user the file held + # nothing. Each pair below plants the two states that look alike (an empty file + # and one huge line) and asserts the wording together with the file on disk. + @id:logs-service-log-longer-than-the-read-bound-is-not-called-empty + Scenario: logs-09 - A server log whose only line is longer than the read bound is not reported as having no output + Given a local server record whose log holds one line longer than the log view reads + When the user reads that server's log + Then the CLI says the newest line is too long to show instead of saying there is no output + + @id:logs-service-log-empty-says-no-output-yet + Scenario: logs-10 - An empty server log is reported as having no output yet + Given a local server record whose log file is empty + When the user reads that server's log + Then the CLI says there is no output yet + + @id:logs-comfyui-log-longer-than-the-read-bound-is-not-called-empty + Scenario: logs-11 - A ComfyUI log whose only line is longer than the read bound is not reported as empty + Given a ComfyUI log holding one line longer than the log view reads + When the user reads the ComfyUI logs + Then the CLI says the ComfyUI log's newest line is too long to show instead of saying it is empty + + @id:logs-comfyui-log-empty-says-empty + Scenario: logs-12 - An empty ComfyUI log is reported as empty + Given a ComfyUI log file that is empty + When the user reads the ComfyUI logs + Then the CLI says the ComfyUI log is empty + + # The `service_logs` tool the assistant and the MCP clients call reads the same + # log through the same bound, and used to answer "Read the last 80 line(s)" with + # an empty tail for both an empty log and one huge line. `rocm mcp-call` runs + # that tool in the CLI; the daemon's copy of it (`rocmd mcp-call`) is covered by + # a `rocmd` unit test, because the harness drives only the `rocm` binary. + @id:logs-service-logs-tool-longer-than-the-read-bound-is-not-called-empty + Scenario: logs-13 - The service_logs tool does not report a log with one over-long line as empty + Given a local server record whose log holds one line longer than the log view reads + When a tool-using client reads that server's log through the service_logs tool + Then the tool says the newest line is too long to show and flags it instead of returning an empty tail unexplained + + @id:logs-service-logs-tool-empty-says-no-output-yet + Scenario: logs-14 - The service_logs tool reports an empty log as having no output yet + Given a local server record whose log file is empty + When a tool-using client reads that server's log through the service_logs tool + Then the tool says the server has no log output yet and does not flag an over-long line + + # `rocm logs` lists the last lines of each command log it finds. A command log + # whose only line is longer than the 4 MiB a log view reads has content but no + # line the read can return whole; listing nothing made the view print "No logs + # found yet." although the file is on disk. It is listed with a line that says + # why nothing is shown. + @id:logs-command-log-longer-than-the-read-bound-is-not-called-absent + Scenario: logs-15 - A command log whose only line is longer than the read bound is not reported as no logs found + Given a recorded command log whose only line is longer than the log view reads + When the user lists the recorded logs + Then the CLI lists that log as having output it could not show instead of saying no logs were found + + # The structured reply's `lines` is how many lines the tool returned, not how + # many it was asked for: asking for 50 lines of a three-line log returns three, + # and the number asked for is reported separately as `lines_requested`. + @id:logs-service-logs-tool-reports-lines-returned + Scenario: logs-16 - The service_logs tool reports the lines it returned, not the number it was asked for + Given a local server record whose log holds three short lines + When a tool-using client asks the service_logs tool for 50 lines of that server's log + Then the tool reports the three lines it returned and the 50 lines it was asked for diff --git a/tests/e2e-cucumber/features/model_catalog.feature b/tests/e2e-cucumber/features/model_catalog.feature index 458dd80d7e..7d355ec8e6 100644 --- a/tests/e2e-cucumber/features/model_catalog.feature +++ b/tests/e2e-cucumber/features/model_catalog.feature @@ -37,3 +37,22 @@ Feature: Model catalog listing Given a recipe catalog whose AIM entry was last verified long ago When the user asks for the model catalog in verbose detail Then the catalog warns that the AIM entry may no longer be current + + # The catalog is also read by tool-using clients — the dashboard's Chat tab + # and any MCP client — and what they were handed was the verbose listing, not + # the recommendation list a person sees. On a developer host that is 189 KB + # against under 1 KB, which is more than a locally served model's whole + # context, so asking the assistant what this machine can run ended with the + # model's provider refusing the request instead of an answer. + @id:model-catalog-tool-client-gets-the-recommendations + Scenario: model-catalog-04 - A tool-using client asking about models gets the recommendations + When a tool-using client asks for the model catalog + Then the client is given the recommended models without the per-recipe diagnostics + And what the client is given is small enough for a locally served model to read + + # The diagnostics are not gone, they are opt-in: a client that wants them + # asks for them, the same way a person types `rocm model --verbose`. + @id:model-catalog-tool-client-can-ask-for-diagnostics + Scenario: model-catalog-05 - A tool-using client can still ask for the full diagnostics + When a tool-using client asks for the model catalog in verbose detail + Then the client is given the per-recipe diagnostics diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index ffbb2fd692..3623c1e80a 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -10,10 +10,19 @@ Feature: Model serving When the user serves a model using its short name Then the output shows the full model name + # A short name is a catalog entry, and an entry can publish a different build + # of the same model per engine, so the names need not be identical -- what must + # hold is that each engine names the build it will serve. + # + # This discriminates nothing yet: the scenario is an expected failure on every + # host while its xfail row stands, and the vLLM run stops before printing a + # plan, so the step panics there whatever the lemonade half did. It becomes + # regression coverage when that row is narrowed. serve-55 and the unit tests in + # serve_cmd.rs are what hold the per-engine refs in the meantime. @id:serve-short-name-consistent-across-engines - Scenario: serve-02 - Short name expansion is consistent across engines + Scenario: serve-02 - Every engine expands a short name rather than serving it When the user serves the same short name with different engines - Then all engines expand to the same full model name + Then every engine expands it to the full name of the model it will serve @id:serve-discoverable-by-name Scenario: serve-03 - A running model server is discoverable by name @@ -70,14 +79,11 @@ Feature: Model serving Then the response contains a model reply And the response identifies the correct model - # HF-checkpoint direct-serve canary (EAI-8026). An `owner/repo:variant` ref - # bypasses Lemonade's model router and runs a packaged llama-server directly - # (`serve_hf_checkpoint`), which bails explicitly if no backend binary is found - # under bin/llamacpp// — unlike the short-recipe-name path above, whose - # managed-lemonade fallback would mask the identical failure behind the - # unrelated EAI-7423 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as - # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a - # fast per-PR canary rather than needing the @nightly large-checkpoint path. + # Explicit HF-checkpoint canary: `owner/repo:variant` runs the selected GPU + # llama-server directly and fails if that backend binary is missing. Named + # catalog entries can also serve directly on Linux when their metadata is + # simple enough; other registered entries use Lemonade's router. Both canaries + # use the same small checkpoint, sharing the cache without another download. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade Scenario: serve-08 - A canonical Hugging Face checkpoint serves and responds to inference Given a managed runtime is active @@ -307,7 +313,7 @@ Feature: Model serving # `--require-api-key` reuse refusal (#604). A running server's key is read once # at launch; reuse cannot retroactively satisfy a demand for auth the server # never got, so the idempotency guard refuses instead of upgrading the record - # in place (see `spawn_managed_engine_child` in `service_launch.rs`). This is the + # in place (see `spawn_managed_engine_child` in `service_launch/spawn.rs`). This is the # local-side proof that the printed stop hint is not just present but correct # and directly runnable: the scenario stops the service with it and checks the # record is left `stopped`, the same proof `@id:service-stop-yes-stops-a- @@ -318,10 +324,10 @@ Feature: Model serving # this guard without a live GPU, same as every other `@requires-gpu` scenario # on this lane. Linux-only: `--require-api-key` forces auth even on a loopback # bind, and the serve path's `ensure_public_bind_engine_supported` guard - # (defined in `service_launch.rs`, called from `serve_cmd.rs`) refuses that combination - # for Lemonade on Windows (its server can't receive the key through the - # Windows detached-spawn primitive), so this scenario cannot reach the - # refusal it exercises there. + # (defined in `service_launch/endpoint_auth.rs`, called from `serve_cmd.rs`) + # refuses that combination for Lemonade on Windows (its server can't receive the + # key through the Windows detached-spawn primitive), so this scenario cannot + # reach the refusal it exercises there. @id:serve-require-api-key-refuses-unauthenticated-reuse @requires-gpu @requires-engine:lemonade @requires-os:linux Scenario: serve-24 - Reusing a running server without a key refuses a --require-api-key request Given a managed runtime is active @@ -359,14 +365,16 @@ Feature: Model serving # verified on (see `HostCapability::engine_available`'s "aim" arm — Radeon # gfx110X/120X and Strix Halo gfx1151, not Instinct); a Strix Halo # self-hosted runner with Docker is where this actually runs. - # `@nightly` keeps both AIM scenarios off the per-PR lanes: each one pulls the - # image and a 9B checkpoint on a cold runner and may wait out the full - # 1200s readiness cap, so the pair on the per-PR Strix Ubuntu job (~25 min - # without them, 45 min cap) ended every run at the cap. They run only when + # `@nightly` keeps both AIM scenarios off the pull-request lanes: each one + # pulls the image and a 9B checkpoint on a cold runner and may wait out the + # full 1200s readiness cap, so the pair on the Strix Ubuntu job (~25 min + # without them, 45 min cap) ended every run at the cap. That job now runs + # only on push, dispatch and nightly, never on pull_request. They run only when # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. - # Unconfirmed: whether the nightly Strix Halo host has Docker; without it both - # skip every night (the nightly job logs whether Docker is reachable). + # On a host without Docker both skip (the nightly job logs whether Docker is + # reachable). serve-27's known failure on Strix Halo Ubuntu is an xfail row in + # expectations.toml, tracked in rocm-cli#1058. # `@serve-timeout:1200` # widens both the harness's readiness poll and the CLI's own # ROCM_CLI_AIM_READY_TIMEOUT_SECS (see isolate_cmd) — a cold-start HF weight @@ -725,7 +733,7 @@ Feature: Model serving # so it cannot be combined with the attached modes, which already stay on the # terminal. The waiting itself (a live server that becomes ready, one that # exits first) is covered by `wait_past_readiness_budget_*` in - # `apps/rocm/src/service_launch.rs`, since a scenario would need a slow engine. + # `apps/rocm/src/service_launch/readiness.rs`, since a scenario would need a slow engine. @id:serve-wait-conflicts-with-attached-modes Scenario: serve-43 - Waiting for readiness cannot be combined with attached mode When the user serves a model with both --wait and --verbose @@ -750,7 +758,7 @@ Feature: Model serving # "stop the service holding that port" advice is checked here by its wording # only; the advice is actually proven by # `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` in - # apps/rocm/src/service_launch.rs, per AGENTS.md section 3. + # apps/rocm/src/service_launch/launch_transaction.rs, per AGENTS.md section 3. # Runs in the merge queue, like its automatic-port sibling serve-auto-port-skips-occupied. @id:serve-explicit-port-reserved @requires-gpu @merge-queue Scenario: serve-45 - Serving on a port another managed service already holds is refused @@ -810,3 +818,125 @@ Feature: Model serving And a stopped managed service is recorded on the default serve port When the user restarts the stopped service Then the restart is refused naming the service that holds the port + + # ROCMAI-597: a served model is also reachable from Anthropic-format clients. + # The serving process publishes an Anthropic-compatible `/v1/messages` endpoint + # beside the OpenAI one — the engine's own when it serves one, else a + # translation front on its own port — and `services list` shows it. Engine- + # agnostic like `serve-readiness-contract`: the served engine follows the host. + # Tool use is translated for non-streaming requests (serve-53); a streaming + # request that offers tools is refused, which only the unit tests cover. + @id:serve-anthropic-endpoint-advertised @requires-gpu + Scenario: serve-50 - A served model advertises an Anthropic-compatible endpoint + Given a managed runtime is active + And a model is being served on GPU + When the user lists running services + Then the service details include an Anthropic-compatible messages endpoint + And the service details still include the existing OpenAI-compatible endpoint + + @id:serve-anthropic-messages-inference @requires-gpu + Scenario: serve-51 - A served model responds to an Anthropic Messages API request + Given a managed runtime is active + And a model is being served on GPU + When the user sends a request to the Anthropic messages endpoint + Then the response is a valid Anthropic Messages API response + And the Anthropic response identifies the correct model + + @id:serve-anthropic-endpoint-readiness @requires-gpu + Scenario: serve-52 - Readiness covers the Anthropic-compatible endpoint + Given a managed runtime is active + And a model is being served on GPU + When the CLI reports the service as ready + Then both the OpenAI-compatible and Anthropic-compatible endpoints are reported healthy + + # ROCMAI-920: tool use is translated in both directions. A request offering + # one tool and requiring it gets a `tool_use` block back. + @id:serve-anthropic-messages-tool-use @requires-gpu + Scenario: serve-53 - A served model handles tool use via the Anthropic Messages API + Given a managed runtime is active + And a model is being served on GPU + When the user sends a tool-use request to the Anthropic messages endpoint + Then the response contains a tool_use content block + And the tool_use block names a tool and arguments from the request + + @id:serve-anthropic-messages-streaming @requires-gpu + Scenario: serve-54 - A served model streams responses via the Anthropic Messages API + Given a managed runtime is active + And a model is being served on GPU + When the user sends a streaming request to the Anthropic messages endpoint + Then the response is delivered as Anthropic-style server-sent events + And the streamed content reassembles into a complete reply + + # The catalog's recommended small assistant for a low-VRAM machine, asked for + # by its short name with no engine named. Whatever engine is chosen for the + # user has to be handed a build of that model it can actually load, so the + # short name alone must be enough. (The dash serve wizard fills the field with + # the recipe's canonical id rather than a short name; the unit tests cover that + # spelling, and it resolves through the same recipe.) + # + # Deliberately NOT tagged @requires-engine: the engine is the CLI's to pick + # here, and pinning one would make the harness attribute the run to an engine + # the scenario never asked for. The host's own default engine is what gates it. + @id:serve-small-assistant-short-name @requires-gpu + Scenario: serve-55 - The recommended small assistant serves from its short name alone + Given a managed runtime is active + And the recommended small assistant is being served by its short name + When the user sends a chat completion request + Then the response contains a model reply + And the response identifies the correct model + + # ROCMAI-863: a Hugging Face repo named without a quantization variant has no + # GGUF file to serve, but serving started an engine for it anyway and left the + # objection to the background process — where, on Windows, nothing the user can + # see ever reports it: an empty log, a success exit, and a server recorded as + # stopped. The refusal now comes before anything starts. It also names no + # quantization of its own: nothing on this path reads the repository's files, so + # a suggested label is a guess, and following a guess fails a second time on a + # repo that ships no GGUF at all. @requires-gpu because a host with no usable + # GPU is turned away earlier, for that reason instead, and never reaches this + # one. No @requires-os deliberately — Windows is the lane the failure was silent + # on, so it is the one most worth watching. + @id:serve-missing-gguf-variant-refused @requires-gpu @requires-engine:lemonade + Scenario: serve-56 - Serving a Hugging Face repo with no quantization variant is refused + When the user serves a Hugging Face repo with no quantization variant on Lemonade + Then serving is refused before any engine starts + And the user is told the model needs an explicit GGUF variant + And no quantization is suggested for a repo the CLI has not inspected + And no local server was recorded + # The pinned Lemonade catalog's smallest vision GGUF needs its mmproj file. + # Serving its registered name must retain the router, not flatten the entry + # into a single-checkpoint direct serve. Use a scenario-owned port and service; + # the additional model download stays off the ordinary per-PR path. + # Linux only: registered names take the router on Windows both with and + # without the direct llama-server route, so a Windows run cannot tell the + # two behaviours apart. + # No expectations row is added for this scenario. The flaky Linux rows cover + # the direct llama-server route, and this scenario takes the router route, so + # those rows' failure mode does not describe it. It is nightly-only. + @id:serve-lemonade-complex-registered-inference @requires-gpu @requires-os:linux @requires-engine:lemonade @nightly + Scenario: serve-57 - A registered vision GGUF retains the Lemonade router and managed lifecycle + Given a managed runtime is active + And a registered vision GGUF is being served on lemonade on its own port + Then the managed engine state confirms the Lemonade router loaded the registered model + When the user sends a chat completion request + Then the response contains a model reply + And the response identifies the correct model + When the user stops the scenario's managed Lemonade service + Then the managed service is stopped and its endpoint is closed + + # Which runtime a registered name takes is user-observable: a simple catalog + # entry (one checkpoint with a variant, no extra files) is served directly by + # the selected GPU llama-server on Linux, while + # `serve-lemonade-complex-registered-inference` covers the router route for an + # entry that needs more. The endpoint answering would not tell the + # two apart, so this reads the engine state's method and backend. + @id:serve-lemonade-simple-registered-direct-route @requires-gpu @requires-os:linux @requires-engine:lemonade + Scenario: serve-58 - A simple registered GGUF name is served directly by the selected GPU backend + Given a managed runtime is active + And a simple registered GGUF is being served on lemonade on its own port + Then the managed engine state records direct serving by the selected GPU backend + When the user sends a chat completion request + Then the response contains a model reply + And the response identifies the correct model + When the user stops the scenario's managed Lemonade service + Then the managed service is stopped and its endpoint is closed diff --git a/tests/e2e-cucumber/features/runtime_lifecycle.feature b/tests/e2e-cucumber/features/runtime_lifecycle.feature index e5e9537edf..5f8da06c42 100644 --- a/tests/e2e-cucumber/features/runtime_lifecycle.feature +++ b/tests/e2e-cucumber/features/runtime_lifecycle.feature @@ -48,10 +48,12 @@ Feature: Runtime lifecycle state machine Given a runtime manifest to import When the user imports the runtime Then the runtime is registered as read-only + And the registry records the import and its folder gains no manifest file When the user imports the same runtime again Then the CLI refuses because it already exists When the user imports it again allowing replacement Then the import succeeds + And the registry records the import and its folder gains no manifest file # `rocm runtimes list` prefixes each row with `*`/`-`/a blank to mark active, # rollback-target, and neither, with nothing else on the page explaining what diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index d3b93cf112..24c61ea166 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -50,6 +50,15 @@ Feature: Runtime configuration # It reads the block's verdict rather than one string, because a torch that has # already run a GPU kernel with this SDK is kept instead of rewritten and reports # a `retained_*` verdict — settled, with nothing installed. + # On a ROCm 10.x runtime the vLLM install pins torch itself and the engine's own + # metadata declares none, so the verdict is `not_applicable (vllm does not pin + # torch)`; the step accepts it only after asking the runtime's interpreter that + # the installed vLLM really requires no torch. + # Nothing compared a pin there, and the device check only proves torch can run a + # kernel, so the same step also imports torch, torchvision, vLLM and its compiled + # extensions in the runtime: a reinstall that leaves a torch and torchvision pair + # unable to register their operators, or a torch the vLLM build does not match, + # fails here instead of passing and breaking every serve. @id:runtime-sdk-reinstall-keeps-engine-consistent @requires-gpu @requires-engine:vllm @nightly Scenario: runtime-03 - Reinstalling the SDK leaves the installed engine able to use the GPU Given a managed runtime with an inference engine already installed @@ -70,14 +79,16 @@ Feature: Runtime configuration # see. This is the same reinstall as scenario 4 with the opt-out set, so what # differs between them is exactly the variable. # - # Four claims across three Thens, because the opt-out is only coherent if all - # four hold: torch was not rewritten; the skip is reported as its own verdict - # rather than folded into the generic `not_applicable`, which would leave the - # user unable to tell whether the variable did anything; the divergence the - # opt-out deliberately leaves behind is not then sold back to that user as a - # runtime to repair by reinstalling the engine — an instruction that would undo - # what they asked for; and the checks the opt-out does not suppress still run, - # because it suppresses the correction, not the diagnosis. + # Four claims, because the opt-out is only coherent if all four hold: torch + # was not rewritten by the alignment; the SDK install itself did not rewrite + # it either — it hands `uv` the torch requirements too, so + # before it honoured the opt-out the engine's torch was replaced by the SDK's + # build in the same command that reported the opt-out, and a GPU on which that + # build opens no device failed the install; the divergence the opt-out + # deliberately leaves behind is not then sold back to that user as a runtime to + # repair by reinstalling the engine — an instruction that would undo what they + # asked for; and the checks the opt-out does not suppress still run, because it + # suppresses the correction, not the diagnosis. # # Same lane as scenario 4 and for the same reasons: a real SDK install and a # real engine, on the serialized nightly GPU runners. `@requires-engine:vllm` @@ -88,6 +99,7 @@ Feature: Runtime configuration And the user has opted out of realigning torch When the user installs the SDK again Then the torch alignment reports the opt-out instead of rewriting torch + And the SDK install leaves the torch the runtime held as it was And the install does not offer to reinstall the engine over the kept torch And the runtime's device health is still reported @@ -129,6 +141,7 @@ Feature: Runtime configuration @id:runtime-install-records-the-real-folder @nightly Scenario: runtime-06 - Previewing an install through a linked runtimes folder names the real folder Given a machine whose runtimes folder is a link to somewhere else + And a uv is already installed When the user previews an SDK install Then the planned runtime folder is inside the folder the link points at And the planned runtime folder is not expressed through the link @@ -144,24 +157,29 @@ Feature: Runtime configuration Then the adoption is refused And the error explains which install types can be adopted - # Both channels now resolve from one canonical aggregate index each, so the - # provenance the preview prints is the whole of what the user can check before - # committing to a multi-GiB install. This pins the nightly half: the source that - # was selected, the version that came back from it, and the layout generation - # that source was read as. `--family` is supplied so the scenario needs no GPU, - # and `@nightly` because it resolves the real index over the network — the same - # cost that keeps scenario runtime-06 off the unserialized mock lane. + # Both channels resolve from one aggregate index each, so the provenance the + # preview prints is the whole of what the user can check before committing to a + # multi-GiB install. This pins the nightly half: the source that was selected, + # the version that came back from it, and the layout generation that source was + # read as. With no pin that is the newest layout's nightly index + # (`nightly.repo.amd.com`); runtime-31 pins the earlier stream. `--family` is + # an exact arch so the scenario needs no GPU, and `@nightly` because it + # resolves the real index over the network — the same cost that keeps scenario + # runtime-06 off the unserialized mock lane. @id:runtime-resolve-canonical-nightly @nightly - Scenario: runtime-08 - Previewing a nightly SDK install reports canonical provenance + Scenario: runtime-08 - Previewing a nightly SDK install reports the newest layout's provenance + Given a uv is already installed When the user dry-runs a nightly SDK install for a known family - Then the SDK preview reports canonical nightly provenance + Then the SDK preview reports next-layout nightly provenance # The release channel is where this went wrong in the field (rocm-cli#271). The # resolver read a per-family index, `repo.amd.com/rocm/whl/{family}`, which is # frozen at 7.13.0 and has no device payloads at all, so every release install # got a stale SDK and a bare `rocm[libraries,devel]` — no GPU backend in it. - # Both halves are fixed by the same canonical model: one flat aggregate index - # for the channel, and the device payload for the chip this host actually has. + # Both halves are fixed by the same model: one flat aggregate index for the + # channel (now `stable.repo.amd.com` for an unpinned install; runtime-32 pins + # the earlier release stream), and the device payload for the chip this host + # actually has. # # The obvious regression test for that history is the one that does not work. # Asserting the broken per-family URL is *absent* passes with the bug present: @@ -182,10 +200,11 @@ Feature: Runtime configuration # WSL hosts that can read the Windows-side target before ROCm passthrough is # ready, while mock hosts with no target skip it. @id:runtime-resolve-canonical-release @requires-gfx-target - Scenario: runtime-09 - Previewing a release SDK install resolves the canonical aggregate for this GPU + Scenario: runtime-09 - Previewing a release SDK install resolves the newest layout's aggregate for this GPU Given a machine with an AMD GPU + And a uv is already installed When the user dry-runs a release SDK install for this host - Then the SDK preview reports canonical release provenance + Then the SDK preview reports next-layout release provenance And the SDK preview requests the device payload for this host's GPU # `rollback` can only ever undo one step, not walk a history. A unit test on @@ -327,17 +346,28 @@ Feature: Runtime configuration # against real hardware (Strix Halo, gfx1151): a fresh SDK's version does not # match Lemonade's packaged pin, Tier 1's install 404s (the pinned build # predates a matching ROCm-version asset), and Tier 2's newest build - # succeeds -- producing exactly the line this scenario asserts. + # succeeds -- producing exactly the line this scenario asserts. That held for + # an SDK whose TheRock tarball Lemonade could download; for a 7.14.x SDK Tier 2 + # fails too (the tarball is not published where Lemonade 11.5.1 looks), which is + # why `expectations.toml` carries a Windows row for this scenario (ROCMAI-899). # # `@requires-engine:lemonade` because vLLM shares the SDK's own runtime # environment and has no llama.cpp backend to align; `@nightly` for the same # reason as the vLLM torch-alignment scenarios above -- a real managed SDK # and a real backend download, not something to repeat on every PR. + # + # The advice `rocm serve` prints for a missing llama-server is `rocm engines + # install lemonade --reinstall`. The Givens install Lemonade and delete the + # selected backend's llama-server, so the last Then can only pass if the + # reinstall put it back. @id:runtime-lemonade-backend-alignment-reported @requires-gpu @requires-os:windows @requires-engine:lemonade @nightly Scenario: runtime-16 - Reinstalling Lemonade reports whether its ROCm backend was aligned Given a managed runtime is active + And the lemonade engine is installed + And the selected Lemonade backend's llama-server binary has been deleted When the user reinstalls the lemonade engine Then the CLI reports that Lemonade's ROCm backend was aligned to the active SDK + And the selected Lemonade backend's llama-server binary is installed # `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` is the exit for a hand-edited # `backend_versions.json` -- the alignment runs on every Lemonade install, so @@ -426,8 +456,8 @@ Feature: Runtime configuration # network are needed and this runs on the ordinary E2E lane. # # Technical details: the host install is planted under the scenario's root - # and exported as ROCM_PATH; its `.info/version` reads `7.20.1-33`. The - # catalog serves one release tarball, ROCm 7.10.0, for `gfx120X-all`. + # and exported as ROCM_PATH; its `.info/version` reads `10.5.1-33`. The + # catalog serves one release tarball, ROCm 10.0.0, for `gfx120X-all`. @id:runtime-install-preview-notes-newer-host-rocm @requires-os:linux Scenario: runtime-20 - Previewing an install on a host with newer ROCm explains why the older SDK is selected Given a release tarball catalog whose newest ROCm is older than the host's @@ -442,8 +472,8 @@ Feature: Runtime configuration # evidence: it shows the same preview printing the note when the host # version is readable and newer. # - # Technical details: `.info/version` reads `7.20.1-33rc1`, which with its - # suffix dropped would be `7.20.1`, newer than the catalog's 7.10.0. + # Technical details: `.info/version` reads `10.5.1-33rc1`, which with its + # suffix dropped would be `10.5.1`, newer than the catalog's 10.0.0. @id:runtime-install-preview-ignores-unreadable-host-rocm @requires-os:linux Scenario: runtime-21 - Previewing an install on a host whose ROCm version cannot be read makes no newer-host claim Given a release tarball catalog whose newest ROCm is older than the host's @@ -512,7 +542,7 @@ Feature: Runtime configuration Scenario: runtime-23 - Previewing an SDK install on a host with no usable Python prepares no Python Given a canonical release pip index fixture And a host with no usable Python and a recording uv - When the user previews a wheel SDK install for arch gfx1200 with no version pin + When the user previews a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release Then the preview says a real install would prepare a managed Python And the preview prepared no Python, as it says @@ -532,7 +562,7 @@ Feature: Runtime configuration Given a canonical release pip index fixture And a host with no usable Python and a recording uv And the managed Python version is overridden to 3.13 - When the user tries to preview a wheel SDK install for arch gfx1200 with no version pin + When the user tries to preview a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release Then the install fails And the preview is refused because the install would refuse that Python @@ -556,6 +586,135 @@ Feature: Runtime configuration And a host with no usable Python and a recording uv And the only Python on the host's path is not usable And managed Python bootstrap is disabled - When the user tries to preview a wheel SDK install for arch gfx1200 with no version pin + When the user tries to preview a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release Then the install fails And the preview is refused because managed Python is disabled, without a promise + + # `rocm runtimes adopt --python` takes the environment folder from where the + # interpreter sits: the folder that holds its `bin` (or `Scripts`) folder. An + # interpreter anywhere else has no environment folder to infer, and a guess (the + # interpreter's grandparent, which is what the command used to adopt) would + # register a folder that is not an environment as a runtime. The command refuses + # instead and names the flag that supplies the folder; nothing is saved. + # + # Linux-only: the stand-in interpreter is a POSIX shell script. The "adopts with + # --root" half is covered by the unit test + # `adopt_request_without_an_inferred_root_asks_for_root`, which shows an explicit + # root clears the refusal; adopting for real needs a genuine ROCm SDK environment. + @id:runtime-adopt-outside-a-venv-asks-for-root @requires-os:linux + Scenario: runtime-26 - Adopting a Python that sits outside any environment's bin folder asks for --root + Given a Python interpreter that sits outside any environment's bin folder + When the user adopts that interpreter without naming an environment folder + Then the adoption is refused + And the refusal asks for the environment folder with --root + And no runtime is registered + + # A preview of a ROCm 10 or newer wheel has `uv pip compile` choose the torch + # and torchvision versions: nothing else does, and the plan's package specs, its + # runtime key and its command are made of them. When a `uv` is already there the + # preview must still run it, so the plan is the one a real install would print. + # This pins that half of the contract next to runtime-28, where no `uv` is there, + # so that skipping the step on a clean host cannot become skipping it everywhere. + # + # The recording `uv` is the one named by `ROCM_CLI_UV_BINARY`, so it counts as + # present, and it answers `uv pip compile` with the fixture's four pinned + # versions. Its log is the evidence the compile ran, once, and the plan + # carries the versions it printed. + # + # Linux only, for the same empty-`PATH` host as runtime-23 (the Python the + # preview would prepare is not what this scenario is about). + @id:runtime-install-preview-of-rocm-10-uses-the-uv-that-is-present @requires-os:linux + Scenario: runtime-27 - Previewing a ROCm 10 SDK install with a uv already present resolves the package versions with it + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a host with no usable Python and a recording uv + And the recording uv answers uv pip compile with the ROCm 10 fixture's versions + When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 + Then the preview requests the gfx1200 device extras + And the preview resolved the versions with the uv that was present + + # A preview is a promise that nothing on the machine changes, and runtime-23 held + # the Python half of it. The other half is `uv`: a preview of a ROCm 10 or newer + # wheel used to go and fetch the managed `uv` (creating the data folder's `tools` + # folder) because `uv pip compile` is the one step that knows the torch and + # torchvision versions. On a host with no `uv` the preview now runs nothing and + # downloads nothing. It prints what it can know without `uv` (the ROCm version, + # the torchaudio pin, the requirements `uv` would be asked about), says in a + # `resolution_note:` line and a progress line what it did not do, and marks the + # lines made from the missing versions as not resolved rather than printing a + # guess. A real install and `rocm update` still fetch `uv` as they always did. + # + # The Then checks the message and the state together: with no `uv` to log, the + # evidence is the folders. No managed `tools` folder, no tool archive in the + # scenario's own cache, no saved configuration, and no torch version anywhere in + # the output, so the unresolved lines are not covering for a guess. runtime-27 + # is the other polarity: with a `uv` present the compile still runs. + # + # Linux only, for the same empty-`PATH` host as runtime-23. The decision is + # unit-tested for every combination of layout, who may fetch `uv` and whether one + # is present by `the_version_step_is_decided_by_layout_access_and_a_present_uv_alone` + # in `apps/rocm/src/therock/resolve/uv_access.rs`, and the plan's wording (written + # by `apps/rocm/src/therock/wheel_plan.rs`) by the tests in + # `apps/rocm/src/therock/resolve/wheel.rs`, on every platform. + @id:runtime-install-preview-of-rocm-10-fetches-no-uv @requires-os:linux + Scenario: runtime-28 - Previewing a ROCm 10 SDK install on a host with no uv fetches none and says what it did not resolve + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a host with no uv and no usable Python + When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 + Then the preview says it did not resolve the ROCm 10 package versions, as it says + + # ROCm 10 installs use Python 3.14 because vLLM, flash-attn and amd-aiter for + # ROCm 10 exist for it alone, but the SDK and PyTorch wheels are published for + # older Pythons too. `--python-version` lets an install that does not need vLLM + # pick one. The preview must plan what the install would do with it: the + # requested interpreter and its wheel tag, not the default's, with the same + # resolution as runtime-27. Only the plan is asserted here; no scenario runs + # resolution with the chosen interpreter. + # + # Linux only, for the same empty-`PATH` host as runtime-23. + @id:runtime-install-preview-of-rocm-10-honours-python-version @requires-os:linux + Scenario: runtime-29 - Previewing a ROCm 10 SDK install with --python-version plans that Python and not the default + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a host with no usable Python and a recording uv + And the recording uv answers uv pip compile with the ROCm 10 fixture's versions + When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 on Python 3.12 + Then the preview plans Python 3.12 and its cp312 wheel tag and not the default + + # The flag is a choice among the interpreters the layout is published for, so a + # request for one it is not published for is refused instead of ignored: a + # canonical (ROCm 7) install, whose one interpreter is 3.12. This scenario covers + # that refusal; the ROCm 10 refusal (a Python the index has no wheels for) is + # unit-tested only. The refusal names the Python that is allowed, and nothing is + # prepared on the way to it. + # + # Linux only, for the same empty-`PATH` host as runtime-23. Each combination of + # layout and version is also unit-tested on every platform + # (`python_version_is_a_choice_among_the_layouts_published_pythons` in + # `apps/rocm/src/therock/python.rs`). + @id:runtime-install-refuses-python-version-it-cannot-honour @requires-os:linux + Scenario: runtime-30 - Previewing an SDK install with a --python-version the layout is not published for is refused + Given a canonical release pip index fixture + And a host with no usable Python and a recording uv + When the user tries to preview a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release on Python 3.13 + Then the install fails + And the preview is refused because the layout has no wheels for that Python, and prepared none + + # The other half of runtime-08. The newest layout took over the nightly channel + # on 2026-08-23; the earlier stream (`rocm.nightlies.amd.com`) stopped the day + # before, so a build from before then exists only there. Pinning one must + # keep resolving it, from that stream and read as that stream's layout, rather + # than failing against the newest layout's index. `@nightly` for the same + # reason as runtime-08: it resolves the real index. + @id:runtime-resolve-earlier-nightly @nightly + Scenario: runtime-31 - Previewing a nightly SDK install pinned to an earlier build reports the earlier stream's provenance + When the user dry-runs a nightly SDK install pinned to a build from the earlier stream + Then the SDK preview reports canonical nightly provenance + + # The other half of runtime-09: an earlier ROCm 7 release is published only on + # the earlier stream (`repo.amd.com`), so pinning one has to keep reading it, + # while the unpinned install of runtime-09 reads the newest layout. Same + # `@requires-gfx-target` as runtime-09, since the preview detects the chip. + @id:runtime-resolve-earlier-release @requires-gfx-target + Scenario: runtime-32 - Previewing a release SDK install pinned to an earlier ROCm 7 release reports the earlier stream's provenance + Given a machine with an AMD GPU + When the user dry-runs a release SDK install pinned to an earlier ROCm 7 release + Then the SDK preview reports canonical release provenance diff --git a/tests/e2e-cucumber/features/service_record_unqueryable.feature b/tests/e2e-cucumber/features/service_record_unqueryable.feature new file mode 100644 index 0000000000..77ecaf04db --- /dev/null +++ b/tests/e2e-cucumber/features/service_record_unqueryable.feature @@ -0,0 +1,49 @@ +Feature: A local server record held by a process rocm cannot query + + # On Windows a live process the account may not open cannot be queried, so + # `rocm` cannot tell it from a dead server whose PID another program took. + # It treats that as possibly running: a stop reports it unverified, and + # `rocm services remove` refuses the record. `remove --force --yes` is the way + # out for exactly that case, and these scenarios drive the whole path through + # the real binary rather than the pieces. + # + # An unqueryable process cannot be staged from outside the binary, so the + # `e2e-test-hooks` build reads ROCM_E2E_UNQUERYABLE_PIDS and answers + # "cannot be queried" for the PIDs it lists; production builds never read it. + # The records are planted as plain JSON, so no scenario needs a GPU. + # + # `service-unqueryable-01` has no OS tag and the hook works on Windows too, + # but no CI job runs this feature on Windows: the Windows job does not run + # scenarios. It has run on Linux only. + # + # A new feature file, as `service_record_cleanup.feature` explains for its + # own: indexes must be sequential per file, and several open branches claim the + # next ones in the files that exist. + + @id:service-unqueryable-remove-force-clears-an-unverified-record + Scenario: service-unqueryable-01 - A record whose process cannot be queried is stopped as unverified, refused by remove, and cleared by remove --force + Given a local server record whose recorded process cannot be queried + When the user stops that local server while its process cannot be queried + Then the CLI reports the stop as not confirmed because the process could not be queried + And every file belonging to that record is still there + When the user removes that local server record while its process cannot be queried + Then the CLI refuses the removal and names the forced removal + And every file belonging to that record is still there + When the user prunes every record whatever its age while its process cannot be queried + Then the CLI leaves the record alone and names the forced removal + And every file belonging to that record is still there + When the user force-removes that local server record while its process cannot be queried + Then the CLI removes the record and says its process could not be queried + And every file belonging to that record is gone + + # `--force` is for a process `rocm` cannot query, never for one it can see + # running. The recorded process here is a real one the scenario started, and a + # stop was requested on the record, so the only thing standing in the way is + # that the process is running. + @id:service-unqueryable-force-never-passes-a-running-process + @requires-os:linux + Scenario: service-unqueryable-02 - --force is refused for a record whose process is confirmed running + Given a local server record whose recorded process is running and whose stop was requested + When the user force-removes that local server record + Then the CLI refuses the forced removal because the record does not qualify + And every file belonging to that record is still there diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index cae0e9859a..2c30645420 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -41,7 +41,11 @@ Feature: Stopping a managed service # # The recorded process is a real one the scenario starts, recorded with the # kernel's own start-time token for it, so the stop's identity check confirms - # it and goes on to terminate it. No GPU and no engine are involved. + # it and goes on to terminate it. No GPU and no running engine are involved: the + # daemon does ask the engine to stop (a real `vllm` engine, reached through the + # `rocm` beside the standalone `rocmd`), and since the planted record names no + # engine process the engine answers that it stopped nothing, which the scenario + # checks along with the report. # # Linux-only: the token is read from `/proc`, which is where `rocmd` reads it # too; without one the identity check is not exercised at all. @@ -59,6 +63,7 @@ Feature: Stopping a managed service Given a managed server whose recorded process is running When the daemon's stop_server tool stops that server Then the tool reports the server stopped, naming the process it stopped + And the tool asked the engine to stop and the engine answered And the recorded process is no longer running And the server's record reads stopped and names no process And the server's endpoint key file is gone @@ -119,3 +124,73 @@ Feature: Stopping a managed service Then the CLI reports the stop as not confirmed and names no stop command And the server's record still names its process and the stop request And the report says only the record was kept, and there is no key file + + # `rocm services stop`'s confirmed half with a real process. service-stop-01 + # plants PID 0, so it never signals anything and cannot show what the stop + # leaves in the record; service-stop-02 shows it for the daemon's tool. This + # runs the CLI's own stop against a real recorded process and checks the + # persisted state it claims: the process exited, and the record names no + # process any more, so a second stop of that record would signal nothing (a + # PID recorded without a start-time token cannot be refuted once the OS + # recycles it). The unit test + # `a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` + # covers the second stop itself. + # + # Linux-only for the same reason as service-stop-02: the recorded process + # carries the start-time token `/proc` gives it. + + @id:service-stop-cli-confirms-a-stop-and-clears-the-record + @requires-os:linux + Scenario: service-stop-06 - `rocm services stop` of a running managed server confirms the stop and leaves no process in its record + Given a managed server whose recorded process is running + When the user stops it with --yes + Then the CLI reports the service as stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + # The Windows half of the stop. A managed engine runs one level below the + # recorded process (the daemon launches it through an `__engine-serve-http` + # process), and `terminate_verified` cannot walk a process tree on Windows, so + # the stop first runs `taskkill /PID /T /F` on the recorded process. + # service-stop-01 plants PID 0 and never reaches that step, and the scenarios + # above need Linux's start-time token, so none of them kills a real tree. + # + # This records a real parent that has started a real child, stops it, and + # checks that BOTH are gone together with the report and the record. Without + # the tree-kill step the stop ends only the recorded process, the child keeps + # running, and the scenario fails on it. No GPU, no engine, no network. The + # record carries no start-time token, like a Windows record written before the + # token was recorded: the case where the PID cannot be refuted, which the tree + # kill and the cleared record exist for. service-stop-08 below is the same stop + # with a token that does not match. + + @id:service-stop-takes-down-the-whole-process-tree-on-windows + @requires-os:windows + Scenario: service-stop-07 - Stopping a managed server on Windows takes down its recorded process together with the process it started + Given a managed server whose recorded process has started another process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the report counts one stopped process + And the recorded process and the process it started are no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + # A recorded PID is not proof that the recorded process still holds it: the + # server can exit by itself and the OS hand its PID to an unrelated program. + # The record pairs the PID with the process's start-time token (Linux's + # `/proc` start time, Windows' creation time), and a stop whose token no longer + # matches treats the recorded process as already gone: it reports the stop, + # clears the record and leaves the process that now holds the PID running, + # instead of signalling a stranger (on Windows, with `taskkill /T /F`, its + # whole subtree). The stranger here is a real process, so "left alone" is + # checked on the process itself. Runs on every OS: the planted token is one + # no live process has. + @id:service-stop-leaves-a-process-that-reused-the-recorded-pid-alone + Scenario: service-stop-08 - Stopping a managed server whose recorded PID now belongs to another process leaves that process running + Given a managed server whose recorded process id now belongs to a different process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the process now holding the recorded process id is still running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/features/therock_next_generation.feature b/tests/e2e-cucumber/features/therock_next_generation.feature index 35c683eccf..6e38e6c1f0 100644 --- a/tests/e2e-cucumber/features/therock_next_generation.feature +++ b/tests/e2e-cucumber/features/therock_next_generation.feature @@ -1,35 +1,51 @@ Feature: TheRock "next" ROCm 10 install layout - # ROCm 10 ships from a different source layout than the canonical multi-arch - # release/nightly streams: a separate pip index and a separate tarball catalog - # whose listing mixes non-release `-tests-` siblings in with the real dist - # archive. The next layout is an explicit extension, never a default: it is - # reachable only by pinning a ROCm version whose major is 10 or newer AND - # naming an exact raw GFX arch, and the canonical streams keep resolving - # exactly as they do today when neither is asked for. + # AMD now publishes every channel to a newer source layout than the canonical + # multi-arch release/nightly streams: a separate pip index and a separate + # tarball catalog per channel (`stable.repo.amd.com`, `nightly.repo.amd.com`), + # whose tarball listing mixes non-release `-tests-` siblings in with the real + # dist archive. The canonical streams no longer receive builds, so a request + # with no pin reads the newer layout, and a pin reads whichever layout holds + # that build: a stable ROCm 10+ release or a nightly built on or after + # 2026-08-23 in the newer layout, an earlier ROCm 7 release or an earlier + # nightly in the canonical stream. The newer layout picks its device payload + # by exact arch, so a request that reads it also needs a raw GFX arch. # - # Scenarios 01-05 point source overrides at loopback fixtures through the - # explicit trust gate. Scenario 06 proves the same overrides are inert when - # the trust gate is absent; scenario 07 supplies the live GPU evidence. + # Scenarios 01-05 and 16-23 point source overrides at loopback fixtures + # through the explicit trust gate. Scenario 06 proves the same overrides are + # inert when the trust gate is absent; scenario 07 supplies the live GPU + # evidence. + # + # A preview of the newer layout has a real `uv pip compile` choose the torch + # and torchvision versions, and a preview never fetches `uv` itself. The + # scenarios below that print resolved package specs (01, 02, 10, 11, 13, 15, + # 20 and 22) therefore start from an installed `uv`, which is the + # precondition they always had when the preview fetched one for them. + # runtime-27 and runtime-28 in `runtime_setup.feature` pin the preview's + # behaviour with a `uv` present and with none. - # The regression this pins: a "next-first" dispatch that probes the ROCm 10 - # sources before the canonical ones. An unpinned release install must never - # look at the next layout at all, so the next fixture is served and asserted - # untouched rather than left unconfigured. - @id:therock-next-01-unpinned-release-stays-canonical - Scenario: therock-next-01 - An unpinned release install stays on the canonical multi-arch source + # The release stream the CLI used to install by default stopped at ROCm 7.14, + # so an install with no pin that kept reading it would silently stay on old + # builds while the current ones sat in the newer layout. The canonical fixture + # is served and asserted untouched rather than left unconfigured, so reading + # it would show up as a failure and not as a coincidence. + @id:therock-next-01-unpinned-release-reads-the-newest-layout + Scenario: therock-next-01 - An unpinned release install reads the newest layout's source Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 with no version pin - Then the preview resolves the canonical release pip index - And the preview reports the canonical multi-arch source layout generation - And the preview never mentions the ROCm 10 pip index + Then the preview resolves the ROCm 10 pip index + And the preview reports the next source layout generation + And the preview never mentions the canonical release pip index - # `--version 10.0.0` plus a raw arch is the whole opt-in. The device payload is - # requested as an exact `device-gfx1200` extra on rocm, torch and torchvision; - # torchaudio carries no device extra because it publishes none. + # `--version 10.0.0` plus a raw arch selects the newer layout explicitly, the + # same layout an unpinned install reads. The device payload is requested as an + # exact `device-gfx1200` extra on rocm, torch and torchvision; torchaudio + # carries no device extra because it publishes none. @id:therock-next-02-wheel-pins-raw-device-extras Scenario: therock-next-02 - A pinned ROCm 10 wheel install resolves the next index with raw device extras Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 Then the preview resolves the ROCm 10 pip index And the preview reports the next source layout generation @@ -148,7 +164,7 @@ Feature: TheRock "next" ROCm 10 install layout # # The serve+inference tail matters because `serve-vllm-inference` # (model_serving.feature) does not cover this route: its runtime comes from - # a plain unpinned `rocm install sdk`, never the ROCm 10 preview source. A + # a plain unpinned `rocm install sdk`, which reads the newest layout. A # discovered wheel that installs cleanly but can't actually load or answer a # request (bad torch realignment, cp314 ABI mismatch, a `uv.toml` fallback # pulling an incompatible transitive dep) would pass every assertion above @@ -195,6 +211,7 @@ Feature: TheRock "next" ROCm 10 install layout @id:therock-next-09-wheel-devel-adds-the-toolchain Scenario: therock-next-10 - A pinned ROCm 10 wheel install adds the toolchain when asked Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 with the toolchain Then the preview resolves the ROCm 10 pip index And the preview requests the gfx1200 device extras with the toolchain @@ -210,6 +227,7 @@ Feature: TheRock "next" ROCm 10 install layout @id:therock-next-11-older-pin-keeps-one-rocm-build Scenario: therock-next-11 - A ROCm 10 wheel install pinned below the newest build installs only that build Given a ROCm 10 pip index fixture that also publishes a newer ROCm 10 build + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 Then the preview resolves the ROCm 10 pip index And every package in the preview is built for ROCm 10.0.0 @@ -235,6 +253,7 @@ Feature: TheRock "next" ROCm 10 install layout @id:therock-next-13-verified-torchaudio-over-newer-rebuild Scenario: therock-next-13 - A ROCm 10 wheel install takes the hardware-verified torchaudio over a newer rebuild Given a ROCm 10 pip index fixture where several torchaudio rebuilds share one ROCm label + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.1.0 Then the preview installs the hardware-verified torchaudio for ROCm 10.1.0, not the newer rebuild @@ -258,10 +277,104 @@ Feature: TheRock "next" ROCm 10 install layout # folder whose name carries a lone `'` rendered bare and left a line no shell # can parse. The assertion reads the line back through a POSIX word-splitter # and requires the install folder to come back as one word. Reuses the - # loopback fixture from therock-next-01, so it resolves without live network - # or GPU and needs no gate. + # loopback fixture from therock-next-01 (an unpinned install reads its ROCm 10 + # index, so it needs an installed `uv` like therock-next-01), so it resolves + # without live network or GPU and needs no gate. @id:therock-next-15-dry-run-command-keeps-a-quoted-folder-whole Scenario: therock-next-15 - The dry-run command line keeps an install folder named with a quote as one argument Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 into a folder whose name has a quote Then the dry-run command line names that folder as a single argument + + # The other half of therock-next-01. An earlier ROCm 7 release is published + # only in the canonical stream, so pinning one must keep reading it, from that + # stream and as that stream's layout, instead of failing against the newer + # layout's index. The newer layout's fixture is served and asserted untouched. + @id:therock-next-16-earlier-release-pin-reads-the-canonical-stream + Scenario: therock-next-16 - A release install pinned to an earlier ROCm 7 release reads the canonical stream + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + When the user previews a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release + Then the preview resolves the canonical release pip index + And the preview reports the canonical multi-arch source layout generation + And the preview never mentions the ROCm 10 pip index + + # No silent fallback, now for the default: with no pin the newer layout is + # the only place the newest build is, and a group label names no exact arch + # for it. The install must refuse and say which flag to pass, not drop back to + # the canonical stream and hand over an old build. The refusal also names + # pinning an earlier release as the way to still install a group family, so + # that command is run here: the message and the behaviour it promises are + # asserted together. + @id:therock-next-17-unpinned-group-family-refused-and-the-named-pin-works + Scenario: therock-next-17 - An unpinned release install refuses a group family, and the earlier-release pin it names installs + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + When the user tries to preview a wheel SDK install for family gfx120X-all with no version pin + Then the install fails + And the failure asks for an exact GPU arch and names --family gfx1200 + When the user previews a wheel SDK install for family gfx120X-all pinned to an earlier ROCm 7 release + Then the preview resolves the canonical release pip index + + # Linux-only for the same reason as therock-next-04. The unpinned tarball + # install reads the newer layout's catalog and still skips the `-tests-` + # sibling that carries a later mtime than the real archive. + @id:therock-next-18-unpinned-tarball-reads-the-newest-catalog @requires-os:linux + Scenario: therock-next-18 - An unpinned release tarball install reads the newest layout's catalog + Given a canonical release tarball fixture and a ROCm 10 tarball fixture with a tests sibling + When the user previews a tarball SDK install for arch gfx1200 with no version pin + Then the preview resolves the ROCm 10 tarball catalog + And the preview reports the next source layout generation + And the preview selects the real tarball artifact + And the preview does not select the tests artifact + + # A tarball catalog can only be pinned in the newer layout, so a pin into the + # earlier release stream is refused. The refusal points at the wheel install, + # and the wheel install it points at is run, so the advice is shown to work. + @id:therock-next-19-earlier-release-tarball-pin-points-to-wheel @requires-os:linux + Scenario: therock-next-19 - A release tarball install pinned to an earlier ROCm 7 release points to a wheel install + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + When the user tries to preview a tarball SDK install for arch gfx1200 pinned to an earlier ROCm 7 release + Then the install fails + And the failure points to a wheel install for that earlier release + + # The nightly channel moved the same way on 2026-08-23. With no pin the + # install reads the newer layout's nightly index; that index publishes a + # `rocm` build before its PyTorch wheels land, so the install takes the newest + # build that has the whole stack and says it skipped the newer one, rather + # than failing on a build it cannot install. + @id:therock-next-20-unpinned-nightly-reads-the-newest-layout-and-skips-an-unbuilt-build + Scenario: therock-next-20 - An unpinned nightly install reads the newest layout's index at its newest installable build + Given an earlier nightly pip index fixture and a newest-layout nightly pip index fixture whose newest build has no PyTorch wheels yet + And a uv is already installed + When the user previews a nightly wheel SDK install for arch gfx1200 with no version pin + Then the preview resolves the newest layout's nightly pip index at its newest installable build + And the preview warns that the newest nightly build has no PyTorch wheels and installs the build before it + + # The earlier nightly stream's last build is 2026-08-22, so a pin to that + # date can only be read from it, as that stream's layout. + @id:therock-next-21-earlier-nightly-pin-reads-the-canonical-stream + Scenario: therock-next-21 - A nightly install pinned to the earlier stream's last build date reads the canonical stream + Given an earlier nightly pip index fixture and a newest-layout nightly pip index fixture whose newest build has no PyTorch wheels yet + When the user previews a nightly wheel SDK install for arch gfx1200 pinned to the earlier stream's last build date + Then the preview resolves the earlier stream's nightly pip index + + # The other side of therock-next-21's boundary: a pin to a date the newer + # layout publishes reads that layout, though both fixtures are served. + @id:therock-next-22-newer-nightly-pin-reads-the-newest-layout + Scenario: therock-next-22 - A nightly install pinned to a build date from the newest layout reads that layout + Given an earlier nightly pip index fixture and a newest-layout nightly pip index fixture whose newest build has no PyTorch wheels yet + And a uv is already installed + When the user previews a nightly wheel SDK install for arch gfx1200 pinned to a build date from the newest layout + Then the preview resolves the newest layout's nightly pip index for that build date + + # The tarball half of therock-next-19, on the nightly channel, where the + # boundary is a date rather than a major. A tarball pin reads only the newest + # layout's catalog, so a pin to the earlier stream's last build date is + # refused, naming the date the newer layout begins and pointing to the wheel + # install, which is then run and resolves that build. + @id:therock-next-23-earlier-nightly-tarball-pin-points-to-wheel @requires-os:linux + Scenario: therock-next-23 - A nightly tarball install pinned to the earlier stream's last build date points to a wheel install + Given an earlier nightly pip index fixture and a newest-layout nightly pip index fixture whose newest build has no PyTorch wheels yet + When the user tries to preview a nightly tarball SDK install for arch gfx1200 pinned to the earlier stream's last build date + Then the install fails + And the failure names the first build date of the newer layout and points to a wheel install that resolves that earlier nightly diff --git a/tests/e2e-cucumber/features/uninstall.feature b/tests/e2e-cucumber/features/uninstall.feature index 286a59c0c2..147f0455e3 100644 --- a/tests/e2e-cucumber/features/uninstall.feature +++ b/tests/e2e-cucumber/features/uninstall.feature @@ -80,3 +80,110 @@ Feature: Uninstall Then the uninstall succeeds And the uninstall reports that nothing would be removed And ROCm CLI has still not created its folders + + # A dry run run by a person at a terminal ends by saying to run it again + # without `--dry-run`. It once said "Choose Review uninstall", a control the + # command line does not have. The promise is the line and the state together: + # the dry run removes nothing, and running the same command again, as the + # line says and answering its prompt, removes what was listed. + @id:uninstall-dry-run-at-a-terminal-says-to-run-it-again-to-remove @requires-os:linux + Scenario: uninstall-07 - At a terminal, a dry run removes nothing and running it again without --dry-run removes what it listed + Given the config folder holds the user's files + When the user dry-runs an uninstall of only the config at a terminal + Then the uninstall succeeds + And the dry run says to run it again without --dry-run + And the config folder still holds the user's files + When the user runs it again without --dry-run at a terminal and answers yes + Then the uninstall succeeds + And the config folder is gone + + # A run whose stdin or stdout is not a terminal (a script, the dashboard's + # `/uninstall`, the daemon's `rocm_command` tool) has no one to tell to run it + # again: the real step belongs to whatever drives it. It lists what it would + # remove, offers no step, and removes nothing. + @id:uninstall-dry-run-outside-a-terminal-offers-no-next-step + Scenario: uninstall-08 - Outside a terminal, a dry run lists what it would remove and offers no step to remove it + Given the config folder holds the user's files + When the user dry-runs an uninstall of only the config + Then the uninstall succeeds + And the dry run lists the config folder + And the dry run offers no step to remove anything + And the config folder still holds the user's files + + # With nothing to remove there is nothing to approve, so even a person at a + # terminal is offered no step. A config folder that is kept and still holds + # the user's files shows the "nothing to remove" is true. + @id:uninstall-dry-run-with-nothing-to-remove-offers-no-next-step @requires-os:linux + Scenario: uninstall-09 - At a terminal, a dry run with nothing to remove offers no step to remove it + Given the config folder holds the user's files + When the user dry-runs an uninstall that keeps everything at a terminal + Then the uninstall succeeds + And the uninstall reports that nothing would be removed + And the dry run offers no step to remove anything + And the config folder still holds the user's files + + # `--force` is a natural guess for skipping the confirmation, and the nearest + # real flag by name is `--force-dev-binaries`, which removes the binaries of a + # development build. `--force` is refused with the flag to use instead, a near + # miss of it never suggests the destructive flag, and nothing is removed. + @id:uninstall-mistyped-force-is-not-answered-with-a-destructive-flag + Scenario: uninstall-10 - A mistyped --force is refused without pointing at the flag that removes development binaries + Given the config folder holds the user's files + When the user uninstalls the config with --force + Then the uninstall is refused and told to use --yes instead + And the config folder still holds the user's files + When the user uninstalls the config with a near miss of --force + Then the error does not point at the flag that removes development binaries + And the config folder still holds the user's files + + @id:uninstall-help-shows-a-dry-run-before-the-real-run + Scenario: uninstall-11 - The uninstall help shows a dry run before the real run + When the user asks for the uninstall help + Then the help shows a dry run before the real run + + # The assistant's uninstall preview (the `rocm_command` tool of the chat + # assistant and the dashboard, called here without a model through the hidden + # `rocm mcp-call`) renders the same plan as a dry run, but it has no flag to + # offer, so it says that removal needs approval. It is the one run here that + # cannot pass --keep-binaries: the preview takes no flags and removes nothing, + # and it is made without --allow-mutation, so a call that would remove + # something is refused. + @id:uninstall-assistant-preview-says-removal-needs-approval + Scenario: uninstall-12 - The assistant's uninstall preview lists what it would remove, says removal needs approval, and removes nothing + Given the config folder holds the user's files + When the assistant previews an uninstall + Then the preview succeeds + And the preview lists the config folder + And the preview says removal needs approval + And the config folder still holds the user's files + + # The `[y/N]` prompt is shared with `rocm fix` and the SDK-overwrite prompt; + # `rocm uninstall` is the one that is cheap to isolate, so the shared helper's + # behaviour is pinned here. A prompt is only reached on a real terminal (without + # one the command is refused, see uninstall-04), so these run under a + # pseudo-terminal. An answer that is neither yes nor no used to be declined with + # no sign it was not understood, and ending the input (Ctrl-D) left the + # cancellation on the prompt line because no newline had been echoed. + @id:uninstall-unrecognized-answer-is-declined-and-said-so @requires-os:linux + Scenario: uninstall-13 - An answer that is neither yes nor no is declined, and the user is told so + Given the config folder holds the user's files + When the user is asked to confirm an uninstall on a real terminal and types an answer that is neither yes nor no + Then the user is told the answer was not understood and is treated as no + And the uninstall is reported as cancelled + And the config folder still holds the user's files + + @id:uninstall-end-of-input-cancels-on-a-fresh-line @requires-os:linux + Scenario: uninstall-14 - Ending the input at the confirmation prompt cancels on a fresh line + Given the config folder holds the user's files + When the user is asked to confirm an uninstall on a real terminal and ends the input instead of answering + Then the cancellation is reported on its own line, not on the prompt line + And the config folder still holds the user's files + + # The other side of uninstall-13: a deliberate no is not an error to explain. + @id:uninstall-plain-no-is-declined-without-complaint @requires-os:linux + Scenario: uninstall-15 - Answering no is declined without any complaint + Given the config folder holds the user's files + When the user is asked to confirm an uninstall on a real terminal and types no + Then the uninstall is reported as cancelled + And the user is not told the answer was misunderstood + And the config folder still holds the user's files diff --git a/tests/e2e-cucumber/features/update.feature b/tests/e2e-cucumber/features/update.feature index 21ad0c1a35..51070f95bd 100644 --- a/tests/e2e-cucumber/features/update.feature +++ b/tests/e2e-cucumber/features/update.feature @@ -92,3 +92,66 @@ Feature: Update report When the user checks for updates against that catalog Then the report calls the runtime the catalog spells differently up to date And the report offers no update for the four-component runtime that is ahead of the catalog + + # `rocm update` is a check: it reads the installed runtimes and the index and + # changes nothing. To tell which wheels fit a wheel runtime it needs the + # interpreter's tag, and on a host with no usable Python it used to go and get + # one: it downloaded a managed `uv`, had `uv` install Python and saved that + # Python's path in the configuration, all before printing the report. The same + # resolution runs for `rocm update --json` (the dashboard's check) and for the + # startup check that runs ahead of other commands, so a read-only look at the + # runtimes could prepare a Python. Now the check works the tag out from the + # interpreter the install would prepare, says it would prepare one, and + # prepares nothing. + # + # The report's note says the check prepared nothing, so the Then checks that + # note and the state together, as runtime-23 does for the install preview: the + # recording `uv` is reached only by going on to prepare a Python, so its log not + # existing means no preparation was attempted, and the managed tools folder and + # the configuration must be absent too. The first Then shows the check still got + # as far as a verdict, so the absences are not just a command that stopped + # early. `--json` and the startup check are pinned by the unit tests in + # `apps/rocm/src/therock/update.rs`. + # + # Linux only, for the same empty-`PATH` host as runtime-23. The canonical + # release index is served from a loopback fixture, so no network is needed. + @id:update-check-prepares-no-python @requires-os:linux + Scenario: update-09 - Checking for updates on a host with no usable Python prepares no Python + Given a canonical release pip index fixture + And a host with no usable Python and a recording uv + And a registered wheel runtime older than the index with no recorded Python + When the user checks for updates against the pip index + Then the report offers the newer runtime and says it would prepare a managed Python + And the update check prepared no Python, as it says + + # The sibling for `--dry-run`, which promises to show what `--apply` would do + # and change nothing. Its plan step resolved the same way and prepared the + # Python before the preview was printed; the apply itself still prepares what it + # needs, which the unit tests keep apart from the check. + # + # Linux only, for the same empty-`PATH` host as update-09. + @id:update-dry-run-prepares-no-python @requires-os:linux + Scenario: update-10 - Previewing an update on a host with no usable Python prepares no Python + Given a canonical release pip index fixture + And a host with no usable Python and a recording uv + And a registered wheel runtime older than the index with no recorded Python + When the user previews applying the update to that wheel runtime + Then the preview says a real install would prepare a managed Python + And the preview prepared no Python, as it says + + # An update installs from the layout its runtime was recorded in, and names no + # pin. A runtime of the earlier release stream recorded with only a family + # label (`gfx120X-all`, which that stream serves) must therefore still update: + # the newest layout needs an exact arch, but the runtime is not being moved + # there. Gating the update on a fresh install's routing would refuse it with + # "requires an exact GPU arch", though its own resolution never needed one. + # The report is the update's plan, so it names the earlier stream it reads. + # + # Linux only, for the same empty-`PATH` host as update-09. + @id:update-apply-keeps-a-grouped-family-runtime-on-its-recorded-stream @requires-os:linux + Scenario: update-11 - Previewing an update of an earlier-stream runtime recorded with a family label keeps it on that stream + Given a canonical release pip index fixture + And a host with no usable Python and a recording uv + And a registered wheel runtime of the earlier release stream with a grouped family + When the user previews applying the update to that wheel runtime + Then the preview resolves the canonical release pip index diff --git a/tests/e2e-cucumber/src/bin/fake-uv.rs b/tests/e2e-cucumber/src/bin/fake-uv.rs index 3227ae5abc..0bd0a4860e 100644 --- a/tests/e2e-cucumber/src/bin/fake-uv.rs +++ b/tests/e2e-cucumber/src/bin/fake-uv.rs @@ -14,10 +14,22 @@ //! arguments it was given (unset: nothing is written). A scenario that must //! show `uv` was never run asserts the file does not exist. //! +//! `FAKE_UV_DRAIN_STDIN` — when set, read standard input to its end before +//! answering (unset: stdin is left alone). The CLI writes the requirements to +//! `uv pip compile -` on stdin before it reads anything back, so a stand-in that +//! answers and exits without reading can make that write fail on a closed pipe, +//! depending on timing. Unset by default because a scenario that never pipes +//! anything in may leave stdin attached to something that never ends. +//! //! A Rust binary rather than a shell script so the scenarios run on Windows //! too. fn main() { + if std::env::var_os("FAKE_UV_DRAIN_STDIN").is_some() { + use std::io::Read as _; + let mut sink = Vec::new(); + let _ = std::io::stdin().read_to_end(&mut sink); + } if let Some(log) = std::env::var_os("FAKE_UV_LOG").map(std::path::PathBuf::from) { use std::io::Write as _; let args: Vec = std::env::args().skip(1).collect(); diff --git a/tests/e2e-cucumber/src/binary_path.rs b/tests/e2e-cucumber/src/binary_path.rs new file mode 100644 index 0000000000..93c959f428 --- /dev/null +++ b/tests/e2e-cucumber/src/binary_path.rs @@ -0,0 +1,100 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! How the harness names the `rocm` binary when it starts the child somewhere +//! else. +//! +//! `ROCM_CLI_BINARY` is either a path (`target/debug/rocm`, `./rocm`, an +//! absolute one) or a bare name (`rocm`) the operating system finds on `PATH`. +//! A scenario that moves the child to another working directory must pin a +//! relative path first, because it was written relative to the harness's own +//! directory. A bare name must be left alone: canonicalising it would find a +//! file called `rocm` in the harness's directory, if there is one, and run that +//! instead of the program `PATH` resolves. + +use std::io; +use std::path::PathBuf; + +/// Whether `binary` names a path rather than a bare program name: it is +/// absolute, or contains a separator (so `./rocm`, `bin/rocm` and `../x/rocm` +/// are paths, and `rocm` is not). +#[must_use] +pub fn names_a_path(binary: &str) -> bool { + std::path::Path::new(binary).is_absolute() || binary.contains(['/', std::path::MAIN_SEPARATOR]) +} + +/// The program to start for `binary` when the child runs from another directory. +/// +/// A path is made absolute with `canonicalize` (and kept as written when that +/// fails); a bare name is returned unchanged without asking. +pub fn program_for_moved_child( + binary: &str, + canonicalize: impl Fn(&str) -> io::Result, +) -> PathBuf { + if names_a_path(binary) { + canonicalize(binary).unwrap_or_else(|_| PathBuf::from(binary)) + } else { + PathBuf::from(binary) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + #[test] + fn paths_are_told_from_bare_names() { + let abs = std::env::temp_dir().join("rocm"); + for (binary, is_path) in [ + ("rocm", false), + ("rocm.exe", false), + ("./rocm", true), + ("bin/rocm", true), + ("../x/rocm", true), + (abs.to_str().expect("UTF-8 temp dir"), true), + ] { + assert_eq!(names_a_path(binary), is_path, "{binary}"); + } + } + + #[test] + fn a_path_is_canonicalised_and_a_bare_name_is_not_even_when_a_local_file_would_match() { + // A canonicalise that succeeds for everything stands for a `rocm` file + // existing in the harness's directory. + let asked = RefCell::new(Vec::new()); + let pin = |binary: &str| { + asked.borrow_mut().push(binary.to_owned()); + Ok(PathBuf::from("/pinned").join(binary)) + }; + + assert_eq!( + program_for_moved_child("rocm", pin), + PathBuf::from("rocm"), + "a bare name stays for PATH to resolve" + ); + assert!( + asked.borrow().is_empty(), + "a bare name must not be looked up on disk" + ); + for binary in ["./rocm", "bin/rocm", "../x/rocm"] { + assert_eq!( + program_for_moved_child(binary, pin), + PathBuf::from("/pinned").join(binary), + "{binary}" + ); + } + assert_eq!(asked.borrow().len(), 3); + } + + #[test] + fn a_path_that_cannot_be_canonicalised_is_kept_as_written() { + assert_eq!( + program_for_moved_child("./gone/rocm", |_| Err(io::Error::from( + io::ErrorKind::NotFound + ))), + PathBuf::from("./gone/rocm") + ); + } +} diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 21425ba99b..d63549166f 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -231,6 +231,17 @@ pub struct PlatformVersions { /// a mislabeled `--channel` flag can't misreport what's actually installed. #[serde(default, skip_serializing_if = "Option::is_none")] pub channel: Option, + /// Exact GFX target the active runtime was installed for, e.g. "gfx1100", + /// from its recorded wheel composition. `None` for a runtime that records + /// none (a tarball or legacy install). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gfx_target: Option, + /// Where the active runtime's packages came from, as recorded in its + /// registry record: the index URL and, in parentheses, the source layout + /// generation, e.g. "https://repo.amd.com/rocm/whl-multi-arch (multi-arch-v2)". + /// Read from the runtime, never from a workflow-passed label. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, } /// Collect component versions for the report from the installed managed runtime. @@ -258,20 +269,33 @@ pub fn collect_versions(runtimes_dir: Option<&std::path::Path>) -> PlatformVersi // ROCm/vLLM/lemonade versions come from the installed managed runtime. Only // available when CI provides a persistent runtimes dir (E2E_SHARED_RUNTIMES_DIR) // — mock has no runtime, and per-scenario isolated installs are gone by now. - if let Some((rocm, channel, root)) = runtimes_dir.and_then(active_runtime_install_root) { - v.rocm = Some(rocm); - v.channel = channel; + if let Some(runtime) = runtimes_dir.and_then(active_runtime_install_root) { + v.rocm = Some(runtime.version); + v.channel = runtime.channel; + v.gfx_target = runtime.gfx_target; + v.source = runtime.source; // vLLM: parse the version out of `.../site-packages/vllm-.dist-info`. - v.vllm = vllm_version_from_venv(&root); + v.vllm = vllm_version_from_venv(&runtime.root); // lemonade: `/engines/lemonade/runtime/lemond --version`. - v.lemonade = lemonade_version(&root); + v.lemonade = lemonade_version(&runtime.root); } v } -/// Read the active managed runtime's `(version, channel, install_root)` from the -/// runtimes registry. Returns `None` when the tree names no single runtime. +/// What the report reads from the active managed runtime's registry record. +#[derive(Debug)] +struct ActiveRuntime { + version: String, + channel: Option, + root: std::path::PathBuf, + gfx_target: Option, + source: Option, +} + +/// Read the active managed runtime's version, channel, install root, exact GFX +/// target and package source from the runtimes registry. Returns `None` when +/// the tree names no single runtime. /// `channel` is `None` for a registry record predating the field. /// /// Which runtime that is comes from [`crate::shared_runtime::runtime_key_to_activate`], @@ -301,9 +325,7 @@ pub fn collect_versions(runtimes_dir: Option<&std::path::Path>) -> PlatformVersi /// fallback yields an in-tree path or nothing. Where every entry is such a /// corpse this returns `None` and the report simply omits the version — absent /// reads as unknown, which is the honest outcome; a wrong version reads as fact. -fn active_runtime_install_root( - runtimes_dir: &std::path::Path, -) -> Option<(String, Option, std::path::PathBuf)> { +fn active_runtime_install_root(runtimes_dir: &std::path::Path) -> Option { let key = crate::shared_runtime::runtime_key_to_activate(runtimes_dir)?; let manifest = runtimes_dir.join("registry").join(format!("{key}.json")); let json: serde_json::Value = @@ -321,7 +343,51 @@ fn active_runtime_install_root( } else { std::path::PathBuf::from(json.get("install_root")?.as_str()?) }; - Some((version, channel, root)) + // Mirrors the CLI's `wheel_composition_device_target` + // (`apps/rocm/src/runtime_registry.rs`): a runtime installed with a family + // label records no `rocm_sdk_target`, only the `device-` extra in its + // package specs. + let gfx_target = json + .pointer("/wheel_composition/rocm_sdk_target") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .or_else(|| { + json.pointer("/wheel_composition/package_specs")? + .as_array()? + .iter() + .filter_map(serde_json::Value::as_str) + .find_map(|spec| { + let extras = spec.strip_prefix("rocm[")?.split_once(']')?.0; + extras + .split(',') + .map(str::trim) + .find_map(|extra| extra.strip_prefix("device-")) + }) + .map(str::to_owned) + }); + let source = json + .get("index_url") + .and_then(serde_json::Value::as_str) + .map(|url| { + // Same two-level fallback as the CLI's `manifest_source_layout`: a + // manifest older than the top-level field carries the generation on + // its wheel composition. + match json + .get("source_layout_generation") + .or_else(|| json.pointer("/wheel_composition/source_layout_generation")) + .and_then(serde_json::Value::as_str) + { + Some(generation) => format!("{url} ({generation})"), + None => url.to_owned(), + } + }); + Some(ActiveRuntime { + version, + channel, + root, + gfx_target, + source, + }) } /// Select the canonical aggregate wheel runtime from `rocm runtimes list` output. @@ -1210,16 +1276,126 @@ Local model engines ) .expect("write marker"); - let (version, channel, root) = - active_runtime_install_root(dir).expect("a runtime is named"); - assert_eq!(version, "7.14.0"); - assert_eq!(channel.as_deref(), Some("nightly")); + let runtime = active_runtime_install_root(dir).expect("a runtime is named"); + assert_eq!(runtime.version, "7.14.0"); + assert_eq!(runtime.channel.as_deref(), Some("nightly")); assert_eq!( - root, + runtime.root, dir.join("wheel").join("release-wheel-multi-arch-7-14-0") ); } + /// The matrix names the exact target and package source the run served on, + /// read from the runtime's own record. A record that predates either field + /// reports none: absent reads as unknown, a guess reads as fact. + #[test] + fn reports_the_active_runtimes_target_and_source_from_its_record() { + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + let dir = tmp.path(); + let key = "nightly-wheel-multi-arch-7-14-0a20260601-0123456789abcdef"; + let registry = dir.join("registry"); + std::fs::create_dir_all(®istry).expect("create registry"); + let install_root = dir.join("wheel").join(key); + std::fs::create_dir_all(&install_root).expect("create install root"); + std::fs::write( + registry.join(format!("{key}.json")), + serde_json::json!({ + "runtime_key": key, + "version": "7.14.0a20260601", + "install_root": install_root, + "channel": "nightly", + "index_url": "https://rocm.nightlies.amd.com/whl-multi-arch", + "source_layout_generation": "multi-arch-v2", + "wheel_composition": { + "source_layout_generation": "multi-arch-v2", + "package_specs": [], + "rocm_sdk_target": "gfx1100" + }, + }) + .to_string(), + ) + .expect("write manifest"); + + let runtime = active_runtime_install_root(dir).expect("one runtime"); + assert_eq!(runtime.gfx_target.as_deref(), Some("gfx1100")); + assert_eq!( + runtime.source.as_deref(), + Some("https://rocm.nightlies.amd.com/whl-multi-arch (multi-arch-v2)") + ); + + let legacy_dir = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + write_manifest( + legacy_dir.path(), + "release-wheel-gfx94x-dcgpu-7-13-0", + "7.13.0", + ); + let legacy = active_runtime_install_root(legacy_dir.path()).expect("one runtime"); + assert_eq!(legacy.gfx_target, None); + assert_eq!(legacy.source, None); + } + + /// A runtime installed with a family label records no `rocm_sdk_target`; the + /// target is the `device-` extra in its package specs, as the CLI's + /// `wheel_composition_device_target` reads it. + #[test] + fn gfx_target_falls_back_to_the_device_extra_in_package_specs() { + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + let dir = tmp.path(); + let key = "release-wheel-gfx110x-all-7-14-0"; + let registry = dir.join("registry"); + std::fs::create_dir_all(®istry).expect("create registry"); + let install_root = dir.join("wheel").join(key); + std::fs::create_dir_all(&install_root).expect("create install root"); + std::fs::write( + registry.join(format!("{key}.json")), + serde_json::json!({ + "runtime_key": key, + "version": "7.14.0", + "install_root": install_root, + "wheel_composition": { + "package_specs": ["rocm[libraries,device-gfx1100]==7.14.0"], + }, + }) + .to_string(), + ) + .expect("write manifest"); + + let runtime = active_runtime_install_root(dir).expect("one runtime"); + assert_eq!(runtime.gfx_target.as_deref(), Some("gfx1100")); + } + + /// A manifest older than the top-level `source_layout_generation` carries it + /// on its wheel composition; the source still names the generation, as the + /// CLI's `manifest_source_layout` does. + #[test] + fn source_generation_falls_back_to_the_wheel_composition() { + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + let dir = tmp.path(); + let key = "release-wheel-multi-arch-7-14-0"; + let registry = dir.join("registry"); + std::fs::create_dir_all(®istry).expect("create registry"); + let install_root = dir.join("wheel").join(key); + std::fs::create_dir_all(&install_root).expect("create install root"); + std::fs::write( + registry.join(format!("{key}.json")), + serde_json::json!({ + "runtime_key": key, + "version": "7.14.0", + "install_root": install_root, + "index_url": "https://example.test/whl", + "wheel_composition": { "source_layout_generation": "multi-arch-v2" }, + }) + .to_string(), + ) + .expect("write manifest"); + + let runtime = active_runtime_install_root(dir).expect("one runtime"); + assert_eq!( + runtime.source.as_deref(), + Some("https://example.test/whl (multi-arch-v2)") + ); + } + /// Several runtimes and no marker: report nothing rather than guess. An /// absent version reads as unknown; a wrong one reads as fact. #[test] diff --git a/tests/e2e-cucumber/src/dash_screen.rs b/tests/e2e-cucumber/src/dash_screen.rs new file mode 100644 index 0000000000..eb8a0c6c2a --- /dev/null +++ b/tests/e2e-cucumber/src/dash_screen.rs @@ -0,0 +1,301 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Pure matching over captured dashboard screens and config files. +//! +//! The cucumber binary has no unit-test harness, so the logic that the +//! dashboard scenarios depend on lives here, where a failing case can be shown. + +use serde_json::{Map, Value}; + +/// Fewest cells after the model cell of an Observe instances row. +/// +/// The cells are `TOK/S TOK/W TTFT TPOT POWER QUEUE KV%` (`instances.rs`). A +/// cell with a unit (`12.00 tok/W`) splits into more than one, so a live row +/// can have more. +pub const MIN_OBSERVE_METRIC_CELLS: usize = 7; + +/// The cells after the model cell of an Observe instances row whose model +/// starts with `model_prefix`. Free text that merely mentions the model, or a +/// row that was cut short, does not match. +/// +/// The row carries no status or port. Use [`instance_detail_matches`] for those. +pub fn observe_row_cells<'a>(line: &'a str, model_prefix: &str) -> Option> { + let mut cells = line.split_whitespace(); + cells.find(|cell| cell.starts_with(model_prefix))?; + let rest: Vec<&str> = cells.collect(); + (rest.len() >= MIN_OBSERVE_METRIC_CELLS).then_some(rest) +} + +/// The port of an endpoint URL such as `http://127.0.0.1:11435/v1`. +pub fn endpoint_port(endpoint: &str) -> Option { + let authority = endpoint.split("://").nth(1)?.split('/').next()?; + let port = authority.rsplit_once(':')?.1; + (!port.is_empty() && port.chars().all(|c| c.is_ascii_digit())).then(|| port.to_string()) +} + +/// True when the screen holds a matching instance-detail summary. +/// +/// The status badge must be `status` with `port: ` on the same line, and +/// a `model:` cell must be followed by a name starting with `model_prefix` +/// (`render_summary` in `instance_detail.rs`). +pub fn instance_detail_matches(screen: &str, model_prefix: &str, status: &str, port: &str) -> bool { + // Panel borders can touch a label (`│model:`), so treat them as separators. + let screen = strip_box_drawing(screen); + let screen = screen.as_str(); + let status_and_port = screen.lines().any(|line| { + let cells: Vec<&str> = line.split_whitespace().collect(); + cells.contains(&status) + && cells + .windows(2) + .any(|pair| pair[0] == "port:" && pair[1] == port) + }); + let model = screen.lines().any(|line| { + let mut cells = line.split_whitespace(); + cells.any(|cell| cell == "model:") + && cells + .next() + .is_some_and(|name| name.starts_with(model_prefix)) + }); + status_and_port && model +} + +/// Replace the box-drawing characters of the terminal UI with spaces. +fn strip_box_drawing(screen: &str) -> String { + screen + .chars() + .map(|c| { + if ('\u{2500}'..='\u{257f}').contains(&c) { + ' ' + } else { + c + } + }) + .collect() +} + +/// Set only the `dashboard.daemon.*` keys of `update` on `existing`, leaving +/// every other key (an active runtime, for example) as it was. +/// +/// # Panics +/// Panics when `update` has no `dashboard.daemon` object; that is a bug in the +/// calling step, not a runtime condition. +pub fn merge_dashboard_daemon_config(existing: Option, update: &Value) -> Value { + let mut root = match existing { + Some(Value::Object(map)) => map, + _ => Map::new(), + }; + let updates = update + .pointer("/dashboard/daemon") + .and_then(Value::as_object) + .expect("update must carry dashboard.daemon keys"); + let dashboard = object_entry(&mut root, "dashboard"); + let daemon = object_entry(dashboard, "daemon"); + for (key, value) in updates { + daemon.insert(key.clone(), value.clone()); + } + Value::Object(root) +} + +/// Turn Lemonade discovery off unless the config already says otherwise. +/// +/// The dashboard daemon probes `127.0.0.1:13305` by default. A scenario that +/// does not exercise discovery must not pick up a real Lemonade server that +/// happens to listen there, so the isolated config starts with discovery off. +/// A scenario that wants discovery writes `enable_lemonade` itself, and that +/// value is kept. +pub fn with_lemonade_discovery_default_off(existing: Option) -> Value { + if existing + .as_ref() + .and_then(|config| config.pointer("/dashboard/daemon/enable_lemonade")) + .is_some() + { + return existing.expect("checked above"); + } + merge_dashboard_daemon_config( + existing, + &serde_json::json!({ "dashboard": { "daemon": { "enable_lemonade": false } } }), + ) +} + +/// The object stored under `key`, replacing a missing or non-object value. +fn object_entry<'a>(parent: &'a mut Map, key: &str) -> &'a mut Map { + let slot = parent + .entry(key.to_string()) + .or_insert_with(|| Value::Object(Map::new())); + if !slot.is_object() { + *slot = Value::Object(Map::new()); + } + slot.as_object_mut().expect("slot is an object") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const MANAGED_ROW: &str = + "Qwen3-0.6B-GGUF — — — — — — —"; + const DETAIL: &str = "│ READY id: svc-1 port: 11435 tp: 1 │\n\ + │ model: Qwen3-0.6B-GGUF gpus: 0 tok/W: — │"; + + #[test] + fn observe_row_matches_a_full_managed_row() { + let cells = observe_row_cells(MANAGED_ROW, "Qwen3-0.6B").expect("row matches"); + assert_eq!(cells.len(), MIN_OBSERVE_METRIC_CELLS); + let with_units = "Qwen3-0.6B-GGUF 41.2 tok/s 0.52 tok/W 50ms 12ms 80W 0/0 3%"; + assert!(observe_row_cells(with_units, "Qwen3-0.6B").is_some()); + } + + #[test] + fn observe_row_rejects_other_model_truncated_row_and_prose() { + let other = MANAGED_ROW.replace("Qwen3-0.6B-GGUF", "Phi-3-mini"); + assert!(observe_row_cells(&other, "Qwen3-0.6B").is_none()); + let truncated = "Qwen3-0.6B-GGUF — — —"; + assert!(observe_row_cells(truncated, "Qwen3-0.6B").is_none()); + let prose = "loaded Qwen3-0.6B-GGUF on the default port"; + assert!(observe_row_cells(prose, "Qwen3-0.6B").is_none()); + } + + #[test] + fn endpoint_port_reads_the_serving_port() { + assert_eq!( + endpoint_port("http://127.0.0.1:11435/v1").as_deref(), + Some("11435") + ); + assert_eq!(endpoint_port("http://localhost/v1"), None); + assert_eq!(endpoint_port("not a url"), None); + } + + #[test] + fn detail_matches_ready_managed_model_on_the_served_port() { + assert!(instance_detail_matches( + DETAIL, + "Qwen3-0.6B", + "READY", + "11435" + )); + } + + /// Captured from a real run: the panel border touches the `model:` label. + const REAL_DETAIL: &str = "│ ╭─╮Node╭─╮Instance · lemonade-qwen3-0-6b-gguf-1792015518018╭───╮──────╮ │\n\ + │ │ │ READY id: lemonade-qwen3-0-6b-gguf-1792015518018 port: 11435 tp: 0 │ │ │\n\ + │ │ — │model: Qwen3-0.6B-GGUF gpus: - tok/W: - gen: - │ │ │\n\ + │ │ │partition: - quantization: - vram: 0 / 0 MiB │ │ │"; + + #[test] + fn detail_matches_the_screen_a_real_run_rendered() { + assert!(instance_detail_matches( + REAL_DETAIL, + "Qwen3-0.6B", + "READY", + "11435" + )); + assert!(!instance_detail_matches( + REAL_DETAIL, + "Qwen3-0.6B", + "READY", + "11436" + )); + assert!(!instance_detail_matches( + &REAL_DETAIL.replace("READY", "STARTING"), + "Qwen3-0.6B", + "READY", + "11435" + )); + } + + #[test] + fn detail_rejects_wrong_status_port_or_model() { + let starting = DETAIL.replace("READY", "STARTING"); + assert!(!instance_detail_matches( + &starting, + "Qwen3-0.6B", + "READY", + "11435" + )); + let stopped = DETAIL.replace("READY", "STOPPED"); + assert!(!instance_detail_matches( + &stopped, + "Qwen3-0.6B", + "READY", + "11435" + )); + let other_port = DETAIL.replace("11435", "8000"); + assert!(!instance_detail_matches( + &other_port, + "Qwen3-0.6B", + "READY", + "11435" + )); + let other_model = DETAIL.replace("Qwen3-0.6B-GGUF", "Phi-3-mini"); + assert!(!instance_detail_matches( + &other_model, + "Qwen3-0.6B", + "READY", + "11435" + )); + // A status word elsewhere on screen does not stand in for the badge on + // the port line. + let elsewhere = "READY\n│ id: svc-1 port: 11435 tp: 1 │\n│ model: Qwen3-0.6B-GGUF │"; + assert!(!instance_detail_matches( + elsewhere, + "Qwen3-0.6B", + "READY", + "11435" + )); + } + + #[test] + fn dashboard_config_merge_keeps_other_keys() { + let existing = json!({ + "runtime": { "active": "therock-7.1" }, + "dashboard": { "theme": "dark", "daemon": { "lemonade_port": 1 } } + }); + let update = json!({ + "dashboard": { "daemon": { "enable_lemonade": false } } + }); + let merged = merge_dashboard_daemon_config(Some(existing), &update); + assert_eq!(merged["runtime"]["active"], "therock-7.1"); + assert_eq!(merged["dashboard"]["theme"], "dark"); + assert_eq!(merged["dashboard"]["daemon"]["lemonade_port"], 1); + assert_eq!(merged["dashboard"]["daemon"]["enable_lemonade"], false); + } + + #[test] + fn lemonade_discovery_defaults_off_and_keeps_other_keys() { + let existing = json!({ "runtime": { "active": "therock-7.1" } }); + for start in [None, Some(existing.clone()), Some(json!("garbage"))] { + let merged = with_lemonade_discovery_default_off(start); + assert_eq!(merged["dashboard"]["daemon"]["enable_lemonade"], false); + } + let merged = with_lemonade_discovery_default_off(Some(existing)); + assert_eq!(merged["runtime"]["active"], "therock-7.1"); + } + + #[test] + fn lemonade_discovery_default_keeps_an_explicit_choice() { + for enabled in [true, false] { + let existing = json!({ + "dashboard": { "daemon": { "enable_lemonade": enabled, "lemonade_port": 9 } } + }); + let merged = with_lemonade_discovery_default_off(Some(existing)); + assert_eq!(merged["dashboard"]["daemon"]["enable_lemonade"], enabled); + assert_eq!(merged["dashboard"]["daemon"]["lemonade_port"], 9); + } + } + + #[test] + fn dashboard_config_merge_starts_from_nothing_or_garbage() { + let update = json!({ "dashboard": { "daemon": { "lemonade_port": 7 } } }); + for existing in [ + None, + Some(json!("not an object")), + Some(json!({"dashboard": 3})), + ] { + let merged = merge_dashboard_daemon_config(existing, &update); + assert_eq!(merged["dashboard"]["daemon"]["lemonade_port"], 7); + } + } +} diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs new file mode 100644 index 0000000000..fb3a513e6f --- /dev/null +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -0,0 +1,1510 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Which device memory pool the serve drain-wait judges, and the floor it +//! demands of it. +//! +//! The wait between serve scenarios (`wait_for_free_vram` in the serving +//! steps) exists to see a just-killed engine hand its device memory back. It +//! has to look at the pool the engine allocates from. On a discrete card that +//! is VRAM. On an APU such as Strix Halo the VRAM total is only the fixed BIOS +//! carveout (512 MiB on the CI hosts) while the engine draws on GTT-backed +//! system memory (about 100 GiB there), so a floor taken from the carveout can +//! never be met and every GPU serve spent its whole drain deadline waiting. +//! `.github/scripts/gpu-preflight.sh` makes the same switch for the same reason. + +use std::io::Read; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +/// Upper bound on the free-memory floor (MiB). +/// +/// Sized so the largest single +/// scenario model can allocate its engine's memory share without tripping its +/// startup check: Qwen3.6-27B plus vLLM's default fraction-of-total KV +/// reservation needs the MI300X mostly clear. Only a data-center GPU has this +/// much, so on smaller pools the floor is capped to 90% of the pool total (see +/// [`required_free_mib`]) — otherwise the check could never pass. Note the cap +/// no longer sits above what vLLM asks for: rocm-cli used to pin utilization +/// below it, and now defers to vLLM's own (higher) default, so the wait has +/// little headroom left and can time out on a device that is merely holding +/// display memory. The wait is best-effort and the serve proceeds regardless, +/// so this costs time rather than correctness; sizing a deliberate margin is +/// follow-up work. +pub const MAX_FREE_FLOOR_MIB: u64 = 150_000; + +/// A VRAM total below this is an APU's BIOS carveout, not private device memory. +/// +/// The engine's allocations come out of GTT instead. It is the +/// 8 GiB floor the Strix Halo preflight lanes pass as +/// `GPU_PREFLIGHT_MIN_FREE_GIB`, which `gpu-preflight.sh` compares the VRAM +/// total against for the same purpose. Every discrete card the lanes run on is +/// far above it (48 GiB and up). The xtask test +/// `the_serve_drain_carveout_limit_is_the_apu_lanes_preflight_floor` fails if +/// this stops matching the lanes' floor. +pub const CARVEOUT_MAX_MIB: u64 = 8 * 1024; + +/// Whether a VRAM total is an APU's carveout rather than private device memory. +/// +/// Both the decision to read GTT at all ([`sample_device_memory`]) and the +/// choice of pool ([`select_drain_target`]) go through this, so they cannot +/// disagree. +#[must_use] +pub const fn is_carveout(vram_total_mib: u64) -> bool { + vram_total_mib < CARVEOUT_MAX_MIB +} + +/// One attempt to read from a device tool. +/// +/// `Absent` and `Unreadable` are kept apart on purpose. With no tool installed +/// there is nothing to wait on (a mock or local run). A tool that is present +/// but stalled, failed, or printed something unparseable is a device that +/// could not be read right now, which must never be mistaken for either "no +/// tooling" or "drained". +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Probe { + /// The tool is not installed. + Absent, + /// The tool is there but gave no usable reading. + Unreadable, + /// The tool ran and exited successfully, but its output was not understood. + /// Like `Unreadable` for every decision, it also carries what the tool + /// printed (see [`output_hint`]) so a log can show what the parser met. + Unparsed(String), + Read(T), +} + +impl Probe { + /// Fall back to `next` when this probe produced no reading. + /// + /// A reading from either wins. Without one, the result is `Absent` only if + /// every tool tried was absent; if either tool ran and printed something not + /// understood the result keeps what it printed (`Unparsed`, the first tool's + /// if both); otherwise any tool that was present and gave nothing makes it + /// `Unreadable`. + #[must_use] + pub fn or_else(self, next: impl FnOnce() -> Self) -> Self { + if matches!(self, Self::Read(_)) { + return self; + } + match (self, next()) { + (_, read @ Self::Read(_)) => read, + (Self::Absent, Self::Absent) => Self::Absent, + (Self::Unparsed(hint), _) | (_, Self::Unparsed(hint)) => Self::Unparsed(hint), + _ => Self::Unreadable, + } + } +} + +impl Probe { + /// Parse a tool's stdout with `f`. Output `f` rejects is not `Unreadable` + /// but `Unparsed`, carrying the first line of it (see [`output_hint`]) so the + /// log says what the tool printed; `source` names the command. + #[must_use] + pub fn parse_text(self, source: &str, f: impl FnOnce(&str) -> Option) -> Probe { + match self { + Self::Absent => Probe::Absent, + Self::Unreadable => Probe::Unreadable, + Self::Unparsed(hint) => Probe::Unparsed(hint), + Self::Read(text) => { + f(&text).map_or_else(|| Probe::Unparsed(output_hint(source, &text)), Probe::Read) + } + } + } +} + +/// The longest first line of tool output [`output_hint`] repeats. +pub const HINT_MAX_CHARS: usize = 120; + +/// What a tool printed, as one bounded, printable line for a log. +/// +/// It is `source`, then the first non-blank line of `text` with anything outside +/// printable ASCII replaced by `?` and cut to [`HINT_MAX_CHARS`] characters. +/// Meant for the header row of a CSV a parser could not match, so the next run +/// shows the real column names. +#[must_use] +pub fn output_hint(source: &str, text: &str) -> String { + let Some(line) = text.lines().map(str::trim).find(|line| !line.is_empty()) else { + return format!("{source} printed no output"); + }; + let clean: String = line + .chars() + .map(|c| { + if c.is_ascii_graphic() || c == ' ' { + c + } else { + '?' + } + }) + .collect(); + let shown = if clean.chars().count() > HINT_MAX_CHARS { + format!( + "{}...", + clean.chars().take(HINT_MAX_CHARS).collect::() + ) + } else { + clean + }; + format!("{source} printed {shown:?}") +} + +/// One memory pool's size and free space, in MiB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Pool { + pub total_mib: u64, + pub free_mib: u64, +} + +impl Pool { + #[must_use] + pub const fn new(total_mib: u64, free_mib: u64) -> Self { + Self { + total_mib, + free_mib, + } + } +} + +/// What the device tooling reported for GPU 0. +/// +/// `gtt` is `None` when GTT was not read: a discrete card, or a host with no +/// tool that reports it. A GTT read +/// that failed never gets here (the sample is `Unreadable`, see +/// [`sample_device_memory`]), so every pool here has a non-zero total, VRAM and +/// GTT alike. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeviceMemory { + pub vram: Pool, + pub gtt: Option, +} + +/// Which pool a [`DrainTarget`] was read from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolKind { + Vram, + Gtt, +} + +/// The pool to wait on, its current state, and the floor it must reach. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DrainTarget { + pub kind: PoolKind, + pub total_mib: u64, + pub free_mib: u64, + pub floor_mib: u64, +} + +impl DrainTarget { + /// Whether the pool has handed back enough memory. + #[must_use] + pub const fn drained(&self) -> bool { + self.free_mib >= self.floor_mib + } + + /// `" MiB free of MiB[ GTT], floor MiB"`. VRAM is the + /// unlabelled default, so the line keeps the shape it always had on discrete + /// cards; only the GTT pool is named. + #[must_use] + pub fn describe(&self) -> String { + let pool = match self.kind { + PoolKind::Vram => "", + PoolKind::Gtt => " GTT", + }; + format!( + "{} MiB free of {} MiB{pool}, floor {} MiB", + self.free_mib, self.total_mib, self.floor_mib + ) + } +} + +/// The free-memory floor to wait for on a pool of `total_mib`: the +/// model-sized ceiling, but never more than 90% of the pool. +/// +/// The 90% cap is a tolerance, not a demand for an empty pool: on the CI Strix +/// Halo hosts' ~100 GiB GTT pool (102 400 MiB, floor 92 160 MiB) about 10 GiB +/// can still be held when the wait ends as drained. That is the accepted cost +/// of a floor that a pool with some memory permanently in use can still meet. +#[must_use] +pub fn required_free_mib(total_mib: u64) -> u64 { + MAX_FREE_FLOOR_MIB.min(total_mib / 100 * 90) +} + +/// What one sample means for the drain wait. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Judgement { + /// No device tool is installed: nothing to wait on (mock/local run). + NoTooling, + /// A tool is present but gave no usable reading. This is not "no tooling" + /// and not "drained": the wait keeps trying. + Unreadable, + /// As `Unreadable`, and the tool's output was not understood: what it + /// printed, for the report (see [`output_hint`]). + Unparsed(String), + /// A reading, and the pool and floor it is held to. + Target(DrainTarget), +} + +/// Turn a sample into what the wait does with it. +#[must_use] +pub fn judge(sample: Probe) -> Judgement { + match sample { + Probe::Absent => Judgement::NoTooling, + Probe::Unreadable => Judgement::Unreadable, + Probe::Unparsed(hint) => Judgement::Unparsed(hint), + Probe::Read(memory) => Judgement::Target(select_drain_target(memory)), + } +} + +/// Consecutive samples with no reading, from a tool that is installed and has +/// never once produced one, after which the wait stops. +/// +/// A tool that never reads is broken or wedged, and waiting out the whole +/// deadline on every serve would add that deadline to each of them. A tool that +/// has read before is not given up on: its next sample may well succeed. +pub const UNREAD_SAMPLES_BEFORE_GIVING_UP: u32 = 3; + +/// The least time worth starting a probe with. With less than this left before +/// the deadline the wait ends on what it has rather than spawn a probe that +/// would be killed before it could answer. +pub const MIN_PROBE_BUDGET: Duration = Duration::from_secs(1); + +/// How long one probe may run: [`SMI_PROBE_TIMEOUT`], but never past `deadline`. +/// +/// A sample makes up to three probes in a row, so without this the last sample +/// could run several probe timeouts beyond the wait's deadline. +#[must_use] +pub fn probe_budget(now: Instant, deadline: Instant) -> Duration { + SMI_PROBE_TIMEOUT.min(deadline.saturating_duration_since(now)) +} + +/// Whether a sample started at `now` has its probes held to less than a full +/// [`SMI_PROBE_TIMEOUT`] by `deadline` (see [`DrainWait::observe_truncated`]). +#[must_use] +pub fn sample_is_truncated(now: Instant, deadline: Instant) -> bool { + probe_budget(now, deadline) < SMI_PROBE_TIMEOUT +} + +/// How the drain wait ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Verdict { + /// No device tool is installed (mock/local run): nothing was waited on. + NoTooling, + /// The pool handed back enough memory. + Drained(DrainTarget), + /// The deadline passed with the pool still below its floor. + NotDrained(DrainTarget), + /// No usable reading at the end: `samples` samples in a row gave none, and + /// `last_reading` is the most recent one the wait did get, if any. `hint` is + /// what a tool printed that was not understood, when one did (the latest + /// such sample, kept until a reading arrives). This is not "not drained": + /// the harness did not observe that. + Unreadable { + last_reading: Option, + samples: u32, + hint: Option, + }, +} + +/// The drain wait's decision state, fed one [`Judgement`] per sample. +#[derive(Debug, Default)] +pub struct DrainWait { + samples: u32, + unread_in_a_row: u32, + last_reading: Option, + hint: Option, +} + +impl DrainWait { + /// Take the next sample. `Some` ends the wait; `None` means keep sampling. + pub fn observe(&mut self, judgement: Judgement) -> Option { + let first = self.samples == 0; + self.samples += 1; + match judgement { + Judgement::NoTooling if first => Some(Verdict::NoTooling), + Judgement::Target(target) => { + self.unread_in_a_row = 0; + self.last_reading = Some(target); + // A reading shows the tool's output is understood after all. + self.hint = None; + target.drained().then_some(Verdict::Drained(target)) + } + Judgement::Unparsed(hint) => { + self.hint = Some(hint); + self.unread() + } + // A tool that vanishes mid-wait is a sample nobody could read. + Judgement::NoTooling | Judgement::Unreadable => self.unread(), + } + } + + /// As [`observe`](Self::observe), for a sample taken when the deadline had + /// already cut its probes short (`truncated`, see [`probe_budget`]). + /// + /// A truncated sample that gave no reading says nothing about the tool: it + /// was stopped by the deadline, not stalled. Counting it would let a probe + /// killed in the last seconds replace a below-floor reading with + /// "unreadable" at the deadline. It is ignored, so the wait ends on the last + /// real reading. A truncated sample that does read counts as usual. + pub fn observe_truncated(&mut self, judgement: Judgement, truncated: bool) -> Option { + if truncated && matches!(judgement, Judgement::Unreadable | Judgement::Unparsed(_)) { + return None; + } + self.observe(judgement) + } + + fn unread(&mut self) -> Option { + self.unread_in_a_row += 1; + (self.last_reading.is_none() && self.unread_in_a_row >= UNREAD_SAMPLES_BEFORE_GIVING_UP) + .then(|| self.unreadable()) + } + + /// The verdict when the deadline passes: not drained if the last sample was + /// a reading, unreadable if it was not. + #[must_use] + pub fn timed_out(&self) -> Verdict { + match self.last_reading { + Some(target) if self.unread_in_a_row == 0 => Verdict::NotDrained(target), + _ => self.unreadable(), + } + } + + fn unreadable(&self) -> Verdict { + Verdict::Unreadable { + last_reading: self.last_reading, + samples: self.unread_in_a_row, + hint: self.hint.clone(), + } + } +} + +/// The one-line `device state:` report for a verdict, after `elapsed_secs` of waiting. +/// +/// It says only what the harness observed: "NOT drained" is for a reading below +/// its floor, never for a sample that gave none, and a carveout-sized VRAM pool +/// judged without GTT data says why that cannot show whether the engine's +/// memory came back. +#[must_use] +pub fn describe_verdict(verdict: Verdict, elapsed_secs: u64) -> String { + match verdict { + Verdict::NoTooling => "device state: no GPU tooling (mock/local run)".to_owned(), + Verdict::Drained(target) => format!("device state: drained ({})", target.describe()), + Verdict::NotDrained(target) => { + let hint = if target.kind == PoolKind::Vram && is_carveout(target.total_mib) { + "; only the BIOS carveout was read because no GTT reading was available (is \ + rocm-smi installed?), so this wait could not follow the memory the engine uses" + } else { + "" + }; + format!( + "device state: NOT drained after {elapsed_secs}s ({}) — a previous engine may \ + still be holding the GPU{hint}", + target.describe() + ) + } + Verdict::Unreadable { + last_reading: None, + samples, + hint, + } => format!( + "device state: could not read device memory ({samples} samples in {elapsed_secs}s; \ + the GPU tool is installed but gave no usable reading{})", + hint_suffix(hint.as_deref()) + ), + Verdict::Unreadable { + last_reading: Some(target), + hint, + .. + } => format!( + "device state: could not read device memory at the end of {elapsed_secs}s (last \ + reading: {}{})", + target.describe(), + hint_suffix(hint.as_deref()) + ), + } +} + +/// `"; "` for a report line, or nothing without one. +fn hint_suffix(hint: Option<&str>) -> String { + hint.map_or_else(String::new, |hint| format!("; {hint}")) +} + +/// Choose the pool the engine allocates from and the floor to hold it to. +/// +/// A VRAM total that [`is_carveout`] means the GTT pool is used, when one was +/// read. Without GTT data the answer is VRAM, which is what every host got +/// before this choice existed; [`sample_device_memory`] only leaves GTT out +/// when the host has no tool that reports it, never when a read failed. +#[must_use] +pub fn select_drain_target(memory: DeviceMemory) -> DrainTarget { + let (kind, pool) = match memory.gtt { + Some(gtt) if is_carveout(memory.vram.total_mib) => (PoolKind::Gtt, gtt), + _ => (PoolKind::Vram, memory.vram), + }; + DrainTarget { + kind, + total_mib: pool.total_mib, + free_mib: pool.free_mib, + floor_mib: required_free_mib(pool.total_mib), + } +} + +/// Read `(total, free)` in MiB from `rocm-smi --showmeminfo --csv` output. +/// +/// `total_column` and `used_column` are substrings of the two header +/// names (`Total Memory` and `Total Used Memory` for VRAM; the `GTT ` forms for +/// GTT); the first data row is the first GPU. Values are bytes in the output. +/// +/// The header is the first line that names both columns, not the first line: +/// the tool can print warning lines before the CSV, and taking one of those as +/// the header would make every sample unreadable. +#[must_use] +pub fn parse_rocm_smi_csv_pool(text: &str, total_column: &str, used_column: &str) -> Option { + let mut lines = text.lines(); + let (total_idx, used_idx) = lines.find_map(|line| { + let cols: Vec<&str> = line.split(',').collect(); + let total_idx = cols.iter().position(|c| c.contains(total_column))?; + let used_idx = cols.iter().position(|c| c.contains(used_column))?; + Some((total_idx, used_idx)) + })?; + let row = lines.next()?; + let vals: Vec<&str> = row.split(',').collect(); + let total: u64 = vals.get(total_idx)?.trim().parse().ok()?; + let used: u64 = vals.get(used_idx)?.trim().parse().ok()?; + let mib = 1024 * 1024; + Some(Pool::new(total / mib, total.saturating_sub(used) / mib)) +} + +/// Read VRAM from `amd-smi metric -m` output. +/// +/// The `TOTAL_VRAM:` and `FREE_VRAM:` lines look like `TOTAL_VRAM: 196592 MB`; +/// the number in front of the unit is taken as MiB. The first GPU's lines win. A +/// zero total parses here; it is [`sample_device_memory`] that treats it as a +/// failed read. +#[must_use] +pub fn parse_amd_smi_vram(text: &str) -> Option { + let field = |name: &str| { + text.lines() + .find_map(|line| line.trim().strip_prefix(name)) + .and_then(|value| value.split_whitespace().next()) + .and_then(|number| number.parse::().ok()) + }; + Some(Pool::new(field("TOTAL_VRAM:")?, field("FREE_VRAM:")?)) +} + +/// Assemble one device-memory sample from a VRAM probe and a GTT probe. +/// +/// `gtt` runs only when the VRAM total [`is_carveout`], so a discrete-card host +/// never makes the extra call. On a carveout host the GTT reading is what the +/// engine's memory comes out of, so: +/// +/// - a GTT reading makes the sample; +/// - a GTT probe that ran but gave nothing usable (a stall, a failure, output +/// not understood, or a zero total, which is a failed read rather than an +/// empty pool) makes the whole sample `Unreadable` (`Unparsed`, with what the +/// tool printed, when it ran but its output was not understood). Judging the tiny +/// carveout instead could report "drained" while the engine still holds its +/// GTT-backed memory. `gpu-preflight.sh` treats a zero GTT total the same way, +/// as an unreadable sample to retry; +/// - no tool that reports GTT at all (`Absent`) leaves `gtt` empty, and the +/// wait judges VRAM as it did before GTT was considered. +pub fn sample_device_memory( + vram: impl FnOnce() -> Probe, + gtt: impl FnOnce() -> Probe, +) -> Probe { + let vram = match vram() { + // A zero total is a failed read, not a pool: its floor would be zero and + // every sample would count as drained. + Probe::Read(pool) if pool.total_mib > 0 => pool, + Probe::Read(_) | Probe::Unreadable => return Probe::Unreadable, + Probe::Unparsed(hint) => return Probe::Unparsed(hint), + Probe::Absent => return Probe::Absent, + }; + if !is_carveout(vram.total_mib) { + return Probe::Read(DeviceMemory { vram, gtt: None }); + } + match gtt() { + Probe::Read(pool) if pool.total_mib > 0 => Probe::Read(DeviceMemory { + vram, + gtt: Some(pool), + }), + Probe::Read(_) | Probe::Unreadable => Probe::Unreadable, + Probe::Unparsed(hint) => Probe::Unparsed(hint), + Probe::Absent => Probe::Read(DeviceMemory { vram, gtt: None }), + } +} + +/// How long one `amd-smi` or `rocm-smi` call may run before it is an unreadable sample. +/// +/// These tools can stall on a wedged driver, and the drain wait is best-effort: +/// a stalled probe must cost one sample, not the worker. +/// +/// This is a behaviour change on discrete lanes: the previous probe ran +/// `amd-smi metric -m` with no timeout, so a slow call (an 8-GPU `metric` call +/// can be slow) simply took as long as it took, and a wedged one hung the +/// worker. Now `amd-smi` is cut off at this bound, or at the time left before +/// the deadline, and the `rocm-smi` fallback is asked next with its own bound, +/// so a sample costs at most two bounds and still gets a reading whenever +/// `rocm-smi` answers in time. It is not unbounded either way, and a tool that +/// never answers is reported as unreadable after a few samples. +pub const SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +const SMI_POLL_INTERVAL: Duration = Duration::from_millis(25); + +/// How long to wait for the reader after the child has exited successfully. +/// +/// The output is already in the pipe by then and the reader only has to be +/// scheduled to hand it over, so the wait ends the moment it does and the bound +/// matters only when it cannot: a loaded runner that has not run the reader +/// thread yet, where a short bound would turn a good sample into an unreadable +/// one (and three of those in a row make the wait give up). A grandchild that +/// inherited the pipe can keep it open for good, so past this bound the reader +/// is abandoned rather than joined. The bound is generous because the cost of +/// that case, one grace per probe, is small next to [`SMI_PROBE_TIMEOUT`] and +/// the probes here do not fork helpers that outlive them. After a kill there is +/// no wait at all: a probe that outlived its deadline is unreadable at once. +const SMI_DRAIN_GRACE: Duration = Duration::from_secs(2); + +/// Run `command` and return its stdout if it exits successfully within `timeout`. +/// +/// A program that is not installed is [`Probe::Absent`]. Any other spawn +/// failure, a non-zero exit, or a timeout is [`Probe::Unreadable`]; on a +/// timeout the child is killed and reaped before this returns. +/// +/// Only the direct child is killed. A grandchild that inherited the stdout +/// pipe keeps the reader thread blocked in `read_to_end` until it exits, so a +/// timed-out probe can leak one thread. The probes here (`amd-smi`, +/// `rocm-smi`) do not fork helpers that outlive them, and the drain wait makes +/// a handful of calls, so this is accepted rather than guarded. +/// +/// Written against `std` alone, so it repeats the spawn, poll, kill and reap +/// steps of `rocm_core::process::run_with_timeout` on purpose: `rocm-core` is +/// only a dev-dependency of this crate (see its entry in `Cargo.toml`) and this +/// module is library code, and a normal dependency would be a new first-party +/// edge that `cargo xtask check-crate-edges` makes a reviewed addition. The two +/// can drift, so a change to either one's kill or drain behaviour should look at +/// the other. +pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe { + let mut child = match command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + { + Ok(child) => child, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Probe::Absent, + Err(_) => return Probe::Unreadable, + }; + let Some(mut pipe) = child.stdout.take() else { + let _ = child.kill(); + let _ = child.wait(); + return Probe::Unreadable; + }; + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let mut bytes = Vec::new(); + let _ = pipe.read_to_end(&mut bytes); + let _ = sender.send(bytes); + }); + + let deadline = Instant::now() + timeout; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() < deadline => thread::sleep(SMI_POLL_INTERVAL), + _ => { + let _ = child.kill(); + let _ = child.wait(); + return Probe::Unreadable; + } + } + }; + if !status.success() { + return Probe::Unreadable; + } + output_after_clean_exit(|grace| receiver.recv_timeout(grace).ok()) +} + +/// The reading from a probe that exited successfully, given a way to wait for +/// the reader's bytes for at most the grace it is passed. A function of the wait +/// so a test can pin the bound that is offered without sleeping. +fn output_after_clean_exit( + wait_for_reader: impl FnOnce(Duration) -> Option>, +) -> Probe { + wait_for_reader(SMI_DRAIN_GRACE).map_or(Probe::Unreadable, |bytes| { + Probe::Read(String::from_utf8_lossy(&bytes).into_owned()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The pools `gpu-preflight.sh` printed on a Strix Halo lane host + /// (`VRAM 203321344/536870912, GTT 41787392/107374182400`, used/total in + /// bytes), and the same reading in MiB. + fn strix_halo() -> DeviceMemory { + DeviceMemory { + vram: Pool { + total_mib: 512, + free_mib: 318, + }, + gtt: Some(Pool { + total_mib: 102_400, + free_mib: 102_360, + }), + } + } + + fn mi300x() -> DeviceMemory { + DeviceMemory { + vram: Pool { + total_mib: 196_592, + free_mib: 196_309, + }, + gtt: Some(Pool { + total_mib: 1_000_000, + free_mib: 1_000_000, + }), + } + } + + #[test] + fn a_carveout_host_is_judged_by_gtt_and_a_healthy_strix_halo_counts_as_drained() { + let target = select_drain_target(strix_halo()); + assert_eq!(target.kind, PoolKind::Gtt); + assert_eq!(target.total_mib, 102_400); + assert_eq!(target.floor_mib, 92_160); + assert!( + target.drained(), + "an idle Strix Halo host must not wait out the drain deadline: {target:?}" + ); + } + + #[test] + fn a_carveout_host_whose_gtt_is_mostly_held_is_not_drained() { + let mut memory = strix_halo(); + memory.gtt = Some(Pool { + total_mib: 102_400, + free_mib: 60_000, + }); + let target = select_drain_target(memory); + assert_eq!(target.kind, PoolKind::Gtt); + assert!(!target.drained(), "{target:?}"); + } + + #[test] + fn a_discrete_card_is_still_judged_by_vram_with_the_model_sized_cap() { + let target = select_drain_target(mi300x()); + assert_eq!(target.kind, PoolKind::Vram); + assert_eq!(target.total_mib, 196_592); + assert_eq!(target.floor_mib, MAX_FREE_FLOOR_MIB); + assert!(target.drained()); + } + + #[test] + fn a_discrete_card_holding_most_of_its_vram_is_not_drained() { + let mut memory = mi300x(); + memory.vram.free_mib = 40_000; + assert!(!select_drain_target(memory).drained()); + } + + #[test] + fn a_host_without_gtt_data_keeps_the_vram_behaviour() { + let mut memory = strix_halo(); + memory.gtt = None; + let target = select_drain_target(memory); + assert_eq!(target.kind, PoolKind::Vram); + assert_eq!( + target.floor_mib, 450, + "the floor the CI log showed for the carveout" + ); + assert!(!target.drained()); + } + + #[test] + fn a_vram_total_at_the_carveout_limit_is_private_memory() { + let mut memory = strix_halo(); + memory.vram = Pool { + total_mib: CARVEOUT_MAX_MIB, + free_mib: CARVEOUT_MAX_MIB, + }; + assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + assert!(is_carveout(CARVEOUT_MAX_MIB - 1)); + assert!(!is_carveout(CARVEOUT_MAX_MIB)); + } + + /// Run `sample_device_memory` with canned probes, reporting whether the GTT + /// probe was consulted. + fn sample(vram: Probe, gtt: Probe) -> (Probe, bool) { + let mut gtt_asked = false; + let result = sample_device_memory( + || vram, + || { + gtt_asked = true; + gtt + }, + ); + (result, gtt_asked) + } + + #[test] + fn a_carveout_host_reads_gtt_and_the_sample_carries_it() { + let (result, gtt_asked) = sample( + Probe::Read(Pool::new(512, 318)), + Probe::Read(Pool::new(102_400, 102_360)), + ); + assert!(gtt_asked, "a carveout host must have its GTT read"); + assert_eq!(result, Probe::Read(strix_halo())); + } + + #[test] + fn a_discrete_host_never_makes_the_gtt_call() { + let (result, gtt_asked) = sample( + Probe::Read(Pool::new(196_592, 196_309)), + Probe::Read(Pool::new(1_000_000, 1_000_000)), + ); + assert!( + !gtt_asked, + "GTT must not be read when VRAM is private memory" + ); + let Probe::Read(memory) = result else { + panic!("a readable VRAM probe makes a sample: {result:?}"); + }; + assert_eq!(memory.gtt, None); + assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + } + + #[test] + fn a_carveout_host_whose_gtt_probe_failed_is_unreadable_not_judged_by_vram() { + // VRAM alone would read as "drained" here (free is above its floor), so + // falling back to it would hide a GTT-backed allocation still held. + let (result, gtt_asked) = sample(Probe::Read(Pool::new(512, 500)), Probe::Unreadable); + assert!(gtt_asked); + assert_eq!(result, Probe::Unreadable); + } + + #[test] + fn a_zero_gtt_total_is_a_failed_reading_not_a_pool() { + let (result, _) = sample( + Probe::Read(Pool::new(512, 500)), + Probe::Read(Pool::new(0, 0)), + ); + assert_eq!(result, Probe::Unreadable); + } + + #[test] + fn a_zero_vram_total_is_a_failed_reading_not_a_drained_host() { + // With no GTT tool the sample would otherwise be judged by a VRAM pool + // whose floor is zero, which is drained by definition. + let (result, gtt_asked) = sample(Probe::Read(Pool::new(0, 0)), Probe::Absent); + assert_eq!(result, Probe::Unreadable); + assert!(!gtt_asked); + } + + #[test] + fn a_carveout_host_with_no_gtt_tool_keeps_the_vram_behaviour() { + let (result, gtt_asked) = sample(Probe::Read(Pool::new(512, 318)), Probe::Absent); + assert!(gtt_asked); + let Probe::Read(memory) = result else { + panic!("a host without a GTT tool still has a VRAM sample: {result:?}"); + }; + assert_eq!(memory.gtt, None); + assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + } + + #[test] + fn a_vram_probe_that_is_absent_or_unreadable_is_passed_through_without_asking_for_gtt() { + assert_eq!( + sample(Probe::Absent, Probe::Read(Pool::new(102_400, 102_360))), + (Probe::Absent, false) + ); + assert_eq!( + sample(Probe::Unreadable, Probe::Read(Pool::new(102_400, 102_360))), + (Probe::Unreadable, false) + ); + } + + #[test] + fn an_absent_tool_is_no_tooling_but_an_unreadable_one_is_not() { + assert_eq!(judge(Probe::Absent), Judgement::NoTooling); + assert_eq!(judge(Probe::Unreadable), Judgement::Unreadable); + } + + #[test] + fn a_reading_is_judged_against_the_pool_the_engine_allocates_from() { + assert_eq!( + judge(Probe::Read(strix_halo())), + Judgement::Target(select_drain_target(strix_halo())) + ); + } + + #[test] + fn a_fallback_tool_supplies_the_reading_the_first_could_not() { + assert_eq!(Probe::Unreadable.or_else(|| Probe::Read(7)), Probe::Read(7)); + assert_eq!(Probe::Absent.or_else(|| Probe::Read(7)), Probe::Read(7)); + assert_eq!( + Probe::Read(1).or_else(|| panic!("the fallback must not run after a reading")), + Probe::Read(1) + ); + } + + #[test] + fn a_fallback_is_absent_only_when_every_tool_was_absent() { + assert_eq!(Probe::::Absent.or_else(|| Probe::Absent), Probe::Absent); + assert_eq!( + Probe::::Absent.or_else(|| Probe::Unreadable), + Probe::Unreadable + ); + assert_eq!( + Probe::::Unreadable.or_else(|| Probe::Absent), + Probe::Unreadable + ); + } + + #[test] + fn rocm_smi_csv_gtt_is_read_in_mib() { + let text = "device,GTT Total Memory (B),GTT Total Used Memory (B)\n\ + card0,107374182400,41787392\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + Some(Pool::new(102_400, 102_360)) + ); + } + + #[test] + fn rocm_smi_csv_vram_keeps_its_original_column_match() { + let text = "device,VRAM Total Memory (B),VRAM Total Used Memory (B)\n\ + card0,536870912,203321344\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "Total Memory", "Total Used Memory"), + Some(Pool::new(512, 318)) + ); + } + + #[test] + fn rocm_smi_csv_without_the_pool_is_none() { + let text = "device,VRAM Total Memory (B),VRAM Total Used Memory (B)\n\ + card0,536870912,203321344\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + None + ); + assert_eq!( + parse_rocm_smi_csv_pool("", "Total Memory", "Total Used Memory"), + None + ); + } + + #[test] + fn rocm_smi_csv_with_warning_lines_before_the_header_is_still_read() { + let text = "WARNING: a device is in a low-power state\n\ + a warning that only says GTT Total Memory\n\ + \n\ + device,GTT Total Memory (B),GTT Total Used Memory (B)\n\ + card0,107374182400,41787392\n\ + card1,1,1\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + Some(Pool::new(102_400, 102_360)), + "the header is the first line naming both columns, and the first row after it is GPU 0" + ); + } + + #[test] + fn rocm_smi_csv_with_a_header_but_no_data_row_is_none() { + let text = "WARNING: nothing to report\n\ + device,GTT Total Memory (B),GTT Total Used Memory (B)\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + None + ); + } + + /// A Strix Halo host whose GTT pool has only `free_mib` MiB free of 102 400. + fn strix_holding(free_mib: u64) -> DeviceMemory { + DeviceMemory { + gtt: Some(Pool { + total_mib: 102_400, + free_mib, + }), + ..strix_halo() + } + } + + /// A sample whose GTT pool is nearly all held: below its 92 160 MiB floor. + fn undrained() -> Judgement { + Judgement::Target(select_drain_target(strix_holding(1_000))) + } + + /// The healthy Strix Halo reading: above its floor. + fn drained() -> Judgement { + Judgement::Target(select_drain_target(strix_halo())) + } + + #[test] + fn a_target_is_described_with_its_pool_and_only_gtt_is_named() { + assert_eq!( + select_drain_target(strix_halo()).describe(), + "102360 MiB free of 102400 MiB GTT, floor 92160 MiB" + ); + assert_eq!( + select_drain_target(mi300x()).describe(), + "196309 MiB free of 196592 MiB, floor 150000 MiB" + ); + } + + #[test] + fn a_drained_sample_ends_the_wait_and_the_line_names_the_pool_it_read() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + let verdict = wait.observe(drained()).expect("a drained sample ends it"); + assert_eq!(verdict, Verdict::Drained(select_drain_target(strix_halo()))); + assert_eq!( + describe_verdict(verdict, 6), + "device state: drained (102360 MiB free of 102400 MiB GTT, floor 92160 MiB)" + ); + } + + #[test] + fn no_tooling_ends_the_wait_on_the_first_sample_only() { + assert_eq!( + DrainWait::default().observe(Judgement::NoTooling), + Some(Verdict::NoTooling) + ); + assert_eq!( + describe_verdict(Verdict::NoTooling, 0), + "device state: no GPU tooling (mock/local run)" + ); + // A tool that disappears mid-wait is a sample nobody could read, not a + // reason to report a mock run. + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + for _ in 0..UNREAD_SAMPLES_BEFORE_GIVING_UP * 2 { + assert_eq!(wait.observe(Judgement::NoTooling), None); + } + } + + #[test] + fn a_tool_that_never_reads_is_given_up_on_and_called_unreadable_not_undrained() { + let mut wait = DrainWait::default(); + for _ in 1..UNREAD_SAMPLES_BEFORE_GIVING_UP { + assert_eq!(wait.observe(Judgement::Unreadable), None); + } + let verdict = wait + .observe(Judgement::Unreadable) + .expect("the last allowed unread sample ends the wait"); + assert_eq!( + verdict, + Verdict::Unreadable { + last_reading: None, + samples: UNREAD_SAMPLES_BEFORE_GIVING_UP, + hint: None, + } + ); + let line = describe_verdict(verdict, 24); + assert_eq!( + line, + "device state: could not read device memory (3 samples in 24s; the GPU tool is \ + installed but gave no usable reading)" + ); + assert!(!line.contains("NOT drained"), "{line}"); + } + + #[test] + fn a_probe_the_deadline_cut_short_does_not_replace_a_below_floor_reading() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe_truncated(undrained(), false), None); + // The last sample's probe was killed by the deadline and gave nothing. + assert_eq!( + wait.observe_truncated(Judgement::Unreadable, true), + None, + "a truncated sample with no reading is not counted" + ); + assert_eq!( + wait.observe_truncated(Judgement::Unparsed("rocm-smi: killed".to_owned()), true), + None + ); + assert_eq!( + wait.timed_out(), + Verdict::NotDrained(select_drain_target(strix_holding(1_000))), + "the wait ends on the last real reading, not on the probe the deadline stopped" + ); + } + + #[test] + fn an_unread_sample_that_was_not_cut_short_still_counts_and_a_truncated_reading_still_reads() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe_truncated(undrained(), false), None); + assert_eq!(wait.observe_truncated(Judgement::Unreadable, false), None); + assert!( + matches!(wait.timed_out(), Verdict::Unreadable { samples: 1, .. }), + "a stalled probe with the full budget is a real miss" + ); + let mut wait = DrainWait::default(); + let verdict = wait.observe_truncated(drained(), true); + assert!( + matches!(verdict, Some(Verdict::Drained(_))), + "a truncated sample that read is judged like any other: {verdict:?}" + ); + } + + #[test] + fn a_tool_that_has_read_is_waited_on_through_unreadable_samples() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(Judgement::Unreadable), None); + assert_eq!(wait.observe(Judgement::Unreadable), None); + assert_eq!(wait.observe(undrained()), None); + for _ in 0..UNREAD_SAMPLES_BEFORE_GIVING_UP * 3 { + assert_eq!(wait.observe(Judgement::Unreadable), None); + } + let verdict = wait.timed_out(); + assert_eq!( + verdict, + Verdict::Unreadable { + last_reading: Some(select_drain_target(strix_holding(1_000))), + samples: UNREAD_SAMPLES_BEFORE_GIVING_UP * 3, + hint: None, + } + ); + assert_eq!( + describe_verdict(verdict, 120), + "device state: could not read device memory at the end of 120s (last reading: \ + 1000 MiB free of 102400 MiB GTT, floor 92160 MiB)" + ); + } + + #[test] + fn the_deadline_after_a_reading_is_not_drained_and_after_a_miss_it_is_not() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + let target = select_drain_target(strix_holding(1_000)); + assert_eq!(wait.timed_out(), Verdict::NotDrained(target)); + assert_eq!( + describe_verdict(wait.timed_out(), 120), + "device state: NOT drained after 120s (1000 MiB free of 102400 MiB GTT, floor \ + 92160 MiB) — a previous engine may still be holding the GPU" + ); + // The most recent sample was one nobody could read: that is not a claim + // about the device. + assert_eq!(wait.observe(Judgement::Unreadable), None); + assert_eq!( + wait.timed_out(), + Verdict::Unreadable { + last_reading: Some(target), + samples: 1, + hint: None, + } + ); + } + + #[test] + fn an_undrained_carveout_judged_without_gtt_says_it_could_not_follow_the_engines_memory() { + // The real assembly path: amd-smi read the carveout, no tool reports GTT. + let (memory, _) = sample(Probe::Read(Pool::new(512, 318)), Probe::Absent); + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(judge(memory)), None); + let line = describe_verdict(wait.timed_out(), 120); + assert!( + line.starts_with( + "device state: NOT drained after 120s (318 MiB free of 512 MiB, floor 450 MiB)" + ), + "{line}" + ); + assert!( + line.contains( + "only the BIOS carveout was read because no GTT reading was available (is \ + rocm-smi installed?)" + ), + "{line}" + ); + // No such note when the pool judged is the one the engine uses. + for held in [ + Verdict::NotDrained(select_drain_target(strix_holding(1_000))), + Verdict::NotDrained(select_drain_target(DeviceMemory { + vram: Pool { + total_mib: 196_592, + free_mib: 1_000, + }, + gtt: None, + })), + ] { + let line = describe_verdict(held, 120); + assert!(!line.contains("carveout"), "{line}"); + } + } + + /// What `amd-smi metric -m` prints, per GPU, with the lines the parser reads + /// among others it must not confuse with them. + const AMD_SMI_GPU0: &str = "GPU: 0\n MEM_USAGE:\n TOTAL_VRAM: 196592 MB\n \ + USED_VRAM: 283 MB\n FREE_VRAM: 196309 MB\n \ + TOTAL_VISIBLE_VRAM: 196592 MB\n FREE_VISIBLE_VRAM: 196309 MB\n"; + + #[test] + fn amd_smi_vram_reads_the_total_and_free_lines_in_mib() { + assert_eq!( + parse_amd_smi_vram(AMD_SMI_GPU0), + Some(Pool::new(196_592, 196_309)) + ); + } + + #[test] + fn amd_smi_vram_takes_the_first_gpu_and_ignores_indentation_and_units() { + let two = format!("{AMD_SMI_GPU0}GPU: 1\n TOTAL_VRAM: 1 MB\n FREE_VRAM: 2 MB\n"); + assert_eq!(parse_amd_smi_vram(&two), Some(Pool::new(196_592, 196_309))); + // Tabs, wide gaps, any unit word and a missing space after the colon all + // read the same way: the number after the name is the value. + assert_eq!( + parse_amd_smi_vram("\tTOTAL_VRAM: 512 MiB\n FREE_VRAM:318\n"), + Some(Pool::new(512, 318)) + ); + assert_eq!( + parse_amd_smi_vram("TOTAL_VRAM: 512\r\nFREE_VRAM: 318 MB\r\n"), + Some(Pool::new(512, 318)) + ); + } + + #[test] + fn amd_smi_vram_needs_both_fields_as_numbers() { + assert_eq!(parse_amd_smi_vram("TOTAL_VRAM: 512 MB\n"), None); + assert_eq!(parse_amd_smi_vram("FREE_VRAM: 318 MB\n"), None); + assert_eq!( + parse_amd_smi_vram("TOTAL_VRAM: N/A\nFREE_VRAM: 318 MB\n"), + None + ); + assert_eq!(parse_amd_smi_vram("no such fields here"), None); + assert_eq!(parse_amd_smi_vram(""), None); + // The visible-VRAM lines are different fields, not the ones asked for. + assert_eq!( + parse_amd_smi_vram("TOTAL_VISIBLE_VRAM: 512 MB\nFREE_VISIBLE_VRAM: 318 MB\n"), + None + ); + } + + #[test] + fn a_zero_amd_smi_total_parses_and_is_then_rejected_as_a_failed_read() { + let zero = "TOTAL_VRAM: 0 MB\nFREE_VRAM: 0 MB\n"; + let parsed = parse_amd_smi_vram(zero).expect("parses"); + assert_eq!(parsed, Pool::new(0, 0)); + let (result, gtt_asked) = sample(Probe::Read(parsed), Probe::Absent); + assert_eq!(result, Probe::Unreadable); + assert!(!gtt_asked); + } + + #[test] + fn output_hint_is_the_first_nonblank_line_bounded_and_printable() { + assert_eq!( + output_hint( + "rocm-smi gtt", + "\n \ndevice,VRAM Total Memory (B)\ncard0,1\n" + ), + "rocm-smi gtt printed \"device,VRAM Total Memory (B)\"" + ); + assert_eq!(output_hint("tool", ""), "tool printed no output"); + assert_eq!(output_hint("tool", " \n\t\n"), "tool printed no output"); + // Non-ASCII and control characters are replaced, not repeated. + assert_eq!( + output_hint("tool", "GTT \u{1b}[31mT\u{f6}tal\u{0}"), + "tool printed \"GTT ?[31mT?tal?\"" + ); + // Quotes and backslashes are escaped by the quoting. + assert_eq!( + output_hint("tool", "a\"b\\c"), + "tool printed \"a\\\"b\\\\c\"" + ); + let long = "x".repeat(HINT_MAX_CHARS + 50); + let hint = output_hint("tool", &long); + assert_eq!( + hint, + format!("tool printed \"{}...\"", "x".repeat(HINT_MAX_CHARS)) + ); + let exact = "y".repeat(HINT_MAX_CHARS); + assert_eq!( + output_hint("tool", &exact), + format!("tool printed \"{exact}\"") + ); + } + + #[test] + fn output_a_parser_rejects_is_unparsed_with_what_it_printed_not_plain_unreadable() { + let header = "device,GTT Total (B),GTT Used (B)\ncard0,107374182400,41787392\n"; + let probe = Probe::Read(header.to_owned()).parse_text("rocm-smi gtt", |text| { + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory") + }); + assert_eq!( + probe, + Probe::Unparsed( + "rocm-smi gtt printed \"device,GTT Total (B),GTT Used (B)\"".to_owned() + ) + ); + // Right headers still read, and a tool that is absent or failed is not + // given a hint it has not earned. + let good = "device,GTT Total Memory (B),GTT Total Used Memory (B)\ncard0,1048576,0\n"; + assert_eq!( + Probe::Read(good.to_owned()).parse_text("x", |t| { + parse_rocm_smi_csv_pool(t, "GTT Total Memory", "GTT Total Used Memory") + }), + Probe::Read(Pool::new(1, 1)) + ); + assert_eq!( + Probe::::Absent.parse_text("x", |_| Some(0_u8)), + Probe::Absent + ); + assert_eq!( + Probe::::Unreadable.parse_text("x", |_| Some(0_u8)), + Probe::Unreadable + ); + } + + #[test] + fn a_fallback_tool_that_printed_something_unknown_keeps_what_it_printed() { + let unparsed = || Probe::::Unparsed("amd-smi printed \"GPU: 0\"".to_owned()); + assert_eq!( + unparsed().or_else(|| Probe::Absent), + unparsed(), + "an absent fallback leaves the first tool's output visible" + ); + assert_eq!(Probe::Absent.or_else(unparsed), unparsed()); + assert_eq!(Probe::Unreadable.or_else(unparsed), unparsed()); + assert_eq!(unparsed().or_else(|| Probe::Unreadable), unparsed()); + assert_eq!(unparsed().or_else(|| Probe::Read(7)), Probe::Read(7)); + // The first tool's output wins when both printed something. + assert_eq!( + unparsed().or_else(|| Probe::Unparsed("rocm-smi printed \"x\"".to_owned())), + unparsed() + ); + } + + #[test] + fn a_carveout_host_whose_gtt_output_was_not_understood_carries_what_it_printed() { + let hint = "rocm-smi gtt printed \"device,GTT Total (B)\""; + let (result, gtt_asked) = sample( + Probe::Read(Pool::new(512, 318)), + Probe::Unparsed(hint.to_owned()), + ); + assert!(gtt_asked); + assert_eq!(result, Probe::Unparsed(hint.to_owned())); + // And a VRAM probe's unparsed output passes through without asking for GTT. + let (result, gtt_asked) = sample(Probe::Unparsed(hint.to_owned()), Probe::Absent); + assert_eq!(result, Probe::Unparsed(hint.to_owned())); + assert!(!gtt_asked); + assert_eq!(judge(result), Judgement::Unparsed(hint.to_owned())); + } + + #[test] + fn what_a_tool_printed_reaches_the_final_line_when_it_never_reads() { + let hint = "rocm-smi --showmeminfo gtt --csv printed \"device,GTT Total (B)\""; + let mut wait = DrainWait::default(); + for _ in 1..UNREAD_SAMPLES_BEFORE_GIVING_UP { + assert_eq!(wait.observe(Judgement::Unparsed(hint.to_owned())), None); + } + let verdict = wait + .observe(Judgement::Unparsed(hint.to_owned())) + .expect("the third unread sample ends the wait"); + assert_eq!( + describe_verdict(verdict, 9), + "device state: could not read device memory (3 samples in 9s; the GPU tool is \ + installed but gave no usable reading; rocm-smi --showmeminfo gtt --csv printed \ + \"device,GTT Total (B)\")" + ); + } + + #[test] + fn what_a_tool_printed_is_kept_through_later_plain_failures_and_dropped_by_a_reading() { + let hint = "rocm-smi printed \"hdr\""; + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(Judgement::Unparsed(hint.to_owned())), None); + // A later timeout does not erase the evidence of the earlier sample. + assert_eq!(wait.observe(Judgement::Unreadable), None); + let Verdict::Unreadable { hint: kept, .. } = wait.timed_out() else { + panic!("two unread samples are unreadable at the deadline"); + }; + assert_eq!(kept.as_deref(), Some(hint)); + // A reading shows the output is understood after all. + assert_eq!(wait.observe(undrained()), None); + assert_eq!(wait.observe(Judgement::Unreadable), None); + let Verdict::Unreadable { + hint: after, + last_reading, + .. + } = wait.timed_out() + else { + panic!("the last sample was unreadable"); + }; + assert_eq!(after, None); + assert!(last_reading.is_some()); + } + + #[test] + fn a_hint_follows_the_last_reading_on_the_line_when_the_deadline_finds_it_unreadable() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + assert_eq!( + wait.observe(Judgement::Unparsed("t printed \"h\"".to_owned())), + None + ); + assert_eq!( + describe_verdict(wait.timed_out(), 120), + "device state: could not read device memory at the end of 120s (last reading: \ + 1000 MiB free of 102400 MiB GTT, floor 92160 MiB; t printed \"h\")" + ); + } + + #[test] + fn a_probe_gets_the_probe_timeout_or_the_time_left_whichever_is_less() { + let now = Instant::now(); + assert_eq!( + probe_budget(now, now + Duration::from_secs(100)), + SMI_PROBE_TIMEOUT + ); + assert_eq!( + probe_budget(now, now + Duration::from_secs(4)), + Duration::from_secs(4) + ); + assert_eq!( + probe_budget(now + Duration::from_secs(5), now), + Duration::ZERO + ); + } + + #[test] + fn a_sample_is_truncated_only_when_less_than_a_full_probe_timeout_is_left() { + let now = Instant::now(); + assert!(!sample_is_truncated(now, now + Duration::from_secs(100))); + assert!( + !sample_is_truncated(now, now + SMI_PROBE_TIMEOUT), + "exactly a full probe timeout left is not truncated" + ); + assert!(sample_is_truncated( + now, + now + SMI_PROBE_TIMEOUT.saturating_sub(Duration::from_millis(1)) + )); + assert!( + sample_is_truncated(now + Duration::from_secs(5), now), + "past the deadline is truncated" + ); + } + + /// A probe that exited successfully has its output in the pipe, and a loaded + /// runner can take longer than the old 200 ms to run the reader thread that + /// hands it over. The wait it is offered must outlast that, the bytes it + /// returns are the reading, and only a reader that never answers (a + /// grandchild holding the pipe) is unreadable. + #[test] + fn a_clean_exit_waits_for_a_slow_reader_and_only_a_silent_one_is_unreadable() { + let mut offered = None; + let out = output_after_clean_exit(|grace| { + offered = Some(grace); + Some(b"MEM 1 2\n".to_vec()) + }); + assert_eq!(out, Probe::Read("MEM 1 2\n".to_owned())); + let offered = offered.expect("the wait was asked for"); + assert!( + offered >= Duration::from_secs(1), + "a reader on a loaded runner was offered only {offered:?}" + ); + assert!( + offered <= SMI_PROBE_TIMEOUT, + "the grace for one probe is {offered:?}, longer than the probe itself" + ); + assert_eq!(output_after_clean_exit(|_| None), Probe::Unreadable); + } + + #[cfg(unix)] + fn shell(script: &str, args: &[&str]) -> Command { + let mut command = Command::new("sh"); + command.arg("-c").arg(script).arg("sh").args(args); + command + } + + #[cfg(unix)] + #[test] + fn a_probe_that_succeeds_returns_its_stdout() { + let out = bounded_stdout(shell("echo hello", &[]), Duration::from_secs(30)); + assert_eq!(out, Probe::Read("hello\n".to_owned())); + } + + #[cfg(unix)] + #[test] + fn a_probe_that_exits_non_zero_is_unreadable() { + let out = bounded_stdout(shell("echo hello; exit 3", &[]), Duration::from_secs(30)); + assert_eq!(out, Probe::Unreadable); + } + + #[test] + fn a_probe_whose_program_is_not_installed_is_absent() { + let out = bounded_stdout( + Command::new("rocm-cli-test-no-such-smi-tool"), + Duration::from_secs(30), + ); + assert_eq!(out, Probe::Absent); + } + + /// A tool that never returns must cost one sample, not the worker, and the + /// child must not be left running or unreaped. A zombie still answers + /// `kill -0`, so a failing `kill -0` shows the child was reaped, not merely + /// signalled. + #[cfg(unix)] + #[test] + fn a_probe_that_outlives_its_deadline_is_killed_reaped_and_unreadable() { + let dir = tempfile::tempdir().expect("tempdir"); + let pid_file = dir.path().join("pid"); + let started = Instant::now(); + let out = bounded_stdout( + shell( + "echo $$ > \"$1\"; exec sleep 60", + &[pid_file.to_str().expect("utf-8 path")], + ), + Duration::from_secs(3), + ); + let elapsed = started.elapsed(); + assert_eq!( + out, + Probe::Unreadable, + "a timed-out probe is an unreadable sample" + ); + assert!( + elapsed < Duration::from_secs(30), + "returned after {elapsed:?}, far past the 3s deadline" + ); + // The shell creates the file and then writes the pid; give a loaded + // runner a moment to finish the write rather than read it half-done. + let mut pid = String::new(); + for _ in 0..100 { + pid = std::fs::read_to_string(&pid_file) + .unwrap_or_default() + .trim() + .to_owned(); + if !pid.is_empty() { + break; + } + thread::sleep(Duration::from_millis(50)); + } + assert!( + !pid.is_empty(), + "the child wrote its pid before the deadline" + ); + let still_there = Command::new("kill") + .args(["-0", &pid]) + .stderr(Stdio::null()) + .status() + .expect("kill runs") + .success(); + assert!(!still_there, "child {pid} was left running or unreaped"); + } + + /// Windows runs only the deadline half; the reaped-child check above needs + /// `kill -0`. + #[cfg(windows)] + #[test] + fn a_probe_that_outlives_its_deadline_returns_unreadable_in_time() { + let mut command = Command::new("ping.exe"); + command.args(["-n", "60", "127.0.0.1"]); + let started = Instant::now(); + let out = bounded_stdout(command, Duration::from_millis(1500)); + assert_eq!(out, Probe::Unreadable); + assert!(started.elapsed() < Duration::from_secs(20)); + } +} diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 01ed93ca03..9ebdca6c6b 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -174,7 +174,9 @@ pub struct ScenarioDecl { /// per-engine canary on the fast path, so it is skipped on ordinary per-PR / /// on-demand runs and only runs in the merge queue, which opts in via /// `E2E_MERGE_QUEUE`. Keeps the PR feedback loop short while still exercising - /// the full serve matrix (on every lane but MI300X) before a change lands. + /// the heavy serves before a change lands. Only the Windows Strix and WSL2 + /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, + /// Strix Ubuntu) do not. pub merge_queue: bool, } @@ -838,6 +840,102 @@ mod tests { } } + #[test] + fn lemonade_serve_failures_track_rocmai_898_without_xfailing_mock_or_windows() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + "serve-lemonade-inference", + "serve-stop-leaves-no-background-helper", + ] { + let scenario = decl(&[&format!("id:{id}")]); + assert!( + matches!( + resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "ROCMAI-898" + ), + "{id}" + ); + assert_eq!( + matrix.serve_timeout_for(id, &cap("strix-ubuntu"), "lemonade"), + Some(90), + "{id}", + ); + for platform in ["mock", "strix-windows"] { + assert_eq!( + resolve(&scenario, &cap(platform), &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {platform}", + ); + } + } + } + + /// The rows that carry `therock_family` and `has_amd_gpu` exist so that the + /// no-GPU lane, where the same scenarios run against MockServer instead of a + /// real Lemonade serve, stays expect-pass. `cap("mock")` cannot prove that: + /// its `os_family` is `other`, so the `os = "linux"` condition excludes it + /// before either GPU guard is consulted. These hosts are Linux, so each guard + /// is the only thing standing between them and an xfail. The two partial + /// hosts isolate the guards from each other: dropping `has_amd_gpu` from a row + /// fails the host that reports only a gfx target, and dropping `therock_family` + /// fails the host that reports only a GPU. + #[test] + fn real_gpu_lemonade_rows_do_not_xfail_linux_hosts_without_a_usable_gpu() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + let linux_lemonade = cap("strix-ubuntu"); + let hosts = [ + ( + "linux, no GPU, no gfx target", + HostCapability { + gfx_target: None, + has_amd_gpu: false, + amd_gpu_count: Some(0), + ..linux_lemonade.clone() + }, + ), + ( + "linux, gfx target reported but no usable GPU", + HostCapability { + has_amd_gpu: false, + amd_gpu_count: Some(0), + ..linux_lemonade.clone() + }, + ), + ( + "linux, usable GPU but no gfx target reported", + HostCapability { + gfx_target: None, + ..linux_lemonade.clone() + }, + ), + ]; + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + ] { + let scenario = decl(&[&format!("id:{id}")]); + for (label, host) in &hosts { + assert_eq!( + resolve(&scenario, host, &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {label}", + ); + } + // The full real-GPU host is the one that must still be xfailed. + assert!( + matches!( + resolve(&scenario, &linux_lemonade, &matrix, Included::default()), + Expectation::ExpectXfail { bug, .. } if bug == "ROCMAI-898" + ), + "{id} on a real GPU host", + ); + } + } + fn decl(tags: &[&str]) -> ScenarioDecl { ScenarioDecl::from_tags(tags) } @@ -1100,10 +1198,11 @@ serve_timeout_secs = 90 #[test] fn aim_serve_scenarios_stay_off_the_per_pr_lanes() { // serve-26/27 pull the AIM image and a 9B checkpoint on a cold runner and - // may each wait out a 1200s readiness cap. Run per-PR on the Strix Ubuntu - // pool they pushed the job (~25 min otherwise) into its 45 min cap on every - // run. They must resolve to Skip unless the nightly opt-in is set, and must - // still be reachable (not Skip for the nightly reason) when it is. + // may each wait out a 1200s readiness cap. Run on every PR on the Strix + // Ubuntu pool (before that lane left the PR path) they pushed the job + // (~25 min otherwise) into its 45 min cap on every run. They must + // resolve to Skip unless the nightly opt-in is set, and must still be + // reachable (not Skip for the nightly reason) when it is. let m = Expectations::default(); for prefix in ["serve-26 -", "serve-27 -"] { let d = ScenarioDecl::from_tags(&model_serving_tags(prefix)); @@ -2257,11 +2356,47 @@ flaky = true )); } + /// The two rows for Lemonade 11.5.1's failed backend alignment apply to native + /// Windows with lemonade and to nothing else: not to Linux hosts (including one whose + /// default engine is vLLM, where the engine named is what decides), and they are not + /// `flaky` (an XPASS means the cause is gone and the row is stale). Neither sets + /// `serve_timeout_secs`: serve-06's own `@serve-timeout` tag takes precedence over a + /// row, so one would change nothing. + #[test] + fn lemonade_alignment_rows_apply_only_to_native_windows_with_lemonade() { + let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + for id in [ + "runtime-lemonade-backend-alignment-reported", + "serve-large-model-inference", + ] { + assert!(m.is_xfail(id, &cap("strix-windows"), "lemonade"), "{id}"); + for linux in ["strix-ubuntu", "mi300x", "wsl2"] { + assert!(!m.is_xfail(id, &cap(linux), "lemonade"), "{id} on {linux}"); + } + // Same Windows host, other engine: the row is about lemonade. + assert!(!m.is_xfail(id, &cap("strix-windows"), "vllm"), "{id} vllm"); + let entries = m.entries_for(id); + assert_eq!(entries.len(), 1, "{id}"); + assert!(!entries[0].flaky, "{id} is deterministic"); + assert_eq!(entries[0].serve_timeout_secs, None, "{id}"); + } + // The row cannot shorten serve-06's wait: its own tag wins, so the harness + // still hands the 2400s the feature file asks for. + let tags = feature_tags( + include_str!("../features/model_serving.feature"), + "serve-large-model-inference", + ); + assert_eq!( + m.serve_timeout_override(&ScenarioDecl::from_tags(&tags), &cap("strix-windows")), + Some(2400) + ); + } + /// serve-40 serves through the same lemonade managed serve as the other /// lemonade serve rows, so its row is scoped the same way: xfail on a /// lemonade Linux host, with a 90s serve wait, and not an expected failure - /// on native Windows. The row must stay `flaky` (a fixed #897 would - /// otherwise turn the Ubuntu lanes red as a stale XPASS) and must name #897. + /// on native Windows. The row must stay `flaky` (a fixed ROCMAI-898 would + /// otherwise turn the Ubuntu lanes red as a stale XPASS) and must name ROCMAI-898. #[test] fn serve_40_row_is_scoped_like_the_other_lemonade_serve_rows() { let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); @@ -2275,7 +2410,7 @@ flaky = true let entries = m.entries_for(id); assert_eq!(entries.len(), 1); assert!(entries[0].flaky); - assert_eq!(entries[0].bug, "#897"); + assert_eq!(entries[0].bug, "ROCMAI-898"); } /// The row's 90s only matters if the harness hands it to the serve steps: diff --git a/tests/e2e-cucumber/src/lemonade_backend.rs b/tests/e2e-cucumber/src/lemonade_backend.rs new file mode 100644 index 0000000000..1cf5f53885 --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_backend.rs @@ -0,0 +1,167 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Where an installed Lemonade runtime keeps the `llama-server` of a backend. +//! +//! The layout mirrors what the Lemonade engine searches when it serves a model +//! directly: `bin/llamacpp//llama-server` under the runtime +//! directory, either flat or one build-numbered subdirectory down. The `rocm` +//! backend is a family of directory names; Vulkan never crosses over to ROCm. + +use std::path::{Path, PathBuf}; + +/// Directory names that hold the `rocm` backend, in lookup order. +const ROCM_DIRS: [&str; 3] = ["rocm-stable", "rocm-nightly", "rocm"]; + +/// The outcome of looking for a backend's `llama-server`. +#[derive(Debug, PartialEq, Eq)] +pub struct ServerLookup { + /// The binary, when one was found. + pub found: Option, + /// Every directory that was searched, in order. + pub searched: Vec, +} + +/// The platform's file name for the `llama-server` binary. +const fn server_binary_name() -> &'static str { + if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + } +} + +/// The directory names a backend may extract into, or `None` for a backend that +/// has no GPU `llama-server` (CPU and unknown names). +pub fn backend_dirs(backend_name: &str) -> Option<&'static [&'static str]> { + match backend_name { + "rocm" => Some(&ROCM_DIRS), + "vulkan" => Some(&["vulkan"]), + _ => None, + } +} + +/// Look for the selected backend's `llama-server` under `runtime_dir`. +/// +/// Callers may rely on whether a binary exists, not on which one is returned when +/// several build-numbered subdirectories each hold one: the engine takes the first +/// in directory order, this returns the first in name order. +/// +/// Returns `Err` for a backend that has no GPU `llama-server`, so a recorded CPU +/// or unknown backend cannot pass as "nothing to find". +pub fn find_llama_server(runtime_dir: &Path, backend_name: &str) -> Result { + let dirs = backend_dirs(backend_name).ok_or_else(|| { + format!("backend `{backend_name}` has no GPU llama-server directory to look in") + })?; + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let binary = server_binary_name(); + let mut searched = Vec::new(); + for dir in dirs { + let dir = llamacpp.join(dir); + searched.push(dir.clone()); + if let Some(found) = binary_in(&dir, binary) { + return Ok(ServerLookup { + found: Some(found), + searched, + }); + } + } + Ok(ServerLookup { + found: None, + searched, + }) +} + +/// `binary` directly in `dir`, or one level down in a subdirectory. +fn binary_in(dir: &Path, binary: &str) -> Option { + let direct = dir.join(binary); + if direct.is_file() { + return Some(direct); + } + let mut subdirs: Vec = std::fs::read_dir(dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect(); + // Sorted only so the result is deterministic; which build wins is not part of + // the contract (see `find_llama_server`). + subdirs.sort(); + subdirs + .into_iter() + .map(|subdir| subdir.join(binary)) + .find(|candidate| candidate.is_file()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn install(runtime: &Path, relative: &str) -> PathBuf { + let dir = runtime.join("bin").join("llamacpp").join(relative); + std::fs::create_dir_all(&dir).expect("create backend dir"); + let binary = dir.join(server_binary_name()); + std::fs::write(&binary, b"x").expect("write binary"); + binary + } + + #[test] + fn finds_a_flat_binary_for_the_selected_backend() { + let root = tempfile::tempdir().expect("tempdir"); + let binary = install(root.path(), "vulkan"); + let lookup = find_llama_server(root.path(), "vulkan").expect("lookup"); + assert_eq!(lookup.found, Some(binary)); + } + + #[test] + fn finds_a_build_numbered_binary_in_the_rocm_family() { + let root = tempfile::tempdir().expect("tempdir"); + let binary = install(root.path(), "rocm-stable/llama-b9752"); + let lookup = find_llama_server(root.path(), "rocm").expect("lookup"); + assert_eq!(lookup.found, Some(binary)); + } + + #[test] + fn rocm_lookup_follows_family_order() { + let root = tempfile::tempdir().expect("tempdir"); + install(root.path(), "rocm"); + let stable = install(root.path(), "rocm-stable"); + let lookup = find_llama_server(root.path(), "rocm").expect("lookup"); + assert_eq!(lookup.found, Some(stable)); + } + + #[test] + fn a_different_backend_does_not_stand_in() { + let root = tempfile::tempdir().expect("tempdir"); + install(root.path(), "vulkan"); + install(root.path(), "cpu"); + let lookup = find_llama_server(root.path(), "rocm").expect("lookup"); + assert_eq!(lookup.found, None); + let llamacpp = root.path().join("bin").join("llamacpp"); + let expected: Vec = ROCM_DIRS.iter().map(|dir| llamacpp.join(dir)).collect(); + assert_eq!(lookup.searched, expected); + } + + #[test] + fn a_directory_without_the_binary_is_not_a_hit() { + let root = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir_all(root.path().join("bin/llamacpp/vulkan/llama-b1")) + .expect("create dir"); + let lookup = find_llama_server(root.path(), "vulkan").expect("lookup"); + assert_eq!(lookup.found, None); + assert_eq!(lookup.searched.len(), 1); + } + + #[test] + fn cpu_and_unknown_backends_are_rejected() { + let root = tempfile::tempdir().expect("tempdir"); + install(root.path(), "cpu"); + for backend in ["cpu", "unknown", ""] { + assert!( + find_llama_server(root.path(), backend).is_err(), + "{backend}" + ); + } + } +} diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs new file mode 100644 index 0000000000..5d4eed66ab --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -0,0 +1,2116 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Read-only backend evidence for a failed Lemonade serve. +//! +//! The evidence is taken in two steps with the service stop between them (see +//! [`backend_evidence_around_stop`]): what is only read from disk (manifest, +//! pins, candidate paths, fingerprints) is collected while the failed service is +//! still in the state being diagnosed, and the `llama-server` probes, which +//! execute a backend binary, run after it has been stopped. + +use sha2::{Digest as _, Sha256}; +use std::fmt::Write as _; +use std::io::Read as _; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncRead, AsyncReadExt as _}; + +use crate::serve_log::{ENGINE_UNKNOWN_EVIDENCE, read_clamped_file}; + +const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_CANDIDATES: usize = 8; +/// Time for the read-only inventory (pins, candidate paths, fingerprints), +/// taken before the service is stopped. +const INVENTORY_BUDGET: Duration = Duration::from_secs(20); +/// Time for the probes, started after the stop returns so a slow stop does not +/// use it up. +const PROBE_BUDGET: Duration = Duration::from_secs(20); +/// Largest executable that is fingerprinted. +/// +/// A fingerprint that is absent on the lane the report is for is a defect, so the +/// cap is set well above a dynamically linked `llama-server` (a few MiB; the HIP +/// kernels live in separate shared libraries). It is still finite: a build that +/// statically linked kernels for several GPU targets can reach hundreds of MiB, and +/// a corrupt cache entry should not be hashed at all. Hashing streams 64 KiB at a +/// time, so memory stays bounded whatever this is; the time is bounded by the +/// inventory budget, which the hashing loop checks between reads. +const MAX_HASH_BYTES: u64 = 512 * 1024 * 1024; + +#[derive(Clone, Copy)] +struct Budgets { + inventory: Duration, + probes: Duration, +} + +const DEFAULT_BUDGETS: Budgets = Budgets { + inventory: INVENTORY_BUDGET, + probes: PROBE_BUDGET, +}; + +/// Runs one `llama-server` flag and renders what came back. A trait so the order +/// of probes against the service stop is testable without executing a binary. +trait Probe: Sync { + fn run( + &self, + server: &Path, + flag: &str, + timeout: Duration, + ) -> impl std::future::Future + Send; +} + +struct RealProbe; + +impl Probe for RealProbe { + async fn run(&self, server: &Path, flag: &str, timeout: Duration) -> String { + probe(server, flag, timeout).await + } +} + +/// Collect the backend evidence for a failed serve of `engine` around the call +/// that stops the service, and return the stop's own status with it. +/// +/// `stop` always runs exactly once, whatever the engine, so a caller cannot +/// forget it or run it twice. For Lemonade the order is fixed: the read-only +/// inventory first, then `stop`, then the probes. A probe executes a backend +/// binary, and llama.cpp builds commonly initialise a GPU backend even for +/// `--version`, so running one next to a still-live router on a shared-memory +/// device could change the very state being diagnosed. Reading files cannot. +/// +/// `manifest_path` is only called for Lemonade, and nothing it does may keep the +/// stop from running: it returns `Err(reason)` when there is no manifest to look +/// for (the report then says `unavailable: `), and a panic inside it is +/// caught and reported the same way, so the stop still runs exactly once and the +/// failure report survives. The report is the one thing a failed serve has left. +/// +/// The report says `engine unknown` when no engine was determined, so a skipped +/// inventory stays visible, and returns no evidence for another engine. +pub async fn backend_evidence_around_stop( + manifest_path: impl FnOnce() -> Result + Send, + engine: Option<&str>, + stop: impl FnOnce() -> String + Send, +) -> (String, Option) { + match engine { + Some("lemonade") => { + let manifest = std::panic::catch_unwind(std::panic::AssertUnwindSafe(manifest_path)) + .unwrap_or_else(|_| Err("finding the runtime manifest failed".to_owned())); + match manifest { + Ok(manifest) => { + let (stop_status, evidence) = + evidence_around_stop(&manifest, DEFAULT_BUDGETS, &RealProbe, stop).await; + (stop_status, Some(evidence)) + } + Err(reason) => ( + stop(), + Some(format!("backend evidence unavailable: {reason}")), + ), + } + } + Some(_) => (stop(), None), + None => (stop(), Some(ENGINE_UNKNOWN_EVIDENCE.to_owned())), + } +} + +async fn evidence_around_stop( + manifest_path: &Path, + budgets: Budgets, + probe: &impl Probe, + stop: impl FnOnce() -> String + Send, +) -> (String, String) { + let inventory = take_inventory(manifest_path, budgets.inventory).await; + let stop_status = stop(); + let probes = inventory.probe_all(probe, budgets.probes).await; + (stop_status, format!("{}{probes}", inventory.report)) +} + +/// What was read from disk before the stop: the report so far, and the +/// candidates the probes will run against once the service is stopped. +struct Inventory { + report: String, + servers: Vec, +} + +impl Inventory { + const fn without_probes(report: String) -> Self { + Self { + report, + servers: Vec::new(), + } + } + + async fn probe_all(&self, probe: &impl Probe, budget: Duration) -> String { + if self.servers.is_empty() { + return String::new(); + } + let mut report = String::new(); + let _ = writeln!( + report, + "probes, run after the service stop was attempted; they inherit the harness \ + environment, not the engine's managed library environment, so a probe loader \ + failure does not establish a serving failure:" + ); + let deadline = Instant::now() + budget; + for server in &self.servers { + let _ = writeln!(report, "probe target: {}", server.display()); + for flag in ["--version", "--help"] { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } + let output = probe.run(server, flag, remaining.min(PROBE_TIMEOUT)).await; + let _ = writeln!(report, "{flag}:\n{output}"); + } + } + report + } +} + +/// What the report says about a file it did not read because it is a symlink. +const SYMLINK_NOT_FOLLOWED: &str = "symlink not followed"; + +/// Whether the last component of `path` is itself a symlink (not followed). Only +/// the non-unix [`open_no_follow`] needs it; unix refuses in the open itself. +#[cfg(not(unix))] +fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) +} + +/// Whether `error` is the refusal [`open_no_follow`] makes for a symlink. +fn is_symlink_refusal(error: &std::io::Error) -> bool { + error.kind() == std::io::ErrorKind::Other && error.to_string() == SYMLINK_NOT_FOLLOWED +} + +/// Open `path` for reading, refusing it when its last component is a symlink. +/// +/// On unix the refusal is made by the open itself (`O_NOFOLLOW`), so there is no +/// window between a check and the open in which the path could be swapped for a +/// link: a link is refused whenever it is there at the moment of the open. On +/// Windows the check is a metadata read followed by an open, so a path swapped for +/// a link between the two is not caught; the evidence is read-only and the swap +/// needs write access to the backend directory, so that is accepted rather than +/// reached for with platform calls this harness does not otherwise make. Only +/// the last component is checked on either platform: a link in a parent directory +/// is followed. +fn open_no_follow(path: &Path) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + .map_err(|error| { + if error.raw_os_error() == Some(libc::ELOOP) { + std::io::Error::other(SYMLINK_NOT_FOLLOWED) + } else { + error + } + }) + } + #[cfg(not(unix))] + { + if is_symlink(path) { + return Err(std::io::Error::other(SYMLINK_NOT_FOLLOWED)); + } + std::fs::File::open(path) + } +} + +/// [`read_clamped`] on a file opened with [`open_no_follow`]: a link inside the +/// backend tree that points outside it must not put the outside file into the +/// report, the same rule the directory walk applies. +fn read_regular_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { + let file = open_no_follow(path)?; + let len = file.metadata()?.len(); + read_clamped_file(file, len, max) +} + +/// Answers "has the inventory's time run out?". Injected so a test can drive the +/// traversal's deadline without sleeping; the real one compares against an +/// [`Instant`]. `Arc` because the scan runs on a blocking thread. +type Expired = std::sync::Arc bool + Send + Sync>; + +/// Record the installed version and pins separately from executable build output: +/// a pin describes what should be installed, not what a runner actually cached. +/// +/// Every candidate is labelled as such; finding a binary does not prove that the +/// router launched it. Nothing here executes anything. No install, download, +/// model load, or cache repair is done. +/// +/// The budget starts here, at entry, and covers the manifest and pin reads, the +/// directory search and the hashing alike. +async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { + let deadline = Instant::now() + budget; + take_inventory_with( + manifest_path, + deadline, + SEARCH_LIMITS, + std::sync::Arc::new(move || Instant::now() >= deadline), + ) + .await +} + +/// What [`scan_runtime`] read from disk. +enum Scan { + /// The manifest could not be used; the report says why. + Unavailable(String), + Found { + /// Manifest fields and pins. + report: String, + discovery: Discovery, + /// Where an empty search looked, when it found nothing and was not cut short + /// by the clock. + empty_note: Option, + }, +} + +/// Everything the inventory reads before it hashes anything: the manifest, the +/// pins and the directory search. Blocking filesystem work, so it runs on a +/// blocking thread; it carries its own clock because a `timeout` around +/// `spawn_blocking` would only drop the join handle and leave this running. +fn scan_runtime(manifest_path: &Path, limits: SearchLimits, expired: &dyn Fn() -> bool) -> Scan { + let mut report = String::new(); + let _ = writeln!(report, "manifest: {}", manifest_path.display()); + let manifest = + match read_regular_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { + serde_json::from_slice::(&bytes).map_err(std::io::Error::other) + }) { + Ok(manifest) => manifest, + Err(error) => return Scan::Unavailable(format!("{report}unavailable: {error}")), + }; + for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { + let _ = writeln!( + report, + "{key}: {}", + manifest.get(key).unwrap_or(&serde_json::Value::Null) + ); + } + let Some(runtime) = manifest + .get("runtime_dir") + .and_then(serde_json::Value::as_str) + else { + return Scan::Unavailable(format!( + "{report}backend inventory unavailable: no runtime_dir" + )); + }; + let runtime = Path::new(runtime); + let pins = runtime.join("resources/backend_versions.json"); + let _ = writeln!(report, "configured pins (not executable identity):"); + match read_regular_clamped(&pins, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!(report, "{}{note}", String::from_utf8_lossy(&bytes)); + } + Err(error) => { + let _ = writeln!(report, "{}: {error}", pins.display()); + } + } + let discovery = discover_candidates(runtime, limits, expired); + // "Nothing found" is only a finding when the search looked everywhere; one cut + // short by the clock or a limit reports that instead (see `take_inventory_with`). + // A candidate-cap stop has candidates by definition, so it never gets here. + let empty_note = (discovery.candidates.is_empty() && discovery.stop.is_none()) + .then(|| empty_search_note(runtime)); + Scan::Found { + report, + discovery, + empty_note, + } +} + +/// One candidate's fingerprint and `version.txt`, rendered. Blocking reads, run on +/// a blocking thread with the inventory's deadline checked between hash reads. +fn inspect_candidate(server: &Path, deadline: Instant) -> String { + let mut report = String::new(); + match fingerprint(server, deadline) { + Ok(hash) => { + let _ = writeln!(report, "sha256: {hash}"); + } + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + if let Some(parent) = server.parent() { + let version = parent.join("version.txt"); + match read_regular_clamped(&version, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!( + report, + "{}: {}{note}", + version.display(), + String::from_utf8_lossy(&bytes) + ); + } + Err(error) if is_symlink_refusal(&error) => { + let _ = writeln!(report, "{}: {SYMLINK_NOT_FOLLOWED}", version.display()); + } + Err(_) => {} + } + } + report +} + +async fn take_inventory_with( + manifest_path: &Path, + deadline: Instant, + limits: SearchLimits, + expired: Expired, +) -> Inventory { + let scan_path = manifest_path.to_owned(); + let scan_clock = expired.clone(); + let scan = + tokio::task::spawn_blocking(move || scan_runtime(&scan_path, limits, scan_clock.as_ref())) + .await; + let (mut report, discovery, empty_note) = match scan { + Ok(Scan::Found { + report, + discovery, + empty_note, + }) => (report, discovery, empty_note), + Ok(Scan::Unavailable(report)) => return Inventory::without_probes(report), + Err(error) => { + return Inventory::without_probes(format!( + "manifest: {}\nunavailable: inventory scan failed: {error}", + manifest_path.display() + )); + } + }; + if let Some(note) = empty_note { + let _ = writeln!(report, "{note}"); + } + match discovery.stop { + Some(Stop::Deadline) => { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + } + Some(Stop::Directories) => { + let _ = writeln!( + report, + "candidate search stopped: {} directories visited (limit)", + discovery.dirs_visited + ); + } + Some(Stop::Entries) => { + let _ = writeln!( + report, + "candidate search stopped: {} directory entries read (limit)", + discovery.entries_read + ); + } + Some(Stop::Candidates) | None => {} + } + if discovery.symlinks_skipped > 0 { + let _ = writeln!( + report, + "{} symlinked entries under the backend directories were not followed", + discovery.symlinks_skipped + ); + } + let mut servers = Vec::new(); + // Said once: when the search itself ended on the clock the line is already in. + let budget_noted = matches!(discovery.stop, Some(Stop::Deadline)); + for server in &discovery.candidates { + if Instant::now() >= deadline { + if !budget_noted { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + } + return Inventory { report, servers }; + } + let _ = writeln!( + report, + "candidate (not proof of selection): {}", + server.display() + ); + servers.push(server.clone()); + let path = server.clone(); + match tokio::task::spawn_blocking(move || inspect_candidate(&path, deadline)).await { + Ok(lines) => report.push_str(&lines), + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + } + if matches!(discovery.stop, Some(Stop::Candidates)) { + let _ = writeln!( + report, + "backend inventory truncated after {MAX_CANDIDATES} candidates \ + (a further one was found and is not listed)" + ); + } + Inventory { report, servers } +} + +/// The GPU backend directories under `bin/llamacpp` that are searched. The cpu +/// backend is left out on purpose: a CPU build is never what a GPU serve should +/// have launched, and listing it would read as one. +const BACKEND_DIRS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan"]; +/// How many directory levels below a backend directory are searched. Lemonade +/// unpacks some llama.cpp builds into a build-numbered directory, and the layout +/// is the router's to choose, so the search is not pinned to one level. +const MAX_SEARCH_DEPTH: usize = 3; +/// Names listed when explaining an empty search. +const MAX_LISTED_NAMES: usize = 16; +/// Directory entries read to build that listing. Bounded so a huge `bin/llamacpp` +/// cannot make the explanation of an empty search expensive; the report says how +/// many names it left out. +const MAX_LISTING_SCAN: usize = 64; + +/// Global bounds on one candidate search. They apply to the whole search, across +/// every backend directory, not per directory: a tree of many shallow directories +/// is as expensive as a deep one. +#[derive(Clone, Copy)] +struct SearchLimits { + /// Directories whose entries are read. + dirs: usize, + /// Directory entries read, in total. + entries: usize, + /// Candidates kept. The search stops as soon as it finds one more than this, + /// which is how it knows the list is cut; the extra one is dropped. + candidates: usize, +} + +const SEARCH_LIMITS: SearchLimits = SearchLimits { + dirs: 128, + entries: 2048, + candidates: MAX_CANDIDATES, +}; + +/// Why a candidate search ended before it had looked everywhere. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Stop { + Deadline, + Directories, + Entries, + Candidates, +} + +#[derive(Debug, Default)] +struct Discovery { + /// In a fixed order: backend directories as listed, then sibling directories + /// sorted by path. + candidates: Vec, + stop: Option, + /// Symlinks met and not followed. A cache that links out of its backend + /// directory, or back into it, is neither searched nor allowed to loop the + /// search. + symlinks_skipped: usize, + dirs_visited: usize, + entries_read: usize, +} + +const fn server_binary() -> &'static str { + if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + } +} + +/// Look for `llama-server` under each backend directory, within `limits` and until +/// `expired` says time is up. Symlinks are never followed: only real directories +/// are entered and only a regular file counts as a candidate, so a link cycle +/// ends and a link out of the tree is not read. +fn discover_candidates( + runtime: &Path, + limits: SearchLimits, + expired: &dyn Fn() -> bool, +) -> Discovery { + let mut search = Search { + limits, + expired, + out: Discovery::default(), + }; + for backend in BACKEND_DIRS { + let dir = runtime.join("bin/llamacpp").join(backend); + match std::fs::symlink_metadata(&dir) { + Ok(meta) if meta.is_dir() => search.visit(&dir, 0), + Ok(meta) if meta.file_type().is_symlink() => search.out.symlinks_skipped += 1, + _ => {} + } + if search.out.stop.is_some() { + break; + } + } + search.out.candidates.truncate(limits.candidates); + search.out +} + +struct Search<'a> { + limits: SearchLimits, + expired: &'a dyn Fn() -> bool, + out: Discovery, +} + +impl Search<'_> { + fn stop(&mut self, why: Stop) { + self.out.stop.get_or_insert(why); + } + + fn visit(&mut self, dir: &Path, depth: usize) { + if self.out.stop.is_some() { + return; + } + if (self.expired)() { + self.stop(Stop::Deadline); + return; + } + if self.out.dirs_visited >= self.limits.dirs { + self.stop(Stop::Directories); + return; + } + self.out.dirs_visited += 1; + let direct = dir.join(server_binary()); + // A symlink named llama-server is not a candidate (and is counted below, + // as a directory entry, not here). + if std::fs::symlink_metadata(&direct).is_ok_and(|meta| meta.is_file()) { + self.out.candidates.push(direct); + } + // One more than the cap: finding it is what shows the list is cut, so + // exactly `candidates` of them is a complete list and not a truncated one. + if self.out.candidates.len() > self.limits.candidates { + self.stop(Stop::Candidates); + return; + } + if depth >= MAX_SEARCH_DEPTH { + return; + } + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut subdirs = Vec::new(); + for entry in entries { + if (self.expired)() { + self.stop(Stop::Deadline); + break; + } + if self.out.entries_read >= self.limits.entries { + self.stop(Stop::Entries); + break; + } + self.out.entries_read += 1; + let Ok(entry) = entry else { + continue; + }; + match entry.file_type() { + Ok(kind) if kind.is_dir() => subdirs.push(entry.path()), + Ok(kind) if kind.is_symlink() => self.out.symlinks_skipped += 1, + _ => {} + } + } + subdirs.sort(); + for subdir in subdirs { + self.visit(&subdir, depth + 1); + } + } +} + +/// Why a search found nothing, in terms of where it looked. An absent binary can +/// mean a backend download that did not finish, a layout the search does not +/// cover, or a build that was never installed, and the report cannot tell which, +/// so it says what was searched and what `bin/llamacpp` actually holds. +fn empty_search_note(runtime: &Path) -> String { + let llamacpp = runtime.join("bin/llamacpp"); + let searched = BACKEND_DIRS + .iter() + .map(|backend| llamacpp.join(backend).display().to_string()) + .collect::>() + .join(", "); + let present = match std::fs::read_dir(&llamacpp) { + Ok(entries) => { + let mut names: Vec = entries + .take(MAX_LISTING_SCAN) + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + let hit_scan_cap = names.len() >= MAX_LISTING_SCAN; + names.sort(); + if names.is_empty() { + format!("{} is empty", llamacpp.display()) + } else { + let omitted = names.len().saturating_sub(MAX_LISTED_NAMES); + names.truncate(MAX_LISTED_NAMES); + let more = match (omitted, hit_scan_cap) { + (0, false) => String::new(), + (_, false) => format!(" and {omitted} more"), + (_, true) => format!( + " and at least {omitted} more (only the first {MAX_LISTING_SCAN} \ + entries were read)" + ), + }; + format!( + "{} contains: {}{more}", + llamacpp.display(), + names.join(", ") + ) + } + } + Err(error) => format!("{} cannot be read: {error}", llamacpp.display()), + }; + format!( + "no llama-server candidates found; searched {searched}, down to {MAX_SEARCH_DEPTH} \ + directory levels below each (the cpu backend is not searched); {present}" + ) +} + +/// `limit` as the report words it: whole MiB when it is one, else bytes. +fn describe_hash_limit(limit: u64) -> String { + const MIB: u64 = 1024 * 1024; + if limit.is_multiple_of(MIB) { + format!("{} MiB", limit / MIB) + } else { + format!("{limit} bytes") + } +} + +fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { + fingerprint_with_limit(server, deadline, MAX_HASH_BYTES) +} + +fn fingerprint_with_limit(server: &Path, deadline: Instant, limit: u64) -> std::io::Result { + let too_large = || { + std::io::Error::other(format!( + "executable exceeds {} hash limit", + describe_hash_limit(limit) + )) + }; + let mut file = open_no_follow(server)?; + // Do not hash an arbitrarily large corrupt cache entry while reporting a + // failure. Never label a partial hash as SHA256. + if file.metadata()?.len() > limit { + return Err(too_large()); + } + let mut hasher = Sha256::new(); + let mut count = 0; + let mut buffer = vec![0; 64 * 1024]; + loop { + // Check inside the worker, not just around its JoinHandle: timing out a + // spawn_blocking await alone would leave the hashing thread running. + if Instant::now() >= deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "hash evidence budget exhausted", + )); + } + let read = file.read(&mut buffer)?; + count += read as u64; + if count > limit { + return Err(too_large()); + } + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +#[derive(Default)] +struct CapturedOutput { + bytes: Vec, + truncated: bool, +} + +impl CapturedOutput { + fn truncation_note(&self, stream: &str) -> String { + if self.truncated { + format!("\n<{stream} truncated at 32 KiB>") + } else { + String::new() + } + } +} + +async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { + let mut command = tokio::process::Command::new(server); + command.arg(flag); + run_probe(command, timeout).await +} + +/// Run `command` with bounded time and output, and render what came back. +/// +/// Split from [`probe`] so the timeout, kill and reap path can be driven with any +/// child, not only a `llama-server` stand-in: shell-script stand-ins exist on unix +/// only, and this path is what must also hold on Windows. +async fn run_probe(mut command: tokio::process::Command, timeout: Duration) -> String { + command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => return format!("probe could not start: {error}"), + }; + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + let mut out = CapturedOutput::default(); + let mut err = CapturedOutput::default(); + let status = if let Ok((status, stdout_result, stderr_result)) = + tokio::time::timeout(timeout, async { + tokio::join!( + child.wait(), + drain_output(stdout, &mut out), + drain_output(stderr, &mut err), + ) + }) + .await + { + let mut status = status.map_or_else(|error| error.to_string(), |status| status.to_string()); + for (stream, result) in [("stdout", stdout_result), ("stderr", stderr_result)] { + if let Err(error) = result { + let _ = write!(status, "\n{stream} capture failed: {error}"); + } + } + status + } else { + // kill_on_drop also covers cancellation of the whole reporter. + // Explicitly wait here so a normal timeout leaves no zombie child. + let mut status = format!("probe timed out after {}s", timeout.as_secs_f64()); + if let Err(error) = child.kill().await { + let _ = write!(status, "; kill/reap failed: {error}"); + } + status + }; + format!( + "{status}\nstdout:\n{}{}\nstderr:\n{}{}", + String::from_utf8_lossy(&out.bytes), + out.truncation_note("stdout"), + String::from_utf8_lossy(&err.bytes), + err.truncation_note("stderr"), + ) +} + +async fn drain_output( + mut reader: impl AsyncRead + Unpin, + retained: &mut CapturedOutput, +) -> std::io::Result<()> { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + let mut buffer = [0; 4096]; + loop { + let count = reader.read(&mut buffer).await?; + if count == 0 { + return Ok(()); + } + // Continue draining both pipes after the cap, without retaining more + // bytes or writing them to disk. A noisy child must not block on output. + let keep = count.min(limit.saturating_sub(retained.bytes.len())); + retained.bytes.extend_from_slice(&buffer[..keep]); + retained.truncated |= keep < count; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn candidates(runtime: &Path) -> Vec { + discover_candidates(runtime, SEARCH_LIMITS, &|| false).candidates + } + + fn touch_server(dir: &Path) -> PathBuf { + let server = dir.join(server_binary()); + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(&server, "").unwrap(); + server + } + + async fn evidence(manifest: &Path) -> String { + evidence_with(manifest, DEFAULT_BUDGETS).await + } + + async fn evidence_with(manifest: &Path, budgets: Budgets) -> String { + evidence_around_stop(manifest, budgets, &RealProbe, String::new) + .await + .1 + } + + fn write_manifest(runtime: &Path) -> PathBuf { + let manifest = runtime.join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + manifest + } + + /// The candidate paths a report lists, parsed back into paths. + /// + /// Compared as paths, never as printed text: the inventory joins + /// `bin/llamacpp` as one string, so on Windows a report line mixes `/` and + /// `\` while a test that builds the expected path another way prints it + /// differently. `Path` equality compares components, so both spellings agree, + /// and a negative check ("this candidate was skipped") cannot pass vacuously + /// just because the two spellings differ. + fn candidate_paths(report: &str) -> Vec { + report + .lines() + .filter_map(|line| line.strip_prefix("candidate (not proof of selection): ")) + .map(PathBuf::from) + .collect() + } + + #[test] + fn executable_identity_does_not_depend_on_the_dynamic_loader() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("not-executable"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + INVENTORY_BUDGET).unwrap(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn hash_limit_accepts_the_boundary_but_never_returns_a_partial_hash() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + let limit = 3 * 64 * 1024 + 5; + let file = std::fs::File::create(&server).unwrap(); + file.set_len(limit).unwrap(); + assert_eq!( + fingerprint_with_limit(&server, Instant::now() + INVENTORY_BUDGET, limit) + .unwrap() + .len(), + 64 + ); + file.set_len(limit + 1).unwrap(); + assert_eq!( + fingerprint_with_limit(&server, Instant::now() + INVENTORY_BUDGET, limit) + .unwrap_err() + .to_string(), + format!("executable exceeds {limit} bytes hash limit") + ); + } + + /// The cap is not the old 32 MiB: a server above it, such as a static HIP build, + /// must still get a fingerprint on the lane this report is for. The file is + /// sparse, so the test hashes zeros rather than holding real data. + #[test] + fn a_server_above_the_old_32_mib_cap_is_still_fingerprinted() { + const OLD_CAP: u64 = 32 * 1024 * 1024; + const { assert!(MAX_HASH_BYTES >= 256 * 1024 * 1024) }; + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::File::create(&server) + .unwrap() + .set_len(OLD_CAP + 1) + .unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + INVENTORY_BUDGET) + .unwrap() + .len(), + 64 + ); + } + + #[test] + fn hash_limit_is_worded_in_the_unit_it_was_set_in() { + assert_eq!(describe_hash_limit(512 * 1024 * 1024), "512 MiB"); + assert_eq!(describe_hash_limit(1000), "1000 bytes"); + } + + #[test] + fn hash_worker_obeys_its_own_deadline() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now()).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); + } + + #[test] + fn inventory_covers_flat_and_nested_gpu_backends_not_cpu() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut wanted = Vec::new(); + for relative in ["rocm-stable", "rocm-nightly/llama-b9752", "vulkan/build"] { + let server = root.path().join("bin/llamacpp").join(relative).join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, "").unwrap(); + wanted.push(server); + } + let cpu = root.path().join("bin/llamacpp/cpu").join(binary); + std::fs::create_dir_all(cpu.parent().unwrap()).unwrap(); + std::fs::write(cpu, "").unwrap(); + assert_eq!(candidates(root.path()), wanted); + } + + #[test] + fn candidate_search_reaches_three_levels_below_a_backend_and_no_further() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-nightly"); + // Literal layouts, not derived from MAX_SEARCH_DEPTH: the depth is the + // behaviour under test, so changing the constant must turn this red. + let found_at_three = backend.join("a/b/llama-b9752").join(server_binary()); + let found_at_two = backend.join("a/build").join(server_binary()); + let beyond = backend.join("a/b/c/llama-b9752").join(server_binary()); + for server in [&found_at_three, &found_at_two, &beyond] { + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + assert_eq!( + candidates(root.path()), + // Sorted by path, so `a/b/...` precedes `a/build/...`. + vec![found_at_three, found_at_two] + ); + } + + /// An empty search says where it looked and what `bin/llamacpp` holds, and does + /// not guess that a download was interrupted: the same report follows from a + /// layout the search does not cover or a build that was never installed. + #[tokio::test] + async fn empty_search_lists_the_searched_locations_and_blames_nothing() { + let root = tempfile::tempdir().unwrap(); + let cpu = root.path().join("bin/llamacpp/cpu"); + std::fs::create_dir_all(&cpu).unwrap(); + std::fs::write(cpu.join(server_binary()), "").unwrap(); + let report = evidence(&write_manifest(root.path())).await; + let llamacpp = root.path().join("bin/llamacpp"); + for backend in BACKEND_DIRS { + assert!( + report.contains(&llamacpp.join(backend).display().to_string()), + "{report}" + ); + } + assert!( + report.contains("no llama-server candidates found"), + "{report}" + ); + assert!( + report.contains(&format!("{} contains: cpu", llamacpp.display())), + "{report}" + ); + assert!( + !report.contains("download") && !report.contains("complete"), + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + + let bare = tempfile::tempdir().unwrap(); + let report = evidence(&write_manifest(bare.path())).await; + assert!(report.contains("cannot be read"), "{report}"); + } + + fn limits(dirs: usize, entries: usize) -> SearchLimits { + SearchLimits { + dirs, + entries, + candidates: MAX_CANDIDATES, + } + } + + /// The clock is read while the tree is walked, not once before it: it reports + /// time up from its third reading on, so the walk must end partway through, + /// with what it had already found and the reason it stopped. + #[test] + fn the_clock_is_checked_during_the_walk() { + let root = tempfile::tempdir().unwrap(); + let wanted: Vec = BACKEND_DIRS + .iter() + .map(|backend| touch_server(&root.path().join("bin/llamacpp").join(backend))) + .collect(); + let calls = std::sync::atomic::AtomicUsize::new(0); + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2 + }); + assert_eq!(found.stop, Some(Stop::Deadline)); + assert_eq!(found.candidates, wanted[..1], "{found:?}"); + let all = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(all.candidates, wanted); + assert_eq!(all.stop, None); + } + + /// The same, through the inventory: a clock that is already up stops the search + /// even though the hashing deadline is far off, so the walk consults the + /// inventory's clock and not only the hash loop's. + #[tokio::test] + async fn an_inventory_whose_time_is_up_skips_the_search() { + let root = tempfile::tempdir().unwrap(); + touch_server(&root.path().join("bin/llamacpp/rocm-stable")); + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| true), + ) + .await; + let report = &inventory.report; + assert!( + report.contains("remaining evidence skipped: evidence budget exhausted"), + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + assert!( + !report.contains("no llama-server candidates found"), + "an unfinished search must not claim to have found nothing: {report}" + ); + assert!(inventory.servers.is_empty()); + } + + /// The scan is blocking filesystem work and must not run on the async executor. + /// On a current-thread runtime the test's own thread is the executor, so a clock + /// read from any other thread shows the walk ran on a blocking one. + #[tokio::test] + async fn the_directory_scan_runs_off_the_async_executor() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("bin/llamacpp/rocm-stable/build")).unwrap(); + let threads = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let recorded = threads.clone(); + take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(move || { + recorded.lock().unwrap().push(std::thread::current().id()); + false + }), + ) + .await; + let threads = threads.lock().unwrap(); + assert!(!threads.is_empty(), "the walk never read the clock"); + let executor = std::thread::current().id(); + assert!( + threads.iter().all(|thread| *thread != executor), + "the scan ran on the async executor" + ); + } + + /// The limits cover the whole search: two backend directories with three + /// subdirectories each read six entries between them, so a limit of four stops + /// the second directory, which a per-directory limit of four would not. + #[test] + fn entry_and_directory_limits_apply_to_the_whole_search() { + let root = tempfile::tempdir().unwrap(); + for backend in ["rocm-stable", "rocm-nightly"] { + for build in ["a", "b", "c"] { + std::fs::create_dir_all(root.path().join("bin/llamacpp").join(backend).join(build)) + .unwrap(); + } + } + let found = discover_candidates(root.path(), limits(100, 4), &|| false); + assert_eq!(found.stop, Some(Stop::Entries), "{found:?}"); + assert_eq!(found.entries_read, 4, "{found:?}"); + + let found = discover_candidates(root.path(), limits(2, 100), &|| false); + assert_eq!(found.stop, Some(Stop::Directories), "{found:?}"); + assert_eq!(found.dirs_visited, 2, "{found:?}"); + + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!( + found.stop, None, + "the default limits are not the small ones" + ); + } + + /// A search that did not finish must not say it found nothing. The tree is the + /// one a reviewer described: an empty `a`, and a real build under `b`, with the + /// cap reached before `b` is entered. Each way a search can stop early reports + /// its own reason and never the completed-empty-search text, which is asserted + /// absent; a search that did finish and found nothing still gets it. (The + /// candidate cap has no row: it stops a search that already has candidates.) + #[tokio::test] + async fn a_search_cut_short_says_why_and_never_claims_an_empty_result() { + const EMPTY_CLAIM: &str = "no llama-server candidates found"; + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(backend.join("a")).unwrap(); + touch_server(&backend.join("b")); + let manifest = write_manifest(root.path()); + let far = Instant::now() + INVENTORY_BUDGET; + let never: Expired = std::sync::Arc::new(|| false); + let always: Expired = std::sync::Arc::new(|| true); + + for (stop, search_limits, clock, stop_line) in [ + ( + Stop::Directories, + limits(2, 100), + never.clone(), + "candidate search stopped: 2 directories visited (limit)", + ), + ( + Stop::Entries, + limits(100, 1), + never.clone(), + "candidate search stopped: 1 directory entries read (limit)", + ), + ( + Stop::Deadline, + SEARCH_LIMITS, + always, + "remaining evidence skipped: evidence budget exhausted", + ), + ] { + let inventory = take_inventory_with(&manifest, far, search_limits, clock).await; + let report = &inventory.report; + assert!(report.contains(stop_line), "{stop:?}: {report}"); + assert!(!report.contains(EMPTY_CLAIM), "{stop:?}: {report}"); + assert!( + !report.contains("candidate (not proof"), + "{stop:?}: {report}" + ); + } + + let finished = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(finished.path().join("bin/llamacpp/rocm-stable/a")).unwrap(); + let inventory = + take_inventory_with(&write_manifest(finished.path()), far, SEARCH_LIMITS, never).await; + assert!( + inventory.report.contains(EMPTY_CLAIM), + "a search that finished and found nothing says so: {}", + inventory.report + ); + } + + /// The candidate cap ends the walk as soon as it is certain the list is cut, + /// instead of the walk finishing and the list being cut afterwards: with twenty + /// candidate directories only the root and the first nine (eight kept, one more + /// to show there are further ones) are entered, in sorted order. + #[test] + fn the_candidate_cap_stops_the_walk_when_it_is_reached() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + let wanted: Vec = (0..20) + .map(|i| touch_server(&backend.join(format!("{i:02}")))) + .collect(); + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.stop, Some(Stop::Candidates)); + assert_eq!(found.candidates, wanted[..MAX_CANDIDATES]); + assert_eq!(found.dirs_visited, 1 + MAX_CANDIDATES + 1, "{found:?}"); + } + + /// Exactly the cap is a complete list, not a truncated one: no stop, and the + /// report prints no truncation note. A ninth candidate makes it truncated, and + /// then the note is printed. Counting the cap itself as "truncated" turns the + /// first half red. + #[tokio::test] + async fn exactly_the_cap_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + for i in 0..MAX_CANDIDATES { + touch_server(&backend.join(format!("{i:02}"))); + } + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.stop, None, "{found:?}"); + assert_eq!(found.candidates.len(), MAX_CANDIDATES); + let report = evidence(&write_manifest(root.path())).await; + assert!(!report.contains("truncated"), "{report}"); + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + + touch_server(&backend.join(format!("{MAX_CANDIDATES:02}"))); + let report = evidence(&write_manifest(root.path())).await; + assert!( + report.contains("backend inventory truncated after 8 candidates"), + "{report}" + ); + } + + /// When the search ends on the clock after finding candidates, and the hashing + /// deadline has passed too, "remaining evidence skipped" is said once, not once + /// by the search and again by the candidate loop. + #[tokio::test] + async fn the_budget_exhausted_line_is_printed_once() { + let root = tempfile::tempdir().unwrap(); + for backend in BACKEND_DIRS { + touch_server(&root.path().join("bin/llamacpp").join(backend)); + } + let calls = std::sync::atomic::AtomicUsize::new(0); + let calls = std::sync::Arc::new(calls); + let counter = calls.clone(); + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + .checked_sub(Duration::from_secs(1)) + .expect("a second before now"), + SEARCH_LIMITS, + std::sync::Arc::new(move || { + counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2 + }), + ) + .await; + let report = &inventory.report; + assert_eq!( + report + .matches("remaining evidence skipped: evidence budget exhausted") + .count(), + 1, + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + } + + /// A symlink is neither entered nor read as a candidate. The tree holds a link + /// back to its own backend directory (a cycle), a link out to a directory with a + /// real `llama-server`, a `llama-server` that is a link to a file outside, and a + /// whole backend directory that is a link. The walk must end, find only the real + /// file, and count what it declined to follow. + #[cfg(unix)] + #[test] + fn symlinks_are_not_followed_and_a_cycle_ends() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let outside_server = touch_server(outside.path()); + let backends = root.path().join("bin/llamacpp"); + let real = touch_server(&backends.join("rocm-stable/real")); + symlink( + backends.join("rocm-stable"), + backends.join("rocm-stable/loop"), + ) + .unwrap(); + symlink(outside.path(), backends.join("rocm-stable/out")).unwrap(); + symlink( + &outside_server, + backends.join("rocm-stable").join(server_binary()), + ) + .unwrap(); + symlink(outside.path(), backends.join("rocm-nightly")).unwrap(); + // A version marker beside the real server that links out of the tree. + std::fs::write(outside.path().join("version.txt"), "OUTSIDE-VERSION").unwrap(); + symlink( + outside.path().join("version.txt"), + backends.join("rocm-stable/real/version.txt"), + ) + .unwrap(); + + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.candidates, vec![real], "{found:?}"); + assert_eq!(found.stop, None, "{found:?}"); + assert_eq!(found.symlinks_skipped, 5, "{found:?}"); + assert_eq!(found.dirs_visited, 2, "{found:?}"); + } + + /// The refusal is made by the open itself, not by a check before it, so a path + /// that becomes a link after any check is still refused. The helper is called + /// directly on a link, which is what a swap after the metadata check looks like + /// to it; removing `O_NOFOLLOW` makes the open succeed and turns this red. A + /// regular file still opens, and a missing one reports its own error, not a + /// refusal. (Unix only: on Windows the check and the open are separate steps.) + #[cfg(unix)] + #[test] + fn the_open_itself_refuses_a_symlink() { + use std::io::Read as _; + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let real = dir.path().join("real"); + std::fs::write(&real, "OUTSIDE").unwrap(); + let link = dir.path().join("link"); + symlink(&real, &link).unwrap(); + + let error = open_no_follow(&link).unwrap_err(); + assert!(is_symlink_refusal(&error), "{error}"); + assert_eq!(error.to_string(), SYMLINK_NOT_FOLLOWED); + + let mut text = String::new(); + open_no_follow(&real) + .unwrap() + .read_to_string(&mut text) + .unwrap(); + assert_eq!(text, "OUTSIDE"); + let missing = open_no_follow(&dir.path().join("missing")).unwrap_err(); + assert!(!is_symlink_refusal(&missing), "{missing}"); + assert_eq!(missing.kind(), std::io::ErrorKind::NotFound); + } + + /// Every file the report reads from the backend tree is refused when it is a + /// symlink, and the report says so, the way the walk reports a link it did not + /// enter. A real `llama-server` sits beside a `version.txt` that links to a file + /// outside the tree, the pins file is a link out too, and so is a manifest: none + /// of the outside content may appear, and each refusal is named. A fingerprint + /// of a path that has become a link is refused as well. + #[cfg(unix)] + #[tokio::test] + async fn linked_files_are_refused_and_named_not_embedded() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + for (name, text) in [ + ("version.txt", "OUTSIDE-VERSION"), + ("pins.json", "OUTSIDE-PINS"), + ("manifest.json", r#"{"version": "OUTSIDE-MANIFEST"}"#), + ("binary", "OUTSIDE-BINARY"), + ] { + std::fs::write(outside.path().join(name), text).unwrap(); + } + let server = touch_server(&root.path().join("bin/llamacpp/rocm-stable")); + std::fs::write(&server, "abc").unwrap(); + symlink( + outside.path().join("version.txt"), + server.parent().unwrap().join("version.txt"), + ) + .unwrap(); + std::fs::create_dir_all(root.path().join("resources")).unwrap(); + symlink( + outside.path().join("pins.json"), + root.path().join("resources/backend_versions.json"), + ) + .unwrap(); + + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| false), + ) + .await; + let report = &inventory.report; + assert!(!report.contains("OUTSIDE"), "{report}"); + assert!( + report.contains(&format!( + "{}: {SYMLINK_NOT_FOLLOWED}", + server.parent().unwrap().join("version.txt").display() + )), + "{report}" + ); + assert!( + report.contains(&format!( + "{}: {SYMLINK_NOT_FOLLOWED}", + root.path() + .join("resources/backend_versions.json") + .display() + )), + "{report}" + ); + // The real server beside the links is still fingerprinted, and the walk + // counts the one link it saw among the entries. + assert!( + report.contains(&format!("sha256: {ABC_SHA256}")), + "{report}" + ); + assert!( + report.contains("1 symlinked entries under the backend directories were not followed"), + "{report}" + ); + + let linked_manifest = root.path().join("linked-manifest.json"); + symlink(outside.path().join("manifest.json"), &linked_manifest).unwrap(); + let inventory = take_inventory_with( + &linked_manifest, + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| false), + ) + .await; + assert!( + !inventory.report.contains("OUTSIDE"), + "{}", + inventory.report + ); + assert!( + inventory + .report + .contains(&format!("unavailable: {SYMLINK_NOT_FOLLOWED}")), + "{}", + inventory.report + ); + + let linked_binary = root.path().join("linked-binary"); + symlink(outside.path().join("binary"), &linked_binary).unwrap(); + assert_eq!( + fingerprint(&linked_binary, Instant::now() + INVENTORY_BUDGET) + .unwrap_err() + .to_string(), + SYMLINK_NOT_FOLLOWED + ); + } + + #[tokio::test] + async fn the_empty_search_listing_is_bounded_and_counts_what_it_omits() { + let root = tempfile::tempdir().unwrap(); + let llamacpp = root.path().join("bin/llamacpp"); + for i in 0..(MAX_LISTED_NAMES + 4) { + std::fs::create_dir_all(llamacpp.join(format!("other-{i:03}"))).unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + assert!(report.contains(" and 4 more"), "{report}"); + assert!(!report.contains("at least"), "{report}"); + + for i in 0..(2 * MAX_LISTING_SCAN) { + std::fs::create_dir_all(llamacpp.join(format!("other-{i:03}"))).unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + let listing = report + .split("contains: ") + .nth(1) + .and_then(|rest| rest.lines().next()) + .unwrap(); + let (names, omitted) = listing.split_once(" and at least ").unwrap(); + assert_eq!(names.split(", ").count(), MAX_LISTED_NAMES, "{report}"); + assert!( + omitted.starts_with(&format!("{} more", MAX_LISTING_SCAN - MAX_LISTED_NAMES)), + "{report}" + ); + assert!( + omitted.contains(&format!("first {MAX_LISTING_SCAN} entries")), + "{report}" + ); + } + + #[tokio::test] + async fn evidence_truncates_candidates_without_hashing_or_probing_the_rest() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + for i in 0..=MAX_CANDIDATES { + let server = root + .path() + .join(format!("bin/llamacpp/rocm-stable/{i:02}/{binary}")); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + assert_eq!(report.matches("sha256:").count(), MAX_CANDIDATES); + assert_eq!(report.matches("--help:").count(), MAX_CANDIDATES); + assert!(report.contains("backend inventory truncated after 8 candidates")); + let skipped = root + .path() + .join(format!("bin/llamacpp/rocm-stable/08/{binary}")); + let listed = candidate_paths(&report); + assert_eq!(listed.len(), MAX_CANDIDATES, "{report}"); + assert!(!listed.contains(&skipped), "{report}"); + let first = root + .path() + .join(format!("bin/llamacpp/rocm-stable/00/{binary}")); + assert!( + listed.contains(&first), + "the comparison must be able to match a listed candidate: {report}" + ); + } + + /// An executable past the hash cap is not fingerprinted, but the report says + /// so instead of omitting the line, and still lists the candidate and probes it. + #[tokio::test] + async fn oversized_executable_reports_why_it_has_no_fingerprint() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::File::create(&server) + .unwrap() + .set_len(MAX_HASH_BYTES + 1) + .unwrap(); + let report = evidence(&write_manifest(root.path())).await; + assert!( + report.contains(&format!( + "sha256 unavailable: executable exceeds {} hash limit", + describe_hash_limit(MAX_HASH_BYTES) + )), + "{report}" + ); + assert!(!report.contains("sha256: "), "{report}"); + assert_eq!(candidate_paths(&report), vec![server], "{report}"); + assert!(report.contains("--version:"), "{report}"); + } + + #[tokio::test] + async fn exhausted_budget_skips_hashing_and_probes() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + let report = evidence_with( + &write_manifest(root.path()), + Budgets { + inventory: Duration::ZERO, + probes: PROBE_BUDGET, + }, + ) + .await; + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + for skipped in ["sha256", "--version:", "--help:", "candidate ("] { + assert!(!report.contains(skipped), "{report}"); + } + } + + type Events = std::sync::Mutex>; + + struct RecordingProbe<'a> { + events: &'a Events, + } + + impl Probe for RecordingProbe<'_> { + async fn run(&self, _server: &Path, flag: &str, _timeout: Duration) -> String { + self.events.lock().unwrap().push(format!("probe {flag}")); + "recorded, not executed".to_owned() + } + } + + /// A lemonade data dir whose manifest sits where the engine writes it, with one + /// small candidate whose SHA-256 is a known value. + fn lemonade_data_dir_with_candidate() -> (tempfile::TempDir, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let paths = rocm_core::AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let runtime = root.path().join("runtime"); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = runtime.join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, b"abc").unwrap(); + let manifest = paths.engine_install_manifest_path("lemonade"); + std::fs::create_dir_all(manifest.parent().unwrap()).unwrap(); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + (root, manifest, server) + } + + const ABC_SHA256: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + + /// The inventory is read while the failed service is still in place and the + /// probes, which execute a backend binary, run only after the stop. + /// + /// The stop deletes the candidate. A fingerprint in the report therefore proves + /// the file was read before the stop, and the recorded events prove the probes + /// came after it; running either step on the wrong side turns this red. + #[tokio::test] + async fn inventory_precedes_the_stop_and_probes_follow_it() { + let (_root, manifest, server) = lemonade_data_dir_with_candidate(); + let events = Events::default(); + let (stop_status, report) = evidence_around_stop( + &manifest, + DEFAULT_BUDGETS, + &RecordingProbe { events: &events }, + || { + events.lock().unwrap().push("stop".to_owned()); + std::fs::remove_file(&server).unwrap(); + "stopped".to_owned() + }, + ) + .await; + assert_eq!(stop_status, "stopped"); + assert_eq!( + *events.lock().unwrap(), + ["stop", "probe --version", "probe --help"], + "{report}" + ); + assert!( + report.contains(&format!("sha256: {ABC_SHA256}")), + "{report}" + ); + assert!(!report.contains("sha256 unavailable"), "{report}"); + let inventory = report.find("candidate (not proof of selection):").unwrap(); + let probes = report.find("probe target:").unwrap(); + assert!(inventory < probes, "{report}"); + assert!(report.contains("recorded, not executed"), "{report}"); + } + + #[tokio::test] + async fn stop_runs_once_for_every_engine_and_an_unknown_engine_stays_visible() { + let stops = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + "stopped".to_owned() + }; + let no_manifest = + || -> Result { panic!("the manifest is only read for lemonade") }; + + assert_eq!( + backend_evidence_around_stop(no_manifest, Some("vllm"), stop).await, + ("stopped".to_owned(), None) + ); + assert_eq!( + backend_evidence_around_stop(no_manifest, None, stop).await, + ( + "stopped".to_owned(), + Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()) + ) + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 2); + + let (_root, manifest, _server) = lemonade_data_dir_with_candidate(); + let (status, evidence) = + backend_evidence_around_stop(|| Ok(manifest), Some("lemonade"), stop).await; + assert_eq!(status, "stopped"); + assert!(evidence.unwrap().contains("manifest: ")); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 3); + } + + /// A Lemonade serve with nothing to look the manifest up in (the scenario has no + /// isolated root) used to panic before the stop, so the stop never ran and the + /// whole failure report was lost. The lookup now fails soft, whether it says so + /// (`Err`) or panics, and the stop still runs exactly once. Removing the panic + /// guard turns the second case red. + #[tokio::test] + async fn a_failing_manifest_lookup_still_runs_the_stop_once_and_reports_why() { + let stops = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + "stopped".to_owned() + }; + let (status, evidence) = backend_evidence_around_stop( + || Err("the scenario has no isolated root".to_owned()), + Some("lemonade"), + stop, + ) + .await; + assert_eq!(status, "stopped"); + assert_eq!( + evidence.as_deref(), + Some("backend evidence unavailable: the scenario has no isolated root") + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 1); + + let (status, evidence) = backend_evidence_around_stop( + || -> Result { panic!("no isolated root") }, + Some("lemonade"), + stop, + ) + .await; + assert_eq!(status, "stopped"); + assert_eq!( + evidence.as_deref(), + Some("backend evidence unavailable: finding the runtime manifest failed") + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn an_unreadable_manifest_is_reported_and_the_service_is_still_stopped() { + let root = tempfile::tempdir().unwrap(); + let events = Events::default(); + let (stop_status, report) = evidence_around_stop( + &root.path().join("missing.json"), + DEFAULT_BUDGETS, + &RecordingProbe { events: &events }, + || "stopped".to_owned(), + ) + .await; + assert_eq!(stop_status, "stopped"); + assert!(report.contains("unavailable:"), "{report}"); + assert!(events.lock().unwrap().is_empty(), "no candidate, no probe"); + assert!(!report.contains("probe target:"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn timed_out_probe_leaves_no_budget_for_the_next_probe() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("bin/llamacpp/rocm-stable/llama-server"); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nexec /bin/sleep 30\n", + ) + .unwrap(); + let report = evidence_with( + &write_manifest(root.path()), + Budgets { + inventory: INVENTORY_BUDGET, + probes: Duration::from_secs(1), + }, + ) + .await; + assert!(report.contains("--version:\nprobe timed out"), "{report}"); + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + assert!(!report.contains("--help:"), "{report}"); + } + + #[tokio::test] + async fn stream_capture_marks_only_bytes_actually_discarded() { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + for length in [0, limit - 1, limit, limit + 1] { + let bytes = vec![b'x'; length]; + let mut captured = CapturedOutput::default(); + drain_output(bytes.as_slice(), &mut captured).await.unwrap(); + assert_eq!(captured.bytes, vec![b'x'; length.min(limit)]); + assert_eq!(captured.truncated, length > limit); + assert_eq!( + captured.truncation_note("stdout"), + if length > limit { + "\n" + } else { + "" + } + ); + } + } + + #[tokio::test] + async fn stream_truncation_survives_timeout_before_eof() { + use tokio::io::AsyncWriteExt as _; + let length = MAX_EVIDENCE_BYTES as usize / 2 + 1; + let (reader, mut writer) = tokio::io::duplex(length); + writer.write_all(&vec![b'x'; length]).await.unwrap(); + let mut captured = CapturedOutput::default(); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + drain_output(reader, &mut captured) + ) + .await + .is_err() + ); + assert_eq!(captured.bytes.len(), length - 1); + assert_eq!( + captured.truncation_note("stderr"), + "\n" + ); + } + + #[tokio::test] + async fn missing_manifest_is_reported_without_panicking() { + let root = tempfile::tempdir().unwrap(); + let report = evidence(&root.path().join("missing.json")).await; + assert!(report.contains("unavailable:"), "{report}"); + } + + #[tokio::test] + async fn failed_download_keeps_pins_but_does_not_invent_an_installed_build() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + std::fs::create_dir_all(runtime.join("resources")).unwrap(); + std::fs::write( + runtime.join("resources/backend_versions.json"), + r#"{"llamacpp":{"rocm-stable":"b9752"}}"#, + ) + .unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = evidence(&manifest).await; + assert!( + report.contains("configured pins (not executable identity):"), + "{report}" + ); + assert!(report.contains("\"rocm-stable\":\"b9752\""), "{report}"); + assert!(report.contains("no llama-server candidates"), "{report}"); + } + + #[tokio::test] + async fn probe_start_error_is_evidence_not_a_second_failure() { + let root = tempfile::tempdir().unwrap(); + let report = probe(&root.path().join("missing"), "--version", PROBE_TIMEOUT).await; + assert!(report.contains("probe could not start:"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn inventory_probes_the_cached_executable_not_a_path_lookup() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + let dir = runtime.join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(&dir).unwrap(); + let server = dir.join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "\ +#!/bin/sh +case \"$1\" in + --version) echo 'version: 9752 (fixture-build)' ;; + --help) echo '--mmap, --no-mmap'; echo 'fixture help stderr' >&2; exit 2 ;; + *) exit 99 ;; +esac +", + ) + .unwrap(); + std::fs::write(dir.join("version.txt"), "cached-build").unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = evidence(&manifest).await; + assert!(report.contains(&server.display().to_string()), "{report}"); + assert!(report.contains("version: 9752 (fixture-build)"), "{report}"); + assert!(report.contains("version.txt: cached-build"), "{report}"); + assert!( + report.contains("--help:\nexit status: 2\nstdout:"), + "{report}" + ); + assert!(!report.contains("capture"), "{report}"); + assert!(!report.contains("Ok("), "{report}"); + assert!(report.contains("--mmap, --no-mmap"), "{report}"); + assert!(report.contains("fixture help stderr"), "{report}"); + assert!( + server.is_file(), + "diagnostics must not repair or remove the cache" + ); + } + + /// Selects what [`probe_child`] does when a probe test re-runs this test binary + /// as its child process. + const PROBE_CHILD_ENV: &str = "ROCM_E2E_PROBE_CHILD"; + + /// Not a test: the child process the probe tests below start. + /// + /// The probe's timeout, kill and reap path has to hold on Windows too, and a + /// shell-script stand-in does not exist there. This test binary does, on every + /// platform, so the tests re-run it with `--ignored --exact` and this function + /// plays the stand-in. It is `#[ignore]`d so a normal run skips it, and does + /// nothing unless [`PROBE_CHILD_ENV`] selects a behaviour. + #[test] + #[ignore = "child process of the probe tests, not a test of its own"] + fn probe_child() { + match std::env::var(PROBE_CHILD_ENV).as_deref() { + Ok("sleep") => { + // With one test thread (RUST_TEST_THREADS=1, or one CPU) libtest + // writes "test ... " first and our output continues that + // line, so start a fresh one: the parent reads the first line that + // begins with the marker. + println!("\npid:{}", std::process::id()); + std::thread::sleep(Duration::from_secs(CHILD_LIFETIME_SECS)); + } + Ok("exit") => println!("child-ran"), + _ => {} + } + } + + fn probe_child_command(behaviour: &str) -> tokio::process::Command { + let module = module_path!().split_once("::").unwrap().1; + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + &format!("{module}::probe_child"), + "--ignored", + "--nocapture", + ]) + .env(PROBE_CHILD_ENV, behaviour); + command + } + + /// How long each attempt at the timeout test lets the helper child run before the + /// probe's timeout fires. The first is short, so a healthy run is quick; the + /// later ones are generous and matter only when the child was too slow to start + /// and print its pid inside the earlier one (a loaded runner, a cold test + /// binary), which is a slow start and not a failure of the probe. + const CHILD_START_ATTEMPTS: [u64; 3] = [5, 20, 60]; + + /// How much longer than the longest attempt the helper child stays alive. The + /// child must still be running when the probe's timeout fires on any attempt: + /// one that had exited by itself would be a normal exit, not a timeout, and the + /// test would fail for a correct probe. + const CHILD_LIFETIME_MARGIN_SECS: u64 = 60; + + /// The helper child's lifetime, derived from the attempt timeouts so the two + /// cannot drift apart. + const CHILD_LIFETIME_SECS: u64 = { + let mut longest = 0; + let mut at = 0; + while at < CHILD_START_ATTEMPTS.len() { + if CHILD_START_ATTEMPTS[at] > longest { + longest = CHILD_START_ATTEMPTS[at]; + } + at += 1; + } + longest + CHILD_LIFETIME_MARGIN_SECS + }; + + /// The helper child outlives every attempt's timeout by at least half a minute + /// (the margin is asserted as a literal so shrinking the constant fails), so the + /// timeout, kill and reap path is what ends it on each retry. Giving the child a + /// literal lifetime that a longer attempt can overtake (it was 30 seconds against + /// a 60-second attempt) turns this red. + #[test] + fn the_helper_child_outlives_every_attempt_deadline() { + for attempt in CHILD_START_ATTEMPTS { + assert!( + CHILD_LIFETIME_SECS >= attempt + 30, + "the child lives {CHILD_LIFETIME_SECS}s but an attempt waits {attempt}s" + ); + } + } + + /// One run of the timeout path against a helper child that libtest runs with + /// `test_threads` test threads: the child is still running when the timeout + /// fires, so the report must say so, return promptly, and leave no process + /// behind. `process_is_running` is false only once the child has been killed + /// and reaped, so a probe that merely abandoned it turns this red. + /// + /// The pid is read back from the report, so an attempt whose child had not + /// started before the timeout (no pid line) is retried with a longer timeout + /// rather than failed; only an attempt that did report a pid is asserted on. + async fn assert_timeout_kills_and_reaps(test_threads: &str) { + let mut last = String::new(); + for seconds in CHILD_START_ATTEMPTS { + let start = Instant::now(); + let mut command = probe_child_command("sleep"); + command.env("RUST_TEST_THREADS", test_threads); + let report = run_probe(command, Duration::from_secs(seconds)).await; + assert!( + report.starts_with(&format!("probe timed out after {seconds}s")), + "{report}" + ); + assert!(!report.contains("kill/reap failed"), "{report}"); + assert!( + start.elapsed() < Duration::from_secs(seconds + 15), + "{report}" + ); + let Some(pid) = report + .lines() + .find_map(|line| line.strip_prefix("pid:")) + .map(|pid| pid.parse::().unwrap()) + else { + last = report; + continue; + }; + assert!(!rocm_core::process_is_running(pid), "{report}"); + return; + } + panic!( + "the child never reported its pid, even with {CHILD_START_ATTEMPTS:?} seconds \ + (test threads {test_threads}): {last}" + ); + } + + /// The timeout path on every platform, with the child run both on one test + /// thread and on several. With one thread (also what a one-CPU host gets) libtest + /// prefixes the child's output with its test-name label, so a pid printed on + /// that same line is not found by a line-start match. The two runs overlap, so + /// this takes one timeout, not two. + #[tokio::test] + async fn a_probe_that_outlives_its_timeout_is_killed_and_reaped() { + tokio::join!( + assert_timeout_kills_and_reaps("1"), + assert_timeout_kills_and_reaps("8"), + ); + } + + /// The normal path on every platform: status and output come back, and the + /// timeout branch is not taken. + #[tokio::test] + async fn a_probe_that_exits_reports_its_status_and_output() { + let report = run_probe(probe_child_command("exit"), Duration::from_mins(1)).await; + let status = if cfg!(windows) { + "exit code: 0\n" + } else { + "exit status: 0\n" + }; + assert!(report.starts_with(status), "{report}"); + assert!(report.contains("child-ran"), "{report}"); + assert!(!report.contains("timed out"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn probe_timeout_is_bounded() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + // exec keeps the sleeper in the child's PID, so cancellation kills it + // rather than leaving a grandchild behind. + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\necho $$\nexec /bin/sleep 30\n", + ) + .unwrap(); + let start = std::time::Instant::now(); + let report = probe(&server, "--help", Duration::from_secs(1)).await; + assert!( + report.contains("probe timed out after 1s\nstdout:"), + "{report}" + ); + assert!(start.elapsed() < Duration::from_secs(5)); + let pid = report + .split("stdout:\n") + .nth(1) + .unwrap() + .lines() + .next() + .unwrap() + .parse::() + .unwrap(); + assert!(!rocm_core::process_is_running(pid), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn exact_stream_limit_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 32768 /dev/zero\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains("exit status: 0"), "{report}"); + assert_eq!(report.bytes().filter(|byte| *byte == 0).count(), 32768); + assert!(!report.contains("capped"), "{report}"); + assert!(!report.contains("truncated"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn noisy_probe_output_is_capped() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 100000 /dev/zero\nhead -c 100000 /dev/zero >&2\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains(""), "{report}"); + assert!(report.ends_with(""), "{report}"); + assert_eq!( + report.bytes().filter(|byte| *byte == 0).count(), + MAX_EVIDENCE_BYTES as usize + ); + assert!(report.len() < MAX_EVIDENCE_BYTES as usize + 300); + } +} diff --git a/tests/e2e-cucumber/src/lemonade_protocol.rs b/tests/e2e-cucumber/src/lemonade_protocol.rs new file mode 100644 index 0000000000..5ab4ab6f5b --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_protocol.rs @@ -0,0 +1,153 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! HTTP protocol fixture, not a real Lemonade runtime or inference engine. +//! Runs on its own thread so blocking CLI subprocesses cannot stall its routes. + +use std::sync::{Arc, Mutex}; + +use axum::{ + Json, Router, + extract::{Request, State}, + middleware::{self, Next}, + response::Response, + routing::get, +}; +use serde_json::{Value, json}; + +use crate::http_server::{self, ServerHandle}; + +#[derive(Debug)] +pub struct LemonadeProtocolServer { + server: ServerHandle, + loaded: Arc>>, + stray: Arc>>, +} + +impl LemonadeProtocolServer { + pub fn start(models: &[&str]) -> Self { + let loaded = Arc::new(Mutex::new(models.iter().map(|s| (*s).to_owned()).collect())); + let stray = Arc::new(Mutex::new(Vec::new())); + let app = Router::new() + .route("/api/v1/health", get(health)) + .route( + "/api/v1/stats", + get(|| async { + Json(json!({ + "tokens_per_second": 37.5, + "time_to_first_token": 0.05, + "input_tokens": 8, + "output_tokens": 16 + })) + }), + ) + .with_state(Arc::clone(&loaded)) + .layer(middleware::from_fn_with_state( + Arc::clone(&stray), + record_stray, + )); + Self { + server: http_server::spawn_on_own_thread(app), + loaded, + stray, + } + } + + pub const fn port(&self) -> u16 { + self.server.port() + } + + pub fn base_url(&self) -> String { + self.server.base_url() + } + + /// Every request that was not a `GET` of the health or stats route, as + /// `" "`. A client that unloads, loads, or stops a model on + /// this server, or probes any other route, shows up here. + pub fn stray_requests(&self) -> Vec { + self.stray.lock().expect("fixture state poisoned").clone() + } + + pub fn set_loaded(&self, models: &[&str]) { + *self.loaded.lock().expect("fixture state poisoned") = + models.iter().map(|s| (*s).to_owned()).collect(); + } +} + +async fn record_stray( + State(stray): State>>>, + request: Request, + next: Next, +) -> Response { + let method = request.method().clone(); + let path = request.uri().path().to_owned(); + let allowed = method == axum::http::Method::GET + && matches!(path.as_str(), "/api/v1/health" | "/api/v1/stats"); + if !allowed { + stray + .lock() + .expect("fixture state poisoned") + .push(format!("{method} {path}")); + } + next.run(request).await +} + +async fn health(State(loaded): State>>>) -> Json { + let loaded = loaded.lock().expect("fixture state poisoned"); + Json(json!({ + "status": "ok", + "model_loaded": loaded.first(), + "all_models_loaded": loaded + .iter() + .map(|model| json!({"model_name": model})) + .collect::>() + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn records_requests_other_than_get_health_and_stats() { + let fixture = LemonadeProtocolServer::start(&["a", "b"]); + let base = fixture.base_url(); + let client = reqwest::Client::new(); + client + .get(format!("{base}/api/v1/health")) + .send() + .await + .unwrap(); + client + .get(format!("{base}/api/v1/stats")) + .send() + .await + .unwrap(); + assert!(fixture.stray_requests().is_empty()); + + client + .post(format!("{base}/api/v1/unload")) + .send() + .await + .unwrap(); + client + .get(format!("{base}/api/v1/models")) + .send() + .await + .unwrap(); + client + .delete(format!("{base}/api/v1/health")) + .send() + .await + .unwrap(); + assert_eq!( + fixture.stray_requests(), + vec![ + "POST /api/v1/unload", + "GET /api/v1/models", + "DELETE /api/v1/health" + ] + ); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index acd458f969..29e3345836 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -2,13 +2,19 @@ // // SPDX-License-Identifier: MIT +pub mod binary_path; pub mod capability; pub mod chat_request; +pub mod dash_screen; +pub mod device_memory; pub mod expectation; pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lemonade_backend; +pub mod lemonade_diagnostics; +pub mod lemonade_protocol; pub mod loopback_http; pub mod mock_server; pub mod model_id; @@ -16,6 +22,7 @@ pub mod monotonic_clock; pub mod paced_download; pub mod palette_rows; pub mod panic_capture; +pub mod planted_path; pub mod polled_deadline; pub mod powershell_text_rule; pub mod quit_and_wait; @@ -25,7 +32,11 @@ pub mod send_until; pub mod serve_log; pub mod serve_plan; pub mod shared_runtime; +pub mod stand_in_uname; +pub mod stray_process; +pub mod tasklist; pub mod terminal_size; +pub mod torch_alignment_outcome; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/planted_path.rs b/tests/e2e-cucumber/src/planted_path.rs new file mode 100644 index 0000000000..e26c46d868 --- /dev/null +++ b/tests/e2e-cucumber/src/planted_path.rs @@ -0,0 +1,179 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `PATH` a scenario hands the `rocm` child when it plants a machine that +//! holds a `rocminfo` of its own. +//! +//! The scenarios about whether `rocminfo` is installed must not be decided by a +//! `rocminfo` the runner happens to have, so the runner's own `PATH` entries +//! that hold one are left out. The entries are written by the harness process +//! but read by the child, and the two can run from different directories (the +//! scenario's `command_dir`), so a relative entry such as `.` means two +//! different directories. [`planted_path_entries`] settles that before it +//! filters: each inherited relative entry is made absolute against the +//! harness's own working directory, and is kept or dropped by what that +//! directory holds. The fixture's own entries are never touched, because a +//! scenario writes them to be read from the child's directory (an empty entry +//! and `.` are the point of two of them). + +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; + +/// The program the runner's entries are checked for. +pub const PLANTED_TOOL: &str = "rocminfo"; + +/// Whether `dir` would make [`PLANTED_TOOL`] resolve. +/// +/// It holds a regular file of that name, as the shared `which` lookup in +/// `rocm-core` requires. A directory called `rocminfo` does not, so it must not +/// get its `PATH` entry dropped. +#[must_use] +pub fn dir_holds_tool(dir: &Path) -> bool { + dir.join(PLANTED_TOOL).is_file() +} + +/// The entries of the planted `PATH`, in order: `fixture` exactly as written, +/// then `fake_bin`, then each non-empty entry of `inherited` that holds no +/// [`PLANTED_TOOL`]. +/// +/// An inherited relative entry is resolved against `harness_cwd` first, checked +/// there with `holds_tool`, and kept in its resolved (absolute) form, so it still +/// names the directory it named when the harness read it after the child has +/// moved. An inherited empty entry is dropped: it would be read as the child's +/// working directory, which the scenario decides and the runner must not. +pub fn planted_path_entries( + fixture: &[PathBuf], + fake_bin: &Path, + inherited: &OsStr, + harness_cwd: &Path, + holds_tool: impl Fn(&Path) -> bool, +) -> Vec { + let kept = std::env::split_paths(inherited) + .filter(|entry| !entry.as_os_str().is_empty()) + // `join` leaves an absolute entry as it is and roots a relative one. + .map(|entry| harness_cwd.join(entry)) + .filter(|resolved| !holds_tool(resolved)); + fixture + .iter() + .cloned() + .chain(std::iter::once(fake_bin.to_owned())) + .chain(kept) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// An absolute directory on any host, so the rows are about the resolution + /// rule and not about one platform's path syntax. + fn abs(name: &str) -> PathBuf { + std::env::temp_dir().join("planted-path-rows").join(name) + } + + fn joined(entries: &[PathBuf]) -> std::ffi::OsString { + std::env::join_paths(entries).expect("no entry holds a separator") + } + + /// The planted entries for a host whose `rocminfo` is in exactly `holding`, + /// with the harness running from `abs("harness")`. + fn planted(fixture: &[PathBuf], inherited: &[PathBuf], holding: &[PathBuf]) -> Vec { + planted_path_entries( + fixture, + &abs("fake-bin"), + &joined(inherited), + &abs("harness"), + |dir| holding.iter().any(|held| held == dir), + ) + } + + #[test] + fn an_inherited_dot_without_a_rocminfo_survives_as_the_harness_directory() { + // The child runs from elsewhere, where a bare `.` would name the planted + // directory and expose its stand-in, so it must not survive as `.`. + assert_eq!( + planted(&[], &[PathBuf::from(".")], &[]), + vec![abs("fake-bin"), abs("harness")] + ); + } + + #[test] + fn an_inherited_dot_with_a_rocminfo_in_the_harness_directory_is_dropped() { + assert_eq!( + planted(&[], &[PathBuf::from(".")], &[abs("harness")]), + vec![abs("fake-bin")] + ); + } + + #[test] + fn an_inherited_absolute_entry_is_checked_and_kept_as_it_is() { + assert_eq!( + planted(&[], &[abs("usr-bin"), abs("rocm-bin")], &[abs("rocm-bin")]), + vec![abs("fake-bin"), abs("usr-bin")] + ); + } + + #[test] + fn an_inherited_relative_name_is_resolved_against_the_harness_directory() { + assert_eq!( + planted( + &[], + &[PathBuf::from("bin"), PathBuf::from("tools")], + &[abs("harness").join("tools")] + ), + vec![abs("fake-bin"), abs("harness").join("bin")] + ); + } + + #[test] + fn inherited_empty_entries_are_dropped_wherever_they_stand() { + assert_eq!( + planted(&[], &[PathBuf::new(), abs("usr-bin"), PathBuf::new()], &[]), + vec![abs("fake-bin"), abs("usr-bin")] + ); + } + + #[test] + fn fixture_entries_come_first_and_are_left_exactly_as_written() { + // The empty entry, `.` and a name are what the scenarios are about; the + // harness directory holding a `rocminfo` makes no difference to them. + let fixture = [PathBuf::new(), PathBuf::from("."), PathBuf::from("odd")]; + assert_eq!( + planted( + &fixture, + &[PathBuf::from(".")], + &[abs("harness"), PathBuf::from("."), PathBuf::from("odd")] + ), + vec![ + PathBuf::new(), + PathBuf::from("."), + PathBuf::from("odd"), + abs("fake-bin") + ] + ); + } + + #[test] + fn a_directory_named_like_the_tool_does_not_hold_it_and_a_file_does() { + let scratch = tempfile::tempdir().expect("scratch directory"); + let with_dir = scratch.path().join("with-dir"); + let with_file = scratch.path().join("with-file"); + std::fs::create_dir_all(with_dir.join(PLANTED_TOOL)).expect("plant a directory"); + std::fs::create_dir_all(&with_file).expect("create the directory"); + std::fs::write(with_file.join(PLANTED_TOOL), "").expect("plant a file"); + + assert!(!dir_holds_tool(&with_dir)); + assert!(dir_holds_tool(&with_file)); + // Through the planting itself: the entry with only a directory of that + // name is kept, the entry with a file is dropped. + let entries = planted_path_entries( + &[], + &abs("fake-bin"), + &joined(&[with_dir.clone(), with_file]), + &abs("harness"), + dir_holds_tool, + ); + assert_eq!(entries, vec![abs("fake-bin"), with_dir]); + } +} diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index bf0ff9d8c1..b990259855 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -27,6 +27,21 @@ //! deleted with the scenario. use std::path::Path; +use std::time::Duration; + +/// Extra time the `rocm serve` process gets beyond the model readiness timeout. +/// +/// The CLI's own readiness wait starts only after its pre-launch work (backend +/// and weights fetch), while the harness clock starts at spawn. Bounding the +/// process by the bare timeout would kill it first and lose the CLI's own +/// diagnostic in a slow cold start. The margin still bounds a stuck serve. +pub const SERVE_CLI_MARGIN_SECS: u64 = 300; + +/// How long to let a `rocm serve` process run: the readiness timeout plus +/// [`SERVE_CLI_MARGIN_SECS`]. Model readiness polling keeps the plain timeout. +pub const fn serve_cli_budget(timeout_secs: u64) -> Duration { + Duration::from_secs(timeout_secs.saturating_add(SERVE_CLI_MARGIN_SECS)) +} /// How many trailing lines of a stalled service's log to quote in a failure. const DEFAULT_TAIL_LINES: usize = 40; @@ -55,6 +70,44 @@ fn parse_log_path(serve_stdout: &str) -> Option<&str> { .filter(|path| !path.is_empty()) } +/// The selected engine, if the serve reached the point of printing its plan. +#[must_use] +pub fn reported_engine(serve_stdout: &str) -> Option<&str> { + serve_stdout + .lines() + .find_map(|line| line.trim().strip_prefix("engine:")) + .map(str::trim) + .filter(|engine| !engine.is_empty()) +} + +/// The engine named on a `rocm serve` command line. +/// +/// Accepts `--engine X` and `--engine=X`. `None` when no engine was requested: +/// the CLI then picks a default, which only the printed plan reveals (see +/// [`reported_engine`]). A value that starts with `--` is another flag, not an +/// engine (`serve m --engine --managed`), so it counts as absent too: reporting +/// `--managed` as the engine would skip the backend inventory without the +/// [`ENGINE_UNKNOWN_EVIDENCE`] line that makes the skip visible. +#[must_use] +pub fn engine_from_serve_args<'a>(args: &[&'a str]) -> Option<&'a str> { + let mut args = args.iter().copied(); + while let Some(arg) = args.next() { + let value = if let Some(inline) = arg.strip_prefix("--engine=") { + Some(inline) + } else if arg == "--engine" { + args.next() + } else { + continue; + }; + return value.filter(|engine| !engine.is_empty() && !engine.starts_with("--")); + } + None +} + +/// What a failed serve's report says when no engine could be determined, so a +/// skipped backend inventory is visible rather than silently absent. +pub const ENGINE_UNKNOWN_EVIDENCE: &str = "engine unknown: backend evidence skipped"; + /// The last `max` lines of `text`. fn tail_lines(text: &str, max: usize) -> String { let lines: Vec<&str> = text.lines().collect(); @@ -119,16 +172,55 @@ fn slugify(name: &str) -> String { /// this is collecting. Seeking keeps the cost fixed however big the log grows. /// /// The log may still be being written, so `len` is a snapshot: the head and tail -/// are always real, and only the elided count can be slightly stale. -fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { +/// are always real, and only the elided count can be slightly stale. A log that +/// grows past the cap after the size was taken is clamped the same way and the +/// note says it grew (see [`read_clamped_file`]); no caller gets a silent +/// truncation. Callers: `archive_service_log` (the note lands on the archived-path +/// line) and the Lemonade report's pins and `version.txt` reads (the note follows +/// the quoted text); its manifest read discards the note, but a truncated JSON +/// manifest fails to parse and the report says so. +pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { + let file = std::fs::File::open(path)?; + let len = file.metadata()?.len(); + read_clamped_file(file, len, max) +} + +/// [`read_clamped`] on an open file whose size was read as `len`. Split out so a +/// test can hand it a `len` that has gone stale, which is what a log that grows +/// between the size check and the read looks like. +/// +/// A file that fits is read in full, but one byte past `max` is asked for: if it +/// is there, the log grew after `len` was taken. That is not allowed to pass as +/// "fits" with the surplus dropped without a word, so the read falls back to the +/// head and tail of the file as it is now and the note says it grew. A file that +/// shrank instead is returned as read, with a note that it changed. +pub(crate) fn read_clamped_file( + mut file: std::fs::File, + len: u64, + max: u64, +) -> std::io::Result<(Vec, String)> { use std::io::{Read as _, Seek as _, SeekFrom}; - let mut file = std::fs::File::open(path)?; - let len = file.metadata()?.len(); + let mut len = len; + let mut grown = false; if len <= max { let mut body = Vec::new(); - file.read_to_end(&mut body)?; - return Ok((body, String::new())); + file.by_ref() + .take(max.saturating_add(1)) + .read_to_end(&mut body)?; + if body.len() as u64 <= max { + return Ok((body, String::new())); + } + len = file.metadata()?.len(); + if len <= max { + body.truncate(usize::try_from(max).unwrap_or(usize::MAX)); + return Ok(( + body, + format!(" (truncated at {max} bytes: the file changed while it was read)"), + )); + } + file.seek(SeekFrom::Start(0))?; + grown = true; } // `len > max >= 2 * half`, so the two halves can never overlap. let half = max / 2; @@ -143,7 +235,15 @@ fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { let mut body = head; body.extend_from_slice(marker.as_bytes()); body.extend_from_slice(&tail); - Ok((body, format!(" ({elided} bytes elided from the middle)"))) + let grown_note = if grown { + "; the file grew while it was read" + } else { + "" + }; + Ok(( + body, + format!(" ({elided} bytes elided from the middle{grown_note})"), + )) } /// Copy the log written by the managed service `serve_stdout` launched into @@ -230,6 +330,8 @@ pub struct ServeAttempt<'a> { /// failed, means the engine may still hold the port and the device — which /// changes how every later failure in the run should be read. pub stop_status: &'a str, + /// Only Lemonade attempts carry the optional backend inventory. + pub backend_evidence: Option<&'a str>, } /// Render one serve attempt's evidence as a failure message. @@ -241,7 +343,7 @@ pub struct ServeAttempt<'a> { #[must_use] pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { use crate::section; - format!( + let mut report = format!( "{}\n{}\n{}\n{}\n{}\n{}\n{}", attempt.headline, section("device state", attempt.device_state), @@ -250,16 +352,32 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { section("service log (tail)", attempt.log_tail), section("service log (full)", attempt.archived_log), section("stop", attempt.stop_status), - ) + ); + if let Some(evidence) = attempt.backend_evidence { + report.push_str("\n--- Lemonade backend evidence ---\n"); + report.push_str(evidence); + } + report } #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, parse_log_path, read_clamped, - serve_attempt_report, service_log_tail, slugify, tail_lines, + MAX_ARCHIVED_LOG_BYTES, SERVE_CLI_MARGIN_SECS, ServeAttempt, archive_service_log, + engine_from_serve_args, parse_log_path, read_clamped, read_clamped_file, reported_engine, + serve_attempt_report, serve_cli_budget, service_log_tail, slugify, tail_lines, }; + #[test] + fn cli_budget_outlasts_the_readiness_timeout_by_a_bounded_margin() { + for timeout in [90, 2400] { + let budget = serve_cli_budget(timeout).as_secs(); + assert!(budget > timeout, "the CLI must outlive the readiness wait"); + assert_eq!(budget, timeout + SERVE_CLI_MARGIN_SECS); + } + assert_eq!(serve_cli_budget(u64::MAX).as_secs(), u64::MAX); + } + const SERVE_PLAN: &str = "\ serve plan requested model: Qwen/Qwen3.5-0.8B @@ -270,6 +388,38 @@ managed service launched readiness: starting "; + #[test] + fn engine_lookup_is_safe_for_missing_or_incomplete_serve_plans() { + assert_eq!(reported_engine(SERVE_PLAN), Some("vllm")); + assert_eq!(reported_engine(" engine: lemonade\n"), Some("lemonade")); + for output in ["", "serve failed", " engine: \n"] { + assert_eq!(reported_engine(output), None); + } + } + + #[test] + fn engine_is_read_from_both_flag_forms_and_absent_otherwise() { + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine", "lemonade", "--managed"]), + Some("lemonade") + ); + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine=lemonade", "--managed"]), + Some("lemonade") + ); + for args in [ + &["serve", "m", "--managed"][..], + &["serve", "m", "--engine"], + &["serve", "m", "--engine="], + &["serve", "m", "--engine", ""], + &["serve", "m", "--engine", "--managed"], + &["serve", "m", "--engine=--managed"], + &["serve", "--engine-version=1"], + ] { + assert_eq!(engine_from_serve_args(args), None, "{args:?}"); + } + } + #[test] fn log_path_is_read_from_the_indented_plan_line() { assert_eq!( @@ -395,6 +545,34 @@ managed service launched read_clamped(&path, max).expect("read log") } + /// A log that grows after its size was read must not be cut at the cap without + /// saying so. `len` is handed in stale (10 bytes under a 20-byte cap) for a file + /// that now holds 60: the read keeps the head and tail of what is there and the + /// note says it grew. Cutting at the cap silently turns this red, and a file + /// that really fits (stale or not) stays verbatim with no note. + #[test] + fn a_log_that_grew_after_its_size_was_read_is_clamped_and_says_so() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("service.log"); + let body: Vec = (0..60u8).map(|i| b'a' + i % 26).collect(); + std::fs::write(&path, &body).expect("write log"); + + let (clamped, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 10, 20).expect("read"); + assert!(note.contains("the file grew while it was read"), "{note}"); + assert!(note.contains("40 bytes elided from the middle"), "{note}"); + let text = String::from_utf8_lossy(&clamped); + assert!(text.starts_with(&String::from_utf8_lossy(&body[..10]).to_string())); + assert!(text.ends_with(&String::from_utf8_lossy(&body[50..]).to_string())); + + let (whole, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 60, 100).expect("read"); + assert_eq!((whole, note), (body.clone(), String::new())); + let (exact, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 60, 60).expect("read"); + assert_eq!((exact, note), (body, String::new())); + } + /// A runaway log is bounded, but never at the cost of the startup banner: /// both ends survive and the gap says so. #[test] @@ -502,7 +680,7 @@ managed service launched /// full log went — none of which the plain readiness timeout reported. #[test] fn a_stalled_serve_report_carries_every_piece_of_evidence() { - let report = serve_attempt_report(&ServeAttempt { + let mut attempt = ServeAttempt { headline: "`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --engine lemonade --managed` \ exited 0, but the endpoint never served `Qwen3-0.6B` within 600s", device_state: "device state: drained (48000 MiB free of 49000 MiB, floor 44100 MiB)", @@ -511,7 +689,15 @@ managed service launched log_tail: "load: failed to load model", archived_log: "archived to service-logs/scenario--svc.log", stop_status: "lemonade-qwen: stopped", - }); + backend_evidence: None, + }; + let report = serve_attempt_report(&attempt); + assert!(!report.contains("Lemonade backend evidence"), "{report}"); + attempt.backend_evidence = Some("fixture backend identity"); + assert_eq!( + serve_attempt_report(&attempt), + format!("{report}\n--- Lemonade backend evidence ---\nfixture backend identity") + ); for expected in [ "exited 0", diff --git a/tests/e2e-cucumber/src/serve_plan.rs b/tests/e2e-cucumber/src/serve_plan.rs index 5723d6f540..96d60d836f 100644 --- a/tests/e2e-cucumber/src/serve_plan.rs +++ b/tests/e2e-cucumber/src/serve_plan.rs @@ -92,10 +92,10 @@ mod tests { #[test] fn a_printed_plan_names_its_resolved_model() { - let plan = "serve plan\n requested model: qwen2.5\n resolved model: Qwen/Qwen2.5-1.5B-Instruct\n engine: lemonade\n"; + let plan = "serve plan\n requested model: qwen2.5\n resolved model: Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M\n engine: lemonade\n"; assert_eq!( attempt("lemonade", plan, "", 0).resolved_model(), - Some("Qwen/Qwen2.5-1.5B-Instruct") + Some("Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M") ); } diff --git a/tests/e2e-cucumber/src/shared_runtime.rs b/tests/e2e-cucumber/src/shared_runtime.rs index be3ab641be..74872c7058 100644 --- a/tests/e2e-cucumber/src/shared_runtime.rs +++ b/tests/e2e-cucumber/src/shared_runtime.rs @@ -344,6 +344,45 @@ mod tests { ); } + /// Whether `err` is Windows refusing a symlink because the account lacks + /// `SeCreateSymbolicLinkPrivilege` (error 1314), which is the default for a + /// non-elevated account without Developer Mode. Never true elsewhere. + fn is_missing_symlink_privilege(err: &std::io::Error) -> bool { + cfg!(windows) && err.raw_os_error() == Some(1314) + } + + /// Create a directory symlink `link` -> `target`. + /// + /// Returns `false`, after saying why on stderr, when this host will not let + /// the account create one, so the caller can skip instead of failing on a + /// condition of the machine and not of the code. Every other error still + /// panics. + fn try_dir_symlink(target: &Path, link: &Path) -> bool { + #[cfg(unix)] + let result = std::os::unix::fs::symlink(target, link); + #[cfg(windows)] + let result = std::os::windows::fs::symlink_dir(target, link); + match result { + Ok(()) => true, + Err(e) if is_missing_symlink_privilege(&e) => { + eprintln!( + "SKIPPED: this account cannot create symlinks ({e}); enable Windows Developer Mode or run elevated to exercise this test" + ); + false + } + Err(e) => panic!("symlink: {e}"), + } + } + + #[test] + fn only_windows_privilege_error_1314_counts_as_a_skippable_symlink_refusal() { + let refused = std::io::Error::from_raw_os_error(1314); + assert_eq!(is_missing_symlink_privilege(&refused), cfg!(windows)); + // An unrelated failure must never be swallowed into a skip. + let other = std::io::Error::from(std::io::ErrorKind::NotFound); + assert!(!is_missing_symlink_privilege(&other)); + } + /// Reaching the tree through a symlinked parent must not disqualify every /// healthy runtime in it. /// @@ -362,10 +401,9 @@ mod tests { installed(&real, "release-wheel-multi-arch-7-14-0"); let link = tmp.path().join("link"); - #[cfg(unix)] - std::os::unix::fs::symlink(&real, &link).expect("symlink"); - #[cfg(windows)] - std::os::windows::fs::symlink_dir(&real, &link).expect("symlink"); + if !try_dir_symlink(&real, &link) { + return; + } // Selection is handed the unresolved spelling, exactly as CI would. assert_eq!( @@ -387,10 +425,9 @@ mod tests { poisoned(&real, "release-wheel-gfx94x-dcgpu-7-13-0"); let link = tmp.path().join("link"); - #[cfg(unix)] - std::os::unix::fs::symlink(&real, &link).expect("symlink"); - #[cfg(windows)] - std::os::windows::fs::symlink_dir(&real, &link).expect("symlink"); + if !try_dir_symlink(&real, &link) { + return; + } assert_eq!(runtime_key_to_activate(&link), None); } diff --git a/tests/e2e-cucumber/src/stand_in_uname.rs b/tests/e2e-cucumber/src/stand_in_uname.rs new file mode 100644 index 0000000000..2be0e2a765 --- /dev/null +++ b/tests/e2e-cucumber/src/stand_in_uname.rs @@ -0,0 +1,72 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The shell script a scenario plants as a stand-in `uname`. +//! +//! `-v` answers a fixed string, so a probe that also asks for the kernel version +//! has nothing to wait for; `-r` runs a fragment the scenario chooses; and every +//! call appends its arguments to a log, so a scenario can count how often the +//! release was asked for. The log lives under the scenario's isolated root, whose +//! path the harness does not control, so it is quoted for the shell rather than +//! put between bare single quotes. + +use std::path::Path; + +/// `value` as one single-quoted shell word: each embedded single quote closes the +/// quotes, is written escaped and reopens them. (`rocm_core::quoting` has the same +/// quoting, but `rocm-core` is only a dev-dependency of this library.) +fn single_quoted(value: &str) -> String { + format!("'{}'", value.replace('\'', r"'\''")) +} + +/// The text of the stand-in `uname`: log the arguments to `log`, answer `-v`, and +/// otherwise run `release_body` (a shell fragment). +#[must_use] +pub fn stand_in_uname_script(release_body: &str, log: &Path) -> String { + let log = single_quoted(&log.to_string_lossy()); + format!( + "#!/bin/sh\necho \"$*\" >> {log}\nif [ \"$1\" = -v ]; then echo '#1 SMP stand-in'; exit 0; fi\n{release_body}\n" + ) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::process::Command; + + /// Plant the script in a directory whose name holds a single quote and a + /// space, run it, and read back what it logged and printed. + fn run_in_awkward_directory(args: &[&str], body: &str) -> (String, String) { + let scratch = tempfile::tempdir().expect("scratch directory"); + let dir = scratch.path().join("it's a dir"); + std::fs::create_dir_all(&dir).expect("create the awkward directory"); + let log = dir.join("uname calls.log"); + let script = dir.join("uname"); + rocm_core::fake_executable::write_fake_executable( + &script, + stand_in_uname_script(body, &log), + ) + .expect("plant the stand-in"); + let output = Command::new(&script).args(args).output().expect("run it"); + assert!(output.status.success(), "{output:?}"); + ( + String::from_utf8_lossy(&output.stdout).into_owned(), + std::fs::read_to_string(&log).expect("the stand-in logged its call"), + ) + } + + #[test] + fn the_log_path_survives_a_single_quote_and_a_space_in_it() { + let (stdout, log) = run_in_awkward_directory(&["-r"], "echo 6.8.0-stand-in"); + assert_eq!(stdout.trim(), "6.8.0-stand-in"); + assert_eq!(log, "-r\n"); + } + + #[test] + fn dash_v_answers_a_fixed_string_and_is_logged_too() { + let (stdout, log) = run_in_awkward_directory(&["-v"], "echo not-reached"); + assert_eq!(stdout.trim(), "#1 SMP stand-in"); + assert_eq!(log, "-v\n"); + } +} diff --git a/tests/e2e-cucumber/src/stray_process.rs b/tests/e2e-cucumber/src/stray_process.rs new file mode 100644 index 0000000000..815b5a9271 --- /dev/null +++ b/tests/e2e-cucumber/src/stray_process.rs @@ -0,0 +1,75 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What scenario teardown does about a process a scenario started and recorded. +//! +//! A scenario that records a process a stop is supposed to take down (a parent +//! and the child it started, say) can fail with one of them left running, and a +//! later stop cannot find a child whose parent is gone. Teardown therefore +//! keeps the child's own identity and ends it itself. Whether to do so is +//! decided here, apart from the system reads, so a table test covers each case +//! on every host: only a process that is still the one recorded is ended, never +//! one that has since exited or whose PID another process has taken. + +/// What the host says about a recorded process now. +/// +/// It is read against the identity the scenario captured for the process. The +/// harness maps the production identity check onto these (`rocm-core` is a +/// dev-dependency of this crate, so the library does not name it). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedState { + /// Running, and the same process (or recorded without a token, which cannot + /// be refuted). + Same, + /// No longer running. + Gone, + /// Running, but provably a different process that took the PID. + Reused, + /// Running, but its identity cannot be confirmed. + Unconfirmed, +} + +/// What teardown does about one recorded process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Leftover { + /// End it: it is still the process the scenario recorded. + Terminate, + /// Leave it: it is gone, or is not (provably) the recorded process. + Leave, +} + +/// The teardown action for a recorded process whose identity reads as `state`. +/// +/// A process that cannot be confirmed as the recorded one is left alone: ending +/// a stranger that happens to hold a recycled PID is worse than leaving a +/// stray five-minute process to time out. +#[must_use] +pub const fn leftover_action(state: RecordedState) -> Leftover { + match state { + RecordedState::Same => Leftover::Terminate, + RecordedState::Gone | RecordedState::Reused | RecordedState::Unconfirmed => Leftover::Leave, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_process_that_is_still_the_recorded_one_is_ended() { + for (state, expected) in [ + // Alive and the same process (or recorded without a token, which + // cannot be refuted): the stray child a failed stop left behind. + (RecordedState::Same, Leftover::Terminate), + // The stop took it down already. + (RecordedState::Gone, Leftover::Leave), + // Its PID has been reused: a stranger. + (RecordedState::Reused, Leftover::Leave), + // Its token cannot be read back, so it cannot be confirmed. + (RecordedState::Unconfirmed, Leftover::Leave), + ] { + assert_eq!(leftover_action(state), expected, "{state:?}"); + } + } +} diff --git a/tests/e2e-cucumber/src/tasklist.rs b/tests/e2e-cucumber/src/tasklist.rs new file mode 100644 index 0000000000..35aea69c67 --- /dev/null +++ b/tests/e2e-cucumber/src/tasklist.rs @@ -0,0 +1,83 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What `tasklist` says about one process, read so that a failed query cannot +//! pass for an answer. +//! +//! A scenario that wants to know whether a process a stop was meant to take down +//! is still running asks `tasklist /FI "PID eq N" /FO CSV /NH`. A match prints a +//! CSV row naming the PID and no match prints an INFO line without one, but the +//! command can also fail (it cannot be started under that name, it is refused, it +//! times out) and print nothing useful. Reading "no row" from such a run as "the +//! process exited" lets the scenario pass while the process may still be running, +//! so the exit status is judged before the text. The read is a pure function of +//! the command's status and output, so a table test covers every case on every +//! host. + +/// Whether `tasklist` reported process `pid` running. +/// +/// `success` is whether the command exited successfully and `stdout` is what it +/// printed. `Ok(true)` is a CSV row for `pid`, `Ok(false)` is a successful run +/// with no such row, and `Err` is a run that did not succeed: its output says +/// nothing about the process, so there is no answer. +/// +/// # Errors +/// +/// Returns a message naming the PID and what the failed run printed when the +/// command did not exit successfully. +pub fn tasklist_reports_running( + success: bool, + stdout: &str, + stderr: &str, + pid: u32, +) -> Result { + if !success { + return Err(format!( + "`tasklist` did not exit successfully, so it cannot say whether process {pid} \ + is still running (stdout: {:?}, stderr: {:?})", + stdout.trim(), + stderr.trim() + )); + } + Ok(stdout.contains(&format!("\"{pid}\""))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const ROW: &str = "\"ping.exe\",\"4242\",\"Console\",\"1\",\"3,044 K\"\r\n"; + const NO_MATCH: &str = "INFO: No tasks are running which match the specified criteria.\r\n"; + + #[test] + fn a_successful_run_with_a_row_for_the_pid_is_running() { + assert_eq!(tasklist_reports_running(true, ROW, "", 4242), Ok(true)); + } + + #[test] + fn a_successful_run_without_a_row_for_the_pid_is_not_running() { + assert_eq!( + tasklist_reports_running(true, NO_MATCH, "", 4242), + Ok(false) + ); + // A row for another PID is not a row for this one, nor is a PID that + // only contains it. + assert_eq!(tasklist_reports_running(true, ROW, "", 424), Ok(false)); + assert_eq!(tasklist_reports_running(true, ROW, "", 7), Ok(false)); + } + + #[test] + fn a_failed_run_is_no_answer_whatever_it_printed() { + // No output to read: "no row" must not become "exited". + let err = tasklist_reports_running(false, "", "Access is denied.", 4242) + .expect_err("a failed query is not an answer"); + assert!( + err.contains("4242") && err.contains("Access is denied."), + "{err}" + ); + // Nor does a row printed before the failure make it one. + assert!(tasklist_reports_running(false, ROW, "", 4242).is_err()); + assert!(tasklist_reports_running(false, NO_MATCH, "", 4242).is_err()); + } +} diff --git a/tests/e2e-cucumber/src/terminal_size.rs b/tests/e2e-cucumber/src/terminal_size.rs index 4481cdf456..b861672487 100644 --- a/tests/e2e-cucumber/src/terminal_size.rs +++ b/tests/e2e-cucumber/src/terminal_size.rs @@ -2,9 +2,11 @@ // // SPDX-License-Identifier: MIT -//! Validation of a requested terminal size before it reaches the emulator. +//! Validation of a requested terminal size, and the order in which the emulated +//! screen and the real PTY take it. use std::num::NonZeroU16; +use std::sync::Mutex; /// `rows` and `cols` as the `NonZeroU16`s the `vt100` emulator takes, or an /// error naming the requested size if either is zero. @@ -21,9 +23,75 @@ pub fn non_zero_dimensions(rows: u16, cols: u16) -> Result<(NonZeroU16, NonZeroU } } +/// Resize the emulated screen first, then the real PTY. +/// +/// `resize_pty` is the call that resizes the real PTY and so signals the child. +/// It runs after the emulated screen has taken the new size and with the +/// emulator unlocked, so the reader thread keeps draining the child's output +/// while it runs. If it fails, the emulated screen goes back to its old size so +/// the two never disagree; that restores the size only, not any cells a shrink +/// truncated, which is acceptable because a failed resize fails the step. +/// +/// The order is the point. The child redraws the moment it learns the new size, +/// and the reader thread feeds whatever it writes to the emulator. Resizing the +/// PTY first and the emulator second leaves a window between the two calls in +/// which a frame laid out for the new width is parsed by the old, narrower +/// screen: its lines wrap and scroll the top rows (the header among them) off, +/// and because a terminal UI only repaints the cells that changed, nothing ever +/// repairs the damage. Any scheduling stall of the harness thread between the +/// two calls (a loaded CI runner) opens that window. With the emulator resized +/// first, whatever the child writes in answer to the resize is parsed at the new +/// size. +/// +/// The same argument is not made, and not tested, for shrinking: a frame the +/// child drew for the old, larger size can reach the already smaller emulator +/// and wrap there, but the child clears and redraws on a resize, so that residue +/// is repainted over. Shrinking relies on the child doing so; a test of it would +/// pass with either order and so would not guard anything. +/// +/// The emulator is deliberately not held locked across `resize_pty`: the reader +/// drains the PTY under that lock, and a PTY resize that waits for the child's +/// output to drain (which a Windows pseudoconsole is not documented to rule out) +/// would then deadlock against it. +/// +/// Pure of any PTY on purpose, like [`non_zero_dimensions`]: the driver lives in +/// the cucumber binary, where plain `#[test]`s never run, so the ordering is +/// covered here by `cargo test -p e2e-cucumber --lib`. +pub fn resize_screen_then_pty( + parser: &Mutex, + rows: u16, + cols: u16, + resize_pty: impl FnOnce() -> Result<(), String>, +) -> Result<(), String> { + // Validate before touching either side, so a zero dimension fails cleanly + // instead of leaving the PTY and the emulated screen at different sizes. + let (vt_rows, vt_cols) = non_zero_dimensions(rows, cols)?; + // Recover rather than fail on a poisoned lock: resizing is still meaningful + // even if some earlier operation panicked while holding it. + let lock = || { + parser + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + }; + let (old_rows, old_cols) = { + let mut parser = lock(); + let old = parser.screen().size(); + parser.screen_mut().set_size(vt_rows, vt_cols); + old + }; + let resized = resize_pty(); + if resized.is_err() { + lock().screen_mut().set_size(old_rows, old_cols); + } + resized +} + #[cfg(test)] mod tests { - use super::non_zero_dimensions; + use super::{non_zero_dimensions, resize_screen_then_pty}; + use std::fmt::Write as _; + use std::sync::{Arc, Mutex, mpsc}; + use std::time::Duration; #[test] fn accepts_non_zero_dimensions_and_keeps_their_order() { @@ -54,4 +122,106 @@ mod tests { Err("cannot resize to rows=0 cols=0: zero dimension".to_owned()) ); } + + /// Cursor-addressed frame of `rows` full-width lines, as a terminal UI + /// writes it after a resize: clear, then every row at an absolute position. + fn frame(rows: u16, cols: u16) -> Vec { + let mut out = String::from("\x1b[2J"); + for row in 0..rows { + let label = format!("row{row:02}"); + let fill = "x".repeat(usize::from(cols) - label.len()); + let _ = write!(out, "\x1b[{};1H{label}{fill}", row + 1); + } + out.into_bytes() + } + + fn parser(rows: u16, cols: u16) -> Arc> { + let (rows, cols) = non_zero_dimensions(rows, cols).expect("a valid test size"); + Arc::new(Mutex::new(vt100::Parser::new(rows, cols, 0))) + } + + fn size(parser: &Mutex) -> (u16, u16) { + let (rows, cols) = parser.lock().unwrap().screen().size(); + (rows.get(), cols.get()) + } + + #[test] + fn a_frame_the_child_draws_in_answer_to_the_resize_lands_on_the_new_size() { + let parser = parser(24, 80); + // The reader thread parses the child's answer the moment it is released, + // under the parser lock, exactly as the driver's reader does. + let (go, released) = mpsc::channel::<()>(); + let (parsed, wait_parsed) = mpsc::channel::<()>(); + let reader = { + let parser = Arc::clone(&parser); + std::thread::spawn(move || { + released + .recv() + .expect("the resize never signalled the child"); + parser.lock().unwrap().process(&frame(40, 120)); + parsed.send(()).unwrap(); + }) + }; + + let resized = resize_screen_then_pty(&parser, 40, 120, || { + // The child redraws as soon as it is signalled, and the harness + // thread does not return from the PTY call until the reader has + // parsed that redraw: the window a PTY-first order leaves open (a + // frame parsed before the emulator has its new size), and, if the + // emulator were kept locked across this call, one the reader could + // never get through, so the wait below would time out. + go.send(()).unwrap(); + wait_parsed + .recv_timeout(Duration::from_secs(5)) + .map_err(|_| { + "the reader could not parse the child's redraw while the PTY resize ran" + .to_owned() + }) + }); + assert_eq!(resized, Ok(())); + reader.join().unwrap(); + + assert_eq!(size(&parser), (40, 120)); + let guard = parser.lock().unwrap(); + let screen = guard.screen(); + for (row, text) in screen.rows(0, 120).enumerate() { + let label = format!("row{row:02}"); + assert_eq!( + text, + format!("{label}{}", "x".repeat(120 - label.len())), + "row {row} was not drawn intact: a frame laid out for 120 columns \ + was parsed by a screen still sized for the old geometry" + ); + } + } + + #[test] + fn a_failed_pty_resize_puts_the_emulated_screen_back_and_reports_the_error() { + let parser = parser(24, 80); + let resized = + resize_screen_then_pty( + &parser, + 40, + 120, + || Err("failed to resize pty: boom".into()), + ); + assert_eq!(resized, Err("failed to resize pty: boom".to_owned())); + assert_eq!(size(&parser), (24, 80)); + } + + #[test] + fn a_zero_dimension_resizes_neither_side() { + let parser = parser(24, 80); + let mut pty_resized = false; + let resized = resize_screen_then_pty(&parser, 0, 120, || { + pty_resized = true; + Ok(()) + }); + assert_eq!( + resized, + Err("cannot resize to rows=0 cols=120: zero dimension".to_owned()) + ); + assert!(!pty_resized, "the PTY was resized for an invalid size"); + assert_eq!(size(&parser), (24, 80)); + } } diff --git a/tests/e2e-cucumber/src/torch_alignment_outcome.rs b/tests/e2e-cucumber/src/torch_alignment_outcome.rs new file mode 100644 index 0000000000..242952e1a8 --- /dev/null +++ b/tests/e2e-cucumber/src/torch_alignment_outcome.rs @@ -0,0 +1,143 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Which `torch_alignment:` outcomes the runtime scenarios accept as settled. + +/// Verdicts that mean the runtime was settled and its torch left in a state the +/// CLI produces on purpose. +/// +/// Four, not one. Whether a reinstall rewrites torch or finds it already correct +/// depends on what the shared pre-warm tree held when the scenario started, and a +/// torch that has already executed a GPU kernel with this SDK is kept exactly as +/// it is and reported as `retained_*`: settled without anything being installed. +pub const SETTLED_VERDICTS: [&str; 4] = [ + "realigned", + "already_aligned", + "retained_sdk_build", + "retained_engine_build", +]; + +/// The reason the CLI prints when the installed engine declares no torch +/// requirement at all. +const NO_PIN_REASON: &str = "(vllm does not pin torch)"; + +/// Whether the installed engine's own package metadata requires torch. +/// +/// Read from the runtime's interpreter by the scenario, never from the CLI's +/// output: the output is the thing being judged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EngineTorchRequirement { + Declared, + Absent, +} + +/// Whether the text after `torch_alignment: ` describes a settled runtime. +/// +/// `not_applicable (vllm does not pin torch)` counts only when the engine really +/// declares no torch requirement. That is the ROCm 10.x discover install: the +/// vLLM wheel declares no torch, and the engine install pins and realigns the +/// torch trio itself, so there is no engine pin for the alignment to settle +/// against. The same text over an +/// engine that does declare one would be the skip this check exists to catch, as +/// would any other `not_applicable` reason, such as a manifest that identifies no +/// SDK torch build. +#[must_use] +pub fn is_settled(verdict_line: &str, engine: EngineTorchRequirement) -> bool { + let verdict_line = verdict_line.trim(); + let verdict = verdict_line.split_whitespace().next().unwrap_or_default(); + if SETTLED_VERDICTS.contains(&verdict) { + return true; + } + verdict == "not_applicable" + && engine == EngineTorchRequirement::Absent + && verdict_line.strip_prefix(verdict).map(str::trim) == Some(NO_PIN_REASON) +} + +/// Judge the result of importing the stack the engine needs (torch, torchvision, +/// the engine and its compiled extensions) in the runtime's own interpreter. +/// +/// Needed where the alignment decided nothing. `not_applicable` says no engine pin +/// was compared, and the device check that follows only proves torch can run a +/// kernel, so a torch and torchvision that cannot register their operators +/// together, or a torch the engine's compiled extensions were not built against, +/// pass both while every `rocm serve` on the runtime fails on import. The last +/// line of the failure is returned, because that is the line that names the cause. +/// +/// # Errors +/// +/// The last non-empty stderr line when the import did not succeed. +pub fn engine_import_verdict(succeeded: bool, stderr: &str) -> Result<(), String> { + if succeeded { + return Ok(()); + } + Err(stderr + .lines() + .rev() + .map(str::trim) + .find(|line| !line.is_empty()) + .unwrap_or("the import failed with no error output") + .to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const NO_PIN: &str = "not_applicable (vllm does not pin torch)"; + + #[test] + fn every_settled_verdict_is_accepted_whatever_the_engine_declares() { + for verdict in SETTLED_VERDICTS { + for engine in [ + EngineTorchRequirement::Declared, + EngineTorchRequirement::Absent, + ] { + assert!(is_settled(verdict, engine), "{verdict} with {engine:?}"); + } + } + } + + #[test] + fn no_pin_is_settled_only_when_the_engine_declares_no_torch() { + assert!(is_settled(NO_PIN, EngineTorchRequirement::Absent)); + assert!(!is_settled(NO_PIN, EngineTorchRequirement::Declared)); + } + + #[test] + fn any_other_not_applicable_reason_stays_unsettled() { + for line in [ + "not_applicable (the runtime manifest does not identify the SDK's torch build)", + "not_applicable (vllm does not pin torch to an exact version (torch>=2.0))", + "not_applicable", + ] { + assert!(!is_settled(line, EngineTorchRequirement::Absent), "{line}"); + } + } + + #[test] + fn a_successful_import_is_accepted_whatever_stderr_holds() { + assert_eq!(engine_import_verdict(true, "UserWarning: noise\n"), Ok(())); + } + + #[test] + fn a_failed_import_names_the_last_line_of_its_error() { + let stderr = "Traceback (most recent call last):\n File \"\", line 1\n\ + RuntimeError: operator torchvision::nms does not exist\n\n"; + assert_eq!( + engine_import_verdict(false, stderr), + Err("RuntimeError: operator torchvision::nms does not exist".to_owned()) + ); + assert_eq!( + engine_import_verdict(false, ""), + Err("the import failed with no error output".to_owned()) + ); + } + + #[test] + fn the_other_unsettled_verdicts_stay_unsettled() { + for line in ["unavailable", "install_failed", "disabled", ""] { + assert!(!is_settled(line, EngineTorchRequirement::Absent), "{line}"); + } + } +} diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index a9f6a391e6..6e8db6c579 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -32,6 +32,7 @@ mod e2e { pub mod engines_steps; pub mod examine_steps; pub mod host_guard; + pub mod lemonade_install_steps; pub mod lifecycle_steps; pub mod logs_steps; pub mod model_catalog_steps; @@ -45,6 +46,7 @@ mod e2e { pub mod serving_steps; pub mod skill_steps; pub mod storage_steps; + pub mod symptom_file_steps; pub mod therock_steps; pub mod tui_driver; pub mod uninstall_steps; @@ -56,6 +58,7 @@ mod e2e { #[derive(Debug, cucumber::World)] pub struct E2eWorld { pub mock: Option, + pub lemonade_protocol: Option, /// Loopback file server used by artifact-prefetch scenarios. Kept on the /// World so it remains alive while the real `rocmd` subprocess downloads. pub artifact_server: Option, @@ -70,11 +73,21 @@ pub struct E2eWorld { /// Listener that deliberately occupies the default serve port while a real /// `rocm serve` invocation proves automatic allocation advances past it. pub automatic_port_guard: Option, + /// How many requests a stand-in Hugging Face endpoint has answered, for + /// scenarios that must show the CLI did or did not reach out. + /// The serving thread stops once the World drops its handle. + pub hub_requests: Option>, /// Cache-marker destination discovered from `rocmd`'s own JSON report. pub artifact_marker_path: Option, pub endpoint: Option, pub model_name: Option, pub chat_response: Option, + /// The reply to a request sent to the served model's Anthropic-compatible + /// `/v1/messages` endpoint. + pub anthropic_response: Option, + /// The raw server-sent-event body of a streaming request sent to the same + /// endpoint. + pub anthropic_stream: Option, pub cli_output: Option, pub cli_outputs: Option>, pub cli_stderr: Option, @@ -85,11 +98,21 @@ pub struct E2eWorld { /// Wall-clock time the last measured `rocm` invocation took, for scenarios /// where the *duration* bounds the behaviour under test — /// `service-cleanup-07`, where a prune that returned while the managed-launch - /// lock was still held cannot have waited for it. It is a one-sided bound: - /// the measurement brackets the whole child process, so a long elapsed time - /// does not establish that the prune blocked. Set by the When step that - /// measures it; `None` everywhere else. + /// lock was still held cannot have waited for it, and `examine-48`, where an + /// inspection that finished long before a probe tool's background process + /// did cannot have waited for it. Both assert the short side of a one-sided + /// bound: the measurement brackets the whole child process, so a long elapsed + /// time does not establish that the command blocked. Set by the When step + /// that measures it; `None` everywhere else. pub cli_elapsed: Option, + /// The `ROCM_CLI_DATA_DIR` value a refusal told the user to set, read from + /// its stderr by the first step that follows that advice, so later steps + /// can follow it again after other commands have replaced `cli_stderr`. + pub suggested_data_dir: Option, + /// The file a refusal told the user to rename and the name to give it, + /// read from its stderr by the step that checks the refusal, for the step + /// that follows that advice after other commands have replaced `cli_stderr`. + pub suggested_settings_rename: Option<(String, String)>, /// Extra environment for `rocm remote` scenarios: a `PATH` carrying the /// tailscale stand-in, and the status document it should serve. Set by a /// Given step so the When steps stay about what the user does. @@ -143,6 +166,11 @@ pub struct E2eWorld { /// records a behavioral precondition here; the When step remains a plain user /// action and consumes the fixture without exposing its mechanism in Gherkin. pub command_env: Vec<(&'static str, std::ffi::OsString)>, + /// The working directory for this scenario's next `rocm` invocation, for a + /// scenario whose subject is what the CLI finds in the directory it is run + /// from. Consumed by [`run_rocm_with_scenario_env`] like `command_env`; `None` + /// leaves the harness's own working directory. + pub command_dir: Option, /// The interactive dash/chat TUI spawned under a pseudo-terminal for this /// scenario, if any (see `e2e::tui_driver`). Torn down in `Drop` before the /// mock server and isolated directory so the child process never outlives @@ -175,6 +203,14 @@ pub struct E2eWorld { /// field rather than borrowing `model_name`, which means a served model and /// has nothing to do with a user's error report. pub skill_symptom: Option, + /// The pasted error a diagnose-59 or diagnose-60 scenario hands to + /// `rocm diagnose --symptom-file`. Its own field for the reason + /// `skill_symptom` is. + pub symptom_paste: Option, + /// The `rocm diagnose --symptom-file` runs a diagnose-59 to diagnose-61 + /// scenario made: one per shell that piped a paste, or one per file the + /// CLI was pointed at. + pub symptom_file_runs: Vec, /// The background helper (`rocm daemon`) a scenario saw running, captured /// while it was known live, so `Drop` can stop it by a verified identity if /// an earlier step failed before the scenario disabled the watcher. @@ -191,6 +227,11 @@ pub struct E2eWorld { /// reads the record, but a real `rocm services stop e2e-mock --yes` would /// then signal the harness process itself. pub recorded_process: Option, + /// The identity of a process the recorded one started (`service-stop-07`'s + /// `ping`). Teardown ends it on its own account, because a stop that fails + /// to take it down leaves it running with a dead parent, where nothing else + /// can find it. + pub recorded_descendant: Option, /// Whether the Doctor report preview in the dashboard console was the CLI's /// refusal envelope (it carries a `refused` key). Recorded when the preview /// is read, so a later step can check the card against what the CLI said. @@ -323,20 +364,26 @@ impl Default for E2eWorld { Self { mock: None, + lemonade_protocol: None, artifact_server: None, paced_download_server: None, comfyui_port_listener: None, automatic_port_guard: None, + hub_requests: None, artifact_marker_path: None, endpoint: None, model_name: None, chat_response: None, + anthropic_response: None, + anthropic_stream: None, cli_output: None, cli_outputs: None, cli_stderr: None, cli_rc: None, serve_runs: None, cli_elapsed: None, + suggested_data_dir: None, + suggested_settings_rename: None, remote_env: Vec::new(), launch_release: None, launch_stage: None, @@ -347,6 +394,7 @@ impl Default for E2eWorld { serve_timeout_override: None, expect_xfail: false, command_env: Vec::new(), + command_dir: None, tui: None, chat_use_mock: false, comfyui_baseline_torch_stack: None, @@ -354,8 +402,11 @@ impl Default for E2eWorld { lifecycle: None, skill_reference: None, skill_symptom: None, + symptom_paste: None, + symptom_file_runs: Vec::new(), helper_identity: None, recorded_process: None, + recorded_descendant: None, dash_report_refused: None, automation_daemon: None, } @@ -704,6 +755,21 @@ impl Drop for E2eWorld { let _ = process.kill(); let _ = process.wait(); } + // The process the recorded one started: ended too when it is still the + // process that was recorded, so a failed scenario does not leave it + // running until its own timeout. + if let Some(descendant) = self.recorded_descendant.take() { + use e2e_cucumber::stray_process::{Leftover, RecordedState, leftover_action}; + let state = match rocm_core::identity_state(&descendant) { + rocm_core::IdentityState::Matches => RecordedState::Same, + rocm_core::IdentityState::Gone => RecordedState::Gone, + rocm_core::IdentityState::Recycled => RecordedState::Reused, + rocm_core::IdentityState::Indeterminate => RecordedState::Unconfirmed, + }; + if leftover_action(state) == Leftover::Terminate { + let _ = rocm_core::terminate_recorded_pid(&descendant); + } + } // A scenario that enabled a watcher started a real detached `rocm daemon`, // which only exits once the watcher is disabled. If a step failed first, // nothing else would stop it, and it would keep polling a config dir @@ -717,6 +783,7 @@ impl Drop for E2eWorld { if let Some(mock) = self.mock.take() { mock.stop(); } + self.lemonade_protocol.take(); self.artifact_server.take(); self.paced_download_server.take(); // A scenario that ran `rocm serve --managed` left a DETACHED supervisor + @@ -894,6 +961,67 @@ pub fn run_rocm(world: &E2eWorld, args: &[&str]) -> (String, String, i32) { ) } +/// Like [`run_rocm`], but gives up after `timeout` instead of blocking forever. +/// +/// [`run_rocm`] uses `Command::output`, which waits for the pipes to close. A +/// `rocm serve --managed` that leaves a supervisor holding an inherited pipe, or +/// that never returns, therefore stalls the whole lane with no report. Here the +/// child writes to files, so an inherited handle cannot keep us waiting, and the +/// child is killed once the deadline passes. A timeout is reported as rc `-1` +/// with a note appended to stderr. +pub fn run_rocm_bounded( + world: &E2eWorld, + args: &[&str], + timeout: std::time::Duration, +) -> (String, String, i32) { + use std::process::Stdio; + + let binary = rocm_binary(); + let dir = tempfile::tempdir().expect("temp dir for bounded command output"); + let out_path = dir.path().join("stdout"); + let err_path = dir.path().join("stderr"); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::from( + std::fs::File::create(&out_path).expect("stdout file"), + )); + cmd.stderr(Stdio::from( + std::fs::File::create(&err_path).expect("stderr file"), + )); + let mut child = cmd + .spawn() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + let deadline = std::time::Instant::now() + timeout; + let mut timed_out = false; + let status = loop { + if let Some(status) = child.try_wait().expect("wait on rocm") { + break Some(status); + } + if std::time::Instant::now() >= deadline { + timed_out = true; + let _ = child.kill(); + let _ = child.wait(); + break None; + } + std::thread::sleep(std::time::Duration::from_millis(200)); + }; + let rc = status.and_then(|s| s.code()).unwrap_or(-1); + let stdout = String::from_utf8_lossy(&std::fs::read(&out_path).unwrap_or_default()).to_string(); + let mut stderr = + String::from_utf8_lossy(&std::fs::read(&err_path).unwrap_or_default()).to_string(); + if timed_out { + stderr = format!( + "{stderr}\n`rocm {}` did not return within {}s and was killed\n", + args.join(" "), + timeout.as_secs() + ); + } + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + (stdout, stderr, rc) +} + /// Run `rocm`, returning stdout, and panic with the full diagnostic bundle /// ([`cli_failure_report`]) if it exits non-zero. /// @@ -1007,16 +1135,68 @@ pub fn run_rocm_with_env_and_unset( ) } +/// Like [`run_rocm`], but without `ROCM_CLI_DATA_DIR`, so the CLI has to work +/// out its data folder from its own settings file as it does for a user who +/// never set it. +/// +/// The home directory is pointed inside the scenario's sandbox as well, so the +/// default data folder the CLI falls back to (`~/.rocm`) is never the real +/// one on the runner. +pub fn run_rocm_without_data_dir_override( + world: &E2eWorld, + args: &[&str], +) -> (String, String, i32) { + let binary = rocm_binary(); + let home = world + .isolated_root + .as_ref() + .expect("scenario has an isolated root") + .path() + .join("home"); + std::fs::create_dir_all(&home).expect("failed to create sandbox home"); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + cmd.env_remove("ROCM_CLI_DATA_DIR"); + cmd.env("HOME", &home); + cmd.env("USERPROFILE", &home); + let output = cmd + .output() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + let rc = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + ( + stdout, + String::from_utf8_lossy(&output.stderr).to_string(), + rc, + ) +} + /// Run `rocm` with the behavioral fixture established by a Given step, then /// consume it so it cannot leak into a later action in the same scenario. pub fn run_rocm_with_scenario_env(world: &mut E2eWorld, args: &[&str]) -> (String, String, i32) { let binary = rocm_binary(); - let mut cmd = std::process::Command::new(&binary); + // A binary given as a relative path was meant relative to this process's + // directory, not to the scenario's, so it is made absolute before the child + // moves. A bare name (`rocm`, found on `PATH`) stays as it is, even if the + // harness's directory happens to hold a file of that name. + let program = if world.command_dir.is_some() { + e2e_cucumber::binary_path::program_for_moved_child(&binary, |path| { + std::fs::canonicalize(path) + }) + } else { + PathBuf::from(&binary) + }; + let mut cmd = std::process::Command::new(program); cmd.args(args); world.isolate_cmd(&mut cmd); for (key, value) in std::mem::take(&mut world.command_env) { cmd.env(key, value); } + if let Some(dir) = world.command_dir.take() { + cmd.current_dir(dir); + } let output = cmd .output() .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); @@ -1232,8 +1412,35 @@ fn load_expectations() -> e2e_cucumber::expectation::Expectations { .unwrap_or_else(|e| panic!("failed to parse expectations.toml: {e}")) } -#[tokio::main] -async fn main() { +/// Stack for the thread that drives the whole suite. +/// +/// The cucumber run is one future polled by `block_on` on the calling thread, so +/// every step future, and the evidence-gathering futures a failing serve step +/// awaits, is held on that thread's stack. Windows gives a main thread 1 MiB +/// against 8 MiB on Linux, and a stalled serve-06 overflowed it (#1003) instead +/// of reporting a failure. A spawned thread's stack is whatever is asked for here. +const SUITE_STACK_BYTES: usize = 64 * 1024 * 1024; + +fn main() { + let suite = std::thread::Builder::new() + .name("e2e-suite".to_owned()) + .stack_size(SUITE_STACK_BYTES) + .spawn(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build the tokio runtime") + .block_on(run_suite()); + }) + .expect("failed to spawn the e2e suite thread"); + if let Err(payload) = suite.join() { + std::panic::resume_unwind(payload); + } +} + +// `block_on` needs no `Send`, and the cucumber writers held across awaits are not. +#[allow(clippy::future_not_send)] +async fn run_suite() { use std::collections::BTreeMap; use std::sync::Mutex; diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 95972ef56e..82176a92d7 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -101,7 +101,7 @@ fn write_harness_autostart_claim(path: &std::path::Path) { /// Re-stamp a claim `plant_inflight_autostart_claim` planted, right before the /// CLI runs. The CLI honours a claim for only 30 seconds -/// (`AUTOSTART_CLAIM_TTL_MS` in `apps/rocm/src/service_launch.rs`). The mock +/// (`AUTOSTART_CLAIM_TTL_MS` in `apps/rocm/src/service_launch/helper_autostart.rs`). The mock /// lane runs up to 64 scenarios at once, and every step blocks a runtime /// worker while its `rocm` child runs, so this scenario's When can start more /// than 30 seconds after its Given. The CLI then rightly reads the claim as diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index 61b37e4faf..c0bca4e7a5 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -34,6 +34,21 @@ //! (`crates/rocm-dash-tui/src/app/tool_approval.rs`) asserts that envelope is collapsed //! out of the chat. //! +//! `comfyui-15` to `comfyui-19` cover the other half of that selection: with +//! nothing chosen, the install recovers the runtime `rocm setup` created before +//! it counts the ready ones, as `rocm serve` does, so a runtime the registry +//! lost is found. They plant a ready runtime and the setup folder that names it, +//! but no registry entry. `comfyui-15` previews the install with `--dry-run`, +//! which prints the `--runtime-id` of the runtime it chose and writes nothing, +//! so the registry still has no entry afterwards; `comfyui-16` runs the install +//! against an existing checkout and finds the entry written back; `comfyui-17` +//! runs the command the preview printed, from the state the preview left, and +//! finds the same entry written back. `comfyui-18` runs that command with a +//! registry that cannot be written and finds the failure to restore the entry +//! reported. `comfyui-19` plants a setup folder whose runtime is no longer +//! usable next to one ready registered runtime and finds the preview using the +//! registered one without registering the unusable one. +//! //! `comfyui-04` covers the source-archive download spinner under a real PTY, //! mirroring `download_progress_pty.feature`'s tarball scenario but for //! `download_and_extract_source`'s in-process `GzDecoder`/`tar` unpack, which @@ -194,6 +209,21 @@ fn rocm_library_filename(shortname: &str) -> String { /// confusing downstream assertion instead of on the fixture. If this step starts /// failing after an edit here, suspect the manifest shape first. fn plant_ready_runtime(data: &Path, key: &str) { + let manifest = plant_ready_runtime_tree(data, key); + let registry = data.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", registry.display())); + std::fs::write( + registry.join(format!("{key}.json")), + serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), + ) + .unwrap_or_else(|e| panic!("failed to write the planted runtime manifest: {e}")); +} + +/// The on-disk half of [`plant_ready_runtime`]: the install root and what the +/// readiness check needs under it. Returns the manifest JSON that describes it, +/// leaving it to the caller whether the registry gets an entry. +fn plant_ready_runtime_tree(data: &Path, key: &str) -> serde_json::Value { let install_root = data.join("runtimes").join("roots").join(key); let sdk_root = install_root.join("sdk"); let sdk_bin = sdk_root.join("bin"); @@ -208,7 +238,7 @@ fn plant_ready_runtime(data: &Path, key: &str) { write_fixture(&amdhip, "stub"); write_fixture(&hipblas, "stub"); - let manifest = serde_json::json!({ + serde_json::json!({ "runtime_key": key, "runtime_id": key, "channel": "release", @@ -229,16 +259,7 @@ fn plant_ready_runtime(data: &Path, key: &str) { ], }, "installed_at_unix_ms": 1, - }); - - let registry = data.join("runtimes").join("registry"); - std::fs::create_dir_all(®istry) - .unwrap_or_else(|e| panic!("failed to create {}: {e}", registry.display())); - std::fs::write( - registry.join(format!("{key}.json")), - serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), - ) - .unwrap_or_else(|e| panic!("failed to write the planted runtime manifest: {e}")); + }) } /// Combined stdout+stderr of the recorded `rocm` invocation. The refusal is an @@ -483,6 +504,34 @@ async fn install_comfyui_without_runtime(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// Runs the `install command:` line of the preview in `world.cli_output`, as a +/// user would copy it: every word after `rocm` is an argument. The runtime key +/// planted for these scenarios needs no shell quoting, so splitting on +/// whitespace is the whole parse; the first word is checked so a line in +/// another shape fails here rather than running something else. +#[when("the user runs the install command the preview printed")] +async fn run_the_install_command_the_preview_printed(world: &mut E2eWorld) { + let stdout = world.cli_output.clone().unwrap_or_default(); + let line = stdout + .lines() + .find_map(|line| line.trim().strip_prefix("install command:")) + .unwrap_or_else(|| panic!("the preview printed no install command:\n{stdout}")); + let words: Vec<&str> = line.split_whitespace().collect(); + assert_eq!( + words.first().copied(), + Some("rocm"), + "the printed command should start with `rocm`: {line}" + ); + assert!( + words.contains(&"--runtime-id") && !words.contains(&"--dry-run"), + "the printed command should name its runtime and not preview: {line}" + ); + let (stdout, stderr, rc) = crate::run_rocm(world, &words[1..]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[then("ComfyUI install is refused as ambiguous")] async fn comfyui_install_refused(world: &mut E2eWorld) { let text = refusal_text(world); @@ -536,6 +585,251 @@ async fn refusal_lists_both_keys(world: &mut E2eWorld) { } } +// --- comfyui-15: the install finds the runtime setup created --- + +/// The one runtime `comfyui-15` plants. Not in [`RUNTIME_KEYS`], which are the +/// two the ambiguity scenario needs. +const SETUP_RUNTIME_KEY: &str = "e2e-setup-created-runtime"; + +fn registry_dir(world: &E2eWorld) -> PathBuf { + data_dir(world).join("runtimes").join("registry") +} + +#[given("a ready ROCm runtime that setup created and the registry has forgotten")] +async fn plant_runtime_setup_created_and_registry_forgot(world: &mut E2eWorld) { + let data = data_dir(world); + let manifest = plant_ready_runtime_tree(&data, SETUP_RUNTIME_KEY); + let install_root = data.join("runtimes").join("roots").join(SETUP_RUNTIME_KEY); + // The install tree's own copy of its manifest is what makes the registry + // entry recoverable. Without it recovery finds nothing to restore and the + // scenario would pass or fail for the wrong reason. + write_fixture( + &install_root.join(".rocm-cli-runtime.json"), + &serde_json::to_string_pretty(&manifest).expect("manifest serialises"), + ); + // Black-box: plain JSON matching the CLI's on-disk config schema. `setup` + // naming the folder is what lets recovery find the tree; there is no + // registry entry, and that is the state under test. + write_fixture( + &root(world).join("config").join("config.json"), + &serde_json::to_string_pretty(&serde_json::json!({ + "setup": { "therock_venv": install_root.display().to_string() }, + })) + .expect("config serialises"), + ); + assert!( + !registry_dir(world) + .join(format!("{SETUP_RUNTIME_KEY}.json")) + .exists(), + "the registry must start without the runtime for this scenario to mean anything" + ); +} + +#[when("the user previews a ComfyUI install without choosing a runtime")] +async fn preview_comfyui_install_without_runtime(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["comfyui", "install", "--dry-run"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the preview selects that runtime")] +async fn preview_selects_setup_runtime(world: &mut E2eWorld) { + let text = refusal_text(world); + assert_eq!( + world.cli_rc, + Some(0), + "the preview should succeed, got:\n{text}" + ); + let stdout = world.cli_output.clone().unwrap_or_default(); + assert!( + stdout.lines().any(|line| line.contains("install command:") + && line.contains(&format!("--runtime-id {SETUP_RUNTIME_KEY}"))), + "the preview should name the runtime setup created in its install command, got:\n{text}" + ); +} + +#[then("the runtime registry has an entry for it again")] +async fn registry_has_the_setup_runtime_again(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{SETUP_RUNTIME_KEY}.json")); + assert!( + entry.is_file(), + "selecting the runtime setup created should have written {} back", + entry.display() + ); +} + +/// A preview promises not to change files, and the registry entry is a file: the +/// runtime it named is not registered by having been previewed. +#[then("the runtime registry still has no entry for it")] +async fn registry_still_lacks_the_setup_runtime(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{SETUP_RUNTIME_KEY}.json")); + assert!( + !entry.exists(), + "a `--dry-run` preview must not write {}", + entry.display() + ); +} + +#[given("a ComfyUI checkout that needs nothing beyond the runtime's torch stack")] +async fn comfyui_checkout_needing_no_dependencies(world: &mut E2eWorld) { + let source_dir = data_dir(world) + .join("runtimes") + .join("roots") + .join(SETUP_RUNTIME_KEY) + .join("apps") + .join("comfyui") + .join("source"); + // Every ComfyUI release has a `main.py`. The requirements name only the + // torch stack the runtime provides, which the install filters out, so it + // reaches no package index. + write_fixture(&source_dir.join("main.py"), ""); + write_fixture( + &source_dir.join("requirements.txt"), + "torch==2.4.0\ntorchvision==0.19.0\ntorchaudio==2.4.0\n", + ); +} + +/// What stands where the registry directory should be for `comfyui-18`. A plain +/// file stops every user, an administrator included, from creating the registry, +/// where a read-only directory would not stop `root`. +const UNWRITABLE_REGISTRY_CONTENTS: &str = "not a directory"; + +#[given("the runtime registry cannot be written")] +async fn runtime_registry_cannot_be_written(world: &mut E2eWorld) { + let registry = registry_dir(world); + assert!( + !registry.exists(), + "the registry must not exist yet for this step to mean anything" + ); + write_fixture(®istry, UNWRITABLE_REGISTRY_CONTENTS); +} + +/// The failing install names the entry it could not restore, so the user learns +/// why, and is not told that setup left no runtime: the runtime exists, and the +/// preview just named it. +#[then("the install is refused because the runtime's registry entry could not be restored")] +async fn install_refused_for_the_registry_entry(world: &mut E2eWorld) { + let text = refusal_text(world); + assert!( + world.cli_rc.is_some_and(|rc| rc != 0), + "the install should fail when the registry cannot take the runtime back, got:\n{text}" + ); + assert!( + text.contains(&format!( + "failed to restore setup runtime `{SETUP_RUNTIME_KEY}`" + )), + "the failure should name the runtime whose entry could not be restored, got:\n{text}" + ); + assert!( + !text.contains("Install ROCm first"), + "the failure must not tell the user to install ROCm when setup left a runtime, got:\n{text}" + ); +} + +#[then("the runtime registry is as it was")] +async fn runtime_registry_is_as_it_was(world: &mut E2eWorld) { + let registry = registry_dir(world); + assert_eq!( + std::fs::read_to_string(®istry).ok().as_deref(), + Some(UNWRITABLE_REGISTRY_CONTENTS), + "a failed restore must leave {} as it found it", + registry.display() + ); +} + +#[then("the install uses the runtime setup created")] +async fn install_uses_the_setup_runtime(world: &mut E2eWorld) { + let text = refusal_text(world); + assert_eq!( + world.cli_rc, + Some(0), + "the install should succeed, got:\n{text}" + ); + let stdout = world.cli_output.clone().unwrap_or_default(); + assert!( + stdout.contains("action: install") && stdout.contains("ROCm runtime: 7.13.0"), + "the install should report the runtime setup created, got:\n{text}" + ); +} + +// --- comfyui-19: an unusable setup runtime beside a ready registered one --- + +/// The setup runtime `comfyui-19` plants and breaks. Not [`SETUP_RUNTIME_KEY`], +/// so the two scenarios cannot be mistaken for each other in a failure. +const STALE_SETUP_RUNTIME_KEY: &str = "e2e-stale-setup-runtime"; + +#[given("a ready ROCm runtime in the registry")] +async fn plant_one_registered_ready_runtime(world: &mut E2eWorld) { + plant_ready_runtime(&data_dir(world), RUNTIME_KEY); +} + +#[given("a setup folder whose runtime is no longer usable")] +async fn plant_setup_folder_with_an_unusable_runtime(world: &mut E2eWorld) { + let data = data_dir(world); + let manifest = plant_ready_runtime_tree(&data, STALE_SETUP_RUNTIME_KEY); + let install_root = data + .join("runtimes") + .join("roots") + .join(STALE_SETUP_RUNTIME_KEY); + write_fixture( + &install_root.join(".rocm-cli-runtime.json"), + &serde_json::to_string_pretty(&manifest).expect("manifest serialises"), + ); + write_fixture( + &root(world).join("config").join("config.json"), + &serde_json::to_string_pretty(&serde_json::json!({ + "setup": { "therock_venv": install_root.display().to_string() }, + })) + .expect("config serialises"), + ); + // The interpreter is what the readiness check needs and what goes missing + // when a setup folder goes stale. Without removing it this runtime would be + // a second ready one and the scenario would assert the wrong thing. + let python = manifest["python_executable"] + .as_str() + .expect("the planted manifest records an interpreter"); + std::fs::remove_file(python).unwrap_or_else(|e| panic!("failed to remove {python}: {e}")); +} + +#[then("the preview selects the registered runtime")] +async fn preview_selects_the_registered_runtime(world: &mut E2eWorld) { + let text = refusal_text(world); + assert_eq!( + world.cli_rc, + Some(0), + "the preview should succeed, got:\n{text}" + ); + let stdout = world.cli_output.clone().unwrap_or_default(); + assert!( + stdout.lines().any(|line| line.contains("install command:") + && line.contains(&format!("--runtime-id {RUNTIME_KEY}"))), + "the preview should name the registered runtime in its install command, got:\n{text}" + ); + // The recovery error names the runtime ("setup runtime `` ... is not + // usable"); other output can say "not usable" about something else, such as + // the Python the install prepares. + assert!( + !text.contains(STALE_SETUP_RUNTIME_KEY), + "the unusable setup runtime must not be reported while another runtime is ready, got:\n{text}" + ); +} + +#[then("the runtime registry holds only the registered runtime")] +async fn registry_holds_only_the_registered_runtime(world: &mut E2eWorld) { + let mut names: Vec = std::fs::read_dir(registry_dir(world)) + .unwrap_or_else(|e| panic!("the registry should be readable: {e}")) + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + assert_eq!( + names, + vec![format!("{RUNTIME_KEY}.json")], + "the unusable setup runtime must not be written into the registry" + ); +} + /// Pacing knobs for `comfyui-04`'s download server, mirroring /// `therock_steps.rs`'s `PACED_TARBALL_*` constants: large enough that several /// chunk boundaries land before the transfer completes, slow enough per chunk diff --git a/tests/e2e-cucumber/tests/e2e/config_steps.rs b/tests/e2e-cucumber/tests/e2e/config_steps.rs index 726b3de098..dfb518892b 100644 --- a/tests/e2e-cucumber/tests/e2e/config_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/config_steps.rs @@ -211,6 +211,395 @@ async fn confirm_key_not_echoed(world: &mut E2eWorld) { ); } +// ── Unreadable settings file ────────────────────────────────────── + +/// The custom ROCm folder the damaged settings file records, inside the +/// scenario's sandbox so the suggested `ROCM_CLI_DATA_DIR` can be used as-is. +/// Its name has a space, so the refusal's advice only works if it quotes it. +fn custom_rocm_folder(world: &E2eWorld) -> std::path::PathBuf { + sandbox(world).join("custom rocm") +} + +/// The path in double quotes right after the first `label` in `stderr`: what +/// a user copies when the refusal tells them which path to use. +fn quoted_after(stderr: &str, label: &str) -> String { + let (_, rest) = stderr + .split_once(label) + .unwrap_or_else(|| panic!("the refusal does not say {label:?}:\n{stderr}")); + let rest = rest + .strip_prefix('"') + .unwrap_or_else(|| panic!("no quoted path after {label:?}:\n{stderr}")); + let (path, _) = rest + .split_once('"') + .unwrap_or_else(|| panic!("an unclosed quote after {label:?}:\n{stderr}")); + path.to_owned() +} + +/// A `config.json` cut off just after it recorded `setup.therock_venv`, the +/// field that tells the CLI where a custom-folder install keeps its runtimes. +/// Black-box: plain bytes, as an interrupted write or a bad hand edit leaves +/// them. Derived from the sandbox, so every Then can rebuild it and compare +/// byte for byte. +fn cut_short_settings(world: &E2eWorld) -> String { + let folder = serde_json::to_string(&custom_rocm_folder(world)) + .expect("failed to encode the custom folder"); + format!( + "{{\n \"active_runtime_key\": \"therock-release-gfx1151-7.13.0\",\n \"setup\": {{\n \ + \"therock_venv\": {folder},\n \"comp" + ) +} + +fn sandbox(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has an isolated root") + .path() + .to_path_buf() +} + +fn settings_path(world: &E2eWorld) -> std::path::PathBuf { + sandbox(world).join("config").join("config.json") +} + +fn renamed_settings_path(world: &E2eWorld) -> std::path::PathBuf { + sandbox(world).join("config").join("config.json.broken") +} + +/// The runtime planted in the custom folder; the damaged settings file names +/// it as the active one. +const CUSTOM_FOLDER_RUNTIME_KEY: &str = "therock-release-gfx1151-7.13.0"; + +#[given( + "the CLI's settings file was cut short just after it recorded a custom ROCm folder holding \ + an installed runtime" +)] +async fn settings_cut_short(world: &mut E2eWorld) { + let path = settings_path(world); + std::fs::create_dir_all(path.parent().expect("settings file has a parent")) + .expect("failed to create config dir"); + std::fs::write(&path, cut_short_settings(world)).expect("failed to plant settings file"); + + // A read-only tarball runtime registered in the custom folder's own + // registry, as `rocm install sdk --prefix` leaves it. Plain JSON matching + // the on-disk manifest schema, black-box like the settings file above. + let folder = custom_rocm_folder(world); + let install_root = folder.join("runtime-gfx1151"); + std::fs::create_dir_all(&install_root).expect("failed to create install root"); + std::fs::write(install_root.join("payload.txt"), "payload") + .expect("failed to write runtime payload"); + let registry = folder.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry).expect("failed to create registry dir"); + let manifest = serde_json::to_string_pretty(&serde_json::json!({ + "runtime_key": CUSTOM_FOLDER_RUNTIME_KEY, + "runtime_id": "therock-release:gfx1151", + "channel": "release", + "format": "tarball", + "family": "gfx1151", + "family_source": "manual", + "version": "7.13.0", + "install_root": install_root, + "selected_artifact_url": format!("https://example.invalid/{CUSTOM_FOLDER_RUNTIME_KEY}.tar.gz"), + "read_only": true, + "installed_at_unix_ms": 1_700_000_000_000u64, + })) + .expect("failed to serialize runtime manifest"); + std::fs::write( + registry.join(format!("{CUSTOM_FOLDER_RUNTIME_KEY}.json")), + manifest, + ) + .expect("failed to write runtime manifest"); +} + +#[when("the user checks the CLI version without naming a data folder")] +async fn version_without_data_dir(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm_without_data_dir_override(world, &["version"]); + record(world, stdout, stderr, rc); +} + +#[when("the user opens the CLI with the data folder the refusal suggested")] +async fn open_with_suggested_data_dir(world: &mut E2eWorld) { + // Take the value from the refusal itself, so this runs the advice as + // printed rather than a value the test already knows. + let folder = suggested_data_dir(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_env(world, &[], &[("ROCM_CLI_DATA_DIR", &folder)]); + record(world, stdout, stderr, rc); +} + +#[when("the user changes a setting with the data folder the refusal suggested")] +async fn change_setting_with_suggested_data_dir(world: &mut E2eWorld) { + let folder = suggested_data_dir(world); + let (stdout, stderr, rc) = crate::run_rocm_with_env( + world, + &["config", "set-telemetry", "off"], + &[("ROCM_CLI_DATA_DIR", &folder)], + ); + record(world, stdout, stderr, rc); +} + +/// The `ROCM_CLI_DATA_DIR` value the refusal told the user to set, read from +/// its stderr — "setting the environment variable ROCM_CLI_DATA_DIR to +/// \"\" ..." — the first time it is needed, and remembered for the +/// steps after that. +fn suggested_data_dir(world: &mut E2eWorld) -> String { + if let Some(folder) = &world.suggested_data_dir { + return folder.clone(); + } + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + let folder = quoted_after( + stderr, + "setting the environment variable ROCM_CLI_DATA_DIR to ", + ); + world.suggested_data_dir = Some(folder.clone()); + folder +} + +#[when( + "the user renames the settings file as the refusal said and opens the CLI without naming a \ + data folder" +)] +async fn open_after_renaming_settings(world: &mut E2eWorld) { + // The paths come from the refusal itself, read by the step that checked + // it, so this follows the advice as printed. + let (from, to) = world + .suggested_settings_rename + .clone() + .expect("the refusal was checked before its advice is followed"); + std::fs::rename(&from, &to).expect("failed to rename the settings file"); + assert_eq!( + std::fs::read_to_string(&to).expect("the renamed settings file is gone"), + cut_short_settings(world), + "renaming the settings file changed its contents" + ); + let (stdout, stderr, rc) = crate::run_rocm_without_data_dir_override(world, &[]); + record(world, stdout, stderr, rc); +} + +#[then("the CLI refuses, naming the settings file, the recorded folder and how to recover")] +async fn refuses_unreadable_settings(world: &mut E2eWorld) { + assert_failed(world); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + let suggestion = format!( + "setting the environment variable ROCM_CLI_DATA_DIR to \"{}\"", + custom_rocm_folder(world).display() + ); + for needle in [ + "cannot tell where rocm-cli keeps its data", + "config.json", + "cannot be read", + "Repair that file", + suggestion.as_str(), + "any command that changes settings still stops on it", + "run on default settings", + ] { + assert!( + stderr.contains(needle), + "expected stderr to contain {needle:?}, got:\n{}", + combined_output(world) + ); + } + // The file to rename is the exact one the CLI read — here under the + // scenario's own config folder — and the name to give it sits beside it. + // Both are read from between their quotes, as a user would copy them. + let from = quoted_after(stderr, "keeps its data: "); + let to = quoted_after(stderr, "Renaming it to "); + assert_eq!(std::path::Path::new(&from), settings_path(world)); + assert_eq!(std::path::Path::new(&to), renamed_settings_path(world)); + world.suggested_settings_rename = Some((from, to)); +} + +#[then("the settings file is left exactly as it was")] +async fn settings_unchanged(world: &mut E2eWorld) { + let after = std::fs::read_to_string(settings_path(world)).expect("settings file is gone"); + assert_eq!( + after, + cut_short_settings(world), + "the CLI rewrote the settings file" + ); +} + +/// Where config-11 keeps the file its settings link points at: outside the +/// config directory, as a dotfiles checkout would be. +fn linked_settings_target(world: &E2eWorld) -> std::path::PathBuf { + sandbox(world).join("dotfiles").join("rocm-config.json") +} + +fn telemetry_mode_in(path: &std::path::Path) -> String { + let text = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display())); + let value: serde_json::Value = serde_json::from_str(&text) + .unwrap_or_else(|e| panic!("{} is not JSON ({e}):\n{text}", path.display())); + value["telemetry"]["mode"] + .as_str() + .unwrap_or_else(|| panic!("{} records no telemetry mode:\n{text}", path.display())) + .to_owned() +} + +#[given("a settings file that only its owner can read")] +async fn private_settings_file(world: &mut E2eWorld) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let path = settings_path(world); + std::fs::create_dir_all(path.parent().expect("settings file has a parent")) + .expect("failed to create config dir"); + std::fs::write(&path, "{}\n").expect("failed to plant settings file"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .expect("failed to make the settings file private"); + } + #[cfg(not(unix))] + { + let _ = world; + panic!("tagged @requires-os:linux: needs Unix permissions"); + } +} + +#[then("the settings file is still readable only by its owner")] +async fn settings_still_private(world: &mut E2eWorld) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(settings_path(world)) + .expect("settings file is gone") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "saving changed the settings file's permissions to {:o}", + mode & 0o777 + ); + } + #[cfg(not(unix))] + { + let _ = world; + panic!("tagged @requires-os:linux: needs Unix permissions"); + } +} + +#[then("the settings file records telemetry as off")] +async fn settings_record_telemetry_off(world: &mut E2eWorld) { + assert_eq!(telemetry_mode_in(&settings_path(world)), "off"); +} + +#[given("a settings file that is a link to a file kept elsewhere")] +async fn linked_settings_file(world: &mut E2eWorld) { + #[cfg(unix)] + { + let target = linked_settings_target(world); + std::fs::create_dir_all(target.parent().expect("link target has a parent")) + .expect("failed to create the dotfiles folder"); + std::fs::write(&target, "{}\n").expect("failed to plant the linked-to file"); + let path = settings_path(world); + std::fs::create_dir_all(path.parent().expect("settings file has a parent")) + .expect("failed to create config dir"); + std::os::unix::fs::symlink(&target, &path).expect("failed to link the settings file"); + } + #[cfg(not(unix))] + { + let _ = world; + panic!("tagged @requires-os:linux: needs Unix links"); + } +} + +#[then("the settings file is still a link to that file")] +async fn settings_still_linked(world: &mut E2eWorld) { + let path = settings_path(world); + let metadata = std::fs::symlink_metadata(&path).expect("settings file is gone"); + assert!( + metadata.file_type().is_symlink(), + "saving replaced the linked settings file with a regular file" + ); + assert_eq!( + std::fs::read_link(&path).expect("cannot read the settings link"), + linked_settings_target(world), + "the settings link now points somewhere else" + ); +} + +#[then("the linked-to file records telemetry as off")] +async fn linked_target_records_telemetry_off(world: &mut E2eWorld) { + assert_eq!(telemetry_mode_in(&linked_settings_target(world)), "off"); +} + +/// The launcher summary's `data dir:` line, from a run that succeeded. +fn launcher_data_dir(world: &E2eWorld) -> String { + assert_ok_contains(world, "active runtime key: "); + let stdout = world.cli_output.as_deref().unwrap_or(""); + stdout + .lines() + .find_map(|line| line.trim().strip_prefix("data dir: ")) + .unwrap_or_else(|| panic!("the CLI did not say where its data is:\n{stdout}")) + .to_owned() +} + +// "On default settings" is what `active runtime key: ` shows: the broken +// file records an active runtime key, and the CLI is not reading it. +#[then("the CLI keeps its data in the recorded folder, on default settings")] +async fn data_in_recorded_folder(world: &mut E2eWorld) { + let data_dir = launcher_data_dir(world); + assert_eq!( + std::path::Path::new(&data_dir), + custom_rocm_folder(world), + "the suggested ROCM_CLI_DATA_DIR did not point the CLI at the recorded folder" + ); +} + +/// The default data folder: `.rocm` under the home directory +/// `run_rocm_without_data_dir_override` gives the CLI. +#[then("the CLI keeps its data in its default folder")] +async fn data_in_default_folder(world: &mut E2eWorld) { + let data_dir = launcher_data_dir(world); + assert_eq!( + std::path::Path::new(&data_dir), + sandbox(world).join("home").join(".rocm"), + "with the settings file renamed, the CLI did not go back to its default folder" + ); +} + +#[when("the user lists runtimes with the data folder the refusal suggested")] +async fn list_runtimes_with_suggested_data_dir(world: &mut E2eWorld) { + let folder = suggested_data_dir(world); + let (stdout, stderr, rc) = crate::run_rocm_with_env( + world, + &["runtimes", "list"], + &[("ROCM_CLI_DATA_DIR", &folder)], + ); + record(world, stdout, stderr, rc); +} + +#[then("the runtime installed in the recorded folder is listed")] +async fn custom_folder_runtime_listed(world: &mut E2eWorld) { + assert_ok_contains(world, CUSTOM_FOLDER_RUNTIME_KEY); +} + +/// The data folder is already the one the refusal suggested, so the advice +/// is the rename alone, naming the exact file and the name to give it; that +/// following it works is the rename step later in the same scenario. +#[then("the CLI refuses to change the setting, naming the settings file to rename")] +async fn refuses_to_change_setting(world: &mut E2eWorld) { + assert_failed(world); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + let rename = format!( + "To start from default settings instead, rename \"{}\" to \"{}\"", + settings_path(world).display(), + renamed_settings_path(world).display() + ); + assert!( + stderr.contains("Repair that file") && stderr.contains(&rename), + "expected the refusal to say {rename:?}, got:\n{}", + combined_output(world) + ); + // The user already set the folder the first refusal suggested, so being + // told to set it again would be wrong. + assert!( + !stderr.contains("then also need the environment variable ROCM_CLI_DATA_DIR"), + "the refusal repeated the ROCM_CLI_DATA_DIR advice the user already followed:\n{}", + combined_output(world) + ); +} + // ── Helpers ──────────────────────────────────────────────────────── fn record(world: &mut E2eWorld, stdout: String, stderr: String, rc: i32) { diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 45915271ef..ddb5b598d9 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -100,6 +100,330 @@ async fn wait_for_all_markers(tui: &mut TuiSession, what: &str, markers: &[&str] .await } +// This is a protocol contract journey: CLI, embedded daemon and TUI are real; +// the HTTP service is a named protocol fixture, not GPU inference. +const PROTOCOL_MODELS: [&str; 2] = ["Protocol-Qwen-4B", "Protocol-Phi-3B"]; + +#[given("an external Lemonade server has two models loaded")] +async fn external_lemonade_protocol(world: &mut E2eWorld) { + let server = e2e_cucumber::lemonade_protocol::LemonadeProtocolServer::start(&PROTOCOL_MODELS); + configure_lemonade_discovery(world, server.port()); + let root = world.isolated_root.as_ref().expect("isolated root").path(); + assert!( + !root.join("data/services").exists(), + "no managed service may be planted" + ); + world.lemonade_protocol = Some(server); +} + +fn configure_lemonade_discovery(world: &E2eWorld, port: u16) { + // Discovery is off by default in the isolated config (see + // `disable_lemonade_discovery_by_default`), so these scenarios turn it on + // explicitly. `lemonade_host` is omitted to exercise its serde default. + // Socket isolation comes from the PTY driver's HOME/XDG environment. + write_dashboard_config( + world, + serde_json::json!({ + "dashboard": { "daemon": { "enable_lemonade": true, "lemonade_port": port, + "discovery_tick_secs": 0.1, "instance_tick_secs": 0.1 } } + }), + ); +} + +#[given("the dashboard watches only managed services")] +async fn dashboard_managed_only(world: &mut E2eWorld) { + // An unrelated live endpoint on the default port must not supply the + // model or external-running status in this real managed-service journey. + write_dashboard_config( + world, + serde_json::json!({"dashboard": {"daemon": {"enable_lemonade": false}}}), + ); +} + +/// Keep a dashboard that is not testing Lemonade discovery away from any real +/// Lemonade server on the default port. Call before every dashboard spawn. +fn disable_lemonade_discovery_by_default(world: &E2eWorld) { + let root = world.isolated_root.as_ref().expect("isolated root").path(); + let dir = root.join("config"); + std::fs::create_dir_all(&dir).expect("create config dir"); + let path = dir.join("config.json"); + let existing = std::fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()); + let config = e2e_cucumber::dash_screen::with_lemonade_discovery_default_off(existing); + std::fs::write( + &path, + serde_json::to_vec(&config).expect("serialize config"), + ) + .expect("write config"); +} + +fn write_dashboard_config(world: &E2eWorld, config: serde_json::Value) { + let root = world.isolated_root.as_ref().expect("isolated root").path(); + let dir = root.join("config"); + std::fs::create_dir_all(&dir).expect("create config dir"); + let path = dir.join("config.json"); + // An earlier step may have written the active runtime here; keep it. + let existing = std::fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()); + let merged = e2e_cucumber::dash_screen::merge_dashboard_daemon_config(existing, &config); + std::fs::write( + &path, + serde_json::to_vec(&merged).expect("serialize config"), + ) + .expect("write config"); +} + +#[then("both loaded models are displayed with throughput shown as unknown")] +async fn external_models_visible(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "both protocol models, with unknown unattributable throughput", + |screen| { + PROTOCOL_MODELS.iter().all(|model| { + screen + .lines() + .any(|line| protocol_tps_cell(line, model) == Some("—")) + }) + }, + default_timeout(), + ) + .await + .expect("external models should be discovered"); +} + +fn protocol_tps_cell<'a>(line: &'a str, model: &str) -> Option<&'a str> { + let mut cells = line.split_whitespace().skip_while(|cell| *cell != model); + cells.next()?; + cells.next() +} + +#[when("the external Lemonade server unloads its second model")] +async fn external_single_model(world: &mut E2eWorld) { + world + .lemonade_protocol + .as_ref() + .expect("fixture") + .set_loaded(&PROTOCOL_MODELS[..1]); +} + +#[then("only the first model is displayed, with its throughput")] +async fn external_single_model_rate(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "single protocol model with unambiguous last-request throughput", + |screen| { + !screen.contains(PROTOCOL_MODELS[1]) + && screen + .lines() + .any(|line| protocol_tps_cell(line, PROTOCOL_MODELS[0]) == Some("38")) + // The rate is the last completed request's, and the screen says so. + && screen.contains("last req tok/s") + }, + default_timeout(), + ) + .await + .expect("single loaded model owns endpoint stats"); +} + +#[when("the external Lemonade server unloads every model")] +async fn external_unload(world: &mut E2eWorld) { + world + .lemonade_protocol + .as_ref() + .expect("fixture") + .set_loaded(&[]); +} + +#[then("no loaded model is displayed and none are reported as discovered")] +async fn external_models_absent(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "unloaded external models disappear", + |screen| { + PROTOCOL_MODELS.iter().all(|m| !screen.contains(m)) + && screen.contains("no serving models discovered") + }, + default_timeout(), + ) + .await + .expect("stale external models must disappear"); +} + +#[when("the external Lemonade server loads both models again")] +async fn external_reload(world: &mut E2eWorld) { + world + .lemonade_protocol + .as_ref() + .expect("fixture") + .set_loaded(&PROTOCOL_MODELS); +} + +#[when("the external Lemonade server stops responding")] +async fn external_unreachable(world: &mut E2eWorld) { + drop(world.lemonade_protocol.take()); +} + +#[then("the external server received no lifecycle request and no managed record exists")] +async fn external_server_untouched(world: &mut E2eWorld) { + // What this proves: the fixture recorded only observational requests (the + // stray-request check below) and the registry holds no record. The health + // read is a sanity check that the fixture still answers; the fixture is an + // in-process thread the dashboard cannot stop, so it is not evidence of + // anything on its own. + let fixture = world.lemonade_protocol.as_ref().expect("fixture"); + let response = reqwest::get(format!("{}/api/v1/health", fixture.base_url())) + .await + .expect("dashboard quit must leave external service alive"); + assert!(response.status().is_success()); + let body: serde_json::Value = response.json().await.expect("health JSON"); + assert_eq!(body["status"], "ok"); + // The dashboard may only observe: any load, unload, stop or other request + // reaching the external server would be a lifecycle action on a service it + // does not own. + assert_eq!( + fixture.stray_requests(), + Vec::::new(), + "the dashboard sent a non-observational request to the external server" + ); + assert_eq!( + body["all_models_loaded"].as_array().expect("models").len(), + 2 + ); + let services = world + .isolated_root + .as_ref() + .expect("root") + .path() + .join("data/services"); + assert!( + !services.exists() + || std::fs::read_dir(services) + .expect("services") + .next() + .is_none(), + "discovery must not adopt an external fixture as a managed service" + ); +} + +#[when("the user selects engine status in the Serving view")] +async fn engine_status_pane(world: &mut E2eWorld) { + let tui = session(world); + tui.use_detail_size().expect("enlarge dashboard"); + tui.send_until("3", "● Serving", default_timeout()) + .await + .expect("Serving view"); + tui.send("gj").expect("select Engines action"); +} + +#[then("Lemonade is shown as externally running without a managed installation")] +async fn external_installation_truth(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "external versus managed engine state", + |screen| { + screen.contains("Lemonade — externally running") + && screen.contains("Managed runtime — not installed") + }, + default_timeout(), + ) + .await + .expect("engine status must distinguish external service and managed install"); +} + +#[then("Lemonade returns to the managed installation status without external availability")] +async fn external_availability_cleared(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "unreachable endpoint no longer counts as external availability", + |screen| { + screen.contains("Lemonade — managed runtime not installed") + && !screen.contains("Lemonade — externally running") + }, + default_timeout(), + ) + .await + .expect("engine status must return to managed inventory after endpoint loss"); +} + +#[then("the real managed Lemonade model is displayed")] +async fn real_lemonade_visible(world: &mut E2eWorld) { + // The preceding real inference assertion identifies this scenario's model. + // Direct llama-server need not expose Lemonade stats: missing throughput is + // truthful and must not cause us to invent a rate to satisfy this journey. + // + // Anchored to a full Observe instances row (the model cell followed by the + // TOK/S .. KV% cells), not to the name appearing anywhere on screen, so a + // stray mention elsewhere cannot satisfy it. The dashboard watches only + // managed services in this scenario, so that row is the registry-owned one. + // The row carries neither status nor port; the next steps read those from + // the instance detail. + session(world) + .wait_for_screen_where( + "an Observe row for the managed Lemonade model", + |screen| { + screen.lines().any(|line| { + e2e_cucumber::dash_screen::observe_row_cells(line, "Qwen3-0.6B").is_some() + }) + }, + default_timeout(), + ) + .await + .expect("dashboard must display this scenario's real served model"); +} + +#[then("the managed model's instance detail shows it ready on the port it was served on")] +async fn real_lemonade_detail_ready(world: &mut E2eWorld) { + let endpoint = world + .endpoint + .clone() + .expect("the serve step sets an endpoint"); + let port = e2e_cucumber::dash_screen::endpoint_port(&endpoint) + .unwrap_or_else(|| panic!("no port in served endpoint {endpoint}")); + session(world) + .wait_for_screen_where( + "instance detail showing the model READY on the served port", + |screen| { + e2e_cucumber::dash_screen::instance_detail_matches( + screen, + "Qwen3-0.6B", + "READY", + &port, + ) + }, + default_timeout(), + ) + .await + .unwrap_or_else(|e| { + panic!("instance detail must show Qwen3-0.6B READY on port {port}: {e}") + }); +} + +#[when("the user closes instance detail")] +async fn close_instance_detail(world: &mut E2eWorld) { + // Esc closes the popup, but with no popup open it opens the menu. So send + // it once and wait for the popup to be gone. + let tui = session(world); + tui.send("\x1b") + .unwrap_or_else(|e| panic!("failed to send Esc: {e}")); + tui.wait_for_screen_where( + "instance detail closed", + |screen| !screen.contains("Instance · "), + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("instance detail did not close: {e}")); +} + +#[then("Lemonade is shown as a managed runtime installation")] +async fn real_lemonade_installed(world: &mut E2eWorld) { + session(world) + .wait_for_screen("Lemonade — managed runtime installed", default_timeout()) + .await + .expect("managed engine Detect evidence must reach dashboard"); +} + // ── Given ────────────────────────────────────────────────────────── #[given("interactive chat uses an offline assistant")] @@ -208,6 +532,7 @@ async fn replay_missing_recording(world: &mut E2eWorld) { )); let _ = std::fs::remove_file(&missing); let path = missing.to_string_lossy().to_string(); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn(world, &["dash", "--replay", &path]) .unwrap_or_else(|e| panic!("failed to spawn dash under a pty: {e}")); world.tui = Some(session); @@ -217,11 +542,116 @@ async fn replay_missing_recording(world: &mut E2eWorld) { async fn open_dashboard_demo(world: &mut E2eWorld) { // `--demo` replays a deterministic synthetic session, so the dashboard // populates with no GPU and no daemon — a stable, mock-tier target. + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn(world, &["dash", "--demo"]) .unwrap_or_else(|e| panic!("failed to open the dashboard: {e}")); world.tui = Some(session); } +/// Cores reported by `dash-50`'s idle recording. A distinctive count so the +/// title assertion cannot match some other panel's text. +const IDLE_CORES: usize = 64; + +/// Baseline braille cells: the bottom dot row of both columns lit. +/// +/// Four consecutive cells cannot appear unless idle samples are drawn at all — +/// before #796 a sample under a few percent of the scale rounded to zero lit +/// dots and left a gap. Four rather than a longer run because the graph is +/// right-aligned and grows two samples per column, so a longer run would be +/// racing the replay's 2-second cadence against the step timeout. +const IDLE_BASELINE_RUN: &str = "⣀⣀⣀⣀"; + +#[then("the CPU graph is drawn as a continuous line")] +async fn cpu_graph_is_continuous(world: &mut E2eWorld) { + session(world) + .wait_for_screen(IDLE_BASELINE_RUN, default_timeout()) + .await + .unwrap_or_else(|e| panic!("the idle CPU graph did not render as a line: {e}")); +} + +#[then("the CPU box reports the host's core count")] +async fn cpu_box_reports_core_count(world: &mut E2eWorld) { + // Observe's CPU slot is too short for the per-core box, so the count has + // to come from the overall box's own title. + // + // Matching `· N cores` alone would prove nothing: it is a suffix of the + // per-core box's own title too, so the step would pass whether or not the + // count was folded. Assert the whole folded title, and that the box it + // would otherwise have come from is absent. + let folded = format!("CPU · 2.0% · {IDLE_CORES} cores"); + session(world) + .wait_for_screen_where( + "the CPU title carries the core count and no per-core box is drawn", + |screen| screen.contains(&folded) && !screen.contains("Per-core"), + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the CPU box did not report the core count: {e}")); +} + +/// Cores for `dash-51`. The side-by-side split wants two columns per core on +/// each half of the CPU slot, and Observe's centre column is about 116 columns +/// at the detail geometry, so 16 cores clear it with room to spare while +/// `IDLE_CORES` does not. Widening the terminal is not the lever: past a +/// threshold Observe grows side docks and the centre column gets narrower. +const SIDE_BY_SIDE_CORES: usize = 16; + +#[when("the user opens the dashboard replaying an idle 16-core host")] +async fn open_dashboard_replaying_idle_16_core_host(world: &mut E2eWorld) { + replay_idle_host(world, SIDE_BY_SIDE_CORES, "idle-16-core-recording.ndjson"); +} + +#[then("the CPU graph and the per-core strip sit side by side")] +async fn cpu_boxes_sit_side_by_side(world: &mut E2eWorld) { + // Both titles on one screen row is what separates this layout from the + // stacked one, where each box owns its own row. + session(world) + .wait_for_screen_where( + "the CPU and per-core titles share a row", + |screen| { + screen + .lines() + .any(|row| row.contains("CPU · ") && row.contains("Per-core · 16 cores")) + }, + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the CPU boxes were not drawn side by side: {e}")); +} + +#[when("the user opens the dashboard replaying an idle host")] +async fn open_dashboard_replaying_idle_host(world: &mut E2eWorld) { + replay_idle_host(world, IDLE_CORES, "idle-host-recording.ndjson"); +} + +/// Replay a quiet host with `cores` idle cores: 2% overall, 1% each. +fn replay_idle_host(world: &mut E2eWorld, cores: usize, file_name: &str) { + let per_core = (0..cores) + .map(|_| "1.0".to_string()) + .collect::>() + .join(","); + let host = format!( + concat!( + r#""cpu_overall_pct":2.0,"cpu_per_core_pct":[{per_core}],"memory_used_mb":0,"#, + r#""memory_total_mb":0,"swap_used_mb":0,"swap_total_mb":0,"#, + r#""disk_read_bps":0,"disk_write_bps":0,"net_rx_bps":0,"net_tx_bps":0"#, + ), + per_core = per_core + ); + replay_recorded_snapshots_with_host( + world, + file_name, + &host, + // The GPU is given a load rather than left at 0: the widget draws an + // idle series as the same baseline run this scenario looks for, so a + // quiet GPU panel could satisfy the assertion without the CPU graph + // ever rendering. + r#"{"device_id":"0","vram_used_mb":0,"vram_total_mb":1024,"gpu_utilization_pct":55.0,"temperature_c":40.0,"power_w":90.0,"clock_mhz":null}"#, + "", + "", + ); +} + /// The one warning `dash-39`'s recording reports. Short and ASCII so it stays on /// one screen row inside the popup and is found by a plain substring match. const RECORDED_WARNING: &str = "amd-smi unavailable (GPU device inaccessible)"; @@ -239,19 +669,47 @@ fn replay_recorded_snapshots( gpus: &str, instances: &str, warnings: &str, +) { + replay_recorded_snapshots_with_host( + world, + file_name, + IDLE_HOST_FIELDS, + gpus, + instances, + warnings, + ); +} + +/// A quiet host: the body most scenarios want, since they assert on GPUs or +/// instances rather than on the host panels. +const IDLE_HOST_FIELDS: &str = concat!( + r#""cpu_overall_pct":0.0,"cpu_per_core_pct":[],"memory_used_mb":0,"#, + r#""memory_total_mb":0,"swap_used_mb":0,"swap_total_mb":0,"#, + r#""disk_read_bps":0,"disk_write_bps":0,"net_rx_bps":0,"net_tx_bps":0"#, +); + +/// As [`replay_recorded_snapshots`], with the snapshot's `host` body supplied +/// by the caller for scenarios that assert on the host panels themselves. +/// `host` is the object body without its braces. +fn replay_recorded_snapshots_with_host( + world: &mut E2eWorld, + file_name: &str, + host: &str, + gpus: &str, + instances: &str, + warnings: &str, ) { let snapshot = |ts_us: u64| { format!( concat!( r#"{{"ts_us":{ts_us},"event":{{"kind":"snapshot","#, r#""timestamp":"2023-11-15T00:00:00Z","#, - r#""host":{{"cpu_overall_pct":0.0,"cpu_per_core_pct":[],"memory_used_mb":0,"#, - r#""memory_total_mb":0,"swap_used_mb":0,"swap_total_mb":0,"#, - r#""disk_read_bps":0,"disk_write_bps":0,"net_rx_bps":0,"net_tx_bps":0}},"#, + r#""host":{{{host}}},"#, r#""gpus":[{gpus}],"gpu_system_info":null,"instances":[{instances}],"#, r#""warnings":[{warnings}]}}}}"#, ), ts_us = ts_us, + host = host, gpus = gpus, instances = instances, warnings = warnings @@ -270,6 +728,7 @@ fn replay_recorded_snapshots( std::fs::write(&recording, recording_text) .unwrap_or_else(|e| panic!("failed to write {}: {e}", recording.display())); let path = recording.to_string_lossy().to_string(); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn(world, &["dash", "--replay", &path]) .unwrap_or_else(|e| panic!("failed to open the dashboard replaying a recording: {e}")); world.tui = Some(session); @@ -292,11 +751,14 @@ async fn open_dashboard_replaying_warning(world: &mut E2eWorld) { } /// One recorded instance per lifecycle state the Services manager colors. The -/// model name carries the state so the scenario can find each row by it. +/// model name carries the state so the scenario can find each row by it. Each is +/// recorded as managed, because the Services manager lists only instances the +/// daemon marked as registry-owned (a recording without the field replays as +/// unmanaged and would leave the overlay empty). fn recorded_instance(id: &str, model: &str, status_json: &str) -> String { format!( concat!( - r#"{{"container_id":"{id}","container_name":"vllm-{id}","status":{status},"#, + r#"{{"container_id":"{id}","container_name":"vllm-{id}","managed":true,"status":{status},"#, r#""model_name":"{model}","gpu_ids":["0"],"tensor_parallel_size":1,"#, r#""vram_used_mb":0,"vram_total_mb":0,"gen_tps":null,"launch_args":[],"env_vars":{{}}}}"#, ), @@ -379,6 +841,7 @@ async fn open_dashboard_in_scenario_folder(world: &mut E2eWorld) { child.display() ) }); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn_in(world, &["dash", "--demo"], &root) .unwrap_or_else(|e| panic!("failed to open the dashboard in {}: {e}", root.display())); world.tui = Some(session); @@ -423,6 +886,7 @@ async fn open_dashboard_in_whitespace_named_folder(world: &mut E2eWorld) { cwd.display() ) }); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn_in(world, &["dash", "--demo"], &cwd) .unwrap_or_else(|e| panic!("failed to open the dashboard in {}: {e}", cwd.display())); world.tui = Some(session); @@ -435,6 +899,10 @@ async fn open_chat(world: &mut E2eWorld) { } else { &["chat"] }; + // `rocm chat` starts the embedded daemon too, and discovery is on by default. + if world.isolated_root.is_some() { + disable_lemonade_discovery_by_default(world); + } let session = TuiSession::spawn(world, args) .unwrap_or_else(|e| panic!("failed to open interactive chat: {e}")); world.tui = Some(session); @@ -458,6 +926,7 @@ async fn machine_with_no_gpu(world: &mut E2eWorld) { #[when("the user opens the dashboard")] async fn open_dashboard(world: &mut E2eWorld) { + disable_lemonade_discovery_by_default(world); let tui = TuiSession::spawn(world, &["dash"]) .unwrap_or_else(|e| panic!("failed to open the dashboard: {e}")); world.tui = Some(tui); @@ -467,6 +936,7 @@ async fn open_dashboard(world: &mut E2eWorld) { async fn open_launcher(world: &mut E2eWorld) { // Bare `rocm` (no subcommand) opens the launcher front door under an // interactive terminal — the PTY slave satisfies `interactive_terminal()`. + disable_lemonade_discovery_by_default(world); let tui = TuiSession::spawn(world, &[]) .unwrap_or_else(|e| panic!("failed to open the launcher: {e}")); world.tui = Some(tui); @@ -524,6 +994,104 @@ async fn open_services_overlay(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("failed to open the services overlay: {e}")); } +#[then("the Services overlay has no managed servers")] +async fn external_services_are_read_only(world: &mut E2eWorld) { + session(world) + .wait_for_screen( + "No managed services. Start one with `rocm serve --managed`.", + default_timeout(), + ) + .await + .expect("discovered external models must not become lifecycle targets"); +} + +#[then("the Services overlay lists the managed model with its port")] +async fn services_overlay_lists_managed_model(world: &mut E2eWorld) { + let model = world.model_name.clone().expect("no model name set"); + let port = + e2e_cucumber::dash_screen::endpoint_port(world.endpoint.as_deref().expect("no endpoint")) + .expect("endpoint has a port"); + let port_cell = format!(":{port}"); + session(world) + .wait_for_screen_where( + "a Services row for the managed model and its port", + |screen| { + !screen.contains("No managed services.") + && screen + .lines() + .any(|line| line.contains(&model) && line.contains(&port_cell)) + }, + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the managed service is not listed in the overlay: {e}")); +} + +#[when("the user asks to stop the listed service")] +async fn user_stages_stop(world: &mut E2eWorld) { + // Sent once: the key stages an approval, and a second copy would land on the + // approval prompt rather than the list. + session(world) + .send("s") + .unwrap_or_else(|e| panic!("failed to send the stop key: {e}")); +} + +#[then("a stop approval prompt names the managed model and its port")] +async fn stop_approval_is_shown(world: &mut E2eWorld) { + let model = world.model_name.clone().expect("no model name set"); + let port = + e2e_cucumber::dash_screen::endpoint_port(world.endpoint.as_deref().expect("no endpoint")) + .expect("endpoint has a port"); + let detail = format!("model: {model} port: {port}"); + session(world) + .wait_for_screen_where( + "a stop approval for the managed service", + |screen| { + screen.contains("Review: stop service") + && screen.contains("Approve (y)") + && screen.contains(&detail) + }, + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("no stop approval prompt appeared: {e}")); +} + +#[when("the user declines the approval prompt")] +async fn user_declines_approval(world: &mut E2eWorld) { + // Esc declines without confirming. The prompt itself going away is the + // proof the key was read; the overlay list stays open behind it. + let tui = session(world); + tui.send("\u{1b}") + .unwrap_or_else(|e| panic!("failed to decline the approval: {e}")); + tui.wait_until_gone("Review: stop service", default_timeout()) + .await + .unwrap_or_else(|e| panic!("the approval prompt did not close: {e}")); +} + +#[when("the user attempts to stop and restart the external models")] +async fn external_lifecycle_keys_do_nothing(world: &mut E2eWorld) { + let tui = session(world); + tui.send("sr").expect("attempt stop and restart"); + // One Escape closes the list only if neither key staged an approval or a + // lifecycle console. In either modal, it dismisses that modal instead and + // leaves the Services overlay open, which the next step then catches. + tui.send("\u{1b}").expect("close Services list"); +} + +#[then("the Services overlay closes without an approval prompt or job")] +async fn external_lifecycle_keys_left_no_modal(world: &mut E2eWorld) { + session(world) + .wait_until_gone(SERVICES_OVERLAY_TITLE, default_timeout()) + .await + .expect("external lifecycle keys must not stage approval or launch a job"); +} + +#[then("both loaded models are still displayed")] +async fn external_models_still_visible(world: &mut E2eWorld) { + external_models_visible(world).await; +} + #[then("the overlay reports the record that is no longer running")] async fn services_overlay_reports_past_attempts(world: &mut E2eWorld) { // The overlay renders only the live instances the daemon scrapes, so the @@ -1646,8 +2214,8 @@ async fn home_view_displayed(world: &mut E2eWorld) { #[then("the hero reports no GPU telemetry available")] async fn hero_reports_no_gpu_telemetry(world: &mut E2eWorld) { let tui = session(world); - // The VRAM/TEMP/POWER mini-sparks only draw in the hero's wide layout - // (`draw_hero_band`'s `hero.width >= 8 && hero.height >= 6` gate). + // The VRAM/TEMP/POWER/T/S rows sit low in the hero, and a short pty + // squeezes the hero until `hero_rows` drops its bottom rows; enlarge first. tui.use_wide_size() .unwrap_or_else(|e| panic!("failed to enlarge the dashboard: {e}")); @@ -1700,6 +2268,38 @@ async fn hero_reports_no_gpu_telemetry(world: &mut E2eWorld) { .await; } +#[then("the hero reports no generation throughput")] +async fn hero_reports_no_generation_throughput(world: &mut E2eWorld) { + let tui = session(world); + // Self-contained rather than relying on a preceding step: cucumber matches + // step text globally, so this has to hold wherever it is written. The T/S + // row sits low in the hero and a short pty drops it, so enlarge and wait + // for the redraw before reading the screen. + tui.use_wide_size() + .unwrap_or_else(|e| panic!("failed to enlarge the dashboard: {e}")); + tui.wait_for_screen_where( + "the hero's T/S row", + |screen| screen.contains("T/S"), + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the hero's T/S row did not render: {e}")); + + let screen = tui.screen_text(); + let row = screen + .lines() + .find(|line| line.contains("T/S")) + .unwrap_or_else(|| panic!("no T/S row on screen:\n{screen}")); + let value = row + .split_once("T/S") + .and_then(|(_, rest)| rest.split('│').next()) + .map_or_else(|| panic!("no text after T/S on row:\n{row}"), str::trim); + assert_eq!( + value, "—", + "T/S should show the \"—\" placeholder when nothing is served:\n{row}" + ); +} + #[then("the dashboard reports live GPU telemetry")] async fn gpu_telemetry_displayed(world: &mut E2eWorld) { let tui = session(world); diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index a95ddf04c6..6e8d5aebfa 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -7,6 +7,11 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::lifecycle_steps; use crate::e2e::tui_driver::{TuiSession, default_timeout}; +// The note the text report prints under a cause the paste did not contribute +// to (diagnose-63/-64) is read from the library's own constant rather than +// typed again here, so the real binary's output is held to the one string the +// renderer prints from. +use rocm_core::diagnose::SYMPTOM_DID_NOT_CONTRIBUTE_NOTE; /// A symptom string that scores a catalog match on both Linux and Windows. It /// keys off `check_1_arch_not_in_wheel` (a `LINUX_AND_WINDOWS` checker), which @@ -331,6 +336,16 @@ async fn user_chose_path_fix(world: &mut E2eWorld) { world.model_name = Some("fix-6-path".to_string()); } +#[given("a user who has chosen the fix for a missing render group")] +async fn user_chose_render_group_fix(world: &mut E2eWorld) { + world.model_name = Some(COMMAND_FAILURE_FIX_ID.to_string()); +} + +#[given("a user who has chosen the fix for a framework wheel built for another ROCm release")] +async fn user_chose_wheel_mismatch_fix(world: &mut E2eWorld) { + world.model_name = Some("fix-8-wheel-rocm".to_string()); +} + #[then( "the printed commands and verify line avoid the constructs known to break in Windows PowerShell" )] @@ -451,6 +466,45 @@ async fn assert_notes_name_the_runtime_only_case(world: &mut E2eWorld) { ); } +#[then( + "the printed verify line checks the user belongs to the group that owns the GPU compute device, and shows that group" +)] +async fn assert_render_group_verify_shows_the_kfd_owner(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no fix preview output"); + let verify = printed_verify_line(output); + // The cause tells the user to join the group that owns `/dev/kfd`, which + // need not be `render`, so the check has to accept that group and show + // it. Text only: the line needs a Linux host to run, and + // `the_render_group_verify_accepts_the_group_that_owns_kfd` in + // crates/rocm-core/src/diagnose/linux.rs runs it with stand-in tools. That + // `rocm diagnose` prints this same line is held by + // `the_render_group_fix_verify_matches_the_catalog_copy` there; both read + // one constant. + assert!( + verify.starts_with("groups ") + && verify.contains("stat -c %G /dev/kfd") + && verify.contains("ls -l /dev/kfd"), + "the verify line must check the group that owns /dev/kfd and list it: {verify}" + ); +} + +#[then("the printed title names the X.Y release, not the major version")] +async fn assert_wheel_mismatch_title_names_the_release(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no fix preview output"); + let title = output + .lines() + .find_map(|line| line.strip_prefix("Fix:")) + .unwrap_or_else(|| panic!("the preview printed no Fix: line:\n{output}")) + .trim(); + // Text only. The comparison the title describes, `X.Y` and not the bare + // major, is held by `the_wheel_mismatch_compares_the_x_y_release_and_says_so` + // in crates/rocm-core/src/diagnose/host.rs. + assert!( + title.contains("X.Y") && !title.contains("major"), + "the title must name the X.Y release it compares: {title}" + ); +} + /// Every `Verify` line the preview printed: the plain one and the Linux one. fn printed_verify_lines(output: &str) -> Vec<&str> { output @@ -2797,6 +2851,90 @@ async fn assert_exit_codes_published(world: &mut E2eWorld) { "the published exit codes are missing `{name}`:\n{catalog:#}" ); } + // A name without a meaning leaves the caller guessing just the same, so + // every published code needs a description that says something. + for name in codes.keys() { + let description = catalog["exit_code_descriptions"][name] + .as_str() + .unwrap_or_default(); + assert!( + !description.trim().is_empty(), + "exit code `{name}` is published without a description:\n{catalog:#}" + ); + } +} + +/// The catalog `rocm fix --json` printed earlier in the scenario. The steps +/// that run the CLI again after it keep their output local, so it stays here. +fn published_catalog(world: &E2eWorld) -> serde_json::Value { + let output = world.cli_output.as_ref().expect("no catalog was read"); + serde_json::from_str(output).expect("catalog parses") +} + +/// The code the catalog publishes for `name`, and its description. +fn published_exit_code(catalog: &serde_json::Value, name: &str) -> (i32, String) { + let code = catalog["exit_codes"][name] + .as_i64() + .unwrap_or_else(|| panic!("no exit code published for `{name}`:\n{catalog:#}")); + let description = catalog["exit_code_descriptions"][name] + .as_str() + .unwrap_or_else(|| panic!("no description published for `{name}`:\n{catalog:#}")) + .to_owned(); + ( + i32::try_from(code).expect("an exit code fits i32"), + description, + ) +} + +/// Whether some line of `stderr` starts with `prefix`. The description promises +/// a line, not the start of stderr: a one-time `rocm:` notice printed at startup +/// can come before either refusal. +fn has_line_starting_with(stderr: &str, prefix: &str) -> bool { + stderr.lines().any(|line| line.starts_with(prefix)) +} + +#[then("asking for a fix it does not offer exits with the code the catalog describes for that")] +async fn assert_unknown_fix_exits_as_published(world: &mut E2eWorld) { + let catalog = published_catalog(world); + let (code, description) = published_exit_code(&catalog, "unknown_id"); + let (_stdout, stderr, rc) = crate::run_rocm(world, &["fix", "fix-does-not-exist"]); + assert_eq!( + rc, code, + "an unknown fix-id must exit with the code the catalog publishes for it; stderr:\n{stderr}" + ); + // The description names the message that sets this refusal apart; the + // refusal has to actually print it, or the advice sends a caller looking + // for a line that never comes. + let marker = "Unknown fix-id:"; + assert!( + description.contains(marker) && has_line_starting_with(&stderr, marker), + "the description must name `{marker}`, and a stderr line must start with it.\n\ + description: {description}\nstderr:\n{stderr}" + ); +} + +#[then( + "a malformed command line exits with that same code, as the catalog warns, and is told apart by its message" +)] +async fn assert_usage_error_shares_the_unknown_id_code(world: &mut E2eWorld) { + let catalog = published_catalog(world); + let (code, description) = published_exit_code(&catalog, "unknown_id"); + assert!( + description.contains("malformed command line") && description.contains("`error:`"), + "the catalog must warn that a usage error exits {code} too, and say how to spot it:\n\ + {description}" + ); + let (_stdout, stderr, rc) = crate::run_rocm(world, &["fix", "--no-such-option"]); + assert_eq!( + rc, code, + "a malformed `rocm fix` command line must exit {code}, as the catalog warns; stderr:\n{stderr}" + ); + // A line for the `error:`, the whole of stderr for the absence: the + // unknown-id message must appear nowhere in a usage error. + assert!( + has_line_starting_with(&stderr, "error:") && !stderr.contains("Unknown fix-id:"), + "a usage error must be distinguishable by its message, as the catalog says:\n{stderr}" + ); } #[then("the CLI refuses and explains that the two cannot be combined")] @@ -2995,6 +3133,53 @@ fn diagnose_verify_line(output: &str, fix_id: &str) -> String { .to_owned() } +#[when("the user diagnoses this machine with a ROCm runtime library load failure")] +async fn user_diagnoses_runtime_library_load_failure(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "diagnose", + "--symptom", + "libamdhip64.so.6: cannot open shared object file", + "--top", + "20", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the wheel mismatch cause and its plan name the X.Y release, not the major version")] +async fn assert_wheel_mismatch_cause_names_the_release(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no diagnose output"); + // The symptom alone raises this cause on every OS it is scoped to, whatever + // the host's own ROCm, so every lane sees the block. Text only: the + // comparison it describes is held by + // `the_wheel_mismatch_compares_the_x_y_release_and_says_so` in + // crates/rocm-core/src/diagnose/host.rs. + let block = cause_block(output, "fix-8-wheel-rocm"); + assert!( + !block.is_empty(), + "no `fix-8-wheel-rocm` cause was reported:\n{output}" + ); + // Only the title and the plan: they are what describe the comparison, and + // another line of the block may say "major" about something else. + let header = block[0]; + assert!( + header.contains("X.Y release") && !header.contains("major"), + "the cause's title must name the X.Y release: {header}" + ); + let plan = block + .iter() + .find_map(|line| line.trim().strip_prefix("plan:")) + .unwrap_or_else(|| panic!("the cause printed no plan:\n{}", block.join("\n"))); + assert!( + plan.contains("X.Y release") && !plan.contains("major"), + "the plan must name the X.Y release: {plan}" + ); +} + #[given("a Linux machine whose shell has a GPU override set")] async fn linux_machine_with_a_gpu_override(world: &mut E2eWorld) { // `rocm diagnose` reads the override from the environment it examines, @@ -3331,3 +3516,959 @@ async fn assert_hip_sdk_advice_follows_the_inspection(world: &mut E2eWorld) { field("hipinfo_present"), ); } + +const TINY_GGUF_LAYERS: u64 = 2; +const TINY_GGUF_KV_HEADS: u64 = 1; +const TINY_GGUF_HEAD_WIDTH: u64 = 64; +const TINY_GGUF_WEIGHT_BYTES: u64 = 1024 * 1024; +const TINY_GGUF_REQUESTED_CONTEXT: u64 = 2048; + +/// A minimal, valid GGUF v3 file: the metadata an estimate reads, one tensor +/// record, and a zeroed weight region of [`TINY_GGUF_WEIGHT_BYTES`]. +fn tiny_gguf_bytes() -> Vec { + fn string(out: &mut Vec, value: &str) { + out.extend_from_slice(&(value.len() as u64).to_le_bytes()); + out.extend_from_slice(value.as_bytes()); + } + fn entry_u32(out: &mut Vec, key: &str, value: u32) { + string(out, key); + out.extend_from_slice(&4_u32.to_le_bytes()); + out.extend_from_slice(&value.to_le_bytes()); + } + let mut metadata = Vec::new(); + string(&mut metadata, "general.architecture"); + metadata.extend_from_slice(&8_u32.to_le_bytes()); + string(&mut metadata, "e2e"); + entry_u32(&mut metadata, "e2e.block_count", TINY_GGUF_LAYERS as u32); + entry_u32(&mut metadata, "e2e.embedding_length", 128); + entry_u32(&mut metadata, "e2e.feed_forward_length", 256); + entry_u32(&mut metadata, "e2e.attention.head_count", 2); + entry_u32( + &mut metadata, + "e2e.attention.head_count_kv", + TINY_GGUF_KV_HEADS as u32, + ); + entry_u32(&mut metadata, "e2e.context_length", 8192); + + let mut tensor = Vec::new(); + string(&mut tensor, "w"); + tensor.extend_from_slice(&1_u32.to_le_bytes()); + tensor.extend_from_slice(&1024_u64.to_le_bytes()); + tensor.extend_from_slice(&0_u32.to_le_bytes()); + tensor.extend_from_slice(&0_u64.to_le_bytes()); + + let mut file = Vec::new(); + file.extend_from_slice(b"GGUF"); + file.extend_from_slice(&3_u32.to_le_bytes()); + file.extend_from_slice(&1_u64.to_le_bytes()); + file.extend_from_slice(&7_u64.to_le_bytes()); + file.extend_from_slice(&metadata); + file.extend_from_slice(&tensor); + // Tensor data starts at the next multiple of the default 32-byte alignment. + file.resize(file.len().div_ceil(32) * 32, 0); + file.resize(file.len() + TINY_GGUF_WEIGHT_BYTES as usize, 0); + file +} + +#[given("a user asking about a small GGUF file on this machine")] +async fn user_asks_about_a_small_gguf(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_path_buf(); + let path = root.join("tiny-e2e-model.gguf"); + std::fs::write(&path, tiny_gguf_bytes()).expect("write the GGUF fixture"); + world.model_name = Some(path.to_string_lossy().into_owned()); +} + +#[when( + "the user asks the CLI whether that file would run at 2048 tokens with a q8_0 KV cache, in machine-readable form" +)] +async fn user_asks_whether_a_gguf_would_run_json(world: &mut E2eWorld) { + let model = world.model_name.clone().expect("no model named"); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "diagnose", + "--model", + &model, + "--context-size", + "2048", + "--kv-cache-type", + "q8_0", + "--json", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when( + "the user asks the CLI whether that file would run at 2048 tokens with a q8_0 KV cache, in human-readable form" +)] +async fn user_asks_whether_a_gguf_would_run_text(world: &mut E2eWorld) { + let model = world.model_name.clone().expect("no model named"); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "diagnose", + "--model", + &model, + "--context-size", + "2048", + "--kv-cache-type", + "q8_0", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn estimate_section(model: &serde_json::Value) -> &serde_json::Value { + model + .get("estimate") + .filter(|value| value.is_object()) + .unwrap_or_else(|| panic!("the model section carries no estimate: {model:#}")) +} + +#[then("the answer is an estimate under its own outcome, and the verdict stays undetermined")] +async fn assert_gguf_answer_is_an_estimate(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "diagnose is a query and exits 0"); + let model = model_section(world); + assert_eq!( + model_verdict(&model), + "undetermined", + "an estimate from a header is not a recipe's verdict: {model:#}" + ); + assert_eq!( + model_field(&model, "undetermined_reason").as_str(), + Some("model_not_curated"), + "{model:#}" + ); + let estimate = estimate_section(&model); + assert_eq!(estimate["context_size"], TINY_GGUF_REQUESTED_CONTEXT); + assert_eq!(estimate["context_size_defaulted"], false); + assert_eq!(estimate["kv_cache_type"], "q8_0"); + assert_eq!(estimate["architecture"], "e2e"); + assert_eq!(estimate["layers"], TINY_GGUF_LAYERS); +} + +#[then("the estimate sizes the file's weights and its KV cache at that context")] +async fn assert_gguf_estimate_sizes_weights_and_cache(world: &mut E2eWorld) { + let model = model_section(world); + let estimate = estimate_section(&model); + let gib = f64::from(1024 * 1024 * 1024_u32); + let weights = estimate["weights_gib"] + .as_f64() + .unwrap_or_else(|| panic!("no weights figure: {estimate:#}")); + assert!( + (weights - TINY_GGUF_WEIGHT_BYTES as f64 / gib).abs() < 1e-9, + "the weights are the file's size after its header: {estimate:#}" + ); + // layers x kv heads x (key + value width) x tokens x 34/32 bytes (q8_0). + let kv_bytes = (TINY_GGUF_LAYERS * TINY_GGUF_KV_HEADS * 2 * TINY_GGUF_HEAD_WIDTH) as f64 + * TINY_GGUF_REQUESTED_CONTEXT as f64 + * 34.0 + / 32.0; + let kv = estimate["kv_cache_gib"] + .as_f64() + .unwrap_or_else(|| panic!("no KV cache figure: {estimate:#}")); + assert!( + (kv - kv_bytes / gib).abs() < 1e-9, + "expected {} GiB of KV cache: {estimate:#}", + kv_bytes / gib + ); +} + +#[then( + "a machine that measured its GPU is told whether the estimate fits, and one that could not is told why" +)] +async fn assert_gguf_estimate_compares_only_a_measured_gpu(world: &mut E2eWorld) { + let model = model_section(world); + let estimate = estimate_section(&model); + let outcome = estimate["outcome"].as_str().unwrap_or_default(); + if estimate["available_gib"].is_number() { + // The outcome the reported figures imply: the verdict is decided on the + // parts as shown (each rounded up to a whole MiB) against the card's + // memory, so the same arithmetic over the JSON says which answer is + // owed on this lane, and a predicate turned the wrong way fails here. + const MIB: f64 = 1024.0 * 1024.0; + let shown_mib: f64 = [ + "weights_bytes", + "kv_cache_bytes", + "compute_buffer_bytes", + "runtime_reserve_bytes", + ] + .iter() + .map(|field| { + let bytes = estimate[field] + .as_u64() + .unwrap_or_else(|| panic!("no {field}: {estimate:#}")); + (bytes as f64 / MIB).ceil() + }) + .sum(); + let available_mib = estimate["available_gib"].as_f64().unwrap_or_default() * 1024.0; + let expected = if shown_mib <= available_mib { + "fits" + } else { + "does_not_fit" + }; + assert_eq!( + outcome, expected, + "{shown_mib} MiB needed against {available_mib} MiB: {estimate:#}" + ); + if outcome == "fits" { + assert!( + estimate["max_context"].as_u64().unwrap_or(0) >= TINY_GGUF_REQUESTED_CONTEXT, + "fits at 2048 tokens but reports a smaller ceiling: {estimate:#}" + ); + } + } else { + assert_eq!(outcome, "cannot_estimate", "{estimate:#}"); + let reason = estimate["cannot_estimate_reason"] + .as_str() + .unwrap_or_default(); + assert!( + matches!( + reason, + "no_gpu" | "accelerator_memory_unknown" | "unified_memory_unreadable" + ), + "no memory was measured, so the reason must say so: {estimate:#}" + ); + assert!( + estimate["total_gib"].is_number(), + "what the file needs is still worth reporting: {estimate:#}" + ); + } +} + +#[then( + "the report is labelled an estimate, or undetermined with the figures, and is never ready or blocked" +)] +async fn assert_gguf_text_report_is_labelled(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "diagnose is a query and exits 0"); + let output = world.cli_output.clone().expect("no diagnose output"); + let model = world.model_name.clone().expect("no model named"); + let header = output + .lines() + .find(|line| line.starts_with(&format!("rocm diagnose --model {model}: "))) + .unwrap_or_else(|| panic!("no model header in:\n{output}")); + for verdict in ["READY", "BLOCKED", "DEGRADED"] { + assert!( + !header.ends_with(verdict) && !header.contains(&format!(": {verdict}")), + "an estimate was reported with the verdict word {verdict}: {header}" + ); + } + // Which label is owed depends on whether this host's memory was compared, + // and the report says which: a compared estimate names the card it was + // compared with. A compared estimate must be labelled ESTIMATED with the + // outcome its own figures imply, and one that was not, UNDETERMINED -- so + // the label is checked on whichever lane runs this, not only accepted. + // A GPU-less hosted runner takes the second branch; a lane that measures + // dedicated VRAM (the MI300X self-hosted lane) takes the first. + let compared = output.contains("the GPU with the most dedicated VRAM that ROCm can see has"); + if compared { + // The label owed follows from the two figures the report prints: the + // total it needs ("= about N MiB") and the card's ("has M MiB"). + let number_after = |marker: &str| -> u64 { + let at = output + .find(marker) + .unwrap_or_else(|| panic!("no `{marker}` in:\n{output}")) + + marker.len(); + output[at..] + .split(|c: char| !c.is_ascii_digit()) + .next() + .and_then(|digits| digits.parse().ok()) + .unwrap_or_else(|| panic!("no figure after `{marker}` in:\n{output}")) + }; + let needed = number_after("= about "); + let offered = number_after("can see has "); + let label = if needed <= offered { + ": ESTIMATED: fits at 2048 tokens" + } else { + ": ESTIMATED: does not fit at 2048 tokens" + }; + assert!( + header.contains(label), + "{needed} MiB needed against {offered} MiB, so `{label}` is owed: {header}" + ); + } else { + assert!( + header.ends_with(": UNDETERMINED"), + "nothing was compared, so the answer is undetermined: {header}" + ); + } + assert!( + output.contains("KV cache (q8_0)"), + "the figures the estimate rests on were not shown:\n{output}" + ); +} + +/// Start a stand-in Hugging Face endpoint on a loopback port and point the CLI at +/// it. It counts every connection. A repository listing request +/// (`/api/models/...`) is answered with `listing` when one is given; every other +/// request, and a listing when none is given, gets an empty 404. It stops when +/// the World lets go of its handle, so a scenario leaves no thread behind. +fn start_stand_in_hub(world: &mut E2eWorld, listing: Option<&'static str>) { + use std::io::{Read as _, Write as _}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind the stand-in hub"); + listener + .set_nonblocking(true) + .expect("make the stand-in hub non-blocking"); + let port = listener.local_addr().expect("hub address").port(); + let requests = Arc::new(AtomicUsize::new(0)); + let seen = Arc::clone(&requests); + std::thread::spawn(move || { + while Arc::strong_count(&seen) > 1 { + match listener.accept() { + Ok((mut stream, _)) => { + seen.fetch_add(1, Ordering::SeqCst); + let _ = stream.set_nonblocking(false); + let _ = stream.set_read_timeout(Some(std::time::Duration::from_secs(5))); + // The request line is all that is needed, and it arrives first. + let mut buf = [0_u8; 2048]; + let read = stream.read(&mut buf).unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..read]); + let is_listing = request + .split_whitespace() + .nth(1) + .is_some_and(|path| path.starts_with("/api/models/")); + let response = match listing { + Some(body) if is_listing => format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ), + _ => "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_owned(), + }; + let _ = stream.write_all(response.as_bytes()); + } + Err(_) => std::thread::sleep(std::time::Duration::from_millis(10)), + } + } + }); + world.hub_requests = Some(requests); + world + .command_env + .push(("HF_ENDPOINT", format!("http://127.0.0.1:{port}").into())); +} + +#[given("a Hugging Face endpoint that records every request made to it")] +async fn a_hub_endpoint_that_counts_requests(world: &mut E2eWorld) { + start_stand_in_hub(world, None); +} + +/// A repository holding one model and its multimodal projector, as llama.cpp +/// vision models ship. Sizes are given so no `HEAD` request is needed. +const MODEL_AND_PROJECTOR_LISTING: &str = r#"{"siblings":[{"rfilename":"e2e-model-Q4_K_M.gguf","size":1048576},{"rfilename":"mmproj-e2e-model-f16.gguf","size":1048576}]}"#; + +#[given("a Hugging Face repository that holds a model and its multimodal projector")] +async fn a_hub_repository_with_a_model_and_a_projector(world: &mut E2eWorld) { + start_stand_in_hub(world, Some(MODEL_AND_PROJECTOR_LISTING)); +} + +#[when("the user asks the CLI whether the projector file would run, naming it exactly")] +async fn user_asks_about_the_projector_file(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "diagnose", + "--model", + "e2e-owner/e2e-repo:mmproj-e2e-model-f16.gguf", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the CLI says the file is a multimodal projector and names the model file instead")] +async fn assert_the_cli_named_the_projector_and_the_model(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{:?}", world.cli_stderr); + let output = world.cli_output.clone().unwrap_or_default(); + assert!( + output.contains( + "`mmproj-e2e-model-f16.gguf` is a multimodal projector (mmproj), not a language model" + ), + "{output}" + ); + assert!(output.contains("e2e-model-Q4_K_M.gguf"), "{output}"); + assert!(!output.contains("no GGUF file matching"), "{output}"); + // Nothing points the user back at the file it refused. + assert!( + !output.contains("rocm serve e2e-owner/e2e-repo:mmproj"), + "the refused projector was offered as a serve command:\n{output}" + ); + // The one model in the repository is offered by its exact name. + assert!( + output.contains( + "verify after fix: rocm diagnose --model e2e-owner/e2e-repo:e2e-model-Q4_K_M.gguf" + ), + "{output}" + ); + assert!(!output.contains("header could not be read"), "{output}"); +} + +#[then("the endpoint was asked only for the repository's file list")] +async fn assert_only_the_listing_was_requested(world: &mut E2eWorld) { + let seen = world + .hub_requests + .as_ref() + .expect("no stand-in hub") + .load(std::sync::atomic::Ordering::SeqCst); + // One listing request; the projector was refused before any header read. + assert_eq!(seen, 1, "the hub was asked {seen} time(s)"); +} + +const HUB_MODEL_REF: &str = "e2e-owner/e2e-repo:Q4_K_M"; + +async fn run_hub_model_question(world: &mut E2eWorld, report: bool) { + // The environment the Given recorded is consumed by one run; keep it for the next. + let env = world.command_env.clone(); + let mut args = vec!["diagnose", "--model", HUB_MODEL_REF, "--json"]; + if report { + args.insert(1, "--report"); + } + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &args); + world.command_env = env; + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when("the user asks the CLI to prepare a problem report that names a model on that endpoint")] +async fn user_prepares_a_report_naming_a_hub_model(world: &mut E2eWorld) { + run_hub_model_question(world, true).await; +} + +#[when("the user asks the CLI the same model question without preparing a report")] +async fn user_asks_the_hub_model_question_without_a_report(world: &mut E2eWorld) { + run_hub_model_question(world, false).await; +} + +#[then("no request reached the endpoint")] +async fn assert_no_request_reached_the_hub(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{:?}", world.cli_stderr); + // A moment for a request that was sent to be counted before the check. + std::thread::sleep(std::time::Duration::from_millis(300)); + let seen = world + .hub_requests + .as_ref() + .expect("no stand-in hub") + .load(std::sync::atomic::Ordering::SeqCst); + assert_eq!(seen, 0, "a prepared report reached the hub {seen} time(s)"); +} + +#[then("the CLI says the model was ignored rather than assessed")] +async fn assert_the_report_said_it_ignored_the_model(world: &mut E2eWorld) { + let stderr = world.cli_stderr.clone().unwrap_or_default(); + assert!( + stderr.contains("--model was ignored") && stderr.contains("nothing about the model"), + "stderr was: {stderr}" + ); +} + +#[then("the CLI does not say the model was ignored")] +async fn assert_the_cli_did_not_say_it_ignored_the_model(world: &mut E2eWorld) { + let stderr = world.cli_stderr.clone().unwrap_or_default(); + assert!(!stderr.contains("was ignored"), "stderr was: {stderr}"); +} + +#[then("the endpoint was asked about the model")] +async fn assert_the_hub_was_asked_about_the_model(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{:?}", world.cli_stderr); + let seen = world + .hub_requests + .as_ref() + .expect("no stand-in hub") + .load(std::sync::atomic::Ordering::SeqCst); + assert!( + seen > 0, + "the same question without --report never reached the endpoint, so the check above \ + proved nothing" + ); +} + +fn run_model_question(world: &mut E2eWorld, args: &[&str]) { + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, args); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn evidence_text(model: &serde_json::Value) -> String { + model_field(model, "evidence") + .as_array() + .map(|lines| { + lines + .iter() + .filter_map(serde_json::Value::as_str) + .collect::>() + .join("\n") + }) + .unwrap_or_default() +} + +#[when("the user asks the CLI for a context size or a KV cache type without naming a model")] +async fn user_asks_for_context_options_without_a_model(world: &mut E2eWorld) { + let mut outcomes = Vec::new(); + for option in [["--context-size", "8192"], ["--kv-cache-type", "q8_0"]] { + let (_, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["diagnose", option[0], option[1]]); + outcomes.push(format!("{}\t{rc}\t{stderr}", option[0])); + } + world.cli_output = Some(outcomes.join("\n\u{0}\n")); +} + +#[then("the CLI refuses each as a usage error that names --model")] +async fn assert_context_options_refused_without_a_model(world: &mut E2eWorld) { + let output = world.cli_output.clone().unwrap_or_default(); + let outcomes: Vec<&str> = output.split("\n\u{0}\n").collect(); + assert_eq!(outcomes.len(), 2, "{output}"); + for outcome in outcomes { + let mut parts = outcome.splitn(3, '\t'); + let (option, rc, stderr) = ( + parts.next().unwrap_or(""), + parts.next().unwrap_or(""), + parts.next().unwrap_or(""), + ); + // clap's usage-error exit code. + assert_eq!( + rc, "2", + "{option} was not refused as a usage error: {stderr}" + ); + assert!( + stderr.contains("--model"), + "{option}: the refusal does not name what is missing: {stderr}" + ); + } +} + +#[when( + "the user asks the CLI whether the smallest curated model would run at 8192 tokens, in machine-readable form" +)] +async fn user_asks_about_a_curated_model_with_a_context_size(world: &mut E2eWorld) { + run_model_question( + world, + &[ + "diagnose", + "--model", + SMALLEST_MODEL_REF, + "--context-size", + "8192", + "--json", + ], + ); +} + +#[then( + "the answer says the context options do not apply to a curated recipe, and carries no estimate" +)] +async fn assert_curated_model_says_context_options_do_not_apply(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{:?}", world.cli_stderr); + let model = model_section(world); + let evidence = evidence_text(&model); + assert!( + evidence.contains("do not apply to a curated recipe"), + "the option was accepted and nothing said it was not applied: {model:#}" + ); + assert!( + model.get("estimate").is_none_or(serde_json::Value::is_null), + "a curated model was estimated from a header it has none of: {model:#}" + ); +} + +fn missing_gguf_path(world: &E2eWorld) -> String { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("no-such-model.gguf") + .to_string_lossy() + .into_owned() +} + +#[when( + "the user asks the CLI whether a GGUF file that does not exist would run, in machine-readable form" +)] +async fn user_asks_about_a_missing_gguf(world: &mut E2eWorld) { + let path = missing_gguf_path(world); + assert!( + !std::path::Path::new(&path).exists(), + "{path} must not exist" + ); + run_model_question(world, &["diagnose", "--model", &path, "--json"]); +} + +#[then("the answer says the file could not be opened and offers no upstream route")] +async fn assert_missing_gguf_could_not_be_opened(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{:?}", world.cli_stderr); + let model = model_section(world); + let path = missing_gguf_path(world); + let evidence = evidence_text(&model); + assert!( + evidence.contains(&format!("could not open {path}")), + "the answer does not say which file could not be opened: {model:#}" + ); + assert_eq!( + estimate_section(&model)["cannot_estimate_reason"], + "header_unreadable", + "{model:#}" + ); + assert!( + model.get("route").is_none_or(serde_json::Value::is_null), + "a path that does not exist was routed to the upstream tracker: {model:#}" + ); +} + +#[then("the plan is to ask again about that path")] +async fn assert_missing_gguf_plan_is_to_check_the_path(world: &mut E2eWorld) { + let model = model_section(world); + let path = missing_gguf_path(world); + let fix = model_field(&model, "fix"); + // The command names the path as given, quoted when the platform's shell + // would split or expand it (the scenario's temp directory decides which). + let verify = fix["verify"].as_str().unwrap_or(""); + let bare = format!("rocm diagnose --model {path}"); + let quoted = format!("rocm diagnose --model '{path}'"); + // Either form asks again with `rocm diagnose`; serving a file that is not + // there would only repeat the error. + assert!(verify == bare || verify == quoted, "{model:#}"); +} + +#[when( + "the user asks the CLI about a repository with no quantization at 8192 tokens, in machine-readable form" +)] +async fn user_asks_about_a_bare_repository_with_a_context_size(world: &mut E2eWorld) { + run_model_question( + world, + &[ + "diagnose", + "--model", + UNCURATED_MODEL_REF, + "--context-size", + "8192", + "--json", + ], + ); +} + +#[then("the answer says the reference names a repository and asks for a quantization")] +async fn assert_bare_repository_asks_for_a_quantization(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{:?}", world.cli_stderr); + let model = model_section(world); + let evidence = evidence_text(&model); + assert!( + evidence.contains(&format!( + "`{UNCURATED_MODEL_REF}` names a repository, not a model file: add `:QUANT`" + )), + "{model:#}" + ); + assert!( + model.get("estimate").is_none_or(serde_json::Value::is_null), + "a repository was estimated as if it were one file: {model:#}" + ); +} + +#[when("the user asks the CLI about a quantization the repository does not hold")] +async fn user_asks_about_a_missing_quantization(world: &mut E2eWorld) { + run_model_question(world, &["diagnose", "--model", "e2e-owner/e2e-repo:Q8_0"]); +} + +#[then("the CLI says nothing matches and lists the model and the projector, marked as a projector")] +async fn assert_missing_quantization_lists_every_gguf(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "{:?}", world.cli_stderr); + let output = world.cli_output.clone().unwrap_or_default(); + assert!( + output.contains( + "no GGUF file matching `Q8_0` in the repository (it has: e2e-model-Q4_K_M.gguf, \ + mmproj-e2e-model-f16.gguf (multimodal projector))" + ), + "{output}" + ); +} + +// ── The CLI version a problem report names ───────────────────────── + +#[given("a simulated Linux machine whose only GPU is a released one")] +async fn simulated_machine_with_a_released_gpu(world: &mut E2eWorld) { + // Planted as in diagnose-40, but with gfx_target_version 110000 (gfx1100), + // which is on the compatibility matrix, so `--report` prepares a report on + // every Linux lane instead of refusing on the runner's own hardware. + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host = root.join("host-root"); + std::fs::create_dir_all(host.join("etc")).expect("failed to create planted etc"); + std::fs::write( + host.join("etc/os-release"), + "ID=ubuntu\nVERSION_ID=\"24.04\"\nPRETTY_NAME=\"Ubuntu 24.04 LTS\"\n", + ) + .expect("failed to write planted os-release"); + let node = host.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); + std::fs::write( + node.join("properties"), + "cpu_cores_count 0\nsimd_count 96\ngfx_target_version 110000\n", + ) + .expect("failed to write planted KFD node properties"); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host.into_os_string())); + // An empty PCI bus, so a runner's real card cannot join the planted one. + let bin = root.join("released-gpu-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); + rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") + .expect("failed to write the fake lspci"); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.push(("PATH", path)); +} + +#[when("the user asks the CLI what that machine's report would carry in machine-readable form")] +async fn user_asks_for_the_simulated_machines_report(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["diagnose", "--report", "--json"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the report names the CLI version that `rocm version` leads with")] +async fn report_names_the_rocm_version_leading_version(world: &mut E2eWorld) { + let out = world.cli_output.clone().expect("no CLI output"); + let stderr = world.cli_stderr.clone().unwrap_or_default(); + assert_eq!(world.cli_rc, Some(0), "{out}\n{stderr}"); + let report: serde_json::Value = serde_json::from_str(&out).unwrap_or_else(|err| { + panic!("the planted released GPU did not produce a JSON report ({err}):\n{out}\n{stderr}") + }); + let reported = report["cli_version"] + .as_str() + .unwrap_or_else(|| panic!("the report has no cli_version:\n{report:#}")); + + let (version_out, version_err, version_rc) = crate::run_rocm(world, &["version"]); + assert_eq!(version_rc, 0, "{version_out}\n{version_err}"); + let first_line = version_out.lines().next().unwrap_or_default().trim(); + let (leading, reference) = first_line + .strip_prefix("rocm-cli ") + .and_then(|rest| rest.split_once(" (")) + .and_then(|(version, rest)| { + rest.split_once(", ") + .map(|(reference, _hash)| (version, reference)) + }) + .unwrap_or_else(|| panic!("expected 'rocm-cli (, )': {first_line}")); + assert_eq!( + reported, leading, + "the report's cli_version is not the version `rocm version` leads with \ + ({first_line}):\n{report:#}" + ); + // Agreement alone would also hold if both surfaces regressed to the same + // wrong value (the frozen workspace version, say), so the version is also + // checked against the one the ref calls for, worked out independently. + assert_ne!( + reference, "unknown", + "version ref did not resolve: {first_line}" + ); + assert_eq!( + reported, + super::examine_steps::expected_leading_version(reference), + "the report's cli_version does not follow from the ref ({first_line})" + ); +} + +// ── Which causes the pasted error contributed to (diagnose-62/-63/-64) ───── + +/// The entry an unneeded GPU override in the environment raises. It reads the +/// environment and a page-fault paste, never an ISA error. +const ENV_MISCONFIG_FIX_ID: &str = "fix-2-unset-override"; + +/// The entry [`KNOWN_SYMPTOM`] scores. No host state scores it on its own. +const PASTE_FIX_ID: &str = "fix-1-arch"; + +#[given("a machine with a known misconfiguration in its environment")] +async fn machine_with_env_misconfiguration(world: &mut E2eWorld) { + // `rocm diagnose` examines its own environment, so setting the override on + // the CLI child is the whole setup, on every platform the entry covers. + world + .command_env + .push(("HSA_OVERRIDE_GFX_VERSION", "11.0.0".into())); +} + +#[given("a pasted error that describes a different failure")] +async fn pasted_error_for_a_different_failure(world: &mut E2eWorld) { + world.model_name = Some(KNOWN_SYMPTOM.to_owned()); +} + +/// Run `rocm diagnose` with `paste` as the symptom, or with no `--symptom` at +/// all when it is `None`. +fn run_diagnose(world: &mut E2eWorld, paste: Option<&str>, json: bool) { + let mut args = vec!["diagnose", "--top", "50"]; + if let Some(paste) = paste { + args.extend(["--symptom", paste]); + } + if json { + args.push("--json"); + } + // The planted host holds for the whole scenario, and diagnose-64 asks + // twice, but running a command consumes the scenario environment. Put it + // back so the second run examines the same host as the first. + let host_env = world.command_env.clone(); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &args); + world.command_env = host_env; + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn diagnose_with_paste(world: &mut E2eWorld, json: bool) { + let paste = world.model_name.clone().expect("no paste set"); + run_diagnose(world, Some(&paste), json); +} + +#[when("the user asks for a diagnosis with that paste, in machine-readable form")] +async fn user_diagnoses_paste_json(world: &mut E2eWorld) { + diagnose_with_paste(world, true); +} + +#[when("the user asks for a diagnosis with that paste")] +async fn user_diagnoses_paste_text(world: &mut E2eWorld) { + diagnose_with_paste(world, false); +} + +#[when("the user asks for a diagnosis without pasting an error, in machine-readable form")] +async fn user_diagnoses_without_paste_json(world: &mut E2eWorld) { + run_diagnose(world, None, true); +} + +#[when("the user asks for a diagnosis without pasting an error")] +async fn user_diagnoses_without_paste_text(world: &mut E2eWorld) { + run_diagnose(world, None, false); +} + +/// The listed causes of the JSON report, each as `(id, symptom_matched)`, +/// failing the step when the run failed or any entry lacks a boolean field. +fn symptom_matched_by_id(world: &E2eWorld) -> Vec<(String, bool)> { + let stdout = world.cli_output.as_deref().expect("no diagnose output"); + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "diagnose failed:\n{stdout}\n{stderr}" + ); + let report: serde_json::Value = serde_json::from_str(stdout) + .unwrap_or_else(|e| panic!("diagnose --json is not JSON ({e}):\n{stdout}")); + let matched = report["matched"] + .as_array() + .unwrap_or_else(|| panic!("no `matched` list:\n{report:#}")); + matched + .iter() + .map(|d| { + let flag = d["symptom_matched"].as_bool().unwrap_or_else(|| { + panic!("every listed cause says whether the paste contributed to it:\n{d:#}") + }); + (d["id"].as_str().unwrap_or_default().to_owned(), flag) + }) + .collect() +} + +/// `symptom_matched` of the entry `id`, failing the step when it is not listed. +fn symptom_matched_of(world: &E2eWorld, id: &str) -> bool { + let listed = symptom_matched_by_id(world); + listed + .iter() + .find(|(listed_id, _)| listed_id == id) + .map_or_else( + || panic!("{id} was not listed: {listed:?}"), + |(_, flag)| *flag, + ) +} + +#[then("the paste is reported as not contributing to the misconfiguration on the machine")] +async fn assert_paste_did_not_contribute_to_env_misconfig(world: &mut E2eWorld) { + assert!( + !symptom_matched_of(world, ENV_MISCONFIG_FIX_ID), + "{ENV_MISCONFIG_FIX_ID} comes from the environment alone, so the paste must not \ + be credited with it" + ); +} + +#[then("the paste is reported as contributing to the cause it describes")] +async fn assert_paste_contributed_to_its_cause(world: &mut E2eWorld) { + assert!( + symptom_matched_of(world, PASTE_FIX_ID), + "{PASTE_FIX_ID} is scored from the paste, so the paste must be credited with it" + ); +} + +#[then("every listed cause is reported as not contributed to by a paste")] +async fn assert_no_cause_credited_without_a_paste(world: &mut E2eWorld) { + let listed = symptom_matched_by_id(world); + assert!( + listed.iter().any(|(id, _)| id == ENV_MISCONFIG_FIX_ID), + "the planted misconfiguration must be listed, or this asserts nothing: {listed:?}" + ); + assert!( + listed.iter().all(|(_, flag)| !flag), + "nothing was pasted, so no cause may be credited to a paste: {listed:?}" + ); +} + +#[then("the misconfiguration on the machine is marked as found from the host alone")] +async fn assert_env_misconfig_marked(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no diagnose output"); + assert_eq!(world.cli_rc, Some(0), "diagnose failed:\n{output}"); + let block = cause_block(output, ENV_MISCONFIG_FIX_ID); + assert!( + !block.is_empty(), + "{ENV_MISCONFIG_FIX_ID} was not reported:\n{output}" + ); + assert!( + block + .iter() + .any(|line| line.trim() == SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "the cause the paste did not contribute to must say so:\n{output}" + ); +} + +#[then("the cause the pasted error describes carries no such mark")] +async fn assert_paste_cause_unmarked(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no diagnose output"); + let block = cause_block(output, PASTE_FIX_ID); + assert!( + !block.is_empty(), + "{PASTE_FIX_ID} was not reported:\n{output}" + ); + assert!( + !block + .iter() + .any(|line| line.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE)), + "the cause the paste contributed to must not be marked as found without it:\n{output}" + ); +} + +#[then("no listed cause is marked as found from the host alone")] +async fn assert_no_cause_marked_without_a_paste(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no diagnose output"); + assert_eq!(world.cli_rc, Some(0), "diagnose failed:\n{output}"); + assert!( + !cause_block(output, ENV_MISCONFIG_FIX_ID).is_empty(), + "the planted misconfiguration must be reported, or this asserts nothing:\n{output}" + ); + assert!( + !output.contains(SYMPTOM_DID_NOT_CONTRIBUTE_NOTE), + "with nothing pasted the mark would sit on every cause and say nothing:\n{output}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/driver_steps.rs b/tests/e2e-cucumber/tests/e2e/driver_steps.rs index a1532cc998..cdb0d7debf 100644 --- a/tests/e2e-cucumber/tests/e2e/driver_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/driver_steps.rs @@ -41,10 +41,9 @@ async fn assert_driver_plan_matches_host_release(world: &mut E2eWorld) { // breaks the arm-selection logic pass by construction. The hosted // `ubuntu-latest` E2E image (26.04 once GitHub's migration completes) and the self-hosted GPU // lane (24.04) both take the apt arm, with their own release codename. - let os_release_text = - rocm_core::hardware_root::read_os_release(rocm_core::hardware_root::identity_path_for_str) - .text() - .to_owned(); + let os_release_text = rocm_core::hardware_root::real_os_release() + .text() + .to_owned(); let Ok(os_release) = rocm_core::os_release::parse(&os_release_text) else { panic!("host's /etc/os-release is unreadable:\n{output}"); }; diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 93f0999f2f..a986bc22f7 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -143,6 +143,44 @@ async fn user_previews_driver_install_plan(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// The leading version `rocm version` must print for a resolved `reference`, +/// derived here from the ref alone rather than read back from the binary, so a +/// build that reports the wrong version for its ref fails instead of agreeing +/// with itself. Mirrors `release_version` in `apps/rocm/build.rs`: an exact +/// `vX.Y.Z` tag, with an optional non-empty pre-release (`-rc.1`) or build +/// (`+meta`) suffix of ASCII letters, digits, `.`, `-` and `+`, and at most 64 +/// characters after the `v` (`RELEASE_TAG_PATTERN` and +/// `RELEASE_VERSION_MAX_LEN` there), is that version without its leading `v`; any other +/// resolved ref (`main`, a feature branch, `vllm`, `v0.2`) is `0.0.0-dev`. +/// The `unknown` ref is not covered: both callers reject it first. +pub(crate) fn expected_leading_version(reference: &str) -> &str { + const NON_RELEASE: &str = "0.0.0-dev"; + let Some(rest) = reference.strip_prefix('v') else { + return NON_RELEASE; + }; + if rest.len() > 64 { + return NON_RELEASE; + } + let core = match rest.split_once(['-', '+']) { + Some((core, suffix)) + if !suffix.is_empty() + && suffix + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'+')) => + { + core + } + Some(_) => return NON_RELEASE, + None => rest, + }; + let components: Vec<&str> = core.split('.').collect(); + let is_release = components.len() == 3 + && components + .iter() + .all(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit())); + if is_release { rest } else { NON_RELEASE } +} + #[then("matching traceable version strings are returned")] async fn assert_version_returned(world: &mut E2eWorld) { let outputs = world.cli_outputs.as_ref().expect("no commands were run"); @@ -185,6 +223,11 @@ async fn assert_version_returned(world: &mut E2eWorld) { !hash.is_empty() && hash.bytes().all(|byte| byte.is_ascii_hexdigit()), "version hash is not hexadecimal: {output}" ); + assert_eq!( + version, + expected_leading_version(reference), + "the leading version does not follow from the ref: {output}" + ); // `rocm version`'s own two lines beyond the build string. Both are printed // unconditionally (either the detected value or a "not detected"/"unmanaged" @@ -1444,6 +1487,27 @@ async fn os_release_with_a_command(world: &mut E2eWorld) { const RUNS_A_COMMAND_ON_LINE_3: &str = "ID=ubuntu\nVERSION_ID=\"24.04\"\nexport ID=debian\nPRETTY_NAME=\"Ubuntu 24.04 LTS\"\n"; +/// Writes each `(name, script)` as an executable in the isolated root's +/// `fake-bin` and puts that directory first on the scenario's `PATH`. +fn put_fake_executables_on_path(world: &mut E2eWorld, executables: &[(&str, &str)]) { + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("fake-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); + for (name, script) in executables { + rocm_core::fake_executable::write_fake_executable(&bin.join(name), script) + .unwrap_or_else(|e| panic!("failed to write the fake {name}: {e}")); + } + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.push(("PATH", path)); +} + /// One GPU node in the kernel topology: a gfx1101 with no PCI address, so it /// matches no `lspci` entry and is listed from the topology alone. Plus an /// `lspci` that lists an empty bus, ahead of any real one, so the PCI probe @@ -1465,15 +1529,7 @@ fn plant_gpu_fixtures(world: &mut E2eWorld) { "cpu_cores_count 0\nsimd_count 120\ngfx_target_version 110001\n", ) .expect("failed to write planted KFD node properties"); - let bin = root.join("fake-bin"); - std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); - rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") - .expect("failed to write the fake lspci"); - let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( - &std::env::var_os("PATH").unwrap_or_default(), - ))) - .expect("PATH entries cannot hold a separator"); - world.command_env.push(("PATH", path)); + put_fake_executables_on_path(world, &[("lspci", "#!/bin/sh\nexit 0\n")]); } #[given("a GPU machine whose release file runs a command on line 3")] @@ -1502,6 +1558,152 @@ async fn gpu_machine_with_unreadable_release_file(world: &mut E2eWorld) { plant_gpu_fixtures(world); } +/// A simulated host root, plus a `lsmod` on `PATH` that lists `amdgpu` and no +/// `amdkfd`, so the module list never answers for kfd. `with_kfd` plants the +/// `/dev/kfd` node. +fn plant_machine_without_amdkfd_module(world: &mut E2eWorld, with_kfd: bool) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host = root.join("host-root"); + std::fs::create_dir_all(host.join("dev")).expect("failed to create planted /dev"); + if with_kfd { + std::fs::write(host.join("dev/kfd"), "").expect("failed to plant /dev/kfd"); + } + put_fake_executables_on_path( + world, + &[( + "lsmod", + "#!/bin/sh\nprintf 'Module Size Used by\\namdgpu 1 0\\ndrm 1 1 amdgpu\\n'\n", + )], + ); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host.into_os_string())); +} + +#[given("a machine whose kernel exposes kfd but lists no amdkfd module")] +async fn machine_with_built_in_kfd(world: &mut E2eWorld) { + plant_machine_without_amdkfd_module(world, true); +} + +#[given("a machine whose kernel has no kfd and lists no amdkfd module")] +async fn machine_without_kfd(world: &mut E2eWorld) { + plant_machine_without_amdkfd_module(world, false); +} + +/// A simulated host root with `/dev/kfd`, no `/proc/modules`, and a `lsmod` +/// on `PATH` that fails: no module list can be read at all. +#[given("a machine whose kernel exposes kfd but whose module list cannot be read")] +async fn machine_with_kfd_and_no_module_list(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host = root.join("host-root"); + std::fs::create_dir_all(host.join("dev")).expect("failed to create planted /dev"); + std::fs::write(host.join("dev/kfd"), "").expect("failed to plant /dev/kfd"); + put_fake_executables_on_path(world, &[("lsmod", "#!/bin/sh\nexit 1\n")]); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host.into_os_string())); +} + +#[then("the inspection reports kfd as loaded")] +async fn assert_kfd_loaded(world: &mut E2eWorld) { + let json = parsed_json(world); + assert_eq!(json["amdkfd_loaded"], serde_json::json!(true), "{json:#}"); +} + +#[then("the inspection reports kfd as not loaded")] +async fn assert_kfd_not_loaded(world: &mut E2eWorld) { + let json = parsed_json(world); + assert_eq!(json["amdkfd_loaded"], serde_json::json!(false), "{json:#}"); +} + +/// A simulated host root with `/sys/bus/pci/drivers/amdgpu` and a `lsmod` that lists +/// neither `amdgpu` nor `amdkfd`: amdgpu built into the kernel. A ROCm install +/// (`ROCM_PATH`) and a `mokutil` reporting Secure Boot enabled are planted too, +/// so the three `rocm diagnose` evidence lines that read `amdgpu_loaded` have +/// everything else they need to fire. +#[given("a machine whose kernel has amdgpu built in and lists no amdgpu module")] +async fn machine_with_built_in_amdgpu(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host = root.join("host-root"); + std::fs::create_dir_all(host.join("sys/bus/pci/drivers/amdgpu")) + .expect("failed to plant /sys/bus/pci/drivers/amdgpu"); + world.plant_unmanaged_rocm(); + put_fake_executables_on_path( + world, + &[ + ( + "lsmod", + "#!/bin/sh\nprintf 'Module Size Used by\\ndrm 1 0\\n'\n", + ), + ("mokutil", "#!/bin/sh\necho 'SecureBoot enabled'\n"), + ], + ); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host.into_os_string())); +} + +/// Both commands read the same planted machine, so the scenario environment +/// (the host root and the fake executables on `PATH`) is restored between +/// them: `run_rocm_with_scenario_env` consumes it. +#[when("the user inspects and diagnoses that machine")] +async fn user_inspects_and_diagnoses_planted_machine(world: &mut E2eWorld) { + let env = world.command_env.clone(); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip", "--json"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); + world.command_env = env; + let (diagnosis, _, diagnose_rc) = + crate::run_rocm_with_scenario_env(world, &["diagnose", "--json"]); + assert_eq!(diagnose_rc, 0, "diagnose is a query and exits 0"); + world.cli_outputs = Some(vec![diagnosis]); +} + +#[then("the inspection reports amdgpu as loaded")] +async fn assert_amdgpu_loaded(world: &mut E2eWorld) { + let json = parsed_json(world); + assert_eq!(json["amdgpu_loaded"], serde_json::json!(true), "{json:#}"); +} + +#[then("the diagnosis cites no missing amdgpu module, DKMS failure or Secure Boot evidence")] +async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { + let diagnosis = &world.cli_outputs.as_ref().expect("no diagnosis was run")[0]; + let report: serde_json::Value = serde_json::from_str(diagnosis) + .unwrap_or_else(|e| panic!("diagnose --json is not JSON ({e}):\n{diagnosis}")); + let evidence: Vec<&str> = report["matched"] + .as_array() + .unwrap_or_else(|| panic!("diagnose --json has no matched array:\n{report:#}")) + .iter() + .flat_map(|m| m["evidence"].as_array().into_iter().flatten()) + .filter_map(serde_json::Value::as_str) + .collect(); + for line in evidence { + assert!( + !rocm_core::diagnose::AMDGPU_MISSING_EVIDENCE + .iter() + .any(|p| line.contains(p)), + "amdgpu was reported missing though the amdgpu PCI driver is registered: {line}" + ); + } +} + #[when("the user inspects that machine in machine-readable form")] async fn user_inspects_planted_machine(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -2369,6 +2571,556 @@ async fn assert_rocdxg_detail_names_the_cache(world: &mut E2eWorld) { ); } +// ── Both reports agree whether `rocminfo` is installed ───────────── + +/// Plant a stand-in `rocminfo` in `dir`. It prints nothing and succeeds, so the +/// probe finds the tool installed and, having seen no agent in its output, does +/// not claim the GPU works: the scenarios turn on whether it is found, not on +/// what it says. +fn plant_stand_in_rocminfo(dir: &std::path::Path) { + std::fs::create_dir_all(dir).expect("failed to create the stand-in rocminfo directory"); + rocm_core::fake_executable::write_fake_executable(&dir.join("rocminfo"), "#!/bin/sh\nexit 0\n") + .expect("failed to write the stand-in rocminfo"); +} + +/// Replace the `PATH` the planted WSL machine gives the next `rocm` run with +/// `entries`, then the machine's `fake-bin` (its stand-in `ldconfig`), then the +/// runner's own entries that hold no `rocminfo`, so a runner with ROCm installed +/// cannot be the reason the tool reads as found. How the runner's entries are +/// resolved and filtered is [`e2e_cucumber::planted_path::planted_path_entries`]. +/// +/// An entry that is an empty path is how a test writes the empty `PATH` entry +/// the shell reads as the current directory. +fn set_planted_wsl_path(world: &mut E2eWorld, entries: &[PathBuf]) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + // The runner's relative entries are resolved from this process's directory + // before they are checked, because the child may run from another one. + let path = std::env::join_paths(e2e_cucumber::planted_path::planted_path_entries( + entries, + &root.join("fake-bin"), + &std::env::var_os("PATH").unwrap_or_default(), + &std::env::current_dir().expect("the harness has a working directory"), + e2e_cucumber::planted_path::dir_holds_tool, + )) + .expect("PATH entries cannot hold a separator"); + world.command_env.retain(|(key, _)| *key != "PATH"); + world.command_env.push(("PATH", path)); +} + +#[given("a WSL machine whose rocminfo is in a PATH directory")] +async fn wsl_machine_with_rocminfo_on_path(world: &mut E2eWorld) { + plant_wsl_machine_with_unregistered_rocdxg(world, ""); + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let tools = root.join("rocminfo-bin"); + plant_stand_in_rocminfo(&tools); + set_planted_wsl_path(world, &[tools]); +} + +/// The machine of `examine-36` and `examine-38`: the only `rocminfo` is in +/// `work`, the directory `rocm` is started from, and `path_entry` is the first +/// entry of its `PATH`. The two scenarios differ in nothing else, so the answer +/// can only turn on whether that entry names the working directory: `.` does, +/// and an empty entry does not (the lookup skips it). That is also what shows +/// `command_dir` reaches the `rocm` child: with `.` the stand-in is found only +/// if `rocm` really runs from `work`. +fn plant_wsl_machine_with_rocminfo_only_in_the_working_directory( + world: &mut E2eWorld, + path_entry: PathBuf, +) { + plant_wsl_machine_with_unregistered_rocdxg(world, ""); + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let work = root.join("work"); + plant_stand_in_rocminfo(&work); + set_planted_wsl_path(world, &[path_entry]); + // A "not installed" answer would also come out if the stand-in could not be + // reached at all, so show it can: started by bare name from `work` with this + // PATH, it must run, which only an entry that names `work` (or the empty + // entry, which `execvp` reads as `work`) makes possible. + let path = world + .command_env + .iter() + .rev() + .find_map(|(key, value)| (*key == "PATH").then(|| value.clone())) + .expect("the planted PATH was just set"); + let reached = std::process::Command::new("rocminfo") + .env("PATH", path) + .current_dir(&work) + .status() + .expect("the stand-in must resolve through the first PATH entry"); + assert!(reached.success(), "the stand-in rocminfo failed: {reached}"); + world.command_dir = Some(work); +} + +#[given( + "a WSL machine whose only rocminfo is in the directory the command runs from, with an empty entry on PATH" +)] +async fn wsl_machine_with_rocminfo_only_behind_an_empty_path_entry(world: &mut E2eWorld) { + plant_wsl_machine_with_rocminfo_only_in_the_working_directory(world, PathBuf::new()); +} + +#[given( + "a WSL machine whose only rocminfo is in the directory the command runs from, with that directory named on PATH" +)] +async fn wsl_machine_with_rocminfo_only_on_a_path_entry_naming_the_working_directory( + world: &mut E2eWorld, +) { + plant_wsl_machine_with_rocminfo_only_in_the_working_directory(world, PathBuf::from(".")); +} + +/// A directory name that is not valid Unicode: the byte 0xff is never UTF-8. +#[cfg(unix)] +fn directory_name_that_is_not_unicode() -> std::ffi::OsString { + use std::os::unix::ffi::OsStrExt as _; + + std::ffi::OsStr::from_bytes(b"rocminfo-bin-\xff").to_owned() +} + +/// Windows file names are UTF-16 and a lone surrogate would stand in for the +/// byte; the scenario that asks for this name is `@requires-os:linux`, so +/// reaching this is a harness mistake. +#[cfg(not(unix))] +fn directory_name_that_is_not_unicode() -> std::ffi::OsString { + panic!("a directory name that is not valid Unicode is planted on Linux only") +} + +#[given( + "a WSL machine whose rocminfo sits in a PATH directory with a name that is not valid Unicode" +)] +async fn wsl_machine_with_rocminfo_in_a_directory_that_is_not_unicode(world: &mut E2eWorld) { + plant_wsl_machine_with_unregistered_rocdxg(world, ""); + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let tools = root.join(directory_name_that_is_not_unicode()); + plant_stand_in_rocminfo(&tools); + set_planted_wsl_path(world, &[tools]); +} + +/// Both reports of one `rocm examine --json` run say whether `rocminfo` is +/// installed: `summary.wsl.rocminfo` from the host summary and `wsl.rocminfo` +/// from the examination. Asked together so the pair cannot disagree unseen. +fn rocminfo_answers(world: &E2eWorld) -> (bool, bool) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + let answer = |pointer: &str| { + json.pointer(pointer) + .and_then(serde_json::Value::as_bool) + .unwrap_or_else(|| panic!("`examine --json` has no boolean at {pointer}:\n{json:#}")) + }; + (answer("/summary/wsl/rocminfo"), answer("/wsl/rocminfo")) +} + +#[then("both reports say rocminfo is installed")] +async fn assert_rocminfo_installed_in_both_reports(world: &mut E2eWorld) { + assert_eq!( + rocminfo_answers(world), + (true, true), + "(host summary, examination)" + ); +} + +#[then("both reports say rocminfo is not installed")] +async fn assert_rocminfo_not_installed_in_either_report(world: &mut E2eWorld) { + assert_eq!( + rocminfo_answers(world), + (false, false), + "(host summary, examination)" + ); +} + +// ── Both reports read the kernel release from one `uname -r` ─────── + +/// The release the stand-in `uname` reports, chosen so no real host has it. +const STAND_IN_KERNEL_RELEASE: &str = "6.8.0-stand-in-kernel"; + +/// Where the stand-in `uname` logs the arguments of every call, one line each, +/// so a scenario can count the reads. Inside the isolated root. +fn stand_in_uname_log(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-calls.log") +} + +/// Plant a `uname` first on the `PATH` of the next `rocm` run. `-r` runs +/// `release_body` (a shell fragment); `-v` answers a fixed string, so the +/// examination's other `uname` read has nothing to wait for. Every call appends +/// its arguments to [`stand_in_uname_log`]. Nothing else on the +/// machine is planted: the stand-in is what `rocm examine` reads on any Linux +/// host, native or WSL, because both stacks run `uname -r` there. +fn plant_stand_in_uname(world: &mut E2eWorld, release_body: &str) { + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the stand-in uname directory"); + let log = stand_in_uname_log(world); + rocm_core::fake_executable::write_fake_executable( + &bin.join("uname"), + e2e_cucumber::stand_in_uname::stand_in_uname_script(release_body, &log), + ) + .expect("failed to write the stand-in uname"); + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths(&inherited))) + .expect("PATH entries cannot hold a separator"); + world.command_env.retain(|(key, _)| *key != "PATH"); + world.command_env.push(("PATH", path)); +} + +#[given("a machine whose uname reports a kernel release")] +async fn machine_whose_uname_reports_a_release(world: &mut E2eWorld) { + plant_stand_in_uname(world, &format!("echo {STAND_IN_KERNEL_RELEASE}")); +} + +#[given("a machine whose uname prints a kernel release and then fails")] +async fn machine_whose_uname_prints_a_release_and_fails(world: &mut E2eWorld) { + plant_stand_in_uname(world, &format!("echo {STAND_IN_KERNEL_RELEASE}; exit 3")); +} + +/// The `probe_failures` entries of an `examine --json` document, whatever they +/// say. +fn probe_failures(json: &serde_json::Value) -> Vec { + json.get("probe_failures") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("`examine --json` has no probe_failures array:\n{json:#}")) + .iter() + .filter_map(serde_json::Value::as_str) + .map(str::to_owned) + .collect() +} + +/// The `probe_failures` of the same machine with a `uname` that works, for a +/// baseline: entries this machine legitimately has for other reasons are not +/// the kernel read's, and a failure the failing `uname` adds, however worded, +/// is. +fn probe_failures_with_a_working_uname(world: &E2eWorld) -> Vec { + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-baseline-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the baseline uname directory"); + rocm_core::fake_executable::write_fake_executable( + &bin.join("uname"), + e2e_cucumber::stand_in_uname::stand_in_uname_script( + &format!("echo {STAND_IN_KERNEL_RELEASE}"), + &world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-baseline-calls.log"), + ), + ) + .expect("failed to write the baseline uname"); + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths(&inherited))) + .expect("PATH entries cannot hold a separator"); + let path = path.to_str().expect("PATH is UTF-8").to_owned(); + let (stdout, stderr, rc) = crate::run_rocm_with_env( + world, + &["examine", "--framework", "skip", "--json"], + &[("PATH", path.as_str())], + ); + assert_eq!( + rc, 0, + "the baseline `rocm examine --json` failed:\n{stderr}" + ); + let json: serde_json::Value = + serde_json::from_str(&stdout).expect("the baseline run emitted JSON"); + probe_failures(&json) +} + +#[then("the summary and the examination report that kernel release")] +async fn assert_both_report_the_kernel_release(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + assert_eq!( + inspection_str(&json, "/summary/kernel"), + STAND_IN_KERNEL_RELEASE, + "{json:#}" + ); + assert_eq!( + inspection_str(&json, "/kernel_release"), + STAND_IN_KERNEL_RELEASE, + "{json:#}" + ); +} + +/// One `rocm examine --json` run asks `uname` for the release once, for the +/// examination and the summary together: the stand-in logged a single `-r`. A +/// second read would show as a second line, and it is what lets a transient +/// failure make the two fields disagree. +#[then("that run asked uname for the kernel release once")] +async fn assert_uname_was_asked_for_the_release_once(world: &mut E2eWorld) { + let log = std::fs::read_to_string(stand_in_uname_log(world)) + .expect("the stand-in uname was never called"); + let release_reads = log.lines().filter(|line| line.trim() == "-r").count(); + assert_eq!(release_reads, 1, "calls of the stand-in uname:\n{log}"); +} + +#[then( + "neither the summary nor the examination reports a kernel release, and neither calls it a failure" +)] +async fn assert_neither_reports_a_kernel_release(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + assert!( + json.pointer("/summary/kernel") + .is_some_and(serde_json::Value::is_null), + "the summary must read no kernel from a uname that failed:\n{json:#}" + ); + assert_eq!(inspection_str(&json, "/kernel_release"), "", "{json:#}"); + // No `probe_failures` entry beyond what the same machine has with a working + // `uname`, so a failure worded in any way cannot slip through. + let baseline = probe_failures_with_a_working_uname(world); + let new_failures: Vec = probe_failures(&json) + .into_iter() + .filter(|failure| !baseline.contains(failure)) + .collect(); + assert_eq!(new_failures, Vec::::new(), "{json:#}"); +} + +// ── Both reports read the CPU from one reader ────────────────────── + +/// The model a planted AMD `/proc/cpuinfo` names, spelled as `/proc/cpuinfo` +/// does, with runs of spaces inside it. +const PLANTED_AMD_CPUINFO: &str = "processor\t: 0\nvendor_id\t: AuthenticAMD\ncpu family\t: 26\nmodel name\t: AMD Ryzen 9 9950X 16-Core Processor \nprocessor\t: 1\nvendor_id\t: AuthenticAMD\nmodel name\t: AMD Ryzen 9 9950X 16-Core Processor \n"; + +/// The same model with its whitespace collapsed: what both reports show. +const PLANTED_AMD_CPU_MODEL: &str = "AMD Ryzen 9 9950X 16-Core Processor"; + +/// An Arm board's cpuinfo: a `Hardware` line, no `model name` and no `vendor_id`. +const PLANTED_HARDWARE_ONLY_CPUINFO: &str = + "processor\t: 0\nBogoMIPS\t: 38.40\nCPU implementer\t: 0x41\nHardware\t: BCM2835\n"; + +/// Plant `cpuinfo` as `/proc/cpuinfo` under a simulated host root inside the +/// scenario's isolated tree, with a `/dev/dxg` when `wsl`, and hand that root to +/// the next `rocm` invocation. The root holds nothing else, so the machine reads +/// as WSL only when asked to and every other re-rooted probe sees an empty one. +fn plant_cpuinfo_machine(world: &mut E2eWorld, cpuinfo: &str, wsl: bool) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("host-root"); + std::fs::create_dir_all(root.join("proc")).expect("failed to create planted /proc"); + std::fs::write(root.join("proc").join("cpuinfo"), cpuinfo) + .expect("failed to write planted cpuinfo"); + if wsl { + std::fs::create_dir_all(root.join("dev")).expect("failed to create planted /dev"); + std::fs::write(root.join("dev").join("dxg"), "").expect("failed to plant /dev/dxg"); + } + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", root.into_os_string())); +} + +#[given( + "a machine that is not WSL whose cpuinfo names an AMD CPU with repeated spaces in its model" +)] +async fn machine_with_an_amd_cpuinfo(world: &mut E2eWorld) { + plant_cpuinfo_machine(world, PLANTED_AMD_CPUINFO, false); +} + +#[given("a machine that is not WSL whose cpuinfo has a Hardware line and no model name")] +async fn machine_with_a_hardware_only_cpuinfo(world: &mut E2eWorld) { + plant_cpuinfo_machine(world, PLANTED_HARDWARE_ONLY_CPUINFO, false); +} + +#[given("a WSL machine whose cpuinfo names an AMD CPU")] +async fn wsl_machine_with_an_amd_cpuinfo(world: &mut E2eWorld) { + plant_cpuinfo_machine(world, PLANTED_AMD_CPUINFO, true); +} + +/// `summary.cpu`, `cpu_model`, `cpu_vendor` and `is_wsl` of one `rocm examine +/// --json` run. +fn cpu_answers(world: &E2eWorld) -> (Option, String, String, bool) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + let summary = json + .pointer("/summary/cpu") + .unwrap_or_else(|| panic!("`examine --json` has no summary.cpu:\n{json:#}")) + .as_str() + .map(str::to_owned); + let is_wsl = json + .pointer("/is_wsl") + .and_then(serde_json::Value::as_bool) + .unwrap_or_else(|| panic!("`examine --json` has no boolean is_wsl:\n{json:#}")); + ( + summary, + inspection_str(&json, "/cpu_model").to_owned(), + inspection_str(&json, "/cpu_vendor").to_owned(), + is_wsl, + ) +} + +#[then( + "the summary and the examination report that CPU model, and the examination reports its vendor as amd" +)] +async fn assert_both_report_the_planted_amd_cpu(world: &mut E2eWorld) { + assert_eq!( + cpu_answers(world), + ( + Some(PLANTED_AMD_CPU_MODEL.to_owned()), + PLANTED_AMD_CPU_MODEL.to_owned(), + "amd".to_owned(), + false + ), + "(summary.cpu, cpu_model, cpu_vendor, is_wsl)" + ); +} + +#[then( + "the summary and the examination report that CPU model, and the examination reports its vendor as unknown" +)] +async fn assert_both_report_the_hardware_line_cpu(world: &mut E2eWorld) { + assert_eq!( + cpu_answers(world), + ( + Some("BCM2835".to_owned()), + "BCM2835".to_owned(), + "unknown".to_owned(), + false + ), + "(summary.cpu, cpu_model, cpu_vendor, is_wsl)" + ); +} + +#[then( + "the summary reports that CPU model, and the examination reports no CPU model and an unknown vendor" +)] +async fn assert_only_the_summary_reports_the_cpu_on_wsl(world: &mut E2eWorld) { + assert_eq!( + cpu_answers(world), + ( + Some(PLANTED_AMD_CPU_MODEL.to_owned()), + String::new(), + "unknown".to_owned(), + true + ), + "(summary.cpu, cpu_model, cpu_vendor, is_wsl)" + ); +} + +// ── A probe tool that leaves a background process behind ─────────────────── + +/// How long the stand-in `lspci`'s background process keeps the pipes it +/// inherited open, which is how long a probe that waits for end of output waits. +const LEFTOVER_PROCESS_SECONDS: u64 = 30; + +/// An inspection that finishes within this did not wait for that process. Half +/// its life, so a loaded runner has the whole of the other half to be slow in. +const INSPECTION_BUDGET: std::time::Duration = std::time::Duration::from_secs(15); + +/// The PCI address and the line the stand-in `lspci` prints for its one GPU. The +/// bus number is one no real machine's first adapter has, so the line cannot be +/// mistaken for a card of the runner's own. +const LEFTOVER_LSPCI_ADDRESS: &str = "0000:fe:00.0"; +const LEFTOVER_LSPCI_LINE: &str = "0000:fe:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi 31 [Radeon RX 7900 XTX] [1002:744c] (rev c8)"; + +#[given( + "a machine whose lspci prints a GPU and leaves a background process holding its output open" +)] +async fn machine_whose_lspci_leaves_a_background_process(world: &mut E2eWorld) { + // A simulated native Linux root: the PCI probe is skipped on WSL, so a runner + // that is a WSL host would otherwise never reach the stand-in. + plant_os_release(world, "ID=ubuntu\nVERSION_ID=\"24.04\"\n"); + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("leftover-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the stand-in lspci's directory"); + // The `sleep` starts in the background of the script, so it inherits the + // script's stdout and stderr, the pipes `rocm` reads the listing from, and + // outlives the script, which exits 0 straight after printing. + rocm_core::fake_executable::write_fake_executable( + &bin.join("lspci"), + format!( + "#!/bin/sh\nprintf '%s\\n' '{LEFTOVER_LSPCI_LINE}'\nsleep {LEFTOVER_PROCESS_SECONDS} &\n" + ), + ) + .expect("failed to write the stand-in lspci"); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.retain(|(key, _)| *key != "PATH"); + world.command_env.push(("PATH", path)); +} + +#[when("the user inspects that machine in machine-readable form and the time it takes is measured")] +async fn user_times_an_inspection_of_the_planted_machine(world: &mut E2eWorld) { + let started = std::time::Instant::now(); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip", "--json"]); + world.cli_elapsed = Some(started.elapsed()); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the inspection finishes long before the background process does")] +async fn assert_the_inspection_did_not_wait_for_the_background_process(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let elapsed = world.cli_elapsed.expect("the inspection was not timed"); + assert!( + elapsed < INSPECTION_BUDGET, + "the inspection took {elapsed:?}, so it waited for the {LEFTOVER_PROCESS_SECONDS}s \ + background process that held `lspci`'s output open" + ); +} + +#[then("the inspection lists the GPU that lspci printed")] +async fn assert_the_gpu_lspci_printed_is_listed(world: &mut E2eWorld) { + let json = parsed_json(world); + let listed = json + .pointer("/gpus") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("`examine --json` has no `gpus` array:\n{json:#}")) + .iter() + .find(|gpu| { + gpu.pointer("/pci_id").and_then(serde_json::Value::as_str) + == Some(LEFTOVER_LSPCI_ADDRESS) + }) + .unwrap_or_else(|| { + panic!("the GPU `lspci` printed at {LEFTOVER_LSPCI_ADDRESS} is not listed:\n{json:#}") + }); + assert_eq!( + listed + .pointer("/is_amd") + .and_then(serde_json::Value::as_bool), + Some(true), + "{listed:#}" + ); +} + // ── `rocm serve` on a WSL host whose ROCDXG library the loader cannot find ── #[when("the user serves a model under the GPU-required default on that machine")] diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs new file mode 100644 index 0000000000..10a98f0295 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -0,0 +1,402 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `lemonade_installs.feature`: what the user is told about the +//! Lemonade copies ROCm CLI puts on disk (ROCMAI-825). +//! +//! Black-box throughout. The two Lemonade trees and the install record are +//! planted as plain files in the layout the engine writes, so no scenario +//! downloads anything; the refusal scenario reaches the engine's free-space +//! check through the `rocm/e2e-test-hooks` override instead of a full drive. + +use std::path::PathBuf; + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +/// Registry key of the planted runtime. Production-shaped (slug, no `:`), so it +/// is also a valid registry file name on Windows. +const RUNTIME_KEY: &str = "release-tarball-gfx1151"; +/// Payload planted in each tree, chosen so the two render as distinct sizes. +const DATA_FOLDER_TREE_BYTES: usize = 3 * 1024 * 1024; +const RUNTIME_TREE_BYTES: usize = 5 * 1024 * 1024; +/// Free space the refusal scenarios report for the install drive: 1 MiB, below +/// what any Lemonade install needs (its archive and extracted files), so the +/// engine refuses before fetching anything. Room for that but not for the +/// multi-gigabyte ROCm backend only warns, and is unit-tested. +const NEARLY_FULL_BYTES: &str = "1048576"; + +fn data_dir(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") +} + +/// Where Lemonade installs when no ROCm runtime is active. +fn data_folder_tree(world: &E2eWorld) -> PathBuf { + data_dir(world).join("engines").join("lemonade") +} + +fn runtime_install_root(world: &E2eWorld) -> PathBuf { + data_dir(world) + .join("runtimes") + .join("tarball") + .join(RUNTIME_KEY) +} + +/// Where Lemonade installs while the planted runtime is active. +fn runtime_tree(world: &E2eWorld) -> PathBuf { + runtime_install_root(world).join("engines").join("lemonade") +} + +/// Lemonade's install record, in the engine's own location. +fn install_record(world: &E2eWorld) -> PathBuf { + data_folder_tree(world) + .join("manifests") + .join("runtime.json") +} + +const fn lemond_name() -> &'static str { + if cfg!(windows) { + "lemond.exe" + } else { + "lemond" + } +} + +/// An extracted tree: the server binary plus a llama.cpp backend payload. +fn plant_lemonade_tree(tree: &std::path::Path, payload_bytes: usize) { + let runtime_dir = tree.join("runtime"); + let backend = runtime_dir.join("bin").join("llamacpp").join("rocm-stable"); + std::fs::create_dir_all(&backend).expect("failed to create backend dir"); + std::fs::write(runtime_dir.join(lemond_name()), b"").expect("failed to write lemond"); + std::fs::write(backend.join("payload.bin"), vec![0_u8; payload_bytes]) + .expect("failed to write backend payload"); +} + +fn stdout(world: &E2eWorld) -> &str { + world + .cli_output + .as_deref() + .expect("no CLI output captured - did the When step run?") +} + +fn combined_output(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or_default(), + world.cli_stderr.as_deref().unwrap_or_default() + ) +} + +#[given("the drive for a new Lemonade install cannot hold even Lemonade itself")] +async fn install_drive_nearly_full(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES", + NEARLY_FULL_BYTES.into(), + )); +} + +#[when("the user installs the Lemonade engine")] +async fn user_installs_lemonade(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["engines", "install", "lemonade", "--yes"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the Lemonade install is refused before anything is downloaded")] +async fn assert_lemonade_install_refused(world: &mut E2eWorld) { + let output = combined_output(world); + // Without the hook the engine sees the real drive and starts a real + // multi-gigabyte install. Name that cause rather than a baffling mismatch. + assert!( + output.contains("Installing Lemonade"), + "no install notice: the binary under test was probably built without the \ + `rocm/e2e-test-hooks` feature, which `cargo xtask e2e` enables:\n{output}" + ); + assert_ne!( + world.cli_rc, + Some(0), + "the install must fail when the drive cannot hold it:\n{output}" + ); + let tree = data_folder_tree(world); + // The claim "Nothing was downloaded" is checked against the disk: no + // archive cache and no extracted runtime. + assert!( + !tree.join("downloads").exists(), + "an archive was downloaded despite the refusal:\n{output}" + ); + assert!( + !tree.join("runtime").exists(), + "a runtime was extracted despite the refusal:\n{output}" + ); + assert!( + !install_record(world).exists(), + "a refused install must not record Lemonade as installed:\n{output}" + ); +} + +#[then("the user is told what the Lemonade install needs and where it would go")] +async fn assert_lemonade_install_notice(world: &mut E2eWorld) { + let output = combined_output(world); + let tree = data_folder_tree(world); + for expected in [ + format!("into {}", tree.display()), + "a download of up to about".to_owned(), + "of disk space (1.0 MiB free on that drive)".to_owned(), + "not enough free disk space to install Lemonade:".to_owned(), + "Nothing was downloaded.".to_owned(), + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } +} + +/// Register the planted runtime: a removable install ROCm CLI made itself +/// (registry entry plus the in-tree copy `runtimes uninstall` checks before +/// deleting the folder), or, with `read_only`, an external one that uninstall +/// only unregisters. +fn plant_runtime(world: &E2eWorld, read_only: bool) { + let install_root = runtime_install_root(world); + std::fs::create_dir_all(&install_root).expect("failed to create runtime root"); + std::fs::write(install_root.join("payload.txt"), "payload") + .expect("failed to write runtime payload"); + let manifest = serde_json::to_string_pretty(&serde_json::json!({ + "runtime_key": RUNTIME_KEY, + "runtime_id": "therock-release:gfx1151", + "channel": "release", + "format": "tarball", + "family": "gfx1151", + "family_source": "manual", + "version": "1.0.0", + "install_root": install_root, + "selected_artifact_url": format!("https://example.invalid/{RUNTIME_KEY}.tar.gz"), + "read_only": read_only, + "installed_at_unix_ms": 1_700_000_000_000u64, + })) + .expect("failed to serialize runtime manifest"); + let registry = data_dir(world).join("runtimes").join("registry"); + std::fs::create_dir_all(®istry).expect("failed to create registry"); + std::fs::write(registry.join(format!("{RUNTIME_KEY}.json")), &manifest) + .expect("failed to write registry entry"); + if !read_only { + std::fs::write(install_root.join(".rocm-cli-runtime.json"), &manifest) + .expect("failed to write in-tree manifest"); + } +} + +/// Write Lemonade's install record naming `runtime_dir`, whether or not +/// anything is there. +fn record_install_at(world: &E2eWorld, runtime_dir: &std::path::Path) { + let record = install_record(world); + std::fs::create_dir_all(record.parent().expect("record has a parent")) + .expect("failed to create record dir"); + std::fs::write( + &record, + serde_json::to_string_pretty(&serde_json::json!({ + "env_id": "lemonade-embeddable-e2e", + "version": "e2e", + "runtime_dir": runtime_dir, + "lemond": runtime_dir.join(lemond_name()), + "lemonade": runtime_dir.join("lemonade"), + "backend_recipe": "llamacpp", + "backend_name": "rocm", + "installed_at_unix_ms": 0, + })) + .expect("failed to serialize install record"), + ) + .expect("failed to write install record"); +} + +#[given("Lemonade was installed in the data folder and again inside a ROCm install")] +async fn plant_two_lemonade_installs(world: &mut E2eWorld) { + plant_runtime(world, false); + // `rocm engines install lemonade` before any runtime, then `rocm serve` + // with the runtime active: one tree each, the record naming the second. + plant_lemonade_tree(&data_folder_tree(world), DATA_FOLDER_TREE_BYTES); + plant_lemonade_tree(&runtime_tree(world), RUNTIME_TREE_BYTES); + record_install_at(world, &runtime_tree(world).join("runtime")); +} + +#[given("the ROCm install that held Lemonade is gone and only the data folder's copy is left")] +async fn plant_surviving_data_folder_copy(world: &mut E2eWorld) { + // The record still names the removed runtime's folder, as it does after a + // runtime was uninstalled by a build that left the record behind. + plant_lemonade_tree(&data_folder_tree(world), DATA_FOLDER_TREE_BYTES); + record_install_at(world, &runtime_tree(world).join("runtime")); +} + +#[given("a read-only ROCm install holds a Lemonade copy")] +async fn plant_lemonade_in_read_only_runtime(world: &mut E2eWorld) { + plant_runtime(world, true); + plant_lemonade_tree(&runtime_tree(world), RUNTIME_TREE_BYTES); + record_install_at(world, &runtime_tree(world).join("runtime")); +} + +/// The report row for the Lemonade tree at `tree`, and the lines under it. +fn lemonade_row<'a>(report: &'a str, tree: &std::path::Path) -> Vec<&'a str> { + let lines: Vec<&str> = report.lines().collect(); + let path = format!("({})", tree.display()); + let at = lines + .iter() + .position(|line| line.trim_start().starts_with("- lemonade (") && line.ends_with(&path)) + .unwrap_or_else(|| panic!("no Lemonade row for {}:\n{report}", tree.display())); + lines[at..] + .iter() + .take_while(|line| { + *line == &lines[at] + || line.trim_start().starts_with("status:") + || line.trim_start().starts_with("note:") + }) + .copied() + .collect() +} + +#[then("the report lists both Lemonade installs with their sizes")] +async fn assert_both_lemonade_rows(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let data_row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + data_row[0].contains("- lemonade (data folder): 3.0 MiB"), + "the data folder's copy must be listed with its size:\n{report}" + ); + let runtime_row = lemonade_row(&report, &runtime_tree(world)); + assert!( + runtime_row[0].contains(&format!( + "- lemonade (inside ROCm install {RUNTIME_KEY}): 5.0 MiB" + )), + "the runtime's copy must be listed with its size:\n{report}" + ); + assert!( + runtime_row + .iter() + .any(|line| line.trim() == "status: in use"), + "the copy the record points at is the one in use:\n{report}" + ); +} + +#[then("the report flags the data folder's Lemonade install as unused")] +async fn assert_data_folder_copy_unused(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let data_row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + data_row + .iter() + .any(|line| line.trim() + == "status: not used right now; Lemonade runs from another install"), + "the data folder's copy must be flagged unused:\n{report}" + ); + assert!( + data_row + .iter() + .any(|line| line.trim_start().starts_with("note: reclaimable")), + "an unused copy must say its space can be reclaimed:\n{report}" + ); + // Reporting only: the tree is still on disk after the report. + assert!(data_folder_tree(world).join("runtime").exists()); +} + +#[then("the report does not call the surviving Lemonade copy unused or reclaimable")] +async fn assert_survivor_not_called_reclaimable(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + row.iter().any(|line| line.trim() + == "status: not known to be used; the install record names a folder with no install"), + "a record naming a folder with no install leaves no copy known to be in use:\n{report}" + ); + assert!( + row.iter() + .all(|line| !line.contains("reclaimable") || line.contains("not marked reclaimable")), + "the only working copy must not be offered for deletion:\n{report}" + ); + assert!( + data_folder_tree(world).join("runtime").exists(), + "reporting must not delete anything" + ); +} + +#[then("the report does not say that copy is removed with its ROCm install")] +async fn assert_read_only_copy_not_removed_with_runtime(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let row = lemonade_row(&report, &runtime_tree(world)); + assert!( + row.iter().all(|line| !line.contains("removed with it")), + "uninstalling an external ROCm install only unregisters it:\n{report}" + ); + assert!( + row.iter() + .any(|line| line + .contains("does not delete that install's folder (external or imported)")), + "the row must say why the copy outlives an uninstall:\n{report}" + ); +} + +#[then("that Lemonade copy is still on disk")] +async fn assert_lemonade_copy_still_on_disk(world: &mut E2eWorld) { + assert!( + runtime_tree(world).join("runtime").exists(), + "uninstalling an external ROCm install must leave its Lemonade copy" + ); +} + +#[when("the user uninstalls the ROCm install holding Lemonade")] +async fn user_uninstalls_runtime_holding_lemonade(world: &mut E2eWorld) { + let stdout = crate::run_rocm_ok(world, &["runtimes", "uninstall", RUNTIME_KEY, "--yes"]); + world.cli_output = Some(stdout); +} + +#[then("the CLI says Lemonade's install was removed with that ROCm install")] +async fn assert_lemonade_removed_with_runtime(world: &mut E2eWorld) { + let output = stdout(world).to_owned(); + let runtime_dir = runtime_tree(world).join("runtime"); + assert!( + output.contains(&format!( + "its install at {} is inside this runtime folder and would be removed with it", + runtime_dir.display() + )), + "the plan must warn that Lemonade goes with the runtime:\n{output}" + ); + assert!( + output.contains(&format!( + "its install at {} was inside this runtime and was removed with it", + runtime_dir.display() + )), + "the result must say Lemonade was removed with the runtime:\n{output}" + ); + assert!( + !runtime_install_root(world).exists(), + "the runtime folder, Lemonade tree included, must be gone:\n{output}" + ); +} + +#[then("Lemonade is no longer recorded as installed")] +async fn assert_lemonade_record_cleared(world: &mut E2eWorld) { + assert!( + !install_record(world).exists(), + "the record still names a deleted tree" + ); + // The CLI's own view agrees: the remaining copy is no longer called unused + // (that needs a record pointing elsewhere), and the deleted one is not listed. + let report = crate::run_rocm_ok(world, &["storage", "report"]); + let data_row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + data_row + .iter() + .any(|line| line.trim() == "status: not recorded as installed"), + "with no record, the remaining copy is not recorded as installed:\n{report}" + ); + assert!( + !report.contains(&runtime_tree(world).display().to_string()), + "the removed copy must not be listed any more:\n{report}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs index 1dd998ecba..aa3d3d0fd4 100644 --- a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs @@ -60,6 +60,12 @@ pub struct LifecycleState { /// A shell profile the installer is pointed at through /// `ROCM_CLI_SHELL_PROFILE`; `None` means the default `~/.bashrc`. custom_profile: Option, + /// The login shell (`$SHELL`) the installer is run under; `None` leaves the + /// runner's own. + login_shell: Option, + /// The fish config seeded before install, which uninstall must restore + /// exactly. + fish_config_before_install: Option, } /// The Windows user `Path` exactly as the registry holds it: its value with @@ -235,6 +241,9 @@ fn run_installer_command( // Isolate the installer's HOME so shell-profile writes and the seeded // ~/.rocm/config.json land in the scenario's tree, never the real home. cmd.env("HOME", &st.home_dir); + // fish's config lives under $XDG_CONFIG_HOME when that is set, which would + // point the installer's fish setup at the runner's real config. + cmd.env_remove("XDG_CONFIG_HOME"); for (key, value) in public_key_env { cmd.env(key, value); } @@ -534,6 +543,8 @@ async fn given_release_tree(world: &mut E2eWorld) { smoke_cache: None, profile_before_install: None, custom_profile: None, + login_shell: None, + fish_config_before_install: None, }); } @@ -590,6 +601,37 @@ async fn given_profile_user_settings(world: &mut E2eWorld) { state_mut(world).profile_before_install = Some(contents); } +/// The fish config the installer writes its PATH block to in this scenario. +fn fish_config(st: &LifecycleState) -> PathBuf { + st.home_dir.join(".config").join("fish").join("config.fish") +} + +/// The completion file the installer writes for fish in this scenario. +fn fish_completions(st: &LifecycleState) -> PathBuf { + st.home_dir + .join(".config") + .join("fish") + .join("completions") + .join("rocm.fish") +} + +#[given("fish is set up with its own settings")] +async fn given_fish_configured(world: &mut E2eWorld) { + // A config directory is enough for the installer to treat fish as installed, + // so the scenario does not depend on the runner having the fish binary. + let config = fish_config(state(world)); + std::fs::create_dir_all(config.parent().expect("fish config has a folder")) + .expect("failed to create the fish config folder"); + let contents = "# the user's own fish settings\nset -gx ROCM_E2E_USER_SETTING 1\n".to_owned(); + std::fs::write(&config, &contents).expect("failed to seed the fish config"); + state_mut(world).fish_config_before_install = Some(contents); +} + +#[given("the login shell is bash")] +async fn given_login_shell_bash(world: &mut E2eWorld) { + state_mut(world).login_shell = Some("/bin/bash".to_owned()); +} + #[given("a signed bundle installed updating the user PATH")] async fn given_installed_updating_user_path(world: &mut E2eWorld) { package_with_key_file(world).await; @@ -779,6 +821,9 @@ async fn install_with_key_file_shell_profile(world: &mut E2eWorld) { st.public_key.to_string_lossy().into_owned(), )]; let mut shell_on = vec![("ROCM_CLI_UPDATE_SHELL_PATH", "1".to_string())]; + if let Some(shell) = &st.login_shell { + shell_on.push(("SHELL", shell.clone())); + } if let Some(profile) = &st.custom_profile { shell_on.push(( "ROCM_CLI_SHELL_PROFILE", @@ -1455,8 +1500,12 @@ async fn then_single_path_marker(world: &mut E2eWorld) { /// The scenario's shell profile holds the installer's marked PATH block exactly /// once. fn assert_single_path_marker(st: &LifecycleState) { - let profile = shell_profile(st); - let contents = std::fs::read_to_string(&profile).unwrap_or_default(); + assert_single_path_marker_in(&shell_profile(st)); +} + +/// Assert `profile` holds exactly one rocm-cli PATH block, and return its text. +fn assert_single_path_marker_in(profile: &Path) -> String { + let contents = std::fs::read_to_string(profile).unwrap_or_default(); let count = contents.matches("# >>> rocm-cli path >>>").count(); assert_eq!( count, @@ -1464,6 +1513,7 @@ fn assert_single_path_marker(st: &LifecycleState) { "expected exactly one rocm-cli PATH marker in {}, found {count}:\n{contents}", profile.display() ); + contents } #[then("examine reads only the isolated config, data, and cache directories")] @@ -1544,6 +1594,270 @@ async fn then_profile_restored(world: &mut E2eWorld) { ); } +#[then("the fish config adds the install directory to PATH once")] +async fn then_fish_config_has_path(world: &mut E2eWorld) { + let st = state(world); + let config = fish_config(st); + let contents = assert_single_path_marker_in(&config); + // The install folder sits outside the scenario's HOME, so the block names + // it literally rather than as `$HOME/...`. + let install_dir = st.install_dir.to_string_lossy(); + assert!( + contents.contains(&format!("set -gx PATH \"{install_dir}\" $PATH")), + "the fish config does not put {install_dir} on PATH in fish syntax:\n{contents}" + ); + assert!( + contents.contains("# the user's own fish settings"), + "the installer dropped the user's own fish settings:\n{contents}" + ); + // The bash profile the login shell asked for keeps POSIX syntax, not fish's. + let bashrc = std::fs::read_to_string(shell_profile(st)).expect("bash profile not found"); + assert!( + bashrc.contains("export PATH=") && !bashrc.contains("set -gx PATH"), + "the bash profile did not keep POSIX syntax:\n{bashrc}" + ); + let out = &st.last_output; + assert!( + out.contains(&config.display().to_string()), + "installer did not report the fish profile:\n{out}" + ); +} + +#[then("the fish completions for the installed rocm are in place")] +async fn then_fish_completions_installed(world: &mut E2eWorld) { + let st = state(world); + let path = fish_completions(st); + let contents = std::fs::read_to_string(&path).expect("fish completions not found"); + assert!( + contents.contains("complete -c rocm"), + "{} is not a rocm completion script:\n{contents}", + path.display() + ); + // Uninstall removes the file only when this line names its own folder. + let marker = format!( + "# rocm-cli fish completions for {}\n", + st.install_dir.display() + ); + assert!( + contents.starts_with(&marker), + "{} does not start with the installer's marker {marker:?}:\n{contents}", + path.display() + ); + let out = &st.last_output; + assert!( + out.contains("fish completions installed:") && out.contains(&path.display().to_string()), + "installer did not report the fish completions:\n{out}" + ); +} + +#[then("the fish config is back to the user's own settings")] +async fn then_fish_config_restored(world: &mut E2eWorld) { + let st = state(world); + let before = st + .fish_config_before_install + .as_deref() + .expect("the fish config was not seeded"); + let contents = std::fs::read_to_string(fish_config(st)).unwrap_or_default(); + assert_eq!( + contents, before, + "uninstall did not restore the fish config to its pre-install contents" + ); +} + +#[then("the fish completions for the uninstalled rocm are gone")] +async fn then_fish_completions_removed(world: &mut E2eWorld) { + let st = state(world); + let path = fish_completions(st); + assert!( + !path.exists(), + "uninstall left {} behind for a rocm that is gone", + path.display() + ); + assert!( + path.parent().is_some_and(Path::is_dir), + "uninstall removed the user's fish completions folder, not just rocm's script" + ); + let out = &st.last_output; + assert!( + out.contains(&format!("removed shell completions {}", path.display())), + "uninstall did not report removing the fish completions:\n{out}" + ); +} + +/// What the user writes over the installer's fish completions in a scenario. +const USERS_OWN_COMPLETIONS: &str = "complete -c rocm -l my-own-flag\n"; + +#[given("the user has replaced the fish completions with their own")] +async fn given_users_own_fish_completions(world: &mut E2eWorld) { + let path = fish_completions(state(world)); + assert!(path.is_file(), "the installer wrote no {}", path.display()); + std::fs::write(&path, USERS_OWN_COMPLETIONS).expect("failed to replace the fish completions"); +} + +/// Set the Unix permission bits of `path`. The scenarios that use this are +/// `@requires-os:linux`; the stub only keeps the Windows build compiling. +#[cfg(unix)] +fn set_unix_mode(path: &Path, mode: u32) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)) +} + +#[cfg(not(unix))] +fn set_unix_mode(_path: &Path, _mode: u32) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Unix permission bits do not apply on this platform", + )) +} + +#[given("the fish completions folder cannot be written to")] +async fn given_fish_completions_folder_read_only(world: &mut E2eWorld) { + let dir = fish_completions(state(world)) + .parent() + .expect("completions have a folder") + .to_path_buf(); + set_unix_mode(&dir, 0o555).expect("failed to make the fish completions folder read-only"); + // Root writes through folder permissions, so the scenario would not reach + // the failure it is about. + let probe = dir.join(".rocm-e2e-probe"); + if std::fs::write(&probe, "").is_ok() { + let _ = std::fs::remove_file(&probe); + let _ = set_unix_mode(&dir, 0o755); + panic!( + "{} is still writable after chmod 0555; this scenario needs a non-root runner", + dir.display() + ); + } +} + +#[given("the user already has their own fish completions")] +async fn given_users_fish_completions_before_install(world: &mut E2eWorld) { + let path = fish_completions(state(world)); + std::fs::create_dir_all(path.parent().expect("completions have a folder")) + .expect("failed to create the fish completions folder"); + std::fs::write(&path, USERS_OWN_COMPLETIONS).expect("failed to seed the fish completions"); +} + +#[then("the installer leaves the user's own fish completions and says why")] +async fn then_installer_leaves_users_fish_completions(world: &mut E2eWorld) { + let st = state(world); + let path = fish_completions(st); + assert_eq!( + std::fs::read_to_string(&path).unwrap_or_default(), + USERS_OWN_COMPLETIONS, + "the installer changed {}, which it did not write", + path.display() + ); + let out = &st.last_output; + assert!( + out.contains(&format!( + "note: fish completions not installed: {} already exists and was not written by \ + this installer", + path.display() + )), + "the installer did not say it left the fish completions:\n{out}" + ); + assert!( + !out.contains("fish completions installed:"), + "the installer reported installing completions it did not write:\n{out}" + ); + let record = std::fs::read_to_string(st.install_dir.join(".rocm-cli-path-edit")) + .expect("the installer wrote no PATH-edit record"); + assert!( + !record.contains("completions "), + "the installer recorded completions it did not write, so uninstall could remove them:\n{record}" + ); +} + +#[then("the user's own fish completions are untouched")] +async fn then_users_fish_completions_untouched(world: &mut E2eWorld) { + let st = state(world); + let path = fish_completions(st); + assert_eq!( + std::fs::read_to_string(&path).unwrap_or_default(), + USERS_OWN_COMPLETIONS, + "uninstall removed or changed {}, which the installer did not write", + path.display() + ); + let out = &st.last_output; + assert!( + !out.contains("removed shell completions"), + "uninstall reported removing completions it should not have touched:\n{out}" + ); +} + +#[then("the fish completions are still in place")] +async fn then_fish_completions_still_in_place(world: &mut E2eWorld) { + let st = state(world); + let path = fish_completions(st); + assert!( + std::fs::read_to_string(&path) + .unwrap_or_default() + .contains("complete -c rocm"), + "{} is gone after an uninstall that keeps the binaries", + path.display() + ); + assert!( + !st.last_output.contains("removed shell completions"), + "uninstall reported removing the fish completions it was told to keep:\n{}", + st.last_output + ); +} + +#[then("uninstall leaves the user's own fish completions in place and says why")] +async fn then_users_fish_completions_kept(world: &mut E2eWorld) { + let st = state(world); + let path = fish_completions(st); + let contents = std::fs::read_to_string(&path).unwrap_or_default(); + assert_eq!( + contents, + USERS_OWN_COMPLETIONS, + "uninstall removed or changed {}, which the installer did not write", + path.display() + ); + let out = &st.last_output; + assert!( + out.contains(&format!( + "fish completions {} left in place: it does not start with the rocm-cli \ + installer's marker, so the installer did not write it", + path.display() + )), + "uninstall did not say why it left the fish completions:\n{out}" + ); + assert!( + !out.contains("removed shell completions"), + "uninstall reported removing completions it kept:\n{out}" + ); +} + +#[then("uninstall warns that the fish completions could not be removed and still finishes")] +async fn then_fish_completions_removal_warned(world: &mut E2eWorld) { + let st = state(world); + let path = fish_completions(st); + let dir = path.parent().expect("completions have a folder"); + // Put the folder back first, so the scratch tree can be cleaned up even if + // an assertion below fails. + let _ = set_unix_mode(dir, 0o755); + let out = &st.last_output; + assert_eq!(st.last_rc, 0, "uninstall did not finish:\n{out}"); + assert!( + out.contains(&format!( + "warning: could not remove shell completions {}", + path.display() + )) && out.contains("The uninstall went on without it"), + "uninstall did not warn about the fish completions:\n{out}" + ); + assert!( + out.contains("uninstall complete"), + "uninstall did not report finishing:\n{out}" + ); + assert!( + path.is_file(), + "{} is gone, so the warning was not true", + path.display() + ); +} + #[then("the shell profile still adds the install directory to PATH")] async fn then_profile_kept(world: &mut E2eWorld) { assert_single_path_marker(state(world)); @@ -1574,10 +1888,13 @@ async fn then_xdg_kept(world: &mut E2eWorld) { #[then("uninstall reports keeping the PATH change with the binaries")] async fn then_reports_path_kept_with_binaries(world: &mut E2eWorld) { + // The message is true because lifecycle-25 also asserts the state it + // promises: the PATH change and the fish completions stay (see + // `then_fish_completions_still_in_place` and the profile step). let out = &state(world).last_output; assert!( out.contains("binary removal disabled by --keep-binaries") - && out.contains("any PATH change the installer made stays too"), + && out.contains("any PATH change and fish completions the installer made stay too"), "uninstall did not say the PATH change stays with the binaries:\n{out}" ); } diff --git a/tests/e2e-cucumber/tests/e2e/logs_steps.rs b/tests/e2e-cucumber/tests/e2e/logs_steps.rs index e6e31444c2..74ddd75b80 100644 --- a/tests/e2e-cucumber/tests/e2e/logs_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/logs_steps.rs @@ -23,26 +23,185 @@ const MATCH_COUNT: usize = 9; #[given("recorded command logs containing several lines about a topic")] async fn plant_command_logs(world: &mut E2eWorld) { + plant_logs(world, false); +} + +#[given( + "recorded command logs containing several lines about a topic and one byte that is not valid text" +)] +async fn plant_command_logs_with_an_invalid_byte(world: &mut E2eWorld) { + plant_logs(world, true); +} + +/// Plant one action log. With `invalid_byte`, an unrelated leading line carries +/// a lone `0xff` — a byte no UTF-8 text contains — which the matching lines +/// after it must survive. +fn plant_logs(world: &E2eWorld, invalid_byte: bool) { let root = world.isolated_root.as_ref().expect("no isolated root"); let cli_logs = root.path().join("data").join("logs").join("cli"); std::fs::create_dir_all(&cli_logs).expect("failed to create cli logs dir"); // Write the matching lines LAST so they fall within the per-file tail window // `rocm logs` reads (a few leading non-matching lines are harmless context). - let mut body = String::new(); + let mut body: Vec = Vec::new(); for i in 0..3 { - let _ = writeln!(body, "2026-01-01T00:00:0{i} unrelated startup line"); + let mut line = String::new(); + let _ = writeln!(line, "2026-01-01T00:00:0{i} unrelated startup line"); + if invalid_byte && i == 1 { + body.extend_from_slice(b"2026-01-01T00:00:01 unrelated line with a stray \xff byte\n"); + } else { + body.extend_from_slice(line.as_bytes()); + } } for i in 1..=MATCH_COUNT { + let mut line = String::new(); let _ = writeln!( - body, + line, "2026-01-01T00:01:00 event {TOPIC} occurred number {i}" ); + body.extend_from_slice(line.as_bytes()); } std::fs::write(cli_logs.join("e2e-probe.log"), body).expect("failed to write log file"); } +/// How far past the 4 MiB a log view reads (`LOG_VIEW_MAX_TAIL_READ` in +/// `rocm-core`) the planted line reaches. +const LOG_VIEW_BOUND_BYTES: usize = 4 * 1024 * 1024; +const OVERSIZE_LINE_BYTES: usize = LOG_VIEW_BOUND_BYTES + 1024; + +/// Plant the record of a stopped local server (the same planting the record +/// cleanup scenarios use) and replace its log with `bytes`. +fn plant_server_record_with_log(world: &E2eWorld, bytes: &[u8]) { + crate::e2e::service_cleanup_steps::plant_record(world, "failed"); + std::fs::write(server_log_path(world), bytes).expect("failed to write the server log"); +} + +fn server_log_path(world: &E2eWorld) -> std::path::PathBuf { + crate::e2e::service_cleanup_steps::services_dir(world).join(format!( + "{}.log", + crate::e2e::service_cleanup_steps::SERVICE_ID + )) +} + +/// The one saved ComfyUI log the ComfyUI log scenarios plant. +fn comfyui_log_path(world: &E2eWorld) -> std::path::PathBuf { + let root = world.isolated_root.as_ref().expect("no isolated root"); + root.path() + .join("data") + .join("apps") + .join("comfyui") + .join("logs") + .join("start-e2e.log") +} + +fn plant_comfyui_log(world: &E2eWorld, bytes: &[u8]) { + let log = comfyui_log_path(world); + std::fs::create_dir_all(log.parent().expect("log has a parent")) + .expect("failed to create the ComfyUI logs dir"); + std::fs::write(&log, bytes).expect("failed to write the ComfyUI log"); +} + +#[given("a ComfyUI log holding one line longer than the log view reads")] +async fn comfyui_with_an_oversize_line(world: &mut E2eWorld) { + plant_comfyui_log(world, &vec![b'x'; OVERSIZE_LINE_BYTES]); +} + +/// The one command log the logs-browser scenario plants. +fn action_log_path(world: &E2eWorld) -> std::path::PathBuf { + let root = world.isolated_root.as_ref().expect("no isolated root"); + root.path() + .join("data") + .join("logs") + .join("cli") + .join("e2e-oversize.log") +} + +#[given("a recorded command log whose only line is longer than the log view reads")] +async fn command_log_with_an_oversize_line(world: &mut E2eWorld) { + let log = action_log_path(world); + std::fs::create_dir_all(log.parent().expect("log has a parent")) + .expect("failed to create the cli logs dir"); + let mut bytes = vec![b'x'; OVERSIZE_LINE_BYTES]; + bytes.push(b'\n'); + std::fs::write(&log, bytes).expect("failed to write the command log"); +} + +#[given("a ComfyUI log file that is empty")] +async fn comfyui_with_an_empty_log(world: &mut E2eWorld) { + plant_comfyui_log(world, b""); +} + +#[given("a local server record whose log holds one line longer than the log view reads")] +async fn server_with_an_oversize_line(world: &mut E2eWorld) { + // No final newline: a server mid-line when it died, or one that logs + // without line breaks. + plant_server_record_with_log(world, &vec![b'x'; OVERSIZE_LINE_BYTES]); +} + +#[given("a local server record whose log holds three short lines")] +async fn server_with_three_lines(world: &mut E2eWorld) { + plant_server_record_with_log(world, b"first line\nsecond line\nthird line\n"); +} + +#[given("a local server record whose log file is empty")] +async fn server_with_an_empty_log(world: &mut E2eWorld) { + plant_server_record_with_log(world, b""); +} + // ── When ─────────────────────────────────────────────────────────── +#[when("the user reads that server's log")] +async fn read_server_log(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "services", + "logs", + crate::e2e::service_cleanup_steps::SERVICE_ID, + ], + ); + record(world, stdout, stderr, rc); +} + +#[when("a tool-using client asks the service_logs tool for 50 lines of that server's log")] +async fn read_fifty_lines_through_the_tool(world: &mut E2eWorld) { + let arguments = serde_json::json!({ + "service_id": crate::e2e::service_cleanup_steps::SERVICE_ID, + "lines": 50, + }) + .to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["mcp-call", "service_logs", "--arguments-json", &arguments], + ); + record(world, stdout, stderr, rc); +} + +#[when("a tool-using client reads that server's log through the service_logs tool")] +async fn read_server_log_through_the_tool(world: &mut E2eWorld) { + // The tool is read-only, so no `--allow-mutation`. The exit code is checked + // by the `Then` steps that parse the reply. + let arguments = + serde_json::json!({ "service_id": crate::e2e::service_cleanup_steps::SERVICE_ID }) + .to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["mcp-call", "service_logs", "--arguments-json", &arguments], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user reads the ComfyUI logs")] +async fn read_comfyui_logs(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["comfyui", "logs"]); + record(world, stdout, stderr, rc); +} + +#[when("the user lists the recorded logs")] +async fn list_logs(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["logs"]); + record(world, stdout, stderr, rc); +} + #[when("the user searches the logs for that topic")] async fn search_topic(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm(world, &["logs", "--search", TOPIC]); @@ -81,6 +240,168 @@ async fn reports_matches(world: &mut E2eWorld) { ); } +#[then("the CLI says the newest line is too long to show instead of saying there is no output")] +async fn says_the_line_is_too_long(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains( + "Recent output: none shown; the newest line is longer than the 4 MiB a log view reads" + ), + "expected the too-long notice, got:\n{out}" + ); + assert!( + !out.contains("no output yet"), + "a log with content must not be reported as having no output:\n{out}" + ); + // The claim and the state it describes: the log on disk holds exactly one + // line, longer than the bound the notice names. + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!( + on_disk.len() > LOG_VIEW_BOUND_BYTES && !on_disk.contains(&b'\n'), + "the log on disk should be one line past the bound, is {} bytes", + on_disk.len() + ); +} + +#[then( + "the CLI says the ComfyUI log's newest line is too long to show instead of saying it is empty" +)] +async fn comfyui_says_the_line_is_too_long(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("No output shown: the newest line is longer than the 4 MiB a log view reads."), + "expected the too-long notice, got:\n{out}" + ); + assert!( + !out.contains("is empty"), + "a log with content must not be reported as empty:\n{out}" + ); + let on_disk = std::fs::read(comfyui_log_path(world)).expect("the ComfyUI log is still there"); + assert!( + on_disk.len() > LOG_VIEW_BOUND_BYTES && !on_disk.contains(&b'\n'), + "the log on disk should be one line past the bound, is {} bytes", + on_disk.len() + ); +} + +#[then( + "the CLI lists that log as having output it could not show instead of saying no logs were found" +)] +async fn lists_the_log_as_not_shown(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains( + "" + ), + "expected the log to be listed with the too-long notice, got:\n{out}" + ); + assert!( + !out.contains("No logs found yet."), + "a log with content must not be reported as absent:\n{out}" + ); + // The claim and the state it describes: the log on disk holds exactly one + // line, longer than the bound the notice names. + let on_disk = std::fs::read(action_log_path(world)).expect("the command log is still there"); + assert!( + on_disk.len() > LOG_VIEW_BOUND_BYTES + && on_disk.ends_with(b"\n") + && !on_disk[..on_disk.len() - 1].contains(&b'\n'), + "the log on disk should be one line past the bound, is {} bytes", + on_disk.len() + ); +} + +#[then("the CLI says the ComfyUI log is empty")] +async fn comfyui_says_empty(world: &mut E2eWorld) { + let out = ok_output(world); + assert!(out.contains("The saved log is empty."), "{out}"); + assert!(!out.contains("longer than"), "{out}"); + let on_disk = std::fs::read(comfyui_log_path(world)).expect("the ComfyUI log is still there"); + assert!(on_disk.is_empty(), "the log on disk should be empty"); +} + +#[then("the CLI says there is no output yet")] +async fn says_no_output_yet(world: &mut E2eWorld) { + let out = ok_output(world); + assert!(out.contains("Recent output: no output yet"), "{out}"); + assert!(!out.contains("longer than"), "{out}"); + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!(on_disk.is_empty(), "the log on disk should be empty"); +} + +/// The reply `rocm mcp-call service_logs` printed, parsed. Asserts the call +/// succeeded and reported no error first, so a failure reads as the CLI's own. +fn service_logs_tool_reply(world: &E2eWorld) -> serde_json::Value { + let out = ok_output(world); + let reply: serde_json::Value = serde_json::from_str(out.trim()) + .unwrap_or_else(|error| panic!("rocm mcp-call did not print JSON ({error}):\n{out}")); + assert_eq!( + reply["isError"], + serde_json::json!(false), + "the tool reported an error:\n{out}" + ); + reply +} + +#[then( + "the tool says the newest line is too long to show and flags it instead of returning an empty tail unexplained" +)] +async fn tool_says_the_line_is_too_long(world: &mut E2eWorld) { + let reply = service_logs_tool_reply(world); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert!( + text.contains("none is shown: the newest line is longer than the 4 MiB a log view reads"), + "expected the too-long notice, got:\n{reply}" + ); + assert!( + !text.contains("no log output yet") && !text.contains("Read the last"), + "a log with content must not be described as empty or as read:\n{reply}" + ); + let content = &reply["structuredContent"]; + assert_eq!(content["tail"], "", "{reply}"); + assert_eq!(content["newest_line_exceeds_read_bound"], true, "{reply}"); + // The claim and the state it describes: the log on disk holds exactly one + // line, longer than the bound the notice names. + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!( + on_disk.len() > LOG_VIEW_BOUND_BYTES && !on_disk.contains(&b'\n'), + "the log on disk should be one line past the bound, is {} bytes", + on_disk.len() + ); +} + +#[then("the tool reports the three lines it returned and the 50 lines it was asked for")] +async fn tool_reports_returned_and_requested_counts(world: &mut E2eWorld) { + let reply = service_logs_tool_reply(world); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert!( + text.starts_with("Read the last 3 line(s) from service"), + "{reply}" + ); + let content = &reply["structuredContent"]; + assert_eq!(content["lines"], 3, "{reply}"); + assert_eq!(content["lines_requested"], 50, "{reply}"); + // The claim and the state it describes: the log on disk holds exactly the + // three lines the reply returned. + let on_disk = + std::fs::read_to_string(server_log_path(world)).expect("the server log is still there"); + assert_eq!(on_disk.lines().count(), 3, "{on_disk}"); + assert_eq!(content["tail"], on_disk.trim_end_matches('\n'), "{reply}"); +} + +#[then("the tool says the server has no log output yet and does not flag an over-long line")] +async fn tool_says_no_output_yet(world: &mut E2eWorld) { + let reply = service_logs_tool_reply(world); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert!(text.contains("has no log output yet"), "{reply}"); + assert!(!text.contains("longer than"), "{reply}"); + let content = &reply["structuredContent"]; + assert_eq!(content["tail"], "", "{reply}"); + assert_eq!(content["newest_line_exceeds_read_bound"], false, "{reply}"); + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!(on_disk.is_empty(), "the log on disk should be empty"); +} + #[then("the CLI reports no matching lines")] async fn reports_no_matches(world: &mut E2eWorld) { let out = ok_output(world); diff --git a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs index b79c66ff17..03a871c4bc 100644 --- a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs @@ -16,6 +16,87 @@ async fn user_asks_for_verbose_catalog(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// Ask the `rocm_command` tool for the catalog, the way the dashboard's Chat +/// tab and any MCP client do. `mcp-call` is the CLI's own entry to that tool, +/// so the same dispatch the dash drives is exercised without a model or a TUI. +fn ask_the_tool_for_the_catalog(world: &mut E2eWorld, args: &[&str]) { + let arguments = serde_json::json!({ "args": args }).to_string(); + let (stdout, _, rc) = crate::run_rocm( + world, + &["mcp-call", "rocm_command", "--arguments-json", &arguments], + ); + world.cli_output = Some(stdout); + world.cli_rc = Some(rc); +} + +/// The catalog text inside the tool's reply, with the call's success asserted +/// first so a failure reads as the CLI's own rather than as a missing field. +fn catalog_text_from_the_tool(world: &E2eWorld) -> String { + let output = world.cli_output.as_ref().expect("no tool reply"); + assert_eq!(world.cli_rc, Some(0), "the tool call failed:\n{output}"); + let reply: serde_json::Value = serde_json::from_str(output.trim()) + .unwrap_or_else(|error| panic!("the tool reply is not JSON ({error}):\n{output}")); + assert_eq!(reply["isError"], serde_json::json!(false), "{reply}"); + reply["content"][0]["text"] + .as_str() + .unwrap_or_else(|| panic!("the tool reply carries no text:\n{output}")) + .to_owned() +} + +#[when("a tool-using client asks for the model catalog")] +async fn tool_client_asks_for_catalog(world: &mut E2eWorld) { + ask_the_tool_for_the_catalog(world, &["model"]); +} + +#[when("a tool-using client asks for the model catalog in verbose detail")] +async fn tool_client_asks_for_verbose_catalog(world: &mut E2eWorld) { + ask_the_tool_for_the_catalog(world, &["model", "--verbose"]); +} + +#[then("the client is given the recommended models without the per-recipe diagnostics")] +async fn tool_client_gets_the_recommendations(world: &mut E2eWorld) { + let text = catalog_text_from_the_tool(world); + // The same two lines the recommendation listing ends with for the built-in + // catalog, so this pins the curated renderer rather than merely "some + // text with a model id in it". + assert!( + text.contains( + "These are recommendations - you can serve any compatible Hugging Face model" + ) && text.contains("rocm serve "), + "expected the recommendation listing:\n{text}" + ); + // `aliases=[` opens every row of the verbose listing and appears nowhere + // in the curated one. + assert!( + !text.contains("aliases=["), + "the per-recipe diagnostics must not be the default:\n{text}" + ); +} + +#[then("what the client is given is small enough for a locally served model to read")] +async fn tool_catalog_is_small_enough_to_read(world: &mut E2eWorld) { + let text = catalog_text_from_the_tool(world); + // 16 KiB is the dash's per-result budget (`TOOL_RESULT_CHAR_BUDGET` in + // `crates/rocm-dash-tui/src/agent/tools.rs`), chosen to leave room in a + // 32k-token context for the system prompt and several turns. Asserted as + // a literal rather than imported so the e2e suite does not take a + // dependency on a dash internal; the listing measured under 2 KB. + assert!( + text.len() < 16 * 1024, + "the catalog handed to a tool client is {} characters, past the dash's budget:\n{text}", + text.len() + ); +} + +#[then("the client is given the per-recipe diagnostics")] +async fn tool_client_gets_the_diagnostics(world: &mut E2eWorld) { + let text = catalog_text_from_the_tool(world); + assert!( + text.contains("model recipes") && text.contains("aliases=["), + "expected the verbose per-recipe listing:\n{text}" + ); +} + #[then("the catalog lists an AIM engine recipe")] async fn catalog_lists_aim_recipe(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no model catalog output"); diff --git a/tests/e2e-cucumber/tests/e2e/planner_steps.rs b/tests/e2e-cucumber/tests/e2e/planner_steps.rs index 17ffe01299..eef456809c 100644 --- a/tests/e2e-cucumber/tests/e2e/planner_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/planner_steps.rs @@ -168,6 +168,30 @@ async fn planning_assistant_with_distinct_plans(world: &mut E2eWorld) { start_planning_assistant(world, vec![examine_plan_reply(), engines_list_plan_reply()]).await; } +/// The model the planning assistant names in an unsafe serve plan. The CLI's own +/// plan for the vague request names no model, so this appearing in the output +/// means the assistant's plan was shown. +const UNSAFE_PLAN_MODEL: &str = "sshleifer/tiny-gpt2"; + +#[given(expr = "a local planning assistant that suggests serving a model with {string}")] +async fn planning_assistant_suggests_serve_with(world: &mut E2eWorld, flags: String) { + let mut args = vec!["serve", UNSAFE_PLAN_MODEL]; + args.extend(flags.split_whitespace()); + let plan = json!({ + "intent": "serve", + "tool_call": {"tool": "rocm", "args": args} + }); + // One reply: a second planning request would be an unscripted HTTP 500. + start_planning_assistant( + world, + vec![chat_text_reply( + BUILTIN_ASSISTANT_MODEL_ID, + &plan.to_string(), + )], + ) + .await; +} + #[given( "a local chat assistant that hands a vague request to the planning assistant, which suggests a different plan each time it is asked" )] @@ -207,6 +231,15 @@ async fn user_approves_vague_request(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +#[when("the user asks the CLI to plan a vague request")] +async fn user_asks_cli_to_plan_vague_request(world: &mut E2eWorld) { + // No `--yes`: the CLI prints the plan and runs nothing. + let (stdout, stderr, rc) = crate::run_rocm(world, &[VAGUE_REQUEST]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("a tool-using client asks the CLI to plan a vague request")] async fn client_asks_for_a_plan(world: &mut E2eWorld) { // The planning tool is read-only, so no `--allow-mutation`. The exit code is @@ -308,6 +341,57 @@ fn assert_plan_text_is_the_first_answer(text: &str) { ); } +#[then(expr = "the CLI plans the request itself and says the planning assistant's plan {string}")] +async fn cli_plans_itself_and_says_why(world: &mut E2eWorld, reason: String) { + let out = cli_output(world); + let rc = world.cli_rc.expect("no command was run"); + assert_eq!( + rc, + 0, + "planning must still succeed with the CLI's own plan:\n{out}\n{}", + world.cli_stderr.as_deref().unwrap_or_default() + ); + // The behavior: the assistant's serve is not the plan shown, and nothing + // ran. The plan comes from the CLI's own planner alone (no provider in its + // name), names none of the assistant's model, and has no execution section. + assert!( + !out.contains(UNSAFE_PLAN_MODEL), + "the planning assistant's unsafe plan must not be shown:\n{out}" + ); + assert!( + out.lines() + .any(|line| line.trim() == "planner: hybrid-parser-v1"), + "the plan shown must be the CLI's own:\n{out}" + ); + assert!( + !out.contains("\nexecution\n"), + "nothing may be executed:\n{out}" + ); + // The same fact, stated positively so it cannot pass vacuously if the + // execution heading ever changes shape: the plan renderer indents everything + // under its one `request plan` heading, so a run that printed anything of its + // own (an `execution` section, a launch report, an error) would add another + // unindented line. + let sections: Vec<&str> = out + .lines() + .filter(|line| !line.trim().is_empty() && !line.starts_with(char::is_whitespace)) + .collect(); + assert_eq!( + sections, + ["request plan"], + "the output must be the plan alone, with nothing run after it:\n{out}" + ); + // The message, asserted together with the behavior above: it gives the + // rule's reason and names the assistant's plan as what was refused. + let note = format!( + "Provider-assisted planner `local` could not resolve this request: provider planner {reason}" + ); + assert!( + out.lines().any(|line| line.contains(¬e)), + "the CLI must say why the assistant's plan was not used (`{note}`):\n{out}" + ); +} + #[then("the plan shown is the one the planning assistant gave")] async fn plan_shown_is_the_assistants(world: &mut E2eWorld) { assert_plan_text_is_the_first_answer(cli_output(world)); diff --git a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs index c127245530..1838780c1e 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs @@ -273,9 +273,19 @@ fn active_runtime_key(world: &E2eWorld) -> String { /// Path to an importable manifest file (not yet in the registry) for the import /// scenario, with its install_root created so the import validates. +/// The folder the manifest to import points at: a runtime somebody else owns. +fn import_install_root(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("runtime-import") +} + fn write_import_manifest(world: &E2eWorld) -> PathBuf { let root = world.isolated_root.as_ref().expect("no isolated root"); - let install_root = root.path().join("runtime-import"); + let install_root = import_install_root(world); std::fs::create_dir_all(&install_root).expect("failed to create import install root"); std::fs::write(install_root.join("payload.txt"), "payload") .expect("failed to write import payload"); @@ -740,6 +750,27 @@ async fn imported_readonly(world: &mut E2eWorld) { ); } +/// What an import creates on disk: the registry entry, and nothing in the +/// folder it points at. The manifest ROCm CLI keeps in an install root is its +/// proof that it owns the folder, and an imported runtime is somebody else's. +/// Unit tests cover the same file set for adopt and setup recovery, which no +/// scenario reaches without a Python environment to probe. +#[then("the registry records the import and its folder gains no manifest file")] +async fn import_writes_registry_entry_only(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{IMPORT_KEY}.json")); + assert!( + entry.is_file(), + "expected a registry entry at {}", + entry.display() + ); + let sidecar = import_install_root(world).join(".rocm-cli-runtime.json"); + assert!( + !sidecar.exists(), + "an import must not write into the folder it registers: {}", + sidecar.display() + ); +} + #[then("the CLI refuses because it already exists")] async fn import_duplicate_refused(world: &mut E2eWorld) { let rc = world.cli_rc.expect("no command rc recorded"); diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 4150c9a679..d0c92f5f5d 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -5,6 +5,9 @@ use cucumber::{given, then, when}; use crate::E2eWorld; +use e2e_cucumber::torch_alignment_outcome::{ + EngineTorchRequirement, engine_import_verdict, is_settled, +}; #[given("a machine with no CLI-managed runtimes")] async fn setup_no_runtimes(world: &mut E2eWorld) { @@ -60,7 +63,7 @@ async fn user_previews_sdk_install(world: &mut E2eWorld) { // this to a plan: no venv, no multi-GiB download. let stdout = crate::run_rocm_ok( world, - &["install", "sdk", "--family", "gfx110X-all", "--dry-run"], + &["install", "sdk", "--family", "gfx1100", "--dry-run"], ); world.cli_output = Some(stdout); } @@ -239,14 +242,22 @@ async fn user_reinstalls_sdk_with_yes(world: &mut E2eWorld) { world.cli_output = Some(stdout); } -/// TheRock package families this step may ask for, in preference order. Real -/// published names, not placeholders: an unknown family is rejected during +/// TheRock GPU targets this step may ask for, in preference order, each paired +/// with the package family it resolves to. The install reads the newest layout, +/// which picks its device payload by exact arch and refuses a grouped family +/// name, so the request names the arch; the family is what the active runtime's +/// `family=` column is compared against. Real published names, not placeholders: +/// an unknown family is rejected during /// resolution, which would exit non-zero for a reason that has nothing to do /// with the consent gate and would still satisfy "the reinstall is refused". /// Each is published as a `therock-dist-linux--.tar.gz` in the -/// canonical release tarball catalog, for the same reason: an artifact the +/// newest layout's release tarball catalog, for the same reason: an artifact the /// catalog does not carry fails resolution short of the gate. -const OTHER_FAMILY_CANDIDATES: &[&str] = &["gfx110X-all", "gfx120X-all", "gfx94X-dcgpu"]; +const OTHER_FAMILY_CANDIDATES: &[(&str, &str)] = &[ + ("gfx1100", "gfx110X-all"), + ("gfx1200", "gfx120X-all"), + ("gfx942", "gfx94X-dcgpu"), +]; #[when("the user installs a different GPU family without confirming")] async fn user_installs_other_family_without_yes(world: &mut E2eWorld) { @@ -257,9 +268,9 @@ async fn user_installs_other_family_without_yes(world: &mut E2eWorld) { // case-preserving `family=` column — the runtime key alone would not do, // since it is lowercase-slugified and would never match `gfx110X-all`. let (runtimes, _, _) = crate::run_rocm(world, &["runtimes", "list"]); - let family = OTHER_FAMILY_CANDIDATES + let (arch, _) = OTHER_FAMILY_CANDIDATES .iter() - .find(|candidate| !runtimes.contains(*candidate)) + .find(|(_, family)| !runtimes.contains(*family)) .copied() .unwrap_or_else(|| { panic!("no candidate family differs from the installed runtimes:\n{runtimes}") @@ -277,7 +288,7 @@ async fn user_installs_other_family_without_yes(world: &mut E2eWorld) { // `active_default_runtime_relation` shared by both formats — with a family // the host has genuinely never held. The refusal still costs only the // catalog listing: it bails before the multi-GiB archive is fetched. - let args = ["install", "sdk", "--format", "tarball", "--family", family]; + let args = ["install", "sdk", "--format", "tarball", "--family", arch]; let (stdout, stderr, rc) = crate::run_rocm(world, &args); world.cli_output = Some(stdout); world.cli_stderr = Some(stderr); @@ -318,44 +329,96 @@ async fn assert_runtime_can_still_use_the_gpu(world: &mut E2eWorld) { /// reading the verdict is what tells the settled states apart from each other and /// from a skip. fn torch_alignment_verdict(output: &str) -> &str { - output - .lines() - .find_map(|line| line.trim().strip_prefix("torch_alignment: ")) - .unwrap_or_else(|| { - panic!("no torch alignment block, so the runtime was never settled:\n{output}") - }) + torch_alignment_lines(output)[0] .split_whitespace() .next() .unwrap_or_default() } -/// Verdicts that mean the runtime was settled and its torch left in a state this -/// tool produces on purpose. +/// The text after `torch_alignment: ` for every block in the output, reason +/// included. Never empty: an install that printed none never settled the runtime. +fn torch_alignment_lines(output: &str) -> Vec<&str> { + let lines: Vec<&str> = output + .lines() + .filter_map(|line| line.trim().strip_prefix("torch_alignment: ")) + .collect(); + assert!( + !lines.is_empty(), + "no torch alignment block, so the runtime was never settled:\n{output}" + ); + lines +} + +/// The vLLM requirement the verdicts are judged against, asked of the runtime only +/// when a `not_applicable` line makes it matter. Every other verdict is decided by +/// its own text, so a probe that cannot run is never reported as an alignment result. +fn engine_requirement_for(lines: &[&str], output: &str) -> EngineTorchRequirement { + if lines.iter().any(|line| line.starts_with("not_applicable")) { + engine_torch_requirement(output) + } else { + EngineTorchRequirement::Declared + } +} + +/// Whether the vLLM installed in the runtime that `output` describes requires +/// torch in its own package metadata. /// -/// Four, not one. Whether a reinstall rewrites torch or finds it already correct -/// depends on what the shared pre-warm tree held when the scenario started, and a -/// torch that has already executed a GPU kernel with this SDK is kept exactly as -/// it is and reported as `retained_*` — settled without anything being installed. -const SETTLED_TORCH_ALIGNMENTS: [&str; 4] = [ - "realigned", - "already_aligned", - "retained_sdk_build", - "retained_engine_build", -]; +/// Asked of the runtime's interpreter rather than read from the CLI's output, so +/// the answer does not depend on the thing being judged. The ROCm 10.x vLLM wheel +/// declares no torch and the engine install pins and realigns the torch trio +/// itself, which is the case where this is `Absent` and `not_applicable` is the +/// right verdict. +fn engine_torch_requirement(output: &str) -> EngineTorchRequirement { + let python = super::examine_steps::field_value(output, "python_executable") + .unwrap_or_else(|| panic!("the install output names no python_executable:\n{output}")); + let probe = std::process::Command::new(python) + .args([ + "-c", + "import importlib.metadata as m, re\n\ + names = [re.match(r'[A-Za-z0-9._-]+', r.split(';')[0].strip()) for r in (m.requires('vllm') or [])]\n\ + print(any(n and n.group(0).lower().replace('_', '-') == 'torch' for n in names))", + ]) + .output() + .unwrap_or_else(|error| panic!("failed to ask {python} for vllm's requirements: {error}")); + assert!( + probe.status.success(), + "asking {python} for vllm's requirements failed: {}", + String::from_utf8_lossy(&probe.stderr) + ); + if String::from_utf8_lossy(&probe.stdout).trim() == "True" { + EngineTorchRequirement::Declared + } else { + EngineTorchRequirement::Absent + } +} -/// Verdicts that mean the question was left unanswered. +/// The runtime's interpreter can import torch, torchvision, vLLM and its compiled +/// extensions together. /// -/// `not_applicable` is the one that would otherwise go unnoticed: it is what a -/// manifest yielding no SDK build produces, which is exactly the repair path for -/// every runtime installed before `sdk_torch` was recorded. `disabled` belongs -/// here for every scenario that did not ask for the opt-out — seeing it means the -/// opt-out was applied to a user who never set it. -const UNSETTLED_TORCH_ALIGNMENTS: [&str; 4] = [ - "torch_alignment: unavailable", - "torch_alignment: install_failed", - "torch_alignment: not_applicable", - "torch_alignment: disabled", -]; +/// Asked only where a `not_applicable` verdict means the alignment compared no pin, +/// so nothing else vouches that the pair torch and torchvision came to rest on can +/// register its operators. `vllm._C` and `vllm._rocm_C` are imported on their own +/// because `import vllm` loads those compiled extensions lazily: a vLLM built +/// against a different torch imports fine and only fails here, on an undefined +/// torch symbol. The CLI installs only ROCm builds, so both are always present. +/// The verdict itself is `engine_import_verdict`. +fn assert_engine_stack_imports(output: &str) { + let python = super::examine_steps::field_value(output, "python_executable") + .unwrap_or_else(|| panic!("the install output names no python_executable:\n{output}")); + let probe = std::process::Command::new(python) + .args([ + "-c", + "import torch, torchvision, vllm, vllm._C, vllm._rocm_C", + ]) + .output() + .unwrap_or_else(|error| panic!("failed to run {python}: {error}")); + if let Err(cause) = engine_import_verdict( + probe.status.success(), + &String::from_utf8_lossy(&probe.stderr), + ) { + panic!("the reinstalled runtime cannot import the stack vLLM needs ({cause}):\n{output}"); + } +} /// The alignment ran, and settled. /// @@ -367,19 +430,23 @@ const UNSETTLED_TORCH_ALIGNMENTS: [&str; 4] = [ #[then("the torch alignment settled rather than being skipped")] async fn assert_torch_alignment_settled(world: &mut E2eWorld) { let output = world.cli_output.as_deref().expect("no install output"); - let verdict = torch_alignment_verdict(output); - assert!( - SETTLED_TORCH_ALIGNMENTS.contains(&verdict), - "torch alignment reached no settled outcome (`{verdict}`):\n{output}" - ); - // Asserted negatively as well, because the check above reads only the first - // block and would pass on a second one that failed. - for unsettled in UNSETTLED_TORCH_ALIGNMENTS { + // Every block, not only the first: a second one that failed would otherwise + // pass behind a first that settled. `disabled` is unsettled here because it + // belongs only to the scenario that asked for the opt-out, and seeing it + // anywhere else means the opt-out was applied to a user who never set it. + // `not_applicable` is unsettled too, except over an engine that declares no + // torch requirement. + let lines = torch_alignment_lines(output); + let engine = engine_requirement_for(&lines, output); + for line in &lines { assert!( - !output.contains(unsettled), - "torch alignment reported `{unsettled}`:\n{output}" + is_settled(line, engine), + "torch alignment reached no settled outcome (`{line}`):\n{output}" ); } + if lines.iter().any(|line| line.starts_with("not_applicable")) { + assert_engine_stack_imports(output); + } // Asserted on the verdict, not on the divergence lines. Requiring // `expected_divergence` whenever `divergence:` appears cannot fail: one render // arm emits both, the verdict first. The property worth protecting is the other @@ -404,11 +471,17 @@ async fn assert_torch_alignment_settled(world: &mut E2eWorld) { /// the day, not of anything the scenario controls. Every other accepted verdict /// is one where torch was kept as it was, which is what the user asked for. /// -/// `not_applicable` is rejected for a reason of its own: it is the generic bucket -/// for "nothing to decide", and folding the opt-out into it leaves the user who -/// set the variable unable to tell whether it took effect. The reason line is -/// read too, so the block names the variable that caused the skip rather than -/// leaving the reader to guess which of several causes applied. +/// `not_applicable` is accepted only over an engine that declares no torch +/// requirement (the ROCm 10.x discover install, where the CLI pins torch itself): +/// there is no alignment to skip, so `disabled` cannot be printed. Any other +/// `not_applicable` is the generic bucket for "nothing to decide", and folding the +/// opt-out into it leaves the user who set the variable unable to tell whether it +/// took effect. In the accepted case the output has to name the variable, which +/// the `torch_kept:` line the next step requires does, so the reader is still told +/// it was honoured. On this route the alignment returns `not_applicable` before it +/// reads the opt-out, so the verdict is the same with or without the variable set +/// and this step does not exercise that gate: the `realigned` rejection above rules +/// out the one failure that matters, and the next step proves the torch was kept. #[then("the torch alignment reports the opt-out instead of rewriting torch")] async fn assert_torch_alignment_opted_out(world: &mut E2eWorld) { let output = world.cli_output.as_deref().expect("no install output"); @@ -418,12 +491,15 @@ async fn assert_torch_alignment_opted_out(world: &mut E2eWorld) { "torch was realigned even though the user opted out:\n{output}" ); // Every settled state except the rewrite, plus the skip the opt-out produces. - let kept_torch = verdict == "disabled" || SETTLED_TORCH_ALIGNMENTS.contains(&verdict); + let lines = torch_alignment_lines(output); + let line = lines[0]; + let kept_torch = + verdict == "disabled" || is_settled(line, engine_requirement_for(&lines, output)); assert!( kept_torch, - "the opt-out left torch in a state this tool does not produce (`{verdict}`):\n{output}" + "the opt-out left torch in a state this tool does not produce (`{line}`):\n{output}" ); - if verdict == "disabled" { + if verdict == "disabled" || verdict == "not_applicable" { assert!( output.contains("ROCM_CLI_DISABLE_TORCH_ALIGNMENT"), "the skipped alignment does not name the variable that skipped it:\n{output}" @@ -431,6 +507,56 @@ async fn assert_torch_alignment_opted_out(world: &mut E2eWorld) { } } +/// The SDK install did not write over the torch the runtime held. +/// +/// Two things have to agree, because either alone can be satisfied without the +/// other. The install says what it kept (`torch_kept:`, naming the torch version +/// it found before running `uv`), and the runtime's own interpreter, asked after +/// the install, reports that same version. The message by itself would certify the +/// string, and the version by itself would pass on a runtime whose torch was +/// rewritten to the same build. +#[then("the SDK install leaves the torch the runtime held as it was")] +async fn assert_sdk_install_kept_torch(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no install output"); + let kept_line = output + .lines() + .find(|line| line.trim_start().starts_with("torch_kept: ")) + .unwrap_or_else(|| { + panic!("the install did not say it kept the runtime's torch:\n{output}") + }); + let Some((_, after_torch)) = kept_line.split_once(" already holds torch ") else { + panic!("`torch_kept:` names no torch version: {kept_line}"); + }; + let kept_version = after_torch + .split([',', ';']) + .next() + .unwrap_or_default() + .trim(); + assert!( + !kept_version.is_empty(), + "`torch_kept:` names no torch version: {kept_line}" + ); + let python = super::examine_steps::field_value(output, "python_executable") + .unwrap_or_else(|| panic!("the install output names no python_executable:\n{output}")); + let probe = std::process::Command::new(python) + .args([ + "-c", + "import importlib.metadata as m; print(m.version('torch'))", + ]) + .output() + .unwrap_or_else(|error| panic!("failed to ask {python} for its torch: {error}")); + assert!( + probe.status.success(), + "asking {python} for its torch failed: {}", + String::from_utf8_lossy(&probe.stderr) + ); + let installed_version = String::from_utf8_lossy(&probe.stdout).trim().to_owned(); + assert_eq!( + installed_version, kept_version, + "the SDK install replaced the torch it said it kept:\n{output}" + ); +} + /// The kept torch is not sold to the user as a runtime to repair. /// /// Two surfaces print a repair for the same divergence and both have to be quiet @@ -534,6 +660,119 @@ async fn assert_lemonade_backend_alignment_reported(world: &mut E2eWorld) { ); } +/// The Lemonade install `rocm serve` consults before it serves a GGUF directly: +/// the selected backend and where its `llama-server` lives. +pub(crate) struct SelectedLemonadeBackend { + manifest: serde_json::Value, + backend: String, + runtime_dir: std::path::PathBuf, +} + +impl SelectedLemonadeBackend { + /// Read the manifest the install wrote: the runtime directory and backend are + /// what the engine's own lookup (`/bin/llamacpp//`) uses. + pub(crate) fn read(world: &E2eWorld) -> Self { + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated state root") + .path(); + let manifest_path = root.join("data/engines/lemonade/manifests/runtime.json"); + let manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).unwrap_or_else(|error| { + panic!("failed to read {}: {error}", manifest_path.display()) + })) + .unwrap_or_else(|error| panic!("failed to parse {}: {error}", manifest_path.display())); + let field = |name: &str| { + manifest[name] + .as_str() + .unwrap_or_else(|| panic!("no `{name}` in {}: {manifest}", manifest_path.display())) + .to_owned() + }; + let (recipe, backend, runtime_dir) = ( + field("backend_recipe"), + field("backend_name"), + std::path::PathBuf::from(field("runtime_dir")), + ); + assert_eq!( + recipe, "llamacpp", + "the selected recipe is not llama.cpp, so direct serving has no llama-server to find: {manifest}" + ); + Self { + manifest, + backend, + runtime_dir, + } + } + + /// The backend Lemonade selected (`rocm` or `vulkan`), as the manifest records it. + pub(crate) fn backend(&self) -> &str { + &self.backend + } + + /// The selected backend's `llama-server`, or the directories searched. + fn server(&self) -> Result { + let lookup = + e2e_cucumber::lemonade_backend::find_llama_server(&self.runtime_dir, &self.backend) + .unwrap_or_else(|reason| panic!("{reason}: {}", self.manifest)); + lookup.found.ok_or_else(|| { + format!( + "no llama-server exists for the selected backend `{}` under {}. Searched (flat, or one build-numbered level down):{}", + self.backend, + self.runtime_dir.display(), + lookup + .searched + .iter() + .fold(String::new(), |mut listing, dir| { + use std::fmt::Write as _; + let _ = write!(listing, "\n {}", dir.display()); + listing + }) + ) + }) + } +} + +/// A precondition for the reinstall advice: `rocm serve` tells a user whose +/// selected backend has no `llama-server` to run +/// `rocm engines install lemonade --reinstall`. That advice only holds if the +/// command brings the file back, so the scenario first installs Lemonade (the +/// plain install, so the later `--reinstall` is what has to repair it). +#[given("the lemonade engine is installed")] +async fn setup_lemonade_engine_installed(world: &mut E2eWorld) { + crate::run_rocm_ok(world, &["engines", "install", "lemonade"]); +} + +/// Damage the install the way the advice is for: delete the selected backend's +/// `llama-server`. Asserts it was there first and is gone after, so the Then that +/// follows a reinstall can only pass if the reinstall restored it. +#[given("the selected Lemonade backend's llama-server binary has been deleted")] +async fn delete_lemonade_backend_server(world: &mut E2eWorld) { + let selected = SelectedLemonadeBackend::read(world); + let binary = selected + .server() + .unwrap_or_else(|reason| panic!("nothing to delete after the install: {reason}")); + std::fs::remove_file(&binary) + .unwrap_or_else(|error| panic!("failed to delete {}: {error}", binary.display())); + assert!( + selected.server().is_err(), + "a llama-server still exists for `{}` after deleting {}", + selected.backend, + binary.display() + ); +} + +/// Checks the file the direct serve route looks for exists. In runtime-16 it runs +/// after the file was deleted, so it is the reinstall that put it back; the +/// lookup is [`e2e_cucumber::lemonade_backend::find_llama_server`]. +#[then("the selected Lemonade backend's llama-server binary is installed")] +async fn assert_lemonade_backend_server_installed(world: &mut E2eWorld) { + let selected = SelectedLemonadeBackend::read(world); + if let Err(searched) = selected.server() { + panic!("after the reinstall, {searched}"); + } +} + /// The CLI names the variable whenever it is set, so a user can tell it was /// read -- this alone does not prove alignment was actually skipped. /// @@ -659,6 +898,9 @@ fn assert_engine_ready(world: &mut E2eWorld) { } #[when("the user dry-runs a nightly SDK install for a known family")] async fn user_dry_runs_nightly_sdk(world: &mut E2eWorld) { + // An exact arch, not the `gfx120X-all` group: an unpinned install reads the + // newest layout, which picks its device payload by exact arch and refuses a + // group. let stdout = crate::run_rocm_ok( world, &[ @@ -667,7 +909,34 @@ async fn user_dry_runs_nightly_sdk(world: &mut E2eWorld) { "--channel", "nightly", "--family", - "gfx120X-all", + "gfx1200", + "--dry-run", + ], + ); + world.cli_output = Some(stdout); +} + +/// The last day the earlier (canonical) nightly stream published a build is +/// 2026-08-22, so a pin before it can only be read from there. +/// +/// The pin is 2026-08-19, not the 20th: the stream has no native Windows +/// (`win_amd64`) torch, torchvision or torchaudio wheel for 2026-08-20, only +/// Linux ones, so on a Windows host that pin finds no mutually compatible +/// stack. The 19th has all four packages on both platforms for cp314, the +/// Python this ROCm layout installs. +#[when("the user dry-runs a nightly SDK install pinned to a build from the earlier stream")] +async fn user_dry_runs_earlier_nightly_sdk(world: &mut E2eWorld) { + let stdout = crate::run_rocm_ok( + world, + &[ + "install", + "sdk", + "--channel", + "nightly", + "--family", + "gfx1200", + "--build-date", + "2026-08-19", "--dry-run", ], ); @@ -706,7 +975,23 @@ fn requested_rocm_extras(rocm_spec: &str) -> &str { #[then("the SDK preview reports canonical nightly provenance")] async fn assert_canonical_nightly_provenance(world: &mut E2eWorld) { - const AGGREGATE: &str = "https://rocm.nightlies.amd.com/whl-multi-arch"; + assert_nightly_provenance( + world, + "https://rocm.nightlies.amd.com/whl-multi-arch", + "multi-arch-v2", + ); +} + +#[then("the SDK preview reports next-layout nightly provenance")] +async fn assert_next_nightly_provenance(world: &mut E2eWorld) { + assert_nightly_provenance( + world, + "https://nightly.repo.amd.com/rocm/whl-next", + "next-v1", + ); +} + +fn assert_nightly_provenance(world: &E2eWorld, aggregate: &str, generation: &str) { let output = world.cli_output.as_deref().expect("no SDK preview output"); assert_eq!( @@ -720,17 +1005,17 @@ async fn assert_canonical_nightly_provenance(world: &mut E2eWorld) { // disagrees with it is worse than either alone. assert_eq!( preview_field(output, "canonical_source"), - AGGREGATE, - "the preview declares a source other than the canonical nightly aggregate:\n{output}" + aggregate, + "the preview declares a source other than the expected nightly aggregate:\n{output}" ); assert_eq!( preview_field(output, "index_url"), - AGGREGATE, - "the nightly install resolved something other than the canonical aggregate:\n{output}" + aggregate, + "the nightly install resolved something other than the expected aggregate:\n{output}" ); assert_eq!( preview_field(output, "source_layout_generation"), - "multi-arch-v2", + generation, "the preview read the aggregate as some other layout:\n{output}" ); // A `selected_rocm_version` line is provenance only if it names the version @@ -758,9 +1043,44 @@ async fn user_dry_runs_release_sdk(world: &mut E2eWorld) { world.cli_output = Some(stdout); } +#[when("the user dry-runs a release SDK install pinned to an earlier ROCm 7 release")] +async fn user_dry_runs_earlier_release_sdk(world: &mut E2eWorld) { + // Same as the unpinned preview above, with the pin the only difference, so + // the two scenarios differ in exactly the thing that picks the stream. + let stdout = crate::run_rocm_ok( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--version", + "7.14.1", + "--dry-run", + ], + ); + world.cli_output = Some(stdout); +} + #[then("the SDK preview reports canonical release provenance")] async fn assert_canonical_release_provenance(world: &mut E2eWorld) { - const AGGREGATE: &str = "https://repo.amd.com/rocm/whl-multi-arch"; + assert_release_provenance( + world, + "https://repo.amd.com/rocm/whl-multi-arch", + "multi-arch-v2", + ); +} + +#[then("the SDK preview reports next-layout release provenance")] +async fn assert_next_release_provenance(world: &mut E2eWorld) { + assert_release_provenance( + world, + "https://stable.repo.amd.com/rocm/whl-next", + "next-v1", + ); +} + +fn assert_release_provenance(world: &E2eWorld, aggregate: &str, generation: &str) { let output = world.cli_output.as_deref().expect("no SDK preview output"); assert_eq!( @@ -774,17 +1094,17 @@ async fn assert_canonical_release_provenance(world: &mut E2eWorld) { // that says so — the URL the resolver chose, not the one it advertises. assert_eq!( preview_field(output, "canonical_source"), - AGGREGATE, - "the preview declares a source other than the canonical release aggregate:\n{output}" + aggregate, + "the preview declares a source other than the expected release aggregate:\n{output}" ); assert_eq!( preview_field(output, "index_url"), - AGGREGATE, - "the release install resolved something other than the flat aggregate index:\n{output}" + aggregate, + "the release install resolved something other than the expected aggregate index:\n{output}" ); assert_eq!( preview_field(output, "source_layout_generation"), - "multi-arch-v2", + generation, "the preview read the aggregate as some other layout:\n{output}" ); // The release channel installs stable versions only, and a stable version @@ -1515,3 +1835,66 @@ async fn assert_freeform_execution_prefix_is_one_word(world: &mut E2eWorld) { &freeform_prefix_with_quote(world), ); } + +/// Where scenario runtime-26 plants its interpreter: inside a folder called `tools`, +/// which is neither `bin` nor `Scripts`, so nothing about its location names an +/// environment. +fn interpreter_outside_a_venv(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("not-a-venv") + .join("tools") + .join("python") +} + +/// A stand-in interpreter that answers the SDK probe with the smallest document the +/// CLI parses. The refusal under test happens after the probe and before anything is +/// read from the environment, so nothing else about the interpreter matters. +/// +/// A POSIX shell script, so the scenario is Linux-only (`@requires-os:linux`). +#[given("a Python interpreter that sits outside any environment's bin folder")] +async fn setup_interpreter_outside_a_venv(world: &mut E2eWorld) { + let python = interpreter_outside_a_venv(world); + std::fs::create_dir_all(python.parent().expect("interpreter has a folder")) + .expect("failed to create the interpreter's folder"); + rocm_core::fake_executable::write_fake_executable( + &python, + "#!/bin/sh\necho '{\"import_ok\": true}'\n", + ) + .expect("failed to write the stand-in interpreter"); +} + +#[when("the user adopts that interpreter without naming an environment folder")] +async fn user_adopts_interpreter_without_root(world: &mut E2eWorld) { + let python = interpreter_outside_a_venv(world) + .to_string_lossy() + .into_owned(); + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "adopt", "--python", &python]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the refusal asks for the environment folder with --root")] +async fn assert_adopt_refusal_asks_for_root(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or(""); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + let combined = format!("{stdout}{stderr}"); + assert!( + combined.contains("could not infer the Python environment folder") + && combined.contains("pass --root"), + "the refusal does not ask for --root:\n{combined}" + ); +} + +#[then("no runtime is registered")] +async fn assert_no_runtime_registered(world: &mut E2eWorld) { + let (stdout, _, _) = crate::run_rocm(world, &["runtimes", "list"]); + assert!( + stdout.contains("installed: none") || stdout.contains("managed_runtimes: 0"), + "the refused adoption left a runtime registered:\n{stdout}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs index e1bdddbfb3..375a3dfa39 100644 --- a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs @@ -18,7 +18,7 @@ use cucumber::{given, then, when}; use crate::E2eWorld; /// Id of the record every scenario here plants. -const SERVICE_ID: &str = "vllm-e2e-cleanup"; +pub(crate) const SERVICE_ID: &str = "vllm-e2e-cleanup"; /// Engine name, which also picks the engine state directory the record's third /// file lives in (`/engines/vllm/state/`). const ENGINE: &str = "vllm"; @@ -62,7 +62,7 @@ fn data_dir(world: &E2eWorld) -> PathBuf { .join("data") } -fn services_dir(world: &E2eWorld) -> PathBuf { +pub(crate) fn services_dir(world: &E2eWorld) -> PathBuf { data_dir(world).join("services") } @@ -96,7 +96,7 @@ fn record_files(world: &E2eWorld) -> Vec { /// /// The engine state file must agree with `status`: the CLI adopts whatever that /// file says before it decides anything else. -fn plant_record(world: &E2eWorld, status: &str) { +pub(crate) fn plant_record(world: &E2eWorld, status: &str) { let services = services_dir(world); let states = engine_state_dir(world); std::fs::create_dir_all(&services).expect("failed to create services dir"); @@ -696,3 +696,201 @@ fn assert_succeeded(world: &E2eWorld) { combined_output(world) ); } + +// ── A record held by a process `rocm` cannot query ───────────────── + +/// A PID no host is using, which the `e2e-test-hooks` build is told to read as +/// "cannot be queried" (`ROCM_E2E_UNQUERYABLE_PIDS`). +const UNQUERYABLE_PID: u32 = 2_000_000_001; + +/// Rewrite fields of the planted manifest. +fn set_manifest_fields(world: &E2eWorld, fields: &[(&str, serde_json::Value)]) { + let manifest = services_dir(world).join(format!("{SERVICE_ID}.json")); + let mut record: serde_json::Value = serde_json::from_slice( + &std::fs::read(&manifest).expect("failed to read the planted manifest"), + ) + .expect("failed to parse the planted manifest"); + for (key, value) in fields { + record[*key] = value.clone(); + } + std::fs::write( + &manifest, + serde_json::to_vec_pretty(&record).expect("failed to serialize record"), + ) + .expect("failed to rewrite the planted manifest"); +} + +/// Make the next `rocm` invocation read [`UNQUERYABLE_PID`] as unqueryable. +fn script_unqueryable(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_UNQUERYABLE_PIDS", + UNQUERYABLE_PID.to_string().into(), + )); +} + +#[given("a local server record whose recorded process cannot be queried")] +async fn record_with_unqueryable_process(world: &mut E2eWorld) { + plant_record(world, "ready"); + set_manifest_fields( + world, + &[("supervisor_pid", serde_json::json!(UNQUERYABLE_PID))], + ); +} + +#[given("a local server record whose recorded process is running and whose stop was requested")] +async fn record_with_running_process(world: &mut E2eWorld) { + plant_record(world, "ready"); + // A real process the scenario owns, so the teardown (which stops every + // record left behind) ends it rather than the harness. + let child = std::process::Command::new("sleep") + .arg("300") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("failed to start the recorded process"); + set_manifest_fields( + world, + &[ + ("supervisor_pid", serde_json::json!(child.id())), + ("stop_requested_unix_ms", serde_json::json!(1)), + ], + ); + world.recorded_process = Some(child); +} + +#[when("the user stops that local server while its process cannot be queried")] +async fn stop_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "stop", SERVICE_ID, "--yes"]); + record(world, stdout, stderr, rc); +} + +#[when("the user removes that local server record while its process cannot be queried")] +async fn remove_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "remove", SERVICE_ID, "--yes"]); + record(world, stdout, stderr, rc); +} + +#[when("the user force-removes that local server record while its process cannot be queried")] +async fn force_remove_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &["services", "remove", SERVICE_ID, "--force", "--yes"], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user prunes every record whatever its age while its process cannot be queried")] +async fn prune_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "prune", "--any-age", "--yes"]); + record(world, stdout, stderr, rc); +} + +#[then("the CLI leaves the record alone and names the forced removal")] +async fn prune_leaves_and_names_force(world: &mut E2eWorld) { + assert_succeeded(world); + let combined = combined_output(world); + assert!( + combined.contains(&format!( + "{SERVICE_ID} is " + )) && combined.contains(&format!( + "process {UNQUERYABLE_PID} it recorded could not be queried - remove it with `rocm services remove {SERVICE_ID} --force --yes`" + )), + "{combined}" + ); +} + +#[when("the user force-removes that local server record")] +async fn force_remove(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["services", "remove", SERVICE_ID, "--force", "--yes"], + ); + record(world, stdout, stderr, rc); +} + +#[then("the CLI reports the stop as not confirmed because the process could not be queried")] +async fn stop_not_confirmed_unverified(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "an unverified stop must exit non-zero:\n{combined}"); + assert!( + combined.contains("Local server stop not confirmed"), + "{combined}" + ); + assert!( + combined.contains(&format!("not signaled: process {UNQUERYABLE_PID}")), + "{combined}" + ); + // The check the stop names is the host's own command, and names the PID. + assert!( + combined.contains(&format!("ps -p {UNQUERYABLE_PID} -o pid,args")) + || combined.contains(&format!("tasklist /FI \"PID eq {UNQUERYABLE_PID}\"")), + "{combined}" + ); + // What the stop says it kept is what is on disk: the record names the + // process and the stop request is recorded. + let manifest: serde_json::Value = serde_json::from_slice( + &std::fs::read(services_dir(world).join(format!("{SERVICE_ID}.json"))) + .expect("the record must still be there"), + ) + .expect("the record must parse"); + assert_eq!(manifest["supervisor_pid"], UNQUERYABLE_PID, "{manifest}"); + assert!(manifest["stop_requested_unix_ms"].is_u64(), "{manifest}"); +} + +#[then("the CLI refuses the removal and names the forced removal")] +async fn refuses_and_names_force(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "{combined}"); + assert!( + combined.contains(&format!( + "process {UNQUERYABLE_PID} it recorded could not be queried" + )), + "{combined}" + ); + // The command the refusal names is the one the next step runs. + assert!( + combined.contains(&format!("rocm services remove {SERVICE_ID} --force --yes")), + "{combined}" + ); +} + +#[then("the CLI removes the record and says its process could not be queried")] +async fn force_removed(world: &mut E2eWorld) { + assert_succeeded(world); + let combined = combined_output(world); + assert!( + combined.contains(&format!( + "--force: process {UNQUERYABLE_PID} could not be queried" + )), + "{combined}" + ); + assert!( + combined.contains("Local server record removed"), + "{combined}" + ); +} + +#[then("the CLI refuses the forced removal because the record does not qualify")] +async fn refuses_force(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "{combined}"); + assert!( + combined.contains("--force removes a record only when"), + "{combined}" + ); + assert!( + combined.contains(&format!("rocm services stop {SERVICE_ID} --yes")), + "{combined}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 31df329d2b..024f0b5fee 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -166,9 +166,171 @@ fn plant_server_with_running_process(world: &mut E2eWorld, host: &str, with_key: .stderr(Stdio::null()) .spawn() .expect("failed to start the process to record"); + let ticks = start_ticks(process.id()); + plant_server_record(world, host, with_key, process, Some(ticks)); +} + +/// A ready, publicly bound server whose recorded PID is a live process that is +/// not the server: what a recycled PID looks like. The record's start-time token +/// is not the process's own, so the stop has to refuse the PID rather than signal +/// it. The token is the process's real one offset by one on Linux, and `1` (a +/// `FILETIME` in 1601, which no live process has) on Windows. +#[given("a managed server whose recorded process id now belongs to a different process")] +async fn managed_server_with_a_recycled_pid(world: &mut E2eWorld) { + let (program, args): (&str, &[&str]) = if cfg!(windows) { + ("ping", &["-n", "300", "127.0.0.1"]) + } else { + ("sleep", &["300"]) + }; + let bystander = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process that holds the recorded PID"); + let stale_ticks = if cfg!(windows) { + 1 + } else { + start_ticks(bystander.id()).wrapping_add(1) + }; + plant_server_record(world, "0.0.0.0", true, bystander, Some(stale_ticks)); +} + +/// The stop's promise about the process that holds the recorded PID, checked on +/// the process: it was not signalled, so it is still running. +#[then("the process now holding the recorded process id is still running")] +async fn process_holding_the_pid_still_runs(world: &mut E2eWorld) { + let process = world + .recorded_process + .as_mut() + .expect("scenario recorded no process"); + assert!( + process + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "the stop signalled a process that is not the recorded server" + ); +} + +/// Where the parent the Windows scenario records writes the PID of the process +/// it started, for the scenario to read. +fn descendant_pid_path(world: &E2eWorld) -> PathBuf { + services_dir(world).join("descendant.pid") +} + +/// A ready, publicly bound server whose recorded supervisor is a parent process +/// that has started a child of its own, recorded the way a Windows record was +/// before Windows recorded a creation time: the PID alone, with no start-time +/// token. (A record written now carries one; this scenario plants the tokenless +/// shape on purpose, the one a PID cannot be refuted by.) +/// +/// The parent is PowerShell, the child `ping` for five minutes: both ship with +/// Windows, need nothing installed, and are disposable. PowerShell starts the +/// child itself, so the child's parent is the recorded PID, which is the +/// relationship `taskkill /T` follows; it writes the child's PID where the +/// scenario can read it and then stays alive. Both outlive the scenario only if +/// the stop fails to take them, and the World's teardown reaps the parent. +#[given("a managed server whose recorded process has started another process")] +async fn managed_server_whose_process_started_another(world: &mut E2eWorld) { + let services = services_dir(world); + std::fs::create_dir_all(&services).expect("failed to create services dir"); + let pid_file = descendant_pid_path(world); + let process = spawn_parent_with_descendant(&pid_file); + let pid = process.id(); + plant_server_record(world, "0.0.0.0", true, process, None); + + // The premise, so the Then step cannot pass for the wrong reason: the + // descendant exists and is running, and so is the parent. + let descendant = wait_for_descendant_pid(&pid_file); + // Kept first, with its creation time where the host reads one, so teardown + // can end the child on its own even when the stop (or a step) fails and the + // parent is gone: a later stop cannot recover a child from a dead parent. + world.recorded_descendant = Some(rocm_core::ProcessIdentity::capture(descendant)); + assert_ne!(descendant, pid, "the child must be a different process"); + assert!( + windows_process_is_running(descendant), + "premise: the process the recorded one started must be running" + ); + assert!( + world + .recorded_process + .as_mut() + .expect("scenario recorded no process") + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "premise: the recorded process must be running" + ); +} + +fn spawn_parent_with_descendant(pid_file: &std::path::Path) -> std::process::Child { + // A single-quoted PowerShell string: an apostrophe in the path is doubled. + let pid_file = pid_file.display().to_string().replace('\'', "''"); + let script = format!( + "$child = Start-Process -FilePath ping.exe -ArgumentList '-n','300','127.0.0.1' \ + -WindowStyle Hidden -PassThru; \ + Set-Content -LiteralPath '{pid_file}' -Value $child.Id; \ + Start-Sleep -Seconds 300" + ); + Command::new("powershell") + .args(["-NoProfile", "-NonInteractive", "-Command", &script]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process to record") +} + +/// The child's PID, once the parent has written it. +fn wait_for_descendant_pid(pid_file: &std::path::Path) -> u32 { + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); + loop { + if let Some(pid) = std::fs::read_to_string(pid_file) + .ok() + .and_then(|text| text.trim().parse().ok()) + { + return pid; + } + assert!( + std::time::Instant::now() < deadline, + "the recorded process never wrote the PID of the process it started" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } +} + +/// Whether Windows has a process `pid`, by `tasklist`: a match prints a CSV +/// row naming it, and no match prints an INFO line without one. A `tasklist` +/// that fails has said nothing about the process, so the scenario fails rather +/// than read it as "exited" (`e2e_cucumber::tasklist`). +fn windows_process_is_running(pid: u32) -> bool { + let output = Command::new("tasklist") + .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) + .output() + .expect("failed to run tasklist"); + e2e_cucumber::tasklist::tasklist_reports_running( + output.status.success(), + &String::from_utf8_lossy(&output.stdout), + &String::from_utf8_lossy(&output.stderr), + pid, + ) + .unwrap_or_else(|message| panic!("{message}")) +} + +/// Write the record for a ready server whose supervisor is `process`, with +/// `ticks` as its start-time token when there is one, and take `process` into +/// the World so the scenario (and its teardown) can check and reap it. +fn plant_server_record( + world: &mut E2eWorld, + host: &str, + with_key: bool, + process: std::process::Child, + ticks: Option, +) { let pid = process.id(); world.recorded_process = Some(process); - let ticks = start_ticks(pid); let services = services_dir(world); std::fs::create_dir_all(&services).expect("failed to create services dir"); @@ -222,6 +384,72 @@ async fn next_stop_unverified(world: &mut E2eWorld) { script_next_stop_outcome(world, "unverified"); } +// ── Given: an engine child the scenario controls ──────────────────── + +/// The standalone daemon's file name: `rocmd`, with the platform's suffix. +const STANDALONE_ROCMD: &str = if cfg!(windows) { "rocmd.exe" } else { "rocmd" }; +/// What the stand-in engine launcher records, one line per time it is run. +const ENGINE_INVOCATIONS: &str = "engine-invocations"; +/// What the stand-in engine launcher read on its stdin, over every run. +const ENGINE_REQUESTS: &str = "engine-requests"; + +/// The directory holding the standalone daemon copy and the stand-in `rocm`. +fn standalone_dir(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("standalone-daemon") +} + +/// Plant a copy of the real `rocmd` with a stand-in `rocm` beside it. +/// +/// A `rocmd` named exactly that reaches an engine through the `rocm` next to it, +/// so the engine child the daemon runs for its `Stop` request is `behaviour`: a +/// shell script that first records its argv and what it read on stdin (into +/// [`ENGINE_INVOCATIONS`] and [`ENGINE_REQUESTS`] beside it), then does what the +/// scenario says. The copy is a plain file, not a link, because the daemon looks +/// for its sibling next to where its executable really is. +fn install_standalone_daemon(world: &E2eWorld, behaviour: &str) { + let dir = standalone_dir(world); + std::fs::create_dir_all(&dir).expect("failed to create the standalone daemon dir"); + let rocmd = std::fs::read(crate::rocmd_binary()).expect("failed to read rocmd"); + rocm_core::fake_executable::write_fake_executable(&dir.join(STANDALONE_ROCMD), rocmd) + .expect("failed to plant the standalone daemon"); + let script = format!( + "#!/bin/sh\n\ + dir=$(dirname \"$0\")\n\ + echo \"$*\" >> \"$dir/{ENGINE_INVOCATIONS}\"\n\ + cat >> \"$dir/{ENGINE_REQUESTS}\"\n\ + {behaviour}\n" + ); + rocm_core::fake_executable::write_fake_executable(&dir.join("rocm"), script) + .expect("failed to plant the stand-in rocm"); +} + +#[given("the engine answers the stop request and then exits with a failing status")] +async fn engine_answers_then_fails(world: &mut E2eWorld) { + install_standalone_daemon( + world, + "echo '{\"ok\":true,\"data\":{\"stopped\":true,\"graceful\":true}}'\n\ + echo 'stand-in engine: exiting with a failing status after answering' >&2\n\ + exit 3", + ); +} + +#[given( + "the engine prints something that is not an answer, complains on stderr and exits with a failing status" +)] +async fn engine_prints_garbage_and_fails(world: &mut E2eWorld) { + install_standalone_daemon( + world, + "printf 'this is not an answer'\n\ + echo 'engine exploded: no container runtime' >&2\n\ + exit 1", + ); +} + // ── When ─────────────────────────────────────────────────────────── /// Runs the pinned, directly-runnable form `serve_summary.rs`'s `stop` row @@ -239,7 +467,25 @@ async fn stop_with_yes(world: &mut E2eWorld) { #[when("the daemon's stop_server tool stops that server")] async fn stop_server_tool(world: &mut E2eWorld) { - let mut command = Command::new(crate::rocmd_binary()); + run_stop_server_tool(world, crate::rocmd_binary()); +} + +/// The same tool, run from the copy of `rocmd` that has a stand-in `rocm` beside +/// it (see [`install_standalone_daemon`]), so the engine's `Stop` request reaches +/// a child the scenario controls. +#[when("the standalone daemon's stop_server tool stops that server")] +async fn standalone_stop_server_tool(world: &mut E2eWorld) { + let rocmd = standalone_dir(world).join(STANDALONE_ROCMD); + assert!( + rocmd.is_file(), + "no standalone daemon was installed: {}", + rocmd.display() + ); + run_stop_server_tool(world, rocmd); +} + +fn run_stop_server_tool(world: &mut E2eWorld, rocmd: PathBuf) { + let mut command = Command::new(rocmd); command.args(["sandbox-tool", "stop_server", "--service-id", SERVICE_ID]); world.isolate_cmd(&mut command); let output = command.output().expect("failed to run rocmd"); @@ -250,13 +496,13 @@ async fn stop_server_tool(world: &mut E2eWorld) { // ── Then ─────────────────────────────────────────────────────────── -// `render_service_action_result` only prints the `service:`/`status:`/ -// `endpoint:` lines when `output.result` carries a nested `service` object -// (as `restart_server` does); `stop_server`'s result is the flat -// `service_id`/`status` shape `stop_internal_managed_service` returns, so none -// of those lines ever print for a stop today. That mismatch is a pre-existing -// gap in that renderer, so this assertion covers only what the command -// actually prints; the on-disk status below is what proves the stop itself. +// `render_service_action_result` prints the `service:` and `status:` lines for +// `stop_server`'s flat `service_id`/`status` result as well as for the nested +// `service` object `restart_server` returns (unit-tested by +// `service_stop_render_prints_the_in_process_stop_report_shape`); `endpoint:` +// still prints only for the nested shape. This step asserts the headline and the +// two lines naming the service and its status; the on-disk status below is what +// proves the stop itself. #[then("the CLI reports the service as stopped")] async fn reports_stopped(world: &mut E2eWorld) { assert_succeeded(world); @@ -265,6 +511,15 @@ async fn reports_stopped(world: &mut E2eWorld) { combined.contains("Local server stopped"), "the stop command must say it stopped the server, got:\n{combined}" ); + let has_line = |wanted: String| combined.lines().any(|line| line.trim() == wanted); + assert!( + has_line(format!("service: {SERVICE_ID}")), + "the stop report must name the service it stopped, got:\n{combined}" + ); + assert!( + has_line("status: stopped".to_owned()), + "the stop report must give the status the service ended in, got:\n{combined}" + ); } #[then("the service record on disk is marked stopped")] @@ -311,21 +566,176 @@ async fn tool_reports_stopped(world: &mut E2eWorld) { ); } +/// The standalone `rocmd` reaches an engine through the `rocm` beside it (it has +/// no engine verb of its own), so its `Stop` request is answered rather than +/// failing with the standalone binary's "unrecognized subcommand". The report +/// carries the engine's answer or the error that took its place. +/// +/// The answer is `stopped: false`, and that is the right one here, not a gap: the +/// planted record names no engine state with a process id, so the real `vllm` +/// engine has nothing it can confirm stopping (`stop_service` answers `false` +/// when no PID is recorded). The recorded supervisor, which the daemon itself +/// signals, is what the other steps check. Asserting the whole answer, not just +/// that it is a boolean, is what tells it from an engine that was never reached. +#[then("the tool asked the engine to stop and the engine answered")] +async fn engine_answered_the_stop(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + let report: serde_json::Value = + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!(engine_stop["attempted"], true, "{report}"); + assert!( + engine_stop.get("error").is_none(), + "the engine's answer was an error instead: {report}" + ); + assert_eq!( + *engine_stop, + serde_json::json!({"attempted": true, "stopped": false, "graceful": false}), + "{report}" + ); +} + +/// The stop report `rocmd sandbox-tool stop_server` printed, with the tool's own +/// exit status checked first: both engine endings below leave the stop itself +/// succeeding. +fn tool_report(world: &E2eWorld) -> serde_json::Value { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "rocmd failed:\n{stdout}\n{}", + world.cli_stderr.as_deref().unwrap_or_default() + ); + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")) +} + +/// The stand-in engine was run once, as `__engine-stdio vllm`, and was sent one +/// line: an engine `stop` request for the planted server. Reads what the +/// stand-in itself recorded, so it holds for the child the daemon ran, not for +/// what the daemon says it ran. +#[then("the engine received one stop request for that server over stdio")] +async fn engine_received_one_stop_request(world: &mut E2eWorld) { + let dir = standalone_dir(world); + let invocations = std::fs::read_to_string(dir.join(ENGINE_INVOCATIONS)) + .expect("the stand-in engine was never run"); + assert_eq!( + invocations, + format!("__engine-stdio {ENGINE}\n"), + "the daemon must run the rocm beside it once, as `__engine-stdio `" + ); + let requests = std::fs::read_to_string(dir.join(ENGINE_REQUESTS)) + .expect("the stand-in engine recorded no request"); + assert_eq!( + requests.lines().count(), + 1, + "one request, one line: {requests:?}" + ); + let request: serde_json::Value = serde_json::from_str(&requests) + .unwrap_or_else(|error| panic!("the request is not JSON ({error}): {requests:?}")); + assert_eq!(request["method"], "stop", "{request}"); + assert_eq!(request["payload"]["service_id"], SERVICE_ID, "{request}"); +} + +/// What the engine printed before it exited non-zero is what the report carries: +/// a stop it answered, with no error taken from the exit status. +#[then("the tool reports the engine's answer to the stop, not the status it exited with")] +async fn engine_answer_survives_a_failing_exit(world: &mut E2eWorld) { + let report = tool_report(world); + assert_eq!( + report["result"]["engine_stop"], + serde_json::json!({"attempted": true, "stopped": true, "graceful": true}), + "{report}" + ); + assert_eq!(report["status"], "stopped", "{report}"); + assert_eq!(report["result"]["stopped"], true, "{report}"); +} + +/// An answer that cannot be read is reported as that, together with the status +/// the engine exited with and what it said on stderr: the status says the answer +/// was cut short, the stderr why. +#[then("the tool reports the stop request failed, with the engine's exit status and stderr")] +async fn unreadable_answer_reported_with_stderr(world: &mut E2eWorld) { + let report = tool_report(world); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!(engine_stop["attempted"], true, "{report}"); + assert_eq!( + engine_stop["error"], + "failed to parse engine response envelope \ + (engine stdio process exited with code 1); \ + stderr: engine exploded: no container runtime", + "{report}" + ); + assert!(engine_stop.get("stopped").is_none(), "{report}"); +} + +/// For a `vllm` server the engine's `Stop` is advisory: the recorded process +/// decides, so a failed request does not leave the server unconfirmed. +#[then("the tool still reports the server stopped")] +async fn tool_still_reports_stopped(world: &mut E2eWorld) { + let report = tool_report(world); + assert_eq!(report["status"], "stopped", "{report}"); + assert_eq!(report["result"]["stopped"], true, "{report}"); +} + #[then("the recorded process is no longer running")] async fn recorded_process_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); +} + +/// The Windows tree kill, observed: the recorded process AND the one it started +/// are gone after the stop. Without the `taskkill /T` step the stop ends only +/// the recorded process (`TerminateProcess` on the root), and the child keeps +/// running, which is what the second check catches. +#[then("the recorded process and the process it started are no longer running")] +async fn recorded_process_and_its_child_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); + let pid = wait_for_descendant_pid(&descendant_pid_path(world)); + // `taskkill` returns once it has asked for the termination; give the exit + // a moment to land before calling the process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + while windows_process_is_running(pid) { + assert!( + std::time::Instant::now() < deadline, + "the stop took the recorded process down but left the process it \ + started (PID {pid}) running: the tree kill did not reach it" + ); + std::thread::sleep(std::time::Duration::from_millis(250)); + } +} + +/// The report says one process was stopped (the recorded one: a tree kill that +/// already took the root is reported as forced, so it counts), checked next to +/// the processes themselves rather than instead of them. +#[then("the report counts one stopped process")] +async fn report_counts_one_stopped_process(world: &mut E2eWorld) { + let combined = combined_output(world); + assert!(combined.contains("stopped processes: 1"), "{combined}"); +} + +fn assert_recorded_process_exited(world: &mut E2eWorld) { let process = world .recorded_process .as_mut() .expect("scenario recorded no process"); // The report's claim, checked against the process itself: it has exited - // (and is reaped here), rather than merely having been signalled. - let exited = process - .try_wait() - .expect("failed to check the recorded process"); - assert!( - exited.is_some(), - "the stop reported the process gone, but it is still running" - ); + // (and is reaped here), rather than merely having been signalled. The stop + // confirms an exit from the OS's exit code, and this handle is signalled by + // the process object's teardown, so allow that teardown a few seconds on a + // loaded host before calling a process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let exited = process + .try_wait() + .expect("failed to check the recorded process"); + if exited.is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the stop reported the process gone, but it is still running" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } } #[then("the server's record reads stopped and names no process")] diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 5e126d1f01..9f5f4a6820 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -11,9 +11,15 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; +use e2e_cucumber::device_memory::{ + DeviceMemory, DrainWait, MIN_PROBE_BUDGET, Pool, Probe, bounded_stdout, describe_verdict, + judge, parse_amd_smi_vram, parse_rocm_smi_csv_pool, probe_budget, sample_device_memory, + sample_is_truncated, +}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, + ServeAttempt, archive_service_log, engine_from_serve_args, reported_engine, + serve_attempt_report, service_log_tail, }; use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; @@ -136,11 +142,15 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// rather than three calls at each site: the log must be read and copied out /// while the service still owns it, because stopping the service both changes /// what the log's last lines say and is the step most likely to fail outright. +/// For Lemonade the backend inventory is read before the stop too, and the +/// `llama-server` probes, which execute a backend binary, run only after it so +/// they cannot contend with the still-running router. /// /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. -fn serve_failure_evidence( - world: &E2eWorld, +async fn serve_failure_evidence( + world: &mut E2eWorld, + engine: Option<&str>, headline: &str, device_state: &str, stdout: &str, @@ -155,7 +165,24 @@ fn serve_failure_evidence( &crate::results_path(), world.current_scenario.as_deref().unwrap_or("scenario"), ); - let stop_status = stop_scenario_services(world); + // The stop sits inside this call because its position relative to the + // backend probes is the point: see `backend_evidence_around_stop`. + let app_paths = world.isolated_root.is_some().then(|| world.app_paths()); + let isolated_root = world + .isolated_root + .as_ref() + .map(|root| root.path().to_owned()); + let (stop_status, backend_evidence) = + e2e_cucumber::lemonade_diagnostics::backend_evidence_around_stop( + move || { + app_paths + .map(|paths| paths.engine_install_manifest_path("lemonade")) + .ok_or_else(|| "the scenario has no isolated root".to_owned()) + }, + engine, + move || stop_services_in(isolated_root.as_deref()), + ) + .await; serve_attempt_report(&ServeAttempt { headline, device_state, @@ -164,6 +191,7 @@ fn serve_failure_evidence( log_tail: &log_tail, archived_log: &archived_log, stop_status: &stop_status, + backend_evidence: backend_evidence.as_deref(), }) } @@ -188,7 +216,16 @@ async fn serve_and_wait( ) -> String { let timeout_secs = serve_timeout_for(world); let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = crate::run_rocm(world, args); + // Bounded: a `rocm serve` that never returns must fail this scenario, not + // stall the lane until the job is cancelled. The bound is the readiness + // timeout plus a margin, so a slow cold start fails through the CLI's own + // diagnostic rather than a harness kill; readiness polling keeps the plain + // timeout. + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + args, + e2e_cucumber::serve_log::serve_cli_budget(timeout_secs), + ); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); @@ -202,7 +239,15 @@ async fn serve_and_wait( }; panic!( "{}", - serve_failure_evidence(world, &headline, &device_state, &stdout, &stderr) + serve_failure_evidence( + world, + engine_from_serve_args(args).or_else(|| reported_engine(&stdout)), + &headline, + &device_state, + &stdout, + &stderr, + ) + .await ); } @@ -268,124 +313,111 @@ async fn ensure_serve_port_free() -> String { wait_for_free_vram().await } -/// Upper bound on the free-VRAM floor (MiB). Sized so the largest single -/// scenario model can allocate its engine's memory share without tripping its -/// startup check: Qwen3.6-27B plus vLLM's default fraction-of-total KV -/// reservation needs the MI300X mostly clear. Only a data-center GPU has this -/// much, so on smaller cards (e.g. Strix Halo's smaller unified-memory pool) the -/// floor is capped to 90% of the device total (see [`required_free_vram_mib`]) — -/// otherwise the check could never pass. Note the cap no longer sits above what -/// vLLM asks for: rocm-cli used to pin utilization below it, and now defers to -/// vLLM's own (higher) default, so the wait has little headroom left and can -/// time out on a device that is merely holding display memory. The wait is -/// best-effort and the serve proceeds regardless, so this costs time rather -/// than correctness; sizing a deliberate margin is follow-up work. -const MAX_FREE_VRAM_FLOOR_MIB: u64 = 150_000; - -/// The free-VRAM floor to wait for on this host: the model-sized ceiling, but -/// never more than 90% of the device's total VRAM. A hardcoded 150 GiB floor is -/// unreachable on a small card (e.g. Strix Halo's ~48 GiB unified VRAM), so -/// `wait_for_free_vram` would burn its full deadline on every serve scenario; -/// scaling to the device keeps the drain-check meaningful everywhere. -fn required_free_vram_mib(total_mib: u64) -> u64 { - MAX_FREE_VRAM_FLOOR_MIB.min(total_mib / 100 * 90) -} - /// How long to wait for a stopped engine to hand its device memory back. const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); -/// Best-effort: wait until the GPU reports enough free VRAM (see -/// [`required_free_vram_mib`]), so a just-killed serve's memory is actually -/// reclaimed before the next serve starts. Queries `amd-smi` then `rocm-smi`; -/// if neither is present (mock/local, no ROCm), returns immediately so non-GPU -/// runs are unaffected. +/// Pause between two samples of the device. +const DRAIN_POLL_INTERVAL: Duration = Duration::from_secs(3); + +/// Best-effort: wait until the pool the engine allocates from reports enough +/// free memory (see [`select_drain_target`]: VRAM on a discrete card, GTT on an +/// APU whose VRAM is only a BIOS carveout), so a just-killed serve's memory is +/// actually reclaimed before the next serve starts. Queries `amd-smi` then +/// `rocm-smi`; if neither is present (mock/local, no ROCm), returns immediately +/// so non-GPU runs are unaffected. /// /// The wait is bounded and best-effort: on timeout the serve still starts, /// because a stale reading must not turn a slow drain into a hard failure. The -/// returned line records which of the two happened — an undrained device is the -/// single most likely reason the serve that follows never becomes ready, and -/// without it the failure looks identical to a genuinely broken serve. +/// returned line records what happened — an undrained device is the single most +/// likely reason the serve that follows never becomes ready, and without it the +/// failure looks identical to a genuinely broken serve. A tool that is installed +/// but never produces a reading is given up on after a few samples +/// (`UNREAD_SAMPLES_BEFORE_GIVING_UP`) and reported as unreadable, not as +/// undrained; the decisions and the wording live in `DrainWait` and +/// `describe_verdict`, where they are tested. async fn wait_for_free_vram() -> String { - // No GPU tooling → nothing to wait on (mock/local). Probe once up front. - let Some(total) = total_vram_mib() else { - return "device state: no GPU tooling (mock/local run)".to_owned(); - }; - let floor = required_free_vram_mib(total); - let deadline = Instant::now() + VRAM_DRAIN_DEADLINE; + // Started before the first probe so that probe counts against the wait. + let started = Instant::now(); + let deadline = started + VRAM_DRAIN_DEADLINE; + let mut wait = DrainWait::default(); loop { - let free = free_vram_mib(); - if let Some(free) = free - && free >= floor - { - return format!( - "device state: drained ({free} MiB free of {total} MiB, floor {floor} MiB)" - ); + // Every probe is held to the time left, so the wait ends at its deadline + // rather than a few probe timeouts after it. A sample started with less + // than a full probe timeout left is `truncated`: if it gives no reading + // that is the deadline's doing, not the tool's, and is not counted. + let truncated = sample_is_truncated(Instant::now(), deadline); + let sample = judge(probe_device_memory(deadline).await); + if let Some(verdict) = wait.observe_truncated(sample, truncated) { + return describe_verdict(verdict, started.elapsed().as_secs()); } - if Instant::now() >= deadline { - let free = free.map_or_else(|| "unreadable".to_owned(), |mib| format!("{mib} MiB")); - return format!( - "device state: NOT drained after {}s ({free} free of {total} MiB, floor \ - {floor} MiB) — a previous engine is still holding the GPU", - VRAM_DRAIN_DEADLINE.as_secs() - ); + let left = deadline.saturating_duration_since(Instant::now()); + if left < MIN_PROBE_BUDGET { + return describe_verdict(wait.timed_out(), started.elapsed().as_secs()); + } + tokio::time::sleep(left.min(DRAIN_POLL_INTERVAL)).await; + if deadline.saturating_duration_since(Instant::now()) < MIN_PROBE_BUDGET { + return describe_verdict(wait.timed_out(), started.elapsed().as_secs()); } - tokio::time::sleep(Duration::from_secs(3)).await; } } -/// Free device VRAM in MiB for GPU 0, via `amd-smi` (then `rocm-smi`). `None` -/// when no such tool exists or its output can't be parsed. -fn free_vram_mib() -> Option { - vram_mib().map(|(_total, free)| free) +/// Device memory for GPU 0 off the async worker: VRAM via `amd-smi` (then +/// `rocm-smi`), plus the GTT pool from `rocm-smi` where the host needs it (see +/// [`sample_device_memory`] for when GTT is read and what a failed read means). +/// `amd-smi` is not asked for GTT: its output has no field this harness has been +/// checked against, whereas the preflight step already reads `rocm-smi`'s GTT +/// rows on these hosts. It runs subprocesses, each bounded by `probe_budget` (at +/// most the probe timeout, never past `deadline`), so the blocking thread it +/// occupies is released. +async fn probe_device_memory(deadline: Instant) -> Probe { + tokio::task::spawn_blocking(move || { + sample_device_memory(|| vram_pool(deadline), || gtt_pool(deadline)) + }) + .await + .unwrap_or(Probe::Unreadable) } -/// Total device VRAM in MiB for GPU 0. `None` when no GPU tool is available — -/// used as the "is there a GPU to wait on at all?" probe in mock/local runs. -fn total_vram_mib() -> Option { - vram_mib().map(|(total, _free)| total) +/// GTT for GPU 0, via `rocm-smi`. The command is named in the report when its +/// output is not understood, so the first run on a host shows the CSV header the +/// column names were meant to match. +/// +/// What is and is not checked: `gpu-preflight.sh` matches the labels +/// `GTT Total Memory` and `GTT Total Used Memory` in `rocm-smi`'s plain-text +/// `--showmeminfo vram gtt` output, so those labels are known to exist there. +/// That `--showmeminfo gtt --csv` uses the same words as its column headers is +/// not checked against real output; it is assumed by analogy with the VRAM CSV. +/// Warning lines before the header are skipped (see [`parse_rocm_smi_csv_pool`]). +fn gtt_pool(deadline: Instant) -> Probe { + use std::process::Command; + let mut command = Command::new("rocm-smi"); + command.args(["--showmeminfo", "gtt", "--csv"]); + bounded_stdout(command, probe_budget(Instant::now(), deadline)) + .parse_text("rocm-smi --showmeminfo gtt --csv", |text| { + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory") + }) } -/// `(total, free)` device VRAM in MiB for GPU 0, via `amd-smi` (then `rocm-smi`). -/// `None` when no such tool exists or its output can't be parsed. -fn vram_mib() -> Option<(u64, u64)> { +/// Device VRAM for GPU 0, via `amd-smi` (then `rocm-smi`). +fn vram_pool(deadline: Instant) -> Probe { use std::process::Command; // amd-smi: lines like " TOTAL_VRAM: 196592 MB" / "FREE_VRAM: 196309 MB". - if let Ok(out) = Command::new("amd-smi").args(["metric", "-m"]).output() - && out.status.success() - { - let text = String::from_utf8_lossy(&out.stdout); - let field = |name: &str| { - text.lines() - .find_map(|l| l.trim().strip_prefix(name)) - .and_then(|v| v.split_whitespace().next()) - .and_then(|n| n.parse::().ok()) - }; - if let (Some(total), Some(free)) = (field("TOTAL_VRAM:"), field("FREE_VRAM:")) { - return Some((total, free)); - } - } + let amd_smi = || { + let mut command = Command::new("amd-smi"); + command.args(["metric", "-m"]); + bounded_stdout(command, probe_budget(Instant::now(), deadline)) + .parse_text("amd-smi metric -m", parse_amd_smi_vram) + }; // rocm-smi fallback: `--showmeminfo vram --csv` → vram total/used per card. - if let Ok(out) = Command::new("rocm-smi") - .args(["--showmeminfo", "vram", "--csv"]) - .output() - && out.status.success() - { - let text = String::from_utf8_lossy(&out.stdout); + let rocm_smi = || { + let mut command = Command::new("rocm-smi"); + command.args(["--showmeminfo", "vram", "--csv"]); // CSV columns include "VRAM Total Memory (B)" and "VRAM Total Used Memory (B)". - // Parse the first data row's total and used to derive free. - let mut lines = text.lines(); - let header = lines.next()?; - let cols: Vec<&str> = header.split(',').collect(); - let total_idx = cols.iter().position(|c| c.contains("Total Memory"))?; - let used_idx = cols.iter().position(|c| c.contains("Total Used Memory"))?; - let row = lines.next()?; - let vals: Vec<&str> = row.split(',').collect(); - let total: u64 = vals.get(total_idx)?.trim().parse().ok()?; - let used: u64 = vals.get(used_idx)?.trim().parse().ok()?; - let mib = 1024 * 1024; - return Some((total / mib, total.saturating_sub(used) / mib)); - } - None + bounded_stdout(command, probe_budget(Instant::now(), deadline)) + .parse_text("rocm-smi --showmeminfo vram --csv", |text| { + parse_rocm_smi_csv_pool(text, "Total Memory", "Total Used Memory") + }) + }; + amd_smi().or_else(rocm_smi) } /// Kill whatever process is listening on `port`. Best-effort and @@ -543,9 +575,16 @@ fn default_engine_serve_target() -> &'static str { /// attempt shares the device with this one, and that must be visible in the /// failure rather than inferred from a serve that looks broken. fn stop_scenario_services(world: &E2eWorld) -> String { - world.isolated_root.as_ref().map_or_else( + stop_services_in(world.isolated_root.as_ref().map(tempfile::TempDir::path)) +} + +/// [`stop_scenario_services`] on the scenario's isolated root rather than the +/// world, so a caller that must hand the stop to a `Send` closure (the world holds +/// PTY handles and is not `Sync`) can capture the path alone. +fn stop_services_in(isolated_root: Option<&std::path::Path>) -> String { + isolated_root.map_or_else( || "not attempted: scenario has no isolated root".to_owned(), - |root| crate::stop_managed_services(root.path()), + crate::stop_managed_services, ) } @@ -636,8 +675,11 @@ async fn serve_gpu_model(world: &mut E2eWorld) { for attempt in 1..=attempts { made = attempt; let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = - crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &["serve", model, "--engine", engine, "--managed"], + e2e_cucumber::serve_log::serve_cli_budget(timeout_secs), + ); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); @@ -658,13 +700,17 @@ async fn serve_gpu_model(world: &mut E2eWorld) { // outcome is quoted: a stop that found no record (the launch had not yet // written one) or failed means the next attempt did NOT get a clean // device, and the failure says so instead of looking like a broken serve. - diagnostics.push(serve_failure_evidence( - world, - &format!("attempt {attempt} of {attempts} (rc={rc})"), - &device_state, - &stdout, - &stderr, - )); + diagnostics.push( + serve_failure_evidence( + world, + Some(engine), + &format!("attempt {attempt} of {attempts} (rc={rc})"), + &device_state, + &stdout, + &stderr, + ) + .await, + ); if attempt == attempts { break; } @@ -762,10 +808,10 @@ async fn serving_started_no_background_helper(world: &mut E2eWorld) { #[given("a canonical Hugging Face GGUF checkpoint is being served on lemonade")] async fn setup_lemonade_hf_checkpoint_model(world: &mut E2eWorld) { - // Forces the `owner/repo:variant` direct-serve path (`serve_hf_checkpoint`, - // EAI-8026) instead of the short-recipe-name router that `setup_lemonade_model` - // exercises. Same underlying checkpoint (unsloth/Qwen3-0.6B-GGUF, Q4_0), so the - // GGUF is already warm in the HF cache when both scenarios run in one job. + // Explicit checkpoints require the selected GPU direct-server backend on + // Linux and Windows. Named entries may also serve directly on Linux, while + // more complex catalog metadata stays with the router. Both scenarios use + // unsloth/Qwen3-0.6B-GGUF:Q4_0, sharing the same HF cache. // // This is the path #260 reports: on Strix Halo Windows the launch exits 0 and // the endpoint never answers. `serve_and_wait` is what makes that outcome @@ -781,6 +827,50 @@ async fn setup_lemonade_hf_checkpoint_model(world: &mut E2eWorld) { .await; } +/// The catalog's recommended small assistant, asked for by a short name with no +/// `--engine`. +/// +/// The engine is deliberately left to the CLI: the model is what decides which +/// one runs, and the point of the scenario is that the catalog hands that engine +/// a model ref it can serve. The expected ref therefore differs per host, and +/// each side is an assertion in its own right: a lemonade host must be given the +/// GGUF build with its quantization variant pinned (a bare `owner/repo` is a +/// safetensors-only repo lemonade refuses, from inside the engine process), and +/// a vLLM host must still be given the safetensors repo the canonical id names. +/// +/// `Qwen2.5-1.5B` is the readiness substring for both, short enough to survive +/// whichever shape the endpoint reports — lemonade's direct-serve path aliases +/// the server to the full `owner/repo:variant` ref, but a router path would name +/// the GGUF artifact instead, as `Qwen3-0.6B` allows for in the steps above. +/// That looseness is why the exact ref is asserted here from the plan the CLI +/// printed instead. Neither later check can tell the GGUF build from the +/// safetensors repo: readiness is a bare substring match, and `model_ids_match` +/// on the chat reply normalizes the variant and the `-GGUF` suffix away — so a +/// wrong variant, or an override that reached the wrong engine, would run green. +#[given("the recommended small assistant is being served by its short name")] +async fn setup_small_assistant_by_short_name(world: &mut E2eWorld) { + let model = if e2e_cucumber::capability::host_capability().effective_serve_engine == "lemonade" + { + "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M" + } else { + "Qwen/Qwen2.5-1.5B-Instruct" + }; + let stdout = serve_and_wait( + world, + &["serve", "qwen-small", "--managed"], + model, + "Qwen2.5-1.5B", + ) + .await; + assert_eq!( + resolved_model(&stdout), + Some(model), + "`rocm serve qwen-small` on a host serving with {} must resolve to {model}:\n{stdout}", + e2e_cucumber::capability::host_capability().effective_serve_engine, + ); + world.cli_output = Some(stdout); +} + #[given("a model is being served on aim")] async fn setup_aim_model(world: &mut E2eWorld) { serve_aim_canary(world).await; @@ -856,6 +946,228 @@ async fn reserve_aim_model(world: &mut E2eWorld) { serve_aim_canary(world).await; } +/// The unmodified v11.5.1 catalog entry carries a required mmproj file. Its +/// public HF checkpoint is unsloth/Qwen3.5-0.8B-GGUF:Qwen3.5-0.8B-UD-Q4_K_XL.gguf. +const COMPLEX_REGISTERED_MODEL: &str = "Qwen3.5-0.8B-GGUF"; + +/// Select only the real service recorded in this scenario's isolated data root. +/// Neither an endpoint found on another port nor a shared runtime grants ownership. +fn complex_model_service(world: &E2eWorld) -> rocm_core::ManagedServiceRecord { + registered_model_service(world, COMPLEX_REGISTERED_MODEL) +} + +/// The scenario-owned managed Lemonade service for `model`. +fn registered_model_service(world: &E2eWorld, model: &str) -> rocm_core::ManagedServiceRecord { + let root = world + .isolated_root + .as_ref() + .expect("no isolated scenario root"); + let services_dir = root.path().join("data/services"); + let mut records = Vec::new(); + for entry in std::fs::read_dir(&services_dir).expect("managed service directory missing") { + let path = entry.expect("unreadable service entry").path(); + if path.extension().and_then(|extension| extension.to_str()) != Some("json") { + continue; + } + let mut record: rocm_core::ManagedServiceRecord = serde_json::from_slice( + &std::fs::read(&path).expect("unreadable managed service record"), + ) + .expect("invalid managed service record"); + record.normalize_paths_for_host(); + if record.engine == "lemonade" + && record.model_ref == model + && Some(record.endpoint_url.as_str()) == world.endpoint.as_deref() + { + records.push(record); + } + } + assert_eq!( + records.len(), + 1, + "expected exactly one scenario-owned {model} service" + ); + records.pop().expect("record count checked") +} + +#[given("a registered vision GGUF is being served on lemonade on its own port")] +async fn setup_complex_registered_lemonade_model(world: &mut E2eWorld) { + serve_registered_on_its_own_port(world, COMPLEX_REGISTERED_MODEL, "Qwen3.5-0.8B").await; +} + +/// A simple registered entry: one checkpoint with a variant and no extra files, +/// so on Linux Lemonade's catalog resolves it to the direct llama-server route. +const SIMPLE_REGISTERED_MODEL: &str = "Qwen3-0.6B-GGUF"; + +#[given("a simple registered GGUF is being served on lemonade on its own port")] +async fn setup_simple_registered_lemonade_model(world: &mut E2eWorld) { + serve_registered_on_its_own_port(world, SIMPLE_REGISTERED_MODEL, "Qwen3-0.6B").await; +} + +async fn serve_registered_on_its_own_port(world: &mut E2eWorld, model: &str, ready_fragment: &str) { + // Reserve an unused loopback port, then release it immediately before the + // ordinary CLI launch. A bind race fails the scenario; never stop a listener + // that happened to claim it, and never invoke the shared-port kill helper. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("cannot reserve serving port"); + let port = listener.local_addr().expect("no reserved address").port(); + world.endpoint = Some(format!("http://127.0.0.1:{port}/v1")); + world.model_name = Some(model.to_owned()); + let port_arg = port.to_string(); + drop(listener); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &[ + "serve", + model, + "--engine", + "lemonade", + "--managed", + "--host", + "127.0.0.1", + "--port", + &port_arg, + ], + e2e_cucumber::serve_log::serve_cli_budget(serve_timeout_for(world)), + ); + let models_url = format!( + "{}/models", + world.endpoint.as_deref().expect("endpoint set") + ); + let ready = rc == 0 + && model_is_ready(&models_url, Some(ready_fragment), serve_timeout_for(world)).await; + assert!( + ready, + "{}", + serve_failure_evidence( + world, + Some("lemonade"), + &format!("registered serve of {model} failed (rc={rc}, endpoint={models_url})"), + "scenario-owned port; no shared listener was stopped", + &stdout, + &stderr, + ) + .await + ); + // Reject readiness from an unrelated listener even if it serves the same model. + let record = registered_model_service(world, model); + assert_eq!(record.port, port); + assert_eq!(record.host, "127.0.0.1"); +} + +/// The service's engine state once it says `ready`, polled for up to ten seconds +/// (the state is written by the serve process, not by the CLI that returned). +async fn wait_for_ready_engine_state( + record: &rocm_core::ManagedServiceRecord, +) -> serde_json::Value { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let state = std::fs::read(&record.engine_state_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + if let Some(state) = state + && state["status"] == "ready" + { + assert_eq!(state["engine"], "lemonade", "{state}"); + return state; + } + assert!( + Instant::now() < deadline, + "ready state missing at {}", + record.engine_state_path.display() + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +#[then("the managed engine state confirms the Lemonade router loaded the registered model")] +async fn assert_complex_model_router(world: &mut E2eWorld) { + let state = wait_for_ready_engine_state(&complex_model_service(world)).await; + assert_eq!(state["load_response"]["method"], "lemonade-cli", "{state}"); + assert_eq!( + state["load_response"]["model_name"], COMPLEX_REGISTERED_MODEL, + "{state}" + ); +} + +#[then("the managed engine state records direct serving by the selected GPU backend")] +async fn assert_simple_model_served_directly(world: &mut E2eWorld) { + let state = + wait_for_ready_engine_state(®istered_model_service(world, SIMPLE_REGISTERED_MODEL)) + .await; + // The route, not just a working endpoint: a router serve records + // `lemonade-cli`, the direct server this method. + assert_eq!( + state["load_response"]["method"], "lemonade-packaged-llama-server", + "{state}" + ); + // Which llama-server served. The direct route only ever runs one from a GPU + // backend directory, so "not CPU" cannot fail here; what can go wrong is a + // crossover, a stale directory of the other backend serving in place of the + // one Lemonade selected. The recorded directory must belong to the selected + // backend. + let selected = crate::e2e::runtime_steps::SelectedLemonadeBackend::read(world); + let dirs = + e2e_cucumber::lemonade_backend::backend_dirs(selected.backend()).unwrap_or_else(|| { + panic!( + "the selected backend `{}` has no GPU llama-server: {state}", + selected.backend() + ) + }); + let served = state["backend_requested"].as_str().unwrap_or_default(); + assert!( + dirs.contains(&served), + "served by `{served}`, which is not a directory of the selected backend `{}` ({dirs:?}): {state}", + selected.backend() + ); +} + +/// The managed Lemonade service of whichever registered model this scenario +/// served (`world.model_name`, set by the serve step), so the stop steps serve +/// both the router and the direct-route scenarios. +fn scenario_registered_service(world: &E2eWorld) -> rocm_core::ManagedServiceRecord { + let model = world + .model_name + .as_deref() + .expect("the scenario has not served a registered model"); + registered_model_service(world, model) +} + +#[when("the user stops the scenario's managed Lemonade service")] +async fn stop_complex_model_service(world: &mut E2eWorld) { + let record = scenario_registered_service(world); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["services", "stop", &record.service_id, "--yes"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the managed service is stopped and its endpoint is closed")] +async fn assert_complex_model_stopped(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "stop failed: {:?}", world.cli_stderr); + let record = scenario_registered_service(world); + let output = world.cli_output.as_deref().expect("no stop command output"); + assert!(output.contains("stopped"), "{output}"); + assert!(output.contains(&record.service_id), "{output}"); + assert_eq!(record.status, "stopped"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let connection = tokio::time::timeout( + Duration::from_secs(1), + tokio::net::TcpStream::connect(("127.0.0.1", record.port)), + ) + .await; + if matches!(connection, Ok(Err(_))) { + return; + } + assert!( + Instant::now() < deadline, + "stopped service still listens on {}", + record.endpoint_url + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + #[given("a large model is being served on GPU")] async fn setup_large_gpu_model(world: &mut E2eWorld) { // Large-model nightly coverage follows the host's serving path. vLLM hosts @@ -945,10 +1257,16 @@ async fn setup_model_detected(world: &mut E2eWorld) { // ── When ─────────────────────────────────────────────────────────── +/// The catalog alias the short-name scenarios (serve-01, serve-02) ask for, held +/// in one place so the Then step can say whether it was expanded without +/// repeating the literal. +const SHORT_NAME_UNDER_TEST: &str = "qwen2.5"; + #[when("the user serves a model using its short name")] async fn user_serves_short_name(world: &mut E2eWorld) { let engine = "vllm"; - let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", SHORT_NAME_UNDER_TEST, "--engine", engine]); // stderr and the exit code ride along so a run that stopped on a refusal reads // as one in the Then step, instead of as "the name was not expanded". world.serve_runs = Some(vec![ServeRun { @@ -964,7 +1282,7 @@ async fn user_serves_multiple_engines(world: &mut E2eWorld) { let mut runs = Vec::new(); for engine in ["lemonade", "vllm"] { let (stdout, stderr, rc) = - crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); + crate::run_rocm(world, &["serve", SHORT_NAME_UNDER_TEST, "--engine", engine]); runs.push(ServeRun { engine: engine.to_owned(), stdout, @@ -1008,7 +1326,11 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { // fn's doc comment. let model = default_engine_serve_target(); let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", model, "--managed"]); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &["serve", model, "--managed"], + e2e_cucumber::serve_log::serve_cli_budget(serve_timeout_for(world)), + ); // The model the CLI resolved (what actually gets served) can differ from the // requested id, so downstream reachability/readiness checks must look for the // resolved model on the shared port; fall back to the requested id. @@ -1041,15 +1363,19 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { &ready_substr, timeout_secs, ); + let stdout = world.cli_output.clone().unwrap_or_default(); + let stderr = world.cli_stderr.clone().unwrap_or_default(); panic!( "{}", serve_failure_evidence( world, + reported_engine(&stdout), &headline, &device_state, - world.cli_output.as_deref().unwrap_or_default(), - world.cli_stderr.as_deref().unwrap_or_default(), + &stdout, + &stderr, ) + .await ); } } @@ -1061,8 +1387,11 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) { // platform. Qwen2.5-0.5B is the smallest vLLM-preferred catalog entry. Omit // `--engine` so the CLI's own default selection is what's exercised. ensure_serve_port_free().await; - let (stdout, stderr, rc) = - crate::run_rocm(world, &["serve", "Qwen/Qwen2.5-0.5B-Instruct", "--managed"]); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &["serve", "Qwen/Qwen2.5-0.5B-Instruct", "--managed"], + e2e_cucumber::serve_log::serve_cli_budget(serve_timeout_for(world)), + ); world.cli_output = Some(stdout); // See the note in user_serves_default_engine: the rc is asserted later, so // stderr must be carried along to explain a non-zero one. @@ -1106,6 +1435,71 @@ async fn user_serves_with_automatic_port(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// ROCMAI-863: serve a real safetensors-only `owner/repo` rather than a made-up +/// one, so the scenario covers the case users hit. There is no GGUF for Lemonade +/// to serve under any variant — the refusal is the only correct outcome, and it +/// must arrive on the terminal, not in a log nobody reads. `--managed` is what +/// `rocm serve` does by default; naming it keeps the scenario on the path the +/// report describes. +/// +/// The id must be one the catalog has NO recipe for. The original report named +/// `Qwen/Qwen2.5-1.5B-Instruct`, but #1044 gave that one a per-engine recipe +/// resolving to `...-GGUF:Q4_K_M`, so the variant arrives from the catalog, the +/// refusal correctly does not fire, and the scenario silently inverted — it +/// served, exited 0, and only the GPU lane noticed. The assertion below keeps +/// that from recurring quietly if this id ever gains a recipe too. +const HF_REPO_WITHOUT_VARIANT: &str = "Qwen/Qwen2.5-3B-Instruct"; + +#[when("the user serves a Hugging Face repo with no quantization variant on Lemonade")] +async fn user_serves_hf_repo_without_variant(world: &mut E2eWorld) { + assert!( + !rocm_core::model_catalog::builtin_model_recipes() + .iter() + .any(|recipe| recipe.matches_ref(HF_REPO_WITHOUT_VARIANT)), + "{HF_REPO_WITHOUT_VARIANT} now has a catalog recipe, which supplies the \ + quantization variant this scenario exists to see refused; pick an id \ + the catalog has no recipe for" + ); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "serve", + HF_REPO_WITHOUT_VARIANT, + "--engine", + "lemonade", + "--managed", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// The refusal has to say what would work, not just that this did not. The +/// `owner/repo:` form is the way out, and it is what the user retypes. +#[then("the user is told the model needs an explicit GGUF variant")] +async fn assert_told_to_name_a_gguf_variant(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("`owner/repo:`"), + "expected the refusal to name the form that would work, got:\n{output}" + ); +} + +/// The discriminating assertion for the second half of ROCMAI-863. The old +/// refusal offered `:Q4_K_M`, a quantization nothing had looked for — on +/// a safetensors-only repo that advice fails again, and the user is no better +/// off than before. Asserted as an absence, because a wording pin would certify +/// whatever text happened to be there rather than that it stopped guessing. +#[then("no quantization is suggested for a repo the CLI has not inspected")] +async fn assert_no_quantization_suggested(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + !output.contains("Q4_K_M") && !output.contains("Q4_0"), + "the refusal must not name a quantization this repo was never checked for, got:\n{output}" + ); +} + #[when("the user sends a chat completion request")] async fn user_sends_completion(world: &mut E2eWorld) { crate::send_chat(world).await; @@ -2123,6 +2517,357 @@ async fn assert_lemonade_recovery_guidance(world: &mut E2eWorld) { ); } +/// The Anthropic endpoint `rocm services list` shows for the first service that +/// has one, waiting for it to appear. +/// +/// The serving process publishes the endpoint a moment after the engine lists its +/// model, so a service the CLI already calls ready may not show it on the first +/// listing. +async fn listed_anthropic_endpoint(world: &mut E2eWorld) -> Option { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); + let found = stdout.lines().find_map(|line| { + line.trim_start() + .strip_prefix("anthropic endpoint: ") + .map(str::to_owned) + }); + world.cli_output = Some(stdout); + if found.is_some() || Instant::now() >= deadline { + return found; + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} + +#[then("the service details include an Anthropic-compatible messages endpoint")] +async fn assert_service_lists_anthropic_endpoint(world: &mut E2eWorld) { + let listed = listed_anthropic_endpoint(world).await; + let output = world.cli_output.clone().unwrap_or_default(); + let endpoint = listed + .unwrap_or_else(|| panic!("no `anthropic endpoint:` row in `services list`:\n{output}")); + assert!( + endpoint.starts_with("http://") && !endpoint.contains("/v1"), + "the Anthropic endpoint is a base URL clients append `/v1/messages` to, got `{endpoint}`" + ); +} + +#[then("the service details still include the existing OpenAI-compatible endpoint")] +async fn assert_service_still_lists_openai_endpoint(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().unwrap_or(""); + assert!( + output + .lines() + .any(|line| line.trim_start().starts_with("endpoint: http") && line.ends_with("/v1")), + "the OpenAI-compatible `endpoint:` row is missing from `services list`:\n{output}" + ); +} + +/// POST `body` to the served model's Anthropic endpoint and return the JSON +/// reply, whatever its status. +async fn post_anthropic_messages(world: &mut E2eWorld, body: serde_json::Value) { + let base = listed_anthropic_endpoint(world) + .await + .expect("no Anthropic endpoint is listed for the served model"); + let budget = Duration::from_secs(crate::inference_timeout_for(world)); + let reply = tokio::spawn(async move { + reqwest::Client::builder() + .timeout(budget) + .build() + .expect("failed to build HTTP client") + .post(format!("{base}/v1/messages")) + .header("anthropic-version", "2023-06-01") + .json(&body) + .send() + .await + .unwrap_or_else(|error| panic!("POST {base}/v1/messages failed: {error}")) + .json::() + .await + .unwrap_or_else(|error| panic!("POST {base}/v1/messages returned non-JSON: {error}")) + }) + .await + .expect("the Anthropic request task panicked"); + world.anthropic_response = Some(reply); +} + +#[when("the user sends a request to the Anthropic messages endpoint")] +async fn user_sends_anthropic_messages_request(world: &mut E2eWorld) { + // Sent under a Claude model name on purpose: an Anthropic client names a + // Claude model, and the endpoint answers for the model that is served. + // The budget leaves room for a reasoning model (Qwen3) to finish thinking + // before it answers: at 32 tokens the whole reply was spent on reasoning + // and the text block came back empty with `stop_reason: max_tokens`. + post_anthropic_messages( + world, + serde_json::json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "Hello"}], + }), + ) + .await; +} + +/// The tool the tool-use scenario offers, and the argument it requires. +const ANTHROPIC_TOOL_NAME: &str = "get_weather"; +const ANTHROPIC_TOOL_ARGUMENT: &str = "city"; + +#[when("the user sends a tool-use request to the Anthropic messages endpoint")] +async fn user_sends_anthropic_tool_use_request(world: &mut E2eWorld) { + // The tool is named in `tool_choice` so the reply does not depend on the + // model choosing to call it: what is under test is the translation of the + // definition out and of the call back, not the model's judgement. + post_anthropic_messages( + world, + serde_json::json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "tools": [{ + "name": ANTHROPIC_TOOL_NAME, + "description": "Get the current weather for a city.", + "input_schema": { + "type": "object", + "properties": {ANTHROPIC_TOOL_ARGUMENT: {"type": "string"}}, + "required": [ANTHROPIC_TOOL_ARGUMENT], + }, + }], + "tool_choice": {"type": "tool", "name": ANTHROPIC_TOOL_NAME}, + "messages": [{"role": "user", "content": "What is the weather in Oslo?"}], + }), + ) + .await; +} + +fn anthropic_tool_use_block(world: &E2eWorld) -> serde_json::Value { + let reply = world + .anthropic_response + .as_ref() + .expect("no Anthropic response"); + reply["content"] + .as_array() + .and_then(|blocks| blocks.iter().find(|block| block["type"] == "tool_use")) + .cloned() + .unwrap_or_else(|| panic!("no tool_use content block in the reply: {reply}")) +} + +#[then("the response contains a tool_use content block")] +async fn assert_anthropic_tool_use_block(world: &mut E2eWorld) { + let block = anthropic_tool_use_block(world); + assert!( + block["id"].as_str().is_some_and(|id| !id.is_empty()), + "the tool_use block has no id: {block}" + ); + assert_eq!( + world.anthropic_response.as_ref().unwrap()["stop_reason"], + "tool_use", + "a reply with a tool call stops for tool use" + ); +} + +#[then("the tool_use block names a tool and arguments from the request")] +async fn assert_anthropic_tool_use_names_request_tool(world: &mut E2eWorld) { + let block = anthropic_tool_use_block(world); + assert_eq!(block["name"], ANTHROPIC_TOOL_NAME, "{block}"); + assert!( + block["input"][ANTHROPIC_TOOL_ARGUMENT] + .as_str() + .is_some_and(|city| !city.is_empty()), + "the tool_use input has no `{ANTHROPIC_TOOL_ARGUMENT}` argument: {block}" + ); +} + +#[then("the response is a valid Anthropic Messages API response")] +async fn assert_valid_anthropic_response(world: &mut E2eWorld) { + let reply = world + .anthropic_response + .as_ref() + .expect("no Anthropic response"); + assert_eq!(reply["type"], "message", "not a Message: {reply}"); + assert_eq!(reply["role"], "assistant", "{reply}"); + assert!( + reply["content"] + .as_array() + .and_then(|blocks| blocks.first()) + .is_some_and(|block| block["type"] == "text" + && block["text"].as_str().is_some_and(|text| !text.is_empty())), + "no text content block in the reply: {reply}" + ); + assert!(reply["stop_reason"].is_string(), "{reply}"); + assert!( + reply["usage"]["input_tokens"].is_u64() && reply["usage"]["output_tokens"].is_u64(), + "usage is missing token counts: {reply}" + ); +} + +#[then("the Anthropic response identifies the correct model")] +async fn assert_anthropic_response_model_correct(world: &mut E2eWorld) { + let reply = world + .anthropic_response + .as_ref() + .expect("no Anthropic response"); + let model = reply["model"].as_str().unwrap_or(""); + let expected = world.model_name.as_deref().unwrap_or(""); + assert!( + model_ids_match(model, expected), + "the Anthropic response model '{model}' does not identify '{expected}'" + ); +} + +#[when("the user sends a streaming request to the Anthropic messages endpoint")] +async fn user_sends_streaming_anthropic_messages_request(world: &mut E2eWorld) { + let base = listed_anthropic_endpoint(world) + .await + .expect("no Anthropic endpoint is listed for the served model"); + let budget = Duration::from_secs(crate::inference_timeout_for(world)); + // Same budget and model name as the non-streaming request above. + let body = serde_json::json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "stream": true, + "messages": [{"role": "user", "content": "Hello"}], + }); + let (content_type, events) = tokio::spawn(async move { + let response = reqwest::Client::builder() + .timeout(budget) + .build() + .expect("failed to build HTTP client") + .post(format!("{base}/v1/messages")) + .header("anthropic-version", "2023-06-01") + .json(&body) + .send() + .await + .unwrap_or_else(|error| panic!("POST {base}/v1/messages failed: {error}")); + assert!( + response.status().is_success(), + "streaming request answered HTTP {}", + response.status() + ); + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned(); + let events = response + .text() + .await + .unwrap_or_else(|error| panic!("reading the event stream failed: {error}")); + (content_type, events) + }) + .await + .expect("the Anthropic streaming request task panicked"); + world.anthropic_stream = Some(format!("content-type: {content_type}\n\n{events}")); +} + +/// The content type and `(event name, data)` pairs of the streamed response. +fn parse_anthropic_stream(world: &E2eWorld) -> (String, Vec<(String, serde_json::Value)>) { + let raw = world + .anthropic_stream + .as_deref() + .expect("no Anthropic streaming response"); + let (header, body) = raw.split_once("\n\n").expect("stream capture is malformed"); + let events = body + .split("\n\n") + .filter(|frame| !frame.trim().is_empty()) + .map(|frame| { + let event = frame + .lines() + .find_map(|line| line.strip_prefix("event: ")) + .unwrap_or_else(|| panic!("an SSE frame has no `event:` line:\n{frame}")); + let data = frame + .lines() + .find_map(|line| line.strip_prefix("data: ")) + .unwrap_or_else(|| panic!("an SSE frame has no `data:` line:\n{frame}")); + let data = serde_json::from_str(data) + .unwrap_or_else(|error| panic!("SSE data is not JSON ({error}):\n{frame}")); + (event.to_owned(), data) + }) + .collect(); + (header.to_owned(), events) +} + +#[then("the response is delivered as Anthropic-style server-sent events")] +async fn assert_anthropic_server_sent_events(world: &mut E2eWorld) { + let (header, events) = parse_anthropic_stream(world); + assert!( + header.contains("text/event-stream"), + "the streaming response is not an event stream: {header}" + ); + let names: Vec<&str> = events.iter().map(|(name, _)| name.as_str()).collect(); + assert_eq!(names.first(), Some(&"message_start"), "{names:?}"); + assert_eq!(names.last(), Some(&"message_stop"), "{names:?}"); + for required in [ + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + ] { + assert!( + names.contains(&required), + "no `{required}` event: {names:?}" + ); + } + assert!( + !names.contains(&"error"), + "the stream carried an error: {events:?}" + ); + let model = events[0].1["message"]["model"].as_str().unwrap_or(""); + let expected = world.model_name.as_deref().unwrap_or(""); + assert!( + model_ids_match(model, expected), + "the streamed message model '{model}' does not identify '{expected}'" + ); +} + +#[then("the streamed content reassembles into a complete reply")] +async fn assert_anthropic_stream_reassembles(world: &mut E2eWorld) { + let (_, events) = parse_anthropic_stream(world); + let text: String = events + .iter() + .filter(|(name, _)| name == "content_block_delta") + .filter_map(|(_, data)| data["delta"]["text"].as_str()) + .collect(); + assert!( + !text.is_empty(), + "the streamed deltas carry no text: {events:?}" + ); + let delta = events + .iter() + .find(|(name, _)| name == "message_delta") + .map(|(_, data)| data) + .expect("no message_delta"); + assert!( + delta["delta"]["stop_reason"].is_string(), + "message_delta has no stop_reason: {delta}" + ); + assert!( + delta["usage"]["output_tokens"] + .as_u64() + .is_some_and(|n| n > 0), + "message_delta reports no output tokens: {delta}" + ); +} + +#[then("both the OpenAI-compatible and Anthropic-compatible endpoints are reported healthy")] +async fn assert_both_endpoints_healthy(world: &mut E2eWorld) { + // `services list` probes each endpoint it shows and marks one that does not + // answer with `(not responding)`; a healthy service carries neither mark. + let listed = listed_anthropic_endpoint(world).await; + let output = world.cli_output.clone().unwrap_or_default(); + assert!( + listed.is_some(), + "no `anthropic endpoint:` row in `services list`:\n{output}" + ); + assert!( + output.contains("status: ready"), + "the OpenAI-compatible endpoint is not reported ready:\n{output}" + ); + assert!( + !output.contains("(not responding)"), + "the Anthropic-compatible endpoint is reported not responding:\n{output}" + ); +} + #[then("an inference request succeeds immediately")] async fn assert_inference_succeeds_now(world: &mut E2eWorld) { // No extra wait: the CLI already reported ready, so inference must work now. @@ -2350,7 +3095,7 @@ async fn plant_oom_launch(world: &mut E2eWorld) { // in the `When`, which left this step a no-op whose deletion changed nothing. // // The consumer of this variable is `e2e_simulate_oom_launch` in - // `apps/rocm/src/service_launch.rs`. Unlike `OOM_FAULT_INJECTION_ENV` (shared + // `apps/rocm/src/service_launch/readiness.rs`. Unlike `OOM_FAULT_INJECTION_ENV` (shared // through `e2e-report`, whose producer and consumer are both test-side), the // consumer here is the shipped binary's crate, which must not depend on the // e2e harness — so the name is spelled out on both sides. A typo fails loudly @@ -2534,26 +3279,42 @@ async fn assert_model_list_is_ascii(world: &mut E2eWorld) { ); } -#[then("all engines expand to the same full model name")] -async fn assert_consistent_expansion(world: &mut E2eWorld) { +/// The build `SHORT_NAME_UNDER_TEST` must expand to on each engine. +/// +/// The two differ on purpose: the catalog recipe behind this alias publishes a +/// quantized GGUF for lemonade and the safetensors repo for vLLM, so equality +/// across engines is the wrong check. Naming both refs is what keeps the step +/// able to fail — on a wrong variant, on a missing expansion, and on an override +/// that reached the wrong engine. +const SHORT_NAME_EXPANSIONS: [(&str, &str); 2] = [ + ("lemonade", "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M"), + ("vllm", "Qwen/Qwen2.5-1.5B-Instruct"), +]; + +#[then("every engine expands it to the full name of the model it will serve")] +async fn assert_short_name_expansion_per_engine(world: &mut E2eWorld) { let runs = world.serve_runs.as_ref().expect("no serve runs"); - assert!(runs.len() >= 2, "need at least 2 serve outputs to compare"); - let mut resolved: Vec = Vec::new(); + assert_eq!( + runs.len(), + SHORT_NAME_EXPANSIONS.len(), + "expected one serve output per engine in {SHORT_NAME_EXPANSIONS:?}" + ); for (i, run) in runs.iter().enumerate() { // A missing `resolved model:` line means the engine never got far enough // to expand the name (e.g. it errored before serving). Fail loudly rather // than defaulting to an empty string — otherwise two engines that both - // fail would produce equal ("") values and pass this check vacuously. - let Some(model) = run.resolved_model().map(str::to_string) else { + // fail would pass this check vacuously. + let Some(resolved) = run.resolved_model() else { panic!("engine #{i}: {}", run.missing_plan_message()); }; - resolved.push(model); - } - let first = &resolved[0]; - for (i, r) in resolved.iter().enumerate().skip(1) { + let (_, expected) = SHORT_NAME_EXPANSIONS + .iter() + .find(|(engine, _)| *engine == run.engine) + .unwrap_or_else(|| panic!("no expected expansion recorded for engine {}", run.engine)); assert_eq!( - r, first, - "inconsistent model name expansion across engines: {resolved:?} (index {i})" + &resolved, expected, + "engine '{}' expanded {SHORT_NAME_UNDER_TEST:?} to {resolved:?}:\n{}", + run.engine, run.stdout ); } } @@ -2652,11 +3413,7 @@ async fn assert_next_automatic_port(world: &mut E2eWorld) { /// Extract the engine name from a serve plan's `engine: ` line. fn selected_engine(output: &str) -> &str { - let Some(engine) = output - .lines() - .find_map(|l| l.trim().strip_prefix("engine:")) - .map(str::trim) - else { + let Some(engine) = reported_engine(output) else { panic!("no 'engine:' line in serve output:\n{output}"); }; engine diff --git a/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs b/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs new file mode 100644 index 0000000000..10d3fd7e48 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs @@ -0,0 +1,405 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `rocm diagnose --symptom-file` (diagnose-59 to diagnose-61): a +//! pasted error handed over on standard input or in a file instead of as a +//! shell argument. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use cucumber::{given, then, when}; + +use crate::E2eWorld; +use crate::e2e::tui_driver::{TuiSession, default_timeout}; + +/// A pasted error that no shell passes through intact as an argument: it +/// starts with `-`, carries double and single quotes, a `$(...)` that bash and +/// PowerShell would run, and a cmd.exe `&` chain. Those commands are there to +/// make the text the kind `--symptom-file` exists for; nothing looks for what +/// they would write, because handed over as data the paste is never parsed by +/// a shell. What the scenarios check is that the text, read from a file and +/// piped by each shell, is diagnosed with the same causes, scores and evidence +/// as the same text passed as one argument, and that typed at a terminal it +/// leaves a screen containing that argument form's whole report. The keyword +/// it carries scores a catalog entry on every platform (see +/// `KNOWN_SYMPTOM` in `diagnose_steps.rs`), so the diagnosis it gets is a real +/// one. ASCII only, because Windows PowerShell 5.1's pipe to a native command +/// encodes text as ASCII and would turn any other character into `?`. +const HOSTILE_PASTE: &str = "-v \"HSA_STATUS_ERROR_INVALID_ISA\" isn't it? \ + $(echo ran > shell-ran-the-paste) & echo ran > shell-ran-the-paste"; + +/// The args every piped run hands `rocm`, for the coverage report. +const PIPED_ARGS: [&str; 4] = ["diagnose", "--symptom-file", "-", "--json"]; + +/// The terminal scenario's command, with the default `--top`, so the cause the +/// paste scores is listed even where causes found on the host outrank it. +const TERMINAL_ARGS: [&str; 3] = ["diagnose", "--symptom-file", "-"]; + +/// Rows of the terminal the paste is typed into: enough for the hint, the +/// echoed paste and a whole report at the default `--top`, because the +/// emulated screen keeps no scrollback and a scrolled-off row reads as absent. +const TERMINAL_ROWS: u16 = 400; + +/// One `rocm diagnose --symptom-file` run: which shell piped it (or which file +/// it was pointed at), and what came back. +#[derive(Debug)] +pub struct SymptomFileRun { + pub label: String, + pub path: Option, + pub rc: i32, + pub stdout: String, + pub stderr: String, +} + +/// The scenario's own directory for the paste and anything a shell writes. +fn paste_dir(world: &E2eWorld) -> PathBuf { + let dir = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("paste"); + std::fs::create_dir_all(&dir).expect("failed to create the paste directory"); + dir +} + +/// Run one shell pipeline in the paste directory, with the scenario's +/// isolated environment, and record it as the `rocm` invocation it wraps. +fn run_piped(world: &E2eWorld, label: &str, mut cmd: Command) -> SymptomFileRun { + cmd.current_dir(paste_dir(world)); + world.isolate_cmd(&mut cmd); + let output = cmd + .output() + .unwrap_or_else(|e| panic!("failed to start {label}: {e}")); + let rc = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + crate::record_command(world.current_scenario.as_deref(), &PIPED_ARGS, rc, &stdout); + SymptomFileRun { + label: label.to_owned(), + path: None, + rc, + stdout, + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + } +} + +/// The pipelines a user on this platform would type: the paste comes from a +/// file through the shell's own pipe, so the shell parses only the command. +#[cfg(not(windows))] +fn platform_pipelines(rocm: &str, paste: &Path) -> Vec<(&'static str, Command)> { + let mut bash = Command::new("bash"); + bash.arg("-c") + .arg(r#"cat -- "$1" | "$0" diagnose --symptom-file - --json"#) + .arg(rocm) + .arg(paste); + vec![("bash", bash)] +} + +#[cfg(windows)] +fn platform_pipelines(rocm: &str, paste: &Path) -> Vec<(&'static str, Command)> { + use std::os::windows::process::CommandExt as _; + + let paste = paste.display().to_string(); + // `/s` strips exactly the outer pair of quotes, leaving each path quoted. + let mut cmd = Command::new("cmd.exe"); + cmd.raw_arg(format!( + r#"/d /s /c "type "{paste}" | "{rocm}" diagnose --symptom-file - --json""# + )); + // PowerShell's pipe to a native command appends a line break, which + // `--symptom-file` removes. The comparison here would not notice if it + // did not (a trailing line break changes no cause), so that rule is held + // by the unit tests in `symptom_input.rs`. `exit $LASTEXITCODE` hands back + // rocm's own exit code rather than PowerShell's success flag. + let single = |text: &str| text.replace('\'', "''"); + let mut powershell = Command::new("powershell.exe"); + powershell.args([ + "-NoProfile", + "-NonInteractive", + "-Command", + &format!( + "Get-Content -Raw -LiteralPath '{}' | & '{}' diagnose --symptom-file - --json; \ + exit $LASTEXITCODE", + single(&paste), + single(rocm) + ), + ]); + vec![("cmd.exe", cmd), ("Windows PowerShell", powershell)] +} + +/// The diagnosis of `paste` handed over as one argument with no shell in +/// between: the reference every other route must agree with. +fn diagnose_as_argument(world: &E2eWorld, paste: &str) -> serde_json::Value { + let flag = format!("--symptom={paste}"); + let stdout = crate::run_rocm_ok(world, &["diagnose", &flag, "--json"]); + let report: serde_json::Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("diagnose --json did not emit JSON ({e}):\n{stdout}")); + assert_eq!( + report.get("has_match").and_then(serde_json::Value::as_bool), + Some(true), + "the paste must be recognised when passed as an argument, or the comparison \ + would be between two empty diagnoses:\n{stdout}" + ); + report +} + +/// What two diagnoses must agree on: every cause, its score and its evidence. +fn causes(report: &serde_json::Value) -> Vec<(String, i64, serde_json::Value)> { + report + .get("matched") + .and_then(serde_json::Value::as_array) + .expect("the report carries a matched list") + .iter() + .map(|cause| { + ( + cause["id"].as_str().unwrap_or_default().to_owned(), + cause["score"].as_i64().unwrap_or_default(), + cause["evidence"].clone(), + ) + }) + .collect() +} + +/// `text` with all whitespace removed, so a line the terminal wrapped +/// compares equal to the same line unwrapped. +fn squeeze(text: &str) -> String { + text.split_whitespace().collect() +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given("a pasted error containing quotes, a command substitution and a leading minus sign")] +async fn a_hostile_paste(world: &mut E2eWorld) { + // Saved without a trailing line break, as a clipboard holds it. + std::fs::write(paste_dir(world).join("paste.txt"), HOSTILE_PASTE) + .expect("failed to save the paste"); + world.symptom_paste = Some(HOSTILE_PASTE.to_owned()); +} + +#[given("an error file that does not exist and one that is not text")] +async fn unusable_error_files(world: &mut E2eWorld) { + let dir = paste_dir(world); + // Starts with a letter, so it is not mistaken for a UTF-16 byte-order mark. + std::fs::write(dir.join("capture.bin"), [b'H', 0xFF, 0xFE, 0x00, 0x81]) + .expect("failed to write the binary file"); + world.symptom_file_runs = vec![ + SymptomFileRun { + label: "missing".into(), + path: Some(dir.join("no-such-error.txt")), + rc: -1, + stdout: String::new(), + stderr: String::new(), + }, + SymptomFileRun { + label: "not text".into(), + path: Some(dir.join("capture.bin")), + rc: -1, + stdout: String::new(), + stderr: String::new(), + }, + ]; +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user pipes it to the diagnosis instead of passing it as an argument")] +async fn pipe_the_paste(world: &mut E2eWorld) { + let paste = paste_dir(world).join("paste.txt"); + let rocm = crate::rocm_binary(); + let runs: Vec = platform_pipelines(&rocm, &paste) + .into_iter() + .map(|(label, cmd)| run_piped(world, label, cmd)) + .collect(); + world.symptom_file_runs = runs; +} + +#[when("the user asks to paste the error at the terminal, pastes it and ends it as told")] +async fn paste_at_the_terminal(world: &mut E2eWorld) { + let paste = world.symptom_paste.clone().expect("no paste set"); + let mut session = TuiSession::spawn(world, &TERMINAL_ARGS) + .unwrap_or_else(|e| panic!("failed to start the diagnosis: {e}")); + session + .grow_rows(TERMINAL_ROWS) + .unwrap_or_else(|e| panic!("failed to grow the terminal: {e}")); + // Waits for the hint's last words too: it is wider than the terminal, so + // its first row can be on screen before the rest has been drawn. + session + .wait_for_screen_where( + "the CLI asked for the paste", + |screen| screen.contains("Paste the error text") && screen.contains("Windows)."), + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the CLI never asked for the paste: {e}")); + // What the hint tells the user to press: Ctrl-D at the start of a line, + // or on Windows Ctrl-Z then Enter. + let end = if cfg!(windows) { "\u{1a}\r" } else { "\u{4}" }; + session + .send(&format!("{paste}\r{end}")) + .unwrap_or_else(|e| panic!("failed to type the paste: {e}")); + let rc = session + .wait_for_exit_code(default_timeout()) + .await + .unwrap_or_else(|e| panic!("the CLI did not finish after the paste was ended: {e}")); + world.cli_output = Some(session.screen_text()); + world.cli_rc = Some(rc); + world.tui = Some(session); +} + +#[when("the user asks the CLI to diagnose the error in each file")] +async fn diagnose_each_file(world: &mut E2eWorld) { + let mut runs = std::mem::take(&mut world.symptom_file_runs); + for run in &mut runs { + let path = run + .path + .as_ref() + .expect("no file set") + .display() + .to_string(); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["diagnose", "--symptom-file", &path, "--json"]); + run.rc = rc; + run.stdout = stdout; + run.stderr = stderr; + } + world.symptom_file_runs = runs; +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then( + "the diagnosis matches the same failure modes as the same text passed safely as an argument" +)] +async fn piped_matches_argument(world: &mut E2eWorld) { + let paste = world.symptom_paste.clone().expect("no paste set"); + let expected = causes(&diagnose_as_argument(world, &paste)); + assert!( + !world.symptom_file_runs.is_empty(), + "no shell piped the paste" + ); + for run in &world.symptom_file_runs { + assert_eq!( + run.rc, 0, + "{} pipe: diagnose should exit 0\nstdout:\n{}\nstderr:\n{}", + run.label, run.stdout, run.stderr + ); + let report: serde_json::Value = serde_json::from_str(&run.stdout).unwrap_or_else(|e| { + panic!( + "{} pipe: diagnose --json did not emit JSON ({e}):\n{}\nstderr:\n{}", + run.label, run.stdout, run.stderr + ) + }); + assert_eq!( + causes(&report), + expected, + "{} pipe: the piped paste was diagnosed differently from the same text \ + passed as an argument", + run.label + ); + } +} + +#[then("the CLI told the user how to end the paste")] +async fn told_how_to_end(world: &mut E2eWorld) { + let screen = world.cli_output.as_deref().expect("no terminal output"); + // The hint's own words. The end-of-input key it names is the one the When + // step pressed, and the CLI finished because of it -- for the Linux key: + // the scenario runs on Linux and WSL only, and the Windows key (Ctrl-Z then + // Enter) is checked by hand in docs/manual-testing.md section 13. The hint + // is wider than the terminal, so where it wraps is ignored. + assert!( + squeeze(screen).contains(&squeeze( + "Paste the error text, then end it with Ctrl-D at the start of a line \ + (Ctrl-Z then Enter on Windows)." + )), + "the terminal never showed how to end the paste:\n{screen}" + ); + assert_eq!( + world.cli_rc, + Some(0), + "the paste ended and diagnose exits 0:\n{screen}" + ); +} + +#[then("the terminal shows the whole report the same text passed safely as an argument gets")] +async fn terminal_paste_matches_argument(world: &mut E2eWorld) { + let paste = world.symptom_paste.clone().expect("no paste set"); + // Establishes that the paste is recognised at all. + diagnose_as_argument(world, &paste); + let flag = format!("--symptom={paste}"); + let reference = crate::run_rocm_ok(world, &["diagnose", &flag]); + // The comparison below shows the paste was used only if the paste changes + // the report. Where causes found on the host alone would print the same + // report, it could not tell a paste that was read from one that was + // dropped, so that is a failure rather than a pass. + let without = crate::run_rocm_ok(world, &["diagnose"]); + assert_ne!( + squeeze(&reference), + squeeze(&without), + "the report is the same with and without the paste on this host, so the terminal \ + run cannot show that the paste was used:\n{reference}" + ); + // The screen also holds the hint and the echoed paste, so it must contain + // the whole report the argument form prints, compared without the + // terminal's line wrapping. + let screen = world.cli_output.as_deref().expect("no terminal output"); + assert!( + squeeze(screen).contains(&squeeze(&reference)), + "the terminal does not contain the report the same text passed as an argument \ + gets.\n--- argument form ---\n{reference}\n--- terminal ---\n{screen}" + ); +} + +#[then("each is refused as a usage error that names the file and says why")] +async fn each_file_refused(world: &mut E2eWorld) { + assert_eq!( + world.symptom_file_runs.len(), + 2, + "both files must have been tried" + ); + for run in &world.symptom_file_runs { + let path = run + .path + .as_ref() + .expect("no file set") + .display() + .to_string(); + let why = if run.label == "missing" { + "could not read the symptom from" + } else { + "is not valid UTF-8" + }; + assert_eq!( + run.rc, 2, + "{} file: a usage error exits 2\nstderr:\n{}", + run.label, run.stderr + ); + for needed in [path.as_str(), why, "Usage: rocm diagnose"] { + assert!( + run.stderr.contains(needed), + "{} file: the refusal must say {needed:?}:\n{}", + run.label, + run.stderr + ); + } + assert!( + !run.stderr.contains("panicked"), + "{} file: refused by a panic:\n{}", + run.label, + run.stderr + ); + } +} + +#[then("nothing is diagnosed")] +async fn nothing_diagnosed(world: &mut E2eWorld) { + for run in &world.symptom_file_runs { + assert!( + run.stdout.trim().is_empty(), + "{} file: a refused file must not produce a diagnosis:\n{}", + run.label, + run.stdout + ); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/therock_steps.rs b/tests/e2e-cucumber/tests/e2e/therock_steps.rs index d07a243d34..f96f4f26db 100644 --- a/tests/e2e-cucumber/tests/e2e/therock_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/therock_steps.rs @@ -44,15 +44,20 @@ const RAW_ARCH: &str = "gfx1200"; const GROUP_FAMILY: &str = "gfx120X-all"; const NEXT_ROCM_VERSION: &str = "10.0.0"; -const NEXT_TORCH_VERSION: &str = "2.10.0+rocm10.0.0"; -const NEXT_TORCHVISION_VERSION: &str = "0.25.0+rocm10.0.0"; +const NEXT_TORCH_VERSION: &str = "2.12.0+rocm10.0.0"; +const NEXT_TORCHVISION_VERSION: &str = "0.27.0+rocm10.0.0"; const NEXT_TORCHAUDIO_VERSION: &str = "2.9.1+rocm10.0.0"; /// A newer ROCm 10 build published beside `NEXT_ROCM_VERSION`, for the scenario /// that pins the older one. const NEWER_NEXT_ROCM_VERSION: &str = "10.1.0"; -const NEWER_NEXT_TORCH_VERSION: &str = "2.11.0+rocm10.1.0"; -const NEWER_NEXT_TORCHVISION_VERSION: &str = "0.26.0+rocm10.1.0"; +const NEWER_NEXT_TORCH_VERSION: &str = "2.13.0+rocm10.1.0"; +const NEWER_NEXT_TORCHVISION_VERSION: &str = "0.28.0+rocm10.1.0"; +/// Published for `NEWER_NEXT_ROCM_VERSION` after `NEWER_NEXT_TORCH_VERSION`, with +/// no torchvision built for it. Left to `uv`, the SDK takes this one and the +/// pair cannot register `torchvision::nms`; the vLLM discover row's series +/// (torch 2.13, torchvision 0.28) is what keeps it off. +const NEWER_NEXT_UNPAIRED_TORCH_VERSION: &str = "2.14.0+rocm10.1.0"; /// The newer build's torchaudio, shaped like the live `whl-next` index's /// `+rocm10.1.0` builds: an original release (CPU-only on the live index), the /// hardware-verified rebuild, and a newer rebuild sharing the same label. The @@ -87,18 +92,33 @@ const NEXT_REAL_TARBALL: &str = "therock-dist-linux-gfx120X-all-10.0.0.tar.gz"; /// The non-release sibling the live catalog publishes beside the real archive. const NEXT_TESTS_TARBALL: &str = "therock-dist-linux-gfx120X-all-tests-10.0.0.tar.gz"; const CURRENT_TARBALL: &str = "therock-dist-linux-gfx120X-all-7.10.0.tar.gz"; -/// The ROCm version `CURRENT_TARBALL` carries, as the preview reports it. -const CURRENT_TARBALL_VERSION: &str = "7.10.0"; +/// The ROCm version `NEXT_REAL_TARBALL` carries, as the preview reports it. +const NEXT_TARBALL_VERSION: &str = "10.0.0"; -/// A host ROCm newer than `CURRENT_TARBALL_VERSION`, reported with a packaging +/// A host ROCm newer than `NEXT_TARBALL_VERSION`, reported with a packaging /// build number the way a system install's `.info/version` reports one. -const HOST_ROCM_NEWER_WITH_BUILD_NUMBER: &str = "7.20.1-33"; +const HOST_ROCM_NEWER_WITH_BUILD_NUMBER: &str = "10.5.1-33"; /// The same host version with a suffix that is neither a build number nor a -/// PEP 440 stage. Dropping the whole suffix would read it as `7.20.1`, newer -/// than `CURRENT_TARBALL_VERSION`; it must be read as "can't tell" instead. -const HOST_ROCM_WITH_UNREADABLE_SUFFIX: &str = "7.20.1-33rc1"; - -/// How many leading bytes of `"Downloading {CURRENT_TARBALL}"` to check for in +/// PEP 440 stage. Dropping the whole suffix would read it as `10.5.1`, newer +/// than `NEXT_TARBALL_VERSION`; it must be read as "can't tell" instead. +const HOST_ROCM_WITH_UNREADABLE_SUFFIX: &str = "10.5.1-33rc1"; + +/// The nightly fixtures. The earlier nightly stream's last build is +/// 2026-08-22 and the newest layout's first is 2026-08-23, so the earlier build +/// can only be read from the earlier stream and the others only from the newest +/// layout's. `UNBUILT` is a `rocm` build published without its PyTorch wheels, +/// as a nightly `rocm` build is for a while after it lands. +const EARLIER_NIGHTLY_ROCM_VERSION: &str = "10.1.0a20260822"; +const EARLIER_NIGHTLY_TORCH_VERSION: &str = "2.11.0+rocm10.1.0a20260822"; +const EARLIER_NIGHTLY_TORCHVISION_VERSION: &str = "0.26.0+rocm10.1.0a20260822"; +const EARLIER_NIGHTLY_TORCHAUDIO_VERSION: &str = "2.11.0+rocm10.1.0a20260822"; +const NEXT_NIGHTLY_ROCM_VERSION: &str = "10.3.0a20261110"; +const NEXT_NIGHTLY_TORCH_VERSION: &str = "2.15.0+rocm10.3.0a20261110"; +const NEXT_NIGHTLY_TORCHVISION_VERSION: &str = "0.29.2+rocm10.3.0a20261110"; +const NEXT_NIGHTLY_TORCHAUDIO_VERSION: &str = "2.11.1+rocm10.3.0a20261110"; +const NEXT_NIGHTLY_UNBUILT_ROCM_VERSION: &str = "10.3.0a20261111"; + +/// How many leading bytes of `"Downloading {NEXT_REAL_TARBALL}"` to check for in /// [`assert_spinner_lines_cleared`] below. Comfortably under the ~48-column /// label budget `assemble_status_line` leaves on an 80-column terminal even at /// the widest realistic progress suffix (see that assertion's comment) — the @@ -269,6 +289,40 @@ const SCRATCH_CACHE_DIR: &str = "scratch-cache"; /// for a reason it does not name, so they are refused up front. #[given("a host with no usable Python and a recording uv")] async fn host_with_no_python_and_recording_uv(world: &mut E2eWorld) { + let scenario_root = host_without_python(world); + let uv = scenario_root + .join("recording-uv-bin") + .join(e2e_cucumber::fixture_binary::fixture_target_name("uv")); + crate::install_fixture_binary("fake-uv", &uv); + world + .command_env + .push(("ROCM_CLI_UV_BINARY", uv.into_os_string())); + world.command_env.push(( + "FAKE_UV_LOG", + scenario_root.join(UV_INVOCATION_LOG).into_os_string(), + )); +} + +/// The same host with no `uv` anywhere: none named by `ROCM_CLI_UV_BINARY` and no +/// managed copy under the scenario's data folder, so a CLI that needs one would +/// have to download it. Nothing here can observe a `uv` run, since there is none to +/// run; what it can observe is the state a download leaves, which is why the +/// scenario's data and cache folders are the evidence. +#[given("a host with no uv and no usable Python")] +async fn host_with_no_uv_and_no_python(world: &mut E2eWorld) { + assert!( + std::env::var_os(rocm_core::uv::UV_BINARY_ENV).is_none(), + "{} is set in the environment running the suite, which would put a uv on this host; \ + unset it", + rocm_core::uv::UV_BINARY_ENV + ); + host_without_python(world); +} + +/// What the hosts these steps build share: `PATH` is one empty folder, so no Python is found +/// and none has been saved, and the CLI has a cache folder of its own. Returns +/// the scenario root. +fn host_without_python(world: &mut E2eWorld) -> std::path::PathBuf { for var in [ "ROCM_CLI_PYTHON", "ROCM_CLI_DISABLE_MANAGED_PYTHON_BOOTSTRAP", @@ -282,10 +336,6 @@ async fn host_with_no_python_and_recording_uv(world: &mut E2eWorld) { let scenario_root = root(world).to_path_buf(); let empty_path = scenario_root.join("no-python-on-path"); std::fs::create_dir_all(&empty_path).expect("failed to create the empty PATH folder"); - let uv = scenario_root - .join("recording-uv-bin") - .join(e2e_cucumber::fixture_binary::fixture_target_name("uv")); - crate::install_fixture_binary("fake-uv", &uv); world .command_env .push(("PATH", empty_path.into_os_string())); @@ -298,13 +348,7 @@ async fn host_with_no_python_and_recording_uv(world: &mut E2eWorld) { world .command_env .push(("ROCM_CLI_CACHE_DIR", scratch_cache.into_os_string())); - world - .command_env - .push(("ROCM_CLI_UV_BINARY", uv.into_os_string())); - world.command_env.push(( - "FAKE_UV_LOG", - scenario_root.join(UV_INVOCATION_LOG).into_os_string(), - )); + scenario_root } #[given("a canonical release pip index fixture and a ROCm 10 pip index fixture")] @@ -336,6 +380,14 @@ async fn pip_index_fixtures(world: &mut E2eWorld) { fn serve_pip_fixtures(world: &mut E2eWorld) { let served = root(world).join("therock-pip-fixtures"); world.artifact_server = Some(LoopbackServer::start(&served)); + arm_pip_fixture_env(world); +} + +/// Points the next CLI call at the running pip fixtures. The scenario's +/// environment is handed to one call and then dropped +/// (`run_rocm_with_scenario_env`), so a scenario that runs the CLI twice has to +/// arm it again before the second; otherwise that call reads the live index. +fn arm_pip_fixture_env(world: &mut E2eWorld) { allow_base_overrides(world); let current = current_pip_base(world); let next = next_pip_base(world); @@ -349,10 +401,12 @@ fn serve_pip_fixtures(world: &mut E2eWorld) { /// Two ROCm 10 builds on one ROCm 10 index, with each package's dependency /// contract shaped like the live preview index's: torch requires its exact -/// `rocm` build and torchvision its exact torch, so pinning `rocm` carries both -/// — but torchaudio requires neither, so nothing in its metadata ties it to a -/// build. `torchaudio` lists the torchaudio builds the index publishes, each -/// with its wheel tag. +/// `rocm` build, so pinning `rocm` carries it. The older build's torchvision +/// requires its exact torch; the newer build's requires any torch, and a newer +/// torch than it was built against is published beside it (see +/// `NEWER_NEXT_UNPAIRED_TORCH_VERSION`). Torchaudio requires neither, so +/// nothing in its metadata ties it to a build. `torchaudio` lists the +/// torchaudio builds the index publishes, each with its wheel tag. fn serve_two_next_builds(world: &mut E2eWorld, torchaudio: &[(&str, &str)]) { let served = root(world).join("therock-pip-fixtures").join("next"); write_fixture(&served.join("index.html"), &aggregate_root_html()); @@ -369,10 +423,25 @@ fn serve_two_next_builds(world: &mut E2eWorld, torchaudio: &[(&str, &str)]) { ), ]; let rocm = builds.map(|(rocm, ..)| (rocm, ANY_WHEEL_TAG, Vec::new())); - let torch = - builds.map(|(rocm, torch, _)| (torch, ANY_WHEEL_TAG, vec![format!("rocm=={rocm}")])); - let torchvision = builds.map(|(_, torch, torchvision)| { - (torchvision, ANY_WHEEL_TAG, vec![format!("torch=={torch}")]) + let mut torch = builds + .map(|(rocm, torch, _)| (torch, ANY_WHEEL_TAG, vec![format!("rocm=={rocm}")])) + .to_vec(); + // A torch newer than the one torchvision was built against, for the same + // `rocm` build, as the live index published for ROCm 10.1.0. + torch.push(( + NEWER_NEXT_UNPAIRED_TORCH_VERSION, + ANY_WHEEL_TAG, + vec![format!("rocm=={NEWER_NEXT_ROCM_VERSION}")], + )); + // The newer build's torchvision names no exact torch, as on the live index, + // so only the series the SDK asks for keeps it off the newer torch above. + let torchvision = builds.map(|(rocm, torch, torchvision)| { + let requires = if rocm == NEWER_NEXT_ROCM_VERSION { + "torch".to_owned() + } else { + format!("torch=={torch}") + }; + (torchvision, ANY_WHEEL_TAG, vec![requires]) }); let torchaudio = torchaudio .iter() @@ -478,6 +547,113 @@ async fn registered_next_wheel_runtime_with_grouped_family(world: &mut E2eWorld) .expect("failed to write runtime manifest"); } +/// Registry key for the planted canonical wheel runtime below. +const OLDER_WHEEL_RUNTIME_KEY: &str = "release-wheel-gfx1200-7-9-0"; + +/// Plants a canonical-layout wheel runtime older than the release index the +/// fixture serves, whose manifest records no Python (a runtime registered before +/// the field was written, or whose recorded interpreter is gone). The update +/// check then has to find an interpreter for itself, which on the host the +/// recording-uv Given builds means preparing one. No `source_layout_generation` +/// is recorded, so the manifest reads as the canonical stream, as an old one does. +#[given("a registered wheel runtime older than the index with no recorded Python")] +async fn registered_older_wheel_runtime_without_python(world: &mut E2eWorld) { + plant_older_wheel_runtime(world, RAW_ARCH); +} + +/// The same runtime of the earlier release stream, recorded with only a family +/// label (`gfx120X-all`) and no exact arch, as an install made with a group +/// label leaves it. Updating it reads the stream it was recorded in, which serves +/// group labels, so it must not be asked for an exact arch. +#[given("a registered wheel runtime of the earlier release stream with a grouped family")] +async fn registered_older_wheel_runtime_with_grouped_family(world: &mut E2eWorld) { + plant_older_wheel_runtime(world, GROUP_FAMILY); +} + +fn plant_older_wheel_runtime(world: &mut E2eWorld, family: &str) { + let install_root = root(world).join("runtime-older-wheel"); + std::fs::create_dir_all(&install_root).expect("failed to create install root"); + let registry = root(world).join("data").join("runtimes").join("registry"); + std::fs::create_dir_all(®istry).expect("failed to create registry dir"); + let manifest = serde_json::to_string_pretty(&serde_json::json!({ + "runtime_key": OLDER_WHEEL_RUNTIME_KEY, + "runtime_id": format!("therock-release:{family}"), + "channel": "release", + "format": "wheel", + "family": family, + "family_source": "manual", + "version": "7.9.0", + "install_root": install_root, + "selected_artifact_url": "https://example.invalid/rocm", + "read_only": false, + "installed_at_unix_ms": 1_700_000_000_000u64, + })) + .expect("failed to serialize runtime manifest"); + std::fs::write( + registry.join(format!("{OLDER_WHEEL_RUNTIME_KEY}.json")), + manifest, + ) + .expect("failed to write runtime manifest"); +} + +#[when("the user checks for updates against the pip index")] +async fn check_updates_against_pip_index(world: &mut E2eWorld) { + preview_ok(world, &["update"]); +} + +#[when("the user previews applying the update to that wheel runtime")] +async fn preview_apply_update_to_older_wheel_runtime(world: &mut E2eWorld) { + preview_ok( + world, + &[ + "update", + "--apply", + "--dry-run", + "--runtime", + OLDER_WHEEL_RUNTIME_KEY, + ], + ); +} + +/// The report reached a verdict (the runtime is behind the index) and says what +/// the update would do about Python, without a path that would be a guess. +#[then("the report offers the newer runtime and says it would prepare a managed Python")] +async fn report_offers_update_and_names_the_python_it_would_prepare(world: &mut E2eWorld) { + assert_contains( + world, + &format!("runtime {OLDER_WHEEL_RUNTIME_KEY} format=wheel"), + "the runtime line", + ); + assert_contains(world, "status=update_available", "the runtime's verdict"); + assert_contains( + world, + &format!("latest={CURRENT_ROCM_VERSION}"), + "the version the index publishes", + ); + assert_contains( + world, + &format!( + "python_note: no usable Python was found; updating would use uv to install managed \ + Python {CANONICAL_MANAGED_PYTHON}" + ), + "the note about the Python an update would prepare", + ); +} + +/// The check says it prepared no Python, and the state it left agrees. Claim and +/// filesystem are checked together: the note alone would pass with the bug +/// present, and so would empty folders alone if the check had stopped before it +/// got to Python. +#[then("the update check prepared no Python, as it says")] +async fn update_check_prepared_no_python(world: &mut E2eWorld) { + assert_contains( + world, + "which this check did not do", + "the note about what the check skipped", + ); + assert_no_python_prepared(world); +} + /// Makes the planted ROCm 10 runtime the active default, so a real install /// without consent reaches the non-interactive refusal: past every point where /// the plan is chosen and announced, but before `uv`, any download or any @@ -648,12 +824,12 @@ const PACED_TARBALL_CHUNK_DELAY: Duration = Duration::from_millis(150); /// `engines_steps.rs`'s file-local `SCREEN_TIMEOUT` convention. const PTY_SCREEN_TIMEOUT: Duration = Duration::from_secs(30); -#[given("a paced canonical release tarball fixture")] +#[given("a paced next-layout release tarball fixture")] async fn paced_tarball_fixture(world: &mut E2eWorld) { let served = root(world).join("therock-paced-tarball-fixture"); write_fixture( - &served.join("tarball").join("current").join("index.html"), - &tarball_index_html(&[(CURRENT_TARBALL, 1_787_000_000.0)]), + &served.join("tarball").join("next").join("index.html"), + &tarball_index_html(&[(NEXT_REAL_TARBALL, 1_787_000_000.0)]), ); // Build a real gzip tarball so `extract_tarball` (auto-detecting `-xf`) has @@ -671,11 +847,11 @@ async fn paced_tarball_fixture(world: &mut E2eWorld) { let payload = deterministic_payload(PACED_TARBALL_PAYLOAD_BYTES); std::fs::write(payload_dir.join("payload.bin"), &payload) .expect("failed to write tarball payload"); - let contents = build_gzip_tarball(&build_dir, CURRENT_TARBALL, "payload").await; + let contents = build_gzip_tarball(&build_dir, NEXT_REAL_TARBALL, "payload").await; world.paced_download_server = Some(PacedDownloadServer::start( &served, - &format!("tarball/current/{CURRENT_TARBALL}"), + &format!("tarball/next/{NEXT_REAL_TARBALL}"), contents, PACED_TARBALL_CHUNK_BYTES, PACED_TARBALL_CHUNK_DELAY, @@ -687,18 +863,17 @@ async fn paced_tarball_fixture(world: &mut E2eWorld) { .expect("paced download server was just started") .base_url(); world.command_env.push(( - "ROCM_CLI_THEROCK_RELEASE_TARBALL_BASE", - format!("{base}/tarball/current/").into(), + "ROCM_CLI_THEROCK_NEXT_TARBALL_BASE", + format!("{base}/tarball/next/").into(), )); } -#[when("the user installs the tarball SDK for family gfx120X-all under a real terminal")] +#[when("the user installs the tarball SDK for arch gfx1200 under a real terminal")] async fn install_tarball_sdk_under_pty(world: &mut E2eWorld) { - // No `--version`: tarball installs only accept an explicit version pin for - // a stable ROCm 10+ selector (see `resolve_tarball_artifact_with_timeout`'s - // rejection message), so an unpinned request is what reaches the canonical - // release catalog here — which the given-step populated with exactly one - // candidate, `CURRENT_TARBALL`. + // No `--version`: an unpinned release install reads the newest layout's + // catalog, which the given-step populated with exactly one candidate, + // `NEXT_REAL_TARBALL`. That layout picks its payload by exact arch, hence + // `RAW_ARCH` rather than a group family. // // This scenario is testing the download spinner, not the post-install // torch-runtime-dependency setup — but a real SDK install completing @@ -720,7 +895,7 @@ async fn install_tarball_sdk_under_pty(world: &mut E2eWorld) { "--format", "tarball", "--family", - GROUP_FAMILY, + RAW_ARCH, "--yes", ], ) @@ -765,7 +940,10 @@ async fn assert_extraction_frame_is_shown(world: &mut E2eWorld) { .as_mut() .expect("no pty session for the tarball install"); session - .wait_for_screen(&format!("Extracting {CURRENT_TARBALL}"), PTY_SCREEN_TIMEOUT) + .wait_for_screen( + &format!("Extracting {NEXT_REAL_TARBALL}"), + PTY_SCREEN_TIMEOUT, + ) .await .unwrap_or_else(|e| panic!("extraction spinner frame never appeared: {e}")); } @@ -791,7 +969,7 @@ async fn assert_spinner_lines_cleared(world: &mut E2eWorld) { // A live progress suffix (kept intact by `assemble_status_line`) can // truncate this label to a fraction of its length on an 80-column // terminal, so the download line never actually contains the full - // "Downloading {CURRENT_TARBALL}" string while it's showing — checking + // "Downloading {NEXT_REAL_TARBALL}" string while it's showing — checking // for that full string here would pass trivially whether or not the line // was cleared. Truncation always keeps the label's head intact and cuts // its tail, so a short prefix is present whenever the line is live and @@ -804,7 +982,7 @@ async fn assert_spinner_lines_cleared(world: &mut E2eWorld) { // stops. The extraction assertion just below, and the ComfyUI scenario's // equivalent check (a spinner with nothing after it to overwrite its // row), are what actually prove `Spinner::clear` runs on `Drop`. - let downloading = format!("Downloading {CURRENT_TARBALL}"); + let downloading = format!("Downloading {NEXT_REAL_TARBALL}"); let downloading_prefix = downloading .get(..DOWNLOADING_PREFIX_LEN) .unwrap_or_else(|| { @@ -820,7 +998,7 @@ async fn assert_spinner_lines_cleared(world: &mut E2eWorld) { "download spinner line was not cleared on completion:\n{screen}" ); assert!( - !screen.contains(&format!("Extracting {CURRENT_TARBALL}")), + !screen.contains(&format!("Extracting {NEXT_REAL_TARBALL}")), "extract spinner line was not cleared on completion:\n{screen}" ); } @@ -865,6 +1043,44 @@ async fn preview_unpinned_wheel_install(world: &mut E2eWorld) { ); } +/// A release pin below ROCm 10 is the one request that still reads the earlier +/// (canonical) release stream, so the scenarios whose subject is that stream +/// say so with a pin instead of leaning on a default that now reads the newest +/// layout. +#[when( + "the user previews a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release" +)] +async fn preview_earlier_release_wheel_install(world: &mut E2eWorld) { + preview_ok( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + RAW_ARCH, + "--version", + CURRENT_ROCM_VERSION, + "--dry-run", + ], + ); +} + +/// A `uv` the preview can use without fetching one. The resolution these +/// scenarios assert on is a real `uv pip compile` against the loopback fixtures, +/// so the stand-in `uv` of runtime-27 would not do; this installs the managed one +/// the way a real install would have (from the network, into the scenario's own +/// data folder, or, when `ROCM_CLI_UV_BINARY` names one, that one). +#[given("a uv is already installed")] +async fn a_uv_is_already_installed(world: &mut E2eWorld) { + let paths = world.app_paths(); + rocm_core::ensure_uv_binary(&paths) + .unwrap_or_else(|error| panic!("failed to install uv for the scenario: {error:#}")); +} + #[when("the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0")] async fn preview_pinned_wheel_install(world: &mut E2eWorld) { preview_ok( @@ -885,6 +1101,84 @@ async fn preview_pinned_wheel_install(world: &mut E2eWorld) { ); } +#[when( + "the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 on Python 3.12" +)] +async fn preview_pinned_wheel_install_on_python_312(world: &mut E2eWorld) { + preview_ok( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + RAW_ARCH, + "--version", + NEXT_ROCM_VERSION, + "--python-version", + "3.12", + "--dry-run", + ], + ); +} + +#[when( + "the user tries to preview a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release on Python 3.13" +)] +async fn try_preview_earlier_release_wheel_install_on_python_313(world: &mut E2eWorld) { + preview( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + RAW_ARCH, + "--version", + CURRENT_ROCM_VERSION, + "--python-version", + "3.13", + "--dry-run", + ], + ); +} + +/// The plan names the requested interpreter and its tag. The default's tag is +/// checked absent too: a plan that printed both would pass the first two checks. +#[then("the preview plans Python 3.12 and its cp312 wheel tag and not the default")] +async fn preview_plans_requested_python(world: &mut E2eWorld) { + assert_contains( + world, + "would use uv to install managed Python 3.12", + "python note", + ); + assert_contains(world, "python_wheel_tag: cp312", "python wheel tag"); + let out = stdout(world); + assert!( + !out.contains("cp314") && !out.contains("Python 3.14"), + "the default Python must not appear in the plan:\n{out}" + ); +} + +#[then( + "the preview is refused because the layout has no wheels for that Python, and prepared none" +)] +async fn preview_refused_for_unpublished_python(world: &mut E2eWorld) { + let stderr = world.cli_stderr.clone().unwrap_or_default(); + assert!( + stderr.contains("--python-version applies only to ROCm 10 and newer installs") + && stderr.contains("this install uses Python 3.12"), + "expected the refusal to name the Python that is allowed:\n{stderr}" + ); + assert_no_python_prepared(world); +} + #[when( "the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 with the toolchain" )] @@ -952,6 +1246,50 @@ async fn preview_pinned_wheel_install_latest_next_version(world: &mut E2eWorld) ); } +#[when("the user tries to preview a wheel SDK install for family gfx120X-all with no version pin")] +async fn try_preview_unpinned_wheel_install_with_group_family(world: &mut E2eWorld) { + preview( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + GROUP_FAMILY, + "--dry-run", + ], + ); +} + +/// The refusal for a group family names pinning an earlier release as the way +/// to still install one, so the pin it names has to work: a group family with +/// an earlier ROCm 7 pin resolves from the earlier stream, which serves groups. +#[when( + "the user previews a wheel SDK install for family gfx120X-all pinned to an earlier ROCm 7 release" +)] +async fn preview_earlier_release_wheel_install_with_group_family(world: &mut E2eWorld) { + arm_pip_fixture_env(world); + preview_ok( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + GROUP_FAMILY, + "--version", + CURRENT_ROCM_VERSION, + "--dry-run", + ], + ); +} + #[when("the user previews a wheel SDK install for family gfx120X-all pinned to ROCm 10.0.0")] async fn preview_pinned_wheel_install_with_group_family(world: &mut E2eWorld) { preview( @@ -1012,6 +1350,92 @@ async fn try_preview_pinned_wheel_install(world: &mut E2eWorld) { ); } +#[when("the user previews a tarball SDK install for arch gfx1200 with no version pin")] +async fn preview_unpinned_tarball_install_for_arch(world: &mut E2eWorld) { + preview_ok( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "tarball", + "--family", + RAW_ARCH, + "--dry-run", + ], + ); +} + +/// A tarball catalog resolves a pin only in the newest layout, so a pin into the +/// earlier release stream is refused and points at the wheel install, which +/// does resolve it. +#[when( + "the user tries to preview a tarball SDK install for arch gfx1200 pinned to an earlier ROCm 7 release" +)] +async fn try_preview_earlier_release_tarball_install(world: &mut E2eWorld) { + preview( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "tarball", + "--family", + RAW_ARCH, + "--version", + CURRENT_ROCM_VERSION, + "--dry-run", + ], + ); +} + +#[then("the failure points to a wheel install for that earlier release")] +async fn failure_points_to_a_wheel_install(world: &mut E2eWorld) { + let reported = format!( + "{}\n{}", + stdout(world), + world.cli_stderr.as_deref().unwrap_or_default() + ); + assert!( + reported.contains("use `--format wheel`"), + "the refusal did not point to a wheel install:\n{reported}" + ); + // The command it points to must itself work: the same pin as a wheel + // install resolves from the earlier stream. + arm_pip_fixture_env(world); + let rc = preview( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + RAW_ARCH, + "--version", + CURRENT_ROCM_VERSION, + "--dry-run", + ], + ); + assert_eq!( + rc, + 0, + "the wheel install the refusal points to failed:\n{}", + stdout(world) + ); + assert_contains( + world, + &format!("selected_rocm_version: {CURRENT_ROCM_VERSION}"), + "the earlier release the refusal points to", + ); +} + #[when("the user previews a tarball SDK install for arch gfx1200 pinned to ROCm 10.0.0")] async fn preview_pinned_tarball_install(world: &mut E2eWorld) { preview_ok( @@ -1156,10 +1580,11 @@ async fn preview_prepared_no_python(world: &mut E2eWorld) { /// The state half of the claim: the recording `uv` was never run (the evidence /// that nothing was prepared), no managed tools folder, no saved configuration, -/// and, as a backstop only, no tool archive in the scenario's cache. Every -/// scenario that calls this previews the canonical layout (no ROCm 10 pin), the -/// only layout whose preview needs no `uv`; a ROCm 10 preview can legitimately -/// create the managed tools folder to resolve package versions. +/// and, as a backstop only, no tool archive in the scenario's cache. It holds for +/// both layouts, because a preview never fetches `uv`: the canonical layout needs +/// none, and the ROCm 10 layout runs one only when it is already present (see +/// runtime-27 and runtime-28). A scenario that gives the preview a `uv` and expects +/// it to run must not call this. fn assert_no_python_prepared(world: &E2eWorld) { let paths = world.app_paths(); let scenario_root = root(world).to_path_buf(); @@ -1247,8 +1672,10 @@ async fn managed_python_version_overridden(world: &mut E2eWorld) { .push(("ROCM_CLI_MANAGED_PYTHON_VERSION", "3.13".into())); } -#[when("the user tries to preview a wheel SDK install for arch gfx1200 with no version pin")] -async fn try_preview_unpinned_wheel_install(world: &mut E2eWorld) { +#[when( + "the user tries to preview a wheel SDK install for arch gfx1200 pinned to an earlier ROCm 7 release" +)] +async fn try_preview_earlier_release_wheel_install(world: &mut E2eWorld) { preview( world, &[ @@ -1260,6 +1687,8 @@ async fn try_preview_unpinned_wheel_install(world: &mut E2eWorld) { "wheel", "--family", RAW_ARCH, + "--version", + CURRENT_ROCM_VERSION, "--dry-run", ], ); @@ -1300,6 +1729,112 @@ async fn preview_refused_for_python_version(world: &mut E2eWorld) { assert_no_python_prepared(world); } +/// The four pins `uv pip compile` would print for the ROCm 10 fixture, one +/// `name==version` per line, which the CLI reads back as the chosen stack. +#[given("the recording uv answers uv pip compile with the ROCm 10 fixture's versions")] +async fn recording_uv_answers_compile(world: &mut E2eWorld) { + let pins = [ + ("rocm", NEXT_ROCM_VERSION), + ("torch", NEXT_TORCH_VERSION), + ("torchvision", NEXT_TORCHVISION_VERSION), + ("torchaudio", NEXT_TORCHAUDIO_VERSION), + ] + .map(|(name, version)| format!("{name}=={version}")) + .join("\n"); + world.command_env.push(("FAKE_UV_STDOUT", pins.into())); + // The CLI pipes the requirements into `uv pip compile -`; see the stand-in. + world.command_env.push(("FAKE_UV_DRAIN_STDIN", "1".into())); +} + +/// A `uv` that is present is still used: it ran `uv pip compile`, once and for +/// nothing else, and the preview did not report the step as skipped. The log is +/// the evidence, since the plan's versions alone could not tell a compile from a +/// guess. +#[then("the preview resolved the versions with the uv that was present")] +async fn preview_resolved_with_present_uv(world: &mut E2eWorld) { + let log_path = root(world).join(UV_INVOCATION_LOG); + let log = std::fs::read_to_string(&log_path).unwrap_or_else(|error| { + panic!( + "the recording uv has no log at {}, so the preview never ran it: {error}", + log_path.display() + ) + }); + let runs: Vec<&str> = log.lines().collect(); + assert!( + matches!(runs.as_slice(), [run] if run.starts_with("pip compile ")), + "the preview should run uv once, for `uv pip compile`, and for nothing else:\n{log}" + ); + let out = stdout(world); + for unresolved in ["resolution_note:", "not resolved by this preview"] { + assert!( + !out.contains(unresolved), + "the preview ran uv but reported the step as skipped ({unresolved:?}):\n{out}" + ); + } +} + +/// What a preview of the ROCm 10 layout says when no `uv` is there to run, and the +/// state it left. The claim and the filesystem are checked together: the note alone +/// would pass with a preview that also downloaded `uv`, and the empty folders alone +/// would pass if the preview had stopped before it got to the question. +#[then("the preview says it did not resolve the ROCm 10 package versions, as it says")] +async fn preview_did_not_resolve_without_uv(world: &mut E2eWorld) { + assert_contains( + world, + "No uv is installed; a real install would fetch it and resolve the torch and torchvision \ + versions with `uv pip compile`. This preview does not.", + "the progress line about what the preview skipped", + ); + assert_contains( + world, + "resolution_note: no uv is installed, so a real install would fetch it and run `uv pip \ + compile` to choose the torch and torchvision versions and to check that the published \ + packages install together, which this preview did not do", + "the note about what the preview skipped", + ); + // Everything made from the versions `uv` would choose is marked, not guessed. + for key in [ + "target", + "runtime_key", + "package_specs", + "command", + "manifest", + ] { + assert_contains( + world, + &format!("\n {key}: not resolved by this preview\n"), + "a line made from the unchosen versions", + ); + } + // What needs no `uv` is still printed, from the same decisions a real install + // takes: the exact device payload, the requirements `uv` would be asked about, + // and the pinned torchaudio. + assert_contains( + world, + &format!("device_target: {RAW_ARCH}"), + "the device target", + ); + assert_version_resolution_extras(world, false); + assert_contains( + world, + &format!("selected_rocm_version: {NEXT_ROCM_VERSION}"), + "the ROCm version", + ); + assert_contains( + world, + &format!("torchaudio_selection: torchaudio {NEXT_TORCHAUDIO_VERSION}, "), + "the torchaudio pin", + ); + for chosen_by_uv in [NEXT_TORCH_VERSION, NEXT_TORCHVISION_VERSION] { + assert!( + !stdout(world).contains(chosen_by_uv), + "the preview named {chosen_by_uv}, which only `uv pip compile` could have chosen:\n{}", + stdout(world) + ); + } + assert_no_python_prepared(world); +} + #[then("the preview resolves the canonical release pip index")] async fn preview_resolves_canonical_pip_index(world: &mut E2eWorld) { let base = current_pip_base(world); @@ -1330,12 +1865,27 @@ async fn preview_never_mentions_next_index(world: &mut E2eWorld) { let next = next_pip_base(world); assert!( !stdout(world).contains(&next), - "an unpinned release install reached the ROCm 10 index {next}:\n{}", + "a release install pinned to an earlier ROCm 7 release reached the ROCm 10 index {next}:\n{}", stdout(world) ); assert!( !stdout(world).contains("next-v1"), - "an unpinned release install selected the next source layout:\n{}", + "a release install pinned to an earlier ROCm 7 release selected the next source layout:\n{}", + stdout(world) + ); +} + +#[then("the preview never mentions the canonical release pip index")] +async fn preview_never_mentions_canonical_index(world: &mut E2eWorld) { + let canonical = current_pip_base(world); + assert!( + !stdout(world).contains(&canonical), + "an unpinned release install read the earlier release stream {canonical}:\n{}", + stdout(world) + ); + assert!( + !stdout(world).contains("multi-arch-v2"), + "an unpinned release install selected the earlier source layout:\n{}", stdout(world) ); } @@ -1422,7 +1972,7 @@ fn assert_version_resolution_extras(world: &mut E2eWorld, include_devel: bool) { }; let expected = format!( "version_resolution_specs: rocm[{extras},device-{RAW_ARCH}]=={NEXT_ROCM_VERSION} \ - torch[device-{RAW_ARCH}] torchvision[device-{RAW_ARCH}] \ + torch[device-{RAW_ARCH}]==2.12.* torchvision[device-{RAW_ARCH}]==0.27.* \ torchaudio=={NEXT_TORCHAUDIO_VERSION}" ); assert_contains(world, &expected, "version resolution extras"); @@ -1821,15 +2371,15 @@ async fn assert_vllm_rocm10_discovery_pins(world: &mut E2eWorld) { async fn release_tarball_catalog_only(world: &mut E2eWorld) { let served = root(world).join("therock-release-tarball-fixture"); write_fixture( - &served.join("tarball").join("current").join("index.html"), - &tarball_index_html(&[(CURRENT_TARBALL, 1_787_000_000.0)]), + &served.join("tarball").join("next").join("index.html"), + &tarball_index_html(&[(NEXT_REAL_TARBALL, 1_787_000_000.0)]), ); world.artifact_server = Some(LoopbackServer::start(&served)); allow_base_overrides(world); let base = server_base(world); world.command_env.push(( - "ROCM_CLI_THEROCK_RELEASE_TARBALL_BASE", - format!("{base}/tarball/current/").into(), + "ROCM_CLI_THEROCK_NEXT_TARBALL_BASE", + format!("{base}/tarball/next/").into(), )); } @@ -1845,8 +2395,8 @@ async fn host_rocm_with_unreadable_suffix(world: &mut E2eWorld) { #[when("the user previews a tarball SDK install")] async fn preview_unpinned_tarball_install(world: &mut E2eWorld) { - // No `--version`: an unpinned release install reaches the canonical - // catalog, which the given-step populated with `CURRENT_TARBALL` alone. + // No `--version`: an unpinned release install reaches the newest layout's + // catalog, which the given-step populated with `NEXT_REAL_TARBALL` alone. // `--family` keeps the preview off host GPU detection. preview_ok( world, @@ -1858,7 +2408,7 @@ async fn preview_unpinned_tarball_install(world: &mut E2eWorld) { "--format", "tarball", "--family", - GROUP_FAMILY, + RAW_ARCH, "--dry-run", ], ); @@ -1868,12 +2418,12 @@ async fn preview_unpinned_tarball_install(world: &mut E2eWorld) { async fn preview_selects_catalog_newest(world: &mut E2eWorld) { assert_contains( world, - &format!("tarball: {CURRENT_TARBALL}"), + &format!("tarball: {NEXT_REAL_TARBALL}"), "selected artifact", ); assert_contains( world, - &format!("latest_version: {CURRENT_TARBALL_VERSION}"), + &format!("latest_version: {NEXT_TARBALL_VERSION}"), "selected artifact version", ); } @@ -1887,7 +2437,7 @@ async fn preview_explains_host_newer(world: &mut E2eWorld) { world, &format!( "version_note: this host reports ROCm {HOST_ROCM_NEWER_WITH_BUILD_NUMBER}, but \ - {CURRENT_TARBALL_VERSION} is the newest TheRock ROCm tarball for this GPU family, \ + {NEXT_TARBALL_VERSION} is the newest TheRock ROCm tarball for this GPU family, \ so it is selected" ), "host-newer version note", @@ -1904,3 +2454,238 @@ async fn preview_does_not_claim_host_newer(world: &mut E2eWorld) { "the preview claimed an unreadable host ROCm version is newer:\n{preview}" ); } + +/// Both nightly streams on one loopback server: the earlier (canonical) one with +/// its last build, and the newest layout's with a build that has its PyTorch +/// wheels and a newer `rocm` build that does not yet. +#[given( + "an earlier nightly pip index fixture and a newest-layout nightly pip index fixture whose newest build has no PyTorch wheels yet" +)] +async fn nightly_pip_index_fixtures(world: &mut E2eWorld) { + let served = root(world).join("therock-pip-fixtures"); + write_pip_index( + &served.join("nightly-earlier"), + [ + ("rocm", EARLIER_NIGHTLY_ROCM_VERSION), + ("torch", EARLIER_NIGHTLY_TORCH_VERSION), + ("torchvision", EARLIER_NIGHTLY_TORCHVISION_VERSION), + ("torchaudio", EARLIER_NIGHTLY_TORCHAUDIO_VERSION), + ], + ); + let next = served.join("nightly-next"); + write_fixture(&next.join("index.html"), &aggregate_root_html()); + let rocm = [ + (NEXT_NIGHTLY_ROCM_VERSION, ANY_WHEEL_TAG, Vec::new()), + (NEXT_NIGHTLY_UNBUILT_ROCM_VERSION, ANY_WHEEL_TAG, Vec::new()), + ]; + let torch = [( + NEXT_NIGHTLY_TORCH_VERSION, + ANY_WHEEL_TAG, + vec![format!("rocm=={NEXT_NIGHTLY_ROCM_VERSION}")], + )]; + let torchvision = [( + NEXT_NIGHTLY_TORCHVISION_VERSION, + ANY_WHEEL_TAG, + vec![format!("torch=={NEXT_NIGHTLY_TORCH_VERSION}")], + )]; + let torchaudio = [(NEXT_NIGHTLY_TORCHAUDIO_VERSION, ANY_WHEEL_TAG, Vec::new())]; + write_wheel_index_builds(&next.join("rocm"), "rocm", &rocm); + write_wheel_index_builds(&next.join("torch"), "torch", &torch); + write_wheel_index_builds(&next.join("torchvision"), "torchvision", &torchvision); + write_wheel_index_builds(&next.join("torchaudio"), "torchaudio", &torchaudio); + world.artifact_server = Some(LoopbackServer::start(&served)); + arm_nightly_fixture_env(world); +} + +/// Points the next CLI call at the running nightly fixtures; see +/// [`arm_pip_fixture_env`] for why a second call has to arm it again. +fn arm_nightly_fixture_env(world: &mut E2eWorld) { + allow_base_overrides(world); + let base = server_base(world); + world.command_env.push(( + "ROCM_CLI_THEROCK_NIGHTLY_PIP_BASE", + format!("{base}/nightly-earlier").into(), + )); + world.command_env.push(( + "ROCM_CLI_THEROCK_NEXT_NIGHTLY_PIP_BASE", + format!("{base}/nightly-next").into(), + )); +} + +/// A tarball catalog can be pinned only in the newest layout, so a nightly pin +/// into the earlier stream is refused. The refusal names the wheel install as +/// the way to still get it, so that command is run. +#[when( + "the user tries to preview a nightly tarball SDK install for arch gfx1200 pinned to the earlier stream's last build date" +)] +async fn try_preview_earlier_nightly_tarball_install(world: &mut E2eWorld) { + preview( + world, + &[ + "install", + "sdk", + "--channel", + "nightly", + "--format", + "tarball", + "--family", + RAW_ARCH, + "--build-date", + "2026-08-22", + "--dry-run", + ], + ); +} + +#[then( + "the failure names the first build date of the newer layout and points to a wheel install that resolves that earlier nightly" +)] +async fn failure_points_to_a_wheel_install_for_the_earlier_nightly(world: &mut E2eWorld) { + let reported = format!( + "{}\n{}", + stdout(world), + world.cli_stderr.as_deref().unwrap_or_default() + ); + assert!( + reported.contains("use `--format wheel`"), + "the refusal did not point to a wheel install:\n{reported}" + ); + assert!( + reported.contains("on or after 2026-08-23"), + "the refusal did not name the first build date of the newer layout:\n{reported}" + ); + arm_nightly_fixture_env(world); + let rc = preview( + world, + &nightly_wheel_preview_args(&["--build-date", "2026-08-22"]), + ); + assert_eq!( + rc, + 0, + "the wheel install the refusal points to failed:\n{}", + stdout(world) + ); + assert_contains( + world, + &format!("selected_rocm_version: {EARLIER_NIGHTLY_ROCM_VERSION}"), + "the earlier nightly the refusal points to", + ); +} + +fn nightly_wheel_preview_args<'a>(pin: &[&'a str]) -> Vec<&'a str> { + let mut args = vec![ + "install", + "sdk", + "--channel", + "nightly", + "--format", + "wheel", + "--family", + RAW_ARCH, + ]; + args.extend_from_slice(pin); + args.push("--dry-run"); + args +} + +#[when("the user previews a nightly wheel SDK install for arch gfx1200 with no version pin")] +async fn preview_unpinned_nightly_wheel_install(world: &mut E2eWorld) { + preview_ok(world, &nightly_wheel_preview_args(&[])); +} + +#[when( + "the user previews a nightly wheel SDK install for arch gfx1200 pinned to the earlier stream's last build date" +)] +async fn preview_earlier_nightly_wheel_install(world: &mut E2eWorld) { + preview_ok( + world, + &nightly_wheel_preview_args(&["--build-date", "2026-08-22"]), + ); +} + +#[when( + "the user previews a nightly wheel SDK install for arch gfx1200 pinned to a build date from the newest layout" +)] +async fn preview_next_nightly_wheel_install(world: &mut E2eWorld) { + preview_ok( + world, + &nightly_wheel_preview_args(&["--build-date", "2026-11-10"]), + ); +} + +fn assert_nightly_preview(world: &E2eWorld, base: &str, version: &str, generation: &str) { + assert_contains( + world, + &format!("canonical_source: {base}"), + "nightly source", + ); + assert_contains( + world, + &format!("index_url: {base}"), + "resolved nightly index", + ); + assert_contains( + world, + &format!("selected_rocm_version: {version}"), + "resolved nightly version", + ); + assert_contains( + world, + &format!("source_layout_generation: {generation}"), + "nightly layout generation", + ); +} + +#[then( + "the preview resolves the newest layout's nightly pip index at its newest installable build" +)] +async fn preview_resolves_next_nightly_index(world: &mut E2eWorld) { + let base = format!("{}/nightly-next", server_base(world)); + assert_nightly_preview(world, &base, NEXT_NIGHTLY_ROCM_VERSION, "next-v1"); + // The plan is built from the chosen build, not the newer one that has no + // PyTorch wheels, so it never names the build it could not install. + assert!( + !stdout(world).contains(&format!("=={NEXT_NIGHTLY_UNBUILT_ROCM_VERSION}")), + "the plan pins the build that has no PyTorch wheels:\n{}", + stdout(world) + ); +} + +#[then( + "the preview warns that the newest nightly build has no PyTorch wheels and installs the build before it" +)] +async fn preview_warns_newest_nightly_has_no_wheels(world: &mut E2eWorld) { + assert_contains( + world, + &format!( + "warning: ROCm {NEXT_NIGHTLY_UNBUILT_ROCM_VERSION} is the newest version in this repository but has no installable PyTorch wheels" + ), + "no-wheels warning", + ); +} + +#[then("the preview resolves the earlier stream's nightly pip index")] +async fn preview_resolves_earlier_nightly_index(world: &mut E2eWorld) { + let base = format!("{}/nightly-earlier", server_base(world)); + assert_nightly_preview(world, &base, EARLIER_NIGHTLY_ROCM_VERSION, "multi-arch-v2"); + // Both fixtures are served; the pin must not reach the newest layout's. + let next = format!("{}/nightly-next", server_base(world)); + assert!( + !stdout(world).contains(&next) && !stdout(world).contains("next-v1"), + "a pin to the earlier stream's last build date reached the newest layout:\n{}", + stdout(world) + ); +} + +#[then("the preview resolves the newest layout's nightly pip index for that build date")] +async fn preview_resolves_next_nightly_index_for_the_build_date(world: &mut E2eWorld) { + let base = format!("{}/nightly-next", server_base(world)); + assert_nightly_preview(world, &base, NEXT_NIGHTLY_ROCM_VERSION, "next-v1"); + // Both fixtures are served; the pin must not fall back to the earlier one. + let earlier = format!("{}/nightly-earlier", server_base(world)); + assert!( + !stdout(world).contains(&earlier) && !stdout(world).contains("multi-arch-v2"), + "a pin to a newest-layout build date reached the earlier stream:\n{}", + stdout(world) + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index f818136f80..61d1757d3c 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -36,7 +36,7 @@ use e2e_cucumber::panic_capture::panic_message; use e2e_cucumber::quit_and_wait::{ExitPoll, QuitError, quit_and_wait as retry_quit_and_wait}; use e2e_cucumber::reader_failure::{ReaderFailure, ReaderFailureObservation}; use e2e_cucumber::send_until::{RetryTiming, TerminalState, send_until as retry_send_until}; -use e2e_cucumber::terminal_size::non_zero_dimensions; +use e2e_cucumber::terminal_size::resize_screen_then_pty; use crate::E2eWorld; @@ -645,28 +645,24 @@ impl TuiSession { /// Resize both the real PTY and the emulated screen to `rows`x`cols`. The /// application receives the normal terminal resize event; assertions /// continue to inspect exactly what a user would see at the new geometry. + /// + /// The emulated screen is resized first and the PTY second, through + /// [`resize_screen_then_pty`]: the other order lets the application's + /// redraw for the new size be parsed by the old, narrower screen whenever + /// this thread is descheduled between the two calls, which corrupts the + /// screen for good (see that function). fn resize_to(&mut self, rows: u16, cols: u16) -> Result<(), String> { - // Validate before resizing the PTY so a zero dimension fails cleanly - // instead of leaving the PTY and the emulated screen at different sizes. - let (vt_rows, vt_cols) = non_zero_dimensions(rows, cols)?; - let size = PtySize { - rows, - cols, - pixel_width: 0, - pixel_height: 0, - }; - self.master - .resize(size) - .map_err(|e| format!("failed to resize pty: {e}"))?; - // As in `screen_snapshot`, recover rather than fail on a poisoned lock — - // resizing is still meaningful even if some earlier operation panicked - // while holding it. - self.parser - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .screen_mut() - .set_size(vt_rows, vt_cols); - Ok(()) + let master = &self.master; + resize_screen_then_pty(&self.parser, rows, cols, || { + master + .resize(PtySize { + rows, + cols, + pixel_width: 0, + pixel_height: 0, + }) + .map_err(|e| format!("failed to resize pty: {e}")) + }) } /// Enlarge to [`DETAIL_ROWS`]x[`DETAIL_COLS`], for journeys that assert diff --git a/tests/e2e-cucumber/tests/e2e/uninstall_steps.rs b/tests/e2e-cucumber/tests/e2e/uninstall_steps.rs index 3b818cffb5..49a16b9436 100644 --- a/tests/e2e-cucumber/tests/e2e/uninstall_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/uninstall_steps.rs @@ -14,6 +14,7 @@ use std::path::{Path, PathBuf}; use cucumber::{given, then, when}; use crate::E2eWorld; +use crate::e2e::tui_driver::{TuiSession, default_timeout}; const USER_FILE: &str = "keep.txt"; @@ -76,6 +77,19 @@ async fn cache_behind_looping_link(world: &mut E2eWorld) { E2eWorld::link_directory(&looping, &looping).expect("failed to plant looping link"); } +/// HOME, the uv cache and the Hugging Face cache inside the scenario's own +/// folder, so nothing a run resolves can reach the runner's shared caches. +fn isolated_env(world: &E2eWorld) -> Vec<(&'static str, String)> { + let root = root(world); + let home = root.join("home"); + std::fs::create_dir_all(&home).expect("failed to create isolated HOME"); + vec![ + ("HOME", home.display().to_string()), + ("UV_CACHE_DIR", root.join("uv-cache").display().to_string()), + ("HF_HOME", root.join("hf-home").display().to_string()), + ] +} + fn run_uninstall(world: &mut E2eWorld, keep: &[&str], settings: &[(&str, &str)]) { run_uninstall_with(world, true, keep, settings); } @@ -89,24 +103,25 @@ fn run_uninstall_with( keep: &[&str], settings: &[(&str, &str)], ) { - let root = root(world).to_path_buf(); - let home = root.join("home"); - std::fs::create_dir_all(&home).expect("failed to create isolated HOME"); - let home = home.display().to_string(); - let uv = root.join("uv-cache").display().to_string(); - let hf = root.join("hf-home").display().to_string(); let mut args = vec!["uninstall", "--keep-binaries"]; if confirmed { args.push("--yes"); } args.extend_from_slice(keep); - let mut envs = vec![ - ("HOME", home.as_str()), - ("UV_CACHE_DIR", &uv), - ("HF_HOME", &hf), - ]; + run_uninstall_args(world, &args, settings); +} + +/// Run `rocm` with exactly `args`, confined to the scenario's own folders like +/// every other uninstall here. +fn run_uninstall_args(world: &mut E2eWorld, args: &[&str], settings: &[(&str, &str)]) { + assert_keeps_binaries(args); + let isolated = isolated_env(world); + let mut envs = isolated + .iter() + .map(|(name, value)| (*name, value.as_str())) + .collect::>(); envs.extend_from_slice(settings); - let (stdout, stderr, rc) = crate::run_rocm_with_env(world, &args, &envs); + let (stdout, stderr, rc) = crate::run_rocm_with_env(world, args, &envs); world.cli_output = Some(stdout); world.cli_stderr = Some(stderr); world.cli_rc = Some(rc); @@ -157,6 +172,145 @@ async fn uninstall_config_unconfirmed(world: &mut E2eWorld) { ); } +/// Run `rocm uninstall` of the config folder only, without `--yes`, under a real +/// terminal, wait for the confirmation prompt and send `keys` as the answer. +/// +/// The piped harness can never reach the prompt (no terminal, so the command is +/// refused first); a pseudo-terminal is the only way to answer it. The screen +/// the user would be looking at is kept as the scenario's output. +async fn answer_uninstall_prompt(world: &mut E2eWorld, keys: &str) { + let root = root(world).to_path_buf(); + let home = root.join("home"); + std::fs::create_dir_all(&home).expect("failed to create isolated HOME"); + let home = home.display().to_string(); + let uv = root.join("uv-cache").display().to_string(); + let hf = root.join("hf-home").display().to_string(); + let config = config_dir(world).display().to_string(); + let mut session = TuiSession::spawn_with_env( + world, + &[ + "uninstall", + "--keep-binaries", + "--keep-data", + "--keep-cache", + ], + &[ + ("HOME", home.as_str()), + ("UV_CACHE_DIR", uv.as_str()), + ("HF_HOME", hf.as_str()), + ("ROCM_CLI_CONFIG_DIR", config.as_str()), + ], + ) + .unwrap_or_else(|e| panic!("failed to open the uninstall prompt: {e}")); + session + .wait_for_screen("Proceed with uninstall? [y/N]:", default_timeout()) + .await + .unwrap_or_else(|e| panic!("the confirmation prompt never appeared: {e}")); + session + .send(keys) + .unwrap_or_else(|e| panic!("failed to answer the prompt: {e}")); + let rc = session + .wait_for_exit_code(default_timeout()) + .await + .unwrap_or_else(|e| panic!("the CLI never exited after the answer: {e}")); + world.cli_output = Some(session.screen_text()); + world.cli_rc = Some(rc); + world.tui = Some(session); +} + +#[when( + "the user is asked to confirm an uninstall on a real terminal and types an answer that is neither yes nor no" +)] +async fn uninstall_prompt_unrecognized(world: &mut E2eWorld) { + answer_uninstall_prompt(world, "maybe\r").await; +} + +#[when( + "the user is asked to confirm an uninstall on a real terminal and ends the input instead of answering" +)] +async fn uninstall_prompt_end_of_input(world: &mut E2eWorld) { + // Ctrl-D on an empty line: the terminal reports end of input and echoes + // nothing, which is what left the cursor on the prompt line. + answer_uninstall_prompt(world, "\x04").await; +} + +#[when("the user is asked to confirm an uninstall on a real terminal and types no")] +async fn uninstall_prompt_no(world: &mut E2eWorld) { + answer_uninstall_prompt(world, "no\r").await; +} + +/// The screen's lines with trailing padding removed. +fn screen_lines(world: &E2eWorld) -> Vec { + output(world) + .lines() + .map(|l| l.trim_end().to_owned()) + .collect() +} + +#[then("the user is told the answer was not understood and is treated as no")] +async fn told_answer_not_understood(world: &mut E2eWorld) { + let lines = screen_lines(world); + let expected = "\"maybe\" is not yes or no; treating it as no."; + assert!( + lines.iter().any(|l| l == expected), + "expected a `{expected}` line on the terminal:\n{}", + output(world) + ); +} + +#[then("the user is not told the answer was misunderstood")] +async fn not_told_answer_misunderstood(world: &mut E2eWorld) { + assert!( + !output(world).contains("is not yes or no"), + "a plain no must not be commented on:\n{}", + output(world) + ); +} + +#[then("the uninstall is reported as cancelled")] +async fn uninstall_reported_cancelled(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no exit code captured"); + assert_eq!( + rc, + 0, + "declining is not a failure, got exit {rc}:\n{}", + output(world) + ); + assert!( + screen_lines(world) + .iter() + .any(|l| l == "uninstall cancelled"), + "expected an `uninstall cancelled` line on its own:\n{}", + output(world) + ); +} + +#[then("the cancellation is reported on its own line, not on the prompt line")] +async fn cancellation_on_its_own_line(world: &mut E2eWorld) { + let lines = screen_lines(world); + let prompt = lines + .iter() + .find(|l| l.starts_with("Proceed with uninstall? [y/N]:")) + .unwrap_or_else(|| panic!("no prompt line on the terminal:\n{}", output(world))); + assert_eq!( + prompt, + "Proceed with uninstall? [y/N]:", + "nothing may follow the prompt after the input ended:\n{}", + output(world) + ); + assert!( + lines.iter().any(|l| l == "uninstall cancelled"), + "expected an `uninstall cancelled` line on its own:\n{}", + output(world) + ); + assert_eq!( + world.cli_rc, + Some(0), + "ending the input declines, it does not fail:\n{}", + output(world) + ); +} + /// The scenario's own home folder, standing in for the user's. fn home_dir(world: &E2eWorld) -> PathBuf { root(world).join("home") @@ -451,3 +605,326 @@ async fn config_intact(world: &mut E2eWorld) { output(world) ); } + +/// Every run here keeps the binaries: the binary under test must survive, and an +/// uninstall that reached it would delete the program the scenario is running. +/// Checked at the point a run starts, so no step can reach it by accident. +fn assert_keeps_binaries(args: &[&str]) { + assert!( + args.contains(&"--keep-binaries"), + "an uninstall in a scenario must pass --keep-binaries: {args:?}" + ); +} + +/// The arguments for uninstalling only the config folder, as a dry run or for real. +fn config_only_args(dry_run: bool) -> Vec<&'static str> { + let mut args = vec![ + "uninstall", + "--keep-binaries", + "--keep-data", + "--keep-cache", + ]; + if dry_run { + args.push("--dry-run"); + } + args +} + +/// Run `rocm` under a pseudo-terminal, confined to the scenario's folders like +/// every other uninstall here, so the command sees a terminal on both streams as +/// a person typing it would. Answers its `[y/N]` prompt when `answer` is given. +async fn run_uninstall_at_terminal(world: &mut E2eWorld, args: &[&str], answer: Option<&str>) { + assert_keeps_binaries(args); + let root = root(world).to_path_buf(); + let home = root.join("home"); + std::fs::create_dir_all(&home).expect("failed to create isolated HOME"); + let home = home.display().to_string(); + let uv = root.join("uv-cache").display().to_string(); + let hf = root.join("hf-home").display().to_string(); + let config = config_dir(world).display().to_string(); + let mut session = TuiSession::spawn_with_env( + world, + args, + &[ + ("HOME", home.as_str()), + ("UV_CACHE_DIR", uv.as_str()), + ("HF_HOME", hf.as_str()), + ("ROCM_CLI_CONFIG_DIR", config.as_str()), + ], + ) + .unwrap_or_else(|e| panic!("failed to run {args:?} at a terminal: {e}")); + if let Some(answer) = answer { + session + .wait_for_screen("[y/N]:", default_timeout()) + .await + .unwrap_or_else(|e| panic!("the confirmation prompt never appeared: {e}")); + session + .send(&format!("{answer}\r")) + .unwrap_or_else(|e| panic!("failed to type the answer: {e}")); + } + let rc = session + .wait_for_exit_code(default_timeout()) + .await + .unwrap_or_else(|e| panic!("the uninstall never exited: {e}")); + world.cli_output = Some(session.screen_text()); + world.cli_stderr = None; + world.cli_rc = Some(rc); + world.tui = Some(session); +} + +#[when("the user dry-runs an uninstall of only the config")] +async fn dry_run_config_only(world: &mut E2eWorld) { + let config = config_dir(world).display().to_string(); + run_uninstall_args( + world, + &config_only_args(true), + &[("ROCM_CLI_CONFIG_DIR", &config)], + ); +} + +#[when("the user dry-runs an uninstall of only the config at a terminal")] +async fn dry_run_config_only_at_terminal(world: &mut E2eWorld) { + run_uninstall_at_terminal(world, &config_only_args(true), None).await; +} + +#[when("the user runs it again without --dry-run at a terminal and answers yes")] +async fn run_it_again_and_confirm(world: &mut E2eWorld) { + // The same arguments as the dry run, without `--dry-run`: the step the + // dry run's last line names. + run_uninstall_at_terminal(world, &config_only_args(false), Some("y")).await; +} + +#[when("the user dry-runs an uninstall that keeps everything at a terminal")] +async fn dry_run_keeping_everything_at_terminal(world: &mut E2eWorld) { + run_uninstall_at_terminal( + world, + &[ + "uninstall", + "--keep-binaries", + "--keep-config", + "--keep-data", + "--keep-cache", + "--dry-run", + ], + None, + ) + .await; +} + +#[then("the dry run says to run it again without --dry-run")] +async fn dry_run_says_to_run_it_again(world: &mut E2eWorld) { + let stdout = output(world); + assert!( + stdout.contains( + "Nothing was removed. To remove these items, run it again without --dry-run." + ), + "the dry run must say how to remove what it listed:\n{stdout}" + ); + assert!( + !stdout.contains("Review uninstall"), + "the command line has no such control:\n{stdout}" + ); +} + +#[then("the dry run lists the config folder")] +async fn dry_run_lists_the_config(world: &mut E2eWorld) { + let stdout = output(world); + let config = config_dir(world).display().to_string(); + assert!( + stdout.contains(&format!("config: {config}")), + "the plan must list the config folder ({config}):\n{stdout}" + ); +} + +#[then("the config folder is gone")] +async fn config_gone(world: &mut E2eWorld) { + let config = config_dir(world); + assert!( + std::fs::symlink_metadata(&config).is_err(), + "running it again as the dry run said did not remove {}:\n{}", + config.display(), + output(world) + ); +} + +#[then("the dry run offers no step to remove anything")] +async fn dry_run_offers_no_step(world: &mut E2eWorld) { + let stdout = output(world); + for phrase in [ + "Nothing was removed.", + "run it again", + "Review uninstall", + "approval", + ] { + assert!( + !stdout.contains(phrase), + "this dry run must not offer {phrase:?}:\n{stdout}" + ); + } +} + +/// The assistant's uninstall preview, called without a model through the hidden +/// `rocm mcp-call` entry point the dashboard and chat use for their tools. +/// +/// The one run here that cannot pass `--keep-binaries`: the preview takes no +/// flags, plans every folder and the binary under test, and renders the plan. It +/// removes nothing, and it is made without `--allow-mutation`, so the tool +/// refuses a call that would remove something instead of running it. +#[when("the assistant previews an uninstall")] +async fn assistant_previews_uninstall(world: &mut E2eWorld) { + let config = config_dir(world).display().to_string(); + let isolated = isolated_env(world); + let mut envs = isolated + .iter() + .map(|(name, value)| (*name, value.as_str())) + .collect::>(); + envs.push(("ROCM_CLI_CONFIG_DIR", &config)); + let arguments = serde_json::json!({ "args": ["uninstall", "--dry-run"] }).to_string(); + let (stdout, stderr, rc) = crate::run_rocm_with_env( + world, + &["mcp-call", "rocm_command", "--arguments-json", &arguments], + &envs, + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// What the preview printed: the command's own text inside the tool result. +fn preview_text(world: &E2eWorld) -> String { + let result: serde_json::Value = serde_json::from_str(output(world)).unwrap_or_else(|error| { + panic!("the tool result is not JSON ({error}):\n{}", output(world)) + }); + result["structuredContent"]["stdout"] + .as_str() + .unwrap_or_else(|| panic!("the tool result has no command output:\n{}", output(world))) + .to_owned() +} + +#[then("the preview succeeds")] +async fn preview_succeeds(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no exit code captured"); + assert!( + rc == 0, + "the preview must exit 0, got {rc}\nstdout:\n{}\nstderr:\n{}", + output(world), + world.cli_stderr.as_deref().unwrap_or("") + ); + let result: serde_json::Value = serde_json::from_str(output(world)).expect("tool result JSON"); + assert_eq!( + result["isError"], + serde_json::Value::Bool(false), + "the tool reported an error:\n{}", + output(world) + ); +} + +#[then("the preview lists the config folder")] +async fn preview_lists_the_config(world: &mut E2eWorld) { + let text = preview_text(world); + let config = config_dir(world).display().to_string(); + assert!( + text.contains(&format!("config: {config}")), + "the preview must list the config folder ({config}):\n{text}" + ); +} + +#[then("the preview says removal needs approval")] +async fn preview_says_removal_needs_approval(world: &mut E2eWorld) { + let text = preview_text(world); + assert!( + text.contains("Nothing was removed. Removal needs your approval."), + "the preview must say removal needs approval:\n{text}" + ); + // The preview has no flag to offer, so it must not point at one. + for phrase in ["run it again", "--dry-run.", "Review uninstall"] { + assert!( + !text.contains(phrase), + "the assistant's preview must not offer {phrase:?}:\n{text}" + ); + } +} + +#[when("the user uninstalls the config with --force")] +async fn uninstall_with_force(world: &mut E2eWorld) { + let config = config_dir(world).display().to_string(); + let mut args = config_only_args(false); + args.push("--force"); + run_uninstall_args(world, &args, &[("ROCM_CLI_CONFIG_DIR", &config)]); +} + +#[when("the user uninstalls the config with a near miss of --force")] +async fn uninstall_with_near_miss_of_force(world: &mut E2eWorld) { + let config = config_dir(world).display().to_string(); + let mut args = config_only_args(false); + args.push("--forc"); + run_uninstall_args(world, &args, &[("ROCM_CLI_CONFIG_DIR", &config)]); +} + +#[then("the uninstall is refused and told to use --yes instead")] +async fn refused_and_told_to_use_yes(world: &mut E2eWorld) { + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + assert_eq!( + world.cli_rc, + Some(1), + "a refused --force exits 1\nstderr:\n{stderr}" + ); + assert!( + stderr.contains("`--force` is not an option of `rocm uninstall`") + && stderr.contains("use `--yes` to skip the confirmation"), + "expected the refusal to name --yes:\n{stderr}" + ); + assert!( + !stderr.contains("force-dev-binaries"), + "the refusal must not name the destructive flag:\n{stderr}" + ); +} + +#[then("the error does not point at the flag that removes development binaries")] +async fn error_does_not_suggest_dev_binaries(world: &mut E2eWorld) { + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + assert_eq!( + world.cli_rc, + Some(2), + "an unknown flag exits 2\nstderr:\n{stderr}" + ); + assert!( + stderr.contains("unexpected argument"), + "expected the unknown-argument error on stderr:\n{stderr}" + ); + assert!( + !stderr.contains("force-dev-binaries"), + "the error must not name the destructive flag:\n{stderr}" + ); +} + +#[when("the user asks for the uninstall help")] +async fn ask_for_uninstall_help(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm_with_env(world, &["uninstall", "--help"], &[]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the help shows a dry run before the real run")] +async fn help_shows_dry_run_first(world: &mut E2eWorld) { + let stdout = output(world); + let examples = stdout.split_once("EXAMPLES:").map_or_else( + || panic!("the help has no examples:\n{stdout}"), + |(_, rest)| rest, + ); + let rows: Vec<&str> = examples + .lines() + .map(str::trim) + .filter(|row| !row.is_empty()) + .collect(); + assert_eq!( + rows.first().copied(), + Some("rocm uninstall --dry-run"), + "the first example must be the dry run:\n{examples}" + ); + assert!( + rows.contains(&"rocm uninstall"), + "the examples must also show the real run:\n{examples}" + ); +} diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index 230f791c8b..47dcb5d9d6 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -33,9 +33,13 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("diagnose.feature", "diagnose"), ("download_progress_pty.feature", "download-progress"), ("driver_install.feature", "driver-install"), + ("engine_ipc.feature", "engine-ipc"), ("engine_shell.feature", "engine-shell"), ("examine.feature", "examine"), ("install_lifecycle.feature", "lifecycle"), + // Not `lemonade`: the file is about the copies Lemonade leaves on disk, + // not about serving with it, which `model_serving.feature` covers. + ("lemonade_installs.feature", "lemonade-installs"), // Not `records`: a key has to say what kind of record, and this file is // about the ones a managed `rocm serve` leaves behind. ("local_server_records.feature", "server-records"), @@ -50,6 +54,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("runtime_lifecycle.feature", "runtime-lifecycle"), ("runtime_setup.feature", "runtime"), ("service_record_cleanup.feature", "service-cleanup"), + ("service_record_unqueryable.feature", "service-unqueryable"), ("service_stop.feature", "service-stop"), ("service_unknown_id.feature", "service-unknown"), ("storage.feature", "storage"), diff --git a/tests/install-sh/run.sh b/tests/install-sh/run.sh new file mode 100755 index 0000000000..d3bd624f01 --- /dev/null +++ b/tests/install-sh/run.sh @@ -0,0 +1,588 @@ +#!/bin/sh +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +# Drive install.sh with a locally signed bundle and a stub `rocm`. +# +# The case that matters: a bash (or zsh) login shell on a machine where fish +# is installed must get the fish PATH block, fish completions, and an uninstall +# record for both. A bash login shell with no fish install must not grow a fish +# config. Re-running must not append a second PATH block. A `rocm` that cannot +# print completions, or a fish config that cannot be written, must not stop the +# install or be reported as done. +# +# Usage: tests/install-sh/run.sh +# Set INSTALL_SH to test another copy of the installer (mutation checks). +# Requires: openssl, tar, sha256sum, and a POSIX sh. No network and no GPU. + +set -eu + +ROOT="$(CDPATH='' cd -- "$(dirname "$0")/../.." && pwd)" +INSTALLER="${INSTALL_SH:-${ROOT}/install.sh}" +# The installer executes the binary it just wrote. /tmp is often mounted +# noexec, so the scratch directory lives on this repo's filesystem. +mkdir -p "${ROOT}/.rocm-work" +WORK="$(mktemp -d "${ROOT}/.rocm-work/install-sh.XXXXXX")" +FAILURES=0 + +cleanup() { + rm -rf "${WORK}" +} +trap cleanup EXIT INT TERM + +fail() { + echo "FAIL: $*" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "ok: $*" +} + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing required command: $1" >&2 + exit 1 + } +} + +for cmd in openssl tar gzip sha256sum awk grep sed find mktemp install; do + need "${cmd}" +done + +TOOLS="${WORK}/tools" +FISH_BIN="${WORK}/fish-bin" +mkdir -p "${TOOLS}" "${FISH_BIN}" "${WORK}/tmp" + +link_tool() { + target="$(command -v "$1")" || { + echo "missing required command: $1" >&2 + exit 1 + } + ln -s "${target}" "${TOOLS}/$1" +} + +for cmd in sh cat awk sed grep rm mkdir mktemp tar gzip openssl sha256sum find head cp uname install dirname; do + link_tool "${cmd}" +done + +# A name on PATH is enough: the installer only tests `command -v fish`. +cat > "${FISH_BIN}/fish" <<'EOF' +#!/bin/sh +exit 0 +EOF +chmod 755 "${FISH_BIN}/fish" + +BUNDLE="${WORK}/bundle/rocm-cli-linux-amd64/bin" +mkdir -p "${BUNDLE}" +# Completions fail once when $HOME/.stub-completions says `fail-once`, and +# print nothing once when it says `empty-once`: the stub consumes the file, so +# the next run works, as a failure that clears by itself would. They also fail +# when the installer hands `rocm` its own stdin, which under `curl | sh` is the +# rest of the script. +cat > "${BUNDLE}/rocm" <<'EOF' +#!/bin/sh +if [ "${1:-}" = "completions" ] && [ "${2:-}" = "fish" ]; then + if IFS= read -r _line; then + printf '%s\n' "rocm read the installer's stdin: ${_line}" >&2 + exit 1 + fi + _mode="$(cat "${HOME}/.stub-completions" 2>/dev/null || true)" + rm -f "${HOME}/.stub-completions" + case "${_mode}" in + fail-once) printf '%s\n' "stub completions failure" >&2; exit 1 ;; + empty-once) exit 0 ;; + esac + printf '%s\n' 'complete -c rocm # stub completions' + exit 0 +fi +printf '%s\n' "unexpected rocm invocation: $*" >&2 +exit 1 +EOF +cat > "${BUNDLE}/rocmd" <<'EOF' +#!/bin/sh +exit 0 +EOF +chmod 755 "${BUNDLE}/rocm" "${BUNDLE}/rocmd" + +ARCHIVE="${WORK}/rocm-cli-linux-amd64.tar.gz" +tar -C "${WORK}/bundle" -czf "${ARCHIVE}" rocm-cli-linux-amd64 +sha256sum "${ARCHIVE}" | awk '{print $1}' > "${ARCHIVE}.sha256" +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${WORK}/private.pem" 2>/dev/null +openssl pkey -in "${WORK}/private.pem" -pubout -out "${WORK}/public.pem" +openssl dgst -sha256 -sign "${WORK}/private.pem" -out "${ARCHIVE}.sig" "${ARCHIVE}" + +BASE_PATH="${TOOLS}" +FISH_PATH="${FISH_BIN}:${TOOLS}" +MARKER='# >>> rocm-cli path >>>' +# Literal installer text, including the unexpanded $HOME and $PATH. +# shellcheck disable=SC2016 +FISH_SNIPPET='if not contains -- "$HOME/.local/bin" $PATH' +# shellcheck disable=SC2016 +POSIX_SNIPPET='case ":$PATH:" in' +COMPLETIONS_FIXTURE='complete -c rocm # stub completions' +# The line install.sh puts above the script so `rocm uninstall` can tell its +# own file from one the user or another install folder wrote. +COMPLETIONS_MARKER='# rocm-cli fish completions for ' + +# What install.sh writes to rocm.fish for an install under home $1. +installed_completions() { + printf '%s%s\n%s' "${COMPLETIONS_MARKER}" "$1/.local/bin" "${COMPLETIONS_FIXTURE}" +} +# The installer's stdin: what `curl | sh` would still have to run. +STDIN_FIXTURE="${WORK}/installer-stdin" +printf '%s\n' 'echo the rest of the installer script' > "${STDIN_FIXTURE}" + +install_failed() { + fail "installer exited non-zero" + echo "---- installer log ----" >&2 + cat "$1" >&2 +} + +# install_case [NAME=value ...] +install_case() { + log="$1" + home="$2" + shell="$3" + path="$4" + update="$5" + shift 5 + mkdir -p "${home}" + env -i \ + HOME="${home}" \ + SHELL="${shell}" \ + PATH="${path}" \ + TMPDIR="${WORK}/tmp" \ + ROCM_CLI_INSTALL_DIR="${home}/.local/bin" \ + ROCM_CLI_ARCHIVE="${ARCHIVE}" \ + ROCM_CLI_SIGNING_PUBLIC_KEY_PATH="${WORK}/public.pem" \ + ROCM_CLI_UPDATE_SHELL_PATH="${update}" \ + "$@" \ + "${INSTALL_SHELL:-${TOOLS}/sh}" "${INSTALLER}" release < "${STDIN_FIXTURE}" > "${log}" 2>&1 || install_failed "${log}" +} + +marker_count() { + if [ -f "$1" ]; then + grep -F -c "${MARKER}" "$1" || true + else + printf '%s\n' 0 + fi +} + +assert_contains() { + file="$1" + needle="$2" + label="$3" + if [ ! -f "${file}" ] || ! grep -F -q -- "${needle}" "${file}"; then + fail "${label}" + if [ -f "${file}" ]; then + echo "---- ${file} ----" >&2 + cat "${file}" >&2 + else + echo "---- ${file} (missing) ----" >&2 + fi + fi +} + +assert_absent_file() { + if [ -e "$1" ]; then + fail "$2" + echo "---- $1 ----" >&2 + cat "$1" >&2 || true + fi +} + +assert_eq() { + if [ "$2" != "$3" ]; then + fail "$1: expected '$3', got '$2'" + fi +} + +assert_file_eq() { + label="$1" + file="$2" + expected="$3" + if [ ! -f "${file}" ]; then + fail "${label}: missing ${file}" + return + fi + actual="$(cat "${file}")" + assert_eq "${label}" "${actual}" "${expected}" +} + +assert_log_contains() { + if ! grep -F -q -- "$2" "$1"; then + fail "$3" + echo "---- log ----" >&2 + cat "$1" >&2 + fi +} + +assert_log_lacks() { + if grep -F -q -- "$2" "$1"; then + fail "$3" + echo "---- log ----" >&2 + cat "$1" >&2 + fi +} + +# Bash login shell, fish on PATH: both profiles, completions, and the record. +bash_home="${WORK}/homes/bash-fish" +install_case "${WORK}/bash-fish.log" "${bash_home}" /bin/bash "${FISH_PATH}" 1 +bash_profile="${bash_home}/.bashrc" +fish_profile="${bash_home}/.config/fish/config.fish" +fish_completions="${bash_home}/.config/fish/completions/rocm.fish" +record="${bash_home}/.local/bin/.rocm-cli-path-edit" +assert_contains "${bash_profile}" "${POSIX_SNIPPET}" "bash profile missing the POSIX PATH block" +if grep -F -q -- "${FISH_SNIPPET}" "${bash_profile}"; then + fail "bash profile received the fish PATH snippet" +fi +assert_contains "${fish_profile}" "${FISH_SNIPPET}" "fish config missing the fish PATH snippet when SHELL is bash" +if grep -F -q -- "${POSIX_SNIPPET}" "${fish_profile}"; then + fail "fish config received the POSIX PATH snippet" +fi +assert_eq "bash marker count" "$(marker_count "${bash_profile}")" 1 +assert_eq "fish marker count" "$(marker_count "${fish_profile}")" 1 +assert_contains "${record}" "profile ${bash_profile}" "uninstall record missing the bash profile" +assert_contains "${record}" "profile ${fish_profile}" "uninstall record missing the fish profile" +# shellcheck disable=SC2016 +assert_contains "${record}" 'path-expr $HOME/.local/bin' "uninstall record missing the path expression" +assert_contains "${record}" "completions ${fish_completions}" "uninstall record missing the fish completions" +assert_eq "completions file" "$(cat "${fish_completions}")" "$(installed_completions "${bash_home}")" +assert_log_contains "${WORK}/bash-fish.log" "shell profile updated:" "installer did not report a shell profile update" +assert_log_contains "${WORK}/bash-fish.log" "profile: ${fish_profile}" "installer did not report the fish profile" +assert_log_contains "${WORK}/bash-fish.log" "fish completions installed:" "installer did not report fish completions" +assert_log_contains "${WORK}/bash-fish.log" "${fish_completions}" "installer did not report the completions path" +if [ "${FAILURES}" -eq 0 ]; then + pass "bash login shell also configures fish" +fi + +# A second install must not append another marked block or another record line. +install_case "${WORK}/bash-fish-again.log" "${bash_home}" /bin/bash "${FISH_PATH}" 1 +assert_eq "bash marker count after reinstall" "$(marker_count "${bash_profile}")" 1 +assert_eq "fish marker count after reinstall" "$(marker_count "${fish_profile}")" 1 +assert_eq "recorded bash profile lines" "$(grep -F -c "profile ${bash_profile}" "${record}" || true)" 1 +assert_eq "recorded fish profile lines" "$(grep -F -c "profile ${fish_profile}" "${record}" || true)" 1 +assert_eq "recorded completions lines" "$(grep -F -c "completions ${fish_completions}" "${record}" || true)" 1 +assert_eq "completions file after reinstall" "$(cat "${fish_completions}")" "$(installed_completions "${bash_home}")" +assert_log_contains "${WORK}/bash-fish-again.log" "fish shell profile already configured:" "reinstall did not report the existing fish profile" +if [ "${FAILURES}" -eq 0 ]; then + pass "reinstall does not duplicate the fish block" +fi + +# Bash, no fish binary and no fish config directory. +plain_home="${WORK}/homes/bash-plain" +install_case "${WORK}/bash-plain.log" "${plain_home}" /bin/bash "${BASE_PATH}" 1 +assert_contains "${plain_home}/.bashrc" "${POSIX_SNIPPET}" "bash profile missing when fish is absent" +assert_absent_file "${plain_home}/.config/fish/config.fish" "bash user with no fish install grew a fish config" +assert_absent_file "${plain_home}/.config/fish/completions/rocm.fish" "bash user with no fish install grew fish completions" +if grep -F -q -- "config.fish" "${plain_home}/.local/bin/.rocm-cli-path-edit"; then + fail "uninstall record names a fish profile that was not written" +fi +if [ "${FAILURES}" -eq 0 ]; then + pass "bash without fish does not grow a fish config" +fi + +# Fish config directory exists, but fish is not on PATH. +dir_home="${WORK}/homes/fish-dir" +mkdir -p "${dir_home}/.config/fish" +printf '%s\n' 'set -gx EDITOR vim' > "${dir_home}/.config/fish/config.fish" +install_case "${WORK}/fish-dir.log" "${dir_home}" /bin/bash "${BASE_PATH}" 1 +assert_contains "${dir_home}/.config/fish/config.fish" "set -gx EDITOR vim" "existing fish config lost its own line" +assert_contains "${dir_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "existing fish config was not given the PATH snippet" +assert_eq "fish marker count for an existing config dir" "$(marker_count "${dir_home}/.config/fish/config.fish")" 1 +assert_contains "${dir_home}/.local/bin/.rocm-cli-path-edit" "profile ${dir_home}/.config/fish/config.fish" "uninstall record missing fish when only the config dir exists" +assert_eq "completions from a config dir" "$(cat "${dir_home}/.config/fish/completions/rocm.fish")" "$(installed_completions "${dir_home}")" +if [ "${FAILURES}" -eq 0 ]; then + pass "an existing fish config dir is configured when fish is not on PATH" +fi + +# A fish config that already puts the install folder on PATH with the user's +# own line is left alone and not recorded: uninstall must not go looking for a +# block the installer never wrote there. +own_home="${WORK}/homes/fish-own-line" +own_config="${own_home}/.config/fish/config.fish" +mkdir -p "${own_home}/.config/fish" +# shellcheck disable=SC2016 +printf '%s\n' 'fish_add_path $HOME/.local/bin' > "${own_config}" +install_case "${WORK}/fish-own-line.log" "${own_home}" /bin/bash "${BASE_PATH}" 1 +# shellcheck disable=SC2016 +assert_eq "fish config with the user's own PATH line" "$(cat "${own_config}")" 'fish_add_path $HOME/.local/bin' +assert_log_contains "${WORK}/fish-own-line.log" "fish shell profile already configured:" "the user's own fish PATH line was not reported as configured" +if grep -F -q -- "profile ${own_config}" "${own_home}/.local/bin/.rocm-cli-path-edit"; then + fail "a fish config holding only the user's own PATH line was recorded" +fi +assert_contains "${own_home}/.local/bin/.rocm-cli-path-edit" "profile ${own_home}/.bashrc" "the bash profile beside the user's own fish line was not recorded" +if [ "${FAILURES}" -eq 0 ]; then + pass "a fish config with the user's own PATH line is not recorded" +fi + +# Login shell is fish. The fish profile is the one the login-shell path writes, +# and completions are still installed. Bash is left alone. +fish_home="${WORK}/homes/fish-login" +install_case "${WORK}/fish-login.log" "${fish_home}" "${FISH_BIN}/fish" "${FISH_PATH}" 1 +assert_contains "${fish_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "fish login shell did not get the fish snippet" +assert_eq "fish login marker count" "$(marker_count "${fish_home}/.config/fish/config.fish")" 1 +assert_absent_file "${fish_home}/.bashrc" "fish login shell also wrote a bash profile" +assert_eq "fish login completions" "$(cat "${fish_home}/.config/fish/completions/rocm.fish")" "$(installed_completions "${fish_home}")" +assert_contains "${fish_home}/.local/bin/.rocm-cli-path-edit" "profile ${fish_home}/.config/fish/config.fish" "fish login shell was not recorded" +# The login-shell step already reported config.fish; a second fish step would +# report it again as "already configured". +assert_log_lacks "${WORK}/fish-login.log" "fish shell profile" "fish login shell configured config.fish twice" +if [ "${FAILURES}" -eq 0 ]; then + pass "fish login shell still configures fish once" +fi + +# zsh keeps the POSIX snippet, and fish is configured beside it. +zsh_home="${WORK}/homes/zsh-fish" +install_case "${WORK}/zsh-fish.log" "${zsh_home}" /bin/zsh "${FISH_PATH}" 1 +assert_contains "${zsh_home}/.zshrc" "${POSIX_SNIPPET}" "zsh profile missing the POSIX PATH block" +assert_contains "${zsh_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "zsh login shell skipped fish" +assert_absent_file "${zsh_home}/.bashrc" "zsh login shell also wrote a bash profile" +assert_contains "${zsh_home}/.local/bin/.rocm-cli-path-edit" "profile ${zsh_home}/.zshrc" "uninstall record missing the zsh profile" +assert_contains "${zsh_home}/.local/bin/.rocm-cli-path-edit" "profile ${zsh_home}/.config/fish/config.fish" "uninstall record missing fish beside zsh" +if [ "${FAILURES}" -eq 0 ]; then + pass "zsh login shell keeps its profile and also configures fish" +fi + +# ksh keeps the POSIX snippet in .kshrc, and fish is configured beside it. +ksh_home="${WORK}/homes/ksh-fish" +install_case "${WORK}/ksh-fish.log" "${ksh_home}" /bin/ksh "${FISH_PATH}" 1 +assert_contains "${ksh_home}/.kshrc" "${POSIX_SNIPPET}" "ksh profile missing the POSIX PATH block" +assert_contains "${ksh_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "ksh login shell skipped fish" +assert_absent_file "${ksh_home}/.bashrc" "ksh login shell also wrote a bash profile" +assert_absent_file "${ksh_home}/.profile" "ksh login shell also wrote a generic profile" +assert_contains "${ksh_home}/.local/bin/.rocm-cli-path-edit" "profile ${ksh_home}/.kshrc" "uninstall record missing the ksh profile" +assert_contains "${ksh_home}/.local/bin/.rocm-cli-path-edit" "profile ${ksh_home}/.config/fish/config.fish" "uninstall record missing fish beside ksh" +if [ "${FAILURES}" -eq 0 ]; then + pass "ksh login shell keeps its profile and also configures fish" +fi + +# Opting out of shell profile updates writes neither profile. +skip_home="${WORK}/homes/skip" +install_case "${WORK}/skip.log" "${skip_home}" /bin/bash "${FISH_PATH}" 0 +assert_absent_file "${skip_home}/.bashrc" "shell update opt-out still wrote bashrc" +assert_absent_file "${skip_home}/.config/fish/config.fish" "shell update opt-out still wrote fish config" +assert_absent_file "${skip_home}/.config/fish/completions/rocm.fish" "shell update opt-out still wrote fish completions" +assert_log_contains "${WORK}/skip.log" "shell profile update skipped" "opt-out did not report the skip" +if [ "${FAILURES}" -eq 0 ]; then + pass "shell profile opt-out leaves fish unconfigured" +fi + +# A profile the user named still receives the login-shell snippet, and fish +# still receives its own. +custom_home="${WORK}/homes/custom" +custom_profile="${custom_home}/dotfiles/rocm-path.sh" +install_case "${WORK}/custom.log" "${custom_home}" /bin/bash "${FISH_PATH}" 1 "ROCM_CLI_SHELL_PROFILE=${custom_profile}" +assert_contains "${custom_profile}" "${POSIX_SNIPPET}" "custom profile missing the POSIX PATH block" +assert_contains "${custom_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "custom profile path skipped fish" +assert_absent_file "${custom_home}/.bashrc" "custom profile path also wrote bashrc" +assert_contains "${custom_home}/.local/bin/.rocm-cli-path-edit" "profile ${custom_profile}" "uninstall record missing the custom profile" +assert_contains "${custom_home}/.local/bin/.rocm-cli-path-edit" "profile ${custom_home}/.config/fish/config.fish" "uninstall record missing fish beside the custom profile" +if [ "${FAILURES}" -eq 0 ]; then + pass "a chosen login profile and fish are both recorded" +fi + +# fish reads $XDG_CONFIG_HOME/fish when that is an absolute path; the PATH +# block, completions, and record follow it there. +xdg_home="${WORK}/homes/xdg" +xdg_config="${xdg_home}/xdg-config" +mkdir -p "${xdg_config}/fish" +install_case "${WORK}/xdg.log" "${xdg_home}" /bin/bash "${BASE_PATH}" 1 "XDG_CONFIG_HOME=${xdg_config}" +assert_contains "${xdg_config}/fish/config.fish" "${FISH_SNIPPET}" "XDG fish config missing the fish PATH snippet" +assert_eq "XDG completions" "$(cat "${xdg_config}/fish/completions/rocm.fish" 2>/dev/null)" "$(installed_completions "${xdg_home}")" +assert_absent_file "${xdg_home}/.config/fish/config.fish" "XDG_CONFIG_HOME was ignored for the fish config" +assert_contains "${xdg_home}/.local/bin/.rocm-cli-path-edit" "profile ${xdg_config}/fish/config.fish" "uninstall record missing the XDG fish profile" +assert_contains "${xdg_home}/.local/bin/.rocm-cli-path-edit" "completions ${xdg_config}/fish/completions/rocm.fish" "uninstall record missing the XDG completions" +# A relative XDG_CONFIG_HOME names a different folder from each directory (fish +# resolves it against its own working directory), so the installer keeps +# ~/.config/fish for it. +rel_home="${WORK}/homes/xdg-relative" +mkdir -p "${rel_home}/.config/fish" +install_case "${WORK}/xdg-relative.log" "${rel_home}" /bin/bash "${BASE_PATH}" 1 "XDG_CONFIG_HOME=relative-config" +assert_contains "${rel_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "a relative XDG_CONFIG_HOME moved the fish config away from ~/.config/fish" +if [ "${FAILURES}" -eq 0 ]; then + pass "fish config follows an absolute XDG_CONFIG_HOME" +fi + +# A rocm that cannot print completions, or prints nothing, still installs +# cleanly with the PATH block on record and writes no completion file. The +# failure here clears by itself (the stub works on its second run), and the +# printed advice, run as printed from a bare environment in a home whose name +# holds a space, then creates the file. +for mode in fail-once empty-once; do + comp_home="${WORK}/homes/spaced home ${mode}" + comp_log="${WORK}/completions-${mode}.log" + comp_path="${comp_home}/.config/fish/completions/rocm.fish" + comp_record="${comp_home}/.local/bin/.rocm-cli-path-edit" + mkdir -p "${comp_home}" + printf '%s\n' "${mode}" > "${comp_home}/.stub-completions" + install_case "${comp_log}" "${comp_home}" /bin/bash "${FISH_PATH}" 1 + assert_absent_file "${comp_home}/.stub-completions" "completions (${mode}) were never attempted" + assert_absent_file "${comp_path}" "completions (${mode}) left a completion file" + assert_log_contains "${comp_log}" "note: could not install fish completions" "completions (${mode}) did not print the note" + assert_log_lacks "${comp_log}" "fish completions installed:" "completions (${mode}) were reported installed" + assert_contains "${comp_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "completions (${mode}) lost the fish PATH block" + assert_contains "${comp_record}" "profile ${comp_home}/.config/fish/config.fish" "completions (${mode}) lost the fish record" + if grep -F -q -- "completions " "${comp_record}"; then + fail "completions (${mode}) recorded a file that was not written" + fi + # The advice names the installed binary by its full path, so it works in a + # terminal that does not have the install folder on PATH yet. + assert_log_contains "${comp_log}" "to try again, run the installer again, or run:" "completions (${mode}) did not offer to run the installer again" + advice="$(grep -F -A2 'to try again, run the installer again, or run:' "${comp_log}" | tail -n 2 | sed 's/^ //')" + if [ -z "${advice}" ]; then + fail "completions (${mode}) printed no advice" + elif ! env -i HOME="${comp_home}" PATH="${TOOLS}" \ + "${TOOLS}/sh" -c "${advice}" < /dev/null; then + fail "completions (${mode}) advice did not run" + fi + assert_eq "completions (${mode}) after the advice" "$(cat "${comp_path}" 2>/dev/null)" "${COMPLETIONS_FIXTURE}" +done +if [ "${FAILURES}" -eq 0 ]; then + pass "completions that cannot be generated leave the PATH block and a working hint" +fi + +# A rocm.fish that is already there and did not come from this installer is the +# user's: a regular file, a link to a dotfiles copy, and a dangling link are all +# left exactly as they are, not recorded, with a note saying why. +USERS_COMPLETIONS='complete -c rocm -l my-own-flag' +for kind in file link markedlink dangling; do + own_home="${WORK}/homes/own-completions-${kind}" + own_log="${WORK}/own-completions-${kind}.log" + own_dir="${own_home}/.config/fish/completions" + own_path="${own_dir}/rocm.fish" + own_target="${own_home}/dotfiles/rocm.fish" + mkdir -p "${own_dir}" "${own_home}/dotfiles" + case "${kind}" in + file) printf '%s\n' "${USERS_COMPLETIONS}" > "${own_path}" ;; + link) + printf '%s\n' "${USERS_COMPLETIONS}" > "${own_target}" + ln -s "${own_target}" "${own_path}" + ;; + markedlink) + # Even a target that carries the installer's marker is not written through. + printf '%s\n%s\n' "${COMPLETIONS_MARKER}/opt/other/bin" "${USERS_COMPLETIONS}" > "${own_target}" + ln -s "${own_target}" "${own_path}" + ;; + dangling) ln -s "${own_target}" "${own_path}" ;; + *) fail "unknown rocm.fish kind ${kind}" ;; + esac + install_case "${own_log}" "${own_home}" /bin/bash "${FISH_PATH}" 1 + assert_log_contains "${own_log}" "note: fish completions not installed: ${own_path} already exists and was not written by this installer" "user's rocm.fish (${kind}) was not reported as left alone" + assert_log_lacks "${own_log}" "fish completions installed:" "user's rocm.fish (${kind}) was reported replaced" + case "${kind}" in + file) assert_eq "user's rocm.fish (${kind})" "$(cat "${own_path}")" "${USERS_COMPLETIONS}" ;; + link) + [ -L "${own_path}" ] || fail "user's rocm.fish link was replaced" + assert_eq "what the user's rocm.fish link points at" "$(cat "${own_target}")" "${USERS_COMPLETIONS}" + ;; + markedlink) + [ -L "${own_path}" ] || fail "user's marked rocm.fish link was replaced" + assert_eq "what the user's marked rocm.fish link points at" "$(tail -n 1 "${own_target}")" "${USERS_COMPLETIONS}" + ;; + dangling) + [ -L "${own_path}" ] || fail "user's dangling rocm.fish link was replaced" + assert_absent_file "${own_target}" "the installer wrote through the user's dangling rocm.fish link" + ;; + *) fail "unknown rocm.fish kind ${kind}" ;; + esac + if grep -F -q -- "completions " "${own_home}/.local/bin/.rocm-cli-path-edit"; then + fail "user's rocm.fish (${kind}) was recorded for uninstall to remove" + fi + assert_contains "${own_home}/.config/fish/config.fish" "${FISH_SNIPPET}" "user's rocm.fish (${kind}) stopped the fish PATH block" +done +# A file another install folder of this installer wrote is replaced, and the +# new install records it. +mk_home="${WORK}/homes/marked-completions" +mkdir -p "${mk_home}/.config/fish/completions" +printf '%s\n%s\n' "${COMPLETIONS_MARKER}/opt/other/bin" "old script" > "${mk_home}/.config/fish/completions/rocm.fish" +install_case "${WORK}/marked-completions.log" "${mk_home}" /bin/bash "${FISH_PATH}" 1 +assert_eq "an earlier installer's rocm.fish" "$(cat "${mk_home}/.config/fish/completions/rocm.fish")" "$(installed_completions "${mk_home}")" +assert_contains "${mk_home}/.local/bin/.rocm-cli-path-edit" "completions ${mk_home}/.config/fish/completions/rocm.fish" "a replaced installer rocm.fish was not recorded" +if [ "${FAILURES}" -eq 0 ]; then + pass "a rocm.fish the installer did not write is left alone, and one it wrote is replaced" +fi + +# The login shell's own profile is recorded only when it holds the installer's +# block: a PATH line the user wrote is theirs, and uninstall must not look for +# a block there. +ob_home="${WORK}/homes/bash-own-line" +mkdir -p "${ob_home}" +# shellcheck disable=SC2016 +printf '%s\n' 'export PATH="$HOME/.local/bin:$PATH"' > "${ob_home}/.bashrc" +install_case "${WORK}/bash-own-line.log" "${ob_home}" /bin/bash "${BASE_PATH}" 1 +assert_eq "bashrc with the user's own PATH line" "$(marker_count "${ob_home}/.bashrc")" 0 +assert_log_contains "${WORK}/bash-own-line.log" "shell profile already configured:" "the user's own PATH line was not reported as configured" +if grep -F -q -- "profile ${ob_home}/.bashrc" "${ob_home}/.local/bin/.rocm-cli-path-edit" 2>/dev/null; then + fail "a bash profile holding only the user's own PATH line was recorded" +fi +if [ "${FAILURES}" -eq 0 ]; then + pass "the login profile is recorded only when it holds the installer's block" +fi + +# A profile that cannot even be created (its folder's name is taken by a file) +# is reported as not saved, and not recorded. This works as root too. +bl_home="${WORK}/homes/profile-blocked" +mkdir -p "${bl_home}" +printf '%s\n' 'a file where a folder is needed' > "${bl_home}/blocker" +install_case "${WORK}/profile-blocked.log" "${bl_home}" /bin/bash "${BASE_PATH}" 1 "ROCM_CLI_SHELL_PROFILE=${bl_home}/blocker/profile" +assert_log_lacks "${WORK}/profile-blocked.log" "shell profile updated:" "an uncreatable profile was reported updated" +assert_log_contains "${WORK}/profile-blocked.log" "is not saved in your shell profile" "an uncreatable profile was not reported" +assert_absent_file "${bl_home}/blocker/profile" "a profile was written through a file" +if grep -F -q -- "profile " "${bl_home}/.local/bin/.rocm-cli-path-edit" 2>/dev/null; then + fail "an uncreatable profile was recorded" +fi +if [ "${FAILURES}" -eq 0 ]; then + pass "a profile that cannot be created is reported, not recorded" +fi + +# A profile that cannot be written is not reported as updated or recorded. +# Root writes through file permissions, so this case needs another user. bash +# runs it too: unlike dash, bash does not stop a function on a failed write +# when its caller tests the result, so only an explicit check catches it. +if [ "$(id -u)" -ne 0 ]; then + for install_shell in "${TOOLS}/sh" "$(command -v bash || true)"; do + [ -n "${install_shell}" ] || continue + shell_label="${install_shell##*/}" + ro_home="${WORK}/homes/read-only-fish-${shell_label}" + ro_log="${WORK}/read-only-${shell_label}.log" + ro_config="${ro_home}/.config/fish/config.fish" + mkdir -p "${ro_home}/.config/fish" + printf '%s\n' 'set -gx EDITOR vim' > "${ro_config}" + chmod 0444 "${ro_config}" + INSTALL_SHELL="${install_shell}" install_case "${ro_log}" "${ro_home}" /bin/bash "${FISH_PATH}" 1 + chmod 0644 "${ro_config}" + assert_eq "read-only fish config (${shell_label})" "$(cat "${ro_config}")" "set -gx EDITOR vim" + assert_log_lacks "${ro_log}" "fish shell profile updated:" "a read-only fish config was reported updated (${shell_label})" + assert_log_contains "${ro_log}" "is not saved in your fish config" "a read-only fish config was not reported (${shell_label})" + if grep -F -q -- "profile ${ro_config}" "${ro_home}/.local/bin/.rocm-cli-path-edit"; then + fail "a read-only fish config was recorded (${shell_label})" + fi + assert_contains "${ro_home}/.bashrc" "${POSIX_SNIPPET}" "a read-only fish config stopped the bash profile (${shell_label})" + + # The login shell's own profile is held to the same rule. + rb_home="${WORK}/homes/read-only-bashrc-${shell_label}" + rb_log="${WORK}/read-only-bashrc-${shell_label}.log" + mkdir -p "${rb_home}" + printf '%s\n' 'export EDITOR=vim' > "${rb_home}/.bashrc" + chmod 0444 "${rb_home}/.bashrc" + INSTALL_SHELL="${install_shell}" install_case "${rb_log}" "${rb_home}" /bin/bash "${BASE_PATH}" 1 + chmod 0644 "${rb_home}/.bashrc" + assert_eq "read-only bashrc (${shell_label})" "$(cat "${rb_home}/.bashrc")" "export EDITOR=vim" + assert_log_lacks "${rb_log}" "shell profile updated:" "a read-only bashrc was reported updated (${shell_label})" + assert_log_contains "${rb_log}" "is not saved in your shell profile" "a read-only bashrc was not reported (${shell_label})" + done + if [ "${FAILURES}" -eq 0 ]; then + pass "a profile that cannot be written is reported, not recorded" + fi +else + echo "skip: read-only fish config case needs a non-root user" +fi + +if [ "${FAILURES}" -ne 0 ]; then + echo "${FAILURES} failed" >&2 + exit 1 +fi +echo "all install.sh fish checks passed" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 6e4b20bd6e..cf12b2609e 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -17,6 +17,7 @@ chrono.workspace = true clap.workspace = true e2e-report = { path = "../crates/e2e-report" } proc-macro2.workspace = true +regex = "1" rocm-core = { path = "../crates/rocm-core" } semver = { workspace = true } serde = { workspace = true } diff --git a/xtask/src/affected.rs b/xtask/src/affected.rs index 850549b324..699ea6623f 100644 --- a/xtask/src/affected.rs +++ b/xtask/src/affected.rs @@ -16,11 +16,17 @@ //! Skipping a test that should have run is the failure mode this guards //! against, so when in doubt it runs everything. //! +//! One rule adds a crate nothing depends on: a changed `proptest-regressions` +//! file, or a Rust file whose diff adds or removes a `proptest!` line, selects +//! `xtask`, which holds the seed guard that reads the whole tree (see +//! `PROPTEST_GUARD_CRATE`). +//! //! As with [`crate::verify_commits`], the decision logic ([`select`], //! [`owning_crate`], [`reverse_closure`]) is pure and unit-tested without git or //! cargo; the I/O lives at the edges ([`load_graph`], [`changed_files`]). use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; use std::process::Command; use anyhow::{Context, Result, bail}; @@ -149,6 +155,21 @@ fn forces_full_workspace(file: &str) -> bool { || file.starts_with(".github/workflows/") } +/// The workspace member whose tests include the proptest seed guard +/// (`every_proptest_regression_file_belongs_to_a_property_test_that_reads_it` +/// in `proptest_regressions_contract.rs`). That guard reads the whole tree, so +/// a change in another crate can fail it: moving a `proptest!` block inside +/// `apps/rocm` leaves its seed behind in `apps/rocm/proptest-regressions/`, and +/// nothing in the reverse-dependency closure of `apps/rocm` runs the guard. +const PROPTEST_GUARD_CRATE: &str = "xtask"; + +/// True for a file under a `proptest-regressions` directory: the saved-failure +/// files the guard above checks. +fn is_proptest_seed_file(file: &str) -> bool { + file.split('/') + .any(|segment| segment == "proptest-regressions") +} + /// True for changed files that cannot affect any Rust crate's build or tests, so /// they neither select a crate nor force a full run. fn is_ignorable(file: &str) -> bool { @@ -158,13 +179,32 @@ fn is_ignorable(file: &str) -> bool { is_markdown || file.starts_with("docs/") || file == "LICENSE" || file.starts_with("LICENSE.") } +/// [`select_with_proptest_sources`] for a caller that has only the changed-file +/// names: a changed file under a `proptest-regressions` directory still selects +/// the `xtask` seed guard, but a Rust file that moved a `proptest!` block is not +/// recognised without its diff. +pub fn select(changed_files: &[String], graph: &Graph) -> Selection { + select_with_proptest_sources(changed_files, &[], graph) +} + /// Pure selection logic: turn the changed-file list into a [`Selection`]. /// +/// `proptest_sources` is the subset of changed Rust files whose diff adds or +/// removes a `proptest!` line ([`proptest_source_files`]). It, and any changed +/// file under a `proptest-regressions` directory, adds `xtask` to the selection +/// so the seed guard runs on the pull request instead of only in the merge +/// queue. When the graph has no `xtask` member the guard cannot be selected, so +/// the answer is the whole workspace. +/// /// Order matters — a single force-full file (e.g. `Cargo.lock`) outweighs any /// number of crate-attributable files. Files owned by a crate seed the changed /// set; ignorable files are dropped; anything left unattributed is treated as /// force-full, because an unrecognized path might affect anything. -pub fn select(changed_files: &[String], graph: &Graph) -> Selection { +pub fn select_with_proptest_sources( + changed_files: &[String], + proptest_sources: &[String], + graph: &Graph, +) -> Selection { let mut changed_crates: BTreeSet = BTreeSet::new(); let mut saw_relevant = false; @@ -172,6 +212,17 @@ pub fn select(changed_files: &[String], graph: &Graph) -> Selection { if forces_full_workspace(file) { return Selection::Workspace; } + if is_proptest_seed_file(file) || proptest_sources.contains(file) { + if !graph + .crate_dirs + .iter() + .any(|(name, _)| name == PROPTEST_GUARD_CRATE) + { + return Selection::Workspace; + } + changed_crates.insert(PROPTEST_GUARD_CRATE.to_string()); + saw_relevant = true; + } if let Some(owner) = owning_crate(file, &graph.crate_dirs) { changed_crates.insert(owner.to_string()); saw_relevant = true; @@ -348,15 +399,38 @@ pub(crate) fn scrub_git_env(cmd: &mut Command) -> &mut Command { /// [`changed_files_between`] in `dir` (the current directory when `None`). fn diff_name_only(dir: Option<&std::path::Path>, base: &str, head: &str) -> Result> { + diff_names(dir, base, head, &[], &[]) +} + +/// List the Rust files changed in `base...HEAD` whose diff adds or removes a +/// line containing `proptest!` (git's `-G` pickaxe, so a block moved out of one +/// file and into another lists both). The pattern also matches a longer name +/// such as `my_proptest!`; an extra hit only adds `xtask` to the selection. +fn proptest_source_files(dir: Option<&Path>, base: &str) -> Result> { + diff_names(dir, base, "HEAD", &["-G", "proptest!"], &["*.rs"]) +} + +/// `git diff --name-only --no-renames base...head [-- ]` in +/// `dir` (the current directory when `None`). +fn diff_names( + dir: Option<&Path>, + base: &str, + head: &str, + options: &[&str], + pathspec: &[&str], +) -> Result> { let range = format!("{base}...{head}"); let mut git = Command::new("git"); if let Some(dir) = dir { scrub_git_env(&mut git).arg("-C").arg(dir); } - let output = git - .args(["diff", "--name-only", "--no-renames", &range]) - .output() - .context("failed to run `git diff`")?; + git.args(["diff", "--name-only", "--no-renames"]) + .args(options) + .arg(&range); + if !pathspec.is_empty() { + git.arg("--").args(pathspec); + } + let output = git.output().context("failed to run `git diff`")?; if !output.status.success() { bail!( "`git diff {range}` failed: {}", @@ -376,7 +450,8 @@ pub fn run(base: Option) -> Result<()> { let base = base.unwrap_or_else(|| DEFAULT_BASE.to_string()); let graph = load_graph()?; let files = changed_files(&base)?; - let selection = select(&files, &graph); + let proptest_sources = proptest_source_files(None, &base)?; + let selection = select_with_proptest_sources(&files, &proptest_sources, &graph); println!("{}", selection.to_cargo_args()); Ok(()) } @@ -385,6 +460,10 @@ pub fn run(base: Option) -> Result<()> { mod tests { use super::*; + fn names(pkgs: &[&str]) -> BTreeSet { + pkgs.iter().map(|p| (*p).to_string()).collect() + } + fn graph() -> Graph { // A small stand-in for the real workspace: core <- protocol <- engine, // plus an independent leaf and the top app depending on everything. @@ -394,6 +473,7 @@ mod tests { ("engine".to_string(), "engines/engine".to_string()), ("leaf".to_string(), "crates/leaf".to_string()), ("app".to_string(), "apps/app".to_string()), + ("xtask".to_string(), "xtask".to_string()), ]; // forward: protocol->core, engine->{core,protocol}, app->{engine,leaf} // reverse: core->{protocol,engine}, protocol->{engine}, engine->{app}, leaf->{app} @@ -403,6 +483,7 @@ mod tests { rev_deps.insert("engine".into(), vec!["app".into()]); rev_deps.insert("leaf".into(), vec!["app".into()]); rev_deps.insert("app".into(), vec![]); + rev_deps.insert("xtask".into(), vec![]); Graph { crate_dirs, rev_deps, @@ -668,6 +749,111 @@ mod tests { assert_eq!(files, ["crates/a/src/moved.rs", "crates/b/src/moved.rs"]); } + #[test] + fn a_changed_proptest_seed_file_selects_the_xtask_guard() { + // The seed guard lives in xtask and reads every crate's seeds; `app` + // has no edge to xtask, so without this rule a seed edit in `app` + // selects only `app`. Both the seed's own crate and the guard run. + let sel = select( + &["apps/app/proptest-regressions/a/b.txt".to_string()], + &graph(), + ); + assert_eq!(sel, Selection::Packages(names(&["app", "xtask"]))); + assert_eq!(sel.to_cargo_args(), "-p app -p xtask"); + } + + #[test] + fn a_changed_proptest_source_selects_the_xtask_guard() { + // The case the guard exists for: a `proptest!` block moves between + // files inside one crate and the seed is forgotten, so no seed file is + // in the change at all; only the diff of the Rust files says so. + let files = [ + "apps/app/src/old.rs".to_string(), + "apps/app/src/new.rs".to_string(), + ]; + assert_eq!( + select_with_proptest_sources(&files, &files, &graph()), + Selection::Packages(names(&["app", "xtask"])) + ); + // The same Rust change with no `proptest!` line in its diff does not. + assert_eq!( + select_with_proptest_sources(&files, &[], &graph()), + Selection::Packages(names(&["app"])) + ); + } + + #[test] + fn a_proptest_change_without_an_xtask_member_runs_the_workspace() { + // Nothing to select the guard with, so fall back rather than skip it. + let mut g = graph(); + g.crate_dirs.retain(|(name, _)| name != "xtask"); + g.rev_deps.remove("xtask"); + let seed = ["apps/app/proptest-regressions/x.txt".to_string()]; + assert_eq!(select(&seed, &g), Selection::Workspace); + let src = ["apps/app/src/x.rs".to_string()]; + assert_eq!( + select_with_proptest_sources(&src, &src, &g), + Selection::Workspace + ); + } + + #[test] + fn only_a_proptest_regressions_directory_counts_as_a_seed_path() { + assert!(is_proptest_seed_file( + "apps/rocm/proptest-regressions/main.txt" + )); + assert!(is_proptest_seed_file( + "crates/c/proptest-regressions/ui/format.txt" + )); + // A file merely named alike, or a Rust file, is not a seed. + assert!(!is_proptest_seed_file( + "apps/rocm/proptest-regressions-notes.md" + )); + assert!(!is_proptest_seed_file( + "apps/rocm/src/proptest-regressions.rs" + )); + assert!(!is_proptest_seed_file("apps/rocm/src/main.rs")); + // Unrelated crate changes still leave xtask out. + assert_eq!( + select(&["apps/app/src/lib.rs".to_string()], &graph()), + Selection::Packages(names(&["app"])) + ); + } + + #[test] + fn proptest_source_files_lists_rust_files_whose_diff_touches_a_proptest_block() { + let dir = tempfile::tempdir().expect("tempdir"); + let repo = dir.path(); + git_in(repo, &["init", "-q", "-b", "main"]); + std::fs::write( + repo.join("moved_from.rs"), + "proptest! { fn a() {} }\nfn keep() {}\n", + ) + .unwrap(); + std::fs::write(repo.join("plain.rs"), "fn plain() {}\n").unwrap(); + std::fs::write(repo.join("notes.md"), "proptest! is mentioned\n").unwrap(); + git_in(repo, &["add", "."]); + git_in(repo, &["commit", "-q", "-m", "base"]); + git_in(repo, &["checkout", "-q", "-b", "topic"]); + // The block moves out of one Rust file and into another. A Rust file + // whose diff has no proptest line, and a Markdown note that does but + // is not Rust, change too; neither may be listed. + std::fs::write(repo.join("moved_from.rs"), "fn keep() {}\n").unwrap(); + std::fs::write(repo.join("moved_to.rs"), "proptest! { fn a() {} }\n").unwrap(); + std::fs::write(repo.join("plain.rs"), "fn plain() { let _ = 1; }\n").unwrap(); + std::fs::write(repo.join("notes.md"), "proptest! is mentioned again\n").unwrap(); + git_in(repo, &["add", "."]); + git_in(repo, &["commit", "-q", "-m", "move"]); + + let mut got = proptest_source_files(Some(repo), "main").expect("diff"); + got.sort(); + assert_eq!(got, ["moved_from.rs", "moved_to.rs"]); + // The plain name-only diff, which the selector otherwise relies on, + // sees all four files: the narrowing above is what the `-G` adds. + let all = diff_name_only(Some(repo), "main", "topic").expect("diff"); + assert_eq!(all.len(), 4, "{all:?}"); + } + #[test] fn cargo_args_rendering() { assert_eq!(Selection::Workspace.to_cargo_args(), "--workspace"); diff --git a/xtask/src/architecture_doc.rs b/xtask/src/architecture_doc.rs new file mode 100644 index 0000000000..336babb8cd --- /dev/null +++ b/xtask/src/architecture_doc.rs @@ -0,0 +1,908 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Require every production source file, and every file of the repository's own +//! tooling and end-to-end harness, to appear in `docs/architecture.md`. +//! +//! `docs/architecture.md` is the contributor-facing module map and is meant to +//! be "updated in the same PR as the code it documents". Nothing enforced that +//! until now. The `lychee` link check (the `docs-links` CI job and the `lychee` +//! prek hook) runs the other way round: it confirms that every link already in +//! the doc still resolves, so a moved or deleted file is caught, but a file the +//! doc never mentions has no link to check and passes silently. +//! +//! This check starts from the source tree instead. Every tracked `.rs` file +//! under `apps/*/src`, `crates/*/src` and `engines/*/src` must be cited by a +//! relative markdown link in `docs/architecture.md` (a complete inline link, +//! `[text](path)`, with its text on one line, that renders as a link: a path +//! inside an HTML comment, a code block or a code span does not count, and +//! neither does a link missing its `[` or `)`, one with an escaped bracket, an +//! image, or the outer brackets of a link that contains another link), +//! or be a test-only helper (see [`is_test_helper`]). Out of scope: `build.rs` +//! files at a crate's root and the `tests/` and `examples/` trees of those +//! crates, which sit outside `/src`. +//! +//! The same applies, without the helper exemption, to every tracked `.rs` file +//! under `xtask/src`, `tests/e2e-cucumber/src` and `tests/e2e-cucumber/tests` +//! (see [`HAND_KEPT_TREES`]): the tooling and the harness are not part of a +//! release, but their tables in the doc used to be kept current by hand, and +//! nothing noticed a file added without a row. The `tests/` tree of that +//! harness is its step definitions and its drift guards, so it is covered as +//! well as its `src/`. Every file in those trees needs a link, including a +//! file named like a helper, because a file added there is not shared test +//! support of a production crate. +//! +//! It reads the git index and the working tree, never a git diff, so the answer +//! does not depend on a base ref. That is the same answer in CI and in the +//! pre-commit hook, where prek stashes unstaged changes first. The pre-push hook +//! runs it too, but it reads the working tree, not the commits being pushed: +//! with a clean tree that is the same thing, and with edits that are staged or +//! unstaged it can pass where CI fails. CI is the gate; the hook is feedback. +//! +//! The doc must use inline `[text](path)` links. Reference-style links, a +//! `?query` suffix and a root-relative path render as working links but are not +//! recognised, so the failure message asks for the inline form. +//! +//! What this does not check: whether an entry's prose still describes the file, +//! and functions that move between two files that are both already cited. + +use std::collections::BTreeSet; +use std::path::Path; +use std::process::Command; + +use anyhow::{Context, Result, bail}; + +use crate::paths; + +/// The module map, relative to the workspace root. +const DOC: &str = "docs/architecture.md"; + +/// The directory `DOC` lives in. Its links are relative to this. +const DOC_DIR: &str = "docs"; + +/// Source roots whose `/src` trees are in scope. +const SOURCE_ROOTS: [&str; 3] = ["apps", "crates", "engines"]; + +/// The repository's own tooling and end-to-end harness: directory prefixes +/// whose `.rs` files are in scope with no helper exemption. +const HAND_KEPT_TREES: [&str; 3] = [ + "xtask/src/", + "tests/e2e-cucumber/src/", + "tests/e2e-cucumber/tests/", +]; + +/// Everything `git ls-files` is asked for: the [`SOURCE_ROOTS`] and the roots +/// the [`HAND_KEPT_TREES`] sit under. +const TRACKED_ROOTS: [&str; 5] = ["apps", "crates", "engines", "xtask", "tests/e2e-cucumber"]; + +/// Whether `path` is a `.rs` file under `//src/` for one of the +/// [`SOURCE_ROOTS`]. +#[allow(clippy::case_sensitive_file_extension_comparisons)] // Rust sources are always lowercase `.rs` +fn in_product_scope(path: &str) -> bool { + let parts: Vec<&str> = path.split('/').collect(); + parts.len() >= 4 + && SOURCE_ROOTS.contains(&parts[0]) + && parts[2] == "src" + && path.ends_with(".rs") +} + +/// Whether `path` is a `.rs` file under one of the [`HAND_KEPT_TREES`]. +#[allow(clippy::case_sensitive_file_extension_comparisons)] // Rust sources are always lowercase `.rs` +fn in_hand_kept_scope(path: &str) -> bool { + path.ends_with(".rs") && HAND_KEPT_TREES.iter().any(|tree| path.starts_with(tree)) +} + +/// Whether `path` is a tracked-file path this check covers. +fn in_scope(path: &str) -> bool { + in_product_scope(path) || in_hand_kept_scope(path) +} + +/// Whether `path` is a test-only helper that does not need an architecture +/// entry: shared test support, or a property-test module. Matched on the file +/// stem so a new helper of the same kind never needs an entry. +fn is_test_helper(path: &str) -> bool { + let name = path.rsplit('/').next().unwrap_or(path); + let stem = name.strip_suffix(".rs").unwrap_or(name); + matches!(stem, "test_support" | "test_env" | "proptests") + || stem.ends_with("_proptests") + || stem.ends_with("_props") + || stem.ends_with("_properties") +} + +/// Resolve one markdown link target found in [`DOC`] to a repo-relative `.rs` +/// path, or `None` if it is not a link to a source file inside the repo. +#[allow(clippy::case_sensitive_file_extension_comparisons)] // Rust sources are always lowercase `.rs` +fn resolve_link(target: &str) -> Option { + let target = target.split('#').next()?; + if target.contains("://") || !target.ends_with(".rs") { + return None; + } + let mut parts: Vec<&str> = vec![DOC_DIR]; + for segment in target.split('/') { + match segment { + "" | "." => {} + ".." => { + // Climbing out of the repo root names nothing in the tree. + parts.pop()?; + } + other => parts.push(other), + } + } + Some(parts.join("/")) +} + +/// The first line of `text` without its terminator, and the byte length of that +/// line including the terminator. +fn next_line(text: &str) -> (&str, usize) { + match text.find('\n') { + Some(end) => (&text[..end], end + 1), + None => (text, text.len()), + } +} + +/// A code-fence marker at the start of `line`: its character and run length. +fn fence_marker(line: &str) -> Option<(u8, usize)> { + let line = line.trim_start_matches([' ', '\t']); + let first = *line.as_bytes().first()?; + if first != b'`' && first != b'~' { + return None; + } + let run = line.bytes().take_while(|&byte| byte == first).count(); + // A backtick fence's info string cannot contain a backtick: that line is an + // inline code span, not a fence. + if run < 3 || (first == b'`' && line[run..].contains('`')) { + return None; + } + Some((first, run)) +} + +/// Whether `line` closes a fence opened with `open`. +fn closes_fence(line: &str, open: (u8, usize)) -> bool { + fence_marker(line).is_some_and(|(character, run)| { + character == open.0 && run >= open.1 && line.trim().bytes().all(|byte| byte == open.0) + }) +} + +/// The byte length of the code span that opens at the start of `text` (which +/// starts with a backtick run), or `None` if no run of the same length closes +/// it. Like a CommonMark code span it cannot cross a blank line, so a stray +/// backtick in one paragraph does not pair with one in a later paragraph. +fn code_span_len(text: &str) -> Option { + let window = &text[..text.find("\n\n").unwrap_or(text.len())]; + let open = window.bytes().take_while(|&byte| byte == b'`').count(); + let mut at = open; + while let Some(offset) = window[at..].find('`') { + let start = at + offset; + let run = window[start..] + .bytes() + .take_while(|&byte| byte == b'`') + .count(); + if run == open { + return Some(start + run); + } + at = start + run; + } + None +} + +/// `doc` with everything that does not render as a link removed: HTML comments, +/// fenced code blocks and inline code spans. A link inside any of them is not +/// part of the rendered module map, so it must not count as a citation. Each +/// removed span leaves a space or newline behind so link text and line +/// structure stay in place. +/// +/// It scans once, left to right, so the usual precedence holds: a backtick +/// inside a comment is part of the comment, and `").map_or(rest.len(), |end| end + 3); + out.push(' '); + continue; + } + if rest.starts_with('`') { + if let Some(length) = code_span_len(rest) { + at += length; + out.push(' '); + } else { + let run = rest.bytes().take_while(|&byte| byte == b'`').count(); + out.push_str(&rest[..run]); + at += run; + } + continue; + } + let Some(character) = rest.chars().next() else { + break; + }; + out.push(character); + at += character.len_utf8(); + } + out +} + +/// Whether the byte at `at` is a backslash that escapes the ASCII punctuation +/// character after it. +fn escapes_next(bytes: &[u8], at: usize) -> bool { + bytes[at] == b'\\' && bytes.get(at + 1).is_some_and(u8::is_ascii_punctuation) +} + +/// `text` with the backslash removed from every escaped punctuation character. +fn unescape(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = String::with_capacity(text.len()); + let mut at = 0; + while at < bytes.len() { + if escapes_next(bytes, at) { + at += 1; + } + // Every byte skipped above or copied here is ASCII, so `at` stays on a + // character boundary. + let Some(character) = text[at..].chars().next() else { + break; + }; + out.push(character); + at += character.len_utf8(); + } + out +} + +/// Advance `at` past spaces, tabs and newlines. +fn skip_space(bytes: &[u8], mut at: usize) -> usize { + while matches!(bytes.get(at), Some(b' ' | b'\t' | b'\n')) { + at += 1; + } + at +} + +/// Parse the rest of an inline link, `text` being everything after its `(`: +/// `destination`, an optional title, and the closing `)`. Returns the +/// destination (its `<...>` delimiters and escapes removed) and the byte length +/// consumed through the `)`, or `None` if it is not a complete inline link. +fn inline_destination(text: &str) -> Option<(String, usize)> { + let bytes = text.as_bytes(); + let mut at = skip_space(bytes, 0); + let destination; + if bytes.get(at) == Some(&b'<') { + let start = at + 1; + at = start; + loop { + match *bytes.get(at)? { + b'>' => break, + b'<' | b'\n' => return None, + _ if escapes_next(bytes, at) => at += 2, + _ => at += 1, + } + } + destination = unescape(&text[start..at]); + at += 1; + } else { + let start = at; + let mut depth = 0usize; + loop { + match bytes.get(at) { + None => return None, + Some(b' ' | b'\t' | b'\n') => break, + Some(b'(') => { + depth += 1; + at += 1; + } + Some(b')') if depth == 0 => break, + Some(b')') => { + depth -= 1; + at += 1; + } + Some(_) if escapes_next(bytes, at) => at += 2, + Some(_) => at += 1, + } + } + destination = unescape(&text[start..at]); + } + at = skip_space(bytes, at); + if let Some(&opener @ (b'"' | b'\'' | b'(')) = bytes.get(at) { + let closer = if opener == b'(' { b')' } else { opener }; + at += 1; + loop { + match *bytes.get(at)? { + byte if byte == closer => break, + _ if escapes_next(bytes, at) => at += 2, + _ => at += 1, + } + } + at = skip_space(bytes, at + 1); + } + (bytes.get(at) == Some(&b')')).then_some((destination, at + 1)) +} + +/// An unmatched `[` while scanning for links. +struct Opener { + /// Preceded by an unescaped `!`: an image, whose destination is not a link. + image: bool, + /// Cleared once a link has formed inside this one: links do not nest, so the + /// outer brackets then render as plain text. + active: bool, +} + +/// The destination of every complete inline link, `[text](destination)`, in +/// `text`. A bracket or parenthesis escaped with a backslash does not count, +/// the `[` must be on the same line as its `]`, and a link missing its `[`, its +/// `)` or its destination is not a link: the rendered page shows it as plain +/// text, so it must not count as a citation either. The same goes for an image, +/// `![alt](destination)`, and for the outer brackets of a link that contains +/// another link; an image inside a link's text is fine. +fn link_destinations(text: &str) -> Vec { + let bytes = text.as_bytes(); + let mut found = Vec::new(); + let mut openers: Vec = Vec::new(); + let mut last_bang = None; + let mut at = 0; + while at < bytes.len() { + if escapes_next(bytes, at) { + at += 2; + continue; + } + match bytes[at] { + b'\n' => openers.clear(), + b'!' => last_bang = Some(at), + b'[' => openers.push(Opener { + image: at > 0 && last_bang == Some(at - 1), + active: true, + }), + b']' => { + if let Some(opener) = openers.pop() + && opener.active + && bytes.get(at + 1) == Some(&b'(') + && let Some((destination, length)) = inline_destination(&text[at + 2..]) + { + if !opener.image { + found.push(destination); + for outer in openers.iter_mut().filter(|outer| !outer.image) { + outer.active = false; + } + } + at += 2 + length; + continue; + } + } + _ => {} + } + at += 1; + } + found +} + +/// Every `.rs` file [`DOC`] links to, as repo-relative paths. Only a complete +/// inline link counts, and not one inside an HTML comment, a code block or a +/// code span. +fn cited_paths(doc: &str) -> BTreeSet { + link_destinations(&link_source_text(doc)) + .iter() + .filter_map(|destination| resolve_link(destination)) + .collect() +} + +/// Check the tracked files against the doc's citations, returning every +/// violation found. +fn check(tracked: &[String], cited: &BTreeSet) -> Vec { + let in_tree: BTreeSet<&str> = tracked + .iter() + .map(String::as_str) + .filter(|path| in_scope(path)) + .collect(); + + let mut violations = Vec::new(); + + for path in &in_tree { + // The helper exemption is for a production crate's shared test support; + // a file in the tooling or harness trees needs its row like any other. + let exempt = in_product_scope(path) && is_test_helper(path); + if !cited.contains(*path) && !exempt { + violations.push(format!( + "`{path}` is not cited in {DOC}: add an entry for it to the module map, as an \ + inline `[text](path)` markdown link relative to docs/ (reference-style links, \ + `?query` suffixes and root-relative paths are not recognised)" + )); + } + } + + violations +} + +/// Tracked files under the [`TRACKED_ROOTS`], as the index lists them. +fn tracked_files(root: &Path) -> Result> { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(["ls-files", "-z", "--"]) + .args(TRACKED_ROOTS) + .output() + .context("failed to run `git ls-files`")?; + if !output.status.success() { + bail!( + "`git ls-files` failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&output.stdout) + .split('\0') + .filter(|path| !path.is_empty()) + .map(str::to_owned) + .collect()) +} + +/// Run the check against the real workspace. +fn violations_in(root: &Path) -> Result> { + let doc = + std::fs::read_to_string(root.join(DOC)).with_context(|| format!("failed to read {DOC}"))?; + Ok(check(&tracked_files(root)?, &cited_paths(&doc))) +} + +/// Fail if a file in scope has no entry in `docs/architecture.md`. +pub fn run() -> Result<()> { + let root = paths::workspace_root()?; + let violations = violations_in(&root)?; + if !violations.is_empty() { + bail!( + "{DOC} is out of step with the source tree ({} problem(s)):\n{}", + violations.len(), + violations.join("\n") + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn files(paths: &[&str]) -> Vec { + paths.iter().map(|path| (*path).to_owned()).collect() + } + + fn cites(paths: &[&str]) -> BTreeSet { + paths.iter().map(|path| (*path).to_owned()).collect() + } + + #[test] + fn cited_file_passes() { + let tracked = files(&["apps/rocm/src/cli.rs"]); + assert!(check(&tracked, &cites(&["apps/rocm/src/cli.rs"])).is_empty()); + } + + #[test] + fn new_uncited_file_fails_and_is_named() { + let tracked = files(&["apps/rocm/src/cli.rs", "apps/rocm/src/brand_new.rs"]); + let violations = check(&tracked, &cites(&["apps/rocm/src/cli.rs"])); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("apps/rocm/src/brand_new.rs")); + assert!(violations[0].contains("is not cited")); + } + + #[test] + fn a_citation_does_not_cover_the_same_name_in_another_crate() { + let tracked = files(&["apps/rocm/src/cli.rs", "apps/rocmd/src/cli.rs"]); + let violations = check(&tracked, &cites(&["apps/rocm/src/cli.rs"])); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("apps/rocmd/src/cli.rs")); + } + + #[test] + fn files_outside_the_source_trees_are_ignored() { + let tracked = files(&[ + "apps/rocm/tests/broken_pipe.rs", + "apps/rocm/build.rs", + "crates/rocm-dash-tui/examples/gen_cast.rs", + "apps/rocm/src/data.json", + "xtask/build.rs", + "xtask/Cargo.toml", + "xtask/tests/anything.rs", + "tests/e2e-cucumber/build.rs", + "tests/e2e-cucumber/features/chat.feature", + "tests/e2e-cucumber/examples/anything.rs", + "tests/other-suite/src/lib.rs", + "tests/e2e-cucumber/srcs/anything.rs", + ]); + assert!(check(&tracked, &cites(&[])).is_empty()); + } + + #[test] + fn a_tooling_or_harness_file_with_no_entry_fails_and_is_named() { + for path in [ + "xtask/src/brand_new.rs", + "xtask/src/nested/brand_new.rs", + "tests/e2e-cucumber/src/brand_new.rs", + "tests/e2e-cucumber/src/bin/brand-new.rs", + "tests/e2e-cucumber/tests/e2e.rs", + "tests/e2e-cucumber/tests/e2e/brand_new_steps.rs", + ] { + let violations = check(&files(&[path]), &cites(&[])); + assert_eq!(violations.len(), 1, "{path} must need an entry"); + assert!(violations[0].contains(path)); + assert!(violations[0].contains("is not cited")); + } + } + + #[test] + fn a_tooling_or_harness_file_with_an_entry_passes() { + let paths = [ + "xtask/src/main.rs", + "tests/e2e-cucumber/src/lib.rs", + "tests/e2e-cucumber/tests/e2e/chat_steps.rs", + ]; + assert!(check(&files(&paths), &cites(&paths)).is_empty()); + } + + #[test] + fn a_citation_does_not_cover_the_same_name_in_the_other_tree() { + let violations = check( + &files(&["xtask/src/paths.rs", "tests/e2e-cucumber/src/paths.rs"]), + &cites(&["xtask/src/paths.rs"]), + ); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("tests/e2e-cucumber/src/paths.rs")); + } + + #[test] + fn a_file_named_like_a_helper_needs_an_entry_in_the_tooling_and_harness_trees() { + let tracked = files(&[ + "xtask/src/test_support.rs", + "tests/e2e-cucumber/src/world_props.rs", + "tests/e2e-cucumber/tests/chat_proptests.rs", + ]); + assert_eq!(check(&tracked, &cites(&[])).len(), 3); + } + + #[test] + fn nested_source_files_are_in_scope() { + let tracked = files(&["crates/rocm-dash-tui/src/ui/tabs/home.rs"]); + let violations = check(&tracked, &cites(&[])); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("ui/tabs/home.rs")); + } + + #[test] + fn test_helpers_need_no_entry() { + let tracked = files(&[ + "apps/rocm/src/test_support.rs", + "apps/rocm/src/therock/test_support.rs", + "crates/rocm-core/src/test_env.rs", + "crates/rocm-core/src/archive_props.rs", + "crates/rocm-core/src/process_tree_properties.rs", + "crates/rocm-core/src/examine_driver_proptests.rs", + "apps/rocm/src/providers/local_http/proptests.rs", + ]); + assert!(check(&tracked, &cites(&[])).is_empty()); + } + + #[test] + fn helper_matching_is_by_whole_file_stem() { + // Names that merely contain a helper word are production files. + for path in [ + "apps/rocm/src/test_supporter.rs", + "apps/rocm/src/properties.rs", + "apps/rocm/src/gpu_props_view.rs", + "apps/rocm/src/proptests_runner.rs", + ] { + assert!( + !is_test_helper(path), + "{path} must not be treated as a helper" + ); + } + } + + #[test] + fn links_resolve_relative_to_the_doc() { + let doc = "[cli](../apps/rocm/src/cli.rs) and [x](./../crates/a/src/b.rs#L1)"; + assert_eq!( + cited_paths(doc), + cites(&["apps/rocm/src/cli.rs", "crates/a/src/b.rs"]) + ); + } + + #[test] + fn only_in_repo_rust_links_count_as_citations() { + let doc = "[md](../README.md) [url](https://example.com/a/src/x.rs) \ + [out](../../outside/src/x.rs) [bare](apps/rocm/src/cli.rs)"; + assert_eq!(cited_paths(doc), cites(&["docs/apps/rocm/src/cli.rs"])); + } + + #[test] + fn a_link_with_a_title_is_still_a_citation() { + let doc = "[cli](../apps/rocm/src/cli.rs \"the command tree\")"; + assert_eq!(cited_paths(doc), cites(&["apps/rocm/src/cli.rs"])); + } + + const CLI: &str = "[cli](../apps/rocm/src/cli.rs)"; + + fn cited_cli(doc: &str) -> bool { + cited_paths(doc).contains("apps/rocm/src/cli.rs") + } + + #[test] + fn commenting_out_the_only_entry_leaves_the_file_uncited() { + let doc = format!(""); + let tracked = files(&["apps/rocm/src/cli.rs"]); + let violations = check(&tracked, &cited_paths(&doc)); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("apps/rocm/src/cli.rs")); + } + + #[test] + fn a_link_in_an_html_comment_is_not_a_citation() { + assert!(!cited_cli(&format!(""))); + assert!(!cited_cli(&format!( + "before\n\nafter" + ))); + // An unterminated comment hides everything after it, as a renderer does. + assert!(!cited_cli(&format!("\n{CLI}"))); + } + + #[test] + fn a_link_in_a_fenced_code_block_is_not_a_citation() { + assert!(!cited_cli(&format!("```\n{CLI}\n```"))); + assert!(!cited_cli(&format!("```md\n{CLI}\n```"))); + assert!(!cited_cli(&format!("~~~\n{CLI}\n~~~"))); + assert!(!cited_cli(&format!(" ```\n {CLI}\n ```"))); + // A longer closing fence closes it; a shorter or different one does not. + assert!(!cited_cli(&format!("````\n```\n{CLI}\n```\n````"))); + assert!(!cited_cli(&format!("~~~\n```\n{CLI}\n```\n~~~"))); + // An unclosed fence runs to the end of the document. + assert!(!cited_cli(&format!("```\n{CLI}"))); + assert!(cited_cli(&format!("```\ncode\n```\n{CLI}"))); + } + + #[test] + fn a_link_in_an_inline_code_span_is_not_a_citation() { + assert!(!cited_cli(&format!("see `{CLI}` for more"))); + assert!(!cited_cli(&format!("see ``a ` b {CLI}`` for more"))); + assert!(cited_cli(&format!("see `code` and {CLI}"))); + } + + #[test] + fn a_code_span_inside_link_text_still_cites() { + assert!(cited_cli("[`cli.rs`](../apps/rocm/src/cli.rs)")); + assert!(cited_cli( + "- [`cli.rs`](../apps/rocm/src/cli.rs) - the command tree" + )); + } + + #[test] + fn a_stray_backtick_does_not_hide_a_later_paragraph() { + assert!(cited_cli(&format!("a ` stray\n\n{CLI}"))); + assert!(cited_cli(&format!("a ` stray {CLI}"))); + } + + #[test] + fn comment_and_code_markers_take_the_usual_precedence() { + // ``"))); + // A fence marker inside a comment does not open a fence. + assert!(cited_cli(&format!("\n{CLI}"))); + // A backtick inside a comment does not open a code span. + assert!(cited_cli(&format!("\n{CLI}\n"))); + } + + #[test] + fn crlf_documents_are_read_the_same() { + assert!(!cited_cli(&format!("```\r\n{CLI}\r\n```\r\n"))); + assert!(cited_cli(&format!("a ` stray\r\n\r\n{CLI}"))); + } + + #[test] + fn an_angle_bracketed_destination_is_a_citation() { + assert!(cited_cli("[cli](<../apps/rocm/src/cli.rs>)")); + assert!(cited_cli("[cli](<../apps/rocm/src/cli.rs#L10>)")); + assert!(cited_cli("[cli]( <../apps/rocm/src/cli.rs> )")); + assert!(cited_cli( + "[cli](<../apps/rocm/src/cli.rs> \"the command tree\")" + )); + // The delimiters are not part of the path: a destination with a space is + // read whole, and an unterminated one is not a link. + assert_eq!( + cited_paths("[x](<../crates/a b/src/c.rs>)"), + cites(&["crates/a b/src/c.rs"]) + ); + assert!(!cited_cli("[cli](<../apps/rocm/src/cli.rs)")); + assert!(!cited_cli("[cli](<../apps/rocm/src/cli.rs\n>)")); + } + + #[test] + fn a_link_missing_its_closing_parenthesis_is_not_a_citation() { + assert!(!cited_cli("[cli](../apps/rocm/src/cli.rs")); + assert!(!cited_cli("[cli](../apps/rocm/src/cli.rs and more text)")); + assert!(!cited_cli( + "[cli](../apps/rocm/src/cli.rs \"unterminated title)" + )); + assert!(!cited_cli( + "[cli](../apps/rocm/src/cli.rs\n\nnext paragraph)" + )); + } + + #[test] + fn a_link_missing_its_opening_bracket_is_not_a_citation() { + assert!(!cited_cli("cli](../apps/rocm/src/cli.rs)")); + // A stray `[` on an earlier line is not this link's text. + assert!(!cited_cli("a [ stray\ncli](../apps/rocm/src/cli.rs)")); + } + + #[test] + fn an_escaped_bracket_or_parenthesis_is_not_a_citation() { + assert!(!cited_cli("\\[cli](../apps/rocm/src/cli.rs)")); + assert!(!cited_cli("[cli\\](../apps/rocm/src/cli.rs)")); + assert!(!cited_cli("[cli]\\(../apps/rocm/src/cli.rs)")); + // An escaped backslash does not escape the bracket after it. + assert!(cited_cli("[cli\\\\](../apps/rocm/src/cli.rs)")); + } + + #[test] + fn breaking_the_only_entry_leaves_the_file_uncited() { + let tracked = files(&["apps/rocm/src/cli.rs"]); + for broken in [ + "[cli](../apps/rocm/src/cli.rs", + "cli](../apps/rocm/src/cli.rs)", + "\\[cli](../apps/rocm/src/cli.rs)", + ] { + let violations = check(&tracked, &cited_paths(broken)); + assert_eq!(violations.len(), 1, "{broken:?} must not cite the file"); + } + } + + #[test] + fn nested_brackets_and_parentheses_do_not_break_a_link() { + assert!(cited_cli("[a [b] c](../apps/rocm/src/cli.rs)")); + assert_eq!( + cited_paths("[x](../crates/a/src/b(1).rs)"), + cites(&["crates/a/src/b(1).rs"]) + ); + } + + #[test] + fn an_image_is_not_a_citation() { + assert!(!cited_cli("![cli](../apps/rocm/src/cli.rs)")); + assert!(!cited_cli("see ![cli](../apps/rocm/src/cli.rs \"title\")")); + let tracked = files(&["apps/rocm/src/cli.rs"]); + let violations = check(&tracked, &cited_paths("![cli](../apps/rocm/src/cli.rs)")); + assert_eq!(violations.len(), 1); + } + + #[test] + fn an_escaped_exclamation_mark_does_not_make_an_image() { + assert!(cited_cli("\\![cli](../apps/rocm/src/cli.rs)")); + assert!(cited_cli("wow! [cli](../apps/rocm/src/cli.rs)")); + } + + #[test] + fn an_image_inside_a_links_text_leaves_the_link_valid() { + let doc = "[![icon](../apps/rocm/src/icon.rs)](../apps/rocm/src/cli.rs)"; + assert_eq!(cited_paths(doc), cites(&["apps/rocm/src/cli.rs"])); + } + + #[test] + fn a_link_containing_a_link_cites_only_the_inner_one() { + // Links cannot nest: the page renders only the inner link, so the outer + // destination is plain text. + let doc = "[cli [d](../apps/rocm/src/d.rs)](../apps/rocm/src/cli.rs)"; + assert_eq!(cited_paths(doc), cites(&["apps/rocm/src/d.rs"])); + assert!(!cited_cli( + "[cli [details](../README.md)](../apps/rocm/src/cli.rs)" + )); + // Plain brackets in the text do not form a link, so the outer one stays. + assert!(cited_cli("[cli [note]](../apps/rocm/src/cli.rs)")); + assert!(cited_cli("[cli [note] x](../apps/rocm/src/cli.rs)")); + // Later links on the line are unaffected. + let doc = "[a [b](../apps/rocm/src/b.rs)](../x.rs) and [c](../apps/rocm/src/c.rs)"; + assert_eq!( + cited_paths(doc), + cites(&["apps/rocm/src/b.rs", "apps/rocm/src/c.rs"]) + ); + } + + #[test] + fn an_escaped_backtick_does_not_open_a_code_span() { + assert!(cited_cli( + "Use \\` for a backtick; [`cli.rs`](../apps/rocm/src/cli.rs)" + )); + // An escaped backslash does not escape the backtick after it. + assert!(!cited_cli("\\\\`[cli](../apps/rocm/src/cli.rs)`")); + // An escaped `<` does not open a comment. + assert!(cited_cli("\\")); + } + + #[test] + fn the_link_forms_the_failure_message_rules_out_are_not_citations() { + // The message tells authors these forms are not recognised; this holds it + // to what the parser does. + assert!(!cited_cli("[cli][c]\n\n[c]: ../apps/rocm/src/cli.rs")); + assert!(!cited_cli("[cli](../apps/rocm/src/cli.rs?plain=1)")); + assert!(!cited_cli("[cli](/apps/rocm/src/cli.rs)")); + let message = &check(&files(&["apps/rocm/src/new.rs"]), &cites(&[]))[0]; + assert!(message.contains("inline `[text](path)`")); + assert!(message.contains("reference-style")); + assert!(message.contains("`?query`")); + assert!(message.contains("root-relative")); + } + + #[test] + fn the_failure_message_for_an_uncited_file_only_asks_for_an_entry() { + let violations = check(&files(&["apps/rocm/src/new.rs"]), &cites(&[])); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("add an entry for it to the module map")); + } + + #[test] + fn several_links_on_a_line_are_all_found() { + let doc = "[a](../apps/rocm/src/a.rs), [b](../apps/rocm/src/b.rs) and [c](<../apps/rocm/src/c.rs>)"; + assert_eq!( + cited_paths(doc), + cites(&[ + "apps/rocm/src/a.rs", + "apps/rocm/src/b.rs", + "apps/rocm/src/c.rs" + ]) + ); + } + + #[test] + fn the_git_listing_includes_the_tooling_and_harness_trees() { + // `check` only sees what `tracked_files` lists, so a listing that stopped + // at the product roots would leave the new scope silently empty. + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory"); + let listed = tracked_files(root).expect("git is readable in a checkout"); + for expected in [ + "xtask/src/architecture_doc.rs", + "tests/e2e-cucumber/src/lib.rs", + "tests/e2e-cucumber/tests/e2e.rs", + ] { + assert!( + listed.iter().any(|path| path == expected), + "{expected} is missing from the tracked-file listing" + ); + } + } + + #[test] + fn current_tree_matches_the_architecture_doc() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory"); + let violations = violations_in(root).expect("git and the doc are readable in a checkout"); + assert!( + violations.is_empty(), + "docs/architecture.md is out of step with the source tree:\n{}", + violations.join("\n") + ); + } +} diff --git a/xtask/src/chat_import_contract.rs b/xtask/src/chat_import_contract.rs new file mode 100644 index 0000000000..b3fbc9f739 --- /dev/null +++ b/xtask/src/chat_import_contract.rs @@ -0,0 +1,568 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: the freeform planner and the leaf modules it shares +//! with `chat.rs` must not import `chat` in production code. +//! +//! `chat.rs` calls the freeform planner (`plan_freeform_request_with_context`), +//! so a planner that also imported `chat` would be a module cycle. The cycle was +//! broken by moving what the planner used from `chat.rs` into leaf modules +//! (`prompt_text.rs`, `rocm_argv.rs`, `tool_policy.rs`) that import nothing from +//! either side. `cargo xtask check-crate-edges` cannot see that rule, because it +//! compares crates, not modules inside `apps/rocm`, so without this test a later +//! change can put the import back and nothing fails. +//! +//! The scan is on `use` paths, not on the word `chat`: modules in this crate are +//! full of identifiers such as `chat_cli_arg_value` and of prose about the chat +//! assistant, and neither is an import. What it flags in production code is +//! `chat` reached as a module: `crate::chat`, `super::chat` or `self::chat` as a +//! path (a `use`, or an inline call such as `crate::chat::f()`), `chat` as a +//! member of a `use crate::{..}` group, and `mod chat`. A raw identifier +//! (`r#chat`) is the same name and is flagged the same way. +//! +//! Test code is exempt, one way only: a `#[cfg(test)] mod name { .. }` block +//! (or `mod name;`, with any attributes and a visibility between the gate and +//! `mod`) is removed before the scan, by matching brackets over text from which +//! comments and literals have already been blanked by [`crate::rust_source`]'s +//! `strip_literals_and_comments`, so a brace inside a string cannot move where +//! the block appears to end. A `#[cfg(test)]` on anything else (a `use`, a +//! function, a field, a variant, an expression) exempts nothing: what it covers +//! is scanned like production code, which fails toward a false alarm rather +//! than a missed import. A test that needs `chat` goes in the test module. +//! +//! Known limits, stated rather than hidden: +//! +//! * Only `#[cfg(test)]` exactly is recognised as a test gate. `#[cfg(all(test, +//! ..))]` reads as production code, which fails toward a false alarm. +//! * The scan covers the files in `GUARDED`, not the modules they import. A +//! module the planner depends on that imports `chat` would re-form the cycle +//! one step away; add it to `GUARDED` if that ever needs pinning. +//! * `chat` is found only as a path segment after `crate::`, `super::` or +//! `self::`, as a member of a group opened by one of those, or as `mod chat`. +//! A route that never writes that path is not seen: an alias (`use crate as +//! root; root::chat::f()`), a glob (`use crate::*;` then `chat::f()`) or a +//! `#[path = ".."] mod` that points at `chat.rs`. None is used in these files. + +#[cfg(test)] +mod tests { + use crate::rust_source::strip_literals_and_comments; + use std::path::Path; + + /// The files that must not import `chat` in production code, relative to + /// the repository root. + /// + /// The planner itself, then the leaf modules it reaches the shared tool + /// policy, argv helpers and prompt text through. The test below fails if + /// one of these no longer exists, so a rename has to update this list + /// rather than quietly dropping the file out of the guard. + const GUARDED: &[&str] = &[ + "apps/rocm/src/freeform_planner.rs", + "apps/rocm/src/prompt_text.rs", + "apps/rocm/src/rocm_argv.rs", + "apps/rocm/src/tool_policy.rs", + ]; + + /// The reserved words, strict and future. Only these keep their `r#` in + /// [`tokenize`]; any other raw identifier is the plain name. + const KEYWORDS: &[&str] = &[ + "abstract", "as", "async", "await", "become", "box", "break", "const", "continue", "crate", + "do", "dyn", "else", "enum", "extern", "false", "final", "fn", "for", "gen", "if", "impl", + "in", "let", "loop", "macro", "match", "mod", "move", "mut", "override", "priv", "pub", + "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "try", + "type", "typeof", "unsafe", "unsized", "use", "virtual", "where", "while", "yield", + ]; + + #[derive(Clone, Debug, PartialEq, Eq)] + enum Tok { + Ident(String), + /// `::` + PathSep, + Punct(char), + } + + #[derive(Clone, Debug)] + struct Token { + tok: Tok, + line: usize, + } + + /// A production-code import of `chat`. + #[derive(Clone, Debug, PartialEq, Eq)] + struct Offense { + line: usize, + /// What was found, for the failure message. + what: &'static str, + } + + /// Split already-stripped source into identifiers, `::` and single + /// punctuation characters, keeping the 1-based line of each. + fn tokenize(stripped: &str) -> Vec { + let chars: Vec = stripped.chars().collect(); + let mut tokens = Vec::new(); + let mut line = 1; + let mut i = 0; + while i < chars.len() { + let c = chars[i]; + if c == '\n' { + line += 1; + i += 1; + } else if c.is_whitespace() { + i += 1; + } else if c == 'r' + && chars.get(i + 1) == Some(&'#') + && chars + .get(i + 2) + .is_some_and(|next| next.is_alphabetic() || *next == '_') + { + // A raw identifier, `r#chat`: one token that names `chat`. + let start = i + 2; + i = start; + while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') { + i += 1; + } + let name: String = chars[start..i].iter().collect(); + // `r#type` is a name that happens to spell a keyword, so it + // keeps its prefix and cannot be mistaken for the keyword. + let name = if KEYWORDS.contains(&name.as_str()) { + format!("r#{name}") + } else { + name + }; + tokens.push(Token { + tok: Tok::Ident(name), + line, + }); + } else if c.is_alphanumeric() || c == '_' { + let start = i; + while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') { + i += 1; + } + tokens.push(Token { + tok: Tok::Ident(chars[start..i].iter().collect()), + line, + }); + } else if c == ':' && chars.get(i + 1) == Some(&':') { + tokens.push(Token { + tok: Tok::PathSep, + line, + }); + i += 2; + } else { + tokens.push(Token { + tok: Tok::Punct(c), + line, + }); + i += 1; + } + } + tokens + } + + fn is_punct(tokens: &[Token], index: usize, wanted: char) -> bool { + tokens + .get(index) + .is_some_and(|t| t.tok == Tok::Punct(wanted)) + } + + fn is_ident(tokens: &[Token], index: usize, wanted: &str) -> bool { + tokens + .get(index) + .is_some_and(|t| matches!(&t.tok, Tok::Ident(name) if name == wanted)) + } + + /// The length of a `#[cfg(test)]` attribute starting at `index`, if one does. + fn cfg_test_attribute_len(tokens: &[Token], index: usize) -> Option { + let matches = is_punct(tokens, index, '#') + && is_punct(tokens, index + 1, '[') + && is_ident(tokens, index + 2, "cfg") + && is_punct(tokens, index + 3, '(') + && is_ident(tokens, index + 4, "test") + && is_punct(tokens, index + 5, ')') + && is_punct(tokens, index + 6, ']'); + matches.then_some(7) + } + + /// The index just past the `(`, `[` or `{` at `open`, once its matching + /// closer has been consumed (the end of the input when it has none). + fn past_balanced(tokens: &[Token], open: usize) -> usize { + let mut depth = 0usize; + let mut i = open; + while i < tokens.len() { + match tokens[i].tok { + Tok::Punct('(' | '[' | '{') => depth += 1, + Tok::Punct(')' | ']' | '}') => { + depth = depth.saturating_sub(1); + if depth == 0 { + return i + 1; + } + } + _ => {} + } + i += 1; + } + tokens.len() + } + + /// The first token of what a gate covers: past any attributes stacked under + /// the gate and a `pub` or `pub(..)` visibility. + fn first_token_gated(tokens: &[Token], mut index: usize) -> usize { + loop { + if is_punct(tokens, index, '#') && is_punct(tokens, index + 1, '[') { + index = past_balanced(tokens, index + 1); + } else if is_ident(tokens, index, "pub") { + index += 1; + if is_punct(tokens, index, '(') { + index = past_balanced(tokens, index); + } + } else { + return index; + } + } + } + + /// The index just past the `#[cfg(test)] mod name { .. }` (or `mod name;`) + /// whose gate ends just before `start`, or `None` when the gate is on + /// anything else. + fn past_test_module(tokens: &[Token], start: usize) -> Option { + let module = first_token_gated(tokens, start); + if !is_ident(tokens, module, "mod") { + return None; + } + let after_name = module + 2; + if is_punct(tokens, after_name, '{') { + Some(past_balanced(tokens, after_name)) + } else if is_punct(tokens, after_name, ';') { + Some(after_name + 1) + } else { + None + } + } + + /// The tokens left once every `#[cfg(test)] mod` block is removed. A gate on + /// anything else removes nothing, so what it covers is scanned as the + /// production code it may be. + fn without_test_modules(tokens: &[Token]) -> Vec { + let mut kept = Vec::new(); + let mut i = 0; + while i < tokens.len() { + if let Some(len) = cfg_test_attribute_len(tokens, i) + && let Some(end) = past_test_module(tokens, i + len) + { + i = end; + continue; + } + kept.push(tokens[i].clone()); + i += 1; + } + kept + } + + fn is_path_root(tokens: &[Token], index: usize) -> bool { + ["crate", "super", "self"] + .iter() + .any(|root| is_ident(tokens, index, root)) + } + + /// Every place `source` reaches the `chat` module outside test code. + /// + /// Takes the raw file text, so a caller (the real-tree test, the fixtures) + /// cannot forget to strip comments and literals first. + fn chat_imports_in_production_code(source: &str) -> Vec { + let tokens = without_test_modules(&tokenize(&strip_literals_and_comments(source))); + let mut offenses = Vec::new(); + // One entry per open `{`: whether it directly follows `crate::`, + // `super::` or `self::`, so its members are children of the root. + let mut groups: Vec = Vec::new(); + for (i, token) in tokens.iter().enumerate() { + match &token.tok { + Tok::Punct('{') => { + let after_root = + i >= 2 && tokens[i - 1].tok == Tok::PathSep && is_path_root(&tokens, i - 2); + groups.push(after_root); + } + Tok::Punct('}') => { + groups.pop(); + } + Tok::Ident(name) if name == "chat" => { + let after_root_path = + i >= 2 && tokens[i - 1].tok == Tok::PathSep && is_path_root(&tokens, i - 2); + let group_member = groups.last() == Some(&true) + && (is_punct(&tokens, i - 1, '{') || is_punct(&tokens, i - 1, ',')); + let module_decl = i >= 1 && is_ident(&tokens, i - 1, "mod"); + let what = if after_root_path { + Some("a path through `crate::chat` (or `super::`/`self::`)") + } else if group_member { + Some("`chat` as a member of a `use crate::{..}` group") + } else if module_decl { + Some("a `mod chat` declaration") + } else { + None + }; + if let Some(what) = what { + offenses.push(Offense { + line: token.line, + what, + }); + } + } + _ => {} + } + } + offenses + } + + /// One failure line per offense, locating it as `file:line`. + fn report(file: &str, offenses: &[Offense]) -> Vec { + offenses + .iter() + .map(|o| format!("{file}:{}: {}", o.line, o.what)) + .collect() + } + + #[test] + fn the_freeform_planner_and_its_leaf_modules_never_import_chat() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory"); + let mut failures = Vec::new(); + for relative in GUARDED { + let source = std::fs::read_to_string(root.join(relative)).unwrap_or_else(|err| { + panic!( + "cannot read {relative}: {err}. If the module moved or was renamed, \ + update GUARDED in xtask/src/chat_import_contract.rs so it stays guarded." + ) + }); + failures.extend(report(relative, &chat_imports_in_production_code(&source))); + } + assert!( + failures.is_empty(), + "these modules import `chat` in non-test code, which closes the cycle \ + `chat` -> freeform planner -> `chat` again:\n {}\n\ + Move whatever the module needs from `chat.rs` into a leaf module \ + (`prompt_text.rs`, `rocm_argv.rs` or `tool_policy.rs`, or a new one that \ + does not import `chat`) and import it from there. `chat` may be \ + referenced from a `#[cfg(test)]` item.", + failures.join("\n ") + ); + } + + #[test] + fn the_guard_finds_an_import_in_the_real_planner_source() { + // Liveness: the fixtures below prove the scan on small strings; this + // proves it on the real file, whose test items and braces it has to + // get past before it reaches the production code. + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory"); + let source = std::fs::read_to_string(root.join(GUARDED[0])).expect("planner source"); + // Counted against the file's own result, so this stays a statement + // about the scanner when the real-tree test above is red. + let baseline = chat_imports_in_production_code(&source).len(); + let with_import = format!("use crate::chat;\n{source}"); + let hits = chat_imports_in_production_code(&with_import); + assert_eq!(hits.len(), baseline + 1, "{hits:?}"); + assert_eq!(hits[0].line, 1, "{hits:?}"); + let at_the_end = format!("{source}\nfn late() {{ crate::chat::f(); }}\n"); + let hits = chat_imports_in_production_code(&at_the_end); + assert_eq!( + hits.len(), + baseline + 1, + "a call appended after the file's `mod tests` is production code: {hits:?}" + ); + } + + fn lines_of(source: &str) -> Vec { + chat_imports_in_production_code(source) + .iter() + .map(|o| o.line) + .collect() + } + + #[test] + fn a_plain_use_of_chat_is_reported_with_its_line() { + let source = "use anyhow::Result;\n\nuse crate::chat;\n"; + let offenses = chat_imports_in_production_code(source); + assert_eq!(offenses.len(), 1, "{offenses:?}"); + assert_eq!( + report("apps/rocm/src/freeform_planner.rs", &offenses), + vec![ + "apps/rocm/src/freeform_planner.rs:3: \ + a path through `crate::chat` (or `super::`/`self::`)" + .to_string() + ] + ); + } + + #[test] + fn every_spelling_of_the_import_is_reported() { + for (source, line) in [ + ("use crate::chat::any_substring;\n", 1), + ("use crate::chat as c;\n", 1), + ("use super::chat::ensure_flag;\n", 1), + ("use self::chat;\n", 1), + ("use crate::{Cli, chat, providers};\n", 1), + ("use crate::{Cli, providers, chat};\n", 1), + ("use crate::{\n Cli,\n chat,\n providers,\n};\n", 3), + ("use crate::{chat::any_substring, Cli};\n", 1), + ("fn f() {\n let _ = crate::chat::f();\n}\n", 2), + ("mod chat;\n", 1), + ("#[cfg(not(test))]\nuse crate::chat;\n", 2), + // A re-export, and the shape the cycle had before the leaf modules: + // `chat` in the import group, then bare `chat::f()` calls. + ("pub use crate::chat::any_substring;\n", 1), + ("pub(crate) use crate::chat::{A, B};\n", 1), + ( + "use crate::{Cli, chat, providers};\nfn f() { chat::any_substring(); }\n", + 1, + ), + ] { + assert_eq!(lines_of(source), vec![line], "{source:?}"); + } + } + + #[test] + fn a_chat_import_inside_a_test_module_is_allowed() { + for source in [ + "fn prod() {}\n\n#[cfg(test)]\nmod tests {\n use crate::chat;\n use crate::{Cli, chat as c};\n #[test]\n fn t() {\n crate::chat::f();\n }\n}\n", + "#[cfg(test)]\n#[allow(dead_code)]\npub(crate) mod helpers {\n pub fn f() -> bool {\n crate::chat::f()\n }\n}\n", + "#[cfg(test)]\n#[path = \"planner_tests.rs\"]\nmod tests;\n", + ] { + assert!( + chat_imports_in_production_code(source).is_empty(), + "{source:?}" + ); + } + } + + #[test] + fn production_code_around_a_test_module_is_still_scanned() { + let source = "use crate::chat::A;\n\ + #[cfg(test)]\nmod tests {\n use crate::chat::B;\n}\n\ + use crate::chat::C;\n\ + #[cfg(test)]\nmod more {\n fn f() { crate::chat::D; }\n}\n\ + fn prod() {\n crate::chat::E;\n}\n"; + assert_eq!(lines_of(source), vec![1, 6, 12]); + } + + #[test] + fn a_test_module_ends_at_its_own_closing_brace_not_at_a_brace_in_a_literal() { + // The `use` between the two literals is test code and stays exempt: a + // `}` read out of the first literal would close the item before it. + let source = "#[cfg(test)]\nmod tests {\n const S: &str = \"}\";\n use crate::chat;\n const T: &str = \"{\";\n}\n\nuse crate::chat;\n"; + assert_eq!(lines_of(source), vec![8]); + } + + #[test] + fn mentions_of_chat_that_are_not_imports_are_ignored() { + for source in [ + "// use crate::chat;\n", + "/// Moved out of `crate::chat` so the planner never imports it.\n", + "/* use crate::chat; */\n", + "const NOTE: &str = \"crate::chat::f\";\n", + "use crate::rocm_argv::chat_cli_arg_value;\n", + "use crate::{providers, tool_policy};\nfn f() { chat_tool_call_is_read_only(); }\n", + "fn f() {\n let chat = 1;\n let _ = chat;\n}\n", + "use crate::providers::chat;\n", + "use crate::{providers::{chat}, Cli};\n", + "use crate::{a::{b, chat}};\n", + "fn f(s: &str) -> bool {\n s.contains(\"chat\")\n}\n", + ] { + assert!( + chat_imports_in_production_code(source).is_empty(), + "{source:?}" + ); + } + } + + #[test] + fn a_raw_identifier_spelling_of_chat_is_reported() { + // `r#chat` names the same module as `chat`, so every form above has a + // raw twin that must fail the guard too. + for (source, line) in [ + ("use crate::r#chat;\n", 1), + ("use crate::{r#chat};\n", 1), + ("use crate::{Cli, r#chat, providers};\n", 1), + ("use crate::{Cli, r#chat as c};\n", 1), + ("use crate::{\n Cli,\n r#chat,\n};\n", 3), + ("use crate::{r#chat::any_substring, Cli};\n", 1), + ("use super::r#chat::ensure_flag;\n", 1), + ("use self::r#chat;\n", 1), + ("mod r#chat;\n", 1), + ("pub mod r#chat;\n", 1), + ("fn f() {\n let _ = crate::r#chat::f();\n}\n", 2), + ("#[cfg(not(test))]\nuse crate::r#chat;\n", 2), + ] { + assert_eq!(lines_of(source), vec![line], "{source:?}"); + } + } + + #[test] + fn raw_identifiers_that_are_not_the_chat_module_are_ignored() { + for source in [ + "use crate::providers::r#chat;\n", + "use crate::r#chat_cli_arg_value;\n", + "fn f() {\n let r#chat = 1;\n let _ = r#chat;\n}\n", + "use crate::{providers::{r#chat}, Cli};\n", + "struct S {\n r#type: u8,\n}\n", + ] { + assert!( + chat_imports_in_production_code(source).is_empty(), + "{source:?}" + ); + } + } + + #[test] + fn a_raw_identifier_is_one_token_and_a_raw_keyword_stays_distinct() { + let names = |source: &str| -> Vec { + tokenize(source) + .into_iter() + .filter_map(|t| match t.tok { + Tok::Ident(name) => Some(name), + _ => None, + }) + .collect() + }; + assert_eq!(names("r#chat"), vec!["chat"]); + assert_eq!(names("crate::r#chat::f"), vec!["crate", "chat", "f"]); + // `r#type` is a field or variable called `type`, not the keyword that + // starts a type alias, so it must not read as one. + assert_eq!(names("r#type"), vec!["r#type"]); + assert_eq!(names("r#match r#fn"), vec!["r#match", "r#fn"]); + // An `r` that is only the start of an ordinary identifier is untouched. + assert_eq!(names("r rest #[x]"), vec!["r", "rest", "x"]); + } + + #[test] + fn a_test_gate_on_anything_but_a_module_exempts_nothing() { + // Only a `#[cfg(test)] mod` block is exempt. A gate on a `use`, a + // function, a field, a variant, a parameter or an expression covers + // nothing the scan skips, so it cannot hide what follows it either: + // this is the shape that let a comparison `<` inside a gated element + // swallow the production elements after it when the scan tried to find + // where each such gate ends. Every line below must still be reported. + for (source, lines) in [ + ("#[cfg(test)]\nuse crate::chat::any_substring;\n", &[2][..]), + ( + "#[cfg(test)]\nfn only_in_tests() {\n crate::chat::f();\n}\n", + &[3], + ), + ( + "struct S {\n #[cfg(test)]\n m: crate::chat::T,\n n: crate::chat::U,\n}\n", + &[3, 4], + ), + ( + "fn f() -> S {\n S {\n #[cfg(test)]\n a: x < 1,\n b: crate::chat::B,\n }\n}\n", + &[5], + ), + ( + "fn f(x: u8) {\n match x {\n #[cfg(test)]\n 0 => crate::chat::f(),\n _ => crate::chat::g(),\n }\n}\n", + &[4, 5], + ), + ("#[cfg(all(test, unix))]\nuse crate::chat;\n", &[2]), + ] { + assert_eq!(lines_of(source), lines, "{source:?}"); + } + } +} diff --git a/xtask/src/dash_text_edit_contract.rs b/xtask/src/dash_text_edit_contract.rs new file mode 100644 index 0000000000..299ed08a36 --- /dev/null +++ b/xtask/src/dash_text_edit_contract.rs @@ -0,0 +1,938 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: a dash-TUI text field edits its `String` through +//! `apply_char_key` / `apply_char_key_filtered` in `ui/widgets.rs`, not through +//! its own "`Char` appends, `Backspace` pops one char" code. +//! +//! The helper replaced the hand-rolled copies, and the convention lives in +//! `AGENTS.md` and `docs/architecture.md`. Prose does not stop the next copy, so +//! this scan does. +//! +//! The rule is per file, not per `match` arm, because the copies were not all +//! arms: `install_manager.rs` had `KeyCode::Char(c) => i.type_char(c)` with the +//! `push` in a separate `fn type_char(&mut self, c: char)`. In the production +//! code of every file under `crates/rocm-dash-tui/src/` other than `ui/widgets.rs`: +//! +//! * a no-argument `.pop()` call in a file that names `KeyCode::Backspace` is +//! reported, and +//! * `.push(x)` (or `.push(*x)`, `.push(x.method())`, `.push(x as char)`, and each +//! of them with a trailing comma such as `.push(x,)`) is reported where `x` is +//! a name the same file binds as a key character (`KeyCode::Char(x)`, +//! `KeyCode::Char(mut x)`) or declares as a `char` parameter (`x: char`), in a +//! file that names `KeyCode::Char(`. +//! +//! Production code means what is left after [`crate::rust_source`]'s +//! `strip_literals_and_comments` blanks comments and literals and every +//! `#[cfg(test)] mod name { .. }` block is blanked too, so a unit test may drive +//! a buffer by hand. A `#[cfg(test)]` on anything else exempts nothing. An +//! out-of-line test module (`#[cfg(test)] mod x;`) is a separate file, which is +//! scanned in full. [`crate::chat_import_contract`] blanks test modules over a +//! token stream instead; this one works over the stripped text because it needs +//! only the module's extent, and the two are not shared. +//! +//! Known limits, stated rather than hidden: +//! +//! * It looks for a `String` edit by shape, not by type. A `.pop()` that takes a +//! value (`let top = stack.pop()`) in a file that handles `Backspace` is a +//! false alarm; rename it, or add the file to `ALLOWED` with the reason. +//! Bindings are per file, not per function, so an unrelated `out.push(c)` is +//! also reported when the same file binds `KeyCode::Char(c)` somewhere else. +//! * Only `.pop()` and `.push(x)` are matched, and a push only when its argument +//! is the bound name alone, as listed above: `.push((x))`, `.push(char::from(x))`, +//! `.push(if shift { x.to_ascii_uppercase() } else { x })` and a call written +//! with a gap before the paren (`.push (x)`, which rustfmt never produces) are +//! not seen. `push_str`, `truncate`, `clear`, `String::pop(&mut b)`, +//! `.insert(..)`, `.remove(..)` and `replace_range(..)` are not scanned. A +//! cursor editor such as the serve wizard's Host field (`InlineEditor` in +//! `ui/serve_wizard.rs`) cannot use the append-only helper, so it is a +//! different thing, and a second copy of one goes unseen. +//! * A file that never names `KeyCode::Char(` is not checked for a `char` +//! push. A typed key can reach such a file through a call from another one (the +//! `install_manager` `type_char` shape, with the arm in a different file), and +//! that split is not seen. `x: char` counts as a binding wherever it appears, +//! as a parameter, a field or a `let`; `x: &char` does not. +//! * Only `#[cfg(test)]` exactly is read as a test gate. `#[cfg(all(test, ..))]` +//! and a file-level `#![cfg(test)]` read as production code, which fails toward +//! a false alarm. `use KeyCode::*` or an alias of `KeyCode` defeats every +//! trigger, since the scan looks for the path `KeyCode::`. +//! * A `Backspace` handler that never pops (`ui/folder_browser.rs` navigates to the +//! parent directory) is not a text edit and is not reported, so it needs no +//! `ALLOWED` entry. +//! * A buffer edited through a wrapper that is itself not in these files is not +//! followed, and a binding renamed on the way to the `push` (`let ch = c;`) is +//! not seen. +//! * An `ALLOWED` entry pins the number of offenses in its file, so a second +//! hand-rolled edit added there fails, but it does not say which one is the +//! documented one: swapping an edit for another keeps the count. + +#[cfg(test)] +mod tests { + use crate::rust_source::strip_literals_and_comments; + use std::path::{Path, PathBuf}; + + /// The one file allowed to own the append/pop edit. + const HELPER_FILE: &str = "crates/rocm-dash-tui/src/ui/widgets.rs"; + + /// The directory the scan covers, relative to the repository root. + const SCANNED_DIR: &str = "crates/rocm-dash-tui/src"; + + /// Files that legitimately keep their own `push` / `pop`: the file, how many + /// offenses it is expected to carry (the documented ones), and the reason. + /// The scan fails when a file carries a different number, so a file that + /// stops needing its entry, and a further hand-rolled edit added next to the + /// documented one, are both caught. + const ALLOWED: &[(&str, usize, &str)] = &[( + "crates/rocm-dash-tui/src/app/actions.rs", + 2, + "the Chat input: its keys are first mapped to a `KeyAction` by a pure, \ + separately tested table, and `apply_action` then pushes and pops the \ + chat buffer from that action, so there is no `KeyCode` match to route \ + through the helper. The two offenses are that push and that pop. The \ + push is found because the file also binds `KeyCode::Char(c)` elsewhere: \ + bindings are file-wide, not followed through `KeyAction::ChatInput(c)`", + )]; + + #[derive(Clone, Debug, PartialEq, Eq)] + struct Offense { + line: usize, + what: String, + } + + fn is_ident_char(c: char) -> bool { + c.is_alphanumeric() || c == '_' + } + + /// Blank `text[from..to]` to spaces, keeping newlines so line numbers hold. + fn blank(chars: &mut [char], from: usize, to: usize) { + for c in &mut chars[from..to] { + if *c != '\n' { + *c = ' '; + } + } + } + + fn skip_ws(chars: &[char], mut i: usize) -> usize { + while chars.get(i).is_some_and(|c| c.is_whitespace()) { + i += 1; + } + i + } + + fn starts_with_at(chars: &[char], i: usize, word: &str) -> bool { + word.chars() + .enumerate() + .all(|(k, w)| chars.get(i + k) == Some(&w)) + } + + /// The index past the bracket group opening at `open`, or the end of input. + fn past_balanced(chars: &[char], open: usize) -> usize { + let mut depth = 0usize; + for (i, c) in chars.iter().enumerate().skip(open) { + match c { + '(' | '[' | '{' => depth += 1, + ')' | ']' | '}' => { + depth = depth.saturating_sub(1); + if depth == 0 { + return i + 1; + } + } + _ => {} + } + } + chars.len() + } + + /// `source` with comments, literals and every `#[cfg(test)] mod` blanked, + /// line structure preserved. + fn production_text(source: &str) -> Vec { + const GATE: &str = "#[cfg(test)]"; + let mut chars: Vec = strip_literals_and_comments(source).chars().collect(); + let mut i = 0; + while i < chars.len() { + if !starts_with_at(&chars, i, GATE) { + i += 1; + continue; + } + let mut j = skip_ws(&chars, i + GATE.len()); + // Attributes stacked under the gate, then a visibility. + loop { + if starts_with_at(&chars, j, "#[") { + j = skip_ws(&chars, past_balanced(&chars, j + 1)); + } else if starts_with_at(&chars, j, "pub") + && !chars.get(j + 3).is_some_and(|c| is_ident_char(*c)) + { + j = skip_ws(&chars, j + 3); + if chars.get(j) == Some(&'(') { + j = skip_ws(&chars, past_balanced(&chars, j)); + } + } else { + break; + } + } + let is_mod = starts_with_at(&chars, j, "mod") + && chars.get(j + 3).is_some_and(|c| c.is_whitespace()); + if !is_mod { + i += GATE.len(); + continue; + } + j = skip_ws(&chars, j + 3); + while chars.get(j).is_some_and(|c| is_ident_char(*c)) { + j += 1; + } + j = skip_ws(&chars, j); + let end = match chars.get(j) { + Some('{') => past_balanced(&chars, j), + Some(';') => j + 1, + _ => { + i += GATE.len(); + continue; + } + }; + blank(&mut chars, i, end); + i = end; + } + chars + } + + fn line_of(chars: &[char], index: usize) -> usize { + 1 + chars[..index].iter().filter(|c| **c == '\n').count() + } + + fn find_all(chars: &[char], word: &str) -> Vec { + (0..chars.len()) + .filter(|i| starts_with_at(chars, *i, word)) + .collect() + } + + /// The identifier starting at `i`, if one does. + fn ident_at(chars: &[char], i: usize) -> Option<(String, usize)> { + let first = chars.get(i)?; + if !(first.is_alphabetic() || *first == '_') { + return None; + } + let mut end = i; + while chars.get(end).is_some_and(|c| is_ident_char(*c)) { + end += 1; + } + Some((chars[i..end].iter().collect(), end)) + } + + /// Names the file binds as a typed-in character: `KeyCode::Char(x)` and + /// `x: char`. + fn char_bindings(chars: &[char]) -> Vec { + let mut names = Vec::new(); + for at in find_all(chars, "KeyCode::Char(") { + let mut i = skip_ws(chars, at + "KeyCode::Char(".len()); + // `mut c` and `ref c` bind the name `c`; the modifier is not it. + while let Some((name, end)) = ident_at(chars, i) { + if name == "mut" || name == "ref" { + i = skip_ws(chars, end); + continue; + } + names.push(name); + break; + } + } + for at in find_all(chars, "char") { + let after = at + "char".len(); + if chars.get(after).is_some_and(|c| is_ident_char(*c)) { + continue; + } + // Walk back over `: ` to the name before it. A `char` that ends a + // longer name (`mychar`) has no `:` before it and stops there. + let mut k = at; + while k > 0 && chars[k - 1].is_whitespace() { + k -= 1; + } + if k == 0 || chars[k - 1] != ':' || (k >= 2 && chars[k - 2] == ':') { + continue; + } + k -= 1; + while k > 0 && chars[k - 1].is_whitespace() { + k -= 1; + } + let end = k; + while k > 0 && is_ident_char(chars[k - 1]) { + k -= 1; + } + if k < end { + names.push(chars[k..end].iter().collect()); + } + } + names + } + + /// Whether the text after a `.push(` argument's leading name (ending at `end`) + /// closes the call with that name as the whole argument: the bare name + /// (`push(c)`), a method on it (`push(c.to_ascii_lowercase())`), a no-op cast + /// to `char` (`push(c as char)`), and any of the first and last with a + /// trailing comma (`push(c,)`, which rustfmt writes when a call spans lines). + /// A second argument (`push(c, 1)`) or a cast to another type + /// (`push(c as u8)`, a `Vec` push) is not a `String` append. + fn is_whole_push_argument(chars: &[char], end: usize) -> bool { + let mut i = skip_ws(chars, end); + if starts_with_at(chars, i, "as") && chars.get(i + 2).is_some_and(|c| c.is_whitespace()) { + let ty = skip_ws(chars, i + 2); + // Any other cast (`as u8`) leaves `i` on the `as`, which closes nothing. + if starts_with_at(chars, ty, "char") + && !chars + .get(ty + "char".len()) + .is_some_and(|c| is_ident_char(*c)) + { + i = skip_ws(chars, ty + "char".len()); + } + } + match chars.get(i) { + Some(')' | '.') => true, + Some(',') => chars.get(skip_ws(chars, i + 1)) == Some(&')'), + _ => false, + } + } + + /// Every hand-rolled append or pop in `source`'s production code. + /// + /// Takes the raw file text, so a caller cannot forget to strip comments, + /// literals and test modules first. + fn hand_rolled_edits(source: &str) -> Vec { + let chars = production_text(source); + let mut offenses = Vec::new(); + let names_backspace = !find_all(&chars, "KeyCode::Backspace").is_empty(); + if names_backspace { + for at in find_all(&chars, ".pop(") { + let close = skip_ws(&chars, at + ".pop(".len()); + if chars.get(close) == Some(&')') { + offenses.push(Offense { + line: line_of(&chars, at), + what: "a `.pop()` in a file that handles `KeyCode::Backspace`".to_owned(), + }); + } + } + } + if !find_all(&chars, "KeyCode::Char(").is_empty() { + let bound = char_bindings(&chars); + for at in find_all(&chars, ".push(") { + let mut arg = skip_ws(&chars, at + ".push(".len()); + // `push(*c)` when the key is matched by reference. + while matches!(chars.get(arg), Some('*' | '&')) { + arg = skip_ws(&chars, arg + 1); + } + let Some((name, end)) = ident_at(&chars, arg) else { + continue; + }; + if is_whole_push_argument(&chars, end) && bound.contains(&name) { + offenses.push(Offense { + line: line_of(&chars, at), + what: format!("`.push({name})` of a typed-in character"), + }); + } + } + } + offenses.sort_by_key(|o| o.line); + offenses + } + + fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + /// Every `.rs` file under the dash-TUI `src/`, as `(repo-relative `/` path, + /// text)`. + fn dash_tui_sources(root: &Path) -> Vec<(String, String)> { + fn walk(root: &Path, dir: &Path, out: &mut Vec<(String, String)>) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + walk(root, &path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + let relative = path + .strip_prefix(root) + .expect("under the root") + .to_string_lossy() + .replace('\\', "/"); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("cannot read {relative}: {err}")); + out.push((relative, text)); + } + } + } + let mut found = Vec::new(); + walk(root, &root.join(SCANNED_DIR), &mut found); + found.sort(); + found + } + + /// The failure lines for `files`: each offense outside the helper file and + /// the allowlist, and each allowlist entry that no longer needs its place. + fn check(files: &[(String, String)]) -> Vec { + let mut failures = Vec::new(); + for (path, source) in files { + if path == HELPER_FILE { + continue; + } + let offenses = hand_rolled_edits(source); + let Some((_, expected, _)) = ALLOWED.iter().find(|(file, _, _)| file == path) else { + for o in offenses { + failures.push(format!( + "{path}:{}: {}. A text field edits its buffer with `apply_char_key` or \ + `apply_char_key_filtered` from ui/widgets.rs, not its own push/pop", + o.line, o.what + )); + } + continue; + }; + let lines = offenses + .iter() + .map(|o| o.line.to_string()) + .collect::>() + .join(", "); + if offenses.is_empty() { + failures.push(format!( + "ALLOWED entry `{path}` is stale: it no longer hand-rolls a text edit. \ + Remove the entry, or the exception outlives its reason" + )); + } else if offenses.len() > *expected { + failures.push(format!( + "ALLOWED entry `{path}` documents {expected} hand-rolled edit(s) and the \ + file now has {} (lines {lines}). A further text field belongs on \ + `apply_char_key`; raise the count only for a reviewed exception", + offenses.len() + )); + } else if offenses.len() < *expected { + failures.push(format!( + "ALLOWED entry `{path}` documents {expected} hand-rolled edit(s) but the \ + file now has {} (lines {lines}): part of the exception is gone. Lower \ + the count to match, or remove the entry if none is left", + offenses.len() + )); + } + } + for (file, _, _) in ALLOWED { + if !files.iter().any(|(path, _)| path == file) { + failures.push(format!( + "ALLOWED entry `{file}` names a file that does not exist; update it in \ + xtask/src/dash_text_edit_contract.rs" + )); + } + } + failures + } + + /// A file with one append and one pop: the shape of the Chat input's pair. + const TWO_EDITS: &str = "\ +fn push(c: KeyCode, q: &mut String) { if let KeyCode::Char(x) = c { q.push(x); } } +fn pop(c: KeyCode, q: &mut String) { if c == KeyCode::Backspace { q.pop(); } } +"; + + /// The failure lines that are about `name`, so a test about one file does + /// not depend on every other file in the tree being clean. + fn in_file<'a>(failures: &'a [String], name: &str) -> Vec<&'a String> { + failures.iter().filter(|f| f.contains(name)).collect() + } + + fn file(path: &str, source: &str) -> (String, String) { + (path.to_owned(), source.to_owned()) + } + + fn lines(offenses: &[Offense]) -> Vec { + offenses.iter().map(|o| o.line).collect() + } + + #[test] + fn the_dash_tui_has_no_hand_rolled_text_edit_outside_the_helper() { + let files = dash_tui_sources(&repo_root()); + let failures = check(&files); + assert!( + failures.is_empty(), + "dash-TUI text fields must edit through the shared helper:\n {}", + failures.join("\n ") + ); + } + + #[test] + fn the_scan_reaches_the_helper_and_the_converted_modules() { + let files = dash_tui_sources(&repo_root()); + for needed in [ + HELPER_FILE, + "crates/rocm-dash-tui/src/ui/logs_view.rs", + "crates/rocm-dash-tui/src/ui/install_manager.rs", + "crates/rocm-dash-tui/src/app/actions.rs", + "crates/rocm-dash-tui/src/ui/folder_browser.rs", + ] { + assert!( + files.iter().any(|(path, _)| path == needed), + "the walk did not reach {needed}" + ); + } + // The helper itself is the shape being banned elsewhere, so it must + // read as an offender here: that is what the exemption is for. Its count + // is not pinned (`check` skips the helper file, so nothing else depends on + // it, and an unrelated edit to `widgets.rs` must not fail this test): it + // must show a push and a pop, the two halves of the edit it implements. + let helper = &files.iter().find(|(p, _)| p == HELPER_FILE).unwrap().1; + let seen: Vec = hand_rolled_edits(helper) + .into_iter() + .map(|offense| offense.what) + .collect(); + assert!( + seen.iter().any(|what| what.contains(".push(")), + "the scan no longer sees the helper's push: {seen:?}" + ); + assert!( + seen.iter().any(|what| what.contains(".pop()")), + "the scan no longer sees the helper's pop: {seen:?}" + ); + } + + #[test] + fn every_allowed_file_carries_a_reason() { + for (file, expected, reason) in ALLOWED { + assert!(*expected > 0, "ALLOWED entry `{file}` expects no offenses"); + assert!( + reason.len() > 40, + "ALLOWED entry `{file}` has no real justification: {reason:?}" + ); + } + } + + #[test] + fn a_hand_rolled_arm_is_reported_with_its_line() { + let source = "\ +fn on_key(l: &mut Logs, key: KeyEvent) { + match key.code { + KeyCode::Char(c) => l.query.push(c), + KeyCode::Backspace => { + l.query.pop(); + } + _ => {} + } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![3, 5]); + } + + #[test] + fn the_helper_split_shape_that_the_arm_scan_would_miss_is_reported() { + // install_manager.rs before the shared helper: the arms call methods, + // and the push/pop live in those methods. + let source = "\ +fn on_key(i: &mut Install, key: KeyEvent) { + match key.code { + KeyCode::Backspace => i.backspace(), + KeyCode::Char(ch) => i.type_char(ch), + _ => {} + } +} +impl Install { + fn type_char(&mut self, c: char) { + match self.field { + Field::Channel => self.channel.push(c), + _ => {} + } + } + fn backspace(&mut self) { + match self.field { + Field::Channel => { + self.channel.pop(); + } + _ => {} + } + } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![11, 18]); + } + + #[test] + fn a_char_parameter_is_a_binding_even_when_no_arm_names_it() { + // Only `x: char` binds `c` here; `KeyCode::Char('x')` just makes this a + // file that handles typed keys. + let source = "\ +fn on_key(code: KeyCode, i: &mut Install) { + if code == KeyCode::Char('x') { i.clear_channel(); } +} +fn type_char(q: &mut String, c: char) { + q.push(c); +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![5]); + } + + #[test] + fn a_binding_by_mut_or_ref_or_a_pushed_deref_or_method_is_reported() { + // One file each: a name bound by any form is bound for the whole file, + // so a shared `c` would let one form's binding cover for another's. + for source in [ + "fn a(code: KeyCode, q: &mut String) {\n if let KeyCode::Char(mut c) = code { q.push(c); }\n}\n", + "fn b(code: &KeyCode, q: &mut String) {\n if let KeyCode::Char(c) = code { q.push(*c); }\n}\n", + "fn d(code: KeyCode, q: &mut String) {\n if let KeyCode::Char(ref c) = code { q.push(c.to_ascii_lowercase()); }\n}\n", + ] { + assert_eq!(lines(&hand_rolled_edits(source)), vec![2], "{source}"); + } + } + + #[test] + fn binding_names_are_whole_identifiers_not_single_letters() { + // `key_char` carries an underscore and `mychar` ends in the word `char`: + // only the first is a char parameter, and only it is pushed. + let source = "\ +fn on_key(code: KeyCode, i: &mut Install) { + if code == KeyCode::Char('x') { i.clear_channel(); } +} +fn typed(q: &mut String, key_char: char) { + q.push(key_char); +} +fn bytes(out: &mut Vec, mychar: u8) { + out.push(mychar); +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![5]); + } + + #[test] + fn a_push_through_two_dereferences_is_reported() { + let source = "\ +fn a(code: &&KeyCode, q: &mut String) { + if let KeyCode::Char(c) = code { q.push(**c); } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![2]); + } + + /// A key arm that appends `shape` to a `String`, wrapped in a function so the + /// whole file is a valid fixture for [`hand_rolled_edits`]. + fn key_arm_pushing(shape: &str) -> String { + format!( + "fn f(code: KeyCode, q: &mut String) {{\n if let KeyCode::Char(c) = code {{\n \ + {shape};\n }}\n}}\n" + ) + } + + #[test] + fn a_push_with_a_trailing_comma_is_reported() { + // `q.push(c,)` is a valid call, and the multi-line form below is what + // rustfmt writes when a call is too long for one line. Both append a + // typed-in character, so both are offenses. + let one_line = key_arm_pushing("q.push(c,)"); + assert_eq!(lines(&hand_rolled_edits(&one_line)), vec![3], "{one_line}"); + let multi_line = key_arm_pushing("q.push(\n c,\n )"); + assert_eq!( + lines(&hand_rolled_edits(&multi_line)), + vec![3], + "{multi_line}" + ); + } + + #[test] + fn the_other_spellings_of_a_pushed_typed_in_char_are_reported() { + let mut missed = Vec::new(); + for (shape, call) in [ + ("spaces around the comma", "q.push( c , )"), + ("a dereference and a comma", "q.push(*c,)"), + ("a method and a comma", "q.push(c.to_ascii_lowercase(),)"), + ("a comment before the paren", "q.push(c /* typed */)"), + ("a cast to char", "q.push(c as char)"), + ("a cast to char and a comma", "q.push(c as char,)"), + ( + "the argument on its own line", + "q.push(\n c\n )", + ), + ] { + let source = key_arm_pushing(call); + if hand_rolled_edits(&source).len() != 1 { + missed.push(shape); + } + } + assert!(missed.is_empty(), "not reported: {missed:?}"); + } + + #[test] + fn a_push_with_more_than_the_typed_in_char_is_not_reported() { + // Not a `String` append of the typed key: two arguments, or a cast to a + // type a `String` cannot take. + for (shape, call) in [ + ("two arguments", "q.push(c, 1)"), + ("two arguments over lines", "q.push(c,\n 1)"), + ("a cast to a byte", "q.push(c as u8,)"), + ("a cast to a byte", "q.push(c as u8)"), + ] { + let source = key_arm_pushing(call); + assert!(hand_rolled_edits(&source).is_empty(), "{shape}: {source}"); + } + } + + #[test] + fn a_pop_with_an_argument_and_a_push_of_a_computed_value_are_ignored() { + let source = "\ +fn f(code: KeyCode, history: &mut History, bytes: &mut Vec) { + if code == KeyCode::Backspace { history.pop(2); } + if let KeyCode::Char(c) = code { bytes.push(c as u8); } +} +"; + assert!(hand_rolled_edits(source).is_empty()); + } + + #[test] + fn a_push_of_something_that_is_not_a_typed_in_char_is_ignored() { + let source = "\ +fn on_key(key: KeyEvent, out: &mut Vec, label: &str) { + match key.code { + KeyCode::Char('x') => out.push(label.to_owned()), + KeyCode::Char(c) if c.is_ascii_digit() => out.push(format!(\"{c}\")), + _ => {} + } + out.push(label); +} +"; + assert!(hand_rolled_edits(source).is_empty()); + } + + #[test] + fn a_pop_or_push_in_a_file_that_handles_no_text_key_is_ignored() { + // `exec.rs` and similar pop a Vec and render chars; neither is a field. + let source = "\ +fn pad(out: &mut String, c: char) { + out.push(c); +} +fn last(words: &mut Vec) -> Option { + words.pop() +} +"; + assert!(hand_rolled_edits(source).is_empty()); + } + + #[test] + fn a_pop_that_takes_a_value_is_still_reported_in_a_backspace_file() { + // The scan cannot tell a `Vec` pop from a `String` pop; the allowlist + // is the way out, documented in the module comment. + let source = "\ +fn f(stack: &mut Vec, code: KeyCode) { + if code == KeyCode::Backspace { + let _ = stack.pop(); + } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![3]); + } + + #[test] + fn code_inside_a_test_module_is_allowed_and_code_around_it_is_not() { + let source = "\ +fn on_key(code: KeyCode, q: &mut String) { + if let KeyCode::Char(c) = code { q.push(c); } +} +#[cfg(test)] +mod tests { + fn drive(code: KeyCode, q: &mut String) { + match code { + KeyCode::Char(c) => q.push(c), + KeyCode::Backspace => { q.pop(); } + _ => {} + } + } +} +fn late(code: KeyCode, q: &mut String) { + if code == KeyCode::Backspace { q.pop(); } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![2, 15]); + } + + #[test] + fn a_test_module_with_stacked_attributes_and_a_visibility_is_still_a_test_module() { + let source = "\ +#[cfg(test)] +#[allow(dead_code)] +pub(crate) mod helpers { + fn drive(code: KeyCode, q: &mut String) { + if let KeyCode::Char(c) = code { q.push(c); } + } +} +#[cfg(test)] +pub mod more { + fn drive(code: KeyCode, q: &mut String) { + if code == KeyCode::Backspace { q.pop(); } + } +} +fn late(code: KeyCode, q: &mut String) { + if code == KeyCode::Backspace { q.pop(); } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![15]); + } + + #[test] + fn an_out_of_line_test_module_declaration_leaves_the_rest_of_the_file_scanned() { + let source = "\ +#[cfg(test)] +mod tests; +fn late(code: KeyCode, q: &mut String) { + if code == KeyCode::Backspace { q.pop(); } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![4]); + } + + #[test] + fn a_test_gate_on_a_function_exempts_nothing() { + let source = "\ +#[cfg(test)] +fn on_key(code: KeyCode, q: &mut String) { + if let KeyCode::Char(c) = code { q.push(c); } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![3]); + } + + #[test] + fn a_test_module_ends_at_its_own_brace_not_at_one_in_a_literal() { + let source = "\ +#[cfg(test)] +mod tests { + fn s() -> &'static str { \"}}\" } +} +fn on_key(code: KeyCode, q: &mut String) { + if let KeyCode::Char(c) = code { q.push(c); } +} +"; + assert_eq!(lines(&hand_rolled_edits(source)), vec![6]); + } + + #[test] + fn mentions_in_comments_and_strings_are_not_edits() { + let source = "\ +// KeyCode::Backspace pops with q.pop(); +/// KeyCode::Char(c) pushes with q.push(c). +fn f(code: KeyCode) -> &'static str { + let _ = KeyCode::Backspace; + \"q.pop(); KeyCode::Char(c) q.push(c)\" +} +"; + assert!(hand_rolled_edits(source).is_empty()); + } + + #[test] + fn the_failure_names_the_file_the_line_and_the_helper() { + let files = [ + file( + "crates/rocm-dash-tui/src/ui/new_form.rs", + "fn k(c: KeyCode, q: &mut String) {\n if let KeyCode::Char(x) = c { q.push(x); }\n}\n", + ), + file(ALLOWED[0].0, TWO_EDITS), + ]; + let failures = check(&files); + assert_eq!(failures.len(), 1, "{failures:?}"); + assert!(failures[0].starts_with("crates/rocm-dash-tui/src/ui/new_form.rs:2:")); + assert!(failures[0].contains("apply_char_key")); + } + + #[test] + fn the_helper_file_and_an_allowed_file_are_not_reported() { + let files = [file(HELPER_FILE, TWO_EDITS), file(ALLOWED[0].0, TWO_EDITS)]; + assert_eq!(check(&files), Vec::::new()); + } + + #[test] + fn an_allowed_file_that_loses_part_of_its_exception_fails_with_the_right_advice() { + // One of the two documented edits is gone: the count is stale, and the + // advice must be to lower it, not to move a further field onto the helper. + let one = TWO_EDITS.lines().next().unwrap().to_owned() + "\n"; + let failures = check(&[file(ALLOWED[0].0, &one)]); + assert_eq!(failures.len(), 1, "{failures:?}"); + assert!(failures[0].contains("documents 2"), "{failures:?}"); + assert!(failures[0].contains("now has 1"), "{failures:?}"); + assert!( + failures[0].contains("part of the exception is gone"), + "{failures:?}" + ); + assert!(!failures[0].contains("further text field"), "{failures:?}"); + } + + #[test] + fn a_further_edit_in_an_allowed_file_fails() { + let three = format!( + "{TWO_EDITS}fn more(c: KeyCode, q: &mut String) {{ if let KeyCode::Char(x) = c {{ q.push(x); }} }}\n" + ); + let failures = check(&[file(ALLOWED[0].0, &three)]); + assert_eq!(failures.len(), 1, "{failures:?}"); + assert!(failures[0].contains("documents 2"), "{failures:?}"); + assert!(failures[0].contains("now has 3"), "{failures:?}"); + assert!(failures[0].contains("further text field"), "{failures:?}"); + assert!(failures[0].contains("apply_char_key"), "{failures:?}"); + } + + #[test] + fn an_allowed_file_that_stops_hand_rolling_fails() { + let files = [file(ALLOWED[0].0, "fn nothing() {}\n")]; + let failures = check(&files); + assert_eq!(failures.len(), 1, "{failures:?}"); + assert!(failures[0].contains("is stale")); + } + + #[test] + fn an_allowed_file_that_no_longer_exists_fails() { + let failures = check(&[]); + assert_eq!(failures.len(), 1, "{failures:?}"); + assert!(failures[0].contains("does not exist")); + } + + #[test] + fn the_guard_finds_a_hand_rolled_arm_added_to_a_converted_module() { + // Liveness on real files: the fixtures prove the scan on small strings; + // this proves it on a file it has to get through, in both of the + // spellings the helper replaced. + let files = dash_tui_sources(&repo_root()); + let logs = "crates/rocm-dash-tui/src/ui/logs_view.rs"; + let original = &files.iter().find(|(p, _)| p == logs).unwrap().1; + let helper_call = "apply_char_key(&mut l.query, code);"; + assert!(original.contains(helper_call), "the mutation target moved"); + assert!(hand_rolled_edits(original).is_empty(), "baseline is clean"); + + let inline = original.replace( + helper_call, + "match code { KeyCode::Char(c) => l.query.push(c), \ + KeyCode::Backspace => { l.query.pop(); } _ => {} }", + ); + let mut mutated = files.clone(); + mutated.iter_mut().find(|(p, _)| p == logs).unwrap().1 = inline; + let all = check(&mutated); + let failures = in_file(&all, "logs_view.rs"); + assert_eq!(failures.len(), 2, "{failures:?}"); + + let appended = format!( + "{original}\nfn late(code: KeyCode, q: &mut String) {{\n \ + if code == KeyCode::Backspace {{ q.pop(); }}\n}}\n" + ); + let mut mutated = files.clone(); + mutated.iter_mut().find(|(p, _)| p == logs).unwrap().1 = appended; + let all = check(&mutated); + let failures = in_file(&all, "logs_view.rs"); + assert_eq!( + failures.len(), + 1, + "after the file's test module: {failures:?}" + ); + + // The same edits inside the file's real `mod tests` are test code. + let in_tests = original.replacen( + "mod tests {", + "mod tests {\n fn drive(code: KeyCode, q: &mut String) {\n \ + match code { KeyCode::Char(c) => q.push(c), \ + KeyCode::Backspace => { q.pop(); } _ => {} }\n }", + 1, + ); + assert_ne!(&in_tests, original, "the test module moved"); + let mut mutated = files; + mutated.iter_mut().find(|(p, _)| p == logs).unwrap().1 = in_tests; + let all = check(&mutated); + let failures = in_file(&all, "logs_view.rs"); + assert!(failures.is_empty(), "{failures:?}"); + } +} diff --git a/xtask/src/e2e_prewarm.rs b/xtask/src/e2e_prewarm.rs index 84cd765f5d..abd07a44cc 100644 --- a/xtask/src/e2e_prewarm.rs +++ b/xtask/src/e2e_prewarm.rs @@ -1130,7 +1130,9 @@ mod tests { /// A real `rocm update` report on a machine with no managed runtime, captured /// verbatim from the built binary rather than written by hand — the parser has - /// to survive the whole document, not an idealized excerpt of it. + /// to survive the whole document, not an idealized excerpt of it. The + /// `cli: installed=` token is the leading version `rocm version` prints; a + /// non-release build reports `0.0.0-dev`, not the workspace package version. /// /// Note the `update_surfaces` block: `runtimes: status=none_configured` is one /// character away from a real `runtime … status=…` entry, and carries a @@ -1141,7 +1143,7 @@ update managed runtimes: none next step: run `rocm install sdk --channel release --dry-run` to resolve a TheRock runtime update_surfaces: - cli: installed=0.1.0 status=not_configured reason=repository-owned CLI update feed is not published yet + cli: installed=0.0.0-dev status=not_configured reason=repository-owned CLI update feed is not published yet engines: status=package_managed packaged=[lemonade,vllm] reason=first-party engine binaries update with the rocm-cli package; data-dir plugins are user-managed model_recipes: status=built_in count=13 reason=external signed recipe index is not configured runtimes: status=none_configured reason=TheRock runtime update checks above are the only live update checks in this build diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index bfd9db279d..63fa727cc5 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -142,7 +142,7 @@ fn label_for_root_report(dir: &Path) -> String { /// Strips the literal `${{ matrix.channel }}` template segment a channel /// matrix inserts into its artifact names — nightly's six GPU lanes and, per -/// ROCMAI-125, `e2e-gpu-strix-ubuntu`'s per-PR `[release, nightly]` leg — so +/// ROCMAI-125, `e2e-gpu-strix-ubuntu`'s `[release, nightly]` leg — so /// a name like `e2e-gpu-${{ matrix.channel }}-report` compares as its /// canonical `e2e-gpu-report` base. This scan is text-based over the raw /// workflow source and does not simulate matrix expansion, so the template @@ -207,8 +207,6 @@ mod tests { "e2e-gpu-rad3-report", "e2e-gpu-mi350p-report", "e2e-gpu-w7900-report", - "e2e-gpu-w7900-n-1-report", - "e2e-gpu-w7900-n-2-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-ubuntu-n-1-report", "e2e-gpu-strix-ubuntu-n-2-report", diff --git a/xtask/src/engine_stdio_client_contract.rs b/xtask/src/engine_stdio_client_contract.rs new file mode 100644 index 0000000000..f311abe55c --- /dev/null +++ b/xtask/src/engine_stdio_client_contract.rs @@ -0,0 +1,301 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: only the shared client may send an engine child its +//! request. +//! +//! `rocm` and `rocmd` used to carry their own copies of the JSON-over-stdio +//! request to an engine process, and the two drifted (different environment, +//! different treatment of a failing exit, a SIGPIPE guard in only one). The +//! single client in `crates/rocm-engine-protocol/src/stdio_client.rs` replaced +//! both. The behavioural tests of that client and of each caller catch a copy +//! that *behaves* differently; none of them can see that a second copy exists, +//! because the regression is a new function nobody calls from a test. +//! +//! So this guard is on the ingredient. A client has to put a request on a +//! child's stdin, which takes a piped stdin, and the request is an engine +//! envelope, so a file that does both is a client. [`CLIENT`] is the only file +//! allowed to; a caller reaches an engine through `run_engine_stdio` and has +//! neither a piped stdin nor a need to name an envelope for a child. A second +//! client has to either use the shared one or add itself to [`ALLOWED`] in a +//! reviewed diff that says why, as for [`crate::crate_edges::ALLOWLIST`]. +//! +//! The two ingredients are looked for anywhere in a file, not on one line, so a +//! copy that builds its `Stdio` in one place and passes it to `.stdin(..)` in +//! another is still seen. That makes the scan coarse on purpose: a file that +//! pipes some child's stdin for any reason and also names an envelope type is +//! reported, and the answer to a false positive is a reviewed [`ALLOWED`] entry. +//! +//! What this does **not** claim: it is a source-text scan, not a call-graph +//! analysis. A copy that builds its stdin another way (a pipe made by hand), or +//! reaches the envelope types under another name, is outside what it can see, +//! and it says nothing about whether a caller then uses the shared client +//! correctly (the callers' own tests pin that). It closes the cheap, likely +//! regression: somebody pasting a spawn-write-read helper back into a binary. +//! +//! Bidirectional, like the other allowlists: an entry that no longer matches +//! fails too, so the exemption does not outlive its reason. + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + /// The one file that is the client. + const CLIENT: &str = "crates/rocm-engine-protocol/src/stdio_client.rs"; + + /// Every file that may both pipe a child's stdin and name an engine + /// envelope, with the reason it is allowed to. + const ALLOWED: &[(&str, &str)] = &[( + CLIENT, + "the shared client: it spawns the engine child, writes the request line \ + to its stdin and reads the answer", + )]; + + /// This file, which the scan must skip: its fixtures quote both ingredients + /// as data, and a text scan cannot tell a fixture string from code. + const SELF_PATH: &str = "xtask/src/engine_stdio_client_contract.rs"; + + /// What names an engine envelope: the request and response types, or the + /// helper that builds a request. + const ENVELOPE_TOKENS: &[&str] = &[ + "EngineRequestEnvelope", + "EngineResponseEnvelope", + "engine_request_envelope", + ]; + + fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + /// Every `.rs` file in the repository as repo-relative, `/`-separated + /// paths. `target/` and dot-directories are skipped so the verdict never + /// depends on state outside the commit under test (the same two exclusions + /// as the other contract tests here). + fn workspace_rust_sources(root: &Path) -> Vec { + let mut found = Vec::new(); + collect_rust_files(root, root, &mut found); + assert!( + !found.is_empty(), + "found no Rust sources to scan -- the walk is broken, not the tree" + ); + found.sort(); + found + } + + fn collect_rust_files(root: &Path, dir: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let skip = path + .file_name() + .and_then(|name| name.to_str()) + .is_none_or(|name| name == "target" || name.starts_with('.')); + if skip { + continue; + } + collect_rust_files(root, &path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") + && let Ok(relative) = path.strip_prefix(root) + { + found.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } + + /// `line` with any `//` comment tail removed, so prose that names the + /// ingredients is not read as code. Crude on purpose (it does not track + /// string literals): that fails open only for a pathological line. + fn strip_comment(line: &str) -> &str { + match line.find("//") { + Some(at) => &line[..at], + None => line, + } + } + + /// Whether `text` pipes a child's stdin *and* names an engine envelope, + /// ignoring comments: the two ingredients of an engine stdio client. + fn is_a_client(text: &str) -> bool { + let code: Vec<&str> = text.lines().map(strip_comment).collect(); + // Anywhere in the file, not on one line: a `Stdio` built on one line and + // handed to `.stdin(..)` on another (a helper that takes it as a + // parameter) is the same ingredient. + let pipes_stdin = code.iter().any(|line| line.contains(".stdin(")) + && code.iter().any(|line| line.contains("piped()")); + let names_envelope = code + .iter() + .any(|line| ENVELOPE_TOKENS.iter().any(|token| line.contains(token))); + pipes_stdin && names_envelope + } + + fn client_files(root: &Path) -> Vec { + workspace_rust_sources(root) + .into_iter() + .filter(|relative| relative != SELF_PATH) + .filter(|relative| { + std::fs::read_to_string(root.join(relative)).is_ok_and(|text| is_a_client(&text)) + }) + .collect() + } + + /// Check a set of client files against [`ALLOWED`] in both directions. + fn check(clients: &[String]) -> Vec { + let unexpected = clients + .iter() + .filter(|file| !ALLOWED.iter().any(|(allowed, _)| allowed == file)) + .map(|file| { + format!( + "`{file}` pipes a child's stdin and names an engine envelope, which is an \ + engine stdio client. Use `rocm_engine_protocol::run_engine_stdio` (in \ + {CLIENT}), or add `{file}` to ALLOWED in \ + xtask/src/engine_stdio_client_contract.rs with the reason a second client \ + is needed" + ) + }); + let stale = ALLOWED + .iter() + .filter(|(allowed, _)| !clients.iter().any(|file| file == allowed)) + .map(|(allowed, _)| { + format!( + "ALLOWED entry `{allowed}` is stale: it no longer pipes a child's stdin and \ + names an engine envelope. Remove it, or the exemption outlives its reason" + ) + }); + unexpected.chain(stale).collect() + } + + #[test] + fn there_is_only_one_engine_stdio_client() { + let violations = check(&client_files(&repo_root())); + assert!( + violations.is_empty(), + "rocm and rocmd share one engine stdio client; a second copy can drift from it \ + (ROCMAI-875):\n{}", + violations.join("\n") + ); + } + + #[test] + fn every_allowed_client_carries_a_reason() { + for (file, reason) in ALLOWED { + assert!( + reason.len() > 40, + "ALLOWED entry `{file}` has no real justification: {reason:?}" + ); + } + } + + /// The shape the two replaced copies had: spawn with a piped stdin, write an + /// envelope to it, read the answer back. + const OLD_STYLE_COPY: &str = " +fn engine_request(engine_binary: &Path, envelope: &EngineRequestEnvelope) -> Result<()> { + let mut command = ProcessCommand::new(engine_binary); + command + .arg(\"__engine-stdio\") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()); + let mut child = command.spawn()?; + serde_json::to_writer(child.stdin.as_mut().unwrap(), envelope)?; + let output = child.wait_with_output()?; + let _answer: EngineResponseEnvelope = serde_json::from_slice(&output.stdout)?; + Ok(()) +} +"; + + #[test] + fn a_pasted_back_copy_of_the_old_client_is_recognised_and_reported() { + assert!(is_a_client(OLD_STYLE_COPY)); + let mut clients: Vec = ALLOWED.iter().map(|(file, _)| (*file).to_owned()).collect(); + clients.push("apps/rocmd/src/engine_ipc.rs".to_owned()); + let violations = check(&clients); + assert_eq!(violations.len(), 1, "{violations:?}"); + assert!(violations[0].contains("apps/rocmd/src/engine_ipc.rs")); + assert!(violations[0].contains("run_engine_stdio")); + } + + #[test] + fn a_copy_that_builds_its_stdio_on_one_line_and_uses_it_on_another_is_recognised() { + let split = " +fn spawn_engine(engine: &Path, stdin: Stdio) -> Result { + Ok(Command::new(engine).stdin(stdin).stdout(Stdio::null()).spawn()?) +} +fn ask(engine: &Path, envelope: &EngineRequestEnvelope) -> Result<()> { + let mut child = spawn_engine(engine, Stdio::piped())?; + serde_json::to_writer(child.stdin.as_mut().unwrap(), envelope)?; + Ok(()) +} +"; + assert!(is_a_client(split)); + } + + #[test] + fn a_caller_of_the_shared_client_is_not_a_client() { + let caller = " +fn engine_request(engine_binary: &Path, envelope: &EngineRequestEnvelope) -> Result { + decode_engine_response(run_engine_stdio(engine_binary, [\"stdio\"], &options, envelope)?) +} +"; + assert!(!is_a_client(caller)); + // Piping stdin for something that is not an engine request is fine. + let unrelated = "let child = Command::new(\"tar\").stdin(Stdio::piped()).spawn()?;"; + assert!(!is_a_client(unrelated)); + // Naming the envelope without a child is fine too (the serving side). + let serving = "fn handle(envelope: EngineRequestEnvelope) -> EngineResponseEnvelope {}"; + assert!(!is_a_client(serving)); + } + + #[test] + fn a_mention_in_a_comment_is_not_a_client() { + let prose = " +// builds an EngineRequestEnvelope and pipes it with .stdin(Stdio::piped()) +/// see EngineResponseEnvelope; the child gets `.stdin(Stdio::piped())` +fn nothing() {} +"; + assert!(!is_a_client(prose)); + } + + #[test] + fn the_client_that_is_allowed_stops_being_allowed_if_it_stops_matching() { + let violations = check(&[]); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("is stale")); + assert!(violations[0].contains(CLIENT)); + } + + #[test] + fn the_self_exclusion_names_a_file_that_exists_and_would_otherwise_be_reported() { + // A rename would leave SELF_PATH pointing at nothing, and the exclusion + // would silently stop applying. + let root = repo_root(); + let text = std::fs::read_to_string(root.join(SELF_PATH)) + .unwrap_or_else(|e| panic!("SELF_PATH `{SELF_PATH}` does not name a real file: {e}")); + assert!( + text.contains("OLD_STYLE_COPY"), + "`{SELF_PATH}` no longer holds the fixture the exclusion exists for" + ); + } + + #[test] + fn the_walk_reaches_the_client_and_both_callers() { + // Without this the contract could pass by scanning nothing. + let files = workspace_rust_sources(&repo_root()); + for expected in [ + CLIENT, + "apps/rocm/src/engine_ipc.rs", + "apps/rocmd/src/engine_ipc.rs", + "xtask/src/engine_stdio_client_contract.rs", + ] { + assert!( + files.iter().any(|file| file == expected), + "the source walk missed `{expected}`, so the contract is scanning less than \ + the tree" + ); + } + } +} diff --git a/xtask/src/env_mutation_contract.rs b/xtask/src/env_mutation_contract.rs index f87709918a..41f2353a63 100644 --- a/xtask/src/env_mutation_contract.rs +++ b/xtask/src/env_mutation_contract.rs @@ -40,6 +40,7 @@ #[cfg(test)] mod tests { + use crate::rust_source::strip_literals_and_comments; use std::path::{Path, PathBuf}; fn repo_root() -> PathBuf { @@ -218,219 +219,6 @@ mod tests { out } - /// Replace the CONTENTS of string literals, char literals and comments with - /// spaces, preserving line structure and everything outside them. - /// - /// The scan below counts braces and looks for call text, and both are - /// wrong if a literal can contribute either. Two real files in this repo - /// break the naive version: - /// - /// * `engines/lemonade/src/lib.rs` contains - /// `ensure_cached_archive("test://archive", ..., |_, destination| {`. - /// Truncating the line at `//` throws away the closure's opening brace, - /// so the enclosing test module reads as closed hundreds of lines early - /// and every mutation after it becomes invisible. - /// * `tests/e2e-cucumber/src/mock_server.rs` contains format strings such - /// as `vllm:num_requests_running{{model=...}}`. Those braces inflate the - /// depth, which drags PRODUCTION code into the scanned region and makes - /// the guard fail a build for a mutation it should not be watching. - /// - /// Both directions are silent, which is the one failure mode a permanent - /// guard must not have, so literals are tokenized rather than approximated. - fn strip_literals_and_comments(text: &str) -> String { - #[derive(Clone, Copy)] - enum State { - Code, - LineComment, - BlockComment(usize), - Str, - RawStr(usize), - Char, - } - - let mut out = String::with_capacity(text.len()); - let mut state = State::Code; - let chars: Vec = text.chars().collect(); - let mut i = 0; - while i < chars.len() { - let c = chars[i]; - let next = chars.get(i + 1).copied(); - match state { - State::Code => { - // A raw string opener: `r"`, `r#"`, `br##"` and so on. Checked - // before the plain-string case so the hashes are counted. - if (c == 'r' || c == 'b') - && let Some((hashes, consumed)) = raw_string_opener(&chars, i) - { - out.extend(std::iter::repeat_n(' ', consumed)); - state = State::RawStr(hashes); - i += consumed; - continue; - } - if c == '/' && next == Some('/') { - state = State::LineComment; - out.push_str(" "); - i += 2; - continue; - } - if c == '/' && next == Some('*') { - state = State::BlockComment(1); - out.push_str(" "); - i += 2; - continue; - } - if c == '"' { - state = State::Str; - out.push(' '); - i += 1; - continue; - } - // A lifetime (`'a`) is not a char literal. A char literal is - // `'x'` or `'\n'`, so require a closing quote nearby. - if c == '\'' && is_char_literal(&chars, i) { - state = State::Char; - out.push(' '); - i += 1; - continue; - } - out.push(c); - i += 1; - } - State::LineComment => { - if c == '\n' { - state = State::Code; - out.push('\n'); - } else { - out.push(' '); - } - i += 1; - } - State::BlockComment(depth) => { - if c == '*' && next == Some('/') { - state = if depth == 1 { - State::Code - } else { - State::BlockComment(depth - 1) - }; - out.push_str(" "); - i += 2; - continue; - } - if c == '/' && next == Some('*') { - state = State::BlockComment(depth + 1); - out.push_str(" "); - i += 2; - continue; - } - out.push(if c == '\n' { '\n' } else { ' ' }); - i += 1; - } - State::Str => { - if c == '\\' { - // The escaped character cannot close the literal, so - // it is consumed here. When it is a NEWLINE -- a line - // continuation, which several strings in this tree use, - // including this file's own assertion message -- the - // newline is not part of the string's value but it is - // part of the file's line structure. Replacing it with - // a space merges two source lines and shifts the line - // number of every offense reported after it, which is - // the `file:line` the assertion promises. - out.push(' '); - out.push(if next == Some('\n') { '\n' } else { ' ' }); - i += 2; - continue; - } - if c == '"' { - state = State::Code; - } - out.push(if c == '\n' { '\n' } else { ' ' }); - i += 1; - } - State::RawStr(hashes) => { - if c == '"' && closing_hashes(&chars, i + 1) >= hashes { - state = State::Code; - out.extend(std::iter::repeat_n(' ', hashes + 1)); - i += hashes + 1; - continue; - } - out.push(if c == '\n' { '\n' } else { ' ' }); - i += 1; - } - State::Char => { - if c == '\\' { - out.push_str(" "); - i += 2; - continue; - } - if c == '\'' { - state = State::Code; - } - out.push(' '); - i += 1; - } - } - } - out - } - - /// `Some((hash_count, chars_consumed))` when a raw-string literal opens at - /// `i` (`r"`, `r#"`, `br##"`, ...). - fn raw_string_opener(chars: &[char], i: usize) -> Option<(usize, usize)> { - let mut j = i; - // Accepting the byte-string `b` here is cosmetic, and deliberately has - // no fixture: removing it only means the caller declines at the `b` and - // matches one character later at the `r`, which opens the same literal - // and strips the same span. The single character of difference is the - // `b` itself, emitted as code rather than as a space. Four things read - // the stripped text — the brace counter, the names in `MUTATIONS`, - // [`serializer_column`] and [`has_test_attribute`] — and a `b` whose only - // neighbour on the right is the stripped literal's run of spaces cannot - // complete a brace, a `set_var(`, a `_TEST_LOCK`/`ScopedTestEnv`, or a - // `#[`. Equivalent on every path that reads it, so a test for it would - // assert nothing. - if chars.get(j) == Some(&'b') { - j += 1; - } - if chars.get(j) != Some(&'r') { - return None; - } - j += 1; - let hash_start = j; - while chars.get(j) == Some(&'#') { - j += 1; - } - if chars.get(j) == Some(&'"') { - Some((j - hash_start, j - i + 1)) - } else { - None - } - } - - fn closing_hashes(chars: &[char], mut i: usize) -> usize { - let start = i; - while chars.get(i) == Some(&'#') { - i += 1; - } - i - start - } - - /// Distinguish `'x'` from a lifetime such as `'a` in `&'a str`. - fn is_char_literal(chars: &[char], i: usize) -> bool { - match chars.get(i + 1) { - // `'\n'`, `'\''`, `'\\'` -- always a literal. - Some('\\') => true, - Some(_) => chars.get(i + 2) == Some(&'\''), - // A `'` as the final character of the file. Deliberately untested: - // the two answers are indistinguishable, because the caller's loop - // ends on the next step either way and the one character of output - // that differs (`'` versus a space) is neither a brace nor part of - // a mutating call's name. Flipping this arm is an equivalent - // mutant, so a fixture for it would assert nothing. - None => false, - } - } - /// Whether `code` carries a test attribute. /// /// The rule is "the attribute path's last segment is `test`", which covers diff --git a/xtask/src/eval_fixtures.rs b/xtask/src/eval_fixtures.rs index 2a16f0aff2..dbc335d0ef 100644 --- a/xtask/src/eval_fixtures.rs +++ b/xtask/src/eval_fixtures.rs @@ -18,6 +18,13 @@ //! when the committed copy differs, so a catalog change cannot leave a stale //! recording behind, the same guard `cargo xtask catalog --check` gives the manifest. //! +//! A case may also name a `host`, which makes the stub answer `rocm examine --json`. +//! The reply is the recorded examination with the few fields the host implies +//! edited, and its `status` recomputed by [`rocm_core::Examination::compute_status`], +//! the rule the probe itself uses. The real command prints those fields plus a +//! `summary` of the host that only the `rocm` binary can build, so the replay +//! leaves `summary` out; the skill reads `status` and `has_amd_gpu` and nothing in it. +//! //! Two things are not rendered. What an OLDER CLI printed was recorded once from a //! real older binary and is committed under `recorded/`. And the current CLI's //! `--version` line is fixed text ([`CURRENT_VERSION`]): it is not the real @@ -54,6 +61,13 @@ const EVALS_FILE: &str = "skills/rocm-doctor/evals/evals.json"; const CURRENT_VERSION: &str = "rocm-cli 0.1.0 (recorded for evals)\n"; /// An older CLI rejects `--json` with a usage error, and clap's usage-error code is 2. const USAGE_ERROR_RC: i32 = 2; +/// What the replay prints when `host` says `rocm examine` itself fails. Fixed +/// text, not recorded from a real failure: the skill reacts to the non-zero exit +/// and relays whatever was printed, and the marker is what the eval pins in +/// `logs_contain`, since only the replay prints it. +const EXAMINE_FAILS_STDERR: &str = "error: failed to read the host (recorded for evals)\n"; +/// A genuine inability to examine propagates as an error, which exits 1. +const EXAMINE_FAILS_RC: i32 = 1; /// The part of the dataset this module reads: which prompt each case id has. #[derive(Debug, Deserialize)] @@ -80,6 +94,26 @@ struct Case { cli: Cli, /// The symptom the diagnosis replay was run with. symptom: String, + /// The machine `rocm examine --json` describes, for a case that needs it. A + /// case without one gets no examine reply, and the stub refuses the command. + #[serde(default)] + host: Option, +} + +/// The host a replayed `rocm examine --json` describes. Each is the recorded +/// examination with only the fields the host implies changed. +#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum Host { + /// Linux with no AMD GPU found: status `no-amd-gpu`. + NoAmdGpu, + /// WSL2 where the Windows host has no AMD GPU: status `wsl`, `has_amd_gpu` + /// false. On WSL the status never says `no-amd-gpu`, so the skill reads the field. + WslHostGpuHidden, + /// Linux with an AMD GPU found: status `ok`. + AmdFound, + /// `rocm examine` itself fails and exits non-zero. + ExamineFails, } #[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)] @@ -116,6 +150,10 @@ struct Replies { fix_json: Reply, diagnose_json: Reply, diagnose_text: Reply, + /// Only for a case that names a `host`; left out of the file otherwise, so a + /// case without one is recorded exactly as it was before this existed. + #[serde(skip_serializing_if = "Option::is_none")] + examine_json: Option, } /// Regenerate (or, with `check`, verify) the committed fixtures. @@ -171,8 +209,57 @@ fn load_examination(root: &Path) -> Result { .with_context(|| format!("{} is not a valid examination", path.display())) } +/// The recorded examination, edited to be the machine `host` names. The host sets +/// the fields it implies whatever the recording held, and `status` is recomputed +/// from them by the probe's own rule, never typed. +fn examination_for(base: &rocm_core::Examination, host: Host) -> rocm_core::Examination { + let mut examination = base.clone(); + // Every host states each field `compute_status` and the replay depend on, so + // what comes out never depends on what the recorded examination happened to + // hold for it. `os_family` and `probe_failures` count: a recording captured + // with a failed probe would otherwise turn `amd-found` into `degraded`. + examination.os_family = "linux".to_owned(); + examination.probe_failures = Vec::new(); + examination.is_wsl = host == Host::WslHostGpuHidden; + examination.wsl = None; + examination.has_amd_gpu = host == Host::AmdFound; + examination.has_discrete_amd = host == Host::AmdFound; + examination.gpus = Vec::new(); + match host { + Host::NoAmdGpu | Host::ExamineFails => {} + Host::WslHostGpuHidden => { + // A WSL2 distro whose Windows host exposes a GPU to it (`/dev/dxg`, + // DXCore) but not an AMD one, so nothing ROCm-side is installed. Typed, + // not probed: the probe fills this from the distro it runs in. + examination.wsl = Some(rocm_core::examine::WslFacts { + version: 2, + dxg_device: true, + dxcore: true, + wsl_lib_dir: true, + ..rocm_core::examine::WslFacts::default() + }); + } + Host::AmdFound => { + // `pci_id` is what the Linux probe reads: the bus address that starts + // the `lspci` line. + examination.gpus = vec![rocm_core::examine::Gpu { + name: "AMD Radeon RX 7900 XTX".to_owned(), + gfx_target: "gfx1100".to_owned(), + pci_id: "0000:c3:00.0".to_owned(), + is_apu: Some(false), + is_amd: true, + }]; + } + } + examination.status = examination.compute_status(); + examination +} + /// Everything the stub says for one case. -fn replies_for(root: &Path, case: &Case, examination: &rocm_core::Examination) -> Result { +fn replies_for(root: &Path, case: &Case, recorded: &rocm_core::Examination) -> Result { + let examination = &case + .host + .map_or_else(|| recorded.clone(), |host| examination_for(recorded, host)); let report = rocm_core::run_diagnose(examination, &case.symptom); let diagnose_json = Reply::ok( serde_json::to_string_pretty(&report).context("failed to render the diagnosis")? + "\n", @@ -195,11 +282,25 @@ fn replies_for(root: &Path, case: &Case, examination: &rocm_core::Examination) - }, ), }; + let examine_json = match case.host { + None => None, + Some(Host::ExamineFails) => Some(Reply { + stdout: String::new(), + stderr: EXAMINE_FAILS_STDERR.to_owned(), + rc: EXAMINE_FAILS_RC, + }), + Some(_) => Some(Reply::ok( + serde_json::to_string_pretty(examination) + .context("failed to render the examination")? + + "\n", + )), + }; Ok(Replies { version, fix_json, diagnose_json, diagnose_text, + examine_json, }) } @@ -372,6 +473,116 @@ mod tests { verify(&root).expect("a freshly generated tree must verify"); } + #[test] + fn a_host_replays_the_status_the_probe_would_compute() { + let root = paths::workspace_root().expect("workspace root"); + let recorded = load_examination(&root).expect("examination"); + let mut seen = Vec::new(); + for case in cases(&root) { + let Some(host) = case.host else { continue }; + seen.push(host); + let replies = replies_for(&root, &case, &recorded).expect("replies"); + let examine = replies + .examine_json + .unwrap_or_else(|| panic!("{} names a host but replays no examine", case.id)); + let (status, has_amd_gpu) = match host { + Host::ExamineFails => { + assert_eq!(examine.rc, EXAMINE_FAILS_RC, "{}", case.id); + assert!(examine.stdout.is_empty(), "{}", case.id); + assert_eq!(examine.stderr, EXAMINE_FAILS_STDERR, "{}", case.id); + continue; + } + Host::NoAmdGpu => ("no-amd-gpu", false), + Host::WslHostGpuHidden => ("wsl", false), + Host::AmdFound => ("ok", true), + }; + let value: serde_json::Value = + serde_json::from_str(&examine.stdout).expect("an examination is JSON"); + assert_eq!(value["status"], status, "{}", case.id); + assert_eq!(value["has_amd_gpu"], has_amd_gpu, "{}", case.id); + } + for host in [ + Host::NoAmdGpu, + Host::WslHostGpuHidden, + Host::AmdFound, + Host::ExamineFails, + ] { + assert!(seen.contains(&host), "no case replays the host {host:?}"); + } + } + + #[test] + fn a_host_does_not_depend_on_what_the_recorded_examination_held() { + /// What one host must come out as, whatever the base held. + struct Expected { + host: Host, + is_wsl: bool, + has_amd_gpu: bool, + gpus: usize, + wsl_facts: bool, + status: &'static str, + } + + // A base that says the opposite of every host, so a host that forgot to + // state a field would show the base's value instead of its own. The + // recorded examination alone cannot tell: it already is a Linux host with + // no AMD GPU, so emptying the `NoAmdGpu` arm changed nothing there. + let root = paths::workspace_root().expect("workspace root"); + let mut hostile = load_examination(&root).expect("examination"); + hostile.is_wsl = true; + hostile.has_amd_gpu = true; + hostile.has_discrete_amd = true; + hostile.gpus = vec![rocm_core::examine::Gpu::default()]; + hostile.wsl = Some(rocm_core::examine::WslFacts::default()); + // `compute_status` reads these two as well, so the base disagrees on them. + hostile.os_family = "other".to_owned(); + hostile.probe_failures = vec!["lspci missing".to_owned()]; + hostile.status = "degraded".to_owned(); + + let expected = |host, is_wsl, has_amd_gpu, gpus, wsl_facts, status| Expected { + host, + is_wsl, + has_amd_gpu, + gpus, + wsl_facts, + status, + }; + for want in [ + expected(Host::NoAmdGpu, false, false, 0, false, "no-amd-gpu"), + // Shares an arm with `NoAmdGpu` today; listed so a later split of that + // arm is still pinned. + expected(Host::ExamineFails, false, false, 0, false, "no-amd-gpu"), + expected(Host::WslHostGpuHidden, true, false, 0, true, "wsl"), + expected(Host::AmdFound, false, true, 1, false, "ok"), + ] { + let host = want.host; + let examination = examination_for(&hostile, host); + assert_eq!(examination.is_wsl, want.is_wsl, "{host:?}"); + assert_eq!(examination.has_amd_gpu, want.has_amd_gpu, "{host:?}"); + assert_eq!(examination.has_discrete_amd, want.has_amd_gpu, "{host:?}"); + assert_eq!(examination.gpus.len(), want.gpus, "{host:?}"); + assert_eq!(examination.wsl.is_some(), want.wsl_facts, "{host:?}"); + assert_eq!(examination.os_family, "linux", "{host:?}"); + assert!(examination.probe_failures.is_empty(), "{host:?}"); + assert_eq!(examination.status, want.status, "{host:?}"); + } + } + + #[test] + fn a_case_with_no_host_records_no_examine_reply() { + // So the cases recorded before `host` existed are byte-for-byte what they + // were, and the stub keeps refusing `examine` for them. + let root = paths::workspace_root().expect("workspace root"); + let mut checked = 0; + for case in cases(&root).iter().filter(|case| case.host.is_none()) { + let path = fixtures_dir(&root).join(&case.id).join(REPLIES_FILE); + let text = fs::read_to_string(&path).expect("replies"); + assert!(!text.contains("examine_json"), "{}", case.id); + checked += 1; + } + assert!(checked > 0, "no case without a host to check"); + } + #[test] fn a_changed_fixture_fails_the_check_and_names_it() { let (_tmp, root) = scratch(); @@ -659,8 +870,8 @@ mod tests { // `hooks.py` is what keeps the recorded `rocm` from running anything but a // replay, so its tests (replay, refusal, PATH and temp-dir cleanup) run // with the rest of this crate. Python is on every lane that runs `cargo - // test` in CI (the smoke steps need it), so on CI a missing Python is a - // failure; on a machine without one the test says so and stops. + // test` in CI, so on CI a missing Python is a failure; on a machine + // without one the test says so and stops. let Some(python) = python() else { assert!( std::env::var_os("CI").is_none(), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 403afef349..e3db2768f6 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -12,14 +12,18 @@ mod affected; mod amd_smi_launch_contract; +mod architecture_doc; mod ascii_output_contract; mod catalog; +mod chat_import_contract; mod coverage; mod crate_edges; +mod dash_text_edit_contract; mod demos; mod e2e; mod e2e_prewarm; mod e2e_report; +mod engine_stdio_client_contract; mod env_mutation_contract; mod eval_fixtures; mod lychee; @@ -29,11 +33,16 @@ mod package; mod paths; mod powershell; mod prepush; +mod prepush_stamp; +mod proptest_regressions_contract; mod release_readiness; mod report_mailbox; mod report_rollup; mod rocm_releases; +#[cfg(test)] +mod rust_source; mod signing; +mod smoke; mod tpn; mod verify_commits; mod verify_pinned_keys; @@ -42,7 +51,7 @@ mod workflow_contract; use std::path::PathBuf; use std::process::ExitCode; -use anyhow::Result; +use anyhow::{Result, bail}; use clap::{Parser, Subcommand}; #[derive(Parser)] @@ -91,7 +100,9 @@ pub(crate) enum Command { /// Assert the release/metadata public keys pinned in the installers and /// `apps/rocm/src/therock/fetch.rs` match the canonical published keys under /// `docs/keys/`, and that any configured CI signing key matches the pinned - /// current release key. A no-op while no canonical keys are published. + /// current release key. Each slot must be assigned exactly once (install.sh + /// must also mark it `readonly`), and a slot whose canonical key is not + /// published yet must be empty. VerifyPinnedKeys, /// Print the workspace crates affected by a git range (changed crates plus /// their transitive dependents) as `cargo` package-selection flags, so CI @@ -118,6 +129,10 @@ pub(crate) enum Command { /// cannot be determined, and prints why. It is skipped only when the /// selection is known and has nothing for the step to check. CI always /// checks the whole workspace. + /// + /// A step that already passed on this exact tree (same cargo command, + /// toolchain and environment, clean working tree) is skipped, and a pass is + /// recorded after a green run; see `xtask/src/prepush_stamp.rs`. PrePushCheck { /// Which step to run. #[arg(value_enum)] @@ -129,6 +144,10 @@ pub(crate) enum Command { /// Print the scope and the cargo command without running it. #[arg(long)] dry_run: bool, + /// Run the step even when this tree already passed it, and record the + /// new result. `ROCM_PRE_PUSH_RERUN=1` does the same, for the hook. + #[arg(long)] + rerun: bool, }, /// Fail if any first-party crate has gained a normal/build dependency /// edge outside the allowlist pinned in `xtask/src/crate_edges.rs`. @@ -140,13 +159,11 @@ pub(crate) enum Command { /// is one limit for every file and no way to exempt one; the definition is /// in `xtask/src/module_size.rs`. CheckModuleSize, - /// The same check as `check-module-size`, but it only reports: it prints the - /// offenders and exits successfully, including when the check itself cannot - /// run (`cargo metadata` fails, a file does not parse). That holds once xtask - /// has built: if xtask or `rocm-core` fails to compile, `cargo xtask` exits - /// non-zero before this code runs. The local pre-commit hook runs this until - /// the limit is enforced. - CheckModuleSizeAdvisory, + /// Fail if a production source file (a `.rs` file under `apps/*/src`, + /// `crates/*/src` or `engines/*/src`), or a `.rs` file under `xtask/src`, + /// `tests/e2e-cucumber/src` or `tests/e2e-cucumber/tests`, has no entry in + /// `docs/architecture.md`. + CheckArchitectureDoc, /// Regenerate the published Doctor catalog manifest from the compiled catalog. Catalog { /// Verify the published manifest is current without writing; exit @@ -367,6 +384,26 @@ pub(crate) enum Command { #[arg(long)] table_out: PathBuf, }, + /// Run the local no-fallback smoke gate: build, then drive the built + /// `rocm`, `rocmd` and engine binaries against a throwaway state root and + /// assert their first-run output — chiefly that every GPU-required path + /// fails loudly rather than falling back to CPU. + Smoke { + /// Build profile to build and smoke. `debug` builds the whole workspace + /// with all targets; `release` builds only the four binaries the gate + /// runs. + #[arg(long, value_enum, default_value_t = smoke::Profile::Debug)] + profile: smoke::Profile, + /// Smoke binaries that are already built instead of building first. + #[arg(long)] + skip_build: bool, + /// Target directory to build into and to find the binaries in, under its + /// profile subdirectory. Relative paths resolve against the workspace + /// root. Defaults to `CARGO_TARGET_DIR`, or `target` under the workspace + /// root; either way the build is given the same directory explicitly. + #[arg(long)] + target_dir: Option, + }, /// Lint PowerShell scripts with PSScriptAnalyzer (parse errors are reported /// too, so this also covers syntax). PowershellLint { @@ -402,14 +439,10 @@ fn run() -> Result<()> { } => signing::verify(public_key.as_deref(), &input, &signature)?, Command::VerifyPinnedKeys => verify_pinned_keys::run()?, Command::Affected { base } => affected::run(base)?, - Command::PrePushCheck { - check, - base, - dry_run, - } => prepush::run(check, base, dry_run)?, + command @ Command::PrePushCheck { .. } => run_pre_push_check(command, prepush::run)?, Command::CheckCrateEdges => crate_edges::run()?, Command::CheckModuleSize => module_size::run()?, - Command::CheckModuleSizeAdvisory => module_size::run_advisory()?, + Command::CheckArchitectureDoc => architecture_doc::run()?, Command::Catalog { check } => catalog::run(check)?, Command::EvalFixtures { check } => eval_fixtures::run(check)?, Command::Coverage { bless } => coverage::run(bless)?, @@ -482,11 +515,35 @@ fn run() -> Result<()> { &out, &table_out, )?, + Command::Smoke { + profile, + skip_build, + target_dir, + } => smoke::run(profile, skip_build, target_dir)?, Command::PowershellLint { shell, files } => powershell::run(shell, files)?, } Ok(()) } +/// The `pre-push-check` arm of [`run`]: hand each argument to `run` as parsed. +/// `run` is a parameter so a test sees exactly what arrives, in particular that +/// `--rerun` is not dropped on the way to the stamp logic. +fn run_pre_push_check( + command: Command, + run: impl FnOnce(prepush::Check, Option, bool, bool) -> Result<()>, +) -> Result<()> { + let Command::PrePushCheck { + check, + base, + dry_run, + rerun, + } = command + else { + bail!("not a pre-push-check command"); + }; + run(check, base, dry_run, rerun) +} + fn main() -> ExitCode { match run() { Ok(()) => ExitCode::SUCCESS, @@ -496,3 +553,67 @@ fn main() -> ExitCode { } } } + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::{Cli, Command}; + + fn rerun(args: &[&str]) -> bool { + let cli = Cli::try_parse_from(args).expect("the arguments parse"); + let Command::PrePushCheck { rerun, .. } = cli.command else { + panic!("expected pre-push-check"); + }; + rerun + } + + /// What the dispatch hands to `prepush::run` for these arguments: the real + /// clap parse, then the `pre-push-check` arm, with `run` recording its input. + fn handed_to_run(args: &[&str]) -> (String, Option, bool, bool) { + let cli = Cli::try_parse_from(args).expect("the arguments parse"); + let mut seen = None; + super::run_pre_push_check(cli.command, |check, base, dry_run, rerun| { + seen = Some((format!("{check:?}"), base, dry_run, rerun)); + Ok(()) + }) + .expect("the dispatch succeeds"); + seen.expect("the arm calls run") + } + + #[test] + fn the_dispatch_hands_every_flag_to_run_unchanged() { + assert_eq!( + handed_to_run(&["xtask", "pre-push-check", "test"]), + ("Test".to_string(), None, false, false) + ); + assert_eq!( + handed_to_run(&[ + "xtask", + "pre-push-check", + "clippy", + "--base", + "origin/main", + "--dry-run", + "--rerun", + ]), + ( + "Clippy".to_string(), + Some("origin/main".to_string()), + true, + true + ) + ); + // `--rerun` alone, and `--dry-run` alone, are not mixed up with each other. + assert!(handed_to_run(&["xtask", "pre-push-check", "test", "--rerun"]).3); + assert!(!handed_to_run(&["xtask", "pre-push-check", "test", "--rerun"]).2); + assert!(handed_to_run(&["xtask", "pre-push-check", "test", "--dry-run"]).2); + assert!(!handed_to_run(&["xtask", "pre-push-check", "test", "--dry-run"]).3); + } + + #[test] + fn pre_push_check_takes_rerun_and_defaults_to_off() { + assert!(rerun(&["xtask", "pre-push-check", "test", "--rerun"])); + assert!(!rerun(&["xtask", "pre-push-check", "test"])); + } +} diff --git a/xtask/src/module_size.rs b/xtask/src/module_size.rs index f13564ab74..5b874c8c54 100644 --- a/xtask/src/module_size.rs +++ b/xtask/src/module_size.rs @@ -67,7 +67,6 @@ use std::collections::BTreeSet; use std::fmt::Write as _; use std::fs; -use std::io::Write; use std::path::{Component, Path, PathBuf}; use anyhow::{Context, Result, anyhow, bail}; @@ -117,7 +116,12 @@ pub(crate) struct Outcome { /// `cargo xtask check-module-size`: fail, naming each file and its count, if any /// source file is over [`MAX_PRODUCTION_LINES`]. pub fn run() -> Result<()> { - let outcome = evaluate()?; + verdict(evaluate()?) +} + +/// The error is what fails the CI step and the pre-commit hook, so the pass and +/// fail branches are split out to be tested without walking the workspace. +fn verdict(outcome: Outcome) -> Result<()> { if outcome.offenders.is_empty() { println!("{}", passed_text(&outcome)); return Ok(()); @@ -125,29 +129,6 @@ pub fn run() -> Result<()> { bail!("{}", failed_text(&outcome)) } -/// `cargo xtask check-module-size-advisory`: the same check, but it only reports. -/// This is the wrapper the local hook runs while the limit is not yet blocking; once -/// xtask has built it cannot fail a commit, including when the check itself cannot -/// run. A compile failure in xtask or `rocm-core` never reaches this function. -pub fn run_advisory() -> Result<()> { - advisory_to(evaluate(), &mut std::io::stdout()); - Ok(()) -} - -/// Print the report. A failed write (a closed pipe, say) is dropped: the wrapper -/// exists so that nothing here can fail a commit. -fn advisory_to(result: Result, out: &mut impl Write) { - let text = match result { - Ok(outcome) if outcome.offenders.is_empty() => passed_text(&outcome), - Ok(outcome) => format!( - "warning: not blocking yet, but this will fail once the limit is enforced\n{}", - failed_text(&outcome) - ), - Err(error) => format!("warning: the module-size check could not run: {error:#}"), - }; - let _ = writeln!(out, "{text}"); -} - fn passed_text(outcome: &Outcome) -> String { format!( "module-size: {} source files checked, none over {MAX_PRODUCTION_LINES} production lines", @@ -155,6 +136,10 @@ fn passed_text(outcome: &Outcome) -> String { ) } +/// The document that states the rule for a file over the limit, which the failure +/// text points at. +const SPLIT_RULE_DOC: &str = "docs/architecture.md"; + fn failed_text(outcome: &Outcome) -> String { let mut text = format!( "{} source file(s) over the limit of {MAX_PRODUCTION_LINES} production lines (of {} checked):", @@ -170,6 +155,10 @@ fn failed_text(outcome: &Outcome) -> String { offender.lines - MAX_PRODUCTION_LINES ); } + let _ = write!( + text, + "\nto fix: split the file in the same PR, as a pure move with its tests moving along with the code; the rule is in {SPLIT_RULE_DOC}" + ); text } @@ -966,6 +955,83 @@ trait T { assert!(offenders(&files, MAX_PRODUCTION_LINES).is_empty()); } + // The CI step and the pre-commit hook both block on this exit status, so it is + // asserted here with the offender the error names. + #[test] + fn an_offender_fails_the_check_and_a_clean_tree_passes() { + let clean = Outcome { + checked: 3, + offenders: Vec::new(), + }; + assert!(verdict(clean).is_ok()); + + let over = Outcome { + checked: 3, + offenders: vec![Offender { + path: "src/big.rs".to_string(), + lines: MAX_PRODUCTION_LINES + 1, + }], + }; + let error = format!("{:#}", verdict(over).unwrap_err()); + assert!( + error.contains("src/big.rs: 1501 production lines (1 over)"), + "{error}" + ); + } + + fn repo_file(relative: &str) -> String { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + fs::read_to_string(root.join(relative)).unwrap() + } + + // The failure text gives remediation advice, so this asserts the advice + // together with the rule it points at: the document exists and states it. + #[test] + fn the_failure_text_points_at_a_split_rule_the_document_states() { + let text = failed_text(&Outcome { + checked: 1, + offenders: vec![Offender { + path: "src/big.rs".to_string(), + lines: MAX_PRODUCTION_LINES + 1, + }], + }); + assert!(text.contains(SPLIT_RULE_DOC), "{text}"); + assert!(text.contains("same PR"), "{text}"); + assert!(text.contains("pure move"), "{text}"); + assert!(text.contains("tests moving along with the code"), "{text}"); + + let doc = repo_file(SPLIT_RULE_DOC); + assert!( + doc.contains("splits it in the same change"), + "{SPLIT_RULE_DOC}" + ); + assert!(doc.contains("a pure move"), "{SPLIT_RULE_DOC}"); + assert!( + doc.contains("the tests moving with the code"), + "{SPLIT_RULE_DOC}" + ); + } + + // The docs say the check blocks merges, and the hook comment calls itself the + // local mirror of this step. Both rest on this step being the bare strict command. + #[test] + fn the_ci_step_runs_the_strict_check_with_no_non_blocking_wrapper() { + let workflow = repo_file(".github/workflows/ci.yml"); + let start = workflow + .find("- name: Source files are within the module-size limit") + .expect("the module-size step is in ci.yml"); + let step = &workflow[start..]; + let step = &step[..step[1..] + .find("\n - name:") + .map_or(step.len(), |n| n + 1)]; + assert!( + step.lines() + .any(|line| line.trim() == "run: cargo xtask check-module-size"), + "{step}" + ); + assert!(!step.contains("continue-on-error"), "{step}"); + } + #[test] fn offenders_are_listed_largest_first() { let files = [ @@ -1447,47 +1513,4 @@ trait T { assert!(error.contains("src/generated.rs"), "{error}"); assert!(error.contains("src/hidden.rs"), "{error}"); } - - // --- the advisory wrapper ---------------------------------------------- - - #[test] - fn the_advisory_wrapper_reports_offenders_and_still_succeeds() { - let outcome = Outcome { - checked: 3, - offenders: vec![Offender { - path: "src/big.rs".to_string(), - lines: 1600, - }], - }; - let mut out = Vec::new(); - advisory_to(Ok(outcome), &mut out); - let text = String::from_utf8(out).unwrap(); - assert!(text.contains("src/big.rs: 1600 production lines"), "{text}"); - assert!(text.contains("not blocking yet"), "{text}"); - } - - #[test] - fn the_advisory_wrapper_succeeds_even_when_the_check_cannot_run() { - let mut out = Vec::new(); - advisory_to(Err(anyhow!("cargo metadata failed")), &mut out); - let text = String::from_utf8(out).unwrap(); - assert!(text.contains("could not run"), "{text}"); - assert!(text.contains("cargo metadata failed"), "{text}"); - } - - #[test] - fn the_advisory_wrapper_succeeds_even_when_the_report_cannot_be_written() { - struct Closed; - impl Write for Closed { - fn write(&mut self, _: &[u8]) -> std::io::Result { - Err(std::io::ErrorKind::BrokenPipe.into()) - } - fn flush(&mut self) -> std::io::Result<()> { - Err(std::io::ErrorKind::BrokenPipe.into()) - } - } - // `advisory_to` returns nothing, so the test is that it returns at all - // instead of panicking; `run_advisory` is `Ok(())` around it. - advisory_to(Err(anyhow!("cargo metadata failed")), &mut Closed); - } } diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 403bb4f3e5..9543c92115 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -31,18 +31,27 @@ //! manual run, it diffs `origin/main...HEAD` like `cargo xtask affected` does. //! `--base` overrides the from-ref. //! +//! A step that already passed on this exact tree is skipped, and a green run is +//! recorded: see [`crate::prepush_stamp`] for what a pass is keyed on (the +//! step's cargo arguments, so a narrow pass never covers a wider run; the tree +//! hash of `HEAD`; the toolchain; the environment) and why a dirty working tree +//! neither uses nor writes one. +//! //! As in [`crate::affected`], the decisions ([`push_from_env`], [`plan`], //! [`hook_selection`], [`resolve_selection`]) are pure and unit-tested; running -//! git and cargo is confined to [`run`], and [`execute`] takes both as -//! parameters so its own behavior (the range it diffs, `--dry-run`, a failing -//! cargo) is tested too. +//! git and cargo is confined to [`run`], and [`execute`] takes both, the stamps +//! and the output as parameters so its own behavior (the range it diffs, +//! `--dry-run`, a failing cargo, a skipped or recorded pass, and what it prints +//! about each) is tested too. use std::collections::BTreeSet; +use std::io::Write; use std::process::Command; use anyhow::{Context, Result, bail}; use crate::affected::{self, DEFAULT_BASE, Graph, Selection}; +use crate::prepush_stamp::{self, Lookup, Stamps}; /// The e2e harness crate. `cargo test` excludes it (its scenarios run through /// `cargo xtask e2e`), and its harness target is linted by its own step. @@ -357,39 +366,94 @@ impl Env { } } +/// What one invocation asks for. +#[derive(Debug, Clone, Default)] +pub struct Request { + /// Replaces the range's base (`--base`). + pub base: Option, + /// Print the plan without running or recording anything (`--dry-run`). + pub dry_run: bool, + /// The refs prek exported. + pub env: Env, +} + /// Run one step: resolve the range (an explicit `base` wins over prek's /// from-ref), pick the affected crates, print the scope, and hand the planned -/// cargo arguments to `run_cargo` unless the step is skipped or `dry_run`. +/// cargo arguments to `run_cargo` unless the step is skipped, `dry_run`, or a +/// pass of exactly this command on this tree is on record (`stamps`). /// -/// `load_graph`, `changed_files` (given the range's base and tip) and -/// `run_cargo` are parameters so the behavior around them is unit-tested. +/// A pass is recorded only after `run_cargo` succeeded. `load_graph`, +/// `changed_files` (given the range's base and tip), `run_cargo` and `out` are +/// parameters so the behavior around them is unit-tested. pub fn execute( check: Check, - base: Option, - dry_run: bool, - env: Env, + request: Request, + stamps: &Stamps<'_>, load_graph: impl FnOnce() -> Result, changed_files: impl FnOnce(&str, &str) -> Result>, mut run_cargo: impl FnMut(&[String]) -> Result<()>, + out: &mut impl Write, ) -> Result<()> { + let label = check.label(); + let Request { base, dry_run, env } = request; let push = push_from_env(base.or(env.from_ref), env.to_ref); let (selection, fallback) = resolve_selection(load_graph, || changed_files(push.base(), &push.to)); - println!("{}", banner(check, &push, &selection, fallback.as_deref())); + writeln!( + out, + "{}", + banner(check, &push, &selection, fallback.as_deref()) + )?; let args = match plan(check, &selection) { Plan::Skip(reason) => { - println!("pre-push-check {}: skipped, {reason}", check.label()); + writeln!(out, "pre-push-check {label}: skipped, {reason}")?; return Ok(()); } Plan::Cargo(args) => args, }; - println!("pre-push-check {}: cargo {}", check.label(), args.join(" ")); + writeln!(out, "pre-push-check {label}: cargo {}", args.join(" "))?; + let key = match stamps.lookup(label, &args) { + Lookup::Hit { name, age_secs } => { + if dry_run { + writeln!( + out, + "pre-push-check {label}: a real run would skip this step, it passed {} ago", + prepush_stamp::age_text(age_secs) + )?; + } else { + let location = stamps.location(); + for line in prepush_stamp::skip_lines(label, &args, age_secs, &name, &location) { + writeln!(out, "{line}")?; + } + } + return Ok(()); + } + Lookup::Miss { key, note } => { + if let Some(note) = note { + writeln!(out, "pre-push-check {label}: {note}")?; + } + key + } + }; if dry_run { return Ok(()); } - run_cargo(&args) + + // `--rerun` replaces the recorded pass: drop it first, so a run that fails + // leaves nothing for the next ordinary run to skip on, and write a fresh one + // only once cargo succeeded. + if stamps.rerun + && let Some(note) = key.as_ref().and_then(|key| stamps.invalidate(key)) + { + writeln!(out, "pre-push-check {label}: {note}")?; + } + run_cargo(&args)?; + if let Some(note) = key.and_then(|key| stamps.record(&key)) { + writeln!(out, "pre-push-check {label}: {note}")?; + } + Ok(()) } /// Run `cargo` with `args`, failing when it does. @@ -405,16 +469,91 @@ fn spawn_cargo(args: &[String]) -> Result<()> { Ok(()) } +/// A process environment variable as text. A value that is not valid Unicode is +/// still a set value (`std::env::var` reports it as an error, which `.ok()` would +/// read as unset, so a non-Unicode `ROCM_PRE_PUSH_RERUN` would be ignored). +fn process_env(name: &str) -> Option { + std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) +} + +/// What the `pre-push-check` command line asked for. +pub struct Flags { + pub base: Option, + pub dry_run: bool, + pub rerun: bool, +} + +/// The world one invocation runs against: where passes are kept, how the tree +/// and toolchain are probed, the clock, and how a process variable is read. +pub struct World<'a> { + pub probe: &'a dyn prepush_stamp::Probe, + pub store: &'a dyn prepush_stamp::Store, + /// Seconds since the epoch. + pub now: u64, + pub env: &'a dyn Fn(&str) -> Option, +} + /// Entry point for the `pre-push-check` subcommand. -pub fn run(check: Check, base: Option, dry_run: bool) -> Result<()> { - execute( +pub fn run(check: Check, base: Option, dry_run: bool, rerun: bool) -> Result<()> { + let store: Box = match prepush_stamp::FsStore::in_git_common_dir(None) + { + Ok(store) => Box::new(store), + Err(error) => Box::new(prepush_stamp::UnavailableStore(format!("{error:#}"))), + }; + let probe = prepush_stamp::SystemProbe::default(); + run_in( check, - base, - dry_run, - Env::from_lookup(|name| std::env::var(name).ok()), + Flags { + base, + dry_run, + rerun, + }, + &World { + probe: &probe, + store: store.as_ref(), + now: prepush_stamp::now_secs(), + env: &process_env, + }, affected::load_graph, affected::changed_files_between, spawn_cargo, + &mut std::io::stdout().lock(), + ) +} + +/// [`run`] with the world injected: this is where `--rerun` and +/// `ROCM_PRE_PUSH_RERUN` reach the stamp logic, so a test drives it with a fake +/// store and sees whether a recorded pass is honored. +pub fn run_in( + check: Check, + flags: Flags, + world: &World<'_>, + load_graph: impl FnOnce() -> Result, + changed_files: impl FnOnce(&str, &str) -> Result>, + run_cargo: impl FnMut(&[String]) -> Result<()>, + out: &mut impl Write, +) -> Result<()> { + let Flags { + base, + dry_run, + rerun, + } = flags; + let stamps = Stamps::new(world.probe, world.store, world.now, rerun, |name| { + (world.env)(name) + }); + let request = Request { + base, + dry_run, + env: Env::from_lookup(|name| (world.env)(name)), + }; + execute( + check, + request, + &stamps, + load_graph, + changed_files, + run_cargo, + out, ) } @@ -423,6 +562,10 @@ mod tests { use std::collections::{BTreeMap, BTreeSet}; use super::*; + use crate::prepush_stamp::fakes::{FakeProbe, MemStore}; + + /// A fixed clock for the stamp tests. + const NOW: u64 = 1_800_000_000; fn packages(names: &[&str]) -> Selection { Selection::Packages(names.iter().map(|name| (*name).to_string()).collect()) @@ -697,7 +840,7 @@ mod tests { "the `windows-debug` job no longer tests the whole workspace; update the banner text" ); assert!( - workflow.contains("\"${{ github.event_name }}\" = \"pull_request\"") + workflow.contains("\"${GITHUB_EVENT_NAME}\" = \"pull_request\"") && workflow.contains("sel=\"--workspace\""), "the `test` job no longer forces `--workspace` outside pull requests; update the banner text" ); @@ -836,24 +979,32 @@ mod tests { } } - /// Run a step against canned inputs; `files` of `None` makes the diff fail. - /// Returns the result, every `(base, tip)` the diff was asked for, and the - /// argument lines handed to cargo. - fn run_step( + /// Everything one `execute` call did. + struct StepRun { + result: Result<()>, + ranges: Ranges, + /// The argument lines handed to cargo. + cargo: Vec, + /// What the step printed. + out: String, + } + + /// Run a step against canned inputs, with the given stamps; `files` of + /// `None` makes the diff fail. + fn run_step_with( check: Check, - base: Option<&str>, - dry_run: bool, - env: Env, + request: Request, + stamps: &Stamps<'_>, files: Option<&[&str]>, cargo_fails: bool, - ) -> (Result<()>, Ranges, Vec) { + ) -> StepRun { let ranges = std::cell::RefCell::new(Vec::new()); let cargo = std::cell::RefCell::new(Vec::new()); + let mut out = Vec::new(); let result = execute( check, - base.map(String::from), - dry_run, - env, + request, + stamps, || Ok(graph()), |base, tip| { ranges @@ -871,8 +1022,44 @@ mod tests { } Ok(()) }, + &mut out, ); - (result, ranges.into_inner(), cargo.into_inner()) + StepRun { + result, + ranges: ranges.into_inner(), + cargo: cargo.into_inner(), + out: String::from_utf8(out).expect("the step prints UTF-8"), + } + } + + /// Run a step with stamps that never apply (the working tree reads as dirty), + /// so these tests see only the range, `--dry-run` and cargo's result. + /// Returns the result, every `(base, tip)` the diff was asked for, and the + /// argument lines handed to cargo. + fn run_step( + check: Check, + base: Option<&str>, + dry_run: bool, + env: Env, + files: Option<&[&str]>, + cargo_fails: bool, + ) -> (Result<()>, Ranges, Vec) { + let probe = FakeProbe::clean("tree"); + *probe.tree.borrow_mut() = Err("the working tree is dirty".to_string()); + let store = MemStore::default(); + let stamps = Stamps { + probe: &probe, + store: &store, + now: NOW, + rerun: false, + }; + let request = Request { + base: base.map(String::from), + dry_run, + env, + }; + let run = run_step_with(check, request, &stamps, files, cargo_fails); + (run.result, run.ranges, run.cargo) } const XTASK_ONLY: Option<&[&str]> = Some(&["xtask/src/main.rs"]); @@ -982,6 +1169,943 @@ mod tests { assert_eq!(Env::from_lookup(|_| None), Env::default()); } + // ---- pass stamps: skip a step that already passed on this exact tree ---- + + const XTASK_TEST: &str = "test -p xtask --all-targets"; + const WIDE_TEST: &str = "test --workspace --all-targets --exclude e2e-cucumber"; + + fn stamps<'a>(probe: &'a FakeProbe, store: &'a MemStore, now: u64, rerun: bool) -> Stamps<'a> { + Stamps { + probe, + store, + now, + rerun, + } + } + + fn run_at( + stamps: &Stamps<'_>, + check: Check, + files: &[&str], + dry_run: bool, + cargo_fails: bool, + ) -> StepRun { + let request = Request { + dry_run, + ..Request::default() + }; + run_step_with(check, request, stamps, Some(files), cargo_fails) + } + + #[test] + fn a_green_run_is_recorded_and_the_next_identical_run_is_skipped_with_the_reason() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let first = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + first.result.expect("the first run"); + assert_eq!(first.cargo, [XTASK_TEST]); + assert_eq!(store.files.borrow().len(), 1, "a green run is recorded"); + + let second = run_at( + &stamps(&probe, &store, NOW + 90, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + second.result.expect("the skipped run"); + assert!( + second.cargo.is_empty(), + "the step must not run again: {:?}", + second.cargo + ); + assert!( + second + .out + .contains(&format!("pre-push-check test: skipped, this tree already passed `cargo {XTASK_TEST}` 1 min ago")), + "{}", + second.out + ); + // The skip names what it does not cover and how to force a run. + assert!( + second.out.contains("user-level cargo configuration"), + "{}", + second.out + ); + assert!(second.out.contains("--rerun"), "{}", second.out); + assert!(second.out.contains("files git ignores"), "{}", second.out); + assert!(second.out.contains("installed tools"), "{}", second.out); + assert!(second.out.contains("flaky tests"), "{}", second.out); + assert!( + second.out.contains(prepush_stamp::RERUN_VAR), + "{}", + second.out + ); + assert!(second.out.contains(""), "{}", second.out); + assert_eq!(store.files.borrow().len(), 1, "a skip records nothing"); + } + + /// A pushed range that lists no changed files means the range is wrong, not + /// that nothing changed, so on the stamped path too it runs the whole + /// workspace: it is never skipped as "no Rust crate", an earlier narrow pass + /// does not cover it, and it is skipped only by its own earlier pass. + #[test] + fn an_empty_changed_file_list_runs_the_whole_workspace_on_the_stamped_path() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let narrow = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!(narrow.cargo, [XTASK_TEST]); + + let empty = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &[], + false, + false, + ); + empty.result.expect("the empty-range run"); + assert_eq!(empty.cargo, [WIDE_TEST], "{}", empty.out); + assert!( + empty + .out + .contains("the pushed range lists no changed files"), + "{}", + empty.out + ); + assert!(!empty.out.contains("skipped"), "{}", empty.out); + assert_eq!(store.files.borrow().len(), 2, "the wide pass is recorded"); + + let again = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &[], + false, + false, + ); + assert!(again.cargo.is_empty(), "{:?}", again.cargo); + assert!( + again + .out + .contains(&format!("this tree already passed `cargo {WIDE_TEST}`")), + "{}", + again.out + ); + + // The lint step falls back the same way and is not skipped as "no crate". + let clippy = run_at( + &stamps(&probe, &store, NOW + 15, false), + Check::Clippy, + &[], + false, + false, + ); + assert_eq!( + clippy.cargo, + ["clippy --workspace --all-targets -- -D warnings"], + "{}", + clippy.out + ); + } + + #[test] + fn a_narrow_pass_never_lets_a_wider_selection_skip() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let narrow = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!(narrow.cargo, [XTASK_TEST]); + + // The same tree, but the push now also touches the lockfile: the whole + // workspace. The earlier `-p xtask` pass says nothing about that. + let wide = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &["Cargo.lock"], + false, + false, + ); + assert_eq!(wide.cargo, [WIDE_TEST], "{}", wide.out); + assert!(!wide.out.contains("skipped"), "{}", wide.out); + assert_eq!(store.files.borrow().len(), 2, "both passes are on record"); + + // And each is skipped on its own next run. + let again = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert!(again.cargo.is_empty(), "{}", again.out); + } + + #[test] + fn a_different_tree_runs_again() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ) + .result + .expect("first"); + *probe.tree.borrow_mut() = Ok("tree-2".to_string()); + let second = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!(second.cargo, [XTASK_TEST], "{}", second.out); + assert_eq!(store.files.borrow().len(), 2); + } + + #[test] + fn a_different_toolchain_or_environment_runs_again() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ) + .result + .expect("first"); + + *probe.toolchain.borrow_mut() = Ok("rustc 1.97.0\ncargo 1.97.0\n".to_string()); + let toolchain = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!( + toolchain.cargo, + [XTASK_TEST], + "a new toolchain must not reuse the pass" + ); + + *probe.env.borrow_mut() = "env-b".to_string(); + let env = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!( + env.cargo, + [XTASK_TEST], + "a new RUSTFLAGS and the like must not reuse the pass" + ); + } + + #[test] + fn a_failed_step_records_nothing_and_a_passed_sibling_still_skips_on_retry() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Clippy, + &files, + false, + false, + ) + .result + .expect("clippy passes"); + let failed = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &files, + false, + true, + ); + failed.result.expect_err("test fails"); + assert_eq!( + store.files.borrow().len(), + 1, + "only the clippy pass is on record" + ); + assert!( + store + .files + .borrow() + .keys() + .all(|name| name.starts_with("clippy-")), + "{:?}", + store.files.borrow().keys() + ); + + // The retry skips the step that passed and re-runs the one that failed. + let clippy = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Clippy, + &files, + false, + false, + ); + assert!(clippy.cargo.is_empty(), "{}", clippy.out); + let test = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &files, + false, + false, + ); + assert_eq!(test.cargo, [XTASK_TEST]); + assert_eq!(store.files.borrow().len(), 2); + } + + #[test] + fn a_dirty_working_tree_neither_skips_nor_records_and_says_so() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("clean run"); + assert_eq!(store.files.borrow().len(), 1); + + *probe.tree.borrow_mut() = + Err("the working tree has 2 uncommitted or untracked path(s)".to_string()); + let dirty = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &files, + false, + false, + ); + assert_eq!( + dirty.cargo, + [XTASK_TEST], + "a recorded pass is not honored on a dirty tree" + ); + assert!( + dirty + .out + .contains("pass stamps are not used: the working tree has 2 uncommitted"), + "{}", + dirty.out + ); + assert_eq!(*store.writes.borrow(), 1, "and no pass is recorded from it"); + } + + #[test] + fn a_tree_the_run_changed_is_not_recorded() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let stamps = stamps(&probe, &store, NOW, false); + let mut out = Vec::new(); + execute( + Check::Test, + Request::default(), + &stamps, + || Ok(graph()), + |_, _| Ok(vec!["xtask/src/main.rs".to_string()]), + |_| { + // cargo rewrote a tracked file (`Cargo.lock`, say) while it ran. + *probe.tree.borrow_mut() = + Err("the working tree has 1 uncommitted path".to_string()); + Ok(()) + }, + &mut out, + ) + .expect("the step itself passed"); + let out = String::from_utf8(out).expect("utf-8"); + assert!( + out.contains("pass not recorded: the working tree has 1 uncommitted"), + "{out}" + ); + assert!( + store.files.borrow().is_empty(), + "nothing is recorded for a tree cargo changed" + ); + } + + /// `--rerun` is used when something the key omits may have changed (user-level + /// cargo configuration, an installed tool). If that rerun fails, the earlier + /// pass must not survive to skip the next ordinary run. + #[test] + fn a_failed_rerun_does_not_leave_the_earlier_pass_to_skip_the_next_ordinary_run() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the first, passing run"); + assert_eq!(store.files.borrow().len(), 1); + + let failed = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + false, + true, + ); + assert!(failed.result.is_err(), "the rerun fails"); + assert_eq!(failed.cargo, [XTASK_TEST], "--rerun must run the step"); + assert!( + store.files.borrow().is_empty(), + "the earlier pass is dropped before the rerun and nothing replaces it" + ); + + let retry = run_at( + &stamps(&probe, &store, NOW + 200, false), + Check::Test, + &files, + false, + false, + ); + retry.result.expect("the ordinary retry"); + assert_eq!( + retry.cargo, + [XTASK_TEST], + "the ordinary retry must run the step: {}", + retry.out + ); + assert!(!retry.out.contains("skipped"), "{}", retry.out); + assert_eq!( + store.files.borrow().len(), + 1, + "the passing retry is recorded" + ); + } + + #[test] + fn a_failed_rerun_keeps_the_passes_of_other_steps_and_other_selections() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + for (check, files) in [ + (Check::Test, &["xtask/src/main.rs"][..]), + (Check::Test, &["Cargo.lock"][..]), + (Check::Clippy, &["xtask/src/main.rs"][..]), + ] { + run_at( + &stamps(&probe, &store, NOW, false), + check, + files, + false, + false, + ) + .result + .expect("a recorded pass"); + } + assert_eq!(store.files.borrow().len(), 3); + + let failed = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &["xtask/src/main.rs"], + false, + true, + ); + assert!(failed.result.is_err()); + assert_eq!( + store.files.borrow().len(), + 2, + "only the stamp the rerun replaces is dropped" + ); + // The sibling passes still skip. + for (check, files) in [ + (Check::Test, &["Cargo.lock"][..]), + (Check::Clippy, &["xtask/src/main.rs"][..]), + ] { + let again = run_at( + &stamps(&probe, &store, NOW + 200, false), + check, + files, + false, + false, + ); + assert!(again.cargo.is_empty(), "{:?}: {}", again.cargo, again.out); + } + } + + #[test] + fn a_dry_run_rerun_leaves_every_stamp_in_place() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the first run"); + let before = store.files.borrow().clone(); + + let dry = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + true, + false, + ); + dry.result.expect("the dry run"); + assert!( + dry.cargo.is_empty(), + "a dry run runs nothing: {:?}", + dry.cargo + ); + assert_eq!( + *store.files.borrow(), + before, + "a dry run neither removes nor rewrites a stamp" + ); + } + + #[test] + fn a_passing_rerun_replaces_the_pass_with_a_fresh_one() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the first run"); + run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the passing rerun"); + assert_eq!( + store.files.borrow().len(), + 1, + "one pass, replaced not added" + ); + let after = run_at( + &stamps(&probe, &store, NOW + 130, false), + Check::Test, + &files, + false, + false, + ); + assert!(after.cargo.is_empty(), "{}", after.out); + assert!( + after.out.contains( + "this tree already passed `cargo test -p xtask --all-targets` under a minute ago" + ), + "the fresh pass is the one that skips: {}", + after.out + ); + } + + #[test] + fn rerun_ignores_a_recorded_pass_and_records_a_fresh_one() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("first"); + let forced = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + false, + false, + ); + assert_eq!(forced.cargo, [XTASK_TEST], "--rerun must run the step"); + assert!( + forced.out.contains("recorded passes are ignored"), + "{}", + forced.out + ); + assert!( + forced.out.contains(prepush_stamp::RERUN_VAR), + "{}", + forced.out + ); + let recorded: Vec = store + .files + .borrow() + .values() + .map(|(text, _)| text.clone()) + .collect(); + assert_eq!(recorded.len(), 1, "the new pass replaces the old one"); + assert!( + recorded[0].contains(&format!("passed-at: {}", NOW + 100)), + "{recorded:?}" + ); + } + + #[test] + fn a_dry_run_reports_a_hit_and_never_runs_or_writes() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + let miss = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + true, + false, + ); + assert!(miss.cargo.is_empty()); + assert!(store.files.borrow().is_empty(), "a dry run records nothing"); + assert!(!miss.out.contains("would skip"), "{}", miss.out); + + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("real run"); + let writes = *store.writes.borrow(); + let hit = run_at( + &stamps(&probe, &store, NOW + 3600, false), + Check::Test, + &files, + true, + false, + ); + assert!(hit.cargo.is_empty()); + assert!( + hit.out + .contains("a real run would skip this step, it passed 1 h ago"), + "{}", + hit.out + ); + assert_eq!(*store.writes.borrow(), writes); + } + + #[test] + fn a_failed_write_is_reported_and_never_fails_the_step() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + *store.fail_writes.borrow_mut() = true; + let run = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + run.result + .expect("a stamp that cannot be written must not fail a green hook"); + assert_eq!(run.cargo, [XTASK_TEST]); + assert!( + run.out.contains( + "pass not recorded: could not write it under (read-only file system)" + ), + "{}", + run.out + ); + } + + #[test] + fn a_store_that_cannot_be_set_up_still_runs_the_step_and_says_why() { + let probe = FakeProbe::clean("tree-1"); + let store = prepush_stamp::UnavailableStore("not a git repository".to_string()); + let stamps = Stamps { + probe: &probe, + store: &store, + now: NOW, + rerun: false, + }; + let run = run_at(&stamps, Check::Test, &["xtask/src/main.rs"], false, false); + run.result.expect("the step runs"); + assert_eq!(run.cargo, [XTASK_TEST]); + assert!( + run.out + .contains("could not read the pass stamp (not a git repository)"), + "{}", + run.out + ); + assert!( + run.out + .contains("pass not recorded: could not write it under the stamp directory"), + "{}", + run.out + ); + } + + #[test] + fn a_corrupted_stamp_is_ignored_and_replaced_by_a_real_one() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("first"); + let name = store.files.borrow().keys().next().expect("a stamp").clone(); + store + .files + .borrow_mut() + .insert(name.clone(), ("not a stamp".to_string(), NOW)); + + let second = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &files, + false, + false, + ); + assert_eq!( + second.cargo, + [XTASK_TEST], + "garbage under the right name must not skip" + ); + assert!( + second + .out + .contains(&format!("ignoring the pass stamp {name}")), + "{}", + second.out + ); + let third = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &files, + false, + false, + ); + assert!( + third.cargo.is_empty(), + "the replacement is a real stamp: {}", + third.out + ); + } + + #[test] + fn a_skipped_selection_neither_looks_for_nor_writes_a_stamp() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let run = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["CHANGELOG.md"], + false, + false, + ); + assert!(run.cargo.is_empty()); + assert!( + run.out.contains("skipped, no Rust crate is affected"), + "{}", + run.out + ); + assert_eq!(*store.writes.borrow(), 0); + } + + #[test] + fn a_head_that_moved_during_the_step_is_not_recorded_and_the_note_names_head() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let stamps = stamps(&probe, &store, NOW, false); + let mut out = Vec::new(); + execute( + Check::Test, + Request::default(), + &stamps, + || Ok(graph()), + |_, _| Ok(vec!["xtask/src/main.rs".to_string()]), + |_| { + // HEAD is a different commit by the time the step has finished. + *probe.tree.borrow_mut() = Ok("tree-2".to_string()); + Ok(()) + }, + &mut out, + ) + .expect("the step itself passed"); + let out = String::from_utf8(out).expect("utf-8"); + assert!( + out.contains("pass not recorded: HEAD moved while the step ran"), + "{out}" + ); + assert!( + store.files.borrow().is_empty(), + "nothing is written for a tree that was not checked" + ); + assert_eq!(*store.writes.borrow(), 0); + } + + /// A stamp for `xtask`'s test step on `tree-1`, written by a normal run. + fn with_a_recorded_pass() -> (FakeProbe, MemStore) { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ) + .result + .expect("the first run"); + assert_eq!(store.files.borrow().len(), 1); + (probe, store) + } + + // ---- run_in: the flag and the variable reach the stamp logic ---- + + /// Drive the real wiring below `main`: `run_in` builds the stamps from the + /// flag and the environment, here over a fake store. With `recorded_pass` + /// a first plain run records a pass for the tree; the run that follows is + /// the one measured, and the result says whether it ran cargo. + fn measured_run_ran_cargo(rerun: bool, rerun_var: Option<&str>, recorded_pass: bool) -> bool { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let env = |name: &str| { + if name == prepush_stamp::RERUN_VAR { + rerun_var.map(String::from) + } else { + None + } + }; + let ran = std::cell::Cell::new(false); + let run_once = |now: u64, rerun: bool| { + ran.set(false); + run_in( + Check::Test, + Flags { + base: None, + dry_run: false, + rerun, + }, + &World { + probe: &probe, + store: &store, + now, + env: &env, + }, + || Ok(graph()), + |_, _| Ok(vec!["xtask/src/main.rs".to_string()]), + |_| { + ran.set(true); + Ok(()) + }, + &mut Vec::new(), + ) + .expect("the step"); + ran.get() + }; + if recorded_pass { + // Nothing is on record yet, so this run runs cargo and records a pass. + assert!(run_once(NOW, false), "the first run runs the step"); + assert_eq!(store.files.borrow().len(), 1, "and records a pass"); + } + run_once(NOW + 60, rerun) + } + + #[test] + fn without_the_flag_or_the_variable_a_recorded_pass_skips_the_step() { + assert!(!measured_run_ran_cargo(false, None, true)); + assert!( + !measured_run_ran_cargo(false, Some("0"), true), + "0 means off" + ); + assert!( + !measured_run_ran_cargo(false, Some(""), true), + "empty means off" + ); + } + + #[test] + fn the_flag_reaches_the_stamp_logic_and_reruns_despite_a_recorded_pass() { + assert!(measured_run_ran_cargo(true, None, true)); + } + + #[test] + fn the_variable_reaches_the_stamp_logic_and_reruns_despite_a_recorded_pass() { + assert!(measured_run_ran_cargo(false, Some("1"), true)); + } + + #[test] + fn with_no_recorded_pass_the_step_runs_either_way() { + assert!(measured_run_ran_cargo(false, None, false)); + assert!(measured_run_ran_cargo(true, None, false)); + } + + #[test] + fn the_rerun_flag_forces_a_run_despite_a_matching_stamp() { + let (probe, store) = with_a_recorded_pass(); + let forced = Stamps::new(&probe, &store, NOW + 5, true, |_| None); + let run = run_at(&forced, Check::Test, &["xtask/src/main.rs"], false, false); + assert_eq!(run.cargo, [XTASK_TEST], "{}", run.out); + assert!( + run.out.contains("recorded passes are ignored"), + "{}", + run.out + ); + assert!(!run.out.contains("skipped"), "{}", run.out); + } + + #[test] + fn the_rerun_variable_forces_a_run_despite_a_matching_stamp() { + let (probe, store) = with_a_recorded_pass(); + let lookup = |name: &str| (name == prepush_stamp::RERUN_VAR).then(|| "1".to_string()); + let forced = Stamps::new(&probe, &store, NOW + 5, false, lookup); + let run = run_at(&forced, Check::Test, &["xtask/src/main.rs"], false, false); + assert_eq!(run.cargo, [XTASK_TEST], "{}", run.out); + assert!( + run.out.contains("recorded passes are ignored"), + "{}", + run.out + ); + } + + #[test] + fn without_the_flag_or_with_the_variable_off_the_same_stamp_still_skips() { + let (probe, store) = with_a_recorded_pass(); + let off = |name: &str| (name == prepush_stamp::RERUN_VAR).then(|| "0".to_string()); + for stamps in [ + Stamps::new(&probe, &store, NOW + 5, false, |_| None), + Stamps::new(&probe, &store, NOW + 5, false, off), + ] { + let run = run_at(&stamps, Check::Test, &["xtask/src/main.rs"], false, false); + assert!(run.cargo.is_empty(), "{}", run.out); + assert!( + run.out.contains("skipped, this tree already passed"), + "{}", + run.out + ); + } + } + // ---- drift guard: every cross-crate read must be covered ---- use std::path::{Path, PathBuf}; diff --git a/xtask/src/prepush_stamp.rs b/xtask/src/prepush_stamp.rs new file mode 100644 index 0000000000..46d5d0290a --- /dev/null +++ b/xtask/src/prepush_stamp.rs @@ -0,0 +1,1618 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Pass stamps for `cargo xtask pre-push-check`: remember that a cargo step +//! already passed on exactly this content, and skip it the next time. +//! +//! The pre-push hook has no memory, so a push retried after a dropped +//! connection, a manual run before the push, an amend that only rewords the +//! message, or the same tree pushed to a second remote re-ran every step on a +//! tree it had just checked. A stamp is written after a step's cargo command +//! exits 0 and is looked up before the next run of that step. +//! +//! **What a stamp stands for.** One step (`clippy`, `clippy-e2e` or `test`), the +//! exact cargo arguments it ran, the git *tree* hash of `HEAD` (not the commit +//! hash, so an amend or a re-commit with identical content still hits), the +//! compiler and cargo identity (`rustc -vV`, `cargo -vV`), and a digest of the +//! `CARGO_*`/`RUST*` environment. Because the arguments are part of the key, a +//! pass recorded for a narrow selection can never let a later, wider selection +//! skip: `-p xtask` and `--workspace` are different stamps. +//! +//! **The working tree must be clean.** Cargo checks the working tree, not the +//! pushed commit, so a stamp keyed on `HEAD`'s tree is only true when the +//! working tree *is* that tree. With uncommitted changes or untracked files +//! (an untracked `mod` target can hide a missing `git add`) a stamp is neither +//! honored nor written, and the step says why. After the run the tree is checked +//! again: if cargo changed it (a rewritten `Cargo.lock`, say), nothing is +//! recorded. +//! +//! **Fail safe.** Every error while locating the store, reading, parsing or +//! verifying a stamp means the step runs; a stamp that does not match its key +//! byte for byte, is unreadable, belongs to a different format, is dated in the +//! future or is older than [`MAX_AGE_SECS`] is ignored; a failed write is +//! reported and never fails the hook. A stamp is only written after cargo +//! succeeded. +//! +//! **Where.** Under the git *common* directory, so every linked worktree of a +//! clone shares the stamps, nothing lands in the work tree (no ignore rule, no +//! chance of committing one), and `cargo clean` does not erase them. Each write +//! prunes stamps older than [`MAX_AGE_SECS`] and keeps at most [`MAX_STAMPS`]. +//! +//! **Not covered**, and said in the skip message: user-level cargo +//! configuration (`~/.cargo/config.toml`), installed tools, files git ignores, +//! network-dependent or flaky tests. `--rerun` or [`RERUN_VAR`] ignores recorded +//! passes: it drops this step's earlier pass before running, so a rerun that +//! fails leaves nothing for the next ordinary run to skip on, and it records a +//! fresh pass when the step succeeds. `SKIP=` still skips a hook +//! outright. The `verify-commits` hook is a separate hook and runs every time. +//! +//! As elsewhere in this crate the decisions are pure and the world is injected: +//! git and the toolchain sit behind [`Probe`], the filesystem behind [`Store`], +//! and the clock is a parameter, so the rules above are unit-tested without +//! cargo, git or a disk. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, bail}; +use sha2::{Digest, Sha256}; + +use crate::affected; + +/// Setting this to anything but empty or `0` ignores recorded passes. +pub const RERUN_VAR: &str = "ROCM_PRE_PUSH_RERUN"; + +/// Directory name under the git common directory. +const STAMP_DIR: &str = "rocm-cli-prepush-passes"; +const STAMP_EXT: &str = ".stamp"; +/// Scratch files a write goes through before it is renamed into place. +const TMP_EXT: &str = ".tmp"; +const FORMAT_LINE: &str = "rocm-cli pre-push pass stamp v1"; + +/// The revision whose hash identifies the checked content. +const HEAD_TREE: &str = "HEAD^{tree}"; + +/// A pass older than this is ignored and pruned: toolchains, dependencies and +/// the registry move on, and a week is longer than any realistic retry loop. +pub const MAX_AGE_SECS: u64 = 7 * 24 * 60 * 60; + +/// At most this many stamps are kept; the oldest go first. +pub const MAX_STAMPS: usize = 64; + +/// A write killed mid-way leaves its scratch file behind until a later write +/// prunes it, once it is older than this: an older scratch file belongs to a +/// write that died, a younger one may be a concurrent writer's and is left alone. +pub const TMP_MAX_AGE_SECS: u64 = 60 * 60; + +/// A stamp dated this far ahead of the clock is treated as corrupt. +const FUTURE_SLACK_SECS: u64 = 5 * 60; + +/// Environment variable prefixes that can change what a cargo step does. +/// Over-including only costs a re-run; leaving one out could skip a step that +/// would now fail. +const ENV_PREFIXES: &[&str] = &["CARGO_", "RUST"]; + +/// Variables under those prefixes that differ between runs without changing the +/// outcome: where artifacts go, terminal styling, and what `cargo run` sets for +/// the `xtask` binary itself. +const ENV_IGNORED: &[&str] = &[ + "CARGO_TARGET_DIR", + "CARGO_MAKEFLAGS", + "CARGO_MANIFEST_DIR", + "CARGO_MANIFEST_PATH", + "CARGO_BIN_NAME", + "CARGO_CRATE_NAME", + "CARGO_PRIMARY_PACKAGE", + "RUST_BACKTRACE", + "RUST_LIB_BACKTRACE", +]; +const ENV_IGNORED_PREFIXES: &[&str] = &["CARGO_TERM_", "CARGO_PKG_", "CARGO_BIN_EXE_"]; + +fn hex_digest(text: &str) -> String { + hex_digest_bytes(text.as_bytes()) +} + +fn hex_digest_bytes(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// Whether `--rerun` or [`RERUN_VAR`] asks for recorded passes to be ignored. +pub fn rerun_requested(flag: bool, lookup: impl Fn(&str) -> Option) -> bool { + flag || lookup(RERUN_VAR).is_some_and(|value| { + let value = value.trim(); + !value.is_empty() && value != "0" + }) +} + +/// Digest of the environment variables that can change a cargo step, in a +/// stable order. Only the digest is kept, never a value. +pub fn env_digest(vars: impl IntoIterator) -> String +where + K: AsRef, + V: AsRef, +{ + // The process environment is read as `OsString`s: a variable holding bytes + // that are not valid Unicode (on Linux, any variable at all) must not abort + // an optional check, so nothing here needs a `String`. Names are matched as + // text, which is enough because every prefix and exempt name is ASCII, and + // values are hashed as the bytes they are. + let mut relevant: Vec<(Vec, Vec)> = vars + .into_iter() + .filter(|(name, _)| { + let name = name.as_ref().to_string_lossy(); + ENV_PREFIXES.iter().any(|prefix| name.starts_with(prefix)) + && !ENV_IGNORED.contains(&name.as_ref()) + && !ENV_IGNORED_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + }) + .map(|(name, value)| { + ( + name.as_ref().as_encoded_bytes().to_vec(), + value.as_ref().as_encoded_bytes().to_vec(), + ) + }) + .collect(); + relevant.sort(); + let mut bytes = Vec::new(); + for (name, value) in relevant { + bytes.extend_from_slice(&name); + bytes.push(b'='); + bytes.extend_from_slice(&value); + bytes.push(b'\n'); + } + hex_digest_bytes(&bytes) +} + +/// Everything one pass depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Key { + step: String, + args: String, + tree: String, + toolchain: String, + env: String, +} + +impl Key { + fn canonical(&self) -> String { + format!( + "step: {}\ncargo: {}\ntree: {}\ntoolchain: {}\nenv: {}\n", + self.step, self.args, self.tree, self.toolchain, self.env + ) + } + + fn file_name(&self) -> String { + let digest = hex_digest(&self.canonical()); + format!("{}-{}{STAMP_EXT}", self.step, &digest[..32]) + } + + fn stamp_text(&self, passed_at: u64) -> String { + format!( + "{FORMAT_LINE}\n{}passed-at: {passed_at}\n", + self.canonical() + ) + } +} + +/// What git and the toolchain say about the state being checked. +pub trait Probe { + /// The tree hash of `HEAD`, only when the working tree is exactly that tree. + /// `Err` carries the reason stamps cannot be trusted right now. + fn clean_tree(&self) -> std::result::Result; + /// A description of the compiler and cargo that will run the step. + fn toolchain(&self) -> std::result::Result; + /// [`env_digest`] of the current environment. + fn env_digest(&self) -> String; +} + +/// Where stamps are kept. Names are file names inside the store. +pub trait Store { + fn read(&self, name: &str) -> Result>; + fn write(&self, name: &str, contents: &str) -> Result<()>; + /// Every stamp as `(name, modified time in seconds since the epoch)`. + fn list(&self) -> Result>; + fn remove(&self, name: &str) -> Result<()>; + /// Where the stamps are, for messages. + fn location(&self) -> String; +} + +/// The outcome of looking for a recorded pass. +#[derive(Debug, PartialEq, Eq)] +pub enum Lookup { + /// A matching pass is on record: skip the step. + Hit { name: String, age_secs: u64 }, + /// Run the step. `key` is `Some` when a pass can be recorded afterwards, and + /// `note` explains anything the user should know (a dirty tree, a rejected + /// stamp, `--rerun`). + Miss { + key: Option, + note: Option, + }, +} + +/// Stamp handling for one invocation, with the world injected. +pub struct Stamps<'a> { + pub probe: &'a dyn Probe, + pub store: &'a dyn Store, + /// Seconds since the epoch. + pub now: u64, + /// Ignore recorded passes (but still record a new one). + pub rerun: bool, +} + +impl<'a> Stamps<'a> { + /// The stamps for one invocation. `rerun_flag` is `--rerun`; `lookup` reads + /// the environment for [`RERUN_VAR`], so tests need no process environment. + pub fn new( + probe: &'a dyn Probe, + store: &'a dyn Store, + now: u64, + rerun_flag: bool, + lookup: impl Fn(&str) -> Option, + ) -> Self { + Self { + probe, + store, + now, + rerun: rerun_requested(rerun_flag, lookup), + } + } +} + +impl Stamps<'_> { + fn key(&self, step: &str, args: &[String]) -> std::result::Result { + let tree = self.probe.clean_tree()?; + let toolchain = hex_digest(&self.probe.toolchain()?); + Ok(Key { + step: step.to_string(), + args: args.join(" "), + tree, + toolchain, + env: self.probe.env_digest(), + }) + } + + /// Look for a pass of `step` run with `args` on the current tree. + pub fn lookup(&self, step: &str, args: &[String]) -> Lookup { + let key = match self.key(step, args) { + Ok(key) => key, + Err(why) => { + return Lookup::Miss { + key: None, + note: Some(format!("pass stamps are not used: {why}")), + }; + } + }; + if self.rerun { + return Lookup::Miss { + key: Some(key), + note: Some(format!( + "recorded passes are ignored because --rerun or {RERUN_VAR} is set" + )), + }; + } + let name = key.file_name(); + let text = match self.store.read(&name) { + Ok(Some(text)) => text, + Ok(None) => { + return Lookup::Miss { + key: Some(key), + note: None, + }; + } + Err(error) => { + return Lookup::Miss { + key: Some(key), + note: Some(format!( + "could not read the pass stamp ({error:#}); running the step" + )), + }; + } + }; + match self.verify(&key, &text) { + Ok(Some(age_secs)) => Lookup::Hit { name, age_secs }, + Ok(None) => Lookup::Miss { + key: Some(key), + note: None, + }, + Err(why) => Lookup::Miss { + key: Some(key), + note: Some(format!("ignoring the pass stamp {name}: {why}")), + }, + } + } + + /// `Ok(Some(age))` for a valid, fresh stamp, `Ok(None)` for an expired one, + /// `Err(reason)` for one that is not what its name says. + fn verify(&self, key: &Key, text: &str) -> std::result::Result, String> { + let expected = format!("{FORMAT_LINE}\n{}", key.canonical()); + let rest = text + .strip_prefix(&expected) + .ok_or("it does not match this step, tree, toolchain and environment")?; + let passed_at: u64 = rest + .strip_prefix("passed-at: ") + .and_then(|value| value.trim().parse().ok()) + .ok_or("its time stamp is unreadable")?; + if passed_at > self.now.saturating_add(FUTURE_SLACK_SECS) { + return Err("it is dated in the future".to_string()); + } + let age = self.now.saturating_sub(passed_at); + Ok((age <= MAX_AGE_SECS).then_some(age)) + } + + /// Drop the recorded pass for exactly this key, before a rerun that will + /// replace it. A rerun exists because something the key does not cover may + /// have changed (user-level cargo configuration, an installed tool), so if it + /// fails the earlier pass must not survive to skip the next ordinary run. + /// Other steps' and other trees' stamps are different files and stay. + /// Returns a note when the stamp could not be removed. + pub fn invalidate(&self, key: &Key) -> Option { + self.store.remove(&key.file_name()).err().map(|error| { + format!( + "could not remove the earlier pass stamp ({error:#}); if this run fails, the \ + next ordinary run may still skip the step" + ) + }) + } + + /// Record that the step passed. Returns a note when nothing was recorded. + /// + /// The tree is probed again first: a stamp is only true when the working + /// tree is still the one the key was computed for, and a step that rewrote + /// it (`Cargo.lock`, a generated file) did not check `HEAD`'s tree. + pub fn record(&self, key: &Key) -> Option { + match self.probe.clean_tree() { + Ok(tree) if tree == key.tree => {} + Ok(_) => return Some("pass not recorded: HEAD moved while the step ran".to_string()), + Err(why) => return Some(format!("pass not recorded: {why}")), + } + if let Err(error) = self + .store + .write(&key.file_name(), &key.stamp_text(self.now)) + { + return Some(format!( + "pass not recorded: could not write it under {} ({error:#})", + self.store.location() + )); + } + self.prune(); + None + } + + /// Drop stamps past [`MAX_AGE_SECS`], then the oldest beyond [`MAX_STAMPS`], + /// and scratch files left by a write that died ([`TMP_MAX_AGE_SECS`]). + /// Best effort: a file that cannot be removed is only wasted space. + fn prune(&self) { + let Ok(mut entries) = self.store.list() else { + return; + }; + entries.retain(|(name, modified)| { + let age = self.now.saturating_sub(*modified); + let expired = if name.ends_with(TMP_EXT) { + age > TMP_MAX_AGE_SECS + } else { + age > MAX_AGE_SECS + }; + if expired { + let _ = self.store.remove(name); + } + !expired + }); + // A scratch file is not a pass: it does not count toward the cap. + entries.retain(|(name, _)| !name.ends_with(TMP_EXT)); + entries.sort_by_key(|(_, modified)| std::cmp::Reverse(*modified)); + for (name, _) in entries.iter().skip(MAX_STAMPS) { + let _ = self.store.remove(name); + } + } + + /// Where the stamps live, for the skip message. + pub fn location(&self) -> String { + self.store.location() + } +} + +/// `3 min`, `2 h`, `1 d`: how long ago a pass was recorded. +pub fn age_text(secs: u64) -> String { + match secs { + 0..=59 => "under a minute".to_string(), + 60..=3599 => format!("{} min", secs / 60), + 3600..=86_399 => format!("{} h", secs / 3600), + _ => format!("{} d", secs / 86_400), + } +} + +/// The lines printed when a step is skipped because its pass is on record. +pub fn skip_lines( + step: &str, + args: &[String], + age_secs: u64, + name: &str, + location: &str, +) -> Vec { + vec![ + format!( + "pre-push-check {step}: skipped, this tree already passed `cargo {}` {} ago \ + (stamp {name})", + args.join(" "), + age_text(age_secs) + ), + format!( + "pre-push-check {step}: that does not cover your user-level cargo configuration, \ + installed tools, files git ignores or flaky tests; run it again with --rerun \ + or {RERUN_VAR}=1, or delete {location}" + ), + ] +} + +// ---- the real world: git, the toolchain, the filesystem ---- + +/// A `git` command for the repository in `dir`. With an explicit directory the +/// repository-selecting variables are removed ([`affected::scrub_git_env`]): git +/// and prek export an absolute `GIT_DIR` to a pre-push hook run from a linked +/// worktree, and `git -C ` would otherwise still act on the repository +/// being pushed. With `None` the environment is kept, because in the hook that +/// inherited `GIT_DIR` correctly names the repository being pushed. +fn git_command(dir: Option<&Path>) -> Command { + let mut git = Command::new("git"); + if let Some(dir) = dir { + affected::scrub_git_env(&mut git).arg("-C").arg(dir); + } + git +} + +/// Run `git` with `args` (in `dir`, or the current directory) and return its +/// trimmed stdout. +fn git_stdout(dir: Option<&Path>, args: &[&str]) -> std::result::Result { + let output = git_command(dir) + .args(args) + .output() + .map_err(|error| format!("could not run git: {error}"))?; + if !output.status.success() { + return Err(format!( + "`git {}` failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +/// ` -vV`, or the reason it could not run. +fn version_text(tool: &std::ffi::OsStr) -> std::result::Result { + let output = Command::new(tool) + .arg("-vV") + .output() + .map_err(|error| format!("could not run {}: {error}", tool.to_string_lossy()))?; + if !output.status.success() { + return Err(format!("`{} -vV` failed", tool.to_string_lossy())); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// [`Probe`] backed by the `git` and toolchain on `PATH`, for the repository in +/// `dir` (the current directory when `None`). +#[derive(Default)] +pub struct SystemProbe { + pub dir: Option, +} + +impl Probe for SystemProbe { + fn clean_tree(&self) -> std::result::Result { + let dir = self.dir.as_deref(); + let status = git_stdout(dir, &["status", "--porcelain", "--untracked-files=normal"])?; + let dirty = status + .lines() + .filter(|line| !line.trim().is_empty()) + .count(); + if dirty > 0 { + return Err(format!( + "the working tree has {dirty} uncommitted or untracked path(s), and cargo checks \ + the working tree rather than HEAD; commit or remove them (`git status` lists them)" + )); + } + git_stdout(dir, &["rev-parse", HEAD_TREE]) + } + + fn toolchain(&self) -> std::result::Result { + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + Ok(format!( + "{}\n{}", + version_text(std::ffi::OsStr::new("rustc"))?, + version_text(&cargo)? + )) + } + + fn env_digest(&self) -> String { + env_digest(std::env::vars_os()) + } +} + +/// Stamps in a directory on disk. +pub struct FsStore { + dir: PathBuf, +} + +impl FsStore { + /// The stamp directory under the git common directory of the repository in + /// `dir` (the current directory when `None`), shared by every worktree of + /// that clone. + pub fn in_git_common_dir(dir: Option<&Path>) -> Result { + let common = git_stdout(dir, &["rev-parse", "--git-common-dir"]) + .map_err(|why| anyhow::anyhow!(why))?; + let common = PathBuf::from(common); + // git prints the path relative to where it ran when it is inside there. + let common = if common.is_absolute() { + common + } else { + let base = match dir { + Some(dir) => dir.to_path_buf(), + None => std::env::current_dir().context("could not read the current directory")?, + }; + base.join(common) + }; + Ok(Self { + dir: common.join(STAMP_DIR), + }) + } + + fn path(&self, name: &str) -> Result { + if name.is_empty() || name == "." || name == ".." || name.contains(['/', '\\']) { + bail!("refusing the stamp name {name:?}"); + } + Ok(self.dir.join(name)) + } +} + +impl Store for FsStore { + fn read(&self, name: &str) -> Result> { + match std::fs::read_to_string(self.path(name)?) { + Ok(text) => Ok(Some(text)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error).context("failed to read the stamp"), + } + } + + fn write(&self, name: &str, contents: &str) -> Result<()> { + std::fs::create_dir_all(&self.dir) + .with_context(|| format!("failed to create {}", self.dir.display()))?; + let target = self.path(name)?; + // Write beside the target and rename, so a reader never sees half a stamp. + let scratch = self + .dir + .join(format!(".{name}.{}{TMP_EXT}", std::process::id())); + let result = std::fs::write(&scratch, contents) + .context("failed to write the stamp") + .and_then(|()| { + std::fs::rename(&scratch, &target).context("failed to move the stamp into place") + }); + if result.is_err() { + // Do not leave the scratch file behind; `prune` only sees what a + // later successful write lists. + let _ = std::fs::remove_file(&scratch); + } + result + } + + fn list(&self) -> Result> { + let mut entries = Vec::new(); + let read_dir = match std::fs::read_dir(&self.dir) { + Ok(read_dir) => read_dir, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(entries), + Err(error) => return Err(error).context("failed to list the stamps"), + }; + for entry in read_dir.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if !name.ends_with(STAMP_EXT) && !name.ends_with(TMP_EXT) { + continue; + } + let modified = entry + .metadata() + .and_then(|meta| meta.modified()) + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_secs()); + entries.push((name, modified)); + } + Ok(entries) + } + + fn remove(&self, name: &str) -> Result<()> { + match std::fs::remove_file(self.path(name)?) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("failed to remove the stamp"), + } + } + + fn location(&self) -> String { + self.dir.display().to_string() + } +} + +/// A store that could not be set up: every read and write reports why, so the +/// step runs and says so. +pub struct UnavailableStore(pub String); + +impl Store for UnavailableStore { + fn read(&self, _name: &str) -> Result> { + bail!("{}", self.0) + } + fn write(&self, _name: &str, _contents: &str) -> Result<()> { + bail!("{}", self.0) + } + fn list(&self) -> Result> { + bail!("{}", self.0) + } + fn remove(&self, _name: &str) -> Result<()> { + bail!("{}", self.0) + } + fn location(&self) -> String { + "the stamp directory".to_string() + } +} + +/// Seconds since the epoch, or 0 when the clock is before it (which makes every +/// stamp look dated in the future, so none is honored). +pub fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()) +} + +#[cfg(test)] +pub(crate) mod fakes { + use std::cell::RefCell; + use std::collections::BTreeMap; + + use anyhow::{Result, bail}; + + use super::{Probe, Store}; + + /// A probe whose answers a test sets, and which counts its calls. + pub struct FakeProbe { + pub tree: RefCell>, + pub toolchain: RefCell>, + pub env: RefCell, + } + + impl FakeProbe { + pub fn clean(tree: &str) -> Self { + Self { + tree: RefCell::new(Ok(tree.to_string())), + toolchain: RefCell::new(Ok("rustc 1.96.0\ncargo 1.96.0\n".to_string())), + env: RefCell::new("env-a".to_string()), + } + } + } + + impl Probe for FakeProbe { + fn clean_tree(&self) -> std::result::Result { + self.tree.borrow().clone() + } + fn toolchain(&self) -> std::result::Result { + self.toolchain.borrow().clone() + } + fn env_digest(&self) -> String { + self.env.borrow().clone() + } + } + + /// An in-memory store; `fail_writes` makes every write error. + #[derive(Default)] + pub struct MemStore { + pub files: RefCell>, + pub fail_writes: RefCell, + pub writes: RefCell, + } + + impl Store for MemStore { + fn read(&self, name: &str) -> Result> { + Ok(self.files.borrow().get(name).map(|(text, _)| text.clone())) + } + fn write(&self, name: &str, contents: &str) -> Result<()> { + if *self.fail_writes.borrow() { + bail!("read-only file system"); + } + *self.writes.borrow_mut() += 1; + // The mtime is the stamp's own `passed-at`, as on a real disk. + let modified = contents + .lines() + .find_map(|line| line.strip_prefix("passed-at: ")) + .and_then(|value| value.parse().ok()) + .unwrap_or(0); + self.files + .borrow_mut() + .insert(name.to_string(), (contents.to_string(), modified)); + Ok(()) + } + fn list(&self) -> Result> { + Ok(self + .files + .borrow() + .iter() + .map(|(name, (_, modified))| (name.clone(), *modified)) + .collect()) + } + fn remove(&self, name: &str) -> Result<()> { + self.files.borrow_mut().remove(name); + Ok(()) + } + fn location(&self) -> String { + "".to_string() + } + } +} + +#[cfg(test)] +mod tests { + use super::fakes::{FakeProbe, MemStore}; + use super::*; + + const NOW: u64 = 1_800_000_000; + + fn args(text: &str) -> Vec { + text.split_whitespace().map(String::from).collect() + } + + fn stamps<'a>(probe: &'a FakeProbe, store: &'a MemStore, now: u64) -> Stamps<'a> { + Stamps { + probe, + store, + now, + rerun: false, + } + } + + /// Record a pass of `test -p xtask` and return the stamp's file name. + fn record_pass(probe: &FakeProbe, store: &MemStore, now: u64) -> String { + let stamps = stamps(probe, store, now); + let Lookup::Miss { key: Some(key), .. } = stamps.lookup("test", &args("test -p xtask")) + else { + panic!("expected a miss with a key"); + }; + assert_eq!(stamps.record(&key), None); + key.file_name() + } + + // ---- the key ---- + + fn base_key() -> Key { + Key { + step: "test".to_string(), + args: "test -p xtask".to_string(), + tree: "tree".to_string(), + toolchain: "toolchain".to_string(), + env: "env".to_string(), + } + } + + #[test] + fn every_part_of_the_key_changes_the_stamp_name() { + let base = base_key(); + let variants = [ + Key { + step: "clippy".into(), + ..base.clone() + }, + Key { + args: "test --workspace".into(), + ..base.clone() + }, + Key { + tree: "other".into(), + ..base.clone() + }, + Key { + toolchain: "other".into(), + ..base.clone() + }, + Key { + env: "other".into(), + ..base.clone() + }, + ]; + for variant in &variants { + assert_ne!(variant.file_name(), base.file_name(), "{variant:?}"); + } + assert_eq!( + base.file_name(), + base_key().file_name(), + "the name is stable" + ); + let name = base.file_name(); + assert!( + name.starts_with("test-") && name.strip_suffix(".stamp").is_some(), + "{name}" + ); + } + + // ---- age: expiry and a clock that disagrees ---- + + #[test] + fn a_pass_is_honored_up_to_the_maximum_age_and_not_a_second_longer() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + record_pass(&probe, &store, NOW); + let at = |now| stamps(&probe, &store, now).lookup("test", &args("test -p xtask")); + assert!(matches!(at(NOW + MAX_AGE_SECS), Lookup::Hit { .. })); + // Expired: a quiet miss that is allowed to record a fresh pass. + match at(NOW + MAX_AGE_SECS + 1) { + Lookup::Miss { key, note } => { + assert!(key.is_some()); + assert_eq!(note, None); + } + Lookup::Hit { .. } => panic!("an expired pass must not skip the step"), + } + } + + #[test] + fn a_stamp_dated_in_the_future_is_ignored_and_says_so() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + let name = record_pass(&probe, &store, NOW + 10 * 60); + match stamps(&probe, &store, NOW).lookup("test", &args("test -p xtask")) { + Lookup::Miss { note, .. } => { + let note = note.expect("a note"); + assert!( + note.contains(&name) && note.contains("dated in the future"), + "{note}" + ); + } + Lookup::Hit { .. } => panic!("a future-dated stamp must not skip the step"), + } + // A little clock skew is tolerated. + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + record_pass(&probe, &store, NOW + 60); + assert!(matches!( + stamps(&probe, &store, NOW).lookup("test", &args("test -p xtask")), + Lookup::Hit { .. } + )); + } + + #[test] + fn a_stamp_from_another_format_or_with_a_bad_time_is_ignored() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + let name = record_pass(&probe, &store, NOW); + let good = store.files.borrow()[&name].0.clone(); + for (broken, why) in [ + ( + good.replace("pass stamp v1", "pass stamp v2"), + "does not match", + ), + (good.replace("tree: t", "tree: u"), "does not match"), + ( + good.replace(&format!("passed-at: {NOW}"), "passed-at: soon"), + "time stamp is unreadable", + ), + ( + good.replace(&format!("passed-at: {NOW}\n"), ""), + "time stamp is unreadable", + ), + ] { + store.files.borrow_mut().insert(name.clone(), (broken, NOW)); + match stamps(&probe, &store, NOW + 1).lookup("test", &args("test -p xtask")) { + Lookup::Miss { note, .. } => { + let note = note.expect("a note"); + assert!(note.contains(why), "{note}"); + } + Lookup::Hit { .. } => panic!("a broken stamp must not skip the step: {why}"), + } + } + } + + // ---- expiry and the size cap ---- + + #[test] + fn recording_prunes_expired_stamps_and_keeps_the_newest_up_to_the_cap() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + { + let mut files = store.files.borrow_mut(); + files.insert( + "old-a.stamp".into(), + (String::new(), NOW - MAX_AGE_SECS - 10), + ); + files.insert( + "old-b.stamp".into(), + (String::new(), NOW - MAX_AGE_SECS - 20), + ); + for index in 0..(MAX_STAMPS + 5) { + files.insert( + format!("fresh-{index:03}.stamp"), + (String::new(), NOW - 1000 + index as u64), + ); + } + } + record_pass(&probe, &store, NOW); + let files = store.files.borrow(); + assert_eq!( + files.len(), + MAX_STAMPS, + "{:?}", + files.keys().collect::>() + ); + assert!(!files.contains_key("old-a.stamp") && !files.contains_key("old-b.stamp")); + assert!( + !files.contains_key("fresh-000.stamp"), + "the oldest beyond the cap go first" + ); + assert!( + files.contains_key(&format!("fresh-{:03}.stamp", MAX_STAMPS + 4)), + "the newest stay" + ); + assert!( + files.keys().any(|name| name.starts_with("test-")), + "and so does the new pass" + ); + } + + #[test] + fn expired_stamps_are_pruned_even_when_the_store_is_far_below_the_cap() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + { + let mut files = store.files.borrow_mut(); + files.insert("old.stamp".into(), (String::new(), NOW - MAX_AGE_SECS - 1)); + files.insert("edge.stamp".into(), (String::new(), NOW - MAX_AGE_SECS)); + files.insert("fresh.stamp".into(), (String::new(), NOW - 60)); + } + record_pass(&probe, &store, NOW); + let files = store.files.borrow(); + assert!( + !files.contains_key("old.stamp"), + "{:?}", + files.keys().collect::>() + ); + assert!( + files.contains_key("edge.stamp"), + "a stamp exactly at the limit is still honored" + ); + assert!(files.contains_key("fresh.stamp")); + } + + // ---- the environment ---- + + fn vars(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect() + } + + #[test] + fn the_environment_digest_follows_cargo_and_rust_settings_only() { + let base = env_digest(vars(&[("PATH", "/bin"), ("RUSTFLAGS", "-D warnings")])); + // Settings that can change a step's outcome change the digest ... + assert_ne!( + base, + env_digest(vars(&[("RUSTFLAGS", "-C target-cpu=native")])) + ); + assert_ne!( + base, + env_digest(vars(&[ + ("RUSTFLAGS", "-D warnings"), + ("CARGO_BUILD_TARGET", "x") + ])) + ); + // ... unrelated or per-run variables do not, whatever their order. + assert_eq!( + base, + env_digest(vars(&[ + ("HOME", "/home/a"), + ("CARGO_TARGET_DIR", "/elsewhere"), + ("CARGO_TERM_COLOR", "always"), + ("CARGO_PKG_NAME", "xtask"), + ("CARGO_MANIFEST_DIR", "/repo/xtask"), + ("RUST_BACKTRACE", "1"), + ("RUSTFLAGS", "-D warnings"), + ])) + ); + assert_eq!( + env_digest(vars(&[("RUSTFLAGS", "a"), ("CARGO_INCREMENTAL", "0")])), + env_digest(vars(&[("CARGO_INCREMENTAL", "0"), ("RUSTFLAGS", "a")])), + ); + } + + #[test] + fn no_environment_value_ends_up_in_a_stamp() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + *probe.env.borrow_mut() = env_digest(vars(&[("CARGO_REGISTRY_TOKEN", "hunter2-secret")])); + let name = record_pass(&probe, &store, NOW); + let text = store.files.borrow()[&name].0.clone(); + assert!( + !text.contains("hunter2") && !text.contains("CARGO_REGISTRY_TOKEN"), + "{text}" + ); + } + + #[test] + fn rerun_comes_from_the_flag_or_a_non_empty_non_zero_variable() { + let var = |value: Option<&'static str>| { + move |name: &str| { + (name == RERUN_VAR) + .then(|| value.map(String::from)) + .flatten() + } + }; + assert!(rerun_requested(true, var(None))); + assert!(rerun_requested(false, var(Some("1")))); + assert!(rerun_requested(false, var(Some("yes")))); + assert!(!rerun_requested(false, var(Some("0")))); + assert!(!rerun_requested(false, var(Some("")))); + assert!(!rerun_requested(false, var(Some(" ")))); + assert!(!rerun_requested(false, var(None))); + } + + #[test] + fn ages_read_as_minutes_hours_or_days() { + assert_eq!(age_text(5), "under a minute"); + assert_eq!(age_text(90), "1 min"); + assert_eq!(age_text(2 * 3600 + 5), "2 h"); + assert_eq!(age_text(3 * 86_400), "3 d"); + } + + // ---- the real store on a real disk ---- + + #[test] + fn the_file_store_round_trips_lists_only_stamps_and_removes_idempotently() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + assert_eq!( + store.read("a.stamp").expect("read"), + None, + "a missing stamp is not an error" + ); + assert!( + store.list().expect("list").is_empty(), + "nor is a missing directory" + ); + + store.write("a.stamp", "one").expect("write"); + store + .write("a.stamp", "two") + .expect("an overwrite replaces it"); + assert_eq!(store.read("a.stamp").expect("read").as_deref(), Some("two")); + std::fs::write(dir.path().join("passes/notes.txt"), "x").expect("a foreign file"); + + let listed = store.list().expect("list"); + assert_eq!(listed.len(), 1, "{listed:?}"); + assert_eq!(listed[0].0, "a.stamp"); + assert!( + listed[0].1 > 1_000_000_000, + "the modified time is read: {listed:?}" + ); + let leftovers = std::fs::read_dir(dir.path().join("passes")) + .expect("dir") + .count(); + assert_eq!( + leftovers, 2, + "no scratch file is left behind (the stamp and the foreign file)" + ); + + store.remove("a.stamp").expect("remove"); + store.remove("a.stamp").expect("removing it again is fine"); + assert_eq!(store.read("a.stamp").expect("read"), None); + } + + #[test] + fn the_file_store_refuses_names_that_leave_its_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + for name in ["../escape.stamp", "a/b.stamp", "a\\b.stamp", "..", ".", ""] { + // The guard itself refuses, not whatever the filesystem then does. + let refusals = [ + store.write(name, "x").unwrap_err(), + store.read(name).unwrap_err(), + store.remove(name).unwrap_err(), + ]; + for error in refusals { + assert!( + format!("{error:#}").contains("refusing the stamp name"), + "{name}: {error:#}" + ); + } + } + assert!(!dir.path().join("escape.stamp").exists()); + } + + // ---- git: the tree hash, the dirty check and the shared location ---- + + fn git_in(dir: &Path, args: &[&str]) -> String { + let output = git_command(Some(dir)) + .args([ + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git {args:?} failed in {}", + dir.display() + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + /// A repository with one commit, in a fresh directory. + fn repo() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + git_in(dir.path(), &["init", "-q"]); + std::fs::write(dir.path().join("a.txt"), "one\n").expect("write"); + git_in(dir.path(), &["add", "a.txt"]); + git_in(dir.path(), &["commit", "-q", "-m", "first"]); + dir + } + + fn probe_for(dir: &Path) -> SystemProbe { + SystemProbe { + dir: Some(dir.to_path_buf()), + } + } + + #[test] + fn a_clean_tree_is_keyed_by_its_tree_hash_which_an_amend_does_not_change() { + let repo = repo(); + let probe = probe_for(repo.path()); + let tree = probe.clean_tree().expect("a clean tree"); + assert_eq!(tree, git_in(repo.path(), &["rev-parse", HEAD_TREE])); + + let commit = git_in(repo.path(), &["rev-parse", "HEAD"]); + git_in(repo.path(), &["commit", "-q", "--amend", "-m", "reworded"]); + assert_ne!( + git_in(repo.path(), &["rev-parse", "HEAD"]), + commit, + "the commit hash changed" + ); + assert_eq!( + probe.clean_tree().expect("still clean"), + tree, + "the tree hash did not" + ); + + std::fs::write(repo.path().join("a.txt"), "two\n").expect("write"); + git_in(repo.path(), &["commit", "-q", "-am", "second"]); + assert_ne!( + probe.clean_tree().expect("clean"), + tree, + "new content is a new tree" + ); + } + + #[test] + fn uncommitted_or_untracked_files_make_the_tree_unusable_for_stamps() { + let repo = repo(); + let probe = probe_for(repo.path()); + probe.clean_tree().expect("clean to start with"); + + std::fs::write(repo.path().join("a.txt"), "changed\n").expect("write"); + let why = probe.clean_tree().expect_err("a modified tracked file"); + assert!(why.contains("1 uncommitted or untracked path"), "{why}"); + assert!( + why.contains("git status"), + "the reason says how to fix it: {why}" + ); + git_in(repo.path(), &["checkout", "--", "a.txt"]); + probe.clean_tree().expect("clean again"); + + // An untracked file can hide a missing `git add`, so it counts too. + std::fs::write(repo.path().join("new.rs"), "mod x;\n").expect("write"); + assert!(probe.clean_tree().is_err()); + } + + #[test] + fn linked_worktrees_share_one_stamp_directory_outside_every_work_tree() { + let repo = repo(); + let linked = tempfile::tempdir().expect("tempdir"); + let linked_path = linked.path().join("wt"); + git_in( + repo.path(), + &[ + "worktree", + "add", + "-q", + linked_path.to_str().expect("utf-8"), + "-b", + "side", + ], + ); + + let main = FsStore::in_git_common_dir(Some(repo.path())).expect("store"); + let side = FsStore::in_git_common_dir(Some(&linked_path)).expect("store"); + let canonical = |store: &FsStore| { + std::fs::canonicalize(store.dir.parent().expect("parent")).expect("exists") + }; + assert_eq!( + canonical(&main), + canonical(&side), + "one directory for the whole clone" + ); + assert!(main.dir.ends_with(STAMP_DIR)); + assert!(canonical(&main).ends_with(".git"), "{:?}", canonical(&main)); + + // A stamp written from one worktree is read from the other, and writing + // it does not dirty either work tree. + main.write("t.stamp", "shared").expect("write"); + assert_eq!( + side.read("t.stamp").expect("read").as_deref(), + Some("shared") + ); + probe_for(repo.path()) + .clean_tree() + .expect("the main work tree is still clean"); + probe_for(&linked_path) + .clean_tree() + .expect("so is the linked one"); + } + + #[test] + fn a_directory_git_does_not_know_has_no_store_and_no_usable_tree() { + let dir = tempfile::tempdir().expect("tempdir"); + // The temp directory is not normally inside a repository; if it were, + // git would answer and there would be nothing to assert. + if git_stdout(Some(dir.path()), &["rev-parse", "--git-dir"]).is_err() { + assert!(probe_for(dir.path()).clean_tree().is_err()); + assert!(FsStore::in_git_common_dir(Some(dir.path())).is_err()); + } + } + + #[test] + fn the_toolchain_is_identified_from_rustc_and_cargo() { + let text = SystemProbe::default() + .toolchain() + .expect("rustc and cargo run in the test environment"); + assert!(text.contains("rustc") && text.contains("cargo"), "{text}"); + } + + // ---- hermetic git: a hostile ambient GIT_DIR must not redirect any of it ---- + + const HOSTILE_CHILD_VAR: &str = "ROCM_STAMP_TEST_HOSTILE_CHILD"; + + #[test] + fn an_explicit_directory_drops_the_repository_selecting_environment() { + let with_dir = git_command(Some(Path::new("scratch"))); + let envs: Vec<_> = with_dir.get_envs().collect(); + for var in [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_COMMON_DIR", + ] { + assert!( + envs.iter() + .any(|(name, value)| name.to_str() == Some(var) && value.is_none()), + "{var} must be removed for a command pointed at a directory" + ); + } + // Without a directory the hook's own environment is kept: there GIT_DIR + // correctly names the repository being pushed. + assert_eq!(git_command(None).get_envs().count(), 0); + } + + /// Does nothing on its own. The test below re-runs this binary with a hostile + /// git environment and this variable set, and this body then drives the real + /// probe, store and test helpers against a scratch repository. + #[test] + fn hostile_environment_child() { + let Some(dir) = std::env::var_os(HOSTILE_CHILD_VAR) else { + return; + }; + let dir = PathBuf::from(dir); + assert!( + std::env::var_os("GIT_DIR").is_some(), + "the child is supposed to run under a hostile GIT_DIR" + ); + let probe = probe_for(&dir); + let tree = git_in(&dir, &["rev-parse", HEAD_TREE]); + assert_eq!( + probe.clean_tree().expect("the scratch repository's tree"), + tree + ); + + // Committing through the helper lands in the scratch repository. + std::fs::write(dir.join("b.txt"), "two\n").expect("write"); + git_in(&dir, &["add", "b.txt"]); + git_in(&dir, &["commit", "-q", "-m", "second"]); + assert_ne!(probe.clean_tree().expect("clean"), tree); + + let store = FsStore::in_git_common_dir(Some(&dir)).expect("store"); + let scratch = std::fs::canonicalize(&dir).expect("canonical"); + let stamp_parent = + std::fs::canonicalize(store.dir.parent().expect("parent")).expect("exists"); + assert!( + stamp_parent.starts_with(&scratch), + "the store must be in the scratch repository, not the outer one: {stamp_parent:?}" + ); + println!("hostile-child-ran"); + } + + #[test] + fn git_driven_behaviour_ignores_a_hostile_git_environment() { + let outer = repo(); + let inner = repo(); + let outer_git = outer.path().join(".git"); + let outer_head = git_in(outer.path(), &["rev-parse", "HEAD"]); + + let output = Command::new(std::env::current_exe().expect("this test binary")) + .args([ + "--exact", + "prepush_stamp::tests::hostile_environment_child", + "--nocapture", + "--test-threads=1", + ]) + .env(HOSTILE_CHILD_VAR, inner.path()) + .env("GIT_DIR", &outer_git) + .env("GIT_WORK_TREE", outer.path()) + .env("GIT_INDEX_FILE", outer_git.join("index")) + .env("GIT_COMMON_DIR", &outer_git) + .output() + .expect("re-run the test binary"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stdout}\n{stderr}"); + assert!( + stdout.contains("hostile-child-ran"), + "the child did not run its body: {stdout}" + ); + + // The outer repository is exactly as it was: no stray commit, nothing staged. + assert_eq!(git_in(outer.path(), &["rev-parse", "HEAD"]), outer_head); + assert_eq!(git_in(outer.path(), &["rev-list", "--count", "HEAD"]), "1"); + assert_eq!(git_in(outer.path(), &["status", "--porcelain"]), ""); + } + + // ---- non-Unicode environment variables must never abort the optional stamp ---- + + #[cfg(unix)] + fn non_unicode(prefix: &str, tail: &[u8]) -> std::ffi::OsString { + use std::os::unix::ffi::OsStringExt; + let mut bytes = prefix.as_bytes().to_vec(); + bytes.extend_from_slice(tail); + std::ffi::OsString::from_vec(bytes) + } + + /// Unicode values hash to the same digest they always did, so stamps recorded + /// before the digest read the environment as bytes stay valid. + #[test] + fn the_environment_digest_of_unicode_values_is_the_digest_of_the_text() { + assert_eq!( + env_digest(vars(&[("RUSTFLAGS", "-D warnings"), ("PATH", "/bin")])), + hex_digest("RUSTFLAGS=-D warnings\n") + ); + } + + #[cfg(unix)] + #[test] + fn an_unrelated_variable_with_non_unicode_bytes_is_ignored_and_does_not_panic() { + let base = env_digest(vars(&[("RUSTFLAGS", "-D warnings")])); + let with_noise = env_digest([ + ( + std::ffi::OsString::from("RUSTFLAGS"), + std::ffi::OsString::from("-D warnings"), + ), + ( + std::ffi::OsString::from("PATH"), + non_unicode("/bin", &[0xff]), + ), + (non_unicode("HOME_", &[0xfe]), std::ffi::OsString::from("x")), + ( + std::ffi::OsString::from("LANG"), + non_unicode("", &[0x80, 0x81]), + ), + ]); + assert_eq!( + with_noise, base, + "unrelated variables never reach the digest" + ); + } + + #[cfg(unix)] + #[test] + fn a_relevant_variable_with_non_unicode_bytes_is_hashed_as_the_bytes_it_is() { + let digest = |value: &[u8]| { + env_digest([( + std::ffi::OsString::from("RUSTFLAGS"), + non_unicode("", value), + )]) + }; + let absent = env_digest(Vec::<(String, String)>::new()); + assert_ne!(digest(&[0xff]), absent, "the variable counts"); + // A lossy conversion would turn both bytes into U+FFFD and collapse them. + assert_ne!( + digest(&[0xff]), + digest(&[0xfe]), + "the bytes are hashed, not a lossy text" + ); + assert_eq!(digest(&[0xff]), digest(&[0xff]), "and the digest is stable"); + + // The same holds when the invalid bytes are in the name, after a prefix. + let named = |tail: &[u8]| { + env_digest([(non_unicode("CARGO_X", tail), std::ffi::OsString::from("1"))]) + }; + assert_ne!(named(&[0xff]), absent); + assert_ne!(named(&[0xff]), named(&[0xfe])); + } + + const NON_UNICODE_CHILD_VAR: &str = "ROCM_STAMP_TEST_NON_UNICODE_CHILD"; + + /// Does nothing on its own. The test below re-runs this binary with + /// non-Unicode bytes in unrelated and in relevant variables, and this body + /// then drives the real probe: `std::env::vars()` would panic here. + #[test] + fn non_unicode_environment_child() { + if std::env::var_os(NON_UNICODE_CHILD_VAR).is_none() { + return; + } + let probe = SystemProbe::default(); + let first = Probe::env_digest(&probe); + assert_eq!(first, Probe::env_digest(&probe), "the digest is stable"); + assert_eq!(first.len(), 64, "a SHA-256 digest: {first}"); + println!("non-unicode-child-ran {first}"); + } + + #[cfg(unix)] + #[test] + fn the_real_probe_computes_its_digest_under_non_unicode_environment_variables() { + let run = |rustflags: &[u8]| { + let output = Command::new(std::env::current_exe().expect("this test binary")) + .args([ + "--exact", + "prepush_stamp::tests::non_unicode_environment_child", + "--nocapture", + "--test-threads=1", + ]) + .env(NON_UNICODE_CHILD_VAR, "1") + .env("ROCM_STAMP_TEST_UNRELATED", non_unicode("v", &[0xff, 0xfe])) + .env(non_unicode("UNRELATED_", &[0xff]), "x") + .env("RUSTFLAGS", non_unicode("", rustflags)) + .output() + .expect("re-run the test binary"); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stdout}\n{stderr}"); + stdout + .split_once("non-unicode-child-ran ") + .and_then(|(_, rest)| rest.split_whitespace().next()) + .unwrap_or_else(|| panic!("the child did not run its body: {stdout}")) + .to_string() + }; + let one = run(&[0xff]); + // A relevant non-Unicode value changes the digest, losslessly. + assert_ne!(one, run(&[0xfe])); + assert_eq!(one, run(&[0xff])); + } + + // ---- scratch files left by a write that died ---- + + #[test] + fn a_write_that_cannot_be_moved_into_place_leaves_no_scratch_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + // A non-empty directory where the stamp should go makes the rename fail. + std::fs::create_dir_all(store.dir.join("a.stamp")).expect("mkdir"); + std::fs::write(store.dir.join("a.stamp").join("inner"), "x").expect("write"); + + let error = store + .write("a.stamp", "x") + .expect_err("a directory is in the way"); + assert!( + format!("{error:#}").contains("failed to move the stamp into place"), + "{error:#}" + ); + let names: Vec = std::fs::read_dir(&store.dir) + .expect("dir") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + names, + ["a.stamp"], + "no scratch file is left behind: {names:?}" + ); + } + + #[test] + fn stale_scratch_files_are_pruned_and_fresh_ones_kept_and_none_count_toward_the_cap() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + { + let mut files = store.files.borrow_mut(); + files.insert( + ".a.stamp.1.tmp".into(), + (String::new(), NOW - TMP_MAX_AGE_SECS - 1), + ); + files.insert( + ".b.stamp.2.tmp".into(), + (String::new(), NOW - TMP_MAX_AGE_SECS), + ); + files.insert(".c.stamp.3.tmp".into(), (String::new(), NOW - 60)); + for index in 0..MAX_STAMPS { + files.insert( + format!("fresh-{index:03}.stamp"), + (String::new(), NOW - 1000 + index as u64), + ); + } + } + record_pass(&probe, &store, NOW); + let files = store.files.borrow(); + let names: Vec<&String> = files.keys().collect(); + assert!( + !files.contains_key(".a.stamp.1.tmp"), + "a dead writer's scratch file goes: {names:?}" + ); + assert!( + files.contains_key(".b.stamp.2.tmp"), + "one exactly at the bound stays" + ); + assert!( + files.contains_key(".c.stamp.3.tmp"), + "a young one may be a live writer's" + ); + let stamps = files + .keys() + .filter(|name| name.ends_with(STAMP_EXT)) + .count(); + assert_eq!( + stamps, MAX_STAMPS, + "scratch files do not push stamps out: {names:?}" + ); + } + + #[test] + fn a_real_store_prunes_a_dead_writers_scratch_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + std::fs::create_dir_all(&store.dir).expect("mkdir"); + let now = now_secs(); + let age_of = |name: &str, secs: u64| { + let path = store.dir.join(name); + std::fs::write(&path, "x").expect("write"); + std::fs::File::options() + .write(true) + .open(&path) + .expect("open") + .set_modified(SystemTime::now() - std::time::Duration::from_secs(secs)) + .expect("set the modified time"); + path + }; + let dead = age_of(".a.stamp.1.tmp", 2 * 60 * 60); + let live = age_of(".b.stamp.2.tmp", 60); + + let probe = FakeProbe::clean("t"); + let stamps = Stamps::new(&probe, &store, now, false, |_| None); + let Lookup::Miss { key: Some(key), .. } = stamps.lookup("test", &args("test -p xtask")) + else { + panic!("expected a miss with a key"); + }; + assert_eq!(stamps.record(&key), None); + assert!( + !dead.exists(), + "the dead writer's scratch file is removed from disk" + ); + assert!(live.exists(), "the young one is left alone"); + } + + // ---- Stamps::new: the --rerun flag and the variable reach the lookup ---- + + #[test] + fn stamps_new_reads_the_flag_and_the_variable() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + let var = + |value: &'static str| move |name: &str| (name == RERUN_VAR).then(|| value.to_string()); + assert!(Stamps::new(&probe, &store, NOW, true, |_| None).rerun); + assert!(Stamps::new(&probe, &store, NOW, false, var("1")).rerun); + assert!(!Stamps::new(&probe, &store, NOW, false, var("0")).rerun); + assert!(!Stamps::new(&probe, &store, NOW, false, |_| None).rerun); + } +} diff --git a/xtask/src/proptest_regressions_contract.rs b/xtask/src/proptest_regressions_contract.rs new file mode 100644 index 0000000000..883d85eeea --- /dev/null +++ b/xtask/src/proptest_regressions_contract.rs @@ -0,0 +1,642 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: every saved proptest failure file must sit where +//! proptest will read it. +//! +//! proptest finds a test's saved failure cases (`cc # shrinks to ...` +//! lines) by the SOURCE FILE the `proptest!` block is written in. With the +//! default `FileFailurePersistence::SourceParallel("proptest-regressions")`, +//! a block in `/src/.rs` reads and writes +//! `/proptest-regressions/.txt`, where `src/` stands for the +//! nearest ancestor directory of the source file that holds a `main.rs` or +//! `lib.rs` (it is `src/` for every crate in this tree). Move the test to another file +//! and its seed file is no longer read: no warning, no failure, and the +//! recorded counterexample is silently never replayed again. That happened +//! twice during module splits in `apps/rocm` -- once for real +//! (`proptest-regressions/main.txt`, whose test had moved to another file) and +//! once caught in review before it merged. +//! +//! This guard walks the whole tree, finds every file under a +//! `proptest-regressions/` directory, and maps it back to the source file +//! proptest would have derived it from. A seed file is an orphan when that +//! source file does not exist, or exists but holds no `proptest!` block. A +//! block is the name, the `!` and an opening delimiter as separate tokens, so +//! `proptest ! { .. }` and `proptest /* note */ ! { .. }` count; a mention in a +//! comment or a string literal, a longer name (`my_proptest!`) and a bang that +//! opens nothing (`proptest != x`) do not. The failure names the file, says +//! which source file it expected, and says what to do about it. +//! +//! What it deliberately does not check: that the seed still reproduces +//! anything (a fixed bug leaves a harmless, still-replayed seed), or which +//! `proptest!` block in a file owns which line (proptest does not record +//! that; all blocks in a file share one seed file). It also assumes the +//! default persistence and that layout. A crate with a `src/bin//main.rs` +//! would be reported as an orphan by mistake: proptest takes `src/bin//` as +//! the directory holding `main.rs` and writes `src/bin/proptest-regressions/main.txt`, +//! which this maps to `src/bin/src/main.rs`. No crate here has such a binary +//! with a property test; avoiding the false alarm needs a directory listing +//! to find the `main.rs`, which the pure mapping does not have. Property tests +//! under a crate's `tests/` directory are not seen either: no ancestor of such a +//! file holds `main.rs` or `lib.rs`, so proptest warns and writes the seed next +//! to the test file, not under `proptest-regressions/`. A test configured with `failure_persistence: None` +//! never writes a seed file, and one with a different `FileFailurePersistence` +//! would put its seeds outside `proptest-regressions/`, where this does not +//! look. `cfg(test)` only, like the other source-scan contracts in this +//! crate: `cargo test -p xtask` runs it, and the merge queue (which runs the +//! whole workspace) is where a change confined to another crate meets it. + +#[cfg(test)] +mod tests { + use crate::rust_source::strip_literals_and_comments; + use std::path::{Path, PathBuf}; + + /// The directory name proptest's default persistence uses. + const REGRESSIONS_DIR: &str = "proptest-regressions"; + + /// The name of the macro that marks a file as holding a property test, + /// which `proptest!` spells with a `!` and a delimiter after it. + const PROPTEST_MACRO: &str = "proptest"; + + fn repo_root() -> PathBuf { + // CARGO_MANIFEST_DIR is the xtask/ crate dir; its parent is the repo + // root (same idiom as workflow_contract::repo_root). + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + /// Every file under `root`, as root-relative `/` paths, sorted. + /// + /// A whole-tree walk rather than a list of crate directories, so a crate + /// added under a new top-level directory is covered without anyone + /// remembering to extend this. `target/` and dot-directories are skipped: + /// the latter include the `.claude/worktrees` checkouts of other + /// branches, and scanning those would make the verdict depend on state + /// outside the commit under test. + fn workspace_files(root: &Path) -> Vec { + let mut found = Vec::new(); + collect_files(root, root, &mut found); + found.sort(); + found + } + + fn collect_files(root: &Path, dir: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let skip = path + .file_name() + .and_then(|name| name.to_str()) + .is_none_or(|name| name == "target" || name.starts_with('.')); + if !skip { + collect_files(root, &path, found); + } + } else { + let relative = path + .strip_prefix(root) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + found.push(relative); + } + } + } + + /// The files that live under a `proptest-regressions/` directory. + fn regression_files(all_files: &[String]) -> Vec { + all_files + .iter() + .filter(|file| file.split('/').any(|part| part == REGRESSIONS_DIR)) + .cloned() + .collect() + } + + /// The source file proptest derives `seed` from, or `None` when `seed` is + /// not shaped like one of its seed files (`/proptest-regressions/.txt`). + /// + /// The crate directory is everything before the `proptest-regressions` + /// component, which is empty for a crate at the repository root. The + /// inverse of proptest's own rule: strip the directory, swap `.txt` for + /// `.rs`, and put the result under the crate's `src/`. proptest's rule is + /// the nearest ancestor holding `main.rs` or `lib.rs`, which is `src/` for + /// every crate in this tree; see the module docs for the layouts where it + /// is not. + fn expected_source(seed: &str) -> Option { + let parts: Vec<&str> = seed.split('/').collect(); + let at = parts.iter().position(|part| *part == REGRESSIONS_DIR)?; + let (crate_dir, below) = (&parts[..at], &parts[at + 1..]); + let (last, dirs) = below.split_last()?; + let stem = last.strip_suffix(".txt").filter(|stem| !stem.is_empty())?; + let mut path: Vec = crate_dir.iter().map(|part| (*part).to_string()).collect(); + path.push("src".to_string()); + path.extend(dirs.iter().map(|part| (*part).to_string())); + path.push(format!("{stem}.rs")); + Some(path.join("/")) + } + + /// True when `source` has a `proptest!` block in code. + /// + /// A `proptest!` named in a comment or a string literal must not keep an + /// orphaned seed alive: a note about where a test moved, whether a `//` + /// line, a trailing `// ...` after code, or a `/* ... */` block (nested + /// ones included), and a test disabled by wrapping it in a block comment, are + /// not property tests. So the text is tokenized first, with the same + /// [`strip_literals_and_comments`] the other source-scan contracts use, + /// which blanks the contents of comments, string literals, raw strings and + /// char literals, and only what is left is searched. + /// + /// What is searched for is the name, the `!` and an opening delimiter as + /// separate tokens, not the text `proptest!`: Rust allows whitespace and + /// comments between them (`proptest ! { .. }`, `proptest /* note */ ! { .. }`), + /// and proptest reads the seed file of such a block all the same. The + /// stripper leaves whitespace alone and blanks a comment to spaces, so + /// skipping whitespace after the name, and again after the `!`, covers + /// both. The name must not be glued to a longer identifier (`my_proptest!` + /// is another macro), and a `!` that does not open a delimiter + /// (`proptest != x`, a variable compared with something) is not an + /// invocation. + /// + /// It still does not parse Rust: a `proptest!` inside another macro's + /// arguments counts, and so does one in a block that is compiled out by a + /// `cfg`. Nothing in this tree does either. + fn has_proptest_block(source: &str) -> bool { + let code = strip_literals_and_comments(source); + code.match_indices(PROPTEST_MACRO).any(|(at, name)| { + let glued_to_a_longer_name = code[..at] + .chars() + .next_back() + .is_some_and(|before| before.is_alphanumeric() || before == '_'); + if glued_to_a_longer_name { + return false; + } + code[at + name.len()..] + .trim_start() + .strip_prefix('!') + .is_some_and(|after_bang| after_bang.trim_start().starts_with(['{', '(', '['])) + }) + } + + /// Why a seed file is not read by any test. + #[derive(Debug, Clone, PartialEq, Eq)] + enum Orphan { + /// Not `.txt` under a `proptest-regressions` directory, so + /// proptest would never produce or read it. + NotASeedFile { seed: String }, + /// The source file proptest would read this seed for does not exist. + SourceMissing { seed: String, expected: String }, + /// The source file exists but no longer holds a `proptest!` block. + NoPropertyTest { seed: String, expected: String }, + } + + /// The seed files nothing reads. + /// + /// `read_source` returns a file's text by root-relative path, or `None` + /// when it does not exist; taking it as a parameter keeps the decision + /// pure so fixtures can stand in for the tree. + fn orphaned_seeds( + seeds: &[String], + read_source: impl Fn(&str) -> Option, + ) -> Vec { + let mut orphans = Vec::new(); + for seed in seeds { + let Some(expected) = expected_source(seed) else { + orphans.push(Orphan::NotASeedFile { seed: seed.clone() }); + continue; + }; + match read_source(&expected) { + None => orphans.push(Orphan::SourceMissing { + seed: seed.clone(), + expected, + }), + Some(text) if !has_proptest_block(&text) => orphans.push(Orphan::NoPropertyTest { + seed: seed.clone(), + expected, + }), + Some(_) => {} + } + } + orphans + } + + /// The failure text: the file, what proptest expected, the likely cause, + /// and the fix. The cause and fix are the same for every kind, so they are + /// written once. + fn describe(orphans: &[Orphan]) -> String { + let lines: Vec = orphans + .iter() + .map(|orphan| match orphan { + Orphan::NotASeedFile { seed } => format!( + " {seed}\n not shaped like a proptest seed file \ + (/{REGRESSIONS_DIR}/.txt), so proptest never reads it" + ), + Orphan::SourceMissing { seed, expected } => format!( + " {seed}\n proptest would read it for {expected}, which does not exist" + ), + Orphan::NoPropertyTest { seed, expected } => format!( + " {seed}\n proptest would read it for {expected}, which has no \ + `{PROPTEST_MACRO}` block" + ), + }) + .collect(); + format!( + "{} proptest seed file(s) that no test reads. proptest names a seed file after the \ + source file of the test (/src/.rs reads \ + /{REGRESSIONS_DIR}/.txt, with src/ the directory holding \ + main.rs or lib.rs), so the likely cause is that the property \ + test moved to another file, or the file was renamed: the saved counterexample is \ + then never replayed again, with no warning. Fix: put its `cc` lines in \ + /{REGRESSIONS_DIR}/: `git mv` the file when nothing is there yet, and append the lines to the \ + existing file when something is (a destination that already has seeds of its own \ + keeps them). Then check it is read by appending a deliberately unparsable line and \ + running the test with `--nocapture` (proptest prints `unparsable line, \ + ignoring`); remove the line again. If the test is gone, delete the seed file.\n{}", + orphans.len(), + lines.join("\n") + ) + } + + /// Orphans among the real files under `root`. + fn orphans_under(root: &Path) -> Vec { + let seeds = regression_files(&workspace_files(root)); + orphaned_seeds(&seeds, |path| std::fs::read_to_string(root.join(path)).ok()) + } + + /// The contract: every seed file is read by a test that still exists. + #[test] + fn every_proptest_regression_file_belongs_to_a_property_test_that_reads_it() { + let orphans = orphans_under(&repo_root()); + assert!(orphans.is_empty(), "{}", describe(&orphans)); + } + + /// The walk reaches the crates' sources, so an empty seed list means + /// "nothing to check" and not "the scan looked in the wrong place". + #[test] + fn the_scan_sees_the_workspace_it_is_guarding() { + let files = workspace_files(&repo_root()); + assert!( + files.iter().any(|file| file == "apps/rocm/src/main.rs"), + "the walk did not reach apps/rocm/src/main.rs -- the walk is broken, not the tree" + ); + } + + fn fixture(files: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let files: Vec<(String, String)> = files + .iter() + .map(|(path, text)| ((*path).to_string(), (*text).to_string())) + .collect(); + move |path| { + files + .iter() + .find(|(candidate, _)| candidate == path) + .map(|(_, text)| text.clone()) + } + } + + fn strings(items: &[&str]) -> Vec { + items.iter().map(|item| (*item).to_string()).collect() + } + + #[test] + fn a_seed_maps_to_the_source_file_proptest_derives_it_from() { + assert_eq!( + expected_source("apps/rocm/proptest-regressions/model_catalog_text.txt").as_deref(), + Some("apps/rocm/src/model_catalog_text.rs") + ); + assert_eq!( + expected_source("apps/rocm/proptest-regressions/providers/local_http/proptests.txt") + .as_deref(), + Some("apps/rocm/src/providers/local_http/proptests.rs") + ); + assert_eq!( + expected_source("crates/rocm-dash-tui/proptest-regressions/ui/format.txt").as_deref(), + Some("crates/rocm-dash-tui/src/ui/format.rs") + ); + // A crate at the repository root has an empty crate directory. + assert_eq!( + expected_source("proptest-regressions/a.txt").as_deref(), + Some("src/a.rs") + ); + } + + #[test] + fn a_file_that_is_not_a_txt_under_a_regressions_directory_has_no_source() { + assert_eq!(expected_source("apps/rocm/proptest-regressions/a.rs"), None); + assert_eq!(expected_source("apps/rocm/proptest-regressions/.txt"), None); + assert_eq!(expected_source("apps/rocm/proptest-regressions/"), None); + assert_eq!(expected_source("apps/rocm/src/a.txt"), None); + } + + #[test] + fn a_seed_whose_test_moved_to_another_file_is_an_orphan() { + // The shape of the real failure: the seed is still named for the file + // the test used to be in, and that file no longer has the test. + let read = fixture(&[ + ("apps/rocm/src/main.rs", "fn main() {}\n"), + ( + "apps/rocm/src/examine_report_text.rs", + "proptest! { fn format_bytes_renders_a_size_in_its_own_unit() {} }\n", + ), + ]); + let orphans = orphaned_seeds(&strings(&["apps/rocm/proptest-regressions/main.txt"]), read); + assert_eq!( + orphans, + vec![Orphan::NoPropertyTest { + seed: "apps/rocm/proptest-regressions/main.txt".to_string(), + expected: "apps/rocm/src/main.rs".to_string(), + }] + ); + } + + #[test] + fn a_seed_whose_source_file_is_gone_is_an_orphan() { + let read = fixture(&[("apps/rocm/src/other.rs", "proptest! {}\n")]); + let orphans = orphaned_seeds(&strings(&["apps/rocm/proptest-regressions/gone.txt"]), read); + assert_eq!( + orphans, + vec![Orphan::SourceMissing { + seed: "apps/rocm/proptest-regressions/gone.txt".to_string(), + expected: "apps/rocm/src/gone.rs".to_string(), + }] + ); + } + + #[test] + fn a_stray_file_in_a_regressions_directory_is_an_orphan() { + let read = fixture(&[]); + let orphans = orphaned_seeds(&strings(&["apps/rocm/proptest-regressions/notes.md"]), read); + assert_eq!( + orphans, + vec![Orphan::NotASeedFile { + seed: "apps/rocm/proptest-regressions/notes.md".to_string(), + }] + ); + } + + #[test] + fn a_seed_beside_its_property_test_is_not_an_orphan() { + let read = fixture(&[ + ( + "apps/rocm/src/model_catalog_text.rs", + "mod tests {\n proptest! {\n #[test] fn p() {}\n }\n}\n", + ), + ( + "crates/rocm-core/src/runtime.rs", + " ::proptest::proptest! { }\n", + ), + ]); + let seeds = strings(&[ + "apps/rocm/proptest-regressions/model_catalog_text.txt", + "crates/rocm-core/proptest-regressions/runtime.txt", + ]); + assert_eq!(orphaned_seeds(&seeds, read), Vec::new()); + } + + #[test] + fn a_comment_naming_proptest_does_not_keep_a_seed_alive() { + // A note about where the test moved to is not a property test. + assert!(!has_proptest_block( + "// the proptest! that was here moved to examine_report_text.rs\n/// proptest! docs\n" + )); + assert!(has_proptest_block(" proptest! {\n")); + let read = fixture(&[( + "apps/rocm/src/main.rs", + "// the proptest! moved to examine_report_text.rs\nfn main() {}\n", + )]); + let orphans = orphaned_seeds(&strings(&["apps/rocm/proptest-regressions/main.txt"]), read); + assert_eq!(orphans.len(), 1); + } + + /// The seed of `apps/rocm/src/main.rs`, whose property test moved away and + /// left only `source` behind: it is an orphan whenever `source` has no + /// `proptest!` in code. + fn orphans_when_main_holds(source: &str) -> Vec { + let files = [("apps/rocm/src/main.rs", source)]; + let seeds = strings(&["apps/rocm/proptest-regressions/main.txt"]); + orphaned_seeds(&seeds, fixture(&files)) + } + + #[test] + fn a_proptest_named_outside_code_does_not_keep_a_seed_alive() { + // Everything a note about a moved test, or a disabled one, can look like + // without being a property test. Each used to be accepted: only a line + // that STARTED with `//` was excluded. + let expected = vec![Orphan::NoPropertyTest { + seed: "apps/rocm/proptest-regressions/main.txt".to_string(), + expected: "apps/rocm/src/main.rs".to_string(), + }]; + let mut accepted = Vec::new(); + for (what, source) in [ + ( + "a trailing line comment", + "fn main() {} // proptest! moved elsewhere\n", + ), + ( + "a block comment on one line", + "fn main() {} /* proptest! moved elsewhere */\n", + ), + ( + "a block comment around the whole property block", + "/*\nproptest! {\n #[test]\n fn p() {}\n}\n*/\nfn main() {}\n", + ), + ( + "a nested block comment", + "/* outer /* inner */ proptest! { fn p() {} } still outer */\nfn main() {}\n", + ), + ( + "a block comment that opens after code", + "fn main() {} /* proptest! {\n fn p() {}\n} */\n", + ), + ( + "a string literal", + "const NOTE: &str = \"the proptest! moved elsewhere\";\n", + ), + ( + "a raw string literal", + "const NOTE: &str = r#\"the \"proptest!\" moved\"#;\n", + ), + ] { + if orphans_when_main_holds(source) != expected { + accepted.push(what); + } + } + assert!( + accepted.is_empty(), + "the guard counted a proptest! outside code as a property test: {accepted:?}" + ); + } + + #[test] + fn text_that_only_resembles_the_macro_does_not_keep_a_seed_alive() { + // The other side of reading `proptest`, `!` and a delimiter as separate + // tokens: the looser match must not accept what is not an invocation. + let expected = vec![Orphan::NoPropertyTest { + seed: "apps/rocm/proptest-regressions/main.txt".to_string(), + expected: "apps/rocm/src/main.rs".to_string(), + }]; + let mut accepted = Vec::new(); + for (what, source) in [ + ("a longer macro name", "my_proptest! { }\n"), + ("a longer macro name, spaced", "my_proptest ! { }\n"), + ("a name that continues", "proptest_cases! { }\n"), + ("the name alone", "use proptest::prelude::*;\n"), + ("a function of that name", "fn proptest() {}\n"), + ( + "a comparison", + "fn f(proptest: u8) -> bool { proptest != 3 }\n", + ), + ("a bang with no delimiter", "const X: u8 = proptest ! 3;\n"), + ("a sibling macro", "proptest::prop_compose! { }\n"), + ("a trailing name", "proptest\n"), + ] { + if orphans_when_main_holds(source) != expected { + accepted.push(what); + } + } + assert!( + accepted.is_empty(), + "the guard counted text that is not a proptest! invocation: {accepted:?}" + ); + } + + #[test] + fn a_property_block_next_to_comments_and_literals_still_counts() { + // The other direction: stripping must not hide a real block. Each is + // a source the guard has to accept, so the stricter scan cannot turn a + // seed that something reads into a false orphan. + let mut hidden = Vec::new(); + for (what, source) in [ + ("after a block comment", "/* note */ proptest! { }\n"), + ("after a line comment", "// note\nproptest! { }\n"), + ( + "after a nested block comment", + "/* a /* b */ c */ proptest! { }\n", + ), + ( + "after a string literal", + "const A: &str = \"x\"; proptest! { }\n", + ), + ( + "after a lifetime", + "fn f<'a>(_: &'a str) {} proptest! { }\n", + ), + ( + "after a char literal", + "const C: char = '\"'; proptest! { }\n", + ), + ( + "with a comment inside the block", + "proptest! {\n // a note\n #[test] fn p() {}\n}\n", + ), + ("through its path", "::proptest::proptest! { }\n"), + ("with a space before the bang", "proptest ! { }\n"), + ("with a tab before the bang", "proptest\t! { }\n"), + ("with the bang on the next line", "proptest\n ! {\n}\n"), + ( + "with a block comment before the bang", + "proptest /* note */ ! { }\n", + ), + ( + "with a nested block comment before the bang", + "proptest/* a /* b */ c */! { }\n", + ), + ( + "with a line comment before the bang", + "proptest // note\n ! { }\n", + ), + ( + "with a space before the bang, through its path", + "::proptest::proptest ! { }\n", + ), + ("with spaces around the bang", "proptest ! {\n}\n"), + ("with parentheses as its delimiter", "proptest! (\n);\n"), + ("with brackets as its delimiter", "proptest![];\n"), + ( + "with a comment between the bang and its delimiter", + "proptest! /* note */ { }\n", + ), + ] { + if !orphans_when_main_holds(source).is_empty() { + hidden.push(what); + } + } + assert!( + hidden.is_empty(), + "the guard did not see a real proptest! block: {hidden:?}" + ); + } + + #[test] + fn the_failure_names_the_file_the_likely_cause_and_the_fix() { + let text = describe(&[Orphan::NoPropertyTest { + seed: "apps/rocm/proptest-regressions/main.txt".to_string(), + expected: "apps/rocm/src/main.rs".to_string(), + }]); + assert!( + text.contains("apps/rocm/proptest-regressions/main.txt"), + "{text}" + ); + assert!(text.contains("apps/rocm/src/main.rs"), "{text}"); + assert!(text.contains("moved to another file"), "{text}"); + assert!(text.contains("git mv"), "{text}"); + // A destination that already has seeds keeps them: the advice must not + // be an unconditional `git mv`, which would replace or refuse there. + assert!(text.contains("append the lines"), "{text}"); + assert!(text.contains("unparsable line, ignoring"), "{text}"); + } + + /// The walk and the mapping together, on a tree on disk: a seed beside its + /// test passes, and the same tree with the test moved out from under it + /// fails. Also covers a nested seed path and a skipped dot-directory. + #[test] + fn a_tree_on_disk_is_orphaned_exactly_when_the_test_moves() { + let tmp = tempfile::tempdir().expect("tempdir"); + let write = |relative: &str, text: &str| { + let path = tmp.path().join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, text).expect("write"); + }; + write("apps/demo/src/main.rs", "fn main() {}\n"); + write("apps/demo/src/sizes.rs", "proptest! { }\n"); + write("apps/demo/src/nested/deep.rs", "proptest! { }\n"); + write( + "apps/demo/proptest-regressions/sizes.txt", + "cc 00 # shrinks to n = 1\n", + ); + write( + "apps/demo/proptest-regressions/nested/deep.txt", + "cc 00 # shrinks to n = 1\n", + ); + // Another checkout's worktree: skipped, so its orphan does not count. + write( + ".claude/worktrees/x/apps/demo/proptest-regressions/stale.txt", + "cc 00\n", + ); + assert_eq!(orphans_under(tmp.path()), Vec::new()); + + // The test moves out of sizes.rs; the seed stays behind. + write("apps/demo/src/sizes.rs", "pub fn size() {}\n"); + write("apps/demo/src/report.rs", "proptest! { }\n"); + assert_eq!( + orphans_under(tmp.path()), + vec![Orphan::NoPropertyTest { + seed: "apps/demo/proptest-regressions/sizes.txt".to_string(), + expected: "apps/demo/src/sizes.rs".to_string(), + }] + ); + + // Re-homing the seed to the file the test is in now clears it. + std::fs::rename( + tmp.path().join("apps/demo/proptest-regressions/sizes.txt"), + tmp.path().join("apps/demo/proptest-regressions/report.txt"), + ) + .expect("git mv stand-in"); + assert_eq!(orphans_under(tmp.path()), Vec::new()); + } +} diff --git a/xtask/src/rust_source.rs b/xtask/src/rust_source.rs new file mode 100644 index 0000000000..48afd297f7 --- /dev/null +++ b/xtask/src/rust_source.rs @@ -0,0 +1,225 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Lexical helpers shared by the source-scan contract tests. +//! +//! The contracts in this crate read Rust sources as text. Each of them has to +//! look at code only: a brace, a call or a macro name inside a string literal +//! or a comment must not count. [`strip_literals_and_comments`] blanks those +//! out while keeping line structure, so every contract that needs the question +//! answered gets the same answer. + +/// Replace the CONTENTS of string literals, char literals and comments with +/// spaces, preserving line structure and everything outside them. +/// +/// A scan that counts braces or looks for call text is wrong if a literal can +/// contribute either. Two real files in this repo break the naive version, as +/// `env_mutation_contract.rs` found: +/// +/// * `engines/lemonade/src/lib.rs` contains +/// `ensure_cached_archive("test://archive", ..., |_, destination| {`. +/// Truncating the line at `//` throws away the closure's opening brace, +/// so the enclosing test module reads as closed hundreds of lines early +/// and every mutation after it becomes invisible. +/// * `tests/e2e-cucumber/src/mock_server.rs` contains format strings such +/// as `vllm:num_requests_running{{model=...}}`. Those braces inflate the +/// depth, which drags PRODUCTION code into the scanned region and makes +/// the guard fail a build for a mutation it should not be watching. +/// +/// Both directions are silent, which is the one failure mode a permanent +/// guard must not have, so literals are tokenized rather than approximated. +pub(crate) fn strip_literals_and_comments(text: &str) -> String { + #[derive(Clone, Copy)] + enum State { + Code, + LineComment, + BlockComment(usize), + Str, + RawStr(usize), + Char, + } + + let mut out = String::with_capacity(text.len()); + let mut state = State::Code; + let chars: Vec = text.chars().collect(); + let mut i = 0; + while i < chars.len() { + let c = chars[i]; + let next = chars.get(i + 1).copied(); + match state { + State::Code => { + // A raw string opener: `r"`, `r#"`, `br##"` and so on. Checked + // before the plain-string case so the hashes are counted. + if (c == 'r' || c == 'b') + && let Some((hashes, consumed)) = raw_string_opener(&chars, i) + { + out.extend(std::iter::repeat_n(' ', consumed)); + state = State::RawStr(hashes); + i += consumed; + continue; + } + if c == '/' && next == Some('/') { + state = State::LineComment; + out.push_str(" "); + i += 2; + continue; + } + if c == '/' && next == Some('*') { + state = State::BlockComment(1); + out.push_str(" "); + i += 2; + continue; + } + if c == '"' { + state = State::Str; + out.push(' '); + i += 1; + continue; + } + // A lifetime (`'a`) is not a char literal. A char literal is + // `'x'` or `'\n'`, so require a closing quote nearby. + if c == '\'' && is_char_literal(&chars, i) { + state = State::Char; + out.push(' '); + i += 1; + continue; + } + out.push(c); + i += 1; + } + State::LineComment => { + if c == '\n' { + state = State::Code; + out.push('\n'); + } else { + out.push(' '); + } + i += 1; + } + State::BlockComment(depth) => { + if c == '*' && next == Some('/') { + state = if depth == 1 { + State::Code + } else { + State::BlockComment(depth - 1) + }; + out.push_str(" "); + i += 2; + continue; + } + if c == '/' && next == Some('*') { + state = State::BlockComment(depth + 1); + out.push_str(" "); + i += 2; + continue; + } + out.push(if c == '\n' { '\n' } else { ' ' }); + i += 1; + } + State::Str => { + if c == '\\' { + // The escaped character cannot close the literal, so + // it is consumed here. When it is a NEWLINE -- a line + // continuation, which several strings in this tree use, + // including this file's own assertion message -- the + // newline is not part of the string's value but it is + // part of the file's line structure. Replacing it with + // a space merges two source lines and shifts the line + // number of every offense reported after it, which is + // the `file:line` the assertion promises. + out.push(' '); + out.push(if next == Some('\n') { '\n' } else { ' ' }); + i += 2; + continue; + } + if c == '"' { + state = State::Code; + } + out.push(if c == '\n' { '\n' } else { ' ' }); + i += 1; + } + State::RawStr(hashes) => { + if c == '"' && closing_hashes(&chars, i + 1) >= hashes { + state = State::Code; + out.extend(std::iter::repeat_n(' ', hashes + 1)); + i += hashes + 1; + continue; + } + out.push(if c == '\n' { '\n' } else { ' ' }); + i += 1; + } + State::Char => { + if c == '\\' { + out.push_str(" "); + i += 2; + continue; + } + if c == '\'' { + state = State::Code; + } + out.push(' '); + i += 1; + } + } + } + out +} + +/// `Some((hash_count, chars_consumed))` when a raw-string literal opens at +/// `i` (`r"`, `r#"`, `br##"`, ...). +fn raw_string_opener(chars: &[char], i: usize) -> Option<(usize, usize)> { + let mut j = i; + // Accepting the byte-string `b` here is cosmetic, and deliberately has + // no fixture: removing it only means the caller declines at the `b` and + // matches one character later at the `r`, which opens the same literal + // and strips the same span. The single character of difference is the + // `b` itself, emitted as code rather than as a space. In + // `env_mutation_contract.rs`, four things read + // the stripped text — the brace counter, the names in `MUTATIONS`, + // [`serializer_column`] and [`has_test_attribute`] — and a `b` whose only + // neighbour on the right is the stripped literal's run of spaces cannot + // complete a brace, a `set_var(`, a `_TEST_LOCK`/`ScopedTestEnv`, or a + // `#[`. Equivalent on every path that reads it, so a test for it would + // assert nothing. + if chars.get(j) == Some(&'b') { + j += 1; + } + if chars.get(j) != Some(&'r') { + return None; + } + j += 1; + let hash_start = j; + while chars.get(j) == Some(&'#') { + j += 1; + } + if chars.get(j) == Some(&'"') { + Some((j - hash_start, j - i + 1)) + } else { + None + } +} + +fn closing_hashes(chars: &[char], mut i: usize) -> usize { + let start = i; + while chars.get(i) == Some(&'#') { + i += 1; + } + i - start +} + +/// Distinguish `'x'` from a lifetime such as `'a` in `&'a str`. +fn is_char_literal(chars: &[char], i: usize) -> bool { + match chars.get(i + 1) { + // `'\n'`, `'\''`, `'\\'` -- always a literal. + Some('\\') => true, + Some(_) => chars.get(i + 2) == Some(&'\''), + // A `'` as the final character of the file. Deliberately untested: + // the two answers are indistinguishable, because the caller's loop + // ends on the next step either way and the one character of output + // that differs (`'` versus a space) is neither a brace nor part of + // a mutating call's name. Flipping this arm is an equivalent + // mutant, so a fixture for it would assert nothing. + None => false, + } +} diff --git a/xtask/src/smoke.rs b/xtask/src/smoke.rs new file mode 100644 index 0000000000..d4dacf5b4f --- /dev/null +++ b/xtask/src/smoke.rs @@ -0,0 +1,2808 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Cross-platform local smoke test for the built `rocm`/`rocmd`/engine binaries. +//! +//! Runs the freshly built binaries against a throwaway state root and asserts +//! their first-run output: the traceable version string, the examine/engines +//! inventory, telemetry config, the freeform command surface, the `rocmd` +//! bridge and sandbox tools, and — the reason this gate exists — that every +//! GPU-required path fails loudly instead of silently falling back to CPU. +//! +//! Replaces the former `scripts/smoke_local.py`, preserving each of its +//! assertions, on the same commands, in the same order. Where it differs from +//! that script, the difference is deliberate: +//! +//! * The binaries are looked for with [`crate::paths::target_dir`], so a +//! configured `CARGO_TARGET_DIR` is honoured. The script always looked under +//! `/target` and so reported every binary missing when that variable +//! pointed elsewhere. The throwaway state root stays at +//! `/target/smoke-local`, as in the script, because it is wiped on every +//! run and a target directory may be shared between checkouts. +//! * `--profile release` builds release binaries, and `--target-dir` is handed +//! to the build as well as the lookup. The script always built debug, so both +//! flags built one place and then looked in another; see [`build_args`]. +//! * A child killed by a signal is reported with the negated signal number +//! (`-9`), as the script's `returncode` was; see [`exit_code_text`]. +//! * The build runs in this process's own environment, not the isolated one. +//! The script built under the isolated environment too, where the redirected +//! `HOME` hid cargo's registry and config from the build; no build script +//! reads a variable the isolation sets. The build uses the `cargo` this task +//! runs under (`$CARGO`, which `cargo xtask` always sets), where the script +//! searched `PATH` and then `~/.cargo/bin`. +//! * Each smoked command has a deadline ([`CHILD_DEADLINE`]): one still running +//! is killed and the gate fails naming it. The script waited indefinitely, so +//! a refusal that regressed into starting a server held the gate until the CI +//! job timed out. The build has no deadline, and its output streams as it +//! runs instead of being printed when it ends. +//! * The script merged each child's stderr into its stdout pipe, truly +//! interleaving them. `std::process` has no portable equivalent, so the two +//! streams are captured separately and concatenated (stdout first). No check +//! depends on the seam: each asserts a substring, reads the first line of the +//! captured output, compares whole outputs for equality (the version +//! surfaces), or requires the whole of it to be one JSON value. (On +//! Linux, each command smoked here was also measured writing to only one of +//! the two.) +//! +//! Smaller differences that change no result today: output is decoded lossily +//! and keeps Windows' `\r` until the line comparisons trim it, where the script +//! decoded strictly and translated newlines; on Windows a crashed child's status +//! prints as a signed code (`-1073741819`), where Python printed it unsigned +//! (`3221225477`); the children inherit the variables `cargo xtask` adds to the +//! environment; and the `managed_runtimes`/`managed_services` failures name the +//! check, `rocm examine`, where the script said `rocm examine first-run state`. + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::io::Read; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use serde_json::Value; + +use crate::paths::{binary_name, workspace_root}; + +/// Build profile whose binaries are smoked. +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum Profile { + Debug, + Release, +} + +impl Profile { + const fn dir_name(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Release => "release", + } + } +} + +/// One of the four binaries the smoke requires to be present. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Bin { + Rocm, + Rocmd, + Lemonade, + Vllm, +} + +impl Bin { + /// Short name used in the "missing smoke binary" failure. + const fn key(self) -> &'static str { + match self { + Self::Rocm => "rocm", + Self::Rocmd => "rocmd", + Self::Lemonade => "lemonade", + Self::Vllm => "vllm", + } + } + + /// File stem of the built artifact, before the platform `.exe` suffix. + const fn stem(self) -> &'static str { + match self { + Self::Rocm => "rocm", + Self::Rocmd => "rocmd", + Self::Lemonade => "rocm-engine-lemonade", + Self::Vllm => "rocm-engine-vllm", + } + } + + const ALL: [Self; 4] = [Self::Rocm, Self::Rocmd, Self::Lemonade, Self::Vllm]; +} + +/// Resolved paths to the binaries under test. +#[derive(Debug, PartialEq, Eq)] +struct Binaries { + dir: PathBuf, +} + +impl Binaries { + const fn new(binary_dir: PathBuf) -> Self { + Self { dir: binary_dir } + } + + fn path(&self, bin: Bin) -> PathBuf { + self.dir.join(binary_name(bin.stem())) + } + + /// Fail naming the first binary that is not a file, as the gate is + /// pointless against a partial build. + /// + /// Where the build was skipped, that is the usual cause, so the failure + /// says how to fix it: running without `--skip-build` builds into the + /// directory searched here, which + /// `the_build_lands_where_the_binaries_are_looked_for` proves. + fn require_all_present(&self, build_skipped: bool) -> Result<()> { + for bin in Bin::ALL { + let path = self.path(bin); + if !path.is_file() { + let hint = if build_skipped { + "\nnothing was built because of --skip-build: drop it to build them, or pass \ + the --profile and --target-dir of the build that produced them" + } else { + "" + }; + bail!( + "missing smoke binary {}: {} (looked in {}){hint}", + bin.key(), + path.display(), + self.dir.display() + ); + } + } + Ok(()) + } +} + +/// A JSON-shaped reply whose payload identity is asserted, rather than a +/// substring of the rendered text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum JsonCheck { + /// Must parse as JSON; nothing further is asserted (`vllm detect`). + Parses, + /// An object whose `protocol` field equals this value. + Protocol(&'static str), + /// A sandbox reply naming `tool` and reporting `ok`. `noun` is the word the + /// failure uses, which the script varied per tool. + SandboxTool { + tool: &'static str, + noun: &'static str, + }, + /// vLLM capabilities: OpenAI-compatible serving, and not CPU. + VllmCapabilities, +} + +impl JsonCheck { + /// Assert the parsed reply, naming what was wrong with it. + fn assert_value(self, value: &Value) -> Result<()> { + match self { + Self::Parses => Ok(()), + Self::Protocol(expected) => { + if json_field_is(value, "protocol", expected) { + Ok(()) + } else { + bail!("unexpected bridge snapshot protocol: {value}") + } + } + Self::SandboxTool { tool, noun } => { + if sandbox_reply_ok(value, tool) { + Ok(()) + } else { + bail!("unexpected sandbox {noun} result: {value}") + } + } + Self::VllmCapabilities => { + if vllm_capabilities_ok(value) { + Ok(()) + } else { + bail!("vllm capabilities did not report GPU-only OpenAI serving: {value}") + } + } + } + } +} + +/// A single command run plus the expectations its output must satisfy. +/// +/// Holding these as data rather than a straight line of calls is what lets the +/// table be checked by unit tests (no duplicate labels, every expectation +/// non-empty) without spawning a single process. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Check { + label: &'static str, + bin: Bin, + args: Vec<&'static str>, + expect_failure: bool, + contains: Vec<&'static str>, + contains_any: Vec<&'static str>, + not_contains: Vec<&'static str>, + json: Option, +} + +/// Token in a check's argv replaced with the ephemeral port chosen for this +/// run. The two serve rejections need a port nothing is listening on, and +/// keeping them in the table is what puts them under the order and expectation +/// pins — they are the checks the gate exists for. +const PORT_TOKEN: &str = "{port}"; + +impl Check { + fn new(label: &'static str, bin: Bin, args: &[&'static str]) -> Self { + Self { + label, + bin, + args: args.to_vec(), + expect_failure: false, + contains: Vec::new(), + contains_any: Vec::new(), + not_contains: Vec::new(), + json: None, + } + } + + /// The command is expected to exit non-zero; a success is the failure. + const fn failing(mut self) -> Self { + self.expect_failure = true; + self + } + + fn contains(mut self, needle: &'static str) -> Self { + self.contains.push(needle); + self + } + + /// At least one of `needles` must appear — for surfaces with more than one + /// accepted wording. + fn contains_any(mut self, needles: &[&'static str]) -> Self { + self.contains_any.extend_from_slice(needles); + self + } + + fn not_contains(mut self, needle: &'static str) -> Self { + self.not_contains.push(needle); + self + } + + /// The reply is parsed as JSON and asserted as a value. + const fn json(mut self, check: JsonCheck) -> Self { + self.json = Some(check); + self + } + + /// Argv with [`PORT_TOKEN`] resolved against the port chosen for this run. + fn resolved_args(&self, port: &str) -> Vec { + self.args + .iter() + .map(|arg| { + if *arg == PORT_TOKEN { + port.to_string() + } else { + (*arg).to_string() + } + }) + .collect() + } + + fn assert_output(&self, output: &str) -> Result<()> { + for needle in &self.contains { + assert_contains(output, needle, self.label)?; + } + if !self.contains_any.is_empty() { + assert_contains_any(output, &self.contains_any, self.label)?; + } + for needle in &self.not_contains { + assert_not_contains(output, needle, self.label)?; + } + if let Some(json) = self.json { + json.assert_value(&parse_json(output, self.label)?)?; + } + Ok(()) + } +} + +fn assert_contains(text: &str, needle: &str, label: &str) -> Result<()> { + if text.contains(needle) { + return Ok(()); + } + bail!("{label} did not contain expected text: {needle}\n{text}") +} + +fn assert_contains_any(text: &str, needles: &[&str], label: &str) -> Result<()> { + if needles.iter().any(|needle| text.contains(needle)) { + return Ok(()); + } + let expected = needles + .iter() + .map(|needle| format!("'{needle}'")) + .collect::>() + .join(" or "); + bail!("{label} did not contain expected text: {expected}\n{text}") +} + +fn assert_not_contains(text: &str, needle: &str, label: &str) -> Result<()> { + if text.contains(needle) { + bail!("{label} contained unexpected text: {needle}\n{text}"); + } + Ok(()) +} + +fn assert_path_missing(path: &Path, label: &str) -> Result<()> { + if path.exists() { + bail!("{label} unexpectedly exists: {}", path.display()); + } + Ok(()) +} + +/// A release tag as `apps/rocm/build.rs`'s `release_version` reads one: the +/// pattern and length are its `RELEASE_TAG_PATTERN` and +/// `RELEASE_VERSION_MAX_LEN` (+1 for the `v`), and its test +/// `release_tag_grammar_is_the_same_everywhere` fails if these differ. +const RELEASE_TAG_PATTERN: &str = r"^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.+-]+)?$"; +const RELEASE_TAG_MAX_LEN: usize = 65; + +fn is_release_tag(git_ref: &str) -> bool { + git_ref.len() <= RELEASE_TAG_MAX_LEN + && regex::Regex::new(RELEASE_TAG_PATTERN) + .expect("the release-tag pattern compiles") + .is_match(git_ref) +} + +/// The version `rocm` must report for a build from `git_ref`, by the same +/// contract as `format_cli_version` in `apps/rocm/build.rs`: an exact release +/// tag `vX.Y.Z` (see [`is_release_tag`]) is reported with that one leading `v` +/// removed, and every other resolved ref, `main` included, as `0.0.0-dev` +/// rather than the workspace package version. It is derived from the ref +/// alone, so a tag build that printed `0.0.0-dev` fails. +fn expected_version(git_ref: &str) -> &str { + if is_release_tag(git_ref) { + &git_ref[1..] + } else { + "0.0.0-dev" + } +} + +/// Accept only the traceable build string `rocm-cli (, )`, +/// reject the placeholder `unknown` ref a build with no resolvable tag/branch +/// emits, and require the version the ref calls for. +fn assert_version_string(text: &str, label: &str) -> Result<()> { + let Some((version, git_ref, _hash)) = parse_version_line(text) else { + bail!("{label} did not match 'rocm-cli (, )':\n{text}"); + }; + if git_ref == "unknown" { + // The ref comes from the build script's `git describe`/`rev-parse`, so a + // detached HEAD or a git-stripped tarball yields `unknown` even though + // nothing is wrong with the binary. Name that, or the gate reads as a + // regression during a bisect. + bail!( + "{label} has an unresolved ref ('unknown'), expected a real tag/branch. \ + A detached HEAD or a source tree without git metadata produces this; \ + build from a branch or tag:\n{text}" + ); + } + let expected = expected_version(git_ref); + if version != expected { + bail!("{label} reported {version:?} for ref {git_ref:?}; expected {expected:?}:\n{text}"); + } + Ok(()) +} + +/// Pure parser behind [`assert_version_string`], mirroring the script's +/// `rocm-cli (\S+) \((\S+), ([0-9a-f]+)\)` full match. +fn parse_version_line(text: &str) -> Option<(&str, &str, &str)> { + let rest = text.trim().strip_prefix("rocm-cli ")?; + let (version, rest) = rest.split_once(' ')?; + let inner = rest.strip_prefix('(')?.strip_suffix(')')?; + let (git_ref, hash) = inner.split_once(", ")?; + let fields_are_words = [version, git_ref] + .iter() + .all(|field| !field.is_empty() && !field.contains(char::is_whitespace)); + let hash_is_lower_hex = !hash.is_empty() + && hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)); + (fields_are_words && hash_is_lower_hex).then_some((version, git_ref, hash)) +} + +/// Python-style truthiness for a JSON field, so the sandbox/capability checks +/// accept exactly what the script accepted. +fn json_truthy(value: &Value, field: &str) -> bool { + match value.get(field) { + None | Some(Value::Null) => false, + Some(Value::Bool(flag)) => *flag, + Some(Value::Number(number)) => number.as_f64().is_some_and(|value| value != 0.0), + Some(Value::String(text)) => !text.is_empty(), + Some(Value::Array(items)) => !items.is_empty(), + Some(Value::Object(entries)) => !entries.is_empty(), + } +} + +fn json_field_is(value: &Value, field: &str, expected: &str) -> bool { + value.get(field).and_then(Value::as_str) == Some(expected) +} + +/// A sandbox tool reply is accepted only when it names the tool that was asked +/// for and reports success. +fn sandbox_reply_ok(value: &Value, tool: &str) -> bool { + // No `is_object` guard is needed: `Value::get` on any non-object is `None`, + // so a bare string or array fails both fields below. + json_field_is(value, "tool", tool) && json_truthy(value, "ok") +} + +/// vLLM must advertise OpenAI-compatible serving and must not advertise CPU. +fn vllm_capabilities_ok(value: &Value) -> bool { + json_truthy(value, "openai_compatible") && !json_truthy(value, "cpu") +} + +fn parse_json(text: &str, label: &str) -> Result { + serde_json::from_str(text) + .with_context(|| format!("{label} did not return valid JSON:\n{text}")) +} + +/// Environment overrides that point the CLI's own state directories, and the +/// platform config, data and cache directories, at `smoke_root`. +/// +/// This is the script's allowlist, not a sandbox: a `ROCM_CLI_*` variable +/// outside the four below is still inherited, so an exported one can change +/// what the gate sees. On Windows `USERPROFILE` is inherited too, as in the +/// script, so code that finds the home directory through it still sees the +/// real one. +fn isolated_env(smoke_root: &Path) -> BTreeMap<&'static str, OsString> { + let mut env = BTreeMap::new(); + env.insert("ROCM_CLI_UPDATE_USER_PATH", OsString::from("0")); + env.insert( + "ROCM_CLI_CONFIG_DIR", + smoke_root.join("rocm-config").into_os_string(), + ); + env.insert( + "ROCM_CLI_DATA_DIR", + smoke_root.join("rocm-data").into_os_string(), + ); + env.insert( + "ROCM_CLI_CACHE_DIR", + smoke_root.join("rocm-cache").into_os_string(), + ); + if cfg!(windows) { + env.insert("APPDATA", smoke_root.join("appdata").into_os_string()); + env.insert( + "LOCALAPPDATA", + smoke_root.join("localappdata").into_os_string(), + ); + } else { + env.insert( + "XDG_CONFIG_HOME", + smoke_root.join("xdg-config").into_os_string(), + ); + env.insert( + "XDG_DATA_HOME", + smoke_root.join("xdg-data").into_os_string(), + ); + env.insert( + "XDG_CACHE_HOME", + smoke_root.join("xdg-cache").into_os_string(), + ); + env.insert("HOME", smoke_root.join("home").into_os_string()); + } + env +} + +/// The cache and data directories the smoke children were pointed at. +fn state_dirs(env: &BTreeMap<&'static str, OsString>) -> (PathBuf, PathBuf) { + ( + PathBuf::from(&env["ROCM_CLI_CACHE_DIR"]), + PathBuf::from(&env["ROCM_CLI_DATA_DIR"]), + ) +} + +/// Bind an ephemeral port and hand back the number, so the serve-rejection +/// checks name a port nothing else on the host is listening on. +fn free_tcp_port() -> Result { + let listener = + TcpListener::bind(("127.0.0.1", 0)).context("failed to bind an ephemeral port")?; + Ok(listener + .local_addr() + .context("failed to read the bound port")? + .port()) +} + +/// How long one smoked command may run. Every check is a first-run status +/// read or a refusal, which take seconds; a refusal that regressed into +/// starting a server would otherwise hold the gate until the CI job's own +/// timeout, without naming the check. +const CHILD_DEADLINE: Duration = Duration::from_mins(3); + +/// How often a running child is polled against its deadline. +const CHILD_POLL: Duration = Duration::from_millis(20); + +/// Spawn one child and return its combined output, failing when the exit status +/// does not match `expect_failure`, or when the child or anything it leaves +/// holding its output is still running at `deadline`. +/// +/// Every child is handed a null stdin rather than inheriting this process's: +/// `rocm chat` without `--prompt` reads a piped stdin to EOF, so an inherited +/// pipe that the harness keeps open would hang the gate depending only on how +/// it was launched. +/// +/// On the deadline the child is killed and the failure names the check and +/// carries what it had written. A process the child started and left behind is +/// not killed (the standard library can only kill the child itself), but the +/// gate does not wait for it either. +fn run_command( + label: &str, + program: &Path, + args: &[&str], + env: &BTreeMap<&'static str, OsString>, + cwd: &Path, + expect_failure: bool, + deadline: Duration, +) -> Result { + println!("smoke: {label}"); + let started = Instant::now(); + let mut command = Command::new(program); + command + .args(args) + .current_dir(cwd) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for (key, value) in env { + command.env(key, value); + } + let mut child = command + .spawn() + .with_context(|| format!("failed to run {}", program.display()))?; + let stdout = OutputPipe::read(child.stdout.take().expect("stdout is piped")); + let stderr = OutputPipe::read(child.stderr.take().expect("stderr is piped")); + let combined = |stdout: &OutputPipe, stderr: &OutputPipe| { + let mut text = stdout.text(); + text.push_str(&stderr.text()); + text.trim().to_string() + }; + + let status = loop { + if let Some(status) = child + .try_wait() + .with_context(|| format!("failed to wait for {label}"))? + { + break status; + } + if started.elapsed() >= deadline { + // Best effort: the child may have exited between the poll and here. + let _ = child.kill(); + let _ = child.wait(); + bail!( + "{label} did not finish within {deadline:?} and was killed; it had written:\n{}", + combined(&stdout, &stderr) + ); + } + std::thread::sleep(CHILD_POLL); + }; + // The child has exited, but a process it started can still hold the pipes + // open; wait for them only until the same deadline. + while !(stdout.closed() && stderr.closed()) { + if started.elapsed() >= deadline { + bail!( + "{label} exited, but a process it started still held its output open after \ + {deadline:?}; it had written:\n{}", + combined(&stdout, &stderr) + ); + } + std::thread::sleep(CHILD_POLL); + } + let combined = combined(&stdout, &stderr); + print_output(&combined); + + if expect_failure { + if status.success() { + bail!("{label} unexpectedly succeeded"); + } + } else if !status.success() { + bail!("{label} exited with status {}", exit_code_text(&status)); + } + Ok(combined) +} + +/// Run the build with its output streamed to this process's terminal as it +/// happens, rather than captured and printed when it ends: a cold build takes +/// minutes, and a captured one shows nothing until then. It has no deadline, +/// for the same reason. +fn run_build(label: &str, program: &Path, args: &[&str], cwd: &Path) -> Result<()> { + println!("smoke: {label}"); + let status = Command::new(program) + .args(args) + .current_dir(cwd) + .stdin(Stdio::null()) + .status() + .with_context(|| format!("failed to run {}", program.display()))?; + if !status.success() { + bail!("{label} exited with status {}", exit_code_text(&status)); + } + Ok(()) +} + +fn print_output(combined: &str) { + if !combined.is_empty() { + println!("{combined}"); + } +} + +/// One of a child's output pipes, drained on its own thread so the child can +/// never block on a full pipe, and readable at any moment, including after a +/// deadline when the pipe may never close. +struct OutputPipe { + bytes: Arc>>, + closed: Arc, +} + +impl OutputPipe { + fn read(mut pipe: impl Read + Send + 'static) -> Self { + let bytes = Arc::new(Mutex::new(Vec::new())); + let closed = Arc::new(AtomicBool::new(false)); + let (sink, done) = (Arc::clone(&bytes), Arc::clone(&closed)); + // Detached on purpose: on a deadline the gate stops waiting for a pipe + // that may never close, and the thread ends with the process. + std::thread::spawn(move || { + let mut chunk = [0_u8; 8192]; + while let Ok(read @ 1..) = pipe.read(&mut chunk) { + sink.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend_from_slice(&chunk[..read]); + } + done.store(true, Ordering::Release); + }); + Self { bytes, closed } + } + + fn closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + + fn text(&self) -> String { + let bytes = self + .bytes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + String::from_utf8_lossy(&bytes).into_owned() + } +} + +/// The script printed Python's `returncode`, which is the negated signal number +/// for a child that was killed. Keep that, so an OOM kill and a segfault stay +/// distinguishable. +fn exit_code_text(status: &std::process::ExitStatus) -> String { + if let Some(code) = status.code() { + return code.to_string(); + } + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + if let Some(signal) = status.signal() { + return format!("-{signal}"); + } + } + "signal".to_string() +} + +/// Every check the gate runs after the version comparison, in the exact order +/// the script ran them. Order is part of the contract: each runs against the +/// same throwaway state root, so a command moved earlier or later sees +/// different state. +/// +/// The two serve rejections, which ran last, need a port chosen at run time; +/// their argv carries [`PORT_TOKEN`], which [`execute`] resolves. +fn checks() -> Vec { + checks_for(Host::current()) +} + +/// The host family the gate runs on, as far as the table depends on it. +/// +/// Native Windows is the only host whose expectations differ, and it is a +/// parameter rather than a `cfg!` inside the table so that a test on any host +/// can pin the native-Windows table too. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Host { + NativeWindows, + Other, +} + +impl Host { + const fn current() -> Self { + if cfg!(windows) { + Self::NativeWindows + } else { + Self::Other + } + } + + /// The expected text for a command that native Windows refuses up front: + /// there `rocm` rules vLLM out before it looks at a runtime, a device + /// policy or a port, so the refusal names the platform rather than the + /// condition the check exercises elsewhere. + const fn unless_vllm_ruled_out(self, elsewhere: &'static str) -> &'static str { + match self { + Self::NativeWindows => VLLM_NATIVE_WINDOWS_REFUSAL, + Self::Other => elsewhere, + } + } +} + +/// What `rocm` prints when it refuses a vLLM serve or install on native +/// Windows (`ensure_engine_allowed_on_platform` in `apps/rocm`). +const VLLM_NATIVE_WINDOWS_REFUSAL: &str = "vLLM is not supported on native Windows"; + +fn checks_for(host: Host) -> Vec { + let mut checks = checks_before_platform_gate(host); + if host == Host::NativeWindows { + checks.extend(windows_only_checks()); + } + checks.extend(checks_after_platform_gate(host)); + checks +} + +fn checks_before_platform_gate(host: Host) -> Vec { + vec![ + Check::new("rocm examine", Bin::Rocm, &["examine"]) + .contains("rocm examine") + .contains("default_engine:") + .contains("managed_runtimes: 0") + .contains("managed_services: 0"), + Check::new("rocm engines list", Bin::Rocm, &["engines", "list"]) + .contains("lemonade") + .contains("vllm"), + Check::new( + "rocm config set telemetry off", + Bin::Rocm, + &["config", "set-telemetry", "off"], + ) + .contains("telemetry mode set to off") + .contains("policy: disabled"), + Check::new("rocm config show", Bin::Rocm, &["config", "show"]) + .contains("telemetry_mode: off") + .contains("telemetry_policy: disabled"), + Check::new( + "rocm engines install requires exact runtime", + Bin::Rocm, + &["engines", "install", "vllm"], + ) + .failing() + .contains(host.unless_vllm_ruled_out("no active ROCm runtime is configured")), + Check::new( + "rocm chat local status", + Bin::Rocm, + &["chat", "--provider", "local"], + ) + .contains_any(&[ + "Provider: local", + "Assistant source: local model on this computer", + ]), + Check::new( + "rocm freeform installed status question", + Bin::Rocm, + &["is rocm installed?"], + ) + .contains("ROCm status") + .contains("Nothing was changed.") + .not_contains("No ROCm action selected"), + Check::new( + "rocm freeform comfyui help question", + Bin::Rocm, + &["how do i setup comfyui"], + ) + .contains("ComfyUI status") + .contains("Nothing was changed.") + .not_contains("No ROCm action selected"), + Check::new( + "rocm freeform comfyui install request", + Bin::Rocm, + &["can you setup comfyui for me"], + ) + .contains("Install ComfyUI") + .contains("approval: required"), + Check::new( + "rocm freeform vllm plan", + Bin::Rocm, + &["serve qwen with vllm"], + ) + .contains("engine: vllm") + .contains("no CPU fallback is implied"), + Check::new( + "rocm freeform tiny gpu recipe plan", + Bin::Rocm, + &["serve tiny-gpt2"], + ) + .contains("model: sshleifer/tiny-gpt2") + .contains("device_policy: gpu_required") + .contains("--device gpu_required") + .contains("approval: required"), + ] +} + +/// The Windows-only check: a tarball SDK install has no Windows path and must +/// be refused rather than half-attempted. +fn windows_only_checks() -> Vec { + vec![ + Check::new( + "windows tarball sdk rejection", + Bin::Rocm, + &["install", "sdk", "--format", "tarball", "--dry-run"], + ) + .failing() + .contains("TheRock tarball installs are not supported on Windows"), + ] +} + +fn checks_after_platform_gate(host: Host) -> Vec { + vec![ + Check::new("rocmd status", Bin::Rocmd, &["status"]).contains("rocmd status"), + Check::new("rocmd bridge snapshot", Bin::Rocmd, &["bridge-snapshot"]) + .json(JsonCheck::Protocol("rocmd-codex-bridge-v0")), + Check::new( + "rocmd sandbox examine snapshot", + Bin::Rocmd, + &["sandbox-run", "examine_snapshot", "--allow-native-fallback"], + ) + .json(JsonCheck::SandboxTool { + tool: "examine_snapshot", + noun: "examine", + }), + Check::new( + "rocmd sandbox list servers", + Bin::Rocmd, + &["sandbox-run", "list_servers", "--allow-native-fallback"], + ) + .json(JsonCheck::SandboxTool { + tool: "list_servers", + noun: "list", + }), + Check::new( + "rocmd sandbox prefetch validation", + Bin::Rocmd, + &[ + "sandbox-run", + "prefetch_artifact", + "--allow-native-fallback", + ], + ) + .failing() + .contains("prefetch_artifact requires"), + Check::new("vllm detect", Bin::Vllm, &["detect"]).json(JsonCheck::Parses), + Check::new("vllm capabilities", Bin::Vllm, &["capabilities"]) + .json(JsonCheck::VllmCapabilities), + Check::new("vllm resolve qwen", Bin::Vllm, &["resolve-model", "qwen"]).contains("qwen"), + Check::new( + "vllm reject cpu", + Bin::Vllm, + &["resolve-model", "qwen", "--device-policy", "cpu_only"], + ) + .failing() + .contains("no CPU fallback is used"), + // The point of the whole gate: a GPU-required serve must refuse rather + // than quietly running on CPU, and an explicit CPU serve must be + // refused too. + Check::new( + "vllm reject required gpu", + Bin::Rocm, + &[ + "serve", + "qwen", + "--engine", + "vllm", + "--device", + "gpu_required", + "--foreground", + "--port", + PORT_TOKEN, + ], + ) + .failing() + .contains(host.unless_vllm_ruled_out("gpu_required")) + .not_contains("CPU fallback"), + Check::new( + "rocm vllm reject cpu serve", + Bin::Rocm, + &[ + "serve", + "qwen", + "--engine", + "vllm", + "--device", + "cpu", + "--foreground", + "--port", + PORT_TOKEN, + ], + ) + .failing() + .contains(host.unless_vllm_ruled_out("CPU mode is not a fallback path")), + ] +} + +/// `cargo build` argv for a smoke run, so the build lands where the binaries are +/// then looked for. The script hard-coded a debug workspace build, which made +/// `--profile release` build one profile and then smoke another. +/// +/// The debug build keeps the script's `--workspace --all-targets`: as the local +/// gate it also catches a test target that no longer compiles. The release +/// build is narrowed to the four binaries the gate runs, for two reasons. A +/// release build of every test and bench target is a heavy price for four +/// executables. And `cargo xtask` is itself a release build of this crate, so a +/// release `--workspace` build would relink the very executable that is running +/// it — which Windows refuses. +/// +/// `target_dir` is the already-resolved directory, so cargo and the binary +/// lookup cannot disagree about a relative path. +fn build_args(profile: Profile, target_dir: Option<&Path>) -> Vec { + let mut args = vec!["build".to_string()]; + match profile { + Profile::Debug => { + args.push("--workspace".to_string()); + args.push("--all-targets".to_string()); + } + Profile::Release => { + args.push("--release".to_string()); + for bin in Bin::ALL { + args.push("-p".to_string()); + args.push(bin.stem().to_string()); + } + } + } + if let Some(dir) = target_dir { + args.push("--target-dir".to_string()); + args.push(dir.display().to_string()); + } + args +} + +/// Resolve a `--target-dir` override against the workspace root once, so the +/// build and the binary lookup are handed the same absolute path. +fn resolve_target_dir(root: &Path, dir: &Path) -> PathBuf { + // `join` returns an absolute `dir` unchanged. + root.join(dir) +} + +/// Directory the built binaries are looked for in: the resolved override, or +/// the active cargo target directory. +fn binary_dir(target_dir: &Path, profile: Profile) -> PathBuf { + target_dir.join(profile.dir_name()) +} + +/// Label printed for the build step, so the log says what was compiled. +const fn build_label(profile: Profile) -> &'static str { + match profile { + // The script's label, kept: this is the build it ran. + Profile::Debug => "build workspace all targets", + Profile::Release => "build release binaries", + } +} + +/// Everything a smoke run decides before it spawns anything, so those decisions +/// can be checked without building or running a binary. +#[derive(Debug)] +struct Plan { + root: PathBuf, + /// Throwaway state root, wiped at the start of every run. + env_root: PathBuf, + env: BTreeMap<&'static str, OsString>, + binaries: Binaries, + /// `cargo` argv for the build, or `None` with `--skip-build`. + build: Option>, +} + +impl Plan { + /// `default_target_dir` is where the binaries are when no `--target-dir` + /// is given: [`run`] passes [`crate::paths::target_dir`], which honours + /// `CARGO_TARGET_DIR`. + fn new( + root: &Path, + default_target_dir: &Path, + profile: Profile, + skip_build: bool, + target_dir_override: Option<&Path>, + ) -> Self { + // Resolved once, and handed to the build as well as the lookup, so the + // build lands where the binaries are looked for even where cargo + // config (`build.target-dir`) would send it elsewhere. + let target_dir = target_dir_override.map_or_else( + || default_target_dir.to_path_buf(), + |dir| resolve_target_dir(root, dir), + ); + // Rooted at the checkout, not the active target dir: a target dir may + // be shared between checkouts, and this directory is wiped at the start + // of every run — two worktrees sharing a target dir would delete each + // other's state mid-run. + let env_root = root.join("target").join("smoke-local"); + Self { + root: root.to_path_buf(), + env: isolated_env(&env_root), + env_root, + binaries: Binaries::new(binary_dir(&target_dir, profile)), + build: (!skip_build).then(|| build_args(profile, Some(&target_dir))), + } + } +} + +/// Runs one smoke command: `(label, program, argv, expect_failure)` to its +/// combined output. [`run`] passes [`run_command`]; tests pass a fake, which is +/// what lets [`execute`] — the gate's whole logic — be tested without binaries. +type Runner<'a> = dyn FnMut(&str, &Path, &[&str], bool) -> Result + 'a; + +/// The gate: the version comparison, every table check in order, and the +/// first-run assertions, against binaries the plan has located. +fn execute(plan: &Plan, port: &str, run: &mut Runner<'_>, say: &mut dyn FnMut(&str)) -> Result<()> { + plan.binaries.require_all_present(plan.build.is_none())?; + say(&run_header(plan)); + + if plan.env_root.exists() { + std::fs::remove_dir_all(&plan.env_root) + .with_context(|| format!("failed to clear {}", plan.env_root.display()))?; + } + std::fs::create_dir_all(&plan.env_root) + .with_context(|| format!("failed to create {}", plan.env_root.display()))?; + + let rocm = plan.binaries.path(Bin::Rocm); + + // The three version surfaces must agree on the build string, and `rocm + // version` reports the active SDK and driver on top of it. + let version_flag = run("rocm --version", &rocm, &["--version"], false)?; + let version_short = run("rocm -V", &rocm, &["-V"], false)?; + if version_flag != version_short { + bail!( + "version flag surfaces returned different output: {version_flag:?} vs {version_short:?}" + ); + } + let version_command = run("rocm version", &rocm, &["version"], false)?; + let build_line = version_command.lines().next().unwrap_or_default(); + if build_line != version_flag { + bail!( + "`rocm version`'s build line does not match `-V`/`--version`: {build_line:?} vs {version_flag:?}" + ); + } + assert_version_string(build_line, "rocm version")?; + assert_contains(&version_command, "ROCm SDK:", "rocm version")?; + assert_contains(&version_command, "GPU driver:", "rocm version")?; + + for check in &checks() { + let program = plan.binaries.path(check.bin); + let args = check.resolved_args(port); + let argv: Vec<&str> = args.iter().map(String::as_str).collect(); + let output = run(check.label, &program, &argv, check.expect_failure)?; + check.assert_output(&output)?; + } + + // Nothing above should have provisioned anything: a first run that quietly + // created a uv cache or a runtime registry would mean a command took an + // install path it was never asked to take. The parent directories come from + // the map the children were given, so renaming one in `isolated_env` cannot + // leave this looking somewhere the CLI was never pointed. + let (cache_dir, data_dir) = state_dirs(&plan.env); + assert_path_missing(&cache_dir.join("uv"), "first-run smoke uv cache")?; + assert_path_missing( + &data_dir.join("runtimes").join("registry"), + "first-run smoke runtime registry", + )?; + Ok(()) +} + +/// The line printed before the first check, naming the binaries and the +/// state root the run uses: +/// `--profile`, `--target-dir` and `CARGO_TARGET_DIR` all move the first, and +/// nothing else in the output says which build was smoked. +fn run_header(plan: &Plan) -> String { + format!( + "smoke: binaries from {}; throwaway state root {}", + plan.binaries.dir.display(), + plan.env_root.display() + ) +} + +/// The runner [`run`] hands to [`execute`]: every child gets the plan's isolated +/// environment and runs in the workspace root. +/// +/// Separate from `run` so a test can prove it. A runner that lost the isolation +/// would not fail the gate — the binaries would read and write the developer's +/// real state, and on a clean machine still print `smoke: ok` — so running the +/// gate cannot catch that; only a test of this function can. +fn child_runner(plan: &Plan) -> impl FnMut(&str, &Path, &[&str], bool) -> Result + '_ { + move |label, program, argv, expect_failure| { + run_command( + label, + program, + argv, + &plan.env, + &plan.root, + expect_failure, + CHILD_DEADLINE, + ) + } +} + +/// Build, then run every smoke check against the built binaries in an isolated +/// state root. +/// +/// Glue between the environment and the tested parts: the gate's logic is +/// [`execute`], the up-front decisions are [`Plan`], and the children's +/// isolation is [`child_runner`]. What is untested is this function's own +/// wiring — the build step, the choice of port, and the hand-off of +/// `child_runner` to `execute` — which runs only when the gate itself is run. +pub fn run(profile: Profile, skip_build: bool, target_dir_override: Option) -> Result<()> { + let root = workspace_root()?; + let plan = Plan::new( + &root, + &crate::paths::target_dir(&root), + profile, + skip_build, + target_dir_override.as_deref(), + ); + + if let Some(args) = &plan.build { + let cargo = PathBuf::from(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())); + let argv: Vec<&str> = args.iter().map(String::as_str).collect(); + // The build inherits this process's environment rather than the isolated + // one. Isolation exists for the binaries under test; under it, a redirected + // `HOME` hides cargo's registry and config from the build. No build + // script reads the variables the isolation sets. + run_build(build_label(profile), &cargo, &argv, &root)?; + } + + let port = free_tcp_port()?.to_string(); + execute(&plan, &port, &mut child_runner(&plan), &mut |line| { + println!("{line}"); + })?; + println!("smoke: ok"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn binary_names_cover_every_required_artifact() { + let binaries = Binaries::new(PathBuf::from("/build/debug")); + let stems: Vec = Bin::ALL + .iter() + .map(|bin| { + binaries + .path(*bin) + .file_name() + .expect("file name") + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + stems, + vec![ + binary_name("rocm"), + binary_name("rocmd"), + binary_name("rocm-engine-lemonade"), + binary_name("rocm-engine-vllm"), + ] + ); + } + + #[test] + fn missing_binary_is_named_with_its_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let binaries = Binaries::new(dir.path().to_path_buf()); + let error = binaries + .require_all_present(false) + .expect_err("an empty dir has no binaries"); + let message = format!("{error}"); + assert!(message.contains("missing smoke binary rocm"), "{message}"); + assert!(message.contains(&binary_name("rocm")), "{message}"); + assert!( + message.contains(&format!("(looked in {})", dir.path().display())), + "{message}" + ); + // A build ran and still produced nothing here: --skip-build is not + // the cause, so the failure must not offer it as the fix. + assert!(!message.contains("--skip-build"), "{message}"); + } + + /// With the build skipped, the failure names the fix as well as the gap. + /// That the fix works — a run without `--skip-build` builds into the + /// directory searched — is `the_build_lands_where_the_binaries_are_looked_for`. + #[test] + fn a_missing_binary_after_a_skipped_build_says_how_to_fix_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let error = Binaries::new(dir.path().to_path_buf()) + .require_all_present(true) + .expect_err("an empty dir has no binaries"); + let message = format!("{error}"); + assert!( + message.contains(&format!("(looked in {})", dir.path().display())), + "{message}" + ); + assert!( + message.contains("nothing was built because of --skip-build: drop it to build them"), + "{message}" + ); + } + + /// The run says which binaries and state root it used, since the flags and + /// `CARGO_TARGET_DIR` move them. + #[test] + fn the_run_header_names_the_binaries_and_the_state_root() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + assert_eq!( + run_header(&plan), + format!( + "smoke: binaries from {}; throwaway state root {}", + plan.binaries.dir.display(), + plan.env_root.display() + ) + ); + assert_ne!(plan.binaries.dir, plan.env_root); + } + + #[test] + fn profile_selects_the_cargo_output_dir() { + assert_eq!(Profile::Debug.dir_name(), "debug"); + assert_eq!(Profile::Release.dir_name(), "release"); + } + + /// The gate's whole content, pinned. + /// + /// With `scripts/smoke_local.py` deleted, nothing else records what this + /// gate asserts: the order and label tests below pass just as happily over + /// a check whose expectations have been quietly dropped. Dropping + /// `.failing()` from a rejection check — turning "this must be refused" + /// into "this must succeed" — is the failure this exists to catch. + /// + /// Update it when a check genuinely changes, and say why in the commit. + /// + /// Both hosts' tables are pinned on every host, so the native-Windows + /// expectations are checked by the Linux CI job too, not only by the + /// Windows one. + #[test] + fn every_check_pins_its_command_and_its_expectations() { + for host in [Host::Other, Host::NativeWindows] { + let rendered: Vec = checks_for(host).iter().map(render_check).collect(); + assert_eq!(rendered, pinned_table(host), "{host:?}"); + } + } + + /// The gate runs the current host's table. + #[test] + fn the_gate_runs_the_current_hosts_table() { + let rendered: Vec = checks().iter().map(render_check).collect(); + let expected = if cfg!(windows) { + pinned_table(Host::NativeWindows) + } else { + pinned_table(Host::Other) + }; + assert_eq!(rendered, expected); + } + + fn pinned_table(host: Host) -> Vec<&'static str> { + let windows = host == Host::NativeWindows; + let mut table = vec![ + r#"rocm examine | rocm ["examine"] | =contains["rocm examine", "default_engine:", "managed_runtimes: 0", "managed_services: 0"]"#, + r#"rocm engines list | rocm ["engines", "list"] | =contains["lemonade", "vllm"]"#, + r#"rocm config set telemetry off | rocm ["config", "set-telemetry", "off"] | =contains["telemetry mode set to off", "policy: disabled"]"#, + r#"rocm config show | rocm ["config", "show"] | =contains["telemetry_mode: off", "telemetry_policy: disabled"]"#, + if windows { + r#"rocm engines install requires exact runtime | rocm ["engines", "install", "vllm"] | !contains["vLLM is not supported on native Windows"]"# + } else { + r#"rocm engines install requires exact runtime | rocm ["engines", "install", "vllm"] | !contains["no active ROCm runtime is configured"]"# + }, + r#"rocm chat local status | rocm ["chat", "--provider", "local"] | =any["Provider: local", "Assistant source: local model on this computer"]"#, + r#"rocm freeform installed status question | rocm ["is rocm installed?"] | =contains["ROCm status", "Nothing was changed."] absent["No ROCm action selected"]"#, + r#"rocm freeform comfyui help question | rocm ["how do i setup comfyui"] | =contains["ComfyUI status", "Nothing was changed."] absent["No ROCm action selected"]"#, + r#"rocm freeform comfyui install request | rocm ["can you setup comfyui for me"] | =contains["Install ComfyUI", "approval: required"]"#, + r#"rocm freeform vllm plan | rocm ["serve qwen with vllm"] | =contains["engine: vllm", "no CPU fallback is implied"]"#, + r#"rocm freeform tiny gpu recipe plan | rocm ["serve tiny-gpt2"] | =contains["model: sshleifer/tiny-gpt2", "device_policy: gpu_required", "--device gpu_required", "approval: required"]"#, + r#"rocmd status | rocmd ["status"] | =contains["rocmd status"]"#, + r#"rocmd bridge snapshot | rocmd ["bridge-snapshot"] | =json(Protocol("rocmd-codex-bridge-v0"))"#, + r#"rocmd sandbox examine snapshot | rocmd ["sandbox-run", "examine_snapshot", "--allow-native-fallback"] | =json(SandboxTool { tool: "examine_snapshot", noun: "examine" })"#, + r#"rocmd sandbox list servers | rocmd ["sandbox-run", "list_servers", "--allow-native-fallback"] | =json(SandboxTool { tool: "list_servers", noun: "list" })"#, + r#"rocmd sandbox prefetch validation | rocmd ["sandbox-run", "prefetch_artifact", "--allow-native-fallback"] | !contains["prefetch_artifact requires"]"#, + r#"vllm detect | vllm ["detect"] | =json(Parses)"#, + r#"vllm capabilities | vllm ["capabilities"] | =json(VllmCapabilities)"#, + r#"vllm resolve qwen | vllm ["resolve-model", "qwen"] | =contains["qwen"]"#, + r#"vllm reject cpu | vllm ["resolve-model", "qwen", "--device-policy", "cpu_only"] | !contains["no CPU fallback is used"]"#, + if windows { + r#"vllm reject required gpu | rocm ["serve", "qwen", "--engine", "vllm", "--device", "gpu_required", "--foreground", "--port", "{port}"] | !contains["vLLM is not supported on native Windows"] absent["CPU fallback"]"# + } else { + r#"vllm reject required gpu | rocm ["serve", "qwen", "--engine", "vllm", "--device", "gpu_required", "--foreground", "--port", "{port}"] | !contains["gpu_required"] absent["CPU fallback"]"# + }, + if windows { + r#"rocm vllm reject cpu serve | rocm ["serve", "qwen", "--engine", "vllm", "--device", "cpu", "--foreground", "--port", "{port}"] | !contains["vLLM is not supported on native Windows"]"# + } else { + r#"rocm vllm reject cpu serve | rocm ["serve", "qwen", "--engine", "vllm", "--device", "cpu", "--foreground", "--port", "{port}"] | !contains["CPU mode is not a fallback path"]"# + }, + ]; + if windows { + let after_freeform = table + .iter() + .position(|row| row.starts_with("rocmd status |")) + .expect("rocmd status row"); + table.insert( + after_freeform, + r#"windows tarball sdk rejection | rocm ["install", "sdk", "--format", "tarball", "--dry-run"] | !contains["TheRock tarball installs are not supported on Windows"]"#, + ); + } + table + } + + /// `=` expects success, `!` expects a non-zero exit. + fn render_check(check: &Check) -> String { + // Debug formatting quotes every element, so the rendering shows argv + // boundaries: `["serve qwen with vllm"]` is one argument, and splitting + // it into four would make `rocm` parse a real `serve` subcommand. The + // JSON expectation is rendered whole for the same reason — its payload + // is what the check asserts, not its variant name. + let mut parts = vec![ + check.label.to_string(), + format!("{} {:?}", check.bin.key(), check.args), + ]; + let mut expectations = vec![if check.expect_failure { + "!".to_string() + } else { + "=".to_string() + }]; + if !check.contains.is_empty() { + expectations.push(format!("contains{:?}", check.contains)); + } + if !check.contains_any.is_empty() { + expectations.push(format!("any{:?}", check.contains_any)); + } + if !check.not_contains.is_empty() { + expectations.push(format!("absent{:?}", check.not_contains)); + } + if let Some(json) = check.json { + expectations.push(format!("json({json:?})")); + } + let head = expectations.remove(0); + parts.push(format!("{head}{}", expectations.join(" "))); + parts.join(" | ") + } + + /// An empty needle is satisfied by every string, so a check carrying one + /// asserts nothing while looking like it does. + #[test] + fn no_check_carries_an_empty_needle() { + for check in [Host::Other, Host::NativeWindows] + .into_iter() + .flat_map(checks_for) + { + for needle in check + .contains + .iter() + .chain(check.contains_any.iter()) + .chain(check.not_contains.iter()) + { + assert!( + !needle.trim().is_empty(), + "{} carries an empty needle", + check.label + ); + } + } + } + + /// The port token is substituted, not passed through to the CLI as a + /// literal — and only where it appears. + #[test] + fn the_port_token_is_resolved_in_argv() { + let check = Check::new( + "demo", + Bin::Rocm, + &["serve", "--port", PORT_TOKEN, "--device", "cpu"], + ); + assert_eq!( + check.resolved_args("54321"), + vec!["serve", "--port", "54321", "--device", "cpu"] + ); + // The token is substituted only as a whole argument, so one embedded in + // a larger argument (`--port={port}`) would reach the CLI verbatim. Catch + // that by substring, which the whole-argument substitution cannot hide. + for check in &checks() { + assert!( + !check + .resolved_args("1") + .iter() + .any(|arg| arg.contains(PORT_TOKEN)), + "{} passes the port token through unresolved", + check.label + ); + } + let embedded = Check::new("demo", Bin::Rocm, &["--port={port}"]); + assert!( + embedded.resolved_args("1")[0].contains(PORT_TOKEN), + "the substring check above must be able to see an embedded token" + ); + } + + /// Order is part of the contract, not an accident of how the table was + /// written: every check shares one throwaway state root, so a command moved + /// earlier or later sees different state. This pins the sequence the + /// `scripts/smoke_local.py` gate ran after its version comparison, minus the + /// platform-gated entry, which the next test places. + #[test] + fn the_check_sequence_matches_the_gate_it_replaces() { + let labels: Vec<&str> = checks_for(Host::Other) + .iter() + .map(|check| check.label) + .collect(); + assert_eq!( + labels, + vec![ + "rocm examine", + "rocm engines list", + "rocm config set telemetry off", + "rocm config show", + "rocm engines install requires exact runtime", + "rocm chat local status", + "rocm freeform installed status question", + "rocm freeform comfyui help question", + "rocm freeform comfyui install request", + "rocm freeform vllm plan", + "rocm freeform tiny gpu recipe plan", + "rocmd status", + "rocmd bridge snapshot", + "rocmd sandbox examine snapshot", + "rocmd sandbox list servers", + "rocmd sandbox prefetch validation", + "vllm detect", + "vllm capabilities", + "vllm resolve qwen", + "vllm reject cpu", + "vllm reject required gpu", + "rocm vllm reject cpu serve", + ] + ); + } + + /// The platform-gated check runs between the freeform plans and the `rocmd` + /// checks, where the script ran it — not appended at the end. + #[test] + fn the_windows_check_sits_between_the_two_halves() { + let before = checks_before_platform_gate(Host::NativeWindows); + assert_eq!( + before.last().expect("a check").label, + "rocm freeform tiny gpu recipe plan" + ); + assert_eq!( + checks_after_platform_gate(Host::NativeWindows) + .first() + .expect("a check") + .label, + "rocmd status" + ); + let labels: Vec<&str> = checks_for(Host::NativeWindows) + .iter() + .map(|check| check.label) + .collect(); + let gated = labels + .iter() + .position(|label| *label == "windows tarball sdk rejection") + .expect("the windows check is in the table"); + assert_eq!(gated, before.len()); + assert!( + !checks_for(Host::Other) + .iter() + .any(|check| check.label == "windows tarball sdk rejection"), + "the windows check runs only on native Windows" + ); + } + + #[test] + fn check_labels_are_unique_across_the_table() { + let mut labels: Vec<&str> = checks_for(Host::NativeWindows) + .iter() + .map(|check| check.label) + .collect(); + let total = labels.len(); + labels.sort_unstable(); + labels.dedup(); + assert_eq!(labels.len(), total, "duplicate check labels: {labels:?}"); + } + + #[test] + fn every_check_asserts_something() { + for check in checks_for(Host::NativeWindows) + .iter() + .chain(checks_for(Host::Other).iter()) + { + assert!( + !check.contains.is_empty() + || !check.contains_any.is_empty() + || !check.not_contains.is_empty() + || check.json.is_some(), + "{} runs a command but asserts nothing", + check.label + ); + assert!(!check.args.is_empty(), "{} has no arguments", check.label); + } + } + + /// The refusals are what the gate exists for, so each must be in the + /// table on every host, expected to fail, and asserting what it printed. + #[test] + fn every_gpu_refusal_is_smoked_as_a_refusal_on_every_host() { + for host in [Host::Other, Host::NativeWindows] { + let table = checks_for(host); + for required in [ + "rocm engines install requires exact runtime", + "vllm reject cpu", + "vllm reject required gpu", + "rocm vllm reject cpu serve", + ] { + let check = table + .iter() + .find(|check| check.label == required) + .unwrap_or_else(|| panic!("{required} is not smoked on {host:?}")); + assert!( + check.expect_failure, + "{required} must expect a refusal on {host:?}" + ); + assert!( + !check.contains.is_empty(), + "{required} must assert the refusal's text on {host:?}" + ); + } + let plan = table + .iter() + .find(|check| check.label == "rocm freeform vllm plan") + .unwrap_or_else(|| panic!("the vllm plan is not smoked on {host:?}")); + assert!(plan.contains.contains(&"no CPU fallback is implied")); + } + } + + #[test] + fn contains_failure_names_the_label_and_the_output() { + let check = Check::new("demo", Bin::Rocm, &["examine"]).contains("expected text"); + let error = check + .assert_output("actual output") + .expect_err("the needle is absent"); + let message = format!("{error}"); + assert!( + message.contains("demo did not contain expected text"), + "{message}" + ); + assert!(message.contains("actual output"), "{message}"); + } + + #[test] + fn contains_any_accepts_either_wording_and_reports_both() { + let check = Check::new("demo", Bin::Rocm, &["chat"]).contains_any(&["first", "second"]); + check + .assert_output("has second here") + .expect("second matches"); + check + .assert_output("has first here") + .expect("first matches"); + + let error = check + .assert_output("neither") + .expect_err("no wording matches"); + let message = format!("{error}"); + assert!(message.contains("'first' or 'second'"), "{message}"); + } + + #[test] + fn not_contains_rejects_the_forbidden_text() { + let check = Check::new("demo", Bin::Rocm, &["serve"]).not_contains("CPU fallback"); + check.assert_output("gpu_required").expect("absent is fine"); + + let error = check + .assert_output("used CPU fallback instead") + .expect_err("the forbidden text is present"); + assert!( + format!("{error}").contains("demo contained unexpected text: CPU fallback"), + "{error}" + ); + } + + #[test] + fn version_line_parses_a_traceable_build_string() { + assert_eq!( + parse_version_line("rocm-cli 0.2.0 (main, a1b2c3d)"), + Some(("0.2.0", "main", "a1b2c3d")) + ); + assert_eq!( + parse_version_line(" rocm-cli 1.0.0-rc.1 (v1.0.0, 0123456789abcdef) "), + Some(("1.0.0-rc.1", "v1.0.0", "0123456789abcdef")) + ); + } + + #[test] + fn version_line_rejects_malformed_and_uppercase_hashes() { + for text in [ + "rocm-cli 0.2.0", + "rocm 0.2.0 (main, a1b2c3d)", + "rocm-cli 0.2.0 (main a1b2c3d)", + "rocm-cli 0.2.0 (main, A1B2C3D)", + "rocm-cli 0.2.0 (main, zzz)", + "rocm-cli 0.2.0 (main, )", + "rocm-cli (main, a1b2c3d)", + ] { + assert_eq!(parse_version_line(text), None, "{text} should not parse"); + } + } + + #[test] + fn an_unresolved_ref_fails_even_though_it_parses() { + assert!(parse_version_line("rocm-cli 0.2.0 (unknown, a1b2c3d)").is_some()); + let error = assert_version_string("rocm-cli 0.2.0 (unknown, a1b2c3d)", "rocm version") + .expect_err("an unknown ref is rejected"); + assert!(format!("{error}").contains("unresolved ref"), "{error}"); + assert!( + format!("{error}").contains( + "A detached HEAD or a source tree without git metadata produces this; \ + build from a branch or tag" + ), + "{error}" + ); + assert_version_string("rocm-cli 0.0.0-dev (main, a1b2c3d)", "rocm version") + .expect("a real ref"); + } + + /// The version must be the one the ref calls for: a release tag's own + /// version, and `0.0.0-dev` for anything else — never the workspace + /// package version a branch build used to report. + #[test] + fn the_reported_version_must_be_the_one_the_ref_calls_for() { + for ok in [ + "rocm-cli 0.0.0-dev (main, a1b2c3d)", + "rocm-cli 0.0.0-dev (vllm, a1b2c3d)", + "rocm-cli 0.2.1 (v0.2.1, a1b2c3d)", + "rocm-cli 1.2.3-rc.1+build.5 (v1.2.3-rc.1+build.5, a1b2c3d)", + "rocm-cli 0.0.0-dev (v1.2.3-rc_1, a1b2c3d)", + ] { + assert_version_string(ok, "rocm version") + .unwrap_or_else(|error| panic!("{ok}: {error}")); + } + for bad in [ + "rocm-cli 0.1.0 (main, a1b2c3d)", + "rocm-cli 0.0.0-dev (v0.2.1, a1b2c3d)", + "rocm-cli v0.2.1 (v0.2.1, a1b2c3d)", + "rocm-cli 1.2.3-rc_1 (v1.2.3-rc_1, a1b2c3d)", + ] { + let error = assert_version_string(bad, "rocm version") + .expect_err(&format!("{bad} is rejected")); + assert!(format!("{error}").contains("expected"), "{error}"); + } + } + + /// The release-tag grammar here is a copy of `apps/rocm/build.rs`'s. + /// `release_tag_grammar_is_the_same_everywhere` there holds the two equal, + /// but it runs only when `rocm` is tested; this runs whenever `xtask` is, + /// so changing the copy here alone is caught on the same run. + #[test] + fn the_release_tag_grammar_matches_build_rs() { + let build_rs = std::fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../apps/rocm/build.rs"), + ) + .expect("read apps/rocm/build.rs") + .replace("\r\n", "\n"); + let pattern = format!(r#"const RELEASE_TAG_PATTERN: &str = r"{RELEASE_TAG_PATTERN}";"#); + assert!(build_rs.contains(&pattern), "build.rs lacks `{pattern}`"); + let bound = format!( + "const RELEASE_VERSION_MAX_LEN: usize = {};", + RELEASE_TAG_MAX_LEN - 1 + ); + assert!(build_rs.contains(&bound), "build.rs lacks `{bound}`"); + } + + /// A release tag is at most 65 characters, `v` included. + #[test] + fn a_release_tag_over_the_length_bound_is_not_a_release() { + let at_limit = format!("v1.2.3-{}", "a".repeat(RELEASE_TAG_MAX_LEN - 7)); + assert_eq!(at_limit.len(), RELEASE_TAG_MAX_LEN); + assert!(is_release_tag(&at_limit)); + assert!(!is_release_tag(&format!("{at_limit}a"))); + } + + /// Each JSON variant must reject a wrong payload, and say which check and + /// which payload was wrong — these are the only failures where the output + /// is a blob rather than prose a reader can scan. + #[test] + fn each_json_check_rejects_a_wrong_payload_by_name() { + let cases = [ + ( + JsonCheck::Protocol("rocmd-codex-bridge-v0"), + serde_json::json!({"protocol": "something-else"}), + "unexpected bridge snapshot protocol", + ), + ( + JsonCheck::SandboxTool { + tool: "examine_snapshot", + noun: "examine", + }, + serde_json::json!({"tool": "examine_snapshot", "ok": false}), + "unexpected sandbox examine result", + ), + ( + JsonCheck::SandboxTool { + tool: "list_servers", + noun: "list", + }, + serde_json::json!({"tool": "list_servers", "ok": false}), + "unexpected sandbox list result", + ), + ( + JsonCheck::VllmCapabilities, + serde_json::json!({"openai_compatible": true, "cpu": true}), + "vllm capabilities did not report GPU-only OpenAI serving", + ), + ]; + for (check, payload, expected) in cases { + let error = check + .assert_value(&payload) + .expect_err("the payload should be rejected"); + let message = format!("{error}"); + assert!(message.contains(expected), "{message}"); + } + } + + #[test] + fn a_parses_only_check_accepts_any_valid_json() { + JsonCheck::Parses + .assert_value(&serde_json::json!({"anything": 1})) + .expect("an object parses"); + JsonCheck::Parses + .assert_value(&serde_json::json!([])) + .expect("an array parses too — `vllm detect` only has to be JSON"); + } + + /// A reply that is valid JSON but not an object must not slip past the + /// identity checks. + #[test] + fn a_non_object_reply_is_rejected() { + for check in [ + JsonCheck::Protocol("rocmd-codex-bridge-v0"), + JsonCheck::VllmCapabilities, + ] { + assert!(check.assert_value(&serde_json::json!("a string")).is_err()); + assert!(check.assert_value(&serde_json::json!([1, 2])).is_err()); + } + } + + #[test] + fn json_truthiness_matches_the_script() { + let value = serde_json::json!({ + "yes": true, + "no": false, + "zero": 0, + "one": 1, + "empty_text": "", + "text": "x", + "empty_list": [], + "list": [1], + "empty_map": {}, + "map": {"k": 1}, + "null": null, + }); + for field in ["yes", "one", "text", "list", "map"] { + assert!(json_truthy(&value, field), "{field} should be truthy"); + } + for field in [ + "no", + "zero", + "empty_text", + "empty_list", + "empty_map", + "null", + "absent", + ] { + assert!(!json_truthy(&value, field), "{field} should be falsy"); + } + } + + #[test] + fn a_sandbox_reply_must_name_its_own_tool_and_report_ok() { + let good = serde_json::json!({"tool": "list_servers", "ok": true}); + assert!(sandbox_reply_ok(&good, "list_servers")); + assert!( + !sandbox_reply_ok(&good, "examine_snapshot"), + "a reply for another tool is not an answer" + ); + + let not_ok = serde_json::json!({"tool": "list_servers", "ok": false}); + assert!(!sandbox_reply_ok(¬_ok, "list_servers")); + + let missing_ok = serde_json::json!({"tool": "list_servers"}); + assert!(!sandbox_reply_ok(&missing_ok, "list_servers")); + + assert!(!sandbox_reply_ok(&serde_json::json!([]), "list_servers")); + } + + #[test] + fn vllm_capabilities_must_be_openai_compatible_and_not_cpu() { + assert!(vllm_capabilities_ok(&serde_json::json!({ + "openai_compatible": true, + "cpu": false, + }))); + assert!(vllm_capabilities_ok( + &serde_json::json!({"openai_compatible": true}) + )); + assert!( + !vllm_capabilities_ok(&serde_json::json!({"openai_compatible": true, "cpu": true})), + "advertising CPU is the regression this check exists for" + ); + assert!(!vllm_capabilities_ok(&serde_json::json!({ + "openai_compatible": false, + "cpu": false, + }))); + } + + #[test] + fn parse_json_failure_names_the_label() { + let error = parse_json("not json", "vllm detect").expect_err("invalid JSON"); + assert!( + format!("{error}").contains("vllm detect did not return valid JSON"), + "{error}" + ); + } + + #[test] + fn isolated_env_confines_every_state_dir_to_the_smoke_root() { + let root = Path::new("/tmp/smoke-local"); + let env = isolated_env(root); + + assert_eq!(env["ROCM_CLI_UPDATE_USER_PATH"], OsString::from("0")); + // Every other entry is a directory, and every one must sit under the + // throwaway root. Asserting only that the keys are present would let any + // one of them be pointed at the developer's real profile unnoticed. + for (key, value) in &env { + if *key == "ROCM_CLI_UPDATE_USER_PATH" { + continue; + } + let value = PathBuf::from(value); + assert!( + value.starts_with(root), + "{key} escaped the smoke root: {value:?}" + ); + } + for key in [ + "ROCM_CLI_CONFIG_DIR", + "ROCM_CLI_DATA_DIR", + "ROCM_CLI_CACHE_DIR", + ] { + assert!(env.contains_key(key), "{key} is not redirected"); + } + } + + #[cfg(windows)] + #[test] + fn isolated_env_redirects_the_windows_profile_dirs() { + let root = Path::new("C:\\smoke"); + let env = isolated_env(root); + for key in ["APPDATA", "LOCALAPPDATA"] { + let value = PathBuf::from(env.get(key).unwrap_or_else(|| panic!("{key} is not set"))); + assert!( + value.starts_with(root), + "{key} points outside the smoke root: {value:?}" + ); + } + assert!(!env.contains_key("HOME")); + } + + // Unix-only: the XDG/HOME arm is compiled out on Windows, where the branch + // above applies instead. Supported hosts are Windows and Linux only. + #[cfg(unix)] + #[test] + fn isolated_env_redirects_the_xdg_dirs_and_home() { + let root = Path::new("/tmp/smoke-local"); + let env = isolated_env(root); + for key in ["XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "HOME"] { + let value = PathBuf::from(env.get(key).unwrap_or_else(|| panic!("{key} is not set"))); + assert!( + value.starts_with(root), + "{key} points outside the smoke root: {value:?}" + ); + } + assert!(!env.contains_key("APPDATA")); + } + + /// A child killed by a signal has no exit code. The script printed Python's + /// negated signal number, which keeps an OOM kill (`-9`) distinguishable + /// from a segfault (`-11`); a bare "signal" would not. + #[cfg(unix)] + #[test] + fn a_signalled_child_keeps_its_signal_number() { + use std::os::unix::process::ExitStatusExt as _; + + assert_eq!(exit_code_text(&std::process::ExitStatus::from_raw(9)), "-9"); + assert_eq!( + exit_code_text(&std::process::ExitStatus::from_raw(11)), + "-11" + ); + } + + #[test] + fn an_exit_code_is_reported_verbatim() { + let status = Command::new(if cfg!(windows) { "cmd" } else { "sh" }) + .args(if cfg!(windows) { + vec!["/C", "exit 3"] + } else { + vec!["-c", "exit 3"] + }) + .status() + .expect("spawn a shell"); + assert_eq!(exit_code_text(&status), "3"); + } + + /// Run a shell script through `run_command`, so it is driven for real rather + /// than reasoned about. Each test supplies the script for both shells: `cmd` + /// is not POSIX — `;` is not a separator there and there is no `cat` — so a + /// shared script would test something different on each platform. + fn run_script( + label: &str, + unix: &str, + windows: &str, + env: &BTreeMap<&'static str, OsString>, + cwd: &Path, + expect_failure: bool, + ) -> Result { + let (program, flag, script) = if cfg!(windows) { + ("cmd", "/C", windows) + } else { + ("sh", "-c", unix) + }; + run_command( + label, + Path::new(program), + &[flag, script], + env, + cwd, + expect_failure, + TEST_DEADLINE, + ) + } + + /// Long enough that no test child that is meant to finish gets near it. + const TEST_DEADLINE: Duration = Duration::from_mins(1); + + /// A child that outlives its deadline is killed and the failure names the + /// check, rather than the gate waiting for it: the regression this guards + /// is a refusal check whose command started a server instead. + #[test] + fn run_command_kills_a_child_that_outlives_its_deadline() { + let dir = tempfile::tempdir().expect("tempdir"); + let (program, args): (&str, &[&str]) = if cfg!(windows) { + ("ping", &["-n", "60", "127.0.0.1"]) + } else { + ("sleep", &["60"]) + }; + let started = Instant::now(); + let error = run_command( + "demo", + Path::new(program), + args, + &BTreeMap::new(), + dir.path(), + true, + Duration::from_millis(500), + ) + .expect_err("a child past its deadline fails the check"); + assert!( + format!("{error}").contains("demo did not finish within 500ms and was killed"), + "{error}" + ); + assert!( + started.elapsed() < Duration::from_secs(30), + "the gate waited for the child instead of killing it" + ); + } + + /// The child is killed, not merely abandoned: `exec` keeps the shell's pid + /// for `sleep`, so the pid the child wrote is the process the deadline must + /// end, and it must be gone when the check fails. What it wrote before the + /// deadline is in the failure. + #[cfg(unix)] + #[test] + fn run_command_leaves_no_child_running_after_its_deadline() { + let dir = tempfile::tempdir().expect("tempdir"); + let error = run_command( + "demo", + Path::new("sh"), + &["-c", "echo $$ > pid; echo waiting; exec sleep 60"], + &BTreeMap::new(), + dir.path(), + false, + Duration::from_millis(500), + ) + .expect_err("a child past its deadline fails the check"); + assert!( + format!("{error}").contains("it had written:\nwaiting"), + "{error}" + ); + let pid = std::fs::read_to_string(dir.path().join("pid")).expect("the child wrote its pid"); + let alive = Command::new("kill") + .args(["-0", pid.trim()]) + .stderr(Stdio::null()) + .status() + .expect("run kill -0") + .success(); + assert!(!alive, "process {} outlived its deadline", pid.trim()); + } + + /// A child that exits but leaves a process holding its output open must + /// not hold the gate either. + #[cfg(unix)] + #[test] + fn run_command_does_not_wait_for_a_process_left_holding_the_output() { + let dir = tempfile::tempdir().expect("tempdir"); + let started = Instant::now(); + let error = run_command( + "demo", + Path::new("sh"), + &["-c", "sleep 60 & echo started"], + &BTreeMap::new(), + dir.path(), + false, + Duration::from_millis(500), + ) + .expect_err("output still open at the deadline fails the check"); + assert!( + format!("{error}") + .contains("demo exited, but a process it started still held its output open"), + "{error}" + ); + assert!(started.elapsed() < Duration::from_secs(30)); + } + + /// The direct form of the closed-stdin check above, which does not depend + /// on what stdin the test harness itself was given: the child's fd 0 must + /// be `/dev/null`, for the smoked commands and for the build alike. + #[cfg(target_os = "linux")] + #[test] + fn every_child_and_the_build_get_dev_null_as_stdin() { + let dir = tempfile::tempdir().expect("tempdir"); + let probe = r#"[ "$(readlink /proc/self/fd/0)" = /dev/null ]"#; + run_command( + "demo", + Path::new("sh"), + &["-c", probe], + &BTreeMap::new(), + dir.path(), + false, + TEST_DEADLINE, + ) + .expect("a smoked command's stdin is /dev/null"); + run_build("build demo", Path::new("sh"), &["-c", probe], dir.path()) + .expect("the build's stdin is /dev/null"); + } + + #[test] + fn run_build_names_the_build_and_its_status_when_it_fails() { + let dir = tempfile::tempdir().expect("tempdir"); + let (program, args): (&str, &[&str]) = if cfg!(windows) { + ("cmd", &["/C", "exit 3"]) + } else { + ("sh", &["-c", "exit 3"]) + }; + let error = run_build("build demo", Path::new(program), args, dir.path()) + .expect_err("the build fails"); + assert!( + format!("{error}").contains("build demo exited with status 3"), + "{error}" + ); + let (program, args): (&str, &[&str]) = if cfg!(windows) { + ("cmd", &["/C", "exit 0"]) + } else { + ("sh", &["-c", "exit 0"]) + }; + run_build("build demo", Path::new(program), args, dir.path()).expect("the build passes"); + } + + fn run_shell(label: &str, unix: &str, windows: &str, expect_failure: bool) -> Result { + let dir = tempfile::tempdir().expect("tempdir"); + run_script( + label, + unix, + windows, + &BTreeMap::new(), + dir.path(), + expect_failure, + ) + } + + /// Lines with `cmd`'s `\r` and trailing spaces removed, so both shells' + /// output compares the same way. + fn lines_of(output: &str) -> Vec { + output.lines().map(|line| line.trim().to_string()).collect() + } + + #[test] + fn run_command_returns_the_trimmed_output_of_a_successful_child() { + let output = + run_shell("demo", "echo hello", "echo hello", false).expect("the child succeeds"); + assert_eq!(output, "hello"); + } + + /// Both streams reach the assertions, stdout first. The script merged them + /// into one pipe; this concatenates them, so a check asserting on stderr must + /// still see it. Exact lines, not substrings: under `cmd` a mis-quoted script + /// echoes its own text, which would satisfy a substring test by accident. + #[test] + fn run_command_captures_stderr_after_stdout() { + let output = run_shell( + "demo", + "echo to-stdout; echo to-stderr 1>&2", + "echo to-stdout& 1>&2 echo to-stderr", + false, + ) + .expect("the child succeeds"); + assert_eq!(lines_of(&output), vec!["to-stdout", "to-stderr"]); + } + + /// The failure names the check and the status, so a red gate says which + /// command broke without the reader going to the transcript. + #[test] + fn run_command_fails_when_an_expected_success_exits_non_zero() { + let error = run_shell("demo", "exit 7", "exit 7", false).expect_err("the child fails"); + let message = format!("{error}"); + assert!(message.contains("demo exited with status 7"), "{message}"); + } + + #[test] + fn run_command_accepts_a_failure_it_was_told_to_expect() { + let output = run_shell( + "demo", + "echo refused 1>&2; exit 1", + "1>&2 echo refused& exit 1", + true, + ) + .expect("a failure was expected"); + assert_eq!(lines_of(&output), vec!["refused"]); + } + + /// The inverse, and the one that matters: a rejection check whose command + /// starts succeeding must fail the gate rather than pass quietly. + #[test] + fn run_command_fails_when_an_expected_failure_succeeds() { + let error = + run_shell("demo", "exit 0", "exit 0", true).expect_err("success is the failure here"); + assert!( + format!("{error}").contains("demo unexpectedly succeeded"), + "{error}" + ); + } + + /// Children are handed a null stdin, so a command that reads to EOF returns + /// instead of hanging the gate — the contract `rocm chat` relies on. + /// + /// This only has teeth where the test process's own stdin is an open pipe or + /// a terminal, as under a local `cargo test`. Where the harness already gives + /// tests a null stdin, an inherited one would also read EOF and this passes + /// either way; nothing in `std::process::Command` exposes the configured + /// stdin for a direct assertion. + #[test] + fn run_command_hands_the_child_a_closed_stdin() { + let output = run_shell("demo", "cat; echo done", "sort& echo done", false) + .expect("the reader sees EOF at once"); + assert_eq!(lines_of(&output), vec!["done"]); + } + + /// The isolated environment and the working directory both reach the child. + /// The directory is proved by a file only it contains, not by comparing + /// paths: a Windows temp directory can come back in 8.3 short form. + #[test] + fn run_command_passes_the_env_and_working_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join("marker-file"), b"").expect("marker"); + let mut env = BTreeMap::new(); + env.insert("SMOKE_PROBE", OsString::from("from-env")); + + let output = run_script( + "demo", + r#"echo "$SMOKE_PROBE" && ls marker-file"#, + "echo %SMOKE_PROBE%& dir /b marker-file", + &env, + dir.path(), + false, + ) + .expect("the probe and the marker are both visible"); + assert_eq!(lines_of(&output), vec!["from-env", "marker-file"]); + } + + /// The JSON branch of `assert_output` is reached through a real check, not + /// only by calling `JsonCheck::assert_value` directly. + #[test] + fn assert_output_runs_the_json_check_it_carries() { + let check = Check::new("demo", Bin::Rocmd, &["bridge-snapshot"]) + .json(JsonCheck::Protocol("rocmd-codex-bridge-v0")); + + check + .assert_output(r#"{"protocol":"rocmd-codex-bridge-v0"}"#) + .expect("the protocol matches"); + + let wrong = check + .assert_output(r#"{"protocol":"other"}"#) + .expect_err("the protocol does not match"); + assert!( + format!("{wrong}").contains("unexpected bridge snapshot protocol"), + "{wrong}" + ); + + let unparsable = check + .assert_output("not json") + .expect_err("not JSON at all"); + assert!( + format!("{unparsable}").contains("demo did not return valid JSON"), + "{unparsable}" + ); + } + + /// [`Bin::stem`] names both the binary the gate looks for and the cargo + /// package the release build passes to `-p`. Hold the two together: each + /// stem must be a workspace package that builds a binary of that same + /// name, so a package or binary rename fails here rather than only under + /// `--profile release`, which no CI job runs. + #[test] + fn every_smoked_binary_is_built_by_the_package_its_stem_names() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask sits in the workspace root"); + let read = |path: &Path| -> toml::Table { + let text = std::fs::read_to_string(path) + .unwrap_or_else(|error| panic!("reading {}: {error}", path.display())); + text.parse() + .unwrap_or_else(|error| panic!("parsing {}: {error}", path.display())) + }; + let workspace = read(&root.join("Cargo.toml")); + let members: Vec<&str> = workspace["workspace"]["members"] + .as_array() + .expect("workspace members") + .iter() + .map(|member| member.as_str().expect("a member is a path")) + .collect(); + for bin in Bin::ALL { + let stem = bin.stem(); + let found: Vec<(&str, toml::Table)> = members + .iter() + .map(|member| (*member, read(&root.join(member).join("Cargo.toml")))) + .filter(|(_, manifest)| manifest["package"]["name"].as_str() == Some(stem)) + .collect(); + assert_eq!( + found.len(), + 1, + "no single workspace package is named {stem}" + ); + let (member, manifest) = &found[0]; + let bins: Vec<&str> = manifest + .get("bin") + .and_then(toml::Value::as_array) + .map(|bins| { + bins.iter() + .filter_map(|bin| bin.get("name").and_then(toml::Value::as_str)) + .collect() + }) + .unwrap_or_default(); + let builds_stem = if bins.is_empty() { + root.join(member).join("src/main.rs").is_file() + } else { + bins.contains(&stem) + }; + assert!( + builds_stem, + "package {stem} ({member}) builds no binary named {stem}" + ); + } + } + + /// `--profile release` must build the profile it then smokes, and a + /// `--target-dir` must reach the build too: otherwise the flags build one + /// place and look in another. + #[test] + fn the_build_lands_where_the_binaries_are_looked_for() { + assert_eq!( + build_args(Profile::Debug, None), + vec!["build", "--workspace", "--all-targets"], + "the debug build keeps the script's whole-workspace build" + ); + assert_eq!( + build_args(Profile::Release, None), + vec![ + "build", + "--release", + "-p", + "rocm", + "-p", + "rocmd", + "-p", + "rocm-engine-lemonade", + "-p", + "rocm-engine-vllm", + ], + "the release build names only the binaries the gate runs" + ); + assert!( + !build_args(Profile::Release, None).contains(&"--workspace".to_string()), + "a release --workspace build would relink the running `cargo xtask`" + ); + + // An absolute path from a real directory, so it is absolute on Windows + // too — `/tmp/...` has no drive and is relative there. + let dir = tempfile::tempdir().expect("tempdir"); + let out = dir.path().join("elsewhere"); + let args = build_args(Profile::Release, Some(&out)); + let target_flag = args + .iter() + .position(|arg| arg == "--target-dir") + .expect("the build is told where to put the binaries"); + assert_eq!(args[target_flag + 1], out.display().to_string()); + + assert_eq!(binary_dir(&out, Profile::Release), out.join("release")); + } + + /// A relative `--target-dir` is resolved once, against the workspace root, + /// so the build and the lookup are given the same path whatever directory + /// cargo happens to run in. + #[test] + fn a_relative_target_dir_resolves_against_the_workspace_root() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path().join("repo"); + let absolute = dir.path().join("abs"); + + assert_eq!( + resolve_target_dir(&root, Path::new("rel")), + root.join("rel") + ); + assert_eq!( + resolve_target_dir(&root, &absolute), + absolute, + "an absolute path is used as given" + ); + + let resolved = Some(resolve_target_dir(&root, Path::new("rel"))); + let args = build_args(Profile::Debug, resolved.as_deref()); + assert_eq!( + args.last().map(String::as_str), + Some(root.join("rel").display().to_string().as_str()), + "the build receives the resolved path, not the relative one" + ); + let plan = Plan::new( + &root, + &dir.path().join("default"), + Profile::Debug, + true, + Some(Path::new("rel")), + ); + assert_eq!(plan.binaries.dir, root.join("rel").join("debug")); + } + + /// The first-run assertions look where the CLI was told to write. + #[test] + fn the_first_run_checks_look_in_the_isolated_state_dirs() { + let root = Path::new("/tmp/smoke-local"); + let env = isolated_env(root); + let (cache_dir, data_dir) = state_dirs(&env); + assert_eq!(cache_dir, PathBuf::from(&env["ROCM_CLI_CACHE_DIR"])); + assert_eq!(data_dir, PathBuf::from(&env["ROCM_CLI_DATA_DIR"])); + assert!(cache_dir.starts_with(root) && data_dir.starts_with(root)); + } + + const FAKE_VERSION: &str = "rocm-cli 0.0.0-dev (main, a1b2c3d)"; + const FAKE_PORT: &str = "45678"; + + /// Output that satisfies a check's own expectations, built from the check — + /// so a test can make one check fail by overriding only that one. + fn satisfying_output(check: &Check) -> String { + if let Some(json) = check.json { + return match json { + JsonCheck::Parses => "{}".to_string(), + JsonCheck::Protocol(protocol) => format!(r#"{{"protocol":"{protocol}"}}"#), + JsonCheck::SandboxTool { tool, .. } => { + format!(r#"{{"tool":"{tool}","ok":true}}"#) + } + JsonCheck::VllmCapabilities => { + r#"{"openai_compatible":true,"cpu":false}"#.to_string() + } + }; + } + let mut lines: Vec<&str> = check.contains.clone(); + lines.extend(check.contains_any.first()); + lines.join("\n") + } + + /// One recorded call: label, program file name, argv, `expect_failure`. + type Call = (String, String, Vec, bool); + + /// A plan over a throwaway tree, with the four binaries present as empty + /// files so `execute` can locate them. + fn fake_plan(dir: &Path) -> Plan { + let target = dir.join("target-dir"); + let bin_dir = target.join("debug"); + std::fs::create_dir_all(&bin_dir).expect("bin dir"); + for bin in Bin::ALL { + std::fs::write(bin_dir.join(binary_name(bin.stem())), b"").expect("stub binary"); + } + Plan::new( + &dir.join("repo"), + &dir.join("default-target"), + Profile::Debug, + true, + Some(&target), + ) + } + + /// Drive `execute` with a fake runner that answers every command with + /// passing output, except where `overrides` says otherwise. `on_call` runs + /// before each answer, so a test can simulate a command's side effects. + fn execute_with( + plan: &Plan, + overrides: &BTreeMap<&str, String>, + on_call: &mut dyn FnMut(&str), + ) -> (Result<()>, Vec) { + let (result, calls, _said) = execute_saying(plan, overrides, on_call); + (result, calls) + } + + /// [`execute_with`], also returning the lines `execute` reported. + fn execute_saying( + plan: &Plan, + overrides: &BTreeMap<&str, String>, + on_call: &mut dyn FnMut(&str), + ) -> (Result<()>, Vec, Vec) { + let table = checks(); + let mut calls = Vec::new(); + let mut said = Vec::new(); + let result = execute( + plan, + FAKE_PORT, + &mut |label, program, argv, expect_failure| { + on_call(label); + calls.push(( + label.to_string(), + program + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(), + argv.iter().map(|arg| (*arg).to_string()).collect(), + expect_failure, + )); + if let Some(output) = overrides.get(label) { + return Ok(output.clone()); + } + Ok(match label { + "rocm --version" | "rocm -V" => FAKE_VERSION.to_string(), + "rocm version" => { + format!("{FAKE_VERSION}\nROCm SDK: not detected\nGPU driver: not detected") + } + _ => satisfying_output( + table + .iter() + .find(|check| check.label == label) + .unwrap_or_else(|| panic!("unexpected command {label}")), + ), + }) + }, + &mut |line| said.push(line.to_string()), + ); + (result, calls, said) + } + + /// The run reports which binaries and state root it uses, once, through + /// the same channel as the rest of its output. + #[test] + fn execute_reports_the_binaries_and_state_root_it_uses() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + let (result, _calls, said) = execute_saying(&plan, &BTreeMap::new(), &mut |_| {}); + result.expect("every check is satisfied"); + assert_eq!(said, vec![run_header(&plan)]); + } + + /// The gate runs the version surfaces, then every table check in table + /// order, each against its own binary, with the port resolved and its + /// expected exit status passed through. + #[test] + fn execute_runs_every_check_in_order_against_the_right_binary() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + let (result, calls) = execute_with(&plan, &BTreeMap::new(), &mut |_| {}); + result.expect("every check is satisfied"); + + let table = checks(); + let mut expected: Vec = vec![ + ( + "rocm --version".into(), + binary_name("rocm"), + vec!["--version".into()], + false, + ), + ( + "rocm -V".into(), + binary_name("rocm"), + vec!["-V".into()], + false, + ), + ( + "rocm version".into(), + binary_name("rocm"), + vec!["version".into()], + false, + ), + ]; + expected.extend(table.iter().map(|check| { + ( + check.label.to_string(), + binary_name(check.bin.stem()), + check.resolved_args(FAKE_PORT), + check.expect_failure, + ) + })); + assert_eq!(calls, expected); + assert!( + calls + .iter() + .any(|(_, _, argv, _)| argv.iter().any(|arg| arg == FAKE_PORT)), + "the serve rejections receive the chosen port" + ); + } + + /// The failure that matters most: a check whose output does not meet its + /// expectations must fail the gate. Without this, a gate that stopped + /// asserting would still print `smoke: ok`. + #[test] + fn execute_fails_a_check_whose_output_misses_its_expectation() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + let overrides = BTreeMap::from([("rocm config show", "telemetry_mode: on".to_string())]); + let (result, _) = execute_with(&plan, &overrides, &mut |_| {}); + let error = result.expect_err("the expectation is unmet"); + assert!( + format!("{error}").contains("rocm config show did not contain expected text"), + "{error}" + ); + } + + #[test] + fn execute_fails_when_the_version_surfaces_disagree() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + + let short = BTreeMap::from([("rocm -V", "rocm-cli 0.1.0 (main, fffffff)".to_string())]); + let (result, _) = execute_with(&plan, &short, &mut |_| {}); + assert!(format!("{}", result.expect_err("-V disagrees")).contains("version flag surfaces"),); + + let long = BTreeMap::from([( + "rocm version", + "rocm-cli 0.1.0 (main, fffffff)\nROCm SDK: not detected\nGPU driver: not detected" + .to_string(), + )]); + let (result, _) = execute_with(&plan, &long, &mut |_| {}); + assert!( + format!("{}", result.expect_err("the build line disagrees")) + .contains("build line does not match"), + ); + } + + /// `rocm version` must carry a traceable build string and report the SDK + /// and driver. Each override keeps the three version surfaces equal, so the + /// earlier equality checks pass and only the assertion under test can fail. + #[test] + fn execute_checks_what_rocm_version_reports() { + let unknown = "rocm-cli 0.1.0 (unknown, a1b2c3d)"; + let cases = [ + ( + unknown, + format!("{unknown}\nROCm SDK: not detected\nGPU driver: not detected"), + "unresolved ref", + ), + ( + FAKE_VERSION, + format!("{FAKE_VERSION}\nGPU driver: not detected"), + "did not contain expected text: ROCm SDK:", + ), + ( + FAKE_VERSION, + format!("{FAKE_VERSION}\nROCm SDK: not detected"), + "did not contain expected text: GPU driver:", + ), + ]; + for (flag, version, expected) in cases { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + let overrides = BTreeMap::from([ + ("rocm --version", flag.to_string()), + ("rocm -V", flag.to_string()), + ("rocm version", version), + ]); + let (result, _) = execute_with(&plan, &overrides, &mut |_| {}); + let error = result.expect_err("the version report is incomplete"); + assert!(format!("{error}").contains(expected), "{error}"); + } + } + + /// A command that provisions a uv cache or a runtime registry fails the + /// gate. The paths are written here independently of the production code, + /// so a typo in either subpath there leaves the check looking somewhere + /// nothing is created, and this test catches it. + #[test] + fn execute_fails_when_a_command_provisions_state() { + for (subpath, expected) in [ + (&["uv"][..], "first-run smoke uv cache"), + ( + &["runtimes", "registry"][..], + "first-run smoke runtime registry", + ), + ] { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + let parent = if subpath[0] == "uv" { + PathBuf::from(&plan.env["ROCM_CLI_CACHE_DIR"]) + } else { + PathBuf::from(&plan.env["ROCM_CLI_DATA_DIR"]) + }; + let provisioned = subpath.iter().fold(parent, |path, part| path.join(part)); + let (result, _) = execute_with(&plan, &BTreeMap::new(), &mut |label| { + if label == "rocm examine" { + std::fs::create_dir_all(&provisioned).expect("simulate provisioning"); + } + }); + let error = result.expect_err("provisioned state is a failure"); + assert!(format!("{error}").contains(expected), "{error}"); + } + } + + /// The state root is wiped before any command runs, so leftovers from a + /// previous run — including a uv cache — neither leak in nor fail the gate. + #[test] + fn execute_starts_from_an_empty_state_root() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + let stale_cache = PathBuf::from(&plan.env["ROCM_CLI_CACHE_DIR"]).join("uv"); + std::fs::create_dir_all(&stale_cache).expect("stale cache"); + let stale_file = plan.env_root.join("stale"); + std::fs::write(&stale_file, b"").expect("stale file"); + + let mut seen_stale = false; + let (result, _) = execute_with(&plan, &BTreeMap::new(), &mut |_| { + seen_stale |= stale_file.exists(); + }); + result.expect("a previous run's leftovers are cleared, not reported"); + assert!(!seen_stale, "a command ran before the state root was wiped"); + } + + #[test] + fn execute_refuses_a_partial_build() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + std::fs::remove_file(plan.binaries.path(Bin::Vllm)).expect("remove one binary"); + let (result, calls) = execute_with(&plan, &BTreeMap::new(), &mut |_| {}); + let message = format!("{}", result.expect_err("a binary is missing")); + assert!(message.contains("missing smoke binary vllm"), "{message}"); + // The fake plan skips the build, so the fix is named. + assert!(message.contains("because of --skip-build"), "{message}"); + assert!(calls.is_empty(), "nothing runs against a partial build"); + } + + /// Every child runs with the plan's isolated environment and in the + /// workspace root. This is what keeps the gate off the developer's real + /// state, and running the gate cannot detect its loss — so it is proved here, + /// through a real shell. Values are compared as the strings that were set, + /// not as resolved paths, which Windows can return in 8.3 short form. + #[test] + fn child_runner_isolates_every_child_and_runs_it_in_the_workspace_root() { + let dir = tempfile::tempdir().expect("tempdir"); + let plan = fake_plan(dir.path()); + std::fs::create_dir_all(&plan.root).expect("workspace root"); + std::fs::write(plan.root.join("marker-file"), b"").expect("marker"); + + let (program, flag, script, profile_var) = if cfg!(windows) { + ( + "cmd", + "/C", + "echo %LOCALAPPDATA%& echo %ROCM_CLI_DATA_DIR%& dir /b marker-file", + "LOCALAPPDATA", + ) + } else { + ( + "sh", + "-c", + r#"echo "$HOME"; echo "$ROCM_CLI_DATA_DIR"; ls marker-file"#, + "HOME", + ) + }; + let output = child_runner(&plan)("probe", Path::new(program), &[flag, script], false) + .expect("the probe runs in the workspace root"); + + let expected = vec![ + plan.env[profile_var].to_string_lossy().into_owned(), + plan.env["ROCM_CLI_DATA_DIR"].to_string_lossy().into_owned(), + "marker-file".to_string(), + ]; + assert_eq!(lines_of(&output), expected); + assert!( + PathBuf::from(&expected[0]).starts_with(&plan.env_root), + "the profile directory the child saw is the throwaway one" + ); + } + + /// The plan builds the profile it then smokes, in the directory it then + /// looks in, and skips the build only when asked. + #[test] + fn the_plan_builds_and_looks_in_the_same_place() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path().join("repo"); + + let default = dir.path().join("default-target"); + let release = Plan::new( + &root, + &default, + Profile::Release, + false, + Some(Path::new("rel")), + ); + assert_eq!( + release.build, + Some(build_args(Profile::Release, Some(&root.join("rel")))) + ); + assert!( + release + .build + .as_ref() + .is_some_and(|args| args.contains(&"--release".to_string())) + ); + assert_eq!(release.binaries.dir, root.join("rel").join("release")); + + let skipped = Plan::new(&root, &default, Profile::Release, true, None); + assert_eq!(skipped.build, None); + + // Without `--target-dir`, the default target dir is used for both, and + // the build is still told it explicitly, so cargo config cannot send + // the binaries somewhere the lookup does not search. + let defaulted = Plan::new(&root, &default, Profile::Debug, false, None); + assert_eq!(defaulted.binaries.dir, default.join("debug")); + assert_eq!( + defaulted.build, + Some(build_args(Profile::Debug, Some(&default))) + ); + + assert_eq!(release.env_root, root.join("target").join("smoke-local")); + assert!(PathBuf::from(&release.env["ROCM_CLI_DATA_DIR"]).starts_with(&release.env_root)); + } + + #[test] + fn the_build_label_names_what_is_built() { + assert_eq!(build_label(Profile::Debug), "build workspace all targets"); + assert_eq!(build_label(Profile::Release), "build release binaries"); + } + + #[test] + fn a_free_port_is_released_and_outside_the_reserved_range() { + // The serve rejections need the port free once it is handed to them: + // a listener still held here would make them fail on the port rather + // than on the refusal they exist to check. Another test running in + // parallel can take a released port before it is bound again, so one + // successful re-bind out of a few attempts is the pass; a port that is + // never released fails every attempt. + let mut failures = Vec::new(); + for _ in 0..5 { + let port = free_tcp_port().expect("an ephemeral port"); + assert!(port > 1023, "{port} is in the reserved range"); + match TcpListener::bind(("127.0.0.1", port)) { + Ok(_) => return, + Err(error) => failures.push(format!("{port}: {error}")), + } + } + panic!("no returned port could be bound again: {failures:?}"); + } + + #[test] + fn path_missing_rejects_a_path_that_exists() { + let dir = tempfile::tempdir().expect("tempdir"); + assert_path_missing(&dir.path().join("absent"), "first-run smoke uv cache") + .expect("an absent path passes"); + + let error = assert_path_missing(dir.path(), "first-run smoke uv cache") + .expect_err("an existing path fails"); + assert!( + format!("{error}").contains("first-run smoke uv cache unexpectedly exists"), + "{error}" + ); + } +} diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 219379b771..752cf97a76 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -15,17 +15,67 @@ //! //! Design notes: //! -//! - **Dormant by default.** Until a canonical `docs/keys/` file exists and is -//! non-empty, its key is skipped. So this passes as a no-op today (empty pinned -//! sentinels, no canonical files) and only starts enforcing once real keys are -//! published. +//! - **What it guarantees.** This catches a pinned key that drifts from +//! `docs/keys/` by mistake or slips past review: a stale or commented-out copy, +//! a second assignment, a key pasted into the wrong slot. It is a textual check +//! of plain installer code, not an interpreter, and does not defend against +//! deliberately obfuscated code (a name assembled at runtime through `eval` or +//! `Invoke-Expression`, an assignment hidden inside a multi-line string, a +//! PowerShell `<# … #>` block comment or a block that never runs, a PowerShell +//! multiple assignment with the name first (`$Name, $x = …`), `foreach` or +//! `[ref]`). Anyone who can commit that can also change +//! `docs/keys/` in the same commit, so review is the control there. +//! - **Dormant slots stay empty.** Until a canonical `docs/keys/` file exists and +//! is non-empty, its slot is not compared, but it must be exactly `""`: the +//! installers trust any non-empty slot, so a key pasted into one with nothing +//! published would be trusted with nothing to compare it against. //! - **Per-constant equality.** Each pinned key is embedded in a specific named //! constant (shell `NAME="…"`, PowerShell string/here-string, Rust `const`). We //! isolate *that constant's own value span*, reduce it to its base64 body, and //! compare it for equality to the canonical key. Comparing the whole file (rather //! than the specific constant) would let a wrong/tampered constant pass whenever //! the canonical body appears anywhere else in the file — e.g. in the `NEXT` slot -//! mid-rotation or a stale comment. +//! mid-rotation or a stale comment. The value must be one well-formed PEM block +//! (the BEGIN line, non-empty lines of base64, the END line, nothing else), +//! free of `$`, backticks and backslashes (Rust may write line breaks as `\n`), +//! and the statement must end with the string (only `;` or a comment may +//! follow), so the text compared is the text the installer ends up holding. +//! - **The assignment the runtime uses.** The shell and PowerShell keep the *last* +//! value they assign, so reading the first textual occurrence of the name could +//! compare a commented-out line, a longer identifier sharing the prefix, or a +//! value later overwritten, while the installer trusts another key. So each +//! source must assign the name exactly once, and that assignment must be the +//! plain declaration as its own statement (leading whitespace allowed): shell +//! `NAME="…"` (optionally after `export`/`readonly`), PowerShell `$Name = "…"` +//! or `$Name = @"…"@` (any case, optionally `$script:`, `$global:`, `$local:` +//! or `$private:`; `$env:Name` is an environment variable, so it never counts +//! as the declaration, only as a write), Rust `const NAME: &str = "…";` +//! (optionally `pub` or `pub(crate)`). Names match as whole identifiers, case-insensitively for +//! PowerShell, and whole-line comments are skipped. Every other write of the +//! name counts towards "exactly once": shell `NAME=` anywhere on a line, +//! `NAME+=`, `${NAME:=…}`, and any other mention of the name, quoted or not, +//! except a `readonly`/`export` command that only names variables (this is how +//! `unset` and `read` name it); PowerShell plain and compound assignment to any +//! reference spelling (`$Name`, `$scope:Name`, `${Name}`), and any mention of +//! the name outside a `$` reference, quoted or not (how `Set-Variable`, +//! `Remove-Variable` and `Set-Item variable:` name it); Rust +//! `const`/`static`/`let` bindings in any scope. Over-counting only fails the +//! check, so prose naming the variable outside a comment line is rejected +//! rather than guessed at. +//! - **Read-only at runtime.** `install.sh` must mark each slot `readonly` after +//! assigning it, so a later write fails instead of silently replacing the trust +//! root. The `readonly` command, or the `readonly NAME="…"` declaration, must +//! start its line (an indented one sits in a block or function that may never +//! run), and a separate command must name the slot itself, with no flags +//! (`-f` names a function, `-p` only prints) and not in a trailing comment. +//! A textual check cannot tell whether an unindented `readonly` line runs, so +//! one inside an `if false` block or an uncalled function whose body is not +//! indented still counts; review is the control there, as above. +//! +//! `install.ps1` does not mark its slots read-only: `irm … | iex` runs it in +//! the caller's session, where a read-only variable would outlive the run and +//! make the next install in that session fail (`-Force` would avoid that, but +//! also defeats the point). //! - **CI cross-check.** When a CI signing public key is configured — the file //! named by `ROCM_CLI_SIGNING_PUBLIC_KEY_PATH`, else the inline //! `ROCM_CLI_SIGNING_PUBLIC_KEY_PEM` release/nightly CI wires from the secret — @@ -39,7 +89,7 @@ use std::ffi::OsString; use std::path::{Path, PathBuf}; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, anyhow, bail}; const PEM_BEGIN: &str = "-----BEGIN PUBLIC KEY-----"; const PEM_END: &str = "-----END PUBLIC KEY-----"; @@ -156,48 +206,447 @@ fn pem_body(text: &str) -> Option { if body.is_empty() { None } else { Some(body) } } -/// Isolate the raw text span assigned to `token` under `embedding`, or `None` if the -/// assignment is not found. PEM payloads contain no `"`, so the first quote after the -/// assignment reliably closes the value. -fn extract_constant<'a>(source: &'a str, token: &str, embedding: Embedding) -> Option<&'a str> { - let key = source.find(token)?; - let after = &source[key + token.len()..]; +impl Embedding { + /// What a whole-line comment starts with in this language. + const fn comment_prefix(self) -> &'static str { + match self { + Self::Shell | Self::PowerShell => "#", + Self::Rust => "//", + } + } + + /// Why a second write defeats the comparison, for error messages. + const fn why_one(self) -> &'static str { + match self { + Self::Shell => "the shell keeps the last value it assigns", + Self::PowerShell => "PowerShell keeps the last value it assigns", + Self::Rust => "another binding can shadow the constant that is compared", + } + } + + /// The one assignment shape the value can be read from, for error messages. + fn supported_form(self, name: &str) -> String { + match self { + Self::Shell => format!("`{name}=\"…\"` as its own statement"), + Self::PowerShell => { + format!("`${name} = \"…\"` or `${name} = @\"…\"@` as its own statement") + } + Self::Rust => format!("`const {name}: &str = \"…\";` as its own statement"), + } + } +} + +/// One occurrence of a pinned name that can set its value. +struct Assignment { + /// 1-based line number, for error messages. + line: usize, + /// Byte offset in the whole source just past the name. + after_name: usize, + /// Whether it is the supported declaration shape whose value can be read. + declaration: bool, +} + +const fn is_ident_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} + +/// The part of a shell line before a trailing comment: a `#` that starts a word, +/// so after any whitespace (a space or a tab), not one inside a word like `a#b`. +fn shell_code(line: &str) -> &str { + line.char_indices() + .find(|&(index, c)| c == '#' && line[..index].ends_with(char::is_whitespace)) + .map_or(line, |(index, _)| &line[..index]) +} + +/// Whether `line` is `readonly`/`export` followed only by names and flags, which +/// sets attributes without changing any value. `export NAME; unset NAME` is not. +fn is_attribute_command(line: &str) -> bool { + let mut words = shell_code(line).split_whitespace(); + matches!(words.next(), Some("readonly" | "export")) + && words.all(|word| { + word.starts_with('-') + || word.bytes().all(is_ident_byte) && !word.as_bytes()[0].is_ascii_digit() + }) +} + +/// Start offsets of `name` in `line` as a whole identifier: a longer identifier +/// that merely starts or ends with `name` is a different variable. +fn identifier_offsets(line: &str, name: &str, ignore_case: bool) -> Vec { + let (haystack, needle) = if ignore_case { + (line.to_ascii_lowercase(), name.to_ascii_lowercase()) + } else { + (line.to_owned(), name.to_owned()) + }; + let bytes = haystack.as_bytes(); + haystack + .match_indices(&needle) + .map(|(start, _)| start) + .filter(|&start| { + let end = start + needle.len(); + (start == 0 || !is_ident_byte(bytes[start - 1])) + && (end == bytes.len() || !is_ident_byte(bytes[end])) + }) + .collect() +} + +/// Classify one whole-identifier occurrence of `name` at `start` in `line`: +/// `None` if it cannot change the value, otherwise whether it is the supported +/// declaration shape. +fn classify(line: &str, start: usize, name: &str, embedding: Embedding) -> Option { + let head = &line[..start]; + let tail = &line[start + name.len()..]; match embedding { Embedding::Shell => { - // `NAME="…"` — the token is immediately followed by `="`. - let open = after.find('"')?; - let rest = &after[open + 1..]; - let close = rest.find('"')?; - Some(&rest[..close]) + // `NAME=`, `NAME+=` (bash) and `${NAME=…}`/`${NAME:=…}` all assign. + if tail.starts_with('=') || tail.starts_with("+=") || tail.starts_with(":=") { + let prefix = head.trim(); + let declaration = tail.starts_with('=') + && (prefix.is_empty() || prefix == "export" || prefix == "readonly"); + return Some(declaration); + } + // `$NAME`, `${NAME}`, `${#NAME}` and `${!NAME}` only read it. + if ["$", "${", "${#", "${!"].iter().any(|r| head.ends_with(r)) { + return None; + } + // `readonly NAME …` / `export NAME …` naming variables only set + // attributes. + if is_attribute_command(line) { + return None; + } + // Any other bare mention is how `unset`, `read`, `getopts` and + // `printf -v` name the variable; treat it as a write and fail closed. + Some(false) + } + Embedding::PowerShell => { + // A variable reference is `$Name`, `$scope:Name`, `${Name}` or + // `${scope:Name}`. Anything else naming the variable — `Set-Variable + // -Name Name`, `New-Variable Name`, `Set-Item variable:Name` — is + // treated as a write: the cmdlet forms are too varied to tell apart, + // and over-counting fails closed. + let mut reference = head; + let mut scope = ""; + if let Some(scoped) = reference.strip_suffix(':') { + reference = + scoped.trim_end_matches(|c: char| c.is_ascii_alphanumeric() || c == '_'); + scope = &scoped[reference.len()..]; + } + let braced = reference.ends_with("${"); + let Some(before) = reference + .strip_suffix("${") + .or_else(|| reference.strip_suffix('$')) + else { + return Some(false); + }; + let mut after = tail; + if braced { + after = after.strip_prefix('}')?; + } + let after = after.trim_start(); + let plain = after.starts_with('=') && !after.starts_with("=="); + let compound = ["+=", "-=", "*=", "/=", "%=", "??="] + .iter() + .any(|op| after.starts_with(op)); + if !(plain || compound) { + return None; + } + // Only a variable scope names this variable, so only it can be the + // declaration. `$env:Name` and other drive-qualified names leave the + // variable unset, but are still counted as a write: a lone one is + // reported as an unsupported form (most likely a mistyped + // declaration), and next to the declaration it fails closed rather + // than being guessed harmless. + let variable_scope = ["", "script", "global", "local", "private"] + .iter() + .any(|s| scope.eq_ignore_ascii_case(s)); + Some(plain && !braced && variable_scope && before.trim().is_empty()) } Embedding::Rust => { - // `const NAME: &str = "…";` — skip to the `=`, then the opening quote. - let eq = after.find('=')?; - let open = after[eq..].find('"')? + eq; - let rest = &after[open + 1..]; - let close = rest.find('"')?; - Some(&rest[..close]) + // `const NAME:`, `static NAME:`, `let NAME =` / `let NAME:` (a shadowing + // binding) all introduce a value under the name; a path `NAME::`, + // comparison `NAME ==` or match arm `NAME =>` does not. + let after = tail.trim_start(); + let binds = (after.starts_with(':') && !after.starts_with("::")) + || (after.starts_with('=') && !after.starts_with("==") && !after.starts_with("=>")); + if !binds { + return None; + } + let words: Vec<&str> = head.split_whitespace().collect(); + let declaration = after.starts_with(':') + && matches!( + words.as_slice(), + ["const"] | ["pub" | "pub(crate)", "const"] + ); + Some(declaration) } - Embedding::PowerShell => { - let here = after.find("@\""); - let quote = after.find('"'); - match (here, quote) { - // Here-string `= @"…"@`: the `@"` quote is the first quote seen. - (Some(h), Some(q)) if q == h + 1 => { - let rest = &after[h + 2..]; - let close = rest.find("\"@")?; - Some(&rest[..close]) - } - // Plain `= "…"`. - (_, Some(q)) => { - let rest = &after[q + 1..]; - let close = rest.find('"')?; - Some(&rest[..close]) + } +} + +/// Whether the occurrence of a name at `start` in `line` is named by a `readonly` +/// command that makes it read-only: at the start of the line (an indented one +/// sits in a block or function that may never run), no flags (`-f` names a +/// function, `-p` only prints), and the name in the command itself rather than +/// its trailing comment. +fn marks_readonly(line: &str, start: usize) -> bool { + let code = shell_code(line); + let mut words = code.split_whitespace(); + // `starts_with` is the start-of-line test; the word check below alone would + // also accept an indented command. + code.starts_with("readonly") + && is_attribute_command(line) + && words.next() == Some("readonly") + && words.all(|word| !word.starts_with('-')) + && start < code.len() +} + +/// Every occurrence of `name` in `source` that can set its value, skipping +/// whole-line comments, plus the lines of `readonly` commands naming it. +/// PowerShell names are case-insensitive. +fn find_assignments( + source: &str, + name: &str, + embedding: Embedding, +) -> (Vec, Vec) { + let ignore_case = matches!(embedding, Embedding::PowerShell); + let mut found = Vec::new(); + let mut readonly_lines = Vec::new(); + let mut offset = 0; + for (index, line) in source.split_inclusive('\n').enumerate() { + if !line.trim_start().starts_with(embedding.comment_prefix()) { + for start in identifier_offsets(line, name, ignore_case) { + match classify(line, start, name, embedding) { + Some(declaration) => found.push(Assignment { + line: index + 1, + after_name: offset + start + name.len(), + declaration, + }), + None if matches!(embedding, Embedding::Shell) + && marks_readonly(line, start) => + { + readonly_lines.push(index + 1); + } + None => {} } - _ => None, } } + offset += line.len(); } + (found, readonly_lines) +} + +/// The single declaration of a pinned constant and the raw text it assigns. +struct Constant<'a> { + value: &'a str, + /// 1-based line of the declaration. + line: usize, + /// Whether the declaration is `readonly NAME=…` or a later `readonly` command + /// names it. Only meaningful for the shell. + readonly: bool, +} + +/// Isolate the raw text span assigned to `token` under `embedding`. +/// +/// The value is read from the *single* assignment of the name, because that is +/// the one the runtime uses. A first-occurrence search read whatever came first — +/// a commented-out line, a longer identifier sharing the prefix, or an assignment +/// later overwritten — while the shell and PowerShell keep the last value they +/// execute. So zero or several assignments is an error, as is a single one in a +/// shape the value cannot be read from. PEM payloads contain no `"`, so the first +/// quote after the opening one reliably closes the value. +fn extract_constant<'a>( + source: &'a str, + token: &str, + embedding: Embedding, +) -> std::result::Result, String> { + let name = match embedding { + Embedding::PowerShell => token.strip_prefix('$').unwrap_or(token), + Embedding::Shell | Embedding::Rust => token, + }; + let (assignments, readonly_lines) = find_assignments(source, name, embedding); + let assignment = match assignments.as_slice() { + [] => return Err(format!("has no assignment of {token}")), + [one] => one, + many => { + let lines = many + .iter() + .map(|a| a.line.to_string()) + .collect::>() + .join(", "); + let declarations: Vec = many + .iter() + .filter(|a| a.declaration) + .map(|a| a.line) + .collect(); + let fix = match declarations.as_slice() { + [line] => format!( + "keep the declaration on line {line} and remove or reword the \ + other lines{}", + match embedding { + Embedding::Rust => "", + Embedding::Shell | Embedding::PowerShell => { + " (any mention of the name outside a variable reference \ + counts as a write)" + } + } + ), + _ => format!("leave exactly one, as {}", embedding.supported_form(name)), + }; + return Err(format!( + "assigns {token} {} times (lines {lines}); {}, so exactly one \ + assignment is allowed: {fix}", + many.len(), + embedding.why_one() + )); + } + }; + if !assignment.declaration { + return Err(format!( + "assigns {token} on line {} in an unsupported form; expected {}", + assignment.line, + embedding.supported_form(name) + )); + } + + let after = &source[assignment.after_name..]; + let until_quote = |value: &'a str| value.find('"').map(|close| &value[..close]); + let unreadable = || { + format!( + "assigns {token} on line {} to a value that is not a double-quoted string", + assignment.line + ) + }; + // The value and the length of the delimiter that closes it: `"@` only for a + // here-string that opened with `@"`, a plain `"` everywhere else. + let plain = |value: &'a str| (value, 1); + let value = match embedding { + // `NAME="…"` — the quote follows the `=` directly. + Embedding::Shell => after.strip_prefix("=\"").and_then(until_quote).map(plain), + // `const NAME: &str = "…";` — skip the type to the `=`, then the quote. + Embedding::Rust => after + .find('=') + .and_then(|eq| after[eq + 1..].trim_start().strip_prefix('"')) + .and_then(until_quote) + .map(plain), + // `$Name = @"…"@` (here-string) or `$Name = "…"`. + Embedding::PowerShell => after + .trim_start() + .strip_prefix('=') + .map(str::trim_start) + .and_then(|rhs| match rhs.strip_prefix("@\"") { + Some(body) => body.find("\"@").map(|close| (&body[..close], 2)), + None => rhs.strip_prefix('"').and_then(until_quote).map(plain), + }), + }; + let (value, closer) = value.ok_or_else(unreadable)?; + + // The statement must end with the string: anything after it (`-replace …`, + // `+ "…"`, a second quoted word, a command) would change the value the + // runtime ends up with, so the text compared here would not be it. + let value_end = value.as_ptr() as usize - source.as_ptr() as usize + value.len(); + let line_rest = source[value_end + closer..].lines().next().unwrap_or(""); + let semicolon = line_rest.trim_start().strip_prefix(';'); + let after_semicolon = semicolon.unwrap_or(line_rest); + let rest = after_semicolon.trim(); + // A shell or PowerShell comment starts a word: `""#x` is the string `#x`, + // not an empty string and a comment, while `"" #x` and `"";#x` are. + let comment = rest.starts_with(embedding.comment_prefix()) + && (matches!(embedding, Embedding::Rust) + || semicolon.is_some() + || after_semicolon.starts_with(char::is_whitespace)); + if !(rest.is_empty() || comment) { + return Err(format!( + "assigns {token} on line {} to a string followed by `{rest}`; the statement \ + must end with the string, so the value compared is the value used", + assignment.line + )); + } + + let line_start = source[..assignment.after_name] + .rfind('\n') + .map_or(0, |n| n + 1); + Ok(Constant { + value, + line: assignment.line, + // Same start-of-line rule as a separate `readonly` command: an indented + // `readonly NAME="…"` may sit in a block that never runs. + readonly: source[line_start..].starts_with("readonly") + || readonly_lines.iter().any(|&line| line > assignment.line), + }) +} + +/// Reduce a pinned constant's raw value to the base64 body of its key, or `None` +/// for an empty slot (`""` exactly: the shell tests a slot with `[ -n … ]`, so a +/// lone space would count as a key there). +/// +/// The value must be one well-formed PUBLIC KEY PEM block and nothing else: the +/// BEGIN line, non-empty lines of base64, the END line. Comparing a looser +/// reading would pass a key that `openssl` cannot load (stray punctuation, a +/// blank line in the body) or, with a second key prepended through `${…}`, one +/// it loads instead. Outside PowerShell, whose installer trims each line, a line +/// may not start with whitespace either. `$`, backticks and backslashes are +/// refused in the installers, +/// which would expand them or keep them verbatim; Rust may write line breaks as +/// `\n`, since a Rust string literal turns them into real ones. +fn pinned_pem_body( + value: &str, + embedding: Embedding, +) -> std::result::Result, String> { + let text = match embedding { + Embedding::Rust => value.replace("\\n", "\n"), + Embedding::Shell | Embedding::PowerShell => value.to_owned(), + }; + if let Some(found) = text.chars().find(|c| matches!(c, '$' | '`' | '\\')) { + return Err(format!( + "holds `{found}`, which the installer would expand or keep verbatim; a \ + pinned key is plain PEM text" + )); + } + if value.is_empty() { + return Ok(None); + } + let malformed = || { + "is not one well-formed PUBLIC KEY PEM block (a BEGIN line, lines of base64, an \ + END line, and nothing else)" + .to_owned() + }; + // install.ps1 trims each line before decoding, so indentation is harmless + // there. install.sh and the Rust source hand the value on verbatim, so outside + // PowerShell a line may not start with whitespace: one rule for both, kept + // stricter than any one PEM reader rather than tracking what each tolerates. + if !matches!(embedding, Embedding::PowerShell) + && text + .lines() + .any(|line| line.starts_with(char::is_whitespace)) + { + return Err( + "has a key line that starts with whitespace; only install.ps1, which trims \ + each line, may indent a pinned key" + .to_owned(), + ); + } + let lines: Vec<&str> = text.lines().map(str::trim).collect(); + // A here-string opens and closes on its own lines; nothing else may be blank. + let start = lines + .iter() + .position(|line| !line.is_empty()) + .ok_or_else(malformed)?; + let end = lines + .iter() + .rposition(|line| !line.is_empty()) + .ok_or_else(malformed)? + + 1; + let [first, body @ .., last] = &lines[start..end] else { + return Err(malformed()); + }; + let base64_line = |line: &&str| { + !line.is_empty() + && line + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '/' | '=')) + }; + if *first != PEM_BEGIN || *last != PEM_END || body.is_empty() || !body.iter().all(base64_line) { + return Err(malformed()); + } + Ok(Some(body.concat())) } /// Check every populated canonical key against its pinned sources. `ci_public_key` @@ -210,24 +659,17 @@ fn check_pinned_keys(root: &Path, ci_public_key: Option<&str>) -> Result text, - _ => { - messages.push(format!( - "{}: no canonical key yet — skipped (dormant)", - entry.name - )); - continue; - } + let canonical_body = match std::fs::read_to_string(&canonical_path) { + Ok(text) if !text.trim().is_empty() => Some(pem_body(&text).with_context(|| { + format!( + "{}: {} is not a valid PUBLIC KEY PEM", + entry.name, entry.canonical + ) + })?), + _ => None, }; - let canonical_body = pem_body(&canonical_text).with_context(|| { - format!( - "{}: {} is not a valid PUBLIC KEY PEM", - entry.name, entry.canonical - ) - })?; if entry.name == CURRENT_RELEASE_KEY { - current_release_body = Some(canonical_body.clone()); + current_release_body.clone_from(&canonical_body); } for source in entry.sources { @@ -235,29 +677,72 @@ fn check_pinned_keys(root: &Path, ci_public_key: Option<&str>) -> Result {} + (Some(_), _) => bail!( + "{}: {} does not embed the canonical key {} in {} (line {}) — the \ + pinned constant and the published key have diverged; copy the key \ + from {} into {}, or, if the published key is the one that is \ + wrong, fix {} and every source that pins it", + entry.name, + source.path, + entry.canonical, + source.token, + constant.line, + entry.canonical, + source.token, + entry.canonical + ), + (None, None) => {} + (None, Some(_)) => bail!( + "{}: {} pins a key in {} on line {}, but {} holds no canonical key; \ + publish the key there or leave the slot empty", + entry.name, + source.path, + source.token, + constant.line, + entry.canonical + ), + } } - messages.push(format!( - "{}: pinned in {}", - entry.name, - entry - .sources - .iter() - .map(|s| s.path) - .collect::>() - .join(", ") - )); + let sources = entry + .sources + .iter() + .map(|s| s.path) + .collect::>() + .join(", "); + messages.push(if canonical_body.is_some() { + format!("{}: pinned in {sources}", entry.name) + } else { + format!( + "{}: no canonical key yet — slots empty in {sources} (dormant)", + entry.name + ) + }); } messages.extend(check_ci_public_key( @@ -424,29 +909,34 @@ mod tests { let shell = "PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"the-current\"\n\ PINNED_RELEASE_PUBLIC_KEY_NEXT=\"the-next\"\n"; assert_eq!( - extract_constant(shell, "PINNED_RELEASE_PUBLIC_KEY_CURRENT", Embedding::Shell), - Some("the-current") + extract_constant(shell, "PINNED_RELEASE_PUBLIC_KEY_CURRENT", Embedding::Shell) + .map(|c| c.value), + Ok("the-current") ); assert_eq!( - extract_constant(shell, "PINNED_RELEASE_PUBLIC_KEY_NEXT", Embedding::Shell), - Some("the-next") + extract_constant(shell, "PINNED_RELEASE_PUBLIC_KEY_NEXT", Embedding::Shell) + .map(|c| c.value), + Ok("the-next") ); let rust = "const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"the-meta\";\n"; assert_eq!( - extract_constant(rust, "PINNED_METADATA_PUBLIC_KEY_PEM", Embedding::Rust), - Some("the-meta") + extract_constant(rust, "PINNED_METADATA_PUBLIC_KEY_PEM", Embedding::Rust) + .map(|c| c.value), + Ok("the-meta") ); let ps = "$PinnedReleasePublicKeyCurrent = @\"\nthe-here\n\"@\n\ $PinnedReleasePublicKeyNext = \"the-plain\"\n"; assert_eq!( - extract_constant(ps, "$PinnedReleasePublicKeyCurrent", Embedding::PowerShell), - Some("\nthe-here\n") + extract_constant(ps, "$PinnedReleasePublicKeyCurrent", Embedding::PowerShell) + .map(|c| c.value), + Ok("\nthe-here\n") ); assert_eq!( - extract_constant(ps, "$PinnedReleasePublicKeyNext", Embedding::PowerShell), - Some("the-plain") + extract_constant(ps, "$PinnedReleasePublicKeyNext", Embedding::PowerShell) + .map(|c| c.value), + Ok("the-plain") ); } @@ -465,11 +955,1120 @@ mod tests { "PINNED_RELEASE_PUBLIC_KEY_CURRENT", Embedding::Shell, ) - .and_then(pem_body) + .ok() + .and_then(|c| pem_body(c.value)) .unwrap(); assert_ne!(current, pem_body(SAMPLE).unwrap()); } + /// A second and third distinct key, used as the canonical next-release and + /// metadata keys. + const NEXT_KEY: &str = "-----BEGIN PUBLIC KEY-----\n\ + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnextnextnextnext+/\n\ + abcDEF0123456789nextnextnextnextnextnextnextnextnextnextABCD==\n\ + -----END PUBLIC KEY-----\n"; + const META: &str = "-----BEGIN PUBLIC KEY-----\n\ + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmetametametameta+/\n\ + abcDEF0123456789metametametametametametametametametametaABCD==\n\ + -----END PUBLIC KEY-----\n"; + + const SH_CURRENT: &str = "PINNED_RELEASE_PUBLIC_KEY_CURRENT"; + const PS_CURRENT: &str = "$PinnedReleasePublicKeyCurrent"; + const RS_META: &str = "PINNED_METADATA_PUBLIC_KEY_PEM"; + + /// The file the check reads `token` from, taken from [`PINNED_KEYS`] so the + /// tests follow the constant when it moves to another file. + fn source_path(token: &str) -> &'static str { + PINNED_KEYS + .iter() + .flat_map(|key| key.sources) + .find(|source| source.token == token) + .map_or_else( + || panic!("{token} is not a pinned source"), + |source| source.path, + ) + } + + /// The Rust file holding the metadata key. + fn rs_path() -> &'static str { + source_path(RS_META) + } + + /// The Rust file's name, as the shorter error fragments name it. + fn rs_file() -> &'static str { + rs_path().rsplit('/').next().expect("a file name") + } + + /// A scratch repository with all three canonical keys published, so every + /// slot is live rather than dormant. + fn scratch_repo(install_sh: &str, install_ps1: &str, therock_rs: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + std::fs::create_dir_all(root.join("docs/keys")).expect("mkdir docs/keys"); + let rust = root.join(rs_path()); + std::fs::create_dir_all(rust.parent().expect("a parent")) + .expect("mkdir the Rust source dir"); + for (file, key) in [ + ("rocm-cli-release-current-public.pem", SAMPLE), + ("rocm-cli-release-next-public.pem", NEXT_KEY), + ("rocm-cli-metadata-public.pem", META), + ] { + std::fs::write(root.join("docs/keys").join(file), key).expect("write canonical key"); + } + std::fs::write(root.join("install.sh"), install_sh).expect("write install.sh"); + std::fs::write(root.join("install.ps1"), install_ps1).expect("write install.ps1"); + std::fs::write(rust, therock_rs).expect("write the Rust source"); + dir + } + + /// A key that differs from every canonical key in its base64 body. + fn wrong() -> String { + SAMPLE.replace("test", "evil") + } + + /// An install.sh whose CURRENT slot is `current` (the caller's lines), with a + /// correct NEXT slot and both marked readonly after them. + fn sh(current: &str) -> String { + format!( + "{current}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n" + ) + } + + fn sh_current() -> String { + format!("PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"{SAMPLE}\"\n") + } + + fn good_sh() -> String { + sh(&sh_current()) + } + + /// An install.ps1 whose CURRENT slot is `current`, with a correct NEXT slot. + fn ps1(current: &str) -> String { + format!("{current}$PinnedReleasePublicKeyNext = @\"\n{NEXT_KEY}\"@\n") + } + + fn ps1_current() -> String { + format!("$PinnedReleasePublicKeyCurrent = @\"\n{SAMPLE}\"@\n") + } + + fn good_ps1() -> String { + ps1(&ps1_current()) + } + + fn good_rs() -> String { + format!("const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"{META}\";\n") + } + + /// Run the check over a scratch repo, require it to fail, and require the + /// error to contain every `expected` fragment, so a test cannot pass on an + /// unrelated failure in another slot. + #[track_caller] + fn rejected(install_sh: &str, install_ps1: &str, therock_rs: &str, expected: &[&str]) { + let repo = scratch_repo(install_sh, install_ps1, therock_rs); + let error = match check_pinned_keys(repo.path(), None) { + Ok(messages) => panic!( + "the check passed, but the source trusts a different key than \ + docs/keys; reported {messages:?}" + ), + Err(error) => format!("{error:#}"), + }; + for fragment in expected { + assert!( + error.contains(fragment), + "expected {fragment:?} in: {error}" + ); + } + } + + #[track_caller] + fn accepted(install_sh: &str, install_ps1: &str, therock_rs: &str) -> Vec { + let repo = scratch_repo(install_sh, install_ps1, therock_rs); + check_pinned_keys(repo.path(), None).unwrap_or_else(|error| panic!("{error:#}")) + } + + #[test] + fn scratch_repo_with_matching_keys_passes() { + let messages = accepted(&good_sh(), &good_ps1(), &good_rs()); + for expected in [ + "release-current: pinned in install.sh, install.ps1", + "release-next: pinned in install.sh, install.ps1", + &format!("metadata: pinned in {}", rs_path()), + ] { + assert!(messages.iter().any(|m| m == expected), "{messages:?}"); + } + } + + /// A slot without a canonical key must be empty: the installers trust any + /// non-empty slot, so a key pasted there would be trusted with nothing to + /// compare it against. + #[test] + fn a_dormant_slot_must_be_empty() { + let empty_next = |sh_next: &str, ps_next: &str, rs: &str| { + let sh = format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{sh_next}\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ); + let ps = format!( + "{}$PinnedReleasePublicKeyNext = \"{ps_next}\"\n", + ps1_current() + ); + let repo = scratch_repo(&sh, &ps, rs); + for file in [ + "rocm-cli-release-next-public.pem", + "rocm-cli-metadata-public.pem", + ] { + std::fs::remove_file(repo.path().join("docs/keys").join(file)).expect("rm"); + } + check_pinned_keys(repo.path(), None).map_err(|error| format!("{error:#}")) + }; + let messages = empty_next( + "", + "", + "const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"\";\n", + ) + .expect("empty dormant slots pass"); + assert!( + messages.iter().any(|m| m + == "release-next: no canonical key yet — slots empty in install.sh, \ + install.ps1 (dormant)"), + "{messages:?}" + ); + + let error = empty_next( + NEXT_KEY, + "", + "const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"\";\n", + ) + .expect_err("a key pasted into a dormant shell slot must fail"); + assert!( + error.contains("install.sh pins a key in PINNED_RELEASE_PUBLIC_KEY_NEXT on line 6") + && error.contains("holds no canonical key"), + "{error}" + ); + let error = empty_next( + "", + NEXT_KEY, + "const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"\";\n", + ) + .expect_err("a key pasted into a dormant PowerShell slot must fail"); + assert!( + error.contains("install.ps1 pins a key in $PinnedReleasePublicKeyNext"), + "{error}" + ); + let error = empty_next("", "", &good_rs()) + .expect_err("a key pasted into a dormant Rust slot must fail"); + assert!( + error.contains(&format!("{} pins a key in {RS_META}", rs_path())), + "{error}" + ); + } + + /// A commented-out line holding the canonical key, above the real assignment + /// holding another key. The runtime never executes the comment, so the value + /// compared must be the real assignment's. + #[test] + fn shell_commented_canonical_above_a_different_assignment_is_rejected() { + let current = format!( + "# {SH_CURRENT}=\"{SAMPLE}\"\n{SH_CURRENT}=\"{}\"\n", + wrong() + ); + rejected( + &sh(¤t), + &good_ps1(), + &good_rs(), + &["install.sh does not embed the canonical key", SH_CURRENT], + ); + } + + #[test] + fn powershell_commented_canonical_above_a_different_assignment_is_rejected() { + let current = format!( + "# {PS_CURRENT} = @\"\n{SAMPLE}\"@\n{PS_CURRENT} = @\"\n{}\"@\n", + wrong() + ); + rejected( + &good_sh(), + &ps1(¤t), + &good_rs(), + &["install.ps1 does not embed the canonical key", PS_CURRENT], + ); + } + + #[test] + fn rust_commented_canonical_above_a_different_const_is_rejected() { + let rs = format!( + "// const {RS_META}: &str = \"{META}\";\nconst {RS_META}: &str = \"{}\";\n", + wrong() + ); + rejected( + &good_sh(), + &good_ps1(), + &rs, + &[ + &format!("{} does not embed the canonical key", rs_path()), + &format!("{RS_META} (line 6)"), + ], + ); + } + + /// The canonical key assigned first and a different one assigned later: the + /// shell keeps the last value it executed. + #[test] + fn shell_reassignment_after_the_canonical_key_is_rejected() { + let current = format!("{}{SH_CURRENT}=\"{}\"\n", sh_current(), wrong()); + rejected( + &sh(¤t), + &good_ps1(), + &good_rs(), + &[ + "install.sh assigns PINNED_RELEASE_PUBLIC_KEY_CURRENT 2 times (lines 1, 6)", + "leave exactly one, as `PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"…\"`", + ], + ); + } + + #[test] + fn powershell_reassignment_after_the_canonical_key_is_rejected() { + let current = format!("{}{PS_CURRENT} = \"{}\"\n", ps1_current(), wrong()); + rejected( + &good_sh(), + &ps1(¤t), + &good_rs(), + &["install.ps1 assigns $PinnedReleasePublicKeyCurrent 2 times (lines 1, 7)"], + ); + } + + /// A longer identifier that merely starts with the token is a different + /// variable, and must not stand in for the real assignment after it. + #[test] + fn shell_identifier_sharing_the_token_as_a_prefix_is_not_the_assignment() { + let current = format!( + "{SH_CURRENT}_FPR=\"{SAMPLE}\"\n{SH_CURRENT}=\"{}\"\n", + wrong() + ); + rejected( + &sh(¤t), + &good_ps1(), + &good_rs(), + &["install.sh does not embed the canonical key", SH_CURRENT], + ); + } + + #[test] + fn powershell_identifier_sharing_the_token_as_a_prefix_is_not_the_assignment() { + let current = format!( + "{PS_CURRENT}Fpr = \"{SAMPLE}\"\n{PS_CURRENT} = \"{}\"\n", + wrong() + ); + rejected( + &good_sh(), + &ps1(¤t), + &good_rs(), + &["install.ps1 does not embed the canonical key", PS_CURRENT], + ); + } + + /// PowerShell variable names are case-insensitive, so a reassignment spelled + /// in another case is the same variable. + #[test] + fn powershell_reassignment_in_a_different_case_is_rejected() { + let current = format!( + "{}$pinnedreleasepublickeycurrent = \"{}\"\n", + ps1_current(), + wrong() + ); + rejected( + &good_sh(), + &ps1(¤t), + &good_rs(), + &["install.ps1", PS_CURRENT, "2 times"], + ); + } + + /// Comments, expansions and other reads of the name around the single real + /// assignment are not assignments and must not disturb the check; nor are + /// longer identifiers that contain the name. + #[test] + fn mentions_and_uses_around_the_single_assignment_pass() { + let sh = format!( + "# {SH_CURRENT} is the \"current\" trust root.\n\ + # {SH_CURRENT}=\"a retired key\"\n\ + {}\ + [ -n \"${{{SH_CURRENT}}}\" ] || exit 1\n\ + [ ${{#{SH_CURRENT}}} -gt 0 ] || exit 1\n\ + : ${{!{SH_CURRENT}}} \"${{!{SH_CURRENT}}}\"\n\ + for pem in \"${SH_CURRENT}\" \"${{PINNED_RELEASE_PUBLIC_KEY_NEXT}}\"; do :; done\n\ + export PINNED_RELEASE_PUBLIC_KEY_NEXT\n\ + {SH_CURRENT}_FPR=\"other\"\n\ + OLD_{SH_CURRENT}=\"other\"\n", + good_sh() + ); + let ps1 = format!( + "# {PS_CURRENT} holds the \"current\" trust root.\n\ + # {PS_CURRENT} = \"a retired key\"\n\ + {}\ + if ([string]::IsNullOrWhiteSpace({PS_CURRENT})) {{ exit 1 }}\n\ + foreach ($p in @({PS_CURRENT}, $PinnedReleasePublicKeyNext)) {{ }}\n\ + $copy = {PS_CURRENT}\n\ + $trimmed = \"$({PS_CURRENT}.Trim())\"\n\ + if ({PS_CURRENT} -eq \"\") {{ }}\n\ + {PS_CURRENT}Fpr = \"other\"\n\ + Set-Variable -Name OldPinnedReleasePublicKeyCurrent -Value \"other\"\n\ + Set-Variable -Name PinnedReleasePublicKeyCurrentFpr -Value \"other\"\n", + good_ps1() + ); + let rs = format!( + "// {RS_META} stays \"empty\" until published.\n\ + // const {RS_META}: &str = \"a retired key\";\n\ + {}\ + fn key() -> &'static str {{ {RS_META}.trim() }}\n\ + fn empty() -> bool {{ {RS_META} == \"\" }}\n\ + fn arm(k: &str) {{ match k {{ {RS_META} => {{}} _ => {{}} }} }}\n\ + const QUOTE: char = '\"';\n\ + const OLD_{RS_META}: &str = \"other\";\n", + good_rs() + ); + accepted(&sh, &ps1, &rs); + } + + /// Every shape that writes the variable counts towards the single-assignment + /// rule, before or after the canonical declaration. + #[test] + fn every_write_form_counts_as_an_assignment() { + let w = wrong(); + let shell_writes = [ + format!("export {SH_CURRENT}=\"{w}\"\n"), + format!("readonly {SH_CURRENT}=\"{w}\"\n"), + format!(" {SH_CURRENT}=\"{w}\"\n"), + format!("true && {SH_CURRENT}=\"{w}\"\n"), + format!(": \"${{{SH_CURRENT}:={w}}}\"\n"), + format!("{SH_CURRENT}+=\"x\"\n"), + format!("unset {SH_CURRENT}\n"), + format!("read -r {SH_CURRENT} < /dev/null\n"), + format!("export {SH_CURRENT}; unset {SH_CURRENT}\n"), + format!("echo \"$(unset {SH_CURRENT})\"\n"), + format!("[ ${{#{SH_CURRENT}}} -gt 0 ] && unset {SH_CURRENT}\n"), + // Quoting the name does not stop `unset` or `read` from writing it. + format!("unset '{SH_CURRENT}'\n"), + format!("read \"{SH_CURRENT}\" < /dev/null\n"), + ]; + for write in &shell_writes { + for current in [ + format!("{}{write}", sh_current()), + format!("{write}{}", sh_current()), + ] { + rejected( + &sh(¤t), + &good_ps1(), + &good_rs(), + &["install.sh", SH_CURRENT, " times (lines "], + ); + } + } + + let mut powershell_writes = vec![ + format!("$script:PinnedReleasePublicKeyCurrent = \"{w}\"\n"), + format!("$Global:PINNEDRELEASEPUBLICKEYCURRENT=\"{w}\"\n"), + format!("${{PinnedReleasePublicKeyCurrent}} = \"{w}\"\n"), + format!("if ($true) {{ {PS_CURRENT} = \"{w}\" }}\n"), + format!("Set-Variable -Name PinnedReleasePublicKeyCurrent -Value \"{w}\"\n"), + format!("New-Variable PinnedReleasePublicKeyCurrent \"{w}\" -Force\n"), + format!("Set-Item variable:PinnedReleasePublicKeyCurrent \"{w}\"\n"), + "Remove-Variable PinnedReleasePublicKeyCurrent\n".to_owned(), + // Quoting the name does not stop a cmdlet from writing it. + format!("Set-Variable -Name 'PinnedReleasePublicKeyCurrent' -Value '{w}'\n"), + format!("Set-Variable -Name \"PinnedReleasePublicKeyCurrent\" -Value \"{w}\"\n"), + format!("Set-Item 'variable:PinnedReleasePublicKeyCurrent' '{w}'\n"), + // A drive-qualified name is not this variable, but counting it fails closed. + format!("$env:PinnedReleasePublicKeyCurrent = \"{w}\"\n"), + ]; + powershell_writes.extend( + ["+=", "-=", "*=", "/=", "%=", "??="] + .iter() + .map(|op| format!("{PS_CURRENT} {op} \"x\"\n")), + ); + for write in &powershell_writes { + for current in [ + format!("{}{write}", ps1_current()), + format!("{write}{}", ps1_current()), + ] { + rejected( + &good_sh(), + &ps1(¤t), + &good_rs(), + &["install.ps1", PS_CURRENT, "2 times"], + ); + } + } + + for write in [ + format!("fn f() {{ let {RS_META} = \"{w}\"; }}\n"), + format!("static {RS_META}: &str = \"{w}\";\n"), + format!("mod m {{ const {RS_META}: &str = \"{w}\"; }}\n"), + format!("/*\nconst {RS_META}: &str = \"{w}\";\n*/\n"), + ] { + for rs in [ + format!("{}{write}", good_rs()), + format!("{write}{}", good_rs()), + ] { + rejected( + &good_sh(), + &good_ps1(), + &rs, + &[rs_path(), RS_META, "2 times"], + ); + } + } + } + + /// The declaration shapes the value is read from, beyond the plain ones. + #[test] + fn supported_declaration_forms_are_read() { + // `readonly NAME=` marks itself; `export NAME=` still needs the readonly line. + let sh = format!( + "readonly {SH_CURRENT}=\"{SAMPLE}\"\n\ + export PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_NEXT\n" + ); + let ps1 = format!( + "$script:pinnedReleasePublicKeyCurrent = \"{SAMPLE}\"\n\ + $Global:PinnedReleasePublicKeyNext = \"{NEXT_KEY}\" # rotation slot\n" + ); + for rs in [ + format!("pub const {RS_META}: &str = \"{META}\"; // metadata trust root\n"), + format!("pub(crate) const {RS_META}: &str = \"{META}\";\n"), + ] { + accepted(&sh, &ps1, &rs); + } + for scope in ["local", "private", "SCRIPT", "global"] { + accepted( + &good_sh(), + &good_ps1().replacen( + "$PinnedReleasePublicKeyCurrent = ", + &format!("${scope}:PinnedReleasePublicKeyCurrent = "), + 1, + ), + &good_rs(), + ); + } + // CRLF line endings, as a Windows checkout of install.ps1 may have. + accepted(&good_sh(), &good_ps1().replace('\n', "\r\n"), &good_rs()); + } + + /// No assignment at all, or a single one the value cannot be read from, is an + /// error naming the file and the variable rather than a silent mismatch. + #[test] + fn missing_or_unreadable_assignment_is_reported() { + rejected( + &sh(""), + &good_ps1(), + &good_rs(), + &["install.sh has no assignment of PINNED_RELEASE_PUBLIC_KEY_CURRENT"], + ); + + // A lone write in a shape the value cannot be read from: conditional, + // compound, braced, a cmdlet, or a drive-qualified name such as `$env:` + // that is not this variable at all. + for lone in [ + format!("Set-Variable -Name PinnedReleasePublicKeyCurrent -Value \"{SAMPLE}\"\n"), + format!("if ($true) {{ {PS_CURRENT} = \"{SAMPLE}\" }}\n"), + format!("{PS_CURRENT} += \"{SAMPLE}\"\n"), + format!("${{PinnedReleasePublicKeyCurrent}} = \"{SAMPLE}\"\n"), + format!("$env:PinnedReleasePublicKeyCurrent = \"{SAMPLE}\"\n"), + format!("$function:PinnedReleasePublicKeyCurrent = \"{SAMPLE}\"\n"), + ] { + rejected( + &good_sh(), + &ps1(&lone), + &good_rs(), + &[ + "install.ps1 assigns $PinnedReleasePublicKeyCurrent on line", + "unsupported form", + ], + ); + } + for lone in [ + format!("[ -n \"$CI\" ] && {SH_CURRENT}=\"{SAMPLE}\"\n"), + format!(": \"${{{SH_CURRENT}:={SAMPLE}}}\"\n"), + format!("{SH_CURRENT}+=\"{SAMPLE}\"\n"), + ] { + rejected( + &sh(&lone), + &good_ps1(), + &good_rs(), + &[ + "install.sh assigns PINNED_RELEASE_PUBLIC_KEY_CURRENT on line", + "unsupported form", + ], + ); + } + for rs in [ + format!("fn f() {{ let {RS_META}: &str = \"{META}\"; }}\n"), + format!("static {RS_META}: &str = \"{META}\";\n"), + ] { + rejected( + &good_sh(), + &good_ps1(), + &rs, + &[&format!("{} assigns", rs_file()), "unsupported form"], + ); + } + + rejected( + &good_sh(), + &ps1(&format!("{PS_CURRENT} = '{SAMPLE}'\n")), + &good_rs(), + &["not a double-quoted string"], + ); + rejected( + &sh(&format!("{SH_CURRENT}=$(printf '%s' \"{SAMPLE}\")\n")), + &good_ps1(), + &good_rs(), + &["not a double-quoted string"], + ); + } + + /// The value must be the key the runtime sees: one well-formed PEM block, + /// and nothing the installer would expand or keep verbatim. + #[test] + fn the_value_must_be_exactly_the_pem_the_runtime_sees() { + const MALFORMED: &str = "is not one well-formed PUBLIC KEY PEM block"; + // A key prepended through an expansion: openssl uses the first PEM. + rejected( + &sh(&format!("{SH_CURRENT}=\"${{EXTRA_PEM}}{SAMPLE}\"\n")), + &good_ps1(), + &good_rs(), + &["install.sh PINNED_RELEASE_PUBLIC_KEY_CURRENT on line 1 holds `$`"], + ); + rejected( + &sh(&format!("{SH_CURRENT}=\"`cat extra.pem`{SAMPLE}\"\n")), + &good_ps1(), + &good_rs(), + &["install.sh PINNED_RELEASE_PUBLIC_KEY_CURRENT on line 1 holds ```, which"], + ); + rejected( + &good_sh(), + &ps1(&format!("{PS_CURRENT} = @\"\n$($extra){SAMPLE}\"@\n")), + &good_rs(), + &["install.ps1 $PinnedReleasePublicKeyCurrent on line 1 holds `$`"], + ); + // In a shell double-quoted string `\n` stays a backslash and an `n`. + rejected( + &sh(&format!( + "{SH_CURRENT}=\"{}\"\n", + SAMPLE.replace('\n', "\\n") + )), + &good_ps1(), + &good_rs(), + &["holds `\\`"], + ); + // Accidental edits that leave the base64 letters intact but a key openssl + // cannot load, or a second block after the first. + let body_line = SAMPLE.lines().nth(1).expect("a body line"); + for broken in [ + SAMPLE.replacen("MIIBIjAN", "MIIBIjAN.-,;", 1), + SAMPLE.replacen(body_line, &format!("{body_line}\n"), 1), + SAMPLE.replacen(body_line, &format!("{body_line}\n----"), 1), + format!("{SAMPLE}{}", wrong()), + format!("junk\n{SAMPLE}"), + format!("{SAMPLE}junk"), + format!("{PEM_BEGIN}\n{PEM_END}\n"), + // A different key type's armour around the right body. + SAMPLE.replace(PEM_BEGIN, "-----BEGIN RSA PUBLIC KEY-----"), + SAMPLE.replace(PEM_END, "-----END RSA PUBLIC KEY-----"), + ] { + rejected( + &sh(&format!("{SH_CURRENT}=\"{broken}\"\n")), + &good_ps1(), + &good_rs(), + &[ + "install.sh PINNED_RELEASE_PUBLIC_KEY_CURRENT on line 1", + MALFORMED, + ], + ); + } + rejected( + &good_sh(), + &ps1(&format!( + "{PS_CURRENT} = @\"\n{}\"@\n", + SAMPLE.replacen("MIIB", "MI;IB", 1) + )), + &good_rs(), + &[ + "install.ps1 $PinnedReleasePublicKeyCurrent on line 1", + MALFORMED, + ], + ); + // A key cut short is well formed but not the canonical key. + let truncated = SAMPLE.replacen(&format!("{body_line}\n"), "", 1); + rejected( + &sh(&format!("{SH_CURRENT}=\"{truncated}\"\n")), + &good_ps1(), + &good_rs(), + &["install.sh does not embed the canonical key"], + ); + // An indented key: install.sh and the Rust source pass the value on + // verbatim, while install.ps1 trims each line before decoding. The message + // names that rule rather than one PEM reader's limits, since the Rust + // source never goes through `openssl`. + for indent in [" ", "\t"] { + let indented = SAMPLE.replace('\n', &format!("\n{indent}")); + let indented = indented.trim_end(); + rejected( + &sh(&format!("{SH_CURRENT}=\"{indent}{indented}\n\"\n")), + &good_ps1(), + &good_rs(), + &[ + "install.sh PINNED_RELEASE_PUBLIC_KEY_CURRENT on line 1 has a key line that starts with whitespace", + "only install.ps1, which trims each line, may indent a pinned key", + ], + ); + rejected( + &good_sh(), + &good_ps1(), + &format!( + "const {RS_META}: &str = \"{}\";\n", + META.replace('\n', &format!("\n{indent}")).trim_end() + ), + &[ + &format!( + "{} {RS_META} on line 1 has a key line that starts with whitespace", + rs_file() + ), + "only install.ps1, which trims each line, may indent a pinned key", + ], + ); + accepted( + &good_sh(), + &ps1(&format!("{PS_CURRENT} = @\"\n{indent}{indented}\n\"@\n")), + &good_rs(), + ); + } + // Rust turns `\n` into a line break, so that form is the same key; any + // other escape is not. + accepted( + &good_sh(), + &good_ps1(), + &format!( + "const {RS_META}: &str = \"{}\";\n", + META.replace('\n', "\\n") + ), + ); + rejected( + &good_sh(), + &good_ps1(), + &format!( + "const {RS_META}: &str = \"{}\";\n", + META.replace('\n', "\\t") + ), + &[&format!("{} {RS_META} on line 1 holds `\\`", rs_file())], + ); + } + + /// Publishing a canonical key while a slot is still empty must fail: an empty + /// slot turns signature verification off in the installers. + #[test] + fn a_published_key_with_an_empty_slot_is_rejected() { + rejected( + &sh(&format!("{SH_CURRENT}=\"\"\n")), + &good_ps1(), + &good_rs(), + &["install.sh does not embed the canonical key", SH_CURRENT], + ); + rejected( + &good_sh(), + &ps1(&format!("{PS_CURRENT} = \"\"\n")), + &good_rs(), + &["install.ps1 does not embed the canonical key", PS_CURRENT], + ); + rejected( + &good_sh(), + &good_ps1(), + &format!("const {RS_META}: &str = \"\";\n"), + &[ + &format!("{} does not embed the canonical key", rs_file()), + RS_META, + ], + ); + } + + /// A canonical file that is empty means "not published yet"; one that holds + /// something other than a key is an error rather than a silent skip. + #[test] + fn canonical_files_are_read_strictly() { + let repo = scratch_repo(&good_sh(), &good_ps1(), &good_rs()); + let next = repo + .path() + .join("docs/keys/rocm-cli-release-next-public.pem"); + std::fs::write(&next, "not a key\n").expect("write garbage"); + let error = format!( + "{:#}", + check_pinned_keys(repo.path(), None).expect_err("garbage") + ); + assert!( + error.contains( + "docs/keys/rocm-cli-release-next-public.pem is not a valid PUBLIC KEY PEM" + ), + "{error}" + ); + std::fs::write(&next, " \n").expect("write blank"); + let error = format!( + "{:#}", + check_pinned_keys(repo.path(), None).expect_err("blank") + ); + assert!( + error.contains("pins a key in PINNED_RELEASE_PUBLIC_KEY_NEXT") + && error.contains("holds no canonical key"), + "a blank canonical file is unpublished, so the filled slot must fail: {error}" + ); + } + + /// The statement must end with the string. Anything after the closing quote + /// changes the value the runtime ends up with: a PowerShell operator or + /// method rewrites it, a second quoted word appends to it, and a trailing + /// shell command turns the assignment into a prefix that leaves it unset. + #[test] + fn nothing_may_follow_the_closing_quote() { + // `"@` closes only a here-string that opened with `@"`. + rejected( + &good_sh(), + &ps1(&format!("{PS_CURRENT} = \"{SAMPLE}\"@\n")), + &good_rs(), + &[ + "install.ps1 assigns $PinnedReleasePublicKeyCurrent on line 1 to a string followed by `@", + ], + ); + for tail in [ + " -replace \"MIIB\", \"MIIC\"", + ".Replace(\"MIIB\", \"MIIC\")", + ".Substring(5)", + " + \"x\"", + ] { + rejected( + &good_sh(), + &ps1(&format!("{PS_CURRENT} = @\"\n{SAMPLE}\"@{tail}\n")), + &good_rs(), + &[ + "install.ps1 assigns $PinnedReleasePublicKeyCurrent on line 1 to a string followed by", + ], + ); + } + for tail in [" true", " | cat", "'extra'", "x", "; unset X", "@"] { + rejected( + &sh(&format!("{SH_CURRENT}=\"{SAMPLE}\"{tail}\n")), + &good_ps1(), + &good_rs(), + &[ + "install.sh assigns PINNED_RELEASE_PUBLIC_KEY_CURRENT on line 1 to a string followed by", + ], + ); + } + rejected( + &good_sh(), + &good_ps1(), + &format!("const {RS_META}: &str = \"{META}\"; const X: u8 = 1;\n"), + &[&format!( + "{} assigns {RS_META} on line 1 to a string followed by", + rs_file() + )], + ); + + // An empty-looking slot that the runtime fills: the dormant NEXT slot has + // no canonical key to compare against, so this is the case that matters. + let dormant = |sh: &str, ps: &str| { + let repo = scratch_repo(sh, ps, &good_rs()); + std::fs::remove_file( + repo.path() + .join("docs/keys/rocm-cli-release-next-public.pem"), + ) + .expect("rm"); + check_pinned_keys(repo.path(), None).map_err(|error| format!("{error:#}")) + }; + let error = dormant( + &format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"'{NEXT_KEY}'\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ), + &good_ps1(), + ) + .expect_err("a quoted key glued to an empty shell slot must fail"); + assert!( + error.contains("PINNED_RELEASE_PUBLIC_KEY_NEXT on line 6 to a string followed by"), + "{error}" + ); + let error = dormant( + &format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ), + &format!( + "{}$PinnedReleasePublicKeyNext = \"\" + @\"\n{NEXT_KEY}\"@\n", + ps1_current() + ), + ) + .expect_err("a here-string added to an empty PowerShell slot must fail"); + assert!( + error.contains("$PinnedReleasePublicKeyNext on line 7 to a string followed by"), + "{error}" + ); + + // `""#junk` is the string `#junk`, and `" "` is not empty to `[ -n … ]`. + for next in ["\"\"#junk", "\" \""] { + let error = dormant( + &format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT={next}\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ), + &good_ps1(), + ) + .expect_err("a slot that only looks empty must fail"); + assert!( + error.contains("PINNED_RELEASE_PUBLIC_KEY_NEXT on line 6"), + "{error}" + ); + } + + // A trailing comment, or `;` before one, is fine. + accepted( + &sh(&format!("{SH_CURRENT}=\"{SAMPLE}\" # current trust root\n")), + &ps1(&format!("{PS_CURRENT} = @\"\n{SAMPLE}\"@ # current\n")), + &format!("const {RS_META}: &str = \"{META}\"; // metadata trust root\n"), + ); + accepted( + &sh(&format!("{SH_CURRENT}=\"{SAMPLE}\";# current\n")), + &ps1(&format!("{PS_CURRENT} = \"{SAMPLE}\"; # current\n")), + &good_rs(), + ); + } + + /// install.sh must mark each slot readonly after assigning it, so a later + /// write fails at runtime rather than silently swapping the trust root. + #[test] + fn install_sh_must_mark_each_slot_readonly() { + let missing = format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n", + sh_current() + ); + rejected( + &missing, + &good_ps1(), + &good_rs(), + &[ + "install.sh does not mark PINNED_RELEASE_PUBLIC_KEY_CURRENT readonly after assigning it on line 1", + ], + ); + let only_next = format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\nreadonly PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ); + rejected( + &only_next, + &good_ps1(), + &good_rs(), + &["does not mark PINNED_RELEASE_PUBLIC_KEY_CURRENT"], + ); + let before = format!( + "readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n{}\ + PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n", + sh_current() + ); + rejected( + &before, + &good_ps1(), + &good_rs(), + &["does not mark PINNED_RELEASE_PUBLIC_KEY_CURRENT"], + ); + let commented = format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n\ + # readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ); + rejected( + &commented, + &good_ps1(), + &good_rs(), + &["does not mark PINNED_RELEASE_PUBLIC_KEY_CURRENT"], + ); + // `export` sets an attribute too, but not this one. + let exported = format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n\ + export PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ); + rejected( + &exported, + &good_ps1(), + &good_rs(), + &["does not mark PINNED_RELEASE_PUBLIC_KEY_CURRENT"], + ); + + // A `readonly` command must name the slot itself, without flags: a read + // of it, a mention in the trailing comment, `-f` (which names a + // function) and `-p` (which only prints) leave the variable writable. + let with_line = |line: &str| { + format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n{line}\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_NEXT\n", + sh_current() + ) + }; + // An indented `readonly` sits in a block or function that may never run + // (`if false; then … fi`), so it does not count either. Each of these + // is told what a qualifying line looks like, since the slot's name is + // already on the line it rejected. + for line in [ + "readonly COPY=\"${PINNED_RELEASE_PUBLIC_KEY_CURRENT}\"", + "readonly OTHER # PINNED_RELEASE_PUBLIC_KEY_CURRENT", + "readonly OTHER\t# PINNED_RELEASE_PUBLIC_KEY_CURRENT", + "readonly -f PINNED_RELEASE_PUBLIC_KEY_CURRENT", + "readonly -p PINNED_RELEASE_PUBLIC_KEY_CURRENT", + "if false; then\n readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT\nfi", + "f() {\n\treadonly PINNED_RELEASE_PUBLIC_KEY_CURRENT\n}", + ] { + rejected( + &with_line(line), + &good_ps1(), + &good_rs(), + &[ + "install.sh does not mark PINNED_RELEASE_PUBLIC_KEY_CURRENT readonly", + "add a line `readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT` after the assignment", + "it must start the line, with no flags, and name the slot outside any comment", + ], + ); + } + // A trailing comment after an assignment is not an attribute command at + // all, so the mention in it counts as a second write of the slot. + rejected( + &with_line("readonly OTHER=1 # PINNED_RELEASE_PUBLIC_KEY_CURRENT"), + &good_ps1(), + &good_rs(), + &[ + "install.sh assigns PINNED_RELEASE_PUBLIC_KEY_CURRENT 2 times (lines 1, 11)", + "keep the declaration on line 1", + ], + ); + + // Accepted: `readonly NAME=` marks itself, and a trailing comment is fine. + accepted( + &format!( + "readonly {SH_CURRENT}=\"{SAMPLE}\"\nPINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_NEXT # rotation slot\n" + ), + &good_ps1(), + &good_rs(), + ); + // The declaration form follows the same start-of-line rule: indented, it + // may sit in a block that never runs, so it does not mark the slot. + rejected( + &format!( + "if true; then\n readonly {SH_CURRENT}=\"{SAMPLE}\"\nfi\n\ + PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_NEXT\n" + ), + &good_ps1(), + &good_rs(), + &[ + "install.sh does not mark PINNED_RELEASE_PUBLIC_KEY_CURRENT readonly after assigning it on line 2", + "it must start the line", + ], + ); + // A comment after a tab is a comment too, so the command still only + // names variables: it marks both slots rather than counting as a second + // write of them. + accepted( + &format!( + "{}PINNED_RELEASE_PUBLIC_KEY_NEXT=\"{NEXT_KEY}\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\t# note\n", + sh_current() + ), + &good_ps1(), + &good_rs(), + ); + // `#` inside a word does not start a comment, so the command is not one + // that only names variables, and the mention counts as a write. + rejected( + &with_line("readonly OTHER a#b PINNED_RELEASE_PUBLIC_KEY_CURRENT"), + &good_ps1(), + &good_rs(), + &["install.sh assigns PINNED_RELEASE_PUBLIC_KEY_CURRENT 2 times"], + ); + } + + /// The real installers and Rust source pass as committed, and the check still + /// bites on them. The copy takes every canonical key the repository + /// publishes, so this keeps holding once the NEXT or metadata key is added. + #[test] + fn the_real_sources_pass_and_their_hardening_is_enforced() { + let real = repo_root(); + let copy = |install_sh: &str| { + let dir = tempfile::tempdir().expect("tempdir"); + let published = PINNED_KEYS + .iter() + .map(|key| key.canonical) + .filter(|path| real.join(path).exists()); + // Every source the check reads except install.sh, which the caller + // supplies, so a source that moves is copied from where it now lives. + let sources = PINNED_KEYS + .iter() + .flat_map(|key| key.sources) + .map(|source| source.path) + .filter(|&path| path != "install.sh"); + for path in published.chain(sources) { + let to = dir.path().join(path); + std::fs::create_dir_all(to.parent().unwrap()).expect("mkdir"); + std::fs::copy(real.join(path), to).expect("copy"); + } + std::fs::write(dir.path().join("install.sh"), install_sh).expect("write install.sh"); + let result = check_pinned_keys(dir.path(), None).map_err(|e| format!("{e:#}")); + (dir, result) + }; + let installer = std::fs::read_to_string(real.join("install.sh")).expect("read install.sh"); + copy(&installer).1.expect("the committed sources pass"); + + // Without its readonly line(s), install.sh fails. + let without_readonly: String = installer + .lines() + .filter(|line| !(is_attribute_command(line) && line.starts_with("readonly "))) + .map(|line| format!("{line}\n")) + .collect::>() + .concat(); + assert_ne!( + without_readonly, installer, + "install.sh has no readonly line" + ); + let error = copy(&without_readonly).1.expect_err("readonly enforced"); + assert!(error.contains("readonly after assigning it"), "{error}"); + + // A commented copy of the real key above a different one fails. + let declaration = "PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"-----BEGIN PUBLIC KEY-----\n"; + let start = installer + .find(declaration) + .expect("the CURRENT declaration"); + let end = start + installer[start..].find("-----\"\n").expect("its end") + 7; + let commented = installer[start..end] + .lines() + .map(|line| format!("# {line}\n")) + .collect::>() + .concat(); + let tampered = format!( + "{}{commented}{}", + &installer[..start], + installer[start..].replacen("MIIB", "MIIC", 1) + ); + let error = copy(&tampered) + .1 + .expect_err("commented canonical above a wrong key"); + assert!( + error.contains("does not embed the canonical key"), + "{error}" + ); + } + #[test] fn ci_public_key_cross_check() { let current = pem_body(SAMPLE).unwrap(); @@ -817,4 +2416,477 @@ mod tests { "an unset variable must resolve to None, not to some other variable" ); } + + /// The canonical current release key exactly as committed, so the checks + /// below run against the real trust root rather than a fixture. + fn canonical_release_key_text() -> String { + std::fs::read_to_string(repo_root().join(PINNED_KEYS[0].canonical)) + .expect("the canonical current release key is committed") + } + + /// The canonical PEM's base64 lines, taken from the raw file text by line + /// position (everything between the header and footer lines) rather than + /// through `pem_body`, so a defect in the normalization under test cannot + /// also shape the expected value. + fn raw_body_lines(pem: &str) -> Vec { + let lines: Vec<&str> = pem.lines().collect(); + assert_eq!( + lines.first(), + Some(&PEM_BEGIN), + "canonical key starts with its header" + ); + let footer = lines + .iter() + .position(|line| *line == PEM_END) + .expect("canonical key has a footer"); + lines[1..footer] + .iter() + .map(|line| (*line).to_owned()) + .collect() + } + + fn pem_from_lines(body: &[String]) -> String { + format!("{PEM_BEGIN}\n{}\n{PEM_END}\n", body.join("\n")) + } + + /// The two installer constants the current release key is pinned in, in the + /// shape the real installers embed them, including the `readonly` line the + /// check requires after the assignments. + fn install_sh(current: &str, extra: &str) -> String { + format!( + "#!/bin/sh\n{extra}PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"{current}\"\n\ + PINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n" + ) + } + + fn install_ps1(current: &str, extra: &str) -> String { + // An empty slot is the one-line `""` form, as in a dormant installer; an + // empty here-string is a malformed key block, not an empty slot. + let slot = if current.is_empty() { + "$PinnedReleasePublicKeyCurrent = \"\"\n".to_owned() + } else { + format!("$PinnedReleasePublicKeyCurrent = @\"\n{current}\"@\n") + }; + format!("{extra}{slot}$PinnedReleasePublicKeyNext = \"\"\n") + } + + /// A repository root holding the real canonical current release key and the + /// given installer sources, for running [`check_pinned_keys`] — the real + /// comparison — end to end. The other canonical keys are absent, so their + /// entries are dormant; the check still reads the metadata key's Rust + /// source to confirm its slot is empty, so the layout holds one. + fn release_layout(canonical: &str, sh: &str, ps1: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir_all(dir.path().join("docs/keys")).expect("create docs/keys"); + std::fs::write(dir.path().join(PINNED_KEYS[0].canonical), canonical) + .expect("write canonical key"); + let rust = dir.path().join(rs_path()); + std::fs::create_dir_all(rust.parent().expect("a parent")) + .expect("create the Rust source dir"); + std::fs::write(rust, "const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"\";\n") + .expect("write the dormant Rust source"); + std::fs::write(dir.path().join("install.sh"), sh).expect("write install.sh"); + std::fs::write(dir.path().join("install.ps1"), ps1).expect("write install.ps1"); + dir + } + + fn diverges(result: &Result>) -> bool { + result + .as_ref() + .is_err_and(|error| error.to_string().contains("have diverged")) + } + + /// Rewrite the installer sources of a [`release_layout`] in place, so a loop + /// over many cases reuses one directory instead of creating one per case. + fn write_sources(root: &Path, sh: &str, ps1: &str) { + std::fs::write(root.join("install.sh"), sh).expect("write install.sh"); + std::fs::write(root.join("install.ps1"), ps1).expect("write install.ps1"); + } + + const BASE64_ALPHABET: &str = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /// The single-character edits applied at one position of a body line: + /// deletion, duplication, substitution by each neighbour in the base64 + /// alphabet (which swaps `+` and `/`, the two symbols a letters-and-digits + /// normalization would drop; `=` padding has none), for a letter the same + /// letter in the other case, and a swap with the next character when the + /// two differ (which a comparison that ignores character order would miss). + fn single_character_edits(line: &str, column: usize) -> Vec { + let alphabet: Vec = BASE64_ALPHABET.chars().collect(); + let chars: Vec = line.chars().collect(); + let original = chars[column]; + // `=` padding has no alphabet neighbours; deletion and duplication + // still apply to it. (The committed key carries no padding today, so + // this only matters after a rotation to one that does.) + let mut replacements = alphabet + .iter() + .position(|&c| c == original) + .map(|index| { + vec![ + alphabet[(index + 1) % alphabet.len()], + alphabet[(index + alphabet.len() - 1) % alphabet.len()], + ] + }) + .unwrap_or_default(); + if original.is_ascii_alphabetic() { + replacements.push(if original.is_ascii_uppercase() { + original.to_ascii_lowercase() + } else { + original.to_ascii_uppercase() + }); + } + + let mut edits: Vec = replacements + .into_iter() + .map(|replacement| { + let mut edited = chars.clone(); + edited[column] = replacement; + edited.into_iter().collect() + }) + .collect(); + let mut deleted = chars.clone(); + deleted.remove(column); + edits.push(deleted.into_iter().collect()); + let mut duplicated = chars.clone(); + duplicated.insert(column, original); + edits.push(duplicated.into_iter().collect()); + if chars.get(column + 1).is_some_and(|&next| next != original) { + let mut swapped = chars; + swapped.swap(column, column + 1); + edits.push(swapped.into_iter().collect()); + } + edits + } + + /// Every single-character drift of the pinned current release key — at + /// every position of the real published key, in either installer — is + /// reported as divergence by the real check. + /// + /// This is the guarantee the whole check exists for: installers must pin + /// exactly the key CI signs releases with, so a one-character slip during a + /// rotation cannot pass. A random key body is caught by any comparison at + /// all; what separates a correct one from a prefix, truncating or + /// over-normalizing comparison is a body that matches everywhere but one + /// place. So rather than sampling, this walks every position and applies + /// each edit [`single_character_edits`] produces there. Look-alike swaps + /// beyond those (`l`/`I`, `O`/`0`) are not enumerated. + #[test] + fn every_single_character_drift_of_the_pinned_key_is_detected() { + let canonical = canonical_release_key_text(); + let body = raw_body_lines(&canonical); + + // Positive control: the untampered layout passes, so every failure below + // is the drift being detected, not the layout being broken. + let root = release_layout( + &canonical, + &install_sh(&canonical, ""), + &install_ps1(&canonical, ""), + ); + let messages = check_pinned_keys(root.path(), None).expect("the intact layout passes"); + assert!( + messages + .iter() + .any(|m| m.starts_with("release-current: pinned in")), + "the intact layout must actually compare the current key, got {messages:?}" + ); + + let mut expected_checks = 0_usize; + let mut checked = 0_usize; + for (line, text) in body.iter().enumerate() { + for (column, original) in text.chars().enumerate() { + let edits = single_character_edits(text, column); + let swaps = usize::from( + text.chars() + .nth(column + 1) + .is_some_and(|next| next != original), + ); + expected_checks += 2 + * (swaps + + match original { + '=' => 2, + letter if letter.is_ascii_alphabetic() => 5, + _ => 4, + }); + for edit in edits { + let mut tampered_body = body.clone(); + tampered_body[line] = edit; + let tampered = pem_from_lines(&tampered_body); + for (sh, ps1, which) in [ + ( + install_sh(&tampered, ""), + install_ps1(&canonical, ""), + "install.sh", + ), + ( + install_sh(&canonical, ""), + install_ps1(&tampered, ""), + "install.ps1", + ), + ] { + write_sources(root.path(), &sh, &ps1); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "{which}: a key differing at line {line}, column {column} was not \ + reported as divergence: {result:?}" + ); + checked += 1; + } + } + } + } + // The walk covered the whole key, counted straight from the committed + // file rather than through `raw_body_lines`, whatever its size: a + // rotation to a different key type or length changes this count, not + // the test. + let walked: usize = body.iter().map(|line| line.chars().count()).sum(); + let committed: usize = canonical + .lines() + .filter(|line| !line.starts_with("-----")) + .map(|line| line.chars().count()) + .sum(); + assert!(committed > 0, "the committed key has a body"); + assert_eq!( + walked, committed, + "the walk must cover every body character of the committed key" + ); + assert_eq!(checked, expected_checks, "every position gets every edit"); + } + + /// A published canonical key whose pinned constant is empty, or missing + /// from the source altogether, is an error — not a match and not a dormant + /// skip. An empty constant is divergence; a missing one is reported as a + /// missing assignment. Each case leaves exactly one entry wrong, so the + /// error must name that entry. + #[test] + fn an_empty_or_missing_constant_against_a_published_key_is_an_error() { + let canonical = canonical_release_key_text(); + let root = release_layout( + &canonical, + &install_sh(&canonical, ""), + &install_ps1(&canonical, ""), + ); + let next_canonical = root.path().join(PINNED_KEYS[1].canonical); + + let cases = [ + ( + "current empty in install.sh", + install_sh("", ""), + install_ps1(&canonical, ""), + false, + "release-current", + "have diverged", + ), + ( + "current empty in install.ps1", + install_sh(&canonical, ""), + install_ps1("", ""), + false, + "release-current", + "have diverged", + ), + ( + "current constant missing from install.sh", + "#!/bin/sh\nPINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n".to_owned(), + install_ps1(&canonical, ""), + false, + "release-current", + "has no assignment of PINNED_RELEASE_PUBLIC_KEY_CURRENT", + ), + // The rotation case: the next key's PEM is published (the same key + // stands in for it) while the installers still carry `NEXT=""`. + ( + "next published but empty in the installers", + install_sh(&canonical, ""), + install_ps1(&canonical, ""), + true, + "release-next", + "have diverged", + ), + ]; + for (label, sh, ps1, publish_next, entry, reported_as) in cases { + if publish_next { + std::fs::write(&next_canonical, &canonical).expect("write canonical next key"); + } else { + let _ = std::fs::remove_file(&next_canonical); + } + write_sources(root.path(), &sh, &ps1); + let result = check_pinned_keys(root.path(), None); + let message = result + .as_ref() + .expect_err(&format!("{label}: expected an error, got {result:?}")) + .to_string(); + assert!( + message.contains(reported_as), + "{label}: expected the error to say `{reported_as}`, got {message}" + ); + assert!( + message.starts_with(&format!("{entry}:")), + "{label}: the error must be reported for {entry}, got {message}" + ); + } + } + + /// An empty canonical file is a key not yet published (dormant), while a + /// pinned source file that does not exist is an error naming the file — + /// neither may read as a match. + #[test] + fn a_blank_canonical_key_is_dormant_and_a_missing_source_is_an_error() { + let canonical = canonical_release_key_text(); + // A dormant slot is empty in every installer; a key pasted into one is + // its own error (`a_dormant_slot_must_be_empty`). + let root = release_layout(" \n", &install_sh("", ""), &install_ps1("", "")); + let messages = + check_pinned_keys(root.path(), None).expect("a blank canonical key is dormant"); + assert!( + messages.iter().any(|m| m + == "release-current: no canonical key yet — slots empty in install.sh, \ + install.ps1 (dormant)"), + "a blank canonical key must be reported dormant, got {messages:?}" + ); + + // Publish the key and pin it in install.sh, so the missing install.ps1 + // is the only fault left. + std::fs::write(root.path().join(PINNED_KEYS[0].canonical), &canonical) + .expect("write canonical key"); + std::fs::write(root.path().join("install.sh"), install_sh(&canonical, "")) + .expect("pin the key in install.sh"); + std::fs::remove_file(root.path().join("install.ps1")).expect("remove install.ps1"); + let error = check_pinned_keys(root.path(), None) + .expect_err("a missing pinned source must be an error"); + assert_eq!( + error.to_string(), + "release-current: source file is missing: install.ps1" + ); + } + + /// A correct copy of the canonical key elsewhere in the same file — in a + /// comment, in another constant, in a stale copy, before or after the + /// constant — never makes a wrong `CURRENT` constant pass. + /// + /// The decoys are placed *before* the constant as well as after it, because + /// a check that reads the first key in the file would only be caught by a + /// decoy that comes first. + #[test] + fn a_correct_key_elsewhere_in_the_file_never_rescues_a_wrong_constant() { + let canonical = canonical_release_key_text(); + let mut body = raw_body_lines(&canonical); + // Flip the case of the first letter on the first body line, a + // one-character drift that stays inside the base64 alphabet. + let flipped: String = { + let mut chars: Vec = body[0].chars().collect(); + let at = chars + .iter() + .position(char::is_ascii_alphabetic) + .expect("a letter to flip"); + chars[at] = if chars[at].is_ascii_uppercase() { + chars[at].to_ascii_lowercase() + } else { + chars[at].to_ascii_uppercase() + }; + chars.into_iter().collect() + }; + body[0] = flipped; + let wrong = pem_from_lines(&body); + assert_fixture_differs(&wrong, &canonical); + + let sh_decoys = [ + format!("# previous key:\n# {}\n", canonical.replace('\n', "\n# ")), + format!("PREVIOUS_RELEASE_PUBLIC_KEY=\"{canonical}\"\n"), + format!("STALE_COPY=\"{canonical}\"\n"), + ]; + let ps1_decoys = [ + format!("# previous key:\n# {}\n", canonical.replace('\n', "\n# ")), + format!("$PreviousReleasePublicKey = @\"\n{canonical}\"@\n"), + format!("$StaleCopy = \"{canonical}\"\n"), + ]; + + for decoy in &sh_decoys { + for sh in [ + install_sh(&wrong, decoy), + format!("{}{decoy}", install_sh(&wrong, "")), + ] { + let root = release_layout(&canonical, &sh, &install_ps1(&canonical, "")); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "install.sh: a wrong CURRENT passed because the canonical key appears \ + elsewhere ({decoy:?}): {result:?}" + ); + } + } + for decoy in &ps1_decoys { + for ps1 in [ + install_ps1(&wrong, decoy), + format!("{}{decoy}", install_ps1(&wrong, "")), + ] { + let root = release_layout(&canonical, &install_sh(&canonical, ""), &ps1); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "install.ps1: a wrong CURRENT passed because the canonical key appears \ + elsewhere ({decoy:?}): {result:?}" + ); + } + } + } + + fn assert_fixture_differs(left: &str, right: &str) { + assert_ne!(left, right, "the fixture must actually differ"); + } + + /// A character outside the base64 alphabet inserted into an installer's + /// copy of the key — whitespace or punctuation — is refused as a malformed + /// key block, in either installer, and the error names the entry and the + /// file. + /// + /// The comparison itself only looks at the base64 alphabet, so this is the + /// well-formedness rule (`pinned_pem_body`) doing the work. It has to: an + /// installer that embeds such a key cannot load it (OpenSSL refuses a `!` or + /// `.`, and `Convert.FromBase64String` throws on anything outside the + /// alphabet), so it would reject every release signed with that key while a + /// comparison alone would pass. + #[test] + fn non_base64_noise_in_an_embedded_key_is_refused_as_malformed() { + let canonical = canonical_release_key_text(); + for noise in [" ", "!", "."] { + // After the first character of the body, which every key has, rather + // than at the start of a line: install.sh refuses a key line that + // starts with whitespace, and that is a separate rule. + let first = canonical + .strip_prefix(&format!("{PEM_BEGIN}\n")) + .and_then(|body| body.chars().next()) + .expect("a body character after the BEGIN line"); + let noisy = canonical.replacen( + &format!("{PEM_BEGIN}\n{first}"), + &format!("{PEM_BEGIN}\n{first}{noise}"), + 1, + ); + assert_fixture_differs(&noisy, &canonical); + for (file, sh, ps1) in [ + ( + "install.sh", + install_sh(&noisy, ""), + install_ps1(&canonical, ""), + ), + ( + "install.ps1", + install_sh(&canonical, ""), + install_ps1(&noisy, ""), + ), + ] { + let root = release_layout(&canonical, &sh, &ps1); + let error = check_pinned_keys(root.path(), None) + .expect_err("a key with stray characters in its body must be refused") + .to_string(); + assert!( + error.starts_with(&format!("release-current: {file} ")) + && error.contains("is not one well-formed PUBLIC KEY PEM block"), + "{noise:?} in {file}: expected a malformed-block error naming {file}, got {error}" + ); + } + } + } } diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 2730c5a13d..f0183849b0 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1620,6 +1620,10 @@ jobs: for (per_pr_job, nightly_job) in [ ("e2e-wsl", "e2e-wsl-nightly"), ("e2e-gpu-strix-windows", "e2e-gpu-nightly-strix-windows"), + // Off the PR path since the two-node PR scope, so the nightly is where + // this lane runs on a schedule: it must not drift back to the + // offline static `native` host. + ("e2e-gpu-strix-ubuntu", "e2e-gpu-nightly-strix"), ] { let per_pr = runs_on_values(job_block(&sh, per_pr_job)); let nightly_values = runs_on_values(job_block(&nightly, nightly_job)); @@ -1654,7 +1658,9 @@ jobs: } } - /// Both self-hosted workflows, so a lane added to either is covered. + /// Both self-hosted workflows that run GPU lanes, so a lane added to either is + /// covered. `nightly-queue.yml` and `queue-run.yml` also run self-hosted but + /// run no GPU lane; `assert_queue_workflow_holds_no_credential` covers them. fn self_hosted_workflows() -> [(&'static str, String); 2] { [ ("e2e-selfhosted.yml", read_workflow("e2e-selfhosted.yml")), @@ -2695,6 +2701,82 @@ jobs: body.contains("GPU_PREFLIGHT_MIN_FREE_GIB:-8") || body.contains("else { 8 }") } + /// `CARVEOUT_MAX_MIB` in MiB, read from the harness source as text. + /// + /// Read as text rather than imported: `xtask` has no edge to `e2e-cucumber` + /// and this check is not a reason to add one. Only the two spellings the + /// constant can reasonably take are understood, ` * 1024` and a bare + /// number, and anything else fails loudly instead of guessing. + fn carveout_max_mib(source: &str) -> u64 { + let line = source + .lines() + .find(|l| { + l.trim_start() + .starts_with("pub const CARVEOUT_MAX_MIB: u64 =") + }) + .expect("device_memory.rs declares `pub const CARVEOUT_MAX_MIB: u64 = ...;`"); + let expr = line + .split_once('=') + .and_then(|(_, rhs)| rhs.split_once(';')) + .map(|(expr, _)| expr.trim().replace('_', "")) + .expect("the constant's value ends in a semicolon"); + let number = |s: &str| { + s.trim() + .parse::() + .unwrap_or_else(|_| panic!("cannot read {s:?} in `{expr}` as a number")) + }; + match expr.split_once('*') { + Some((a, b)) => number(a) * number(b), + None => number(&expr), + } + } + + #[test] + fn carveout_max_mib_reads_both_spellings() { + let spelled = |v: &str| format!("/// doc\npub const CARVEOUT_MAX_MIB: u64 = {v};\n"); + assert_eq!(carveout_max_mib(&spelled("8 * 1024")), 8192); + assert_eq!(carveout_max_mib(&spelled("8192")), 8192); + assert_eq!(carveout_max_mib(&spelled("8_192")), 8192); + } + + #[test] + fn the_serve_drain_carveout_limit_is_the_apu_lanes_preflight_floor() { + // The serve drain wait (`tests/e2e-cucumber/src/device_memory.rs`) treats + // a VRAM total under `CARVEOUT_MAX_MIB` as an APU's BIOS carveout and + // judges GTT instead. Its doc says that limit is the floor the APU lanes + // pass as `GPU_PREFLIGHT_MIN_FREE_GIB`, and nothing tied the two together: + // moving either one left the other quietly wrong, with every unit test + // green. A raised limit would send a small discrete card to GTT; a lowered + // floor with the limit left behind would judge a real carveout by VRAM. + // + // The APU floor is the smallest one any lane sets (the discrete lanes set + // a larger one), so this follows the lanes rather than a copy of 8. + let mut floors_gib = Vec::new(); + for (_, text) in self_hosted_workflows() { + for step in gpu_preflight_steps(&text) { + if let Some(floor) = preflight_floor(&step) { + floors_gib.push(floor.parse::().unwrap_or_else(|_| { + panic!("a preflight floor of {floor:?} is not a whole number of GiB") + })); + } + } + } + let apu_floor_gib = *floors_gib + .iter() + .min() + .expect("the self-hosted lanes set a preflight floor"); + let source = + std::fs::read_to_string(repo_root().join("tests/e2e-cucumber/src/device_memory.rs")) + .expect("reading tests/e2e-cucumber/src/device_memory.rs"); + assert_eq!( + carveout_max_mib(&source), + apu_floor_gib * 1024, + "CARVEOUT_MAX_MIB no longer equals the APU lanes' GPU_PREFLIGHT_MIN_FREE_GIB \ + ({apu_floor_gib} GiB); change them together, or the serve drain wait picks \ + the wrong memory pool" + ); + } + #[test] fn every_apu_preflight_block_measures_the_gtt_pool() { // A gfx1151 APU reports its BIOS carveout as VRAM total — 512 MiB on the @@ -2751,7 +2833,7 @@ jobs: #[test] fn every_native_linux_preflight_delegates_to_the_shared_script() { // What the deduplication (#294) actually bought, and the only thing - // keeping the next #446 from landing eight times: the eight native Linux + // keeping the next #446 from landing seven times: the seven native Linux // preflights are now one script plus a floor. The drift test below can no // longer catch a block left behind, because a block left behind would // carry its own body and simply not be compared against anything. @@ -2790,8 +2872,8 @@ jobs: } } assert_eq!( - delegating, 8, - "expected the eight native Linux preflight lanes to delegate to \ + delegating, 7, + "expected the seven native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2803,7 +2885,7 @@ jobs: // where re-inlining drift grows back: a lane that keeps the delegation // and adds a line of its own above it — one more `pkill -f`, a lane-local // retry, a second floor — satisfies `step.contains(SHARED_PREFLIGHT)` - // and the floor check and leaves that test green, while the eight bodies + // and the floor check and leaves that test green, while the seven bodies // are no longer one body. That is how the original six inline copies // diverged from their PowerShell mirrors: nothing compared them to each // other, only to a substring. @@ -2812,8 +2894,8 @@ jobs: // not_drift` and `both_wsl_lanes_settle_the_clock_before_running_the_ // suite` pin their pairs, and then pin the shape they agree on: a single // `bash