From 7854b9ca920a5b5078c57fdcd6e54c6af94110b6 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 11 Nov 2026 18:59:32 +0200 Subject: [PATCH 1/5] test(e2e): fail lanes that test nothing (ROCMAI-388, ROCMAI-450) A self-hosted GPU lane could pass without testing its GPU. The capability probe ignored `rocm examine`'s exit status, so a probe that failed read as a host with no GPU and every @requires-gpu scenario resolved to skip. Skipped scenarios never reach report.json, and the only outcome-based exit iterated its results, so a run in which everything was filtered out exited 0. Lanes now declare E2E_EXPECT_GPU: 1 on the self-hosted GPU lanes and the queue lane script, 0 on the WSL lanes, whose distro is deliberately left without ROCm's WSL driver bridge. The harness fails before any scenario when the probe could not read the binary (spawn failure, non-zero exit, empty output) or a declared GPU is missing, and after the run when nothing executed or, on a GPU-declaring lane, no @requires-gpu scenario executed. A workflow contract test holds every self-hosted lane to the declaration. Refs: #383 Signed-off-by: Roman Inflianskas --- .github/workflows/e2e-selfhosted.yml | 30 +- .github/workflows/nightly.yml | 27 +- crates/e2e-report/src/consolidated/mod.rs | 15 + crates/e2e-report/src/lib.rs | 14 + docs/architecture.md | 3 +- docs/ci-hardware-testing.md | 43 +- docs/nightly-on-the-queue.md | 6 +- docs/testing.md | 28 ++ scripts/run-e2e-lane.sh | 4 + tests/e2e-cucumber/README.md | 35 +- tests/e2e-cucumber/src/capability.rs | 462 +++++++++++++++++++++- tests/e2e-cucumber/src/expectation.rs | 22 ++ tests/e2e-cucumber/src/lane_verdict.rs | 421 ++++++++++++++++++++ tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/tests/e2e.rs | 67 +++- xtask/src/workflow_contract.rs | 119 +++++- 16 files changed, 1251 insertions(+), 46 deletions(-) create mode 100644 tests/e2e-cucumber/src/lane_verdict.rs diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 96b63f1b9..c1581a054 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -254,6 +254,9 @@ jobs: ) continue-on-error: false env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # Bound serve readiness below the 35-min job cap so a serve that never comes # ready fails the scenario with a real error instead of hanging until the # job is cancelled. 300s is ample for a real MI300X vLLM cold-start; @@ -515,6 +518,9 @@ jobs: - channel: nightly sdk_version: n-2 env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_SERVE_TIMEOUT_SECS: "300" # A pool host can be busy with other CI work between GPU preflight and # the TUI checks, so a frame that renders well inside the 30s default on @@ -669,6 +675,9 @@ jobs: ) continue-on-error: false env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # A pool host can be busy with other CI work between GPU preflight and # the TUI checks, so a frame that renders well inside the 30s default on # an idle runner can miss it. Raise the wait budget rather than let that @@ -1020,6 +1029,13 @@ jobs: ) continue-on-error: false env: + # Declared GPU-less on purpose: the distro is left without ROCm's WSL + # driver bridge (see the "NO librocdxg provisioning step" note below), so + # the probe finds no usable GPU and @requires-gpu scenarios are not + # applicable. The harness reports that rather than failing on it; set "1" + # here if this lane ever provisions a GPU. Forwarded into the guest via + # WSLENV in the run step. + E2E_EXPECT_GPU: "0" # wsl.exe emits UTF-16 by default; without this its own error/status text # comes back mangled ("T h e f i l e c a n n o t ...") when captured # through PowerShell, which is exactly what a genuine wsl.exe error looks @@ -1296,12 +1312,14 @@ jobs: # today: @requires-gpu scenarios already resolve to skip here, since # `rocm-smi` is absent too (see the GPU preflight below). If this lane # ever needs a real GPU, provision it through the CLI as a separate, - # explicitly-not-under-test step rather than restoring the old script. + # explicitly-not-under-test step rather than restoring the old script, + # and flip the job's E2E_EXPECT_GPU to "1". # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU # access under WSL is exactly what this lane is proving out, so an absent - # rocm-smi is a reported condition, not a job failure: the capability probe - # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # rocm-smi is a reported condition, not a job failure: the lane declares + # E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not + # applicable instead of failing on them, and the rest still runs. A # GPU that IS present but stays held fails the job, since that would # corrupt the serve scenarios' results — though "held" means less here # than on the native lanes: this body keeps only the free-memory floor and @@ -1323,7 +1341,7 @@ jobs: MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" if ! command -v rocm-smi >/dev/null 2>&1; then - echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) @@ -1423,7 +1441,7 @@ jobs: sleep 5 done if [ "$saw_vram" -eq 0 ]; then - echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi if [ "$unmeasurable" -eq 1 ]; then @@ -1480,7 +1498,7 @@ jobs: # Forward the job-level E2E_* env vars into the guest: `wsl.exe` # does not share the calling process's environment unless a # variable is named in WSLENV. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' + $env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7aca0e072..a4e40432f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -416,6 +416,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # Include @nightly scenarios (the large-model serve). See src/expectation.rs. E2E_INCLUDE_NIGHTLY: "1" # Serve-readiness floor; a `@serve-timeout` tag lengthens it for the large @@ -530,6 +533,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -617,6 +623,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -884,6 +893,13 @@ jobs: matrix: channel: [release, nightly] env: + # Declared GPU-less on purpose: the distro is left without ROCm's WSL + # driver bridge (see the "NO librocdxg provisioning step" note below), so + # the probe finds no usable GPU and @requires-gpu scenarios are not + # applicable. The harness reports that rather than failing on it; set "1" + # here if this lane ever provisions a GPU. Forwarded into the guest via + # WSLENV in the run step. + E2E_EXPECT_GPU: "0" # wsl.exe emits UTF-16 by default; without this its own error/status text # comes back mangled when captured through PowerShell. Matches the # ubuntu/WSL composite action's own env. @@ -1083,8 +1099,9 @@ jobs: # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU # access under WSL is exactly what this lane is proving out, so an absent - # rocm-smi is a reported condition, not a job failure: the capability probe - # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # rocm-smi is a reported condition, not a job failure: the lane declares + # E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not + # applicable instead of failing on them, and the rest still runs. A # GPU that IS present but stays held fails the job, since that would # corrupt the serve scenarios' results. # @@ -1105,7 +1122,7 @@ jobs: MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" if ! command -v rocm-smi >/dev/null 2>&1; then - echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) @@ -1205,7 +1222,7 @@ jobs: sleep 5 done if [ "$saw_vram" -eq 0 ]; then - echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi if [ "$unmeasurable" -eq 1 ]; then @@ -1256,7 +1273,7 @@ jobs: shell: powershell run: | # See .github/scripts/Invoke-WslBash.ps1. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF' + $env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 9a51e9036..74fdb1fb0 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -897,6 +897,21 @@ mod tests { assert_eq!(r.status_text(), "EMPTY"); } + /// The artifact a lane uploads when the harness refuses to run (a failed + /// capability probe, or a declared GPU the probe did not find): the harness + /// creates `report.json` empty before probing and exits before writing + /// `platform.json`. That lane must read as needing attention, never green. + #[test] + fn a_lane_that_stopped_before_any_scenario_is_not_ok() { + let dir = tempfile::tempdir().expect("temp dir"); + let report = dir.path().join("report.json"); + std::fs::write(&report, "").expect("write empty report.json"); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(r.manifest.is_none(), "no platform.json was written"); + assert!(!r.ok(), "an empty run must not count as healthy"); + assert_eq!(r.status_text(), "EMPTY"); + } + #[test] fn rendering_uses_the_bundle_not_the_artifacts_on_disk() { // The render functions must work from what `load_platform_reports` diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 3eaae3947..cbe1403af 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -36,3 +36,17 @@ pub use single_report::generate; /// typo previously would not fail to compile or fail a test, silently turning /// `@requires-oom-fault-injection` into a skip (green suite, zero coverage). pub const OOM_FAULT_INJECTION_ENV: &str = "ROCM_E2E_OOM_FAULT_INJECTION"; + +/// Environment variable in which a CI lane declares whether it has a GPU. +/// +/// `1` means the lane runs on GPU hardware, so the `e2e-cucumber` harness fails +/// before any scenario runs when its capability probe finds no usable AMD GPU, +/// and after the run when no `@requires-gpu` scenario executed. `0` means the +/// lane deliberately has no usable GPU (the WSL lanes, whose distro is left +/// without ROCm's WSL driver bridge). Unset or empty leaves the lane undeclared, +/// as on the GitHub-hosted lanes and local runs; any other value is an error. +/// +/// Shared by the harness, which reads it, and xtask's workflow contract test, +/// which checks that every self-hosted E2E lane sets it, so the two cannot name +/// different variables. +pub const EXPECT_GPU_ENV: &str = "E2E_EXPECT_GPU"; diff --git a/docs/architecture.md b/docs/architecture.md index 8d7c17a37..981f8eb6b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -786,7 +786,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | --- | --- | --- | | [`src/lib.rs`](../tests/e2e-cucumber/src/lib.rs) | Library root for the test targets and the demo binary: declares the helper modules, re-exports `e2e-report` as `report`, and holds small shared helpers such as `cli_failure_report`. | flat | | [`src/binary_path.rs`](../tests/e2e-cucumber/src/binary_path.rs) | Pins a relative `ROCM_CLI_BINARY` path before a scenario runs the child from another directory, and leaves a bare name for `PATH` to resolve. | flat | -| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | +| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, records why either could not be read (`ProbeError`: not started, non-zero exit, no output), and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | | [`src/chat_request.rs`](../tests/e2e-cucumber/src/chat_request.rs) | The chat round trip against a served endpoint (model discovery, then one chat completion), run on its own task so its timeout measures the server. | flat | | [`src/dash_screen.rs`](../tests/e2e-cucumber/src/dash_screen.rs) | Pure matching over captured dashboard screens and config files, so the logic the dashboard scenarios depend on has unit tests. | flat | | [`src/device_memory.rs`](../tests/e2e-cucumber/src/device_memory.rs) | Which device memory pool (VRAM, or GTT-backed memory on an APU) the drain-wait between serve scenarios judges, how it is sampled, and the free-memory floor. | flat | @@ -798,6 +798,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat | | [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat | | [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat | +| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no `@requires-gpu` scenario) executed. | flat | | [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat | | [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat | | [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat | diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 65cc0a953..766ab12bd 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -156,13 +156,13 @@ scenario to pass / xfail / skip for that host from its `@id` and `@requires-*` tags, a capability probe, and `expectations.toml` — there is no separate tier flag or tag filter to maintain. -| Job | Workflow | Platform | Runner labels | -|---|---|---|---| -| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | -| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | -| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | -| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | -| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | +| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) | +|---|---|---|---|---| +| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | unset | +| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | `1` | +| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | `1` | +| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `1` | +| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `0` | On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` is skipped; it still runs on `push` and dispatch. Over 7 days 35% of its merge-queue jobs (98 of 281) @@ -171,6 +171,17 @@ not wait for it. On a PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo pool). +The last column is the lane's declaration of whether it has a GPU. On a lane +that declares `1`, the harness fails before any scenario runs if its capability +probe finds no usable AMD GPU, and after the run if no `@requires-gpu` scenario +executed; on every lane it fails if the probe could not read the binary under +test or if no scenario executed at all. `nightly.yml`'s self-hosted lanes +declare the same way (`0` only for `e2e-wsl-nightly`), as does every lane of +[`run-e2e-lane.sh`](../scripts/run-e2e-lane.sh). `every_self_hosted_lane_declares_whether_it_has_a_gpu` +in [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) fails when a +self-hosted lane declares neither. See the [E2E README](../tests/e2e-cucumber/README.md)'s +"Lane verdict". + `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact names, so that lane proves the nightly package channel boots. It no longer runs @@ -227,8 +238,8 @@ resolve to skip here, and known bugs resolve to xfail from The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, and `e2e-wsl`) run on AMD GPU systems, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving -scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as -described below. +scenarios that the mock job cannot. The WSL lane declares no usable GPU +(`E2E_EXPECT_GPU=0`) and its GPU preflight is advisory, as described below. `e2e-wsl` runs on the DevLab Dispatch pool. Its WSL2/Ubuntu-24.04 distro is built fresh inside every job (see above), so there is nothing of its own for @@ -242,9 +253,13 @@ preflight, toolchain bootstrap, shared-runtime pre-warm, then the full suite with no hand filtering. It covers WSL host detection, the Windows-to-WSL execution boundary, and whatever GPU access WSL exposes on that machine. The GPU preflight is advisory here precisely because GPU-on-WSL is -what the lane is proving out: where it is unavailable the capability probe -resolves those scenarios to not-applicable and the rest of the suite still -runs. Scenarios the product deliberately routes around on WSL carry +what the lane is proving out. The lane declares `E2E_EXPECT_GPU=0` (forwarded +into the guest through `WSLENV`), because its distro is deliberately left +without the driver bridge (see "What the WSL distro needs"): the capability +probe resolves `@requires-gpu` scenarios to not-applicable, the harness prints +a note saying so rather than failing, and the rest of the suite still runs. If +the probe finds a usable GPU anyway, the harness notes that the declaration is +stale and runs the GPU scenarios. Scenarios the product deliberately routes around on WSL carry `@requires-bare-metal`; scenarios whose premise *is* a WSL host carry `@requires-wsl`, and this is the only lane that runs them. @@ -269,7 +284,9 @@ driver scenarios assert on the plan that command produces — a lane that pre-installed it would hand those scenarios a host already in the state the command under test exists to reach. That costs no coverage today, because `rocm-smi` is absent too, so `@requires-gpu` scenarios resolve to skip either -way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`). +way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`). This is why +the lane declares `E2E_EXPECT_GPU=0`; a lane that provisions the bridge +declares `1` instead. GPU coverage additionally needs ROCm's WSL passthrough to be complete — `/dev/dxg` and dxcore alone are not enough, `librocdxg.so` must be present and diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index e4fae4508..4c01a589c 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -125,7 +125,11 @@ limit (see the run's inputs above). Each channel starts with the GPU preflight, channel's job in `nightly.yml` does, so a model server the previous channel's suite left on the GPU is reclaimed first; the lane log shows it. Both always run, so a failing channel does not hide the other, and the lane fails if either -did. +did. The default command also declares that every lane has a GPU +(`E2E_EXPECT_GPU`, default `1`), so a channel whose suite finds no usable AMD +GPU, or runs no `@requires-gpu` scenario, fails instead of passing with every +GPU scenario skipped; see "Lane verdict" in the +[E2E README](../tests/e2e-cucumber/README.md). A lane starts cold, so its first steps fetch from the network: the build packages, the rustup installer, the toolchain and the crates for the host diff --git a/docs/testing.md b/docs/testing.md index e8cf19058..6200d2ce9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -951,6 +951,34 @@ export ROCM_TEST_APK_REPOS="--repository http://dl-cdn.alpinelinux.org/alpine/v3 --repository http://dl-cdn.alpinelinux.org/alpine/v3.20/community" ``` +## E2E runs that test nothing fail + +`cargo xtask e2e` fails a run whose result could not mean anything, instead of +reporting it green: + +- before any scenario runs, when the capability probe could not read the binary + under test: `rocm examine` or `rocm engines list` could not be started (check + `ROCM_CLI_BINARY`), exited non-zero, or printed nothing; +- after the run, when no scenario executed, which includes a + `cargo xtask e2e -- -n ` whose regex matches no scenario name. + +`E2E_EXPECT_GPU` declares whether the host has a GPU. With `1`, the run also +fails before any scenario if the probe found no usable AMD GPU, and after the +run if no `@requires-gpu` scenario executed. With `0`, it only notes that GPU +scenarios are not applicable (and that the declaration is stale if a GPU is +found). Unset, as on a local run, it makes no GPU claim. Every self-hosted CI +lane sets it; see "Lane verdict" in the [E2E README](../tests/e2e-cucumber/README.md) +and the Platforms table in [ci-hardware-testing.md](ci-hardware-testing.md). The +checks are unit-tested in [lane_verdict.rs](../tests/e2e-cucumber/src/lane_verdict.rs) +and, for each way a probe can fail, in +[capability.rs](../tests/e2e-cucumber/src/capability.rs): + +```bash +cargo test -p e2e-cucumber --lib lane_verdict +cargo test -p e2e-cucumber --lib capability::tests::a_probe +cargo test -p xtask every_self_hosted_lane_declares_whether_it_has_a_gpu +``` + ## CI test selection On pull requests, CI does not test the whole workspace. The `test` job runs diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh index 68e8569f8..08c3dad0c 100755 --- a/scripts/run-e2e-lane.sh +++ b/scripts/run-e2e-lane.sh @@ -22,6 +22,9 @@ # both (default "release nightly") # E2E_LANE_WORK where the build and runtimes go (default /var/tmp/e2e-lane) # LANE_OUT where reported files go (default /out) +# E2E_EXPECT_GPU whether the lane has a GPU (default 1: every queue lane is +# given one); the suite fails rather than passing with every +# GPU scenario skipped. See tests/e2e-cucumber/README.md. set -euo pipefail @@ -39,6 +42,7 @@ if [[ -n "${CARGO_BUILD_TARGET:-}" ]]; then fi export E2E_INCLUDE_NIGHTLY="${E2E_INCLUDE_NIGHTLY-1}" +export E2E_EXPECT_GPU="${E2E_EXPECT_GPU-1}" export E2E_SERVE_TIMEOUT_SECS="${E2E_SERVE_TIMEOUT_SECS:-300}" # The channels are checked before anything is built, so a typo costs nothing. diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d871ab21d..f10410125 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -64,7 +64,7 @@ runs the suite. Extra arguments after `--` are forwarded to the cucumber CLI. # All features: cargo xtask e2e -# Filter by scenario name: +# Filter by scenario name (bypasses capability gating; see below): cargo xtask e2e -- -n "model name" # Stop on first failure: @@ -101,6 +101,37 @@ E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e | `ROCM_CLI_CONFIG_DIR` | (temp dir) | Isolated config directory per scenario | | `ROCM_CLI_DATA_DIR` | (temp dir) | Isolated data directory per scenario | | `ROCM_CLI_CACHE_DIR` | (temp dir) | Isolated cache directory per scenario | +| `E2E_EXPECT_GPU` | unset (undeclared) | Whether the lane has a GPU: `1` = GPU expected, `0` = deliberately none, unset or empty = no claim; any other value fails the run. Set on every self-hosted CI lane; see "Lane verdict" below | + +### Lane verdict + +Scenarios that resolve to skip never reach `report.json`, so scenario results +alone cannot tell a lane that tested its GPU from one on which every GPU +scenario was skipped. The harness checks the run itself, before and after the +scenarios: + +- **Before any scenario runs**, it fails when the capability probe could not + read the binary under test (`rocm examine` or `rocm engines list` could not be + started, exited non-zero, or printed nothing), naming the command, its exit + status and the tail of its stderr. On a lane that sets `E2E_EXPECT_GPU=1` it + also fails when the probe found no usable AMD GPU, naming the detected gfx + target, WSL flag, driver status and visibility mask it decided from. +- **After the run**, it fails when no scenario executed (for example a `-n` + filter that matches nothing), and on an `E2E_EXPECT_GPU=1` lane when no + `@requires-gpu` scenario executed. + +`E2E_EXPECT_GPU=0` is for a lane that deliberately has no usable GPU, such as +the WSL lanes: the harness prints a note that `@requires-gpu` scenarios are not +applicable, and a note that the declaration is stale if the probe finds a GPU +anyway; neither fails the run. Both checks are in +[`src/lane_verdict.rs`](src/lane_verdict.rs). + +A cucumber `-n`/`--name` or `--tags` filter on the command line replaces the +harness's own scenario filter, so the scenarios it selects run without the +capability gating, the opt-in sets and the expectation matrix described below, +and the after-run GPU check does not apply. Use it to run a known scenario on a +host that can satisfy it; use `E2E_ONLY_LIFECYCLE` with `E2E_ALSO_RUN_IDS` to +select scenarios with the gating kept. ## Tags and per-scenario expectations @@ -137,7 +168,7 @@ Scenarios carry stable-id and capability tags: | Tag | Meaning | |---|---| | `@id:-` | Stable scenario id, prefixed with its feature's key. Keys the expectation matrix and the report grid; every scenario has one. | -| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. | +| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. On a lane that sets `E2E_EXPECT_GPU=1` a host with none fails the run instead; see "Lane verdict" above. | | `@requires-multi-gpu` | Premise is a host with **more than one** AMD GPU present. Resolves to **skip** wherever the probed device count is not known to exceed one: a single-GPU host such as Strix Halo, and any host whose count could not be read (non-Linux, or WSL, which exposes neither a KFD topology nor an amdgpu DRM card). `@requires-gpu` cannot express this — it asks only whether *a* device is usable — so scenarios carry both. The count comes from the capability probe's `amd_gpu_count`, which mirrors the product's KFD+DRM rule rather than counting `amd-smi list` — the gate must ask the same "how many devices are present" question `--gpu` validation is built on, and `amd-smi` is only a best-effort fallback that may be absent or disagree. | | `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. | | `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. | diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index d8857f43c..9e9fdfed4 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -164,6 +164,17 @@ pub struct HostCapability { /// running the suite. Gates `@requires-dir-permissions`; see /// [`probe_dir_permissions_bind`]. pub dir_permissions_bind: bool, + /// `examine`'s `driver_status:` line, empty when it printed none. Only + /// decides GPU usability on WSL (see [`host_has_usable_gpu`]); kept so a + /// lane whose GPU went missing can say why. + pub driver_status: String, + /// Why the probe could not read the binary under test, if it could not: + /// the first failure of `rocm examine` or `rocm engines list` (see + /// [`ProbeError`]). When set, the rest of this struct holds the same + /// no-GPU values an empty `examine` would give, so callers never panic, and + /// the harness refuses to run (see `lane_verdict::preflight_verdict`). + #[serde(skip_serializing_if = "Option::is_none")] + pub probe_error: Option, } /// Environment variable `xtask e2e` sets to `1` when it compiled the binary @@ -258,7 +269,11 @@ pub fn collect_versions(runtimes_dir: Option<&std::path::Path>) -> PlatformVersi // OS distro from `examine` — ALWAYS collected (every platform, incl. mock and // hosts without a shared runtime dir). Isolated throwaway root reads the host. if let Ok(tmp) = tempfile::TempDir::with_prefix("rocm-e2e-ver-") { - let examine = run_probe(tmp.path(), &["examine"]); + // Best-effort, unlike the capability probe: a failed `examine` here + // only leaves the report heading's OS blank. The capability probe ran + // the same command before any scenario and already refused the run if + // it failed. + let examine = run_probe(tmp.path(), &["examine"]).unwrap_or_default(); v.os = examine .lines() .find_map(|l| l.trim().strip_prefix("distro:")) @@ -486,6 +501,29 @@ fn probe_host_capability() -> HostCapability { // what the scenarios see. let examine = run_probe(root, &["examine"]); let engines = run_probe(root, &["engines", "list"]); + capability_from_probes(examine, engines, root) +} + +/// Build a [`HostCapability`] from the two probes' results; `root` is the +/// throwaway directory the filesystem probes write under. +/// +/// Split from [`probe_host_capability`] so a test can hand it a failed probe +/// without a binary to run. +fn capability_from_probes( + examine: Result, + engines: Result, + root: &std::path::Path, +) -> HostCapability { + // The first failure is the one reported; either one means the facts below + // were not read from the binary, so they fall back to the no-GPU values an + // empty dump parses to. + let probe_error = examine + .as_ref() + .err() + .or_else(|| engines.as_ref().err()) + .map(ToString::to_string); + let examine = examine.unwrap_or_default(); + let engines = engines.unwrap_or_default(); let ExamineFacts { os_family, @@ -516,6 +554,8 @@ fn probe_host_capability() -> HostCapability { oom_fault_injection, case_sensitive_fs, dir_permissions_bind, + driver_status, + probe_error, } } @@ -581,18 +621,161 @@ fn probe_case_sensitive_fs(dir: &std::path::Path) -> bool { probe().unwrap_or(false) } -/// Run `rocm ` with an isolated config/data/cache root, returning stdout -/// (empty string on any failure — the probe must never panic the suite). -fn run_probe(root: &std::path::Path, args: &[&str]) -> String { - let mut cmd = std::process::Command::new(rocm_binary()); +/// Why a probe of the binary under test produced nothing the harness can read. +/// +/// Every variant is fatal to the run: an unreadable probe parses as a host with +/// no GPU, so every capability-gated scenario would resolve to skip and the run +/// would pass without testing them. `command` is the binary and arguments as +/// run, so the message names exactly what failed. +#[derive(Debug)] +pub enum ProbeError { + /// The binary could not be started at all (wrong path, not executable). + Spawn { + command: String, + source: std::io::Error, + }, + /// It ran and exited non-zero: the CLI exits 0 for every finding, so this + /// means it could not examine the host at all. + Exit { + command: String, + status: std::process::ExitStatus, + stderr_tail: String, + }, + /// It exited 0 but printed nothing on stdout. + EmptyOutput { command: String }, +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Spawn { command, source } => write!( + f, + "could not start `{command}`: {source}. Set ROCM_CLI_BINARY to the rocm binary \ + under test, or run the suite through `cargo xtask e2e`, which builds it and \ + sets ROCM_CLI_BINARY" + ), + Self::Exit { + command, + status, + stderr_tail, + } => write!( + f, + "`{command}` exited with {status}\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::EmptyOutput { command } => { + write!(f, "`{command}` exited 0 but produced no output on stdout") + } + } + } +} + +impl std::error::Error for ProbeError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Spawn { source, .. } => Some(source), + Self::Exit { .. } | Self::EmptyOutput { .. } => None, + } + } +} + +/// How many trailing stderr lines a [`ProbeError::Exit`] keeps. +const PROBE_STDERR_TAIL_LINES: usize = 20; + +/// Turn one probe's spawn result into its stdout, or the reason it has none. +/// +/// Pure, so each outcome is unit-tested without a binary to run. Stdout that is +/// only whitespace counts as empty: no `examine` field can be parsed from it. +pub fn classify_probe_output( + command: &str, + result: std::io::Result, +) -> Result { + let out = result.map_err(|source| ProbeError::Spawn { + command: command.to_owned(), + source, + })?; + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + let lines: Vec<&str> = stderr.lines().collect(); + let tail = lines[lines.len().saturating_sub(PROBE_STDERR_TAIL_LINES)..].join("\n"); + return Err(ProbeError::Exit { + command: command.to_owned(), + status: out.status, + stderr_tail: tail, + }); + } + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + if stdout.trim().is_empty() { + return Err(ProbeError::EmptyOutput { + command: command.to_owned(), + }); + } + Ok(stdout) +} + +/// Run `rocm ` with an isolated config/data/cache root and return its +/// stdout, or why there is none. Never panics: the caller decides whether a +/// failure is fatal. +fn run_probe(root: &std::path::Path, args: &[&str]) -> Result { + run_probe_with(std::ffi::OsStr::new(&rocm_binary()), root, args) +} + +/// [`run_probe`] with the binary passed in, so a test can name one without +/// setting a process-wide environment variable. +pub fn run_probe_with( + binary: &std::ffi::OsStr, + root: &std::path::Path, + args: &[&str], +) -> Result { + let mut cmd = std::process::Command::new(binary); cmd.args(args); cmd.env("ROCM_CLI_CONFIG_DIR", root.join("config")); cmd.env("ROCM_CLI_DATA_DIR", root.join("data")); cmd.env("ROCM_CLI_CACHE_DIR", root.join("cache")); - match cmd.output() { - Ok(out) => String::from_utf8_lossy(&out.stdout).into_owned(), - Err(_) => String::new(), - } + let command = std::iter::once(binary.to_string_lossy().into_owned()) + .chain(args.iter().map(|a| (*a).to_owned())) + .collect::>() + .join(" "); + classify_probe_output(&command, cmd.output()) +} + +/// One line naming the probe facts that decide GPU usability, for a lane whose +/// declared GPU the probe did not find. +/// +/// Reads the visibility masks from the environment the way +/// [`host_has_usable_gpu`] does; [`probe_facts_summary_with_mask`] is the pure +/// form. +#[must_use] +pub fn probe_facts_summary(cap: &HostCapability) -> String { + let hip = std::env::var_os("HIP_VISIBLE_DEVICES"); + let rocr = std::env::var_os("ROCR_VISIBLE_DEVICES"); + probe_facts_summary_with_mask(cap, hip.as_deref(), rocr.as_deref()) +} + +/// Pure form of [`probe_facts_summary`]. Names the mask that is in force by the +/// same precedence as [`selected_visibility_mask`]: HIP wins when set, even +/// empty. +#[must_use] +pub fn probe_facts_summary_with_mask( + cap: &HostCapability, + hip: Option<&std::ffi::OsStr>, + rocr: Option<&std::ffi::OsStr>, +) -> String { + let mask = match (hip, rocr) { + (Some(v), _) => format!("HIP_VISIBLE_DEVICES={:?}", v.to_string_lossy()), + (None, Some(v)) => format!("ROCR_VISIBLE_DEVICES={:?}", v.to_string_lossy()), + (None, None) => "no visibility mask".to_owned(), + }; + let driver_status = if cap.driver_status.is_empty() { + "" + } else { + cap.driver_status.as_str() + }; + format!( + "detected_gfx_target={}, wsl={}, driver_status={driver_status}, {mask}", + cap.gfx_target.as_deref().unwrap_or(""), + cap.is_wsl, + ) } /// The `examine` fields the capability probe reads. @@ -992,6 +1175,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }; assert!(strix.engine_available("lemonade")); // vLLM adapter is "built-in" but cannot start on Windows / non-dcgpu. @@ -1009,6 +1194,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }; assert!(mi300x.engine_available("vllm")); assert!(mi300x.engine_available("lemonade")); @@ -1503,4 +1690,261 @@ Local model engines std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o755)).expect("restore"); assert_eq!(probe_dir_permissions_bind(tmp.path()), refused); } + + /// An exit status that reports failure with code 1, built the way each + /// platform encodes it, so the non-zero-exit row runs on both CI gates. + fn failed_status() -> std::process::ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + // A wait status: the exit code sits in the second byte. + std::process::ExitStatus::from_raw(1 << 8) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(1) + } + } + + fn succeeded_status() -> std::process::ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + } + + fn output( + status: std::process::ExitStatus, + stdout: &str, + stderr: &str, + ) -> std::process::Output { + std::process::Output { + status, + stdout: stdout.as_bytes().to_vec(), + stderr: stderr.as_bytes().to_vec(), + } + } + + /// The regression: a probe that exits non-zero is an error even when it + /// printed something, rather than stdout parsed as a GPU-less host. + #[test] + fn a_probe_that_exits_non_zero_is_an_error_with_its_stderr_tail() { + let result = classify_probe_output( + "rocm examine", + Ok(output( + failed_status(), + "rocm examine\n os: linux\n", + "warning: early\nError: could not read the host\n", + )), + ); + let Err(error @ ProbeError::Exit { .. }) = result else { + panic!("a non-zero exit must be ProbeError::Exit, got {result:?}"); + }; + assert_eq!( + error.to_string(), + format!( + "`rocm examine` exited with {}\n--- stderr (last lines) ---\n\ + warning: early\nError: could not read the host", + failed_status() + ) + ); + } + + #[test] + fn the_stderr_tail_keeps_only_the_last_lines() { + let stderr = (1..=30) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + let Err(ProbeError::Exit { stderr_tail, .. }) = + classify_probe_output("rocm examine", Ok(output(failed_status(), "", &stderr))) + else { + panic!("a non-zero exit must be ProbeError::Exit"); + }; + let expected: Vec = (11..=30).map(|i| format!("line {i}")).collect(); + assert_eq!(stderr_tail, expected.join("\n")); + } + + #[test] + fn a_probe_that_exits_zero_with_output_returns_its_stdout() { + let stdout = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n"; + assert_eq!( + classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, ""))) + .expect("exit 0 with output is a reading"), + stdout + ); + } + + #[test] + fn a_probe_that_exits_zero_with_no_output_is_an_error() { + for stdout in ["", " \n\n"] { + let result = + classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, ""))); + let Err(error @ ProbeError::EmptyOutput { .. }) = result else { + panic!("empty stdout {stdout:?} must be ProbeError::EmptyOutput, got {result:?}"); + }; + assert_eq!( + error.to_string(), + "`rocm examine` exited 0 but produced no output on stdout" + ); + } + } + + #[test] + fn a_probe_whose_binary_does_not_exist_is_a_spawn_error_naming_it() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let missing = tmp.path().join("no-such-rocm"); + let result = run_probe_with(missing.as_os_str(), tmp.path(), &["examine"]); + let Err(error @ ProbeError::Spawn { .. }) = result else { + panic!("a missing binary must be ProbeError::Spawn, got {result:?}"); + }; + let ProbeError::Spawn { command, source } = &error else { + unreachable!() + }; + assert_eq!(command, &format!("{} examine", missing.display())); + assert_eq!(source.kind(), std::io::ErrorKind::NotFound); + assert_eq!( + error.to_string(), + format!( + "could not start `{} examine`: {source}. Set ROCM_CLI_BINARY to the rocm binary \ + under test, or run the suite through `cargo xtask e2e`, which builds it and \ + sets ROCM_CLI_BINARY", + missing.display() + ) + ); + } + + const GPU_EXAMINE: &str = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n wsl: false\n driver_status: ok\n"; + + /// A failed probe records why, and leaves the struct at the no-GPU values + /// so no caller panics. + #[test] + fn a_failed_examine_records_the_error_and_falls_back_to_no_gpu() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Err(ProbeError::EmptyOutput { + command: "rocm examine".to_owned(), + }), + Ok("lemonade\n".to_owned()), + tmp.path(), + ); + assert_eq!( + cap.probe_error.as_deref(), + Some("`rocm examine` exited 0 but produced no output on stdout") + ); + assert!(!cap.has_amd_gpu); + assert_eq!(cap.gfx_target, None); + assert_eq!(cap.platform_slug, "mock"); + } + + /// The `engines list fails, examine OK` row: the GPU facts were read, but + /// the probe still failed, so the run must not proceed on them. + #[test] + fn a_failed_engines_list_is_recorded_even_when_examine_succeeded() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Ok(GPU_EXAMINE.to_owned()), + Err(ProbeError::Exit { + command: "rocm engines list".to_owned(), + status: failed_status(), + stderr_tail: "boom".to_owned(), + }), + tmp.path(), + ); + assert_eq!( + cap.probe_error, + Some(format!( + "`rocm engines list` exited with {}\n--- stderr (last lines) ---\nboom", + failed_status() + )) + ); + assert!(cap.available_engines.is_empty()); + } + + #[test] + fn the_first_failure_is_the_one_recorded() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Err(ProbeError::EmptyOutput { + command: "rocm examine".to_owned(), + }), + Err(ProbeError::EmptyOutput { + command: "rocm engines list".to_owned(), + }), + tmp.path(), + ); + assert_eq!( + cap.probe_error.as_deref(), + Some("`rocm examine` exited 0 but produced no output on stdout") + ); + } + + #[test] + fn successful_probes_record_no_error_and_keep_the_driver_status() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Ok(GPU_EXAMINE.to_owned()), + Ok("lemonade\nvllm\n".to_owned()), + tmp.path(), + ); + assert_eq!(cap.probe_error, None); + assert_eq!(cap.gfx_target.as_deref(), Some("gfx1151")); + assert_eq!(cap.driver_status, "ok"); + // `probe_error` is left out of platform.json when there is none. + let json = serde_json::to_value(&cap).expect("serializes"); + assert!(json.get("probe_error").is_none(), "{json}"); + } + + #[test] + fn the_probe_facts_summary_names_the_mask_in_force() { + let cap = HostCapability { + os_family: "linux".to_owned(), + is_wsl: true, + gfx_target: Some("gfx1151".to_owned()), + has_amd_gpu: false, + amd_gpu_count: None, + available_engines: vec![], + effective_serve_engine: "lemonade".to_owned(), + platform_slug: "mock".to_owned(), + oom_fault_injection: false, + case_sensitive_fs: true, + dir_permissions_bind: true, + driver_status: "wsl_rocdxg_missing".to_owned(), + probe_error: None, + }; + let base = "detected_gfx_target=gfx1151, wsl=true, driver_status=wsl_rocdxg_missing"; + assert_eq!( + probe_facts_summary_with_mask(&cap, None, None), + format!("{base}, no visibility mask") + ); + assert_eq!( + probe_facts_summary_with_mask( + &cap, + Some(std::ffi::OsStr::new("")), + Some(std::ffi::OsStr::new("0")) + ), + format!("{base}, HIP_VISIBLE_DEVICES=\"\"") + ); + assert_eq!( + probe_facts_summary_with_mask(&cap, None, Some(std::ffi::OsStr::new("1"))), + format!("{base}, ROCR_VISIBLE_DEVICES=\"1\"") + ); + let none = HostCapability { + gfx_target: None, + is_wsl: false, + driver_status: String::new(), + ..cap + }; + assert_eq!( + probe_facts_summary_with_mask(&none, None, None), + "detected_gfx_target=, wsl=false, driver_status=, no visibility mask" + ); + } } diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 6771d09dc..9643b346e 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -745,6 +745,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, "strix-ubuntu" => HostCapability { os_family: "linux".into(), @@ -758,6 +760,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, "strix-windows" => HostCapability { os_family: "windows".into(), @@ -771,6 +775,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, // A WSL2 dev box with the ROCm passthrough in place: `os_family` is // `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes @@ -793,6 +799,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, // The same box without the passthrough: the gfx target is reported // by the Windows-side driver but ROCm cannot reach it, so the probe @@ -809,6 +817,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, // The hosted WSL runner before the ROCm passthrough is complete: // Windows still reports the gfx target, but the CLI cannot use it. @@ -824,6 +834,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, _ => HostCapability { os_family: "other".into(), @@ -837,6 +849,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, } } @@ -1824,6 +1838,8 @@ serve_timeout_secs = 90 let folded = HostCapability { case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, ..cap("wsl2") }; let both = decl(&[ @@ -1875,6 +1891,8 @@ serve_timeout_secs = 90 // scenario run and pass without testing anything. let as_root = HostCapability { dir_permissions_bind: false, + driver_status: String::new(), + probe_error: None, ..cap("wsl2") }; match resolve(&d, &as_root, &m, none) { @@ -1929,6 +1947,8 @@ serve_timeout_secs = 90 ); let host = HostCapability { dir_permissions_bind: false, + driver_status: String::new(), + probe_error: None, ..cap("strix-windows") }; assert_ne!(host.os_family, "linux", "fixture must fail the OS premise"); @@ -1965,6 +1985,8 @@ reason = "unrelated open bug" let folded = HostCapability { case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, ..cap("strix-ubuntu") }; assert!(matches!( diff --git a/tests/e2e-cucumber/src/lane_verdict.rs b/tests/e2e-cucumber/src/lane_verdict.rs new file mode 100644 index 000000000..ed562eb9d --- /dev/null +++ b/tests/e2e-cucumber/src/lane_verdict.rs @@ -0,0 +1,421 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether a run's outcome says anything about the lane it ran on. +//! +//! Scenario results alone cannot say that. Scenarios the capability probe +//! resolves to skip never reach `report.json`, so a run in which every scenario +//! was skipped, or none was selected, has no failure to report and passes. +//! That is what a GPU lane does when its probe cannot read the binary under +//! test, or when the GPU is not visible: every `@requires-gpu` scenario +//! resolves to skip, and the lane is green without having tested a GPU. +//! +//! Two checks close that gap, both pure so every row of their state matrix is a +//! unit test: +//! +//! - [`preflight_verdict`], before any scenario runs: a probe that failed is +//! fatal on every lane, and so is a lane that declares a GPU (`1` in +//! [`EXPECT_GPU_ENV`]) where the probe found none. +//! - [`post_run_verdict`], after the run: a run that executed no scenario is +//! fatal on every lane, and so is a GPU-declaring lane on which no +//! `@requires-gpu` scenario executed. + +use std::ffi::OsStr; + +pub use e2e_report::EXPECT_GPU_ENV; + +/// What a lane says about its GPU in [`EXPECT_GPU_ENV`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuDeclaration { + /// `1`: the lane runs on GPU hardware, so a run that found or used none is + /// broken. + Expected, + /// `0`: the lane deliberately has no usable GPU, so `@requires-gpu` + /// scenarios are not applicable on it. + NotExpected, + /// Unset or empty: no claim either way (GitHub-hosted lanes, local runs). + Undeclared, +} + +/// Read [`EXPECT_GPU_ENV`]'s value. +/// +/// Anything other than `1`, `0`, empty or unset is an error rather than a guess, since a misspelt declaration that read +/// as "undeclared" would switch the GPU check off without anyone noticing. +pub fn parse_gpu_declaration(value: Option<&OsStr>) -> Result { + match value.map(OsStr::to_str) { + None | Some(Some("")) => Ok(GpuDeclaration::Undeclared), + Some(Some("1")) => Ok(GpuDeclaration::Expected), + Some(Some("0")) => Ok(GpuDeclaration::NotExpected), + Some(_) => Err(format!( + "invalid {EXPECT_GPU_ENV} value {:?}: use 1 (this lane has a GPU), 0 (it \ + deliberately has none), or leave it unset", + value.unwrap_or_default().to_string_lossy() + )), + } +} + +/// What a check concluded: a reason to fail the run, and lines to print either +/// way. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Verdict { + /// Why the run must fail; `None` when it may proceed or pass. + pub fatal: Option, + /// Informational lines that do not change the outcome. + pub notes: Vec, +} + +/// The verdict before any scenario runs, from the capability probe and the +/// lane's declaration. +/// +/// `probe_error` is [`crate::capability::HostCapability::probe_error`]; +/// `probe_facts` is the one-line summary of the facts the GPU decision was made +/// from ([`crate::capability::probe_facts_summary`]), quoted when a declared GPU +/// is missing. +#[must_use] +pub fn preflight_verdict( + probe_error: Option<&str>, + has_amd_gpu: bool, + probe_facts: &str, + declaration: GpuDeclaration, +) -> Verdict { + let mut verdict = Verdict::default(); + if let Some(error) = probe_error { + verdict.fatal = Some(format!( + "the host capability probe could not read the rocm binary under test, so every \ + capability-gated scenario would resolve to skip and the run would pass without \ + testing them: {error}" + )); + return verdict; + } + match (declaration, has_amd_gpu) { + (GpuDeclaration::Expected, false) => { + verdict.fatal = Some(format!( + "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but the capability probe found \ + no usable AMD GPU ({probe_facts}), so every @requires-gpu scenario would \ + resolve to skip" + )); + } + (GpuDeclaration::NotExpected, true) => verdict.notes.push(format!( + "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0) but the capability probe \ + found one ({probe_facts}); the declaration is stale, and @requires-gpu scenarios \ + will run" + )), + (GpuDeclaration::NotExpected, false) => verdict.notes.push(format!( + "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0); @requires-gpu scenarios \ + are not applicable here" + )), + (GpuDeclaration::Expected, true) | (GpuDeclaration::Undeclared, _) => {} + } + verdict +} + +/// How many `@requires-gpu` scenarios the run resolved and how many of those +/// executed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GpuScenarioCounts { + /// Scenarios tagged `@requires-gpu` or `@requires-multi-gpu` that the + /// harness resolved, whether or not they ran. + pub resolved: usize, + /// Of those, how many reached `report.json`. + pub executed: usize, +} + +/// The verdict after the run. +/// +/// `executed` counts the scenarios that reached `report.json`. `gpu` is `None` +/// when the harness's scenario filter never ran — a cucumber `--name`/`--tags` +/// filter on the command line replaces it — so nothing is known about which +/// scenarios needed a GPU and only the zero-executed check applies. +#[must_use] +pub fn post_run_verdict( + executed: usize, + gpu: Option, + declaration: GpuDeclaration, +) -> Verdict { + let mut verdict = Verdict::default(); + if executed == 0 { + verdict.fatal = Some( + "0 scenarios executed: a run that executes nothing tests nothing. Check the \ + cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and E2E_ALSO_RUN_IDS, and the \ + host capability line above" + .to_owned(), + ); + return verdict; + } + let Some(gpu) = gpu else { + return verdict; + }; + match declaration { + GpuDeclaration::Expected if gpu.executed == 0 => { + verdict.fatal = Some(format!( + "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but no @requires-gpu scenario \ + executed ({} resolved, none ran)", + gpu.resolved + )); + } + GpuDeclaration::NotExpected => verdict.notes.push(format!( + "{} of {} @requires-gpu scenario(s) did not run on this lane ({EXPECT_GPU_ENV}=0)", + gpu.resolved.saturating_sub(gpu.executed), + gpu.resolved + )), + GpuDeclaration::Expected | GpuDeclaration::Undeclared => {} + } + verdict +} + +#[cfg(test)] +mod tests { + use super::{ + EXPECT_GPU_ENV, GpuDeclaration, GpuScenarioCounts, Verdict, parse_gpu_declaration, + post_run_verdict, preflight_verdict, + }; + use std::ffi::OsStr; + + const FACTS: &str = + "detected_gfx_target=, wsl=false, driver_status=, no visibility mask"; + const PROBE_ERROR: &str = "`rocm examine` exited with exit status: 1"; + + const ALL: [GpuDeclaration; 3] = [ + GpuDeclaration::Expected, + GpuDeclaration::NotExpected, + GpuDeclaration::Undeclared, + ]; + + #[test] + fn the_variable_name_is_the_one_the_workflows_set() { + assert_eq!(EXPECT_GPU_ENV, "E2E_EXPECT_GPU"); + } + + #[test] + fn only_one_zero_and_unset_or_empty_are_declarations() { + assert_eq!(parse_gpu_declaration(None), Ok(GpuDeclaration::Undeclared)); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new(""))), + Ok(GpuDeclaration::Undeclared) + ); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new("1"))), + Ok(GpuDeclaration::Expected) + ); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new("0"))), + Ok(GpuDeclaration::NotExpected) + ); + } + + #[test] + fn a_misspelt_declaration_is_an_error_not_undeclared() { + for value in ["true", "yes", " 1", "2"] { + assert_eq!( + parse_gpu_declaration(Some(OsStr::new(value))), + Err(format!( + "invalid E2E_EXPECT_GPU value {value:?}: use 1 (this lane has a GPU), 0 (it \ + deliberately has none), or leave it unset" + )), + ); + } + } + + // Row: probe failed (non-zero exit, spawn failure, empty output, or + // `engines list` failing after `examine` succeeded all reach here as one + // `probe_error`). Fatal under every declaration, GPU or not. + #[test] + fn a_failed_probe_is_fatal_under_every_declaration() { + for declaration in ALL { + for has_amd_gpu in [false, true] { + let verdict = preflight_verdict(Some(PROBE_ERROR), has_amd_gpu, FACTS, declaration); + assert_eq!( + verdict, + Verdict { + fatal: Some(format!( + "the host capability probe could not read the rocm binary under \ + test, so every capability-gated scenario would resolve to skip and \ + the run would pass without testing them: {PROBE_ERROR}" + )), + notes: vec![], + }, + "{declaration:?}, gpu={has_amd_gpu}" + ); + } + } + } + + // Row: probe succeeded, no usable GPU, declared 1. + #[test] + fn a_declared_gpu_the_probe_did_not_find_is_fatal_and_names_the_facts() { + let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::Expected); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but the capability probe found no \ + usable AMD GPU (detected_gfx_target=, wsl=false, driver_status=, \ + no visibility mask), so every @requires-gpu scenario would resolve to skip" + ) + ); + assert!(verdict.notes.is_empty()); + } + + // Row: probe succeeded with a GPU, declared 1 — runs, nothing to say. + #[test] + fn a_declared_gpu_that_is_present_runs_without_comment() { + assert_eq!( + preflight_verdict(None, true, FACTS, GpuDeclaration::Expected), + Verdict::default() + ); + } + + // Row: probe succeeded with a GPU, declared 0 — runs, with a stale note. + #[test] + fn a_declared_absent_gpu_that_is_present_is_a_note_not_a_failure() { + let verdict = preflight_verdict(None, true, FACTS, GpuDeclaration::NotExpected); + assert_eq!(verdict.fatal, None); + assert_eq!( + verdict.notes, + [format!( + "this lane declares no usable GPU (E2E_EXPECT_GPU=0) but the capability probe \ + found one ({FACTS}); the declaration is stale, and @requires-gpu scenarios will \ + run" + )] + ); + } + + // Row: probe succeeded, no GPU, declared 0 — runs, says why GPU scenarios + // are not applicable. + #[test] + fn a_declared_absent_gpu_that_is_absent_runs_with_a_note() { + let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::NotExpected); + assert_eq!(verdict.fatal, None); + assert_eq!( + verdict.notes, + [ + "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are \ + not applicable here" + ] + ); + } + + // Rows: probe succeeded, undeclared — today's behaviour, GPU or not. + #[test] + fn an_undeclared_lane_runs_without_comment_either_way() { + for has_amd_gpu in [false, true] { + assert_eq!( + preflight_verdict(None, has_amd_gpu, FACTS, GpuDeclaration::Undeclared), + Verdict::default(), + "gpu={has_amd_gpu}" + ); + } + } + + // Post-run row: nothing executed — fatal under every declaration, whether + // or not the harness's own filter ran (a `-n` filter matching nothing is + // the `None` case). + #[test] + fn a_run_that_executed_nothing_is_fatal_under_every_declaration() { + let gpu_cases = [ + None, + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }), + Some(GpuScenarioCounts { + resolved: 4, + executed: 0, + }), + ]; + for declaration in ALL { + for gpu in gpu_cases { + let verdict = post_run_verdict(0, gpu, declaration); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "0 scenarios executed: a run that executes nothing tests nothing. Check \ + the cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and \ + E2E_ALSO_RUN_IDS, and the host capability line above" + ), + "{declaration:?}, {gpu:?}" + ); + } + } + } + + // Post-run row: declared 1, scenarios ran but no GPU one did. + #[test] + fn a_gpu_lane_on_which_no_gpu_scenario_ran_is_fatal() { + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::Expected, + ); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu scenario \ + executed (7 resolved, none ran)" + ) + ); + } + + #[test] + fn a_gpu_lane_on_which_a_gpu_scenario_ran_passes() { + assert_eq!( + post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 1, + }), + GpuDeclaration::Expected, + ), + Verdict::default() + ); + } + + // Post-run row: a CLI filter bypassed the harness's filter, so which + // scenarios need a GPU is unknown — only the zero check applies. + #[test] + fn a_cli_filtered_run_is_not_held_to_the_gpu_check() { + for declaration in ALL { + assert_eq!( + post_run_verdict(3, None, declaration), + Verdict::default(), + "{declaration:?}" + ); + } + } + + #[test] + fn a_no_gpu_lane_says_how_many_gpu_scenarios_did_not_run() { + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::NotExpected, + ); + assert_eq!(verdict.fatal, None); + assert_eq!( + verdict.notes, + ["7 of 7 @requires-gpu scenario(s) did not run on this lane (E2E_EXPECT_GPU=0)"] + ); + } + + #[test] + fn an_undeclared_lane_is_not_held_to_the_gpu_check() { + assert_eq!( + post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::Undeclared, + ), + Verdict::default() + ); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 29e334583..1d10fe54b 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -12,6 +12,7 @@ pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lane_verdict; pub mod lemonade_backend; pub mod lemonade_diagnostics; pub mod lemonade_protocol; diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index ae41672ba..dfec6d765 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -260,6 +260,9 @@ struct Resolution { engine: String, feature: String, scenario: String, + /// Tagged `@requires-gpu` or `@requires-multi-gpu`, for the post-run check + /// that a GPU-declaring lane executed at least one of them. + requires_gpu: bool, } /// Resolve a CI-provided shared-directory env var to a validated, existing path. @@ -1457,6 +1460,7 @@ async fn run_suite() { use e2e_cucumber::expectation::{ Expectation, ScenarioDecl, lifecycle_lane_includes, resolve, unmatched_also_run_ids, }; + use e2e_cucumber::lane_verdict; use e2e_cucumber::monotonic_clock::MonotonicClockWriter; let dir = results_dir(); @@ -1517,6 +1521,24 @@ async fn run_suite() { cap.platform_slug, cap.os_family, cap.has_amd_gpu, cap.effective_serve_engine, ); + // Refuse to run when the result could not mean anything: a probe that could + // not read the binary, or a lane that declares a GPU the probe did not find, + // would resolve every GPU scenario to skip and pass. See `lane_verdict`. + let gpu_declaration = lane_verdict::parse_gpu_declaration( + std::env::var_os(lane_verdict::EXPECT_GPU_ENV).as_deref(), + ) + .unwrap_or_else(|error| lane_failure(&error)); + let preflight = lane_verdict::preflight_verdict( + cap.probe_error.as_deref(), + cap.has_amd_gpu, + &e2e_cucumber::capability::probe_facts_summary(cap), + gpu_declaration, + ); + print_lane_notes(&preflight.notes); + if let Some(fatal) = &preflight.fatal { + lane_failure(fatal); + } + // Shared record of each scenario's resolved expectation, keyed by @id. // Populated by `filter_run` (which sees every scenario, run or skipped) so // the post-run evaluation and platform.json can reconcile by id — including @@ -1639,6 +1661,7 @@ async fn run_suite() { engine, feature: feature.name.clone(), scenario: scenario.name.clone(), + requires_gpu: decl.requires_gpu || decl.requires_multi_gpu, }, ); // Two scenarios sharing an `@id` would silently overwrite each @@ -1751,6 +1774,27 @@ async fn run_suite() { } } + // A run with nothing to fail passes; check it executed something, and on + // a GPU-declaring lane something that needed the GPU. `resolutions` is empty + // when a cucumber `--name`/`--tags` filter replaced `filter_run`, and then + // which scenarios need a GPU is unknown. + let gpu_counts = (!resolutions.is_empty()).then(|| { + let gpu_ids: std::collections::BTreeSet<&str> = resolutions + .iter() + .filter(|(_, r)| r.requires_gpu) + .map(|(id, _)| id.as_str()) + .collect(); + lane_verdict::GpuScenarioCounts { + resolved: gpu_ids.len(), + executed: actual + .iter() + .filter(|(id, _)| gpu_ids.contains(id.as_str())) + .count(), + } + }); + let post_run = lane_verdict::post_run_verdict(actual.len(), gpu_counts, gpu_declaration); + print_lane_notes(&post_run.notes); + eprintln!( "Reconciliation: {xfail_count} xfail (failed as expected), {} XPASS ({} flaky, {} stale), {} unexpected failure(s).", flaky_xpass.len() + stale_xpass.len(), @@ -1763,7 +1807,7 @@ async fn run_suite() { "XPASS (flaky, tolerated): '{x}' passed this run; the intermittent bug remains tracked." ); } - if !stale_xpass.is_empty() || !unexpected_fail.is_empty() { + if !stale_xpass.is_empty() || !unexpected_fail.is_empty() || post_run.fatal.is_some() { for x in &stale_xpass { eprintln!( "XPASS: '{x}' is expected to fail on this host but PASSED \u{2014} the bug appears \ @@ -1775,6 +1819,27 @@ async fn run_suite() { "FAIL: '{f}' was expected to pass on this host but FAILED \u{2014} a regression." ); } + if let Some(fatal) = &post_run.fatal { + lane_failure(fatal); + } std::process::exit(1); } } + +/// Print a lane-verdict failure as a GitHub annotation and a plain line, then +/// exit non-zero. The annotation encodes newlines (a probe's stderr tail) as +/// `%0A`, which is how a workflow command carries more than one line. +fn lane_failure(message: &str) -> ! { + eprintln!( + "::error::E2E lane check failed: {}", + message.replace('\n', "%0A") + ); + eprintln!("E2E lane check failed: {message}"); + std::process::exit(1); +} + +fn print_lane_notes(notes: &[String]) { + for note in notes { + eprintln!("::notice::E2E lane: {note}"); + } +} diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index d71237d32..d3e74ccd3 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1718,6 +1718,98 @@ jobs: } } + /// Self-hosted E2E lanes that declare no usable GPU, each with the reason. + /// Every other self-hosted lane must declare one. + const NO_GPU_LANES: [(&str, &str); 2] = [ + ( + "e2e-wsl", + "the WSL distro is deliberately left without ROCm's WSL driver bridge, \ + which `rocm install driver` installs under test", + ), + ( + "e2e-wsl-nightly", + "the nightly twin of e2e-wsl, set up the same way", + ), + ]; + + /// Every self-hosted E2E lane declares whether it has a GPU, so the harness + /// can fail a GPU lane whose probe found none instead of passing it with + /// every `@requires-gpu` scenario skipped. + /// + /// Derived from the workflows like the GPU-preflight check above, so a new + /// lane is held to it the day it lands. The declaration is read from the + /// job-level `env:` mapping only: a step-level one would leave the other + /// steps, and a reader of the job header, without it. A lane that runs the + /// suite inside WSL must also name the variable in every `WSLENV` it sets, + /// since `wsl.exe` passes nothing else into the guest. + #[test] + fn every_self_hosted_lane_declares_whether_it_has_a_gpu() { + let name = e2e_report::EXPECT_GPU_ENV; + let mut no_gpu_seen = Vec::new(); + for (workflow, text) in self_hosted_workflows() { + for (job, _) in self_hosted_e2e_jobs(&text) { + let block = job_block(&text, &job); + let env = job_mapping(block, "env"); + let exempt = NO_GPU_LANES.iter().any(|(lane, _)| *lane == job); + match env.get(name).map(String::as_str) { + Some("1") => assert!( + !exempt, + "{workflow} job `{job}` is listed in NO_GPU_LANES but declares \ + {name}: \"1\"; drop it from the list" + ), + Some("0") => { + assert!( + exempt, + "{workflow} job `{job}` declares {name}: \"0\" but is not in \ + NO_GPU_LANES; a self-hosted lane without a GPU needs its reason \ + recorded there" + ); + no_gpu_seen.push(job.clone()); + } + other => panic!( + "{workflow} job `{job}` runs on self-hosted hardware but its job-level \ + `env:` sets {name} to {other:?}; it must be \"1\" (a GPU lane) or \ + \"0\" (listed in NO_GPU_LANES)" + ), + } + let wslenvs: Vec<&str> = block + .lines() + .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = ")) + .collect(); + assert!( + !block.contains("Invoke-WslBash.ps1") || !wslenvs.is_empty(), + "{workflow} job `{job}` runs commands in WSL but sets no WSLENV, so \ + {name} never reaches the suite" + ); + for wslenv in wslenvs { + let vars = wslenv.trim_matches(|c| c == '\'' || c == '"'); + assert!( + vars.split(':') + .any(|var| var.split('/').next() == Some(name)), + "{workflow} job `{job}` sets WSLENV without {name}, so the suite \ + inside WSL reads the lane as undeclared: {wslenv}" + ); + } + } + } + for (lane, _) in NO_GPU_LANES { + assert!( + no_gpu_seen.iter().any(|seen| seen == lane), + "NO_GPU_LANES lists `{lane}`, which no self-hosted workflow declares \ + {name}: \"0\"; drop the stale entry" + ); + } + // The queue lanes run through this script, not a workflow job; every + // queue lane is given a GPU. + let script = std::fs::read_to_string(repo_root().join("scripts/run-e2e-lane.sh")) + .expect("reading the lane script"); + let export = format!("export {name}=\"${{{name}-1}}\""); + assert!( + script.lines().any(|line| line.trim() == export), + "scripts/run-e2e-lane.sh must declare its lanes' GPU with `{export}`" + ); + } + /// A job's `steps:` items as `(name, whole step)` pairs, in file order; /// `None` when the job has no `steps:` list. `block` is what `job_block` /// returns. @@ -5195,14 +5287,16 @@ fi let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) .expect("read hardware testing docs"); - let documented: Vec> = - markdown_table_rows(&docs, "| Job | Workflow | Platform | Runner labels |") - .into_iter() - .filter(|row| { - row.get(1) - .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") - }) - .collect(); + let documented: Vec> = markdown_table_rows( + &docs, + "| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) |", + ) + .into_iter() + .filter(|row| { + row.get(1) + .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") + }) + .collect(); let documented_jobs: Vec = documented.iter().map(|row| row[0].clone()).collect(); let declared_jobs: Vec = lanes.iter().map(|(job, _)| format!("`{job}`")).collect(); @@ -5230,6 +5324,15 @@ fi flattened_list_items(runs_on), "the documented runner labels for `{job}` must match its actual runs-on" ); + let declared = job_mapping(job_block(&self_hosted, job), "env") + .get(e2e_report::EXPECT_GPU_ENV) + .map(|value| format!("`{value}`")); + assert_eq!( + row.get(4), + declared.as_ref(), + "the GPU declared cell for `{job}` must match its job-level {}", + e2e_report::EXPECT_GPU_ENV + ); } // Derived from the same scan `every_uploaded_e2e_artifact_has_a_name_the_report_can_label` From 73c99238be4c14418a6fb2b9d80bbd5993d043b5 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 20:02:22 +0200 Subject: [PATCH 2/5] test(e2e): bound probes, flag failed lanes (ROCMAI-388, ROCMAI-450) The lane checks added in the previous commit could still be defeated in two ways, and this closes both. A capability probe that never returned hung the whole run instead of refusing it. Each probe command (`rocm examine`, `rocm engines list`) now runs under a deadline, E2E_PROBE_TIMEOUT_SECS (whole seconds, 120 by default), and is killed and reaped when it passes. ProbeError gains Timeout, Wait (the child could not be waited on) and OutputNotClosed (the command exited but a process it started still holds its output), each naming the command and keeping the stderr read so far. The post-run version probe shares the deadline but only blanks the version columns. A lane failed by the post-run check (no scenario executed, or no @requires-gpu scenario on an E2E_EXPECT_GPU=1 lane) exited non-zero but still read green in the consolidated report when every scenario it resolved was a skip. The reason is now written to platform.json as `lane_failure`, and the report renders that lane as FAIL with the reason, in both the step summary and the HTML report. The single "Needs attention" list is split into a platform heading (lane-check failures) and a scenario heading (the expectation grid), so neither renderer shows two identical headings. Release readiness counts such a row, whose Fail cell is 0 and so never reaches the Total row, and does not count a row twice when it also has failed scenarios. Annotation and notice lines now escape `%`, CR and LF as the runner expects, so a probe's stderr tail can no longer truncate or corrupt a workflow command. The decisions the harness takes around these checks (reading the declaration, counting GPU scenarios, building the manifest, the final exit and the exact annotation lines) move out of tests/e2e.rs into pure functions in lane_verdict.rs, each unit-tested; the harness keeps only the glue that runs the probe and the scenarios and writes the files. Refs: #383 Signed-off-by: Roman Inflianskas --- crates/e2e-report/src/components.rs | 2 + crates/e2e-report/src/consolidated/html.rs | 53 +- .../e2e-report/src/consolidated/markdown.rs | 39 +- crates/e2e-report/src/consolidated/mod.rs | 271 ++++++- .../e2e-report/src/consolidated/sidecars.rs | 4 + crates/e2e-report/src/lib.rs | 4 +- docs/architecture.md | 6 +- docs/ci-hardware-testing.md | 6 + docs/testing.md | 13 +- tests/e2e-cucumber/Cargo.toml | 9 +- tests/e2e-cucumber/README.md | 23 +- tests/e2e-cucumber/src/capability.rs | 692 +++++++++++++++++- tests/e2e-cucumber/src/device_memory.rs | 3 +- tests/e2e-cucumber/src/expectation.rs | 36 + tests/e2e-cucumber/src/lane_verdict.rs | 534 +++++++++++++- tests/e2e-cucumber/tests/e2e.rs | 105 ++- xtask/src/release_readiness.rs | 62 +- 17 files changed, 1715 insertions(+), 147 deletions(-) diff --git a/crates/e2e-report/src/components.rs b/crates/e2e-report/src/components.rs index 4015af617..ab008eb6a 100644 --- a/crates/e2e-report/src/components.rs +++ b/crates/e2e-report/src/components.rs @@ -215,6 +215,8 @@ const STYLE: &str = r#" .platform-row { display: flex; align-items: center; gap: 8px; cursor: pointer; font-size: 1.05rem; } .platform-name { font-weight: 700; flex: 1; } .empty-note { color: #999; font-style: italic; padding: 8px 0; } + .lane-failure { color: #c62828; background: #ffebee; border-left: 4px solid #c62828; + padding: 8px 12px; margin: 8px 0; white-space: pre-wrap; } .scenario { border: 1px solid #e0e0e0; border-top: none; padding: 12px; background: #fff; } .scenario:last-child { border-radius: 0 0 4px 4px; } .scenario-row { display: flex; align-items: center; gap: 8px; cursor: pointer; } diff --git a/crates/e2e-report/src/consolidated/html.rs b/crates/e2e-report/src/consolidated/html.rs index 6c4717152..47e52ed5a 100644 --- a/crates/e2e-report/src/consolidated/html.rs +++ b/crates/e2e-report/src/consolidated/html.rs @@ -14,8 +14,9 @@ use std::path::Path; use maud::{Markup, html}; use super::{ - CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, PlatformReport, - PlatformReports, platform_cell, + CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, + PLATFORM_ATTENTION_HEADING, PlatformReport, PlatformReports, SCENARIO_ATTENTION_HEADING, + platform_cell, }; use crate::components::{feature_group, now_utc, page_shell, stats_bar}; @@ -103,6 +104,7 @@ pub fn generate_consolidated( h2 { "Platforms" } (matrix_table(reports)) (legend()) + (lane_failures_html(reports)) (expectation_grid_html(&bundle.grid)) @@ -193,7 +195,7 @@ pub(super) fn expectation_grid_html(grid: &Grid) -> Markup { } } @if !problems.is_empty() { - h3 { "Needs attention" } + h3 { (SCENARIO_ATTENTION_HEADING) } ul { @for (platform_label, id, outcome, detail) in &problems { li { @@ -246,7 +248,8 @@ fn matrix_table(reports: &[PlatformReport]) -> Markup { td.num { (fail) } td.num { (skip) } td.num { @if xf > 0 || r.is_known_bugs { (xf) } @else { "n/a" } } - td class=(if r.ok() { "status-pass" } else { "status-fail" }) { + td class=(if r.ok() { "status-pass" } else { "status-fail" }) + title=[r.lane_failure()] { (r.status_text()) } td { (stats_bar(&r.stats)) } @@ -288,7 +291,10 @@ fn legend() -> Markup { " — PASS when the platform is in its expected state (every " "expect-pass scenario passed and every known bug failed as " "expected); FAIL on an unexpected failure or an XPASS (a known " - "bug that unexpectedly passed — remove its @expected-failure tag)." + "bug that unexpectedly passed — remove its @expected-failure tag), " + "or when the harness failed the lane after the run (for example, no " + "scenario executed); the reason is listed under \u{201c}" + (PLATFORM_ATTENTION_HEADING) "\u{201d} and in the platform's section." } li { b { "Xfail" } " — count of known-bug scenarios that failed as expected." } li { b { "Skip" } " — scenarios not applicable on this platform (not run)." } @@ -297,6 +303,26 @@ fn legend() -> Markup { } } +/// The platforms the harness failed after their run, with each reason: a +/// lane-check failure has no grid cell of its own (an all-skip lane has only +/// skip cells), so it is listed here. +fn lane_failures_html(reports: &[PlatformReport]) -> Markup { + let failed: Vec<(&str, &str)> = reports + .iter() + .filter_map(|r| r.lane_failure().map(|reason| (r.label.as_str(), reason))) + .collect(); + html! { + @if !failed.is_empty() { + h3 { (PLATFORM_ATTENTION_HEADING) } + ul { + @for (label, reason) in &failed { + li { b { "Lane check failed" } " in " code { (label) } ": " (reason) } + } + } + } + } +} + fn platform_section(report: &PlatformReport) -> Markup { let badge_class = if report.ok() { "badge-pass" @@ -310,6 +336,9 @@ fn platform_section(report: &PlatformReport) -> Markup { span.platform-name { (report.label) } span.elapsed { (report.stats.total) " scenarios" } } + @if let Some(reason) = report.lane_failure() { + p.lane-failure { b { "Lane check failed: " } (reason) } + } @if report.features.is_empty() { p.empty-note { "No report.json data for this platform." } } @else { @@ -365,12 +394,18 @@ mod tests { // now would be building the bundle's grid from `&[]` in // `load_platform_reports`. Give this one a platform.json via // write_platform and assert the grid heading actually renders. - let report = feature_json(&[(&["id:serve-a"], &["passed"])]); + // `serve-b` fails though expected to pass, so the grid's attention + // list renders too. + let report = feature_json(&[ + (&["id:serve-a"], &["passed"]), + (&["id:serve-b"], &["failed"]), + ]); let platform = r#"{ "platform_slug": "mock", "capability": {"effective_serve_engine": "none"}, "expectations": [ - {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"} + {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"}, + {"id":"serve-b","feature":"Serving","scenario":"serve-02 - second","expected":"pass"} ] }"#; let (_d, path) = write_platform(&report, platform); @@ -384,6 +419,10 @@ mod tests { html.contains("Expectation grid (scenario × platform)"), "grid heading missing from output:\n{html}" ); + assert!( + html.contains(&format!("

{SCENARIO_ATTENTION_HEADING}

")), + "scenario attention heading missing from output:\n{html}" + ); } #[test] diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs index d2d041373..af90cab7b 100644 --- a/crates/e2e-report/src/consolidated/markdown.rs +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -11,8 +11,9 @@ //! crate root by `lib.rs`. use super::{ - CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, PlatformReports, - ScenarioReference, platform_cell, + CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, + PLATFORM_ATTENTION_HEADING, PlatformReports, SCENARIO_ATTENTION_HEADING, ScenarioReference, + platform_cell, }; use crate::command_coverage::command_coverage_markdown; @@ -113,13 +114,22 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ failed, or a known bug that unexpectedly passed (its tag is now stale). \ - **Status** is PASS unless a platform has any Fail.\n", + **Status** is PASS unless a platform has any Fail, or the harness failed \ + its lane check after the run (listed under {PLATFORM_ATTENTION_HEADING}).\n", ); - // Call out anything that needs a human: XPASS (fixed bug, stale tag) and + // Call out anything that needs a human: a failed post-run lane check + // (`lane_failure` in platform.json), XPASS (fixed bug, stale tag) and // untagged failures in a known-bugs run. let mut notes = Vec::new(); for r in reports { + if let Some(reason) = r.lane_failure() { + notes.push(format!( + "- **Lane check failed** in _{}_: {}", + r.label, + reason.replace('\n', " ") + )); + } for name in &r.xfail.xpass { notes.push(format!( "- **XPASS** in _{}_: `{}` is tagged `@expected-failure` but passed — remove the tag.", @@ -136,7 +146,7 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { } } if !notes.is_empty() { - out.push_str("\n### Needs attention\n\n"); + let _ = write!(out, "\n### {PLATFORM_ATTENTION_HEADING}\n\n"); for n in notes { out.push_str(&n); out.push('\n'); @@ -244,7 +254,7 @@ fn expectation_grid_markdown(grid: &Grid, scenarios: &ScenarioReference) -> Stri // Needs-attention list from reconciliation problems. let problems = grid.problems(); if !problems.is_empty() { - out.push_str("\n### Needs attention\n\n"); + let _ = write!(out, "\n### {SCENARIO_ATTENTION_HEADING}\n\n"); for (platform_label, id, outcome, detail) in problems { let bug = detail .and_then(|d| d.bug.as_deref()) @@ -360,8 +370,10 @@ mod tests { b.path().to_path_buf(), )]; let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("Needs attention")); - assert!(md.contains("XPASS")); + assert!( + md.contains("\n### Needs attention: platforms\n\n- **XPASS** in _"), + "the XPASS bullet must sit directly under the platform attention heading:\n{md}" + ); } #[test] @@ -394,7 +406,12 @@ mod tests { reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ failed, or a known bug that unexpectedly passed (its tag is now stale). \ - **Status** is PASS unless a platform has any Fail.\n"; + **Status** is PASS unless a platform has any Fail, or the harness failed \ + its lane check after the run (listed under Needs attention: platforms).\n"; + // Pins the wording only. That a lane-check failure turns Status FAIL and + // is listed under "Needs attention: platforms" is proved by + // `consolidated::tests::an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason` + // and `consolidated::tests::a_recorded_lane_failure_is_named_in_the_summary_and_the_html`. assert!( md.contains(expected), "summary explanation text changed:\n{md}" @@ -410,6 +427,10 @@ mod tests { md.contains(OTHER_LANES_NON_BLOCKING), "summary must render OTHER_LANES_NON_BLOCKING ({OTHER_LANES_NON_BLOCKING:?}):\n{md}" ); + assert!( + md.contains(&format!("(listed under {PLATFORM_ATTENTION_HEADING})")), + "summary must render PLATFORM_ATTENTION_HEADING ({PLATFORM_ATTENTION_HEADING:?}):\n{md}" + ); } #[test] diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 74fdb1fb0..cfab6c013 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -194,6 +194,14 @@ const MOCK_GATES_PR: &str = "gates the PR"; /// See [`MOCK_GATES_PR`]. const OTHER_LANES_NON_BLOCKING: &str = "non-blocking"; +/// Heading of the platform-level attention list (lane-check failures; in the +/// markdown summary also stale `@expected-failure` tags and regressions). +/// Distinct from [`SCENARIO_ATTENTION_HEADING`], so a report with both lists +/// never shows two identical headings. Shared by both renderers. +const PLATFORM_ATTENTION_HEADING: &str = "Needs attention: platforms"; +/// Heading of the expectation grid's per-scenario attention list. +const SCENARIO_ATTENTION_HEADING: &str = "Needs attention: scenarios"; + /// One platform column of the reconciled (scenario-id × platform) grid. struct GridColumn { /// Platform identity from the manifest (e.g. "mi300x", "strix-halo", "mock"). @@ -571,12 +579,17 @@ struct ReconciledTally { skip: u32, /// Cells that need a human: unexpected-fail, XPASS, or ran-when-N/A. problems: u32, + /// The harness failed the run after it finished (`lane_failure` in + /// `platform.json`), e.g. because no scenario executed. No cell shows that: + /// a run whose every scenario resolved to skip has only skip cells. + lane_failed: bool, } impl ReconciledTally { - /// `true` when nothing needs attention (no unexpected-fail / XPASS / ran-when-NA). + /// `true` when nothing needs attention: no unexpected-fail / XPASS / + /// ran-when-NA cell, and no failed lane check. const fn ok(&self) -> bool { - self.problems == 0 + self.problems == 0 && !self.lane_failed } const fn status_text(&self) -> &'static str { @@ -592,7 +605,10 @@ fn reconciled_tally( manifest: &PlatformManifest, actual: &std::collections::HashMap, ) -> ReconciledTally { - let mut t = ReconciledTally::default(); + let mut t = ReconciledTally { + lane_failed: manifest.lane_failure.is_some(), + ..ReconciledTally::default() + }; for exp in &manifest.expectations { match CellOutcome::reconcile(&exp.expected, exp.flaky, actual.get(&exp.id).copied()) { CellOutcome::Pass => t.pass += 1, @@ -662,6 +678,12 @@ impl PlatformReport { } } + /// Why the harness's post-run lane check failed this run, from + /// `platform.json`; `None` when it passed or the artifact predates it. + fn lane_failure(&self) -> Option<&str> { + self.manifest.as_ref()?.lane_failure.as_deref() + } + /// The channel that governs this platform's rendering. See /// [`resolve_channel`]. fn effective_channel(&self) -> Option<&str> { @@ -713,9 +735,11 @@ impl PlatformReport { /// Display counts for the summary table, reconciliation-aware so the numbers /// agree with [`Self::status_text`]. Returns `(total, pass, fail, skip, /// xfail)`. With a `platform.json`, `fail` counts only cells needing - /// attention (unexpected-fail / XPASS / ran-when-NA) — a known bug failing as - /// expected lands in `xfail`, not `fail`. Without one, falls back to raw - /// junit stats. + /// attention (unexpected-fail / XPASS / ran-when-NA / no result) — a known + /// bug failing as expected lands in `xfail`, not `fail`. A post-run lane + /// failure (`lane_failure`) has no cell, so such a row can read `FAIL` + /// with a `fail` of 0. Without a `platform.json`, falls back to raw junit + /// stats. const fn display_counts(&self) -> (u32, u32, u32, u32, u32) { if let Some(t) = &self.tally { let total = t.pass + t.xfail + t.skip + t.problems; @@ -767,6 +791,32 @@ pub struct PlatformReports { scenarios: ScenarioReference, } +/// One platform's health as the consolidated report renders it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlatformStatus { + /// Whether the row is green. + pub ok: bool, + /// The Status cell: `PASS`, `FAIL` or `EMPTY`. + pub status: &'static str, + /// The recorded lane-check failure, if any. + pub lane_failure: Option, +} + +/// Load one platform's `report.json` (and its sibling `platform.json`) the way +/// the consolidated report does, and return its status. +/// +/// Public so the harness that writes `platform.json` can test that what it +/// writes is read back as intended, keeping producer and consumer tied. +#[must_use] +pub fn platform_status(label: &str, report_json: &Path) -> PlatformStatus { + let report = PlatformReport::load(label.to_owned(), report_json); + PlatformStatus { + ok: report.ok(), + status: report.status_text(), + lane_failure: report.lane_failure().map(str::to_owned), + } +} + /// Parse every `(label, json_path)` input into a [`PlatformReport`], in input /// order. The one place `report.json`/`platform.json`/`commands.jsonl` are /// read from disk — [`Grid::build`] and [`scenario_reference`] consume the @@ -897,10 +947,16 @@ mod tests { assert_eq!(r.status_text(), "EMPTY"); } - /// The artifact a lane uploads when the harness refuses to run (a failed - /// capability probe, or a declared GPU the probe did not find): the harness - /// creates `report.json` empty before probing and exits before writing - /// `platform.json`. That lane must read as needing attention, never green. + /// The artifact a lane uploads when the harness's PRE-run check refuses to + /// run (a failed capability probe, a declared GPU the probe did not find, + /// an invalid `E2E_EXPECT_GPU`): the harness creates `report.json` empty + /// before probing and exits before writing `platform.json`. That lane must + /// read as needing attention, never green. A POST-run lane-check failure + /// writes `platform.json` with `lane_failure` instead; see the tests below. + /// + /// The report code already behaved this way before the harness could exit + /// early, so this is not a regression test; it pins that behaviour because + /// the early exit now relies on it to keep a refused lane off green in the grid. #[test] fn a_lane_that_stopped_before_any_scenario_is_not_ok() { let dir = tempfile::tempdir().expect("temp dir"); @@ -912,6 +968,201 @@ mod tests { assert_eq!(r.status_text(), "EMPTY"); } + /// `platform.json` for a lane whose every resolved scenario was a skip, + /// with or without a recorded post-run lane failure. + fn all_skip_platform(lane_failure: Option<&str>) -> String { + let failure = lane_failure.map_or(String::new(), |reason| { + format!(r#""lane_failure": {},"#, serde_json::json!(reason)) + }); + format!( + r#"{{"platform_slug":"mi300x","capability":{{"effective_serve_engine":"vllm"}},{failure} + "expectations":[{{"id":"serve-a","feature":"Serving","expected":"skip"}}]}}"# + ) + } + + /// The regression: a GPU lane on which every scenario resolved to skip + /// was failed by the harness ("no @requires-gpu scenario executed"), yet + /// its row read PASS, since a skip cell is not a problem. + #[test] + fn an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason() { + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu \ + scenario executed (1 resolved, none ran)"; + let (_dir, report) = write_platform("[]", &all_skip_platform(Some(reason))); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + assert_eq!(r.lane_failure(), Some(reason)); + assert_eq!( + platform_status("e2e-gpu-report", &report), + PlatformStatus { + ok: false, + status: "FAIL", + lane_failure: Some(reason.to_owned()), + } + ); + } + + /// Unchanged: an all-skip lane the harness did not fail (an undeclared or + /// `E2E_EXPECT_GPU=0` lane) stays green. + #[test] + fn an_all_skip_lane_without_a_lane_failure_still_reads_pass() { + let (dir, report) = write_platform("[]", &all_skip_platform(None)); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(r.ok()); + assert_eq!(r.status_text(), "PASS"); + assert_eq!(r.lane_failure(), None); + + // ...and neither rendering shows any lane-failure element. (The legend + // and the explanation paragraph name the heading in prose, so the + // checks are on the elements themselves.) + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert!(!md.contains("### Needs attention: platforms"), "{md}"); + assert!(!md.contains("**Lane check failed**"), "{md}"); + let html_out = dir.path().join("out.html"); + html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default()) + .expect("html written"); + let html = std::fs::read_to_string(&html_out).expect("html read"); + assert!( + !html.contains("

Needs attention: platforms

"), + "{html}" + ); + assert!( + html.contains(r#"PASS"#), + "the Status cell carries no tooltip:\n{html}" + ); + assert!(!html.contains(r#"

"#), "{html}"); + } + + /// A `-n` filter that matched nothing: no expectations were resolved and + /// nothing ran, and the harness recorded "0 scenarios executed". + #[test] + fn a_zero_expectation_lane_reads_fail_only_with_a_recorded_failure() { + let failed = + r#"{"platform_slug":"mock","lane_failure":"0 scenarios executed","expectations":[]}"#; + let (_dir, report) = write_platform("[]", failed); + let r = PlatformReport::load("e2e-report".into(), &report); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + + let (_dir2, report) = write_platform("[]", r#"{"platform_slug":"mock","expectations":[]}"#); + let r = PlatformReport::load("e2e-report".into(), &report); + assert!(r.ok(), "no recorded failure keeps today's reading"); + assert_eq!(r.status_text(), "PASS"); + } + + /// Both renderings name the reason, not just a FAIL. + #[test] + fn a_recorded_lane_failure_is_named_in_the_summary_and_the_html() { + // A newline, markup characters and a quote, so both the markdown's + // newline folding and the HTML escaping are exercised. + let reason = "0 scenarios executed\nsee & \"tail\""; + let html_reason = "0 scenarios executed\nsee <stderr> & "tail""; + let (dir, report) = write_platform("[]", &all_skip_platform(Some(reason))); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + + let md = markdown::consolidated_summary_markdown(&bundle); + assert!( + md.contains( + "\n### Needs attention: platforms\n\n- **Lane check failed** in _MI300X \ + Linux_: 0 scenarios executed see & \"tail\"\n" + ), + "{md}" + ); + + let html_out = dir.path().join("out.html"); + html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default()) + .expect("html written"); + let html = std::fs::read_to_string(&html_out).expect("html read"); + // The Status cell's tooltip. + assert!( + html.contains(&format!( + r#"FAIL"# + )), + "status tooltip:\n{html}" + ); + // The platform section's paragraph. + assert!( + html.contains(&format!( + r#"

Lane check failed: {html_reason}

"# + )), + "lane failure paragraph:\n{html}" + ); + // The platform-level attention list. + assert!( + html.contains(&format!( + r"

Needs attention: platforms

  • Lane check failed in MI300X Linux: {html_reason}
" + )), + "attention list:\n{html}" + ); + // ...styled, so it reads as a failure. + assert!( + html.contains(".lane-failure {"), + "CSS rule for .lane-failure" + ); + // The legend says where the reason is. + assert!( + html.contains( + "or when the harness failed the lane after the run (for example, no scenario \ + executed); the reason is listed under \u{201c}Needs attention: platforms\u{201d} \ + and in the platform's section." + ), + "legend:\n{html}" + ); + } + + /// A lane on which scenarios ran and passed, but which the harness still + /// failed (on a GPU lane, no `@requires-gpu` scenario among them): its row + /// reads FAIL with a Fail count of 0, since no cell failed. Rows of exactly + /// this shape are what `total_e2e_failures` in xtask counts as failures. + #[test] + fn a_lane_that_ran_scenarios_and_was_failed_reads_fail_with_zero_failed_cells() { + let reason = "no @requires-gpu scenario executed (3 resolved, none ran)"; + let platform = format!( + r#"{{"platform_slug":"mi300x","lane_failure":{},"expectations":[ + {{"id":"examine-a","feature":"F","expected":"pass"}}]}}"#, + serde_json::json!(reason) + ); + let (_dir, report) = write_platform( + &feature_json(&[(&["id:examine-a"], &["passed"])]), + &platform, + ); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert_eq!(r.display_counts(), (1, 1, 0, 0, 0)); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert!( + md.contains("| MI300X | Linux | 1 | 1 | 0 | 0 | n/a | FAIL |"), + "{md}" + ); + assert!(md.contains("| **Total** | | 1 | 1 | 0 | 0 | 0 | |"), "{md}"); + } + + /// Both attention lists can appear in one summary, under different + /// headings, each once. + #[test] + fn the_two_attention_lists_have_distinct_headings() { + let platform = r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed", + "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#; + let (_dir, report) = + write_platform(&feature_json(&[(&["id:examine-a"], &["failed"])]), platform); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert_eq!( + md.matches("### Needs attention: platforms\n").count(), + 1, + "{md}" + ); + assert_eq!( + md.matches("### Needs attention: scenarios\n").count(), + 1, + "{md}" + ); + assert_eq!(md.matches("### Needs attention\n").count(), 0, "{md}"); + } + #[test] fn rendering_uses_the_bundle_not_the_artifacts_on_disk() { // The render functions must work from what `load_platform_reports` diff --git a/crates/e2e-report/src/consolidated/sidecars.rs b/crates/e2e-report/src/consolidated/sidecars.rs index b1ff32386..28de5088e 100644 --- a/crates/e2e-report/src/consolidated/sidecars.rs +++ b/crates/e2e-report/src/consolidated/sidecars.rs @@ -54,6 +54,10 @@ pub(super) struct PlatformManifest { /// Component versions (OS/ROCm/vLLM/lemonade); absent in older artifacts. #[serde(default)] pub(super) versions: PlatformVersions, + /// Why the harness's post-run lane check failed the run (for example "0 + /// scenarios executed"); absent when it passed, and in older artifacts. + #[serde(default)] + pub(super) lane_failure: Option, #[serde(default)] pub(super) expectations: Vec, } diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index cbe1403af..3b16cbfbb 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -17,8 +17,8 @@ mod single_report; pub use consolidated::html::{RunMeta, generate_consolidated}; pub use consolidated::markdown::consolidated_summary_markdown; pub use consolidated::{ - ADVISORY_QUEUE_MARKER, PlatformReports, artifact_name_for_platform_slug, load_platform_reports, - queue_artifact_name, queue_lane_gpu, + ADVISORY_QUEUE_MARKER, PlatformReports, PlatformStatus, artifact_name_for_platform_slug, + load_platform_reports, platform_status, queue_artifact_name, queue_lane_gpu, }; pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id}; pub use single_report::generate; diff --git a/docs/architecture.md b/docs/architecture.md index 981f8eb6b..666d38f81 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -633,7 +633,7 @@ HTML and markdown reporting for the cucumber E2E suite, consumed only by `xtask` | [`lib.rs`](../crates/e2e-report/src/lib.rs) | Crate root: module declarations and the public API re-exports. | flat | | [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) | The CLI command-coverage audit table: which `rocm` commands a platform's run exercised and whether they passed. | domain | | [`components.rs`](../crates/e2e-report/src/components.rs) | Shared maud HTML fragments plus the CSS and timestamp helpers both report generators use. | flat | -| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, plus the report text both renderers share. | domain | +| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, `platform_status` (one platform's status, public so the harness can test the `platform.json` it writes), plus the report text both renderers share. | domain | | [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) | Artifact-name identity: parses an artifact or directory name into platform, OS and channel, and owns the platform-label rule. | domain | | [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) | The HTML renderer for the consolidated report. It does not render `PlatformVersions::provenance()`: the GFX target and package source appear only in the Markdown output (the wiki page and the step summary that shares its renderer). | domain | | [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) | The markdown renderer, including the table piped into `$GITHUB_STEP_SUMMARY`. | domain | @@ -786,7 +786,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | --- | --- | --- | | [`src/lib.rs`](../tests/e2e-cucumber/src/lib.rs) | Library root for the test targets and the demo binary: declares the helper modules, re-exports `e2e-report` as `report`, and holds small shared helpers such as `cli_failure_report`. | flat | | [`src/binary_path.rs`](../tests/e2e-cucumber/src/binary_path.rs) | Pins a relative `ROCM_CLI_BINARY` path before a scenario runs the child from another directory, and leaves a bare name for `PATH` to resolve. | flat | -| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, records why either could not be read (`ProbeError`: not started, non-zero exit, no output), and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | +| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, records why either could not be read (`ProbeError`: not started, or could not be waited on, non-zero exit, no output, still running at its deadline (killed and reaped), or output left open after exit), keeping the stderr read so far, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | | [`src/chat_request.rs`](../tests/e2e-cucumber/src/chat_request.rs) | The chat round trip against a served endpoint (model discovery, then one chat completion), run on its own task so its timeout measures the server. | flat | | [`src/dash_screen.rs`](../tests/e2e-cucumber/src/dash_screen.rs) | Pure matching over captured dashboard screens and config files, so the logic the dashboard scenarios depend on has unit tests. | flat | | [`src/device_memory.rs`](../tests/e2e-cucumber/src/device_memory.rs) | Which device memory pool (VRAM, or GTT-backed memory on an APU) the drain-wait between serve scenarios judges, how it is sampled, and the free-memory floor. | flat | @@ -798,7 +798,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat | | [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat | | [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat | -| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no `@requires-gpu` scenario) executed. | flat | +| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no `@requires-gpu` scenario) executed. Also holds the decisions [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) takes around those checks, each unit-tested: reading the declaration, counting GPU scenarios, building the `platform.json` manifest that records a lane failure (`finish_run`), the final exit, and the exact escaped `::error::`/`::notice::` lines. The harness itself runs the probe and the scenarios, classifies their results, writes the files and exits; that glue has no test. | flat | | [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat | | [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat | | [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat | diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 766ab12bd..52f0cab64 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -642,6 +642,12 @@ job, naming the lane, rather than silently dropping out of the grid; on a `continue-on-error: true` lane that is the only red signal for such a failure. The nightly report job applies the same check to every nightly lane. +A lane the harness failed after its run (no scenario executed, or no +`@requires-gpu` scenario on an `E2E_EXPECT_GPU=1` lane) records the reason in +its `platform.json`, and its row reads FAIL with the reason under "Needs +attention: platforms" (step summary and HTML report alike), even when every +scenario it resolved was a skip. + ### Timeouts on the Strix lanes Every Strix Halo lane raises two budgets rather than letting a slow host read diff --git a/docs/testing.md b/docs/testing.md index 6200d2ce9..ddef6f00d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -958,9 +958,16 @@ reporting it green: - before any scenario runs, when the capability probe could not read the binary under test: `rocm examine` or `rocm engines list` could not be started (check - `ROCM_CLI_BINARY`), exited non-zero, or printed nothing; + `ROCM_CLI_BINARY`), or could not be waited on, exited non-zero, printed + nothing, was still running after `E2E_PROBE_TIMEOUT_SECS` (whole seconds ≥ 1; + anything else uses 120), or exited but left its output open. The post-run + version probe (`collect_versions`) uses the same deadline but fails quietly, + only blanking the version columns, and does not refuse the run; - after the run, when no scenario executed, which includes a - `cargo xtask e2e -- -n ` whose regex matches no scenario name. + `cargo xtask e2e -- -n ` whose regex matches no scenario name. The + reason is written into `platform.json`, so `cargo xtask e2e-report` shows that + lane as FAIL with the reason under "Needs attention: platforms", in both its + markdown summary and its HTML report. `E2E_EXPECT_GPU` declares whether the host has a GPU. With `1`, the run also fails before any scenario if the probe found no usable AMD GPU, and after the @@ -975,7 +982,7 @@ and, for each way a probe can fail, in ```bash cargo test -p e2e-cucumber --lib lane_verdict -cargo test -p e2e-cucumber --lib capability::tests::a_probe +cargo test -p e2e-cucumber --lib capability::tests cargo test -p xtask every_self_hosted_lane_declares_whether_it_has_a_gpu ``` diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 3efc130f5..cbe30a22f 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -90,10 +90,11 @@ shlex.workspace = true # itself, and `xtask check-crate-edges` exempts dev edges by design — so keeping # it here needs no allowlist entry. # -# For the same reason the library's `device_memory::bounded_stdout` cannot call -# `rocm_core::process::run_with_timeout`: it repeats that helper's spawn, poll, -# kill and reap steps on `std` alone, deliberately, and a normal dependency -# would be a new edge to review. A change to either one should look at the other. +# For the same reason the library's `device_memory::bounded_stdout` and +# `capability::output_within` cannot call `rocm_core::process::run_with_timeout`: +# each repeats that helper's spawn, poll, kill and reap steps on `std` alone, +# deliberately, and a normal dependency would be a new edge to review. A change +# to any one of the three should look at the other two. rocm-core = { path = "../../crates/rocm-core" } # `test-util` pauses tokio's clock in the retry loops' unit tests # (`#[tokio::test(start_paused = true)]`), so their timing assertions read a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index f10410125..16fa7429a 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -101,6 +101,7 @@ E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e | `ROCM_CLI_CONFIG_DIR` | (temp dir) | Isolated config directory per scenario | | `ROCM_CLI_DATA_DIR` | (temp dir) | Isolated data directory per scenario | | `ROCM_CLI_CACHE_DIR` | (temp dir) | Isolated cache directory per scenario | +| `E2E_PROBE_TIMEOUT_SECS` | 120 | How long each capability-probe command (`rocm examine`, `rocm engines list`) may run before it is killed and the run is refused: whole seconds ≥ 1; anything else uses 120. The post-run version probe (`collect_versions`) uses the same deadline but fails quietly, only blanking the version columns, and does not refuse the run | | `E2E_EXPECT_GPU` | unset (undeclared) | Whether the lane has a GPU: `1` = GPU expected, `0` = deliberately none, unset or empty = no claim; any other value fails the run. Set on every self-hosted CI lane; see "Lane verdict" below | ### Lane verdict @@ -111,14 +112,24 @@ scenario was skipped. The harness checks the run itself, before and after the scenarios: - **Before any scenario runs**, it fails when the capability probe could not - read the binary under test (`rocm examine` or `rocm engines list` could not be - started, exited non-zero, or printed nothing), naming the command, its exit - status and the tail of its stderr. On a lane that sets `E2E_EXPECT_GPU=1` it - also fails when the probe found no usable AMD GPU, naming the detected gfx - target, WSL flag, driver status and visibility mask it decided from. + read the binary under test, naming the command and saying why: + `rocm examine` or `rocm engines list` could not be started (with the error), + or could not be waited on, exited non-zero (with the exit + status and the tail of its stderr), printed nothing, was still running after + `E2E_PROBE_TIMEOUT_SECS` (default 120; it is killed, and the message gives + the deadline and the stderr tail so far), or exited but left its output + open, held by a process it started (with the stderr tail so far). On a lane + that sets `E2E_EXPECT_GPU=1` it also fails when the probe found no usable AMD + GPU, naming the detected gfx target, WSL flag, driver status and visibility + mask it decided from. A run refused here writes no `platform.json`, and the + consolidated report shows it as EMPTY. - **After the run**, it fails when no scenario executed (for example a `-n` filter that matches nothing), and on an `E2E_EXPECT_GPU=1` lane when no - `@requires-gpu` scenario executed. + `@requires-gpu` scenario executed. The reason is also recorded as + `lane_failure` in `platform.json`, so the consolidated report shows the lane + as FAIL, even when every scenario it resolved was a skip, and gives the reason + under "Needs attention: platforms" (in both the step summary and the HTML + report) and in the lane's section of the HTML report. `E2E_EXPECT_GPU=0` is for a lane that deliberately has no usable GPU, such as the WSL lanes: the harness prints a note that `@requires-gpu` scenarios are not diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 9e9fdfed4..aa8f9fb04 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -16,7 +16,7 @@ //! changes engine support, so the unit tests below guard it. //! //! The AMD GPU *count* is re-implemented for the same reason (see -//! [`probe_amd_gpu_count`]): no `rocm` command reports one, so a scenario whose +//! `probe_amd_gpu_count`): no `rocm` command reports one, so a scenario whose //! premise is a multi-GPU host has nothing else to gate on. //! //! `examine`'s `default_engine` now reports the host's real engine rather than the @@ -28,10 +28,12 @@ use std::sync::OnceLock; -/// Name of the `rocm` binary under test (mirrors the test binary's -/// `rocm_binary()`): `ROCM_CLI_BINARY` or plain `rocm`. -fn rocm_binary() -> String { - std::env::var("ROCM_CLI_BINARY").unwrap_or_else(|_| "rocm".to_string()) +/// Name of the `rocm` binary under test: `ROCM_CLI_BINARY` or plain `rocm`, +/// read through `env` so a test can supply it without a process-wide variable. +/// Like the test binary's `rocm_binary()`, except that a value that is not +/// UTF-8 is used as given here rather than replaced by `rocm`. +fn rocm_binary(env: &impl Fn(&str) -> Option) -> std::ffi::OsString { + env("ROCM_CLI_BINARY").unwrap_or_else(|| "rocm".into()) } /// Whether a container runtime is usable here. @@ -132,11 +134,11 @@ pub struct HostCapability { pub gfx_target: Option, /// Whether an AMD GPU is present AND usable here, so `@requires-gpu` /// scenarios can run. A real `detected_gfx_target`, plus a ready ROCm - /// driver path on WSL — see [`host_has_usable_gpu`]. + /// driver path on WSL — see `host_has_usable_gpu`. pub has_amd_gpu: bool, /// How many AMD GPUs are PRESENT on this host, before any visibility mask. /// `None` where the count could not be determined (non-Linux, unreadable - /// sysfs). Gates `@requires-multi-gpu`; see [`probe_amd_gpu_count`]. + /// sysfs). Gates `@requires-multi-gpu`; see `probe_amd_gpu_count`. pub amd_gpu_count: Option, /// Engine adapters the binary reports as present. Both builtins are always /// "built-in", so this is NOT the same as "can start here" — use @@ -154,18 +156,18 @@ pub struct HostCapability { /// exposes it), so `xtask e2e` reports it via the [`OOM_FAULT_INJECTION_ENV`] /// environment variable — set only when xtask built the binary itself with /// the feature, and absent for a prebuilt `ROCM_CLI_BINARY` (the self-hosted - /// lanes' shipping release build). See [`probe_host_capability`]. + /// lanes' shipping release build). See `probe_host_capability`. pub oom_fault_injection: bool, /// Whether two file names differing only in letter case are two files under /// the temp root the scenarios' isolated roots are created in. Gates - /// `@requires-case-sensitive-fs`; see [`probe_case_sensitive_fs`]. + /// `@requires-case-sensitive-fs`; see `probe_case_sensitive_fs`. pub case_sensitive_fs: bool, /// Whether a directory made read-only refuses new files to the user /// running the suite. Gates `@requires-dir-permissions`; see - /// [`probe_dir_permissions_bind`]. + /// `probe_dir_permissions_bind`. pub dir_permissions_bind: bool, /// `examine`'s `driver_status:` line, empty when it printed none. Only - /// decides GPU usability on WSL (see [`host_has_usable_gpu`]); kept so a + /// decides GPU usability on WSL (see `host_has_usable_gpu`); kept so a /// lane whose GPU went missing can say why. pub driver_status: String, /// Why the probe could not read the binary under test, if it could not: @@ -623,9 +625,10 @@ fn probe_case_sensitive_fs(dir: &std::path::Path) -> bool { /// Why a probe of the binary under test produced nothing the harness can read. /// -/// Every variant is fatal to the run: an unreadable probe parses as a host with -/// no GPU, so every capability-gated scenario would resolve to skip and the run -/// would pass without testing them. `command` is the binary and arguments as +/// Every variant is fatal to the run unless the caller ignores it, as +/// `collect_versions` does: an unreadable probe parses as a host with no GPU, +/// so every capability-gated scenario would resolve to skip and the run would +/// pass without testing them. `command` is the binary and arguments as /// run, so the message names exactly what failed. #[derive(Debug)] pub enum ProbeError { @@ -643,6 +646,26 @@ pub enum ProbeError { }, /// It exited 0 but printed nothing on stdout. EmptyOutput { command: String }, + /// It was still running when the deadline ([`PROBE_TIMEOUT_ENV`]) passed, + /// so it was killed and reaped. `stderr_tail` is what it had written to + /// stderr by then. + Timeout { + command: String, + after: std::time::Duration, + stderr_tail: String, + }, + /// It started, but waiting for it to exit failed. + Wait { + command: String, + source: std::io::Error, + }, + /// It exited 0, but its stdout was still open `grace` later: a process it + /// started is holding the pipe, so what it printed cannot be read in full. + OutputNotClosed { + command: String, + grace: std::time::Duration, + stderr_tail: String, + }, } impl std::fmt::Display for ProbeError { @@ -663,6 +686,31 @@ impl std::fmt::Display for ProbeError { "`{command}` exited with {status}\n{}", crate::section("stderr (last lines)", stderr_tail) ), + Self::Timeout { + command, + after, + stderr_tail, + } => write!( + f, + "`{command}` was still running after {after:?}, so it was killed; set \ + {PROBE_TIMEOUT_ENV} to allow it longer\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::Wait { command, source } => write!( + f, + "`{command}` was started, but waiting for it to exit failed: {source}" + ), + Self::OutputNotClosed { + command, + grace, + stderr_tail, + } => write!( + f, + "`{command}` exited 0, but its stdout was still open {grace:?} later (a \ + process it started may be holding it), so what it printed cannot be read \ + in full\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), Self::EmptyOutput { command } => { write!(f, "`{command}` exited 0 but produced no output on stdout") } @@ -673,15 +721,25 @@ impl std::fmt::Display for ProbeError { impl std::error::Error for ProbeError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { - Self::Spawn { source, .. } => Some(source), - Self::Exit { .. } | Self::EmptyOutput { .. } => None, + Self::Spawn { source, .. } | Self::Wait { source, .. } => Some(source), + Self::Exit { .. } + | Self::EmptyOutput { .. } + | Self::Timeout { .. } + | Self::OutputNotClosed { .. } => None, } } } -/// How many trailing stderr lines a [`ProbeError::Exit`] keeps. +/// How many trailing stderr lines a [`ProbeError`] keeps. const PROBE_STDERR_TAIL_LINES: usize = 20; +/// The last [`PROBE_STDERR_TAIL_LINES`] lines of a probe's stderr. +fn stderr_tail(stderr: &[u8]) -> String { + let stderr = String::from_utf8_lossy(stderr); + let lines: Vec<&str> = stderr.lines().collect(); + lines[lines.len().saturating_sub(PROBE_STDERR_TAIL_LINES)..].join("\n") +} + /// Turn one probe's spawn result into its stdout, or the reason it has none. /// /// Pure, so each outcome is unit-tested without a binary to run. Stdout that is @@ -695,13 +753,10 @@ pub fn classify_probe_output( source, })?; if !out.status.success() { - let stderr = String::from_utf8_lossy(&out.stderr); - let lines: Vec<&str> = stderr.lines().collect(); - let tail = lines[lines.len().saturating_sub(PROBE_STDERR_TAIL_LINES)..].join("\n"); return Err(ProbeError::Exit { command: command.to_owned(), status: out.status, - stderr_tail: tail, + stderr_tail: stderr_tail(&out.stderr), }); } let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); @@ -713,19 +768,125 @@ pub fn classify_probe_output( Ok(stdout) } +/// How a bounded probe run ended (see `output_within`). +#[derive(Debug)] +pub enum Finished { + /// The child exited. `stdout_drained` is false when its stdout was still + /// open after the drain grace; `output` then holds what had been read by + /// then. Stderr is read the same way, but one left open does not fail the + /// probe: it carries only diagnostics. + Exited { + output: std::process::Output, + stdout_drained: bool, + }, + /// The child was still running at the deadline and has been killed and + /// reaped; `stderr` is what it had written by then. + TimedOut { stderr: Vec }, + /// Waiting for the child failed; it has been killed. + WaitFailed(std::io::Error), +} + +/// [`classify_probe_output`] for a bounded run: a timeout and an output pipe +/// left open are errors of their own, never read as "no output". +pub fn classify_finished( + command: &str, + result: std::io::Result, + timeout: std::time::Duration, + grace: std::time::Duration, +) -> Result { + match result { + Err(source) => classify_probe_output(command, Err(source)), + Ok(Finished::TimedOut { stderr }) => Err(ProbeError::Timeout { + command: command.to_owned(), + after: timeout, + stderr_tail: stderr_tail(&stderr), + }), + Ok(Finished::WaitFailed(source)) => Err(ProbeError::Wait { + command: command.to_owned(), + source, + }), + Ok(Finished::Exited { + output, + stdout_drained: false, + }) if output.status.success() => Err(ProbeError::OutputNotClosed { + command: command.to_owned(), + grace, + stderr_tail: stderr_tail(&output.stderr), + }), + Ok(Finished::Exited { output, .. }) => classify_probe_output(command, Ok(output)), + } +} + /// Run `rocm ` with an isolated config/data/cache root and return its /// stdout, or why there is none. Never panics: the caller decides whether a /// failure is fatal. fn run_probe(root: &std::path::Path, args: &[&str]) -> Result { - run_probe_with(std::ffi::OsStr::new(&rocm_binary()), root, args) + run_probe_using(root, args, |name| std::env::var_os(name)) +} + +/// `run_probe` with the environment lookup passed in. +/// +/// The binary comes from `ROCM_CLI_BINARY` and the deadline from +/// [`PROBE_TIMEOUT_ENV`], both read through `env`, so a test can drive both +/// without a process-wide variable. +pub fn run_probe_using( + root: &std::path::Path, + args: &[&str], + env: impl Fn(&str) -> Option, +) -> Result { + run_probe_with( + &rocm_binary(&env), + root, + args, + probe_timeout_from_env(env(PROBE_TIMEOUT_ENV).as_deref()), + ) } -/// [`run_probe`] with the binary passed in, so a test can name one without -/// setting a process-wide environment variable. +/// Environment variable that overrides how long one probe command may run, +/// in whole seconds. +pub const PROBE_TIMEOUT_ENV: &str = "E2E_PROBE_TIMEOUT_SECS"; + +/// How long one probe command may run when [`PROBE_TIMEOUT_ENV`] is unset. +/// +/// `rocm examine` normally answers in seconds; this is generous so a loaded +/// runner does not fail a good probe, while a wedged one still fails the lane +/// with a named reason long before the job's own timeout. +pub const DEFAULT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(2); + +/// The probe deadline from [`PROBE_TIMEOUT_ENV`]'s value. +/// +/// Accepts any whole number of seconds of at least 1, with surrounding +/// whitespace ignored; a value too large to add to the current time is waited +/// on without a deadline (see `output_within`). Anything else (unset, empty, +/// zero, negative, a fraction, a non-number, a number past `u64`, or a value +/// that is not UTF-8) gives [`DEFAULT_PROBE_TIMEOUT`] rather than an error. +#[must_use] +pub fn probe_timeout_from_env(value: Option<&std::ffi::OsStr>) -> std::time::Duration { + value + .and_then(std::ffi::OsStr::to_str) + .and_then(|v| v.trim().parse::().ok()) + .filter(|secs| *secs >= 1) + .map_or(DEFAULT_PROBE_TIMEOUT, std::time::Duration::from_secs) +} + +/// `run_probe` with the binary and deadline passed in, so a test can name +/// them without setting process-wide environment variables. pub fn run_probe_with( binary: &std::ffi::OsStr, root: &std::path::Path, args: &[&str], + timeout: std::time::Duration, +) -> Result { + run_probe_bounded(binary, root, args, timeout, PROBE_DRAIN_GRACE) +} + +/// [`run_probe_with`] with the drain grace passed in too, for tests. +fn run_probe_bounded( + binary: &std::ffi::OsStr, + root: &std::path::Path, + args: &[&str], + timeout: std::time::Duration, + grace: std::time::Duration, ) -> Result { let mut cmd = std::process::Command::new(binary); cmd.args(args); @@ -736,14 +897,145 @@ pub fn run_probe_with( .chain(args.iter().map(|a| (*a).to_owned())) .collect::>() .join(" "); - classify_probe_output(&command, cmd.output()) + classify_finished(&command, output_within(cmd, timeout, grace), timeout, grace) +} + +/// How often [`output_within`] checks whether the child has exited. +const PROBE_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); + +/// How long [`output_within`] waits, after the child exited or was killed, for +/// its output pipes to close. Only a process it started that inherited a pipe +/// and outlives it can hold one open past this. +const PROBE_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(5); + +/// A pipe being read on its own thread into a buffer the caller can read at any +/// time, so bytes read before a timeout or a stuck drain are not lost. +struct PipeReader { + bytes: std::sync::Arc>>, + done: std::sync::mpsc::Receiver<()>, +} + +impl PipeReader { + fn spawn(pipe: Option) -> Self { + let bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let (sender, done) = std::sync::mpsc::channel(); + let sink = std::sync::Arc::clone(&bytes); + std::thread::spawn(move || { + if let Some(mut pipe) = pipe { + let mut chunk = [0u8; 8192]; + loop { + match pipe.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(n) => sink + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend_from_slice(&chunk[..n]), + } + } + } + let _ = sender.send(()); + }); + Self { bytes, done } + } + + /// Wait until the pipe closes or `deadline` passes; whether it closed. + fn wait_closed(&self, deadline: std::time::Instant) -> bool { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + self.done.recv_timeout(left).is_ok() + } + + /// What has been read so far. + fn snapshot(&self) -> Vec { + self.bytes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } +} + +/// `command.output()`, bounded by `timeout`: a child still running then is +/// killed and reaped. Either way the output pipes get `grace` to close, and +/// whatever was read by then is kept. +/// +/// std-only, polling `try_wait`, so it behaves the same on Linux and Windows, +/// with each output pipe read on its own thread so a chatty child cannot block +/// on a full pipe. It repeats the spawn, poll, kill and reap steps of +/// `device_memory::bounded_stdout`; a change to the kill or drain behaviour of +/// one should look at the other. It differs in three ways: +/// +/// - it keeps stderr and the exit status, which the probe reports; +/// - it drains after a kill too, where `bounded_stdout` gives up at once, +/// because the stderr written before the deadline is the only evidence of +/// why `rocm examine` hung; +/// - its grace is 5 s (`PROBE_DRAIN_GRACE`) rather than 2 s, because it runs +/// three times per suite (two capability probes and `collect_versions`) +/// rather than once per sample, so a generous wait for a loaded runner to +/// hand over the bytes costs little. +/// +/// Only the direct child is killed: a process it started that inherited a pipe +/// can keep that pipe open past the grace. The reader thread for that pipe is +/// then left detached until the pipe closes, and the bytes it had read are +/// used as they are. A leaked thread per stuck probe is accepted in a test +/// harness that runs three probes and exits. +fn output_within( + mut command: std::process::Command, + timeout: std::time::Duration, + grace: std::time::Duration, +) -> std::io::Result { + use std::process::Stdio; + + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout = PipeReader::spawn(child.stdout.take()); + let stderr = PipeReader::spawn(child.stderr.take()); + + // `None` only for a deadline too far off to represent: wait without one. + let deadline = std::time::Instant::now().checked_add(timeout); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if deadline.is_none_or(|deadline| std::time::Instant::now() < deadline) => { + std::thread::sleep(PROBE_POLL_INTERVAL); + } + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Ok(Finished::WaitFailed(error)); + } + } + }; + let drained_by = std::time::Instant::now() + grace; + let stdout_closed = stdout.wait_closed(drained_by); + // Waited for so the tail is complete when it can be, but not required. + let _ = stderr.wait_closed(drained_by); + Ok(match status { + None => Finished::TimedOut { + stderr: stderr.snapshot(), + }, + Some(status) => Finished::Exited { + output: std::process::Output { + status, + stdout: stdout.snapshot(), + stderr: stderr.snapshot(), + }, + stdout_drained: stdout_closed, + }, + }) } /// One line naming the probe facts that decide GPU usability, for a lane whose /// declared GPU the probe did not find. /// /// Reads the visibility masks from the environment the way -/// [`host_has_usable_gpu`] does; [`probe_facts_summary_with_mask`] is the pure +/// `host_has_usable_gpu` does; [`probe_facts_summary_with_mask`] is the pure /// form. #[must_use] pub fn probe_facts_summary(cap: &HostCapability) -> String { @@ -753,7 +1045,7 @@ pub fn probe_facts_summary(cap: &HostCapability) -> String { } /// Pure form of [`probe_facts_summary`]. Names the mask that is in force by the -/// same precedence as [`selected_visibility_mask`]: HIP wins when set, even +/// same precedence as `selected_visibility_mask`: HIP wins when set, even /// empty. #[must_use] pub fn probe_facts_summary_with_mask( @@ -1801,7 +2093,12 @@ Local model engines fn a_probe_whose_binary_does_not_exist_is_a_spawn_error_naming_it() { let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); let missing = tmp.path().join("no-such-rocm"); - let result = run_probe_with(missing.as_os_str(), tmp.path(), &["examine"]); + let result = run_probe_with( + missing.as_os_str(), + tmp.path(), + &["examine"], + DEFAULT_PROBE_TIMEOUT, + ); let Err(error @ ProbeError::Spawn { .. }) = result else { panic!("a missing binary must be ProbeError::Spawn, got {result:?}"); }; @@ -1947,4 +2244,343 @@ Local model engines "detected_gfx_target=, wsl=false, driver_status=, no visibility mask" ); } + + /// A shell command line, run as `(binary, args)` on each CI gate. + fn shell_line(line: &str) -> (std::ffi::OsString, Vec) { + #[cfg(unix)] + { + ("sh".into(), vec!["-c".to_owned(), line.to_owned()]) + } + #[cfg(windows)] + { + ("cmd".into(), vec!["/C".to_owned(), line.to_owned()]) + } + } + + fn arg_refs(args: &[String]) -> Vec<&str> { + args.iter().map(String::as_str).collect() + } + + /// Prints `done` after about two seconds. + fn two_second_command() -> (std::ffi::OsString, Vec) { + #[cfg(unix)] + { + shell_line("sleep 2; echo done") + } + #[cfg(windows)] + { + shell_line("ping -n 3 127.0.0.1 >nul & echo done") + } + } + + /// The env lookup `run_probe` would get, with only the given variables set. + fn env_of( + vars: Vec<(&'static str, std::ffi::OsString)>, + ) -> impl Fn(&str) -> Option { + move |name| { + vars.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone()) + } + } + + /// `E2E_PROBE_TIMEOUT_SECS` reaches the probe through the same lookup + /// `run_probe` uses (no process-wide variable), and the Timeout message's + /// advice holds: the same command that timed out at 1 second completes when + /// the variable allows it longer. + #[test] + fn the_probe_timeout_variable_sets_the_deadline_and_raising_it_clears_the_timeout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = two_second_command(); + let command = format!("{} {}", binary.to_string_lossy(), args.join(" ")); + + let short = run_probe_using( + tmp.path(), + &arg_refs(&args), + env_of(vec![ + ("ROCM_CLI_BINARY", binary.clone()), + (PROBE_TIMEOUT_ENV, "1".into()), + ]), + ); + let Err(error @ ProbeError::Timeout { .. }) = short else { + panic!("a 1 s deadline on a 2 s command must time out, got {short:?}"); + }; + let ProbeError::Timeout { after, .. } = &error else { + unreachable!() + }; + assert_eq!(*after, std::time::Duration::from_secs(1)); + assert_eq!( + error.to_string(), + format!( + "`{command}` was still running after 1s, so it was killed; set \ + E2E_PROBE_TIMEOUT_SECS to allow it longer\n--- stderr (last lines): (empty) ---" + ) + ); + + // Following that advice clears the condition. + let longer = run_probe_using( + tmp.path(), + &arg_refs(&args), + env_of(vec![ + ("ROCM_CLI_BINARY", binary), + (PROBE_TIMEOUT_ENV, "5".into()), + ]), + ) + .expect("with 5 s allowed, the 2 s command completes"); + assert_eq!(longer.trim(), "done"); + } + + /// The regression: a probe that never finishes must become an error at its + /// deadline, carrying what it had written to stderr, and be killed AND + /// reaped. A zombie still answers `kill -0`, so a failing `kill -0` shows + /// the child was reaped, not merely signalled. + #[cfg(unix)] + #[test] + fn a_probe_still_running_at_its_deadline_is_killed_reaped_and_keeps_its_stderr() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let pid_file = tmp.path().join("pid"); + let script = format!( + "echo $$ > '{}'; echo early-err >&2; exec sleep 30", + pid_file.display() + ); + let started = std::time::Instant::now(); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", &script], + std::time::Duration::from_secs(3), + std::time::Duration::from_secs(2), + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(30), + "returned after {:?}: it waited for the child instead of killing it", + started.elapsed() + ); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail, "early-err"); + // The shell creates the file and then writes the pid; give a loaded + // runner a moment to finish the write rather than read it half-done. + let mut pid = String::new(); + for _ in 0..100 { + pid = std::fs::read_to_string(&pid_file).unwrap_or_default(); + if !pid.trim().is_empty() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!( + !pid.trim().is_empty(), + "the child wrote its pid before the deadline" + ); + let still_there = std::process::Command::new("kill") + .args(["-0", pid.trim()]) + .stderr(std::process::Stdio::null()) + .status() + .expect("kill runs") + .success(); + assert!( + !still_there, + "child {} was left running or unreaped", + pid.trim() + ); + } + + /// Stderr written after the kill still reaches the caller: the drain runs + /// after a timeout too. A grandchild that inherited stderr writes `late` + /// half a second after the 1 s deadline killed its parent, and the grace + /// leaves time to collect it. + #[cfg(unix)] + #[test] + fn stderr_written_after_the_kill_is_drained_into_the_timeout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", "(sleep 1.5; echo late >&2) & exec sleep 30"], + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(10), + ); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail, "late"); + } + + /// Windows half of the test above: the deadline and the stderr kept from + /// before it. `cmd` is killed but the `ping` it started holds the pipe, so + /// the bytes reach the caller only through the shared buffer. The reaping + /// check needs `kill -0` and runs on Unix only. + #[cfg(windows)] + #[test] + fn a_probe_still_running_at_its_deadline_is_reported_with_its_stderr() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = shell_line("echo early-err 1>&2 & ping -n 10 127.0.0.1 >nul"); + let started = std::time::Instant::now(); + let result = run_probe_bounded( + &binary, + tmp.path(), + &arg_refs(&args), + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + ); + assert!(started.elapsed() < std::time::Duration::from_secs(10)); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail.trim(), "early-err"); + } + + /// A probe that exited 0 while a process it started still holds its + /// stdout is reported as such, not as "produced no output". Unix only: it + /// needs a background process that inherits the pipe, which `sh` gives + /// reliably; the classification itself is tested on both gates below. + #[cfg(unix)] + #[test] + fn a_probe_whose_output_stays_open_is_reported_as_that_not_as_no_output() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", "echo real-output; echo warn >&2; sleep 3 &"], + std::time::Duration::from_secs(10), + std::time::Duration::from_millis(300), + ); + let Err(error @ ProbeError::OutputNotClosed { .. }) = &result else { + panic!("an output left open must be ProbeError::OutputNotClosed, got {result:?}"); + }; + assert_eq!( + error.to_string(), + "`sh -c echo real-output; echo warn >&2; sleep 3 &` exited 0, but its stdout was \ + still open 300ms later (a process it started may be holding it), so what it \ + printed cannot be read in full\n--- stderr (last lines) ---\nwarn" + ); + } + + /// Only stdout carries the reading: a process that keeps just stderr open + /// does not fail the probe. Unix only, like the test above. + #[cfg(unix)] + #[test] + fn a_probe_whose_stderr_alone_stays_open_still_returns_its_stdout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", "echo real-output; sleep 3 >/dev/null &"], + std::time::Duration::from_secs(10), + std::time::Duration::from_millis(300), + ); + assert_eq!( + result.expect("stdout closed, so a reading").trim(), + "real-output" + ); + } + + #[test] + fn an_undrained_exit_is_classified_by_its_status() { + let grace = std::time::Duration::from_secs(5); + let undrained = |status| { + Ok(Finished::Exited { + output: output(status, "partial", "err"), + stdout_drained: false, + }) + }; + let ok = classify_finished("rocm examine", undrained(succeeded_status()), grace, grace); + assert!( + matches!(&ok, Err(ProbeError::OutputNotClosed { stderr_tail, .. }) if stderr_tail == "err"), + "{ok:?}" + ); + // A non-zero exit is reported as one, with what stderr had by then. + let failed = classify_finished("rocm examine", undrained(failed_status()), grace, grace); + assert!( + matches!(&failed, Err(ProbeError::Exit { stderr_tail, .. }) if stderr_tail == "err"), + "{failed:?}" + ); + // A drained exit is classified as before. + let drained = classify_finished( + "rocm examine", + Ok(Finished::Exited { + output: output(succeeded_status(), "rocm examine\n", "partial err"), + stdout_drained: true, + }), + grace, + grace, + ); + // Stdout closed but stderr may not have: the reading is used. + assert_eq!(drained.expect("a reading"), "rocm examine\n"); + let wait_failed = classify_finished( + "rocm examine", + Ok(Finished::WaitFailed(std::io::Error::other("no child"))), + grace, + grace, + ); + let Err(error @ ProbeError::Wait { .. }) = &wait_failed else { + panic!("a failed wait must be ProbeError::Wait, got {wait_failed:?}"); + }; + assert_eq!( + error.to_string(), + "`rocm examine` was started, but waiting for it to exit failed: no child" + ); + let timed_out = classify_finished( + "rocm examine", + Ok(Finished::TimedOut { + stderr: b"a\nb\n".to_vec(), + }), + grace, + grace, + ); + assert!( + matches!(&timed_out, Err(ProbeError::Timeout { stderr_tail, after, .. }) + if stderr_tail == "a\nb" && *after == grace), + "{timed_out:?}" + ); + } + + #[test] + fn a_probe_that_finishes_in_time_returns_its_stdout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = shell_line("echo probe-ok"); + let stdout = run_probe_with(&binary, tmp.path(), &arg_refs(&args), DEFAULT_PROBE_TIMEOUT) + .expect("a quick command is a reading"); + assert_eq!(stdout.trim(), "probe-ok"); + // A deadline too far off to add to the current time must not panic. + let stdout = run_probe_with( + &binary, + tmp.path(), + &arg_refs(&args), + std::time::Duration::MAX, + ) + .expect("an unbounded deadline still reads"); + assert_eq!(stdout.trim(), "probe-ok"); + } + + #[test] + fn the_probe_deadline_reads_whole_positive_seconds_else_the_default() { + assert_eq!(probe_timeout_from_env(None), DEFAULT_PROBE_TIMEOUT); + assert_eq!(DEFAULT_PROBE_TIMEOUT.as_secs(), 120); + for value in ["5", " 5 ", "5\n"] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(value))), + std::time::Duration::from_secs(5), + "{value:?}" + ); + } + // No upper bound: a huge value is honoured, and `output_within`'s + // `checked_add` keeps it from overflowing (see the `Duration::MAX` run + // in `a_probe_that_finishes_in_time_returns_its_stdout`). + for secs in [86_401, u64::MAX] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(&secs.to_string()))), + std::time::Duration::from_secs(secs) + ); + } + for value in ["", "0", "-3", "1.5", "soon", "18446744073709551616"] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(value))), + DEFAULT_PROBE_TIMEOUT, + "{value:?}" + ); + } + } } diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index fb3a513e6..0ff33664a 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -585,7 +585,8 @@ const SMI_DRAIN_GRACE: Duration = Duration::from_secs(2); /// module is library code, and a normal dependency would be a new first-party /// edge that `cargo xtask check-crate-edges` makes a reviewed addition. The two /// can drift, so a change to either one's kill or drain behaviour should look at -/// the other. +/// the other. The same applies to `capability::output_within`, the capability +/// probe's copy of these steps, which also keeps stderr and the exit status. pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe { let mut child = match command .stdin(Stdio::null()) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9643b346e..1a032e058 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -482,6 +482,12 @@ pub struct PlatformManifest<'a> { pub capability: &'a HostCapability, /// Component versions (OS/ROCm/vLLM/lemonade) for the report heading. pub versions: crate::capability::PlatformVersions, + /// Why the post-run lane check failed the run + /// (`lane_verdict::post_run_verdict`), so the consolidated report shows the + /// lane as failed even when every scenario it resolved was a skip. Left out + /// when the check passed. + #[serde(skip_serializing_if = "Option::is_none")] + pub lane_failure: Option, pub expectations: Vec, } @@ -2142,6 +2148,36 @@ reason = "unrelated open bug" )); } + /// What the lane-verdict notes say about `@requires-gpu` scenarios (see + /// `lane_verdict::preflight_verdict`): `has_amd_gpu` alone decides their + /// skip, other gates aside. `resolve` takes no lane declaration, so + /// `E2E_EXPECT_GPU` does not enter into it. + #[test] + fn requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not() { + let m = Expectations::default(); + let d = decl(&["id:gpu-premise", "requires-gpu"]); + let included = Included { + nightly: false, + lifecycle: false, + docker: false, + merge_queue: false, + }; + let with_gpu = cap("strix-ubuntu"); + assert!(with_gpu.has_amd_gpu); + assert_eq!( + resolve(&d, &with_gpu, &m, included), + Expectation::ExpectPass + ); + let without_gpu = HostCapability { + has_amd_gpu: false, + ..with_gpu + }; + assert!(matches!( + resolve(&d, &without_gpu, &m, included), + Expectation::Skip { .. } + )); + } + #[test] fn non_gpu_scenario_always_runs() { let m = Expectations::default(); diff --git a/tests/e2e-cucumber/src/lane_verdict.rs b/tests/e2e-cucumber/src/lane_verdict.rs index ed562eb9d..a2c63df93 100644 --- a/tests/e2e-cucumber/src/lane_verdict.rs +++ b/tests/e2e-cucumber/src/lane_verdict.rs @@ -20,6 +20,18 @@ //! - [`post_run_verdict`], after the run: a run that executed no scenario is //! fatal on every lane, and so is a GPU-declaring lane on which no //! `@requires-gpu` scenario executed. +//! +//! The module also holds the decisions `tests/e2e.rs` takes around those two +//! checks, so they are unit-tested too: [`preflight`] (the declaration plus +//! the pre-run verdict), [`needs_gpu`] and [`gpu_scenario_counts`] (the input to +//! the post-run check), [`run_failed`] (the final exit, combining the +//! harness's own XPASS and failure classification with the post-run verdict), +//! [`finish_run`] (the post-run verdict and the `platform.json` manifest that +//! records it), and [`failure_lines`] and [`note_line`] (the exact lines +//! printed, with the annotation escaped by [`workflow_command`]). The harness +//! itself still sets up the run, probes the host, reads its other environment +//! variables, classifies the scenario results, writes the files, and does the +//! printing and exiting; that glue has no test of its own. use std::ffi::OsStr; @@ -137,8 +149,7 @@ pub fn post_run_verdict( if executed == 0 { verdict.fatal = Some( "0 scenarios executed: a run that executes nothing tests nothing. Check the \ - cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and E2E_ALSO_RUN_IDS, and the \ - host capability line above" + cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and E2E_ALSO_RUN_IDS" .to_owned(), ); return verdict; @@ -164,11 +175,175 @@ pub fn post_run_verdict( verdict } +/// What the harness carries forward from the pre-run check when it passes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Preflight { + /// The lane's parsed [`EXPECT_GPU_ENV`] declaration, for the post-run check. + pub declaration: GpuDeclaration, + /// Lines to print before the run. + pub notes: Vec, +} + +/// The pre-run lane check, from the raw [`EXPECT_GPU_ENV`] value and the +/// probe's results: `Err` with the reason when the run must not start. +/// +/// An invalid declaration is reported before anything the probe found, since +/// the verdict depends on the declaration. The harness only prints the notes +/// or the refusal and exits. +pub fn preflight( + declaration_value: Option<&OsStr>, + probe_error: Option<&str>, + has_amd_gpu: bool, + probe_facts: &str, +) -> Result { + let declaration = parse_gpu_declaration(declaration_value)?; + let verdict = preflight_verdict(probe_error, has_amd_gpu, probe_facts, declaration); + match verdict.fatal { + Some(fatal) => Err(fatal), + None => Ok(Preflight { + declaration, + notes: verdict.notes, + }), + } +} + +/// Whether a scenario counts toward the post-run GPU check. +/// +/// It does when tagged `@requires-gpu` or `@requires-multi-gpu`. `@requires-gfx-target` does not +/// count, since such a scenario reads the chip name without using the GPU. +#[must_use] +pub const fn needs_gpu(decl: &crate::expectation::ScenarioDecl) -> bool { + decl.requires_gpu || decl.requires_multi_gpu +} + +/// Count the GPU scenarios the harness resolved and how many of them executed. +/// +/// `resolved` is each scenario the harness's filter saw, as `(id, needs_gpu)`; +/// `executed` is the id of each scenario in `report.json`. `None` when nothing +/// was resolved: a cucumber `--name`/`--tags` filter replaced the harness's +/// filter, so which scenarios need a GPU is unknown (see [`post_run_verdict`]). +/// An executed id the harness did not resolve as a GPU scenario is not counted. +pub fn gpu_scenario_counts<'a>( + resolved: impl IntoIterator, + executed: impl IntoIterator, +) -> Option { + let mut any_resolved = false; + let gpu_ids: std::collections::BTreeSet<&str> = resolved + .into_iter() + .inspect(|_| any_resolved = true) + .filter_map(|(id, gpu)| gpu.then_some(id)) + .collect(); + if !any_resolved { + return None; + } + Some(GpuScenarioCounts { + resolved: gpu_ids.len(), + executed: executed + .into_iter() + .filter(|id| gpu_ids.contains(id)) + .count(), + }) +} + +/// Whether the run exits non-zero after reconciliation. +/// +/// Any one of a stale XPASS, an unexpected failure (both counted by the +/// harness) or a fatal post-run verdict is enough. Parse and hook errors exit +/// earlier, in the harness. +#[must_use] +pub const fn run_failed(stale_xpass: usize, unexpected_fail: usize, post_run: &Verdict) -> bool { + stale_xpass > 0 || unexpected_fail > 0 || post_run.fatal.is_some() +} + +/// A GitHub Actions workflow command (`::error::…`, `::notice::…`) carrying +/// `message` as its data. +/// +/// The data is escaped as the runner expects — `%` first, then CR and LF — so a +/// probe's stderr tail can neither cut the annotation short at a newline nor +/// have a literal `%0A` in it read back as one. +#[must_use] +pub fn workflow_command(command: &str, message: &str) -> String { + let data = message + .replace('%', "%25") + .replace('\r', "%0D") + .replace('\n', "%0A"); + format!("::{command}::{data}") +} + +/// One scenario the harness's filter resolved, as [`finish_run`] needs it. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedRun<'s> { + pub id: &'s str, + pub feature: &'s str, + pub scenario: &'s str, + pub engine: &'s str, + pub expectation: &'s crate::expectation::Expectation, + /// [`needs_gpu`] of the scenario's tags. + pub needs_gpu: bool, +} + +/// The post-run lane check, and the `platform.json` manifest that records its +/// failure as `lane_failure` so the consolidated report shows the lane red. +/// +/// `resolved` is every scenario the harness's filter saw (empty when a CLI +/// filter replaced it); `executed` is the id of each scenario in +/// `report.json`. +#[must_use] +pub fn finish_run<'a>( + cap: &'a crate::capability::HostCapability, + versions: crate::capability::PlatformVersions, + resolved: &[ResolvedRun<'_>], + executed: &[&str], + declaration: GpuDeclaration, +) -> (Verdict, crate::expectation::PlatformManifest<'a>) { + let gpu = gpu_scenario_counts( + resolved.iter().map(|r| (r.id, r.needs_gpu)), + executed.iter().copied(), + ); + let verdict = post_run_verdict(executed.len(), gpu, declaration); + let manifest = crate::expectation::PlatformManifest { + platform_slug: &cap.platform_slug, + capability: cap, + versions, + lane_failure: verdict.fatal.clone(), + expectations: resolved + .iter() + .map(|r| { + crate::expectation::ResolvedScenario::new( + r.id, + r.feature, + r.scenario, + r.engine, + r.expectation, + ) + }) + .collect(), + }; + (verdict, manifest) +} + +/// The lines the harness prints when a lane check fails: a GitHub `::error::` +/// annotation, then the same message as a plain line for anyone reading the +/// log outside Actions. +#[must_use] +pub fn failure_lines(message: &str) -> [String; 2] { + let line = format!("E2E lane check failed: {message}"); + [workflow_command("error", &line), line] +} + +/// The `::notice::` line the harness prints for a lane-check note. +#[must_use] +pub fn note_line(note: &str) -> String { + workflow_command("notice", &format!("E2E lane: {note}")) +} + #[cfg(test)] mod tests { use super::{ - EXPECT_GPU_ENV, GpuDeclaration, GpuScenarioCounts, Verdict, parse_gpu_declaration, - post_run_verdict, preflight_verdict, + EXPECT_GPU_ENV, GpuDeclaration, GpuScenarioCounts, Preflight, ResolvedRun, Verdict, + failure_lines, finish_run, gpu_scenario_counts, needs_gpu, note_line, + parse_gpu_declaration, post_run_verdict, preflight, preflight_verdict, run_failed, + workflow_command, }; use std::ffi::OsStr; @@ -270,6 +445,9 @@ mod tests { fn a_declared_absent_gpu_that_is_present_is_a_note_not_a_failure() { let verdict = preflight_verdict(None, true, FACTS, GpuDeclaration::NotExpected); assert_eq!(verdict.fatal, None); + // Pins the wording only. That @requires-gpu scenarios do run on a host + // whose probe found a GPU, whatever the declaration, is proved by + // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`. assert_eq!( verdict.notes, [format!( @@ -286,6 +464,9 @@ mod tests { fn a_declared_absent_gpu_that_is_absent_runs_with_a_note() { let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::NotExpected); assert_eq!(verdict.fatal, None); + // Pins the wording only. That @requires-gpu scenarios resolve to skip + // (not applicable) where the probe found no GPU is proved by + // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`. assert_eq!( verdict.notes, [ @@ -331,7 +512,7 @@ mod tests { Some( "0 scenarios executed: a run that executes nothing tests nothing. Check \ the cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and \ - E2E_ALSO_RUN_IDS, and the host capability line above" + E2E_ALSO_RUN_IDS" ), "{declaration:?}, {gpu:?}" ); @@ -359,6 +540,32 @@ mod tests { ); } + // Post-run row: declared 1, scenarios ran, and the harness's filter ran but + // resolved no GPU scenario at all (none selected): still fatal, since the + // lane promised GPU coverage it did not get. + #[test] + fn a_gpu_lane_that_resolved_no_gpu_scenario_is_fatal() { + let verdict = post_run_verdict( + 5, + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }), + GpuDeclaration::Expected, + ); + assert_eq!( + verdict, + Verdict { + fatal: Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu scenario \ + executed (0 resolved, none ran)" + .to_owned() + ), + notes: vec![], + } + ); + } + #[test] fn a_gpu_lane_on_which_a_gpu_scenario_ran_passes() { assert_eq!( @@ -418,4 +625,321 @@ mod tests { Verdict::default() ); } + + // --- the harness glue, as pure functions --- + + #[test] + fn preflight_refuses_an_invalid_declaration_before_reading_the_probe() { + let invalid = Err( + "invalid E2E_EXPECT_GPU value \"yes\": use 1 (this lane has a GPU), \ + 0 (it deliberately has none), or leave it unset" + .to_owned(), + ); + // A probe that also failed, or found no GPU, must not displace the + // declaration error: the declaration is checked first. + for (probe_error, has_amd_gpu) in [(None, true), (Some(PROBE_ERROR), false), (None, false)] + { + assert_eq!( + preflight(Some(OsStr::new("yes")), probe_error, has_amd_gpu, FACTS), + invalid, + "probe_error={probe_error:?}, gpu={has_amd_gpu}" + ); + } + } + + #[test] + fn preflight_refuses_a_failed_probe_and_a_missing_declared_gpu() { + let refused = preflight(None, Some(PROBE_ERROR), true, FACTS).expect_err("probe failed"); + assert!(refused.ends_with(PROBE_ERROR), "{refused}"); + let refused = + preflight(Some(OsStr::new("1")), None, false, FACTS).expect_err("declared GPU missing"); + assert!( + refused.starts_with("this lane declares a GPU (E2E_EXPECT_GPU=1)"), + "{refused}" + ); + } + + #[test] + fn preflight_carries_the_declaration_and_notes_into_the_run() { + assert_eq!( + preflight(Some(OsStr::new("0")), None, false, FACTS), + Ok(Preflight { + declaration: GpuDeclaration::NotExpected, + notes: vec![ + "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu \ + scenarios are not applicable here" + .to_owned() + ], + }) + ); + assert_eq!( + preflight(Some(OsStr::new("1")), None, true, FACTS), + Ok(Preflight { + declaration: GpuDeclaration::Expected, + notes: vec![], + }) + ); + } + + #[test] + fn gpu_and_multi_gpu_scenarios_need_a_gpu_and_a_gfx_target_one_does_not() { + let decl = crate::expectation::ScenarioDecl::from_tags; + assert!(needs_gpu(&decl(&["id:a", "requires-gpu"]))); + assert!(needs_gpu(&decl(&["id:b", "requires-multi-gpu"]))); + assert!(!needs_gpu(&decl(&["id:c", "requires-gfx-target"]))); + assert!(!needs_gpu(&decl(&["id:d"]))); + } + + #[test] + fn a_cli_filtered_run_resolved_nothing_so_has_no_gpu_counts() { + assert_eq!(gpu_scenario_counts([], ["serve-a", "serve-b"]), None); + } + + #[test] + fn gpu_counts_take_resolved_from_the_filter_and_executed_from_the_report() { + let resolved = [ + ("gpu-ran", true), + ("multi-gpu-only", true), + ("gpu-skipped", true), + ("cpu-ran", false), + ]; + // `cpu-ran` executed but is not a GPU scenario, and `untagged` was + // never resolved: neither counts as an executed GPU scenario. + let executed = ["gpu-ran", "multi-gpu-only", "cpu-ran", "untagged"]; + assert_eq!( + gpu_scenario_counts(resolved, executed), + Some(GpuScenarioCounts { + resolved: 3, + executed: 2, + }) + ); + } + + #[test] + fn gpu_counts_with_nothing_executed_are_zero_not_none() { + assert_eq!( + gpu_scenario_counts([("gpu", true), ("cpu", false)], []), + Some(GpuScenarioCounts { + resolved: 1, + executed: 0, + }) + ); + assert_eq!( + gpu_scenario_counts([("cpu", false)], ["cpu"]), + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }) + ); + } + + #[test] + fn the_run_fails_on_any_one_of_its_three_reasons() { + let fatal = Verdict { + fatal: Some("0 scenarios executed".to_owned()), + notes: vec![], + }; + let clean = Verdict::default(); + assert!(!run_failed(0, 0, &clean)); + assert!(run_failed(1, 0, &clean)); + assert!(run_failed(0, 1, &clean)); + assert!(run_failed(0, 0, &fatal)); + // A note alone never fails the run. + let noted = Verdict { + fatal: None, + notes: vec!["stale".to_owned()], + }; + assert!(!run_failed(0, 0, ¬ed)); + } + + #[test] + fn workflow_command_data_escapes_percent_cr_and_lf() { + assert_eq!( + workflow_command("error", "100% failed\r\nstderr: a%0Ab"), + "::error::100%25 failed%0D%0Astderr: a%250Ab" + ); + assert_eq!(workflow_command("notice", "plain"), "::notice::plain"); + } + + #[test] + fn a_failure_prints_an_escaped_annotation_then_the_plain_message() { + assert_eq!( + failure_lines("probe failed\n--- stderr ---\n100% full"), + [ + "::error::E2E lane check failed: probe failed%0A--- stderr ---%0A100%25 full" + .to_owned(), + "E2E lane check failed: probe failed\n--- stderr ---\n100% full".to_owned(), + ] + ); + } + + #[test] + fn a_note_prints_as_an_escaped_notice() { + assert_eq!( + note_line("7 of 7 @requires-gpu scenario(s) did not run (100%)"), + "::notice::E2E lane: 7 of 7 @requires-gpu scenario(s) did not run (100%25)" + ); + } + + /// `finish_run`'s manifest, written as the harness writes it and read back + /// with the consolidated report's own loader: the post-run lane failure + /// it records makes the lane red there, and a lane that passed the check + /// stays green. Ties the producer to the consumer through the function + /// `tests/e2e.rs` calls. + #[test] + fn finish_runs_manifest_reads_back_as_the_verdict_in_the_report() { + let cap = crate::capability::HostCapability { + os_family: "linux".into(), + is_wsl: false, + gfx_target: Some("gfx1151".into()), + has_amd_gpu: true, + amd_gpu_count: Some(1), + available_engines: vec!["lemonade".into()], + effective_serve_engine: "lemonade".into(), + platform_slug: "strix-halo-linux".into(), + oom_fault_injection: false, + case_sensitive_fs: true, + dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, + }; + let pass = crate::expectation::Expectation::ExpectPass; + let skip = crate::expectation::Expectation::Skip { + reason: "no usable AMD GPU".to_owned(), + }; + let resolved_with = |gpu_expectation| { + [ + ResolvedRun { + id: "examine-version", + feature: "Examine", + scenario: "examine-01", + engine: "lemonade", + expectation: &pass, + needs_gpu: false, + }, + ResolvedRun { + id: "serve-gpu", + feature: "Serving", + scenario: "serve-01", + engine: "vllm", + expectation: gpu_expectation, + needs_gpu: true, + }, + ] + }; + // Writes what the harness writes and reads it back: the verdict, the + // report's status, and `platform.json`'s id -> expected mapping. + // `executed` is also what report.json holds; every listed id passed. + let status_of = |resolved: &[ResolvedRun<'_>], executed: &[&str]| { + let (verdict, manifest) = finish_run( + &cap, + crate::capability::PlatformVersions::default(), + resolved, + executed, + GpuDeclaration::Expected, + ); + let dir = tempfile::TempDir::with_prefix("rocm-e2e-manifest-").expect("temp dir"); + let elements: Vec = executed + .iter() + .map(|id| { + format!( + r#"{{"name":"{id}","tags":[{{"name":"id:{id}"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"passed"}}}}]}}"# + ) + }) + .collect(); + let report = dir.path().join("report.json"); + std::fs::write( + &report, + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{}]}}]"#, + elements.join(",") + ), + ) + .expect("report.json"); + let platform = dir.path().join("platform.json"); + std::fs::write( + &platform, + serde_json::to_string_pretty(&manifest).expect("serializes"), + ) + .expect("platform.json"); + let written: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&platform).expect("read back")) + .expect("valid json"); + let mapping: Vec> = written["expectations"] + .as_array() + .expect("an expectations array") + .iter() + .map(|e| { + ["id", "feature", "scenario", "effective_engine", "expected"] + .iter() + .map(|key| e[*key].as_str().unwrap_or_default().to_owned()) + .collect() + }) + .collect(); + ( + verdict, + e2e_report::platform_status("e2e-report", &report), + mapping, + ) + }; + // id, feature, scenario, effective_engine, expected — per expectation. + let mapping_of = |rows: &[[&str; 5]]| -> Vec> { + rows.iter() + .map(|row| row.iter().map(|field| (*field).to_owned()).collect()) + .collect() + }; + + // Every cell is pass or skip, so only the recorded lane failure can + // make this row FAIL. + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu \ + scenario executed (1 resolved, none ran)"; + let (verdict, failed, mapping) = status_of(&resolved_with(&skip), &["examine-version"]); + assert_eq!(verdict.fatal.as_deref(), Some(reason)); + assert_eq!( + mapping, + mapping_of(&[ + [ + "examine-version", + "Examine", + "examine-01", + "lemonade", + "pass" + ], + ["serve-gpu", "Serving", "serve-01", "vllm", "skip"], + ]) + ); + assert_eq!( + failed, + e2e_report::PlatformStatus { + ok: false, + status: "FAIL", + lane_failure: Some(reason.to_owned()), + } + ); + + let (verdict, passed, mapping) = + status_of(&resolved_with(&pass), &["examine-version", "serve-gpu"]); + assert_eq!(verdict.fatal, None); + assert_eq!( + mapping, + mapping_of(&[ + [ + "examine-version", + "Examine", + "examine-01", + "lemonade", + "pass" + ], + ["serve-gpu", "Serving", "serve-01", "vllm", "pass"], + ]) + ); + assert_eq!( + passed, + e2e_report::PlatformStatus { + ok: true, + status: "PASS", + lane_failure: None, + } + ); + } } diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index dfec6d765..588cac32d 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1524,20 +1524,15 @@ async fn run_suite() { // Refuse to run when the result could not mean anything: a probe that could // not read the binary, or a lane that declares a GPU the probe did not find, // would resolve every GPU scenario to skip and pass. See `lane_verdict`. - let gpu_declaration = lane_verdict::parse_gpu_declaration( + let preflight = lane_verdict::preflight( std::env::var_os(lane_verdict::EXPECT_GPU_ENV).as_deref(), - ) - .unwrap_or_else(|error| lane_failure(&error)); - let preflight = lane_verdict::preflight_verdict( cap.probe_error.as_deref(), cap.has_amd_gpu, &e2e_cucumber::capability::probe_facts_summary(cap), - gpu_declaration, - ); + ) + .unwrap_or_else(|fatal| lane_failure(&fatal)); print_lane_notes(&preflight.notes); - if let Some(fatal) = &preflight.fatal { - lane_failure(fatal); - } + let gpu_declaration = preflight.declaration; // Shared record of each scenario's resolved expectation, keyed by @id. // Populated by `filter_run` (which sees every scenario, run or skipped) so @@ -1661,7 +1656,7 @@ async fn run_suite() { engine, feature: feature.name.clone(), scenario: scenario.name.clone(), - requires_gpu: decl.requires_gpu || decl.requires_multi_gpu, + requires_gpu: lane_verdict::needs_gpu(&decl), }, ); // Two scenarios sharing an `@id` would silently overwrite each @@ -1717,31 +1712,41 @@ async fn run_suite() { // OS is always collected (every platform, incl. mock); ROCm/vLLM/lemonade come // from the shared runtimes tree CI provides (E2E_SHARED_RUNTIMES_DIR) — absent // on mock (no runtime) and on local runs (per-scenario dirs already dropped), - // where those render as "n/a". Best-effort. + // where those render as "n/a". Best-effort. Collected before the lane check + // so that, once decided, nothing stands between it and platform.json. let shared = shared_runtimes_dir(); let versions = e2e_cucumber::capability::collect_versions(shared.as_deref()); + // A run with nothing to fail passes; check it executed something, and on + // a GPU-declaring lane something that needed the GPU. `resolutions` is empty + // when a cucumber `--name`/`--tags` filter replaced `filter_run`, and then + // which scenarios need a GPU is unknown. The verdict goes into platform.json, + // so the consolidated report shows a failed lane check as red. + let resolved: Vec> = resolutions + .iter() + .map(|(id, r)| lane_verdict::ResolvedRun { + id, + feature: &r.feature, + scenario: &r.scenario, + engine: &r.engine, + expectation: &r.expectation, + needs_gpu: r.requires_gpu, + }) + .collect(); + let executed: Vec<&str> = actual.iter().map(|(id, _)| id.as_str()).collect(); + let (post_run, manifest) = + lane_verdict::finish_run(cap, versions, &resolved, &executed, gpu_declaration); + print_lane_notes(&post_run.notes); + // Write the platform.json sidecar (probed capability + every resolution, - // including skips) for the central report's expected-vs-actual grid. - let manifest = e2e_cucumber::expectation::PlatformManifest { - platform_slug: &cap.platform_slug, - capability: cap, - versions, - expectations: resolutions - .iter() - .map(|(id, r)| { - e2e_cucumber::expectation::ResolvedScenario::new( - id, - &r.feature, - &r.scenario, - &r.engine, - &r.expectation, - ) - }) - .collect(), - }; - if let Ok(json) = serde_json::to_string_pretty(&manifest) { - let _ = std::fs::write(dir.join("platform.json"), json); + // including skips, + any lane failure) for the central report's grid. + let written = serde_json::to_string_pretty(&manifest) + .map_err(|error| error.to_string()) + .and_then(|json| { + std::fs::write(dir.join("platform.json"), json).map_err(|error| error.to_string()) + }); + if let Err(error) = written { + eprintln!("warning: could not write platform.json: {error}"); } // Classify actual results against the matrix. A deterministic XPASS is @@ -1774,27 +1779,6 @@ async fn run_suite() { } } - // A run with nothing to fail passes; check it executed something, and on - // a GPU-declaring lane something that needed the GPU. `resolutions` is empty - // when a cucumber `--name`/`--tags` filter replaced `filter_run`, and then - // which scenarios need a GPU is unknown. - let gpu_counts = (!resolutions.is_empty()).then(|| { - let gpu_ids: std::collections::BTreeSet<&str> = resolutions - .iter() - .filter(|(_, r)| r.requires_gpu) - .map(|(id, _)| id.as_str()) - .collect(); - lane_verdict::GpuScenarioCounts { - resolved: gpu_ids.len(), - executed: actual - .iter() - .filter(|(id, _)| gpu_ids.contains(id.as_str())) - .count(), - } - }); - let post_run = lane_verdict::post_run_verdict(actual.len(), gpu_counts, gpu_declaration); - print_lane_notes(&post_run.notes); - eprintln!( "Reconciliation: {xfail_count} xfail (failed as expected), {} XPASS ({} flaky, {} stale), {} unexpected failure(s).", flaky_xpass.len() + stale_xpass.len(), @@ -1807,7 +1791,7 @@ async fn run_suite() { "XPASS (flaky, tolerated): '{x}' passed this run; the intermittent bug remains tracked." ); } - if !stale_xpass.is_empty() || !unexpected_fail.is_empty() || post_run.fatal.is_some() { + if lane_verdict::run_failed(stale_xpass.len(), unexpected_fail.len(), &post_run) { for x in &stale_xpass { eprintln!( "XPASS: '{x}' is expected to fail on this host but PASSED \u{2014} the bug appears \ @@ -1826,20 +1810,17 @@ async fn run_suite() { } } -/// Print a lane-verdict failure as a GitHub annotation and a plain line, then -/// exit non-zero. The annotation encodes newlines (a probe's stderr tail) as -/// `%0A`, which is how a workflow command carries more than one line. +/// Print a lane-check failure (see `lane_verdict::failure_lines`) and exit +/// non-zero. fn lane_failure(message: &str) -> ! { - eprintln!( - "::error::E2E lane check failed: {}", - message.replace('\n', "%0A") - ); - eprintln!("E2E lane check failed: {message}"); + for line in e2e_cucumber::lane_verdict::failure_lines(message) { + eprintln!("{line}"); + } std::process::exit(1); } fn print_lane_notes(notes: &[String]) { for note in notes { - eprintln!("::notice::E2E lane: {note}"); + eprintln!("{}", e2e_cucumber::lane_verdict::note_line(note)); } } diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index 9d8d47c79..1c85f2e26 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -428,13 +428,13 @@ fn queue_absent_reason(markdown: &str) -> Option { /// for every per-platform row that reports a non-`PASS` `Status` with a `Fail` /// cell of `0`. /// -/// The second part exists because an `EMPTY` platform (no `report.json`, or -/// one that's empty/unparseable) contributes `0` to every numeric column — -/// including `Fail` — so the Total row's own Fail cell has no record that the -/// platform was ever there at all. A genuine `FAIL` status always has a -/// nonzero own `Fail` cell (already folded into the Total row), so this only -/// adds for platforms the Total row would otherwise read as having zero -/// failures. +/// The second part exists because two kinds of failed row contribute `0` to +/// the `Fail` column, so the Total row's own Fail cell has no record of them: +/// an `EMPTY` platform (no `report.json`, or one that's empty/unparseable), +/// and a `FAIL` platform whose only problem is the harness's post-run lane +/// check (`lane_failure` in its `platform.json`, e.g. "0 scenarios executed"), +/// which fails no scenario cell. A `FAIL` row with a nonzero own `Fail` cell +/// is already folded into the Total row and is not counted again. /// /// A row for a shared-queue lane the queue does not block on (its platform /// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the @@ -795,6 +795,54 @@ mod tests { assert_eq!(total_e2e_failures(md), Some(1)); } + /// A lane the harness failed after its run with no failed scenario cell + /// (every scenario passed or was skipped) reads `FAIL` with a `Fail` cell + /// of 0; the Total row cannot show it, so the row itself must count. One + /// that also has a failed cell is already in the Total row and is counted + /// once, not twice. + #[test] + fn total_e2e_failures_counts_a_lane_check_failure_with_no_failed_cells() { + // Real renderer output over a platform.json that records a lane + // failure, with the one scenario passing or failing. + let render = |step_status: &str| { + let dir = tempfile::tempdir().expect("tempdir"); + let report = dir.path().join("report.json"); + std::fs::write( + &report, + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s0","tags":[{{"name":"id:examine-a"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{step_status}"}}}}]}}]}}]"# + ), + ) + .expect("write report"); + std::fs::write( + dir.path().join("platform.json"), + r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed", + "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#, + ) + .expect("write platform.json"); + e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&[( + "e2e-gpu-report".to_string(), + report, + )])) + }; + + let md = render("passed"); + assert!( + md.contains("| 1 | 1 | 0 | 0 | n/a | FAIL |") + && md.contains("| **Total** | | 1 | 1 | 0 |"), + "expected a FAIL row with a Fail cell of 0:\n{md}" + ); + assert_eq!(total_e2e_failures(&md), Some(1), "{md}"); + + let md = render("failed"); + assert!( + md.contains("| 1 | 0 | 1 | 0 | n/a | FAIL |") + && md.contains("| **Total** | | 1 | 0 | 1 |"), + "expected a FAIL row with a Fail cell of 1:\n{md}" + ); + assert_eq!(total_e2e_failures(&md), Some(1), "{md}"); + } + #[test] fn total_e2e_failures_leaves_out_the_queue_lanes_the_queue_lets_fail() { // Real producer output, so the marker the parser looks for is the one the From dcf2e21f67497b90c20a7060ab6b63a7a6ea8389 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 11:44:09 +0200 Subject: [PATCH 3/5] test(e2e): tighten lane-verdict tests and docs (ROCMAI-388, ROCMAI-450) The GPU-lane check already counts @requires-multi-gpu scenarios as GPU scenarios, but its messages, doc comments and docs named @requires-gpu alone. The fatal reason and the E2E_EXPECT_GPU=0 note now name both tags, and the docs say that GPU scenarios the run's own configuration leaves out (@nightly, @merge-queue, outside an E2E_ONLY_LIFECYCLE selection) count as resolved but never as executed. New tests: the E2E_EXPECT_GPU=0 note's "N of M" arithmetic when some GPU scenarios ran (4 of 7), and a probe that reads stdin getting EOF at once rather than blocking until its deadline. Like its rocm-core sibling, the stdin test catches a missing stdin setting but does not pin null against inherit. The constant-equals-itself test is dropped; the workflow contract test already ties the name to the workflows. The workflow contract test now also fails when the step that runs the suite sets E2E_EXPECT_GPU itself, overriding the job-level declaration, when a lane has no such step, and when that step runs in WSL without a WSLENV of its own naming the variable. Stale comments corrected: the workflows now say the Windows lane's E2E_EXPECT_GPU=1 check backstops its best-effort GPU preflight, and that a harness preflight refusal shows as an EMPTY row rather than a missing report; run-e2e-lane.sh says a set-but-empty E2E_EXPECT_GPU leaves the lane undeclared. The wsl fixture gets the driver status a passthrough host reports, so tests built from it no longer reset the field. The markdown report folds any whitespace run in a lane-failure reason to one space, CRLF included, not only LF. Refs: #383 Signed-off-by: Roman Inflianskas --- .github/workflows/e2e-selfhosted.yml | 12 ++- .github/workflows/nightly.yml | 24 +++-- crates/e2e-report/src/consolidated/html.rs | 2 +- .../e2e-report/src/consolidated/markdown.rs | 8 +- crates/e2e-report/src/consolidated/mod.rs | 25 +++-- crates/e2e-report/src/lib.rs | 6 +- docs/architecture.md | 2 +- docs/ci-hardware-testing.md | 12 +-- docs/nightly-on-the-queue.md | 4 +- docs/testing.md | 6 +- scripts/run-e2e-lane.sh | 4 +- tests/e2e-cucumber/README.md | 6 +- tests/e2e-cucumber/src/capability.rs | 94 +++++++++++++++---- tests/e2e-cucumber/src/expectation.rs | 10 +- tests/e2e-cucumber/src/lane_verdict.rs | 69 ++++++++------ xtask/src/workflow_contract.rs | 82 +++++++++++++--- 16 files changed, 258 insertions(+), 108 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index c1581a054..af1e1f60e 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -255,7 +255,7 @@ jobs: continue-on-error: false env: # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU - # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). E2E_EXPECT_GPU: "1" # Bound serve readiness below the 35-min job cap so a serve that never comes # ready fails the scenario with a real error instead of hanging until the @@ -519,7 +519,7 @@ jobs: sdk_version: n-2 env: # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU - # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). E2E_EXPECT_GPU: "1" E2E_SERVE_TIMEOUT_SECS: "300" # A pool host can be busy with other CI work between GPU preflight and @@ -676,7 +676,7 @@ jobs: continue-on-error: false env: # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU - # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). E2E_EXPECT_GPU: "1" # A pool host can be busy with other CI work between GPU preflight and # the TUI checks, so a frame that renders well inside the 30s default on @@ -740,8 +740,10 @@ jobs: # available so a wedged/held GPU fails fast instead of hanging to the cap. # BEST-EFFORT because the Windows ROCm GPU query tool isn't verified here — # if no rocm-smi is found we WARN and continue rather than false-fail a - # working runner. When rocm-smi IS present we poll free memory the same way - # as the Linux jobs and fail only after the ceiling. + # working runner; the harness's E2E_EXPECT_GPU=1 check is the backstop + # that fails the lane if `rocm examine` reports no usable GPU at all. + # When rocm-smi IS present we poll free memory the same way as the Linux + # jobs and fail only after the ceiling. # # WHAT THIS DOES NOT CATCH, and the Linux lanes now do. This is a # free-memory floor ONLY. A fixed 8 GiB against a whole ~62 GiB gfx1151 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a4e40432f..5e02a8d11 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -417,7 +417,7 @@ jobs: channel: [release, nightly] env: # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU - # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). E2E_EXPECT_GPU: "1" # Include @nightly scenarios (the large-model serve). See src/expectation.rs. E2E_INCLUDE_NIGHTLY: "1" @@ -534,7 +534,7 @@ jobs: channel: [release, nightly] env: # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU - # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" @@ -624,7 +624,7 @@ jobs: channel: [release, nightly] env: # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU - # or no @requires-gpu scenario runs (see tests/e2e-cucumber/README.md). + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" @@ -1342,7 +1342,10 @@ jobs: # a failed step there does not turn the lane's check red. Only the # WSL lanes need this guard: every other lane writes report.json # directly in-job, with no guest-to-host copy step that can fail on - # its own. + # its own. A harness preflight refusal is not one of those: it + # leaves an empty report.json that uploads fine, so the missing-report + # check does not catch it; its EMPTY row in the report is the signal + # (a post-run lane failure shows as a FAIL row with lane_failure). if-no-files-found: error # The nightly lanes each uploaded a report nobody joined up, so reading a @@ -1517,11 +1520,14 @@ jobs: fi cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY" - # A lane that dies before writing report.json (preflight, build, a crash - # before results exist) uploads nothing, and `discover()` can only render - # the reports that arrived, so that lane would drop out of the grid with - # no row and no FAIL. On a `continue-on-error` lane nothing else is red - # either. So check it here: every lane that ran (any `needs` result but + # A lane that dies before writing report.json (the GPU preflight step, a + # build failure, a crash before results exist) uploads nothing. The + # harness's own preflight refusal is different: it leaves an empty + # report.json that does upload, and shows as an EMPTY row (a post-run + # lane failure as a FAIL row with lane_failure), not as missing. + # `discover()` can only render the reports that arrived, so a lane that + # uploads nothing would drop out of the grid with no row and no FAIL. On + # a `continue-on-error` lane nothing else is red either. So check it here: every lane that ran (any `needs` result but # `skipped`) must have uploaded its report artifact. This reads the run's # artifact list rather than e2e-artifacts/, because download-artifact # flattens a single match into the root and loses its name. The list also diff --git a/crates/e2e-report/src/consolidated/html.rs b/crates/e2e-report/src/consolidated/html.rs index 47e52ed5a..042980bdf 100644 --- a/crates/e2e-report/src/consolidated/html.rs +++ b/crates/e2e-report/src/consolidated/html.rs @@ -357,7 +357,7 @@ mod tests { use crate::consolidated::{load_platform_reports, load_reports}; #[test] - fn legend_mock_and_other_platform_items_are_unchanged() { + fn legend_mock_and_other_platform_items_are_pinned() { // Pins the two gating items byte for byte, so building them from shared // fragments (see `MOCK_GATES_PR`) cannot change what the HTML // report's legend says. Text only: that Mock is the required check and diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs index af90cab7b..89691ba5f 100644 --- a/crates/e2e-report/src/consolidated/markdown.rs +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -124,10 +124,14 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { let mut notes = Vec::new(); for r in reports { if let Some(reason) = r.lane_failure() { + // Written by the harness's `finish_run` from `post_run_verdict`'s + // fixed strings, so not escaped; folded onto the bullet's line. + // Carrying a pre-run reason (which quotes probe stderr) into the + // report would need escaping here. notes.push(format!( "- **Lane check failed** in _{}_: {}", r.label, - reason.replace('\n', " ") + reason.split_whitespace().collect::>().join(" ") )); } for name in &r.xfail.xpass { @@ -377,7 +381,7 @@ mod tests { } #[test] - fn summary_explanation_paragraphs_are_unchanged() { + fn summary_explanation_paragraphs_are_pinned() { // Pins the explanatory text under the summary table byte for byte, so // building it from shared fragments (see `MOCK_GATES_PR`) cannot // change a character of what the step summary shows. This pins the text diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index cfab6c013..5c6a2cf3a 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -981,12 +981,13 @@ mod tests { } /// The regression: a GPU lane on which every scenario resolved to skip - /// was failed by the harness ("no @requires-gpu scenario executed"), yet + /// was failed by the harness ("no @requires-gpu (or @requires-multi-gpu) + /// scenario executed"), yet /// its row read PASS, since a skip cell is not a problem. #[test] fn an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason() { - let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu \ - scenario executed (1 resolved, none ran)"; + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (1 resolved, none ran)"; let (_dir, report) = write_platform("[]", &all_skip_platform(Some(reason))); let r = PlatformReport::load("e2e-gpu-report".into(), &report); assert!(!r.ok()); @@ -1069,6 +1070,15 @@ mod tests { ), "{md}" ); + // A CRLF, or any run of whitespace, folds to one space too. Defensive: + // no current producer of `lane_failure` emits CR or LF. + let (_crlf_dir, crlf_report) = + write_platform("[]", &all_skip_platform(Some("first\r\n second"))); + let crlf_md = markdown::consolidated_summary_markdown(&load_platform_reports(&[( + "e2e-gpu-report".to_owned(), + crlf_report, + )])); + assert!(crlf_md.contains("Linux_: first second\n"), "{crlf_md}"); let html_out = dir.path().join("out.html"); html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default()) @@ -1112,12 +1122,13 @@ mod tests { } /// A lane on which scenarios ran and passed, but which the harness still - /// failed (on a GPU lane, no `@requires-gpu` scenario among them): its row + /// failed (on a GPU lane, no GPU scenario among them): its row /// reads FAIL with a Fail count of 0, since no cell failed. Rows of exactly /// this shape are what `total_e2e_failures` in xtask counts as failures. #[test] fn a_lane_that_ran_scenarios_and_was_failed_reads_fail_with_zero_failed_cells() { - let reason = "no @requires-gpu scenario executed (3 resolved, none ran)"; + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (3 resolved, none ran)"; let platform = format!( r#"{{"platform_slug":"mi300x","lane_failure":{},"expectations":[ {{"id":"examine-a","feature":"F","expected":"pass"}}]}}"#, @@ -2031,8 +2042,8 @@ mod tests { #[test] fn renderers_take_the_pr_gating_phrases_from_the_shared_constants() { // What this proves, and what it does not. The rendered text is pinned - // verbatim by `summary_explanation_paragraphs_are_unchanged` (markdown) - // and `legend_mock_and_other_platform_items_are_unchanged` (HTML), and + // verbatim by `summary_explanation_paragraphs_are_pinned` (markdown) + // and `legend_mock_and_other_platform_items_are_pinned` (HTML), and // each of those also asserts that the rendered output contains // `MOCK_GATES_PR` and `OTHER_LANES_NON_BLOCKING`; together they catch a // renderer that hardcodes the words while a constant's value moves on. diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 3b16cbfbb..b63ce8fed 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -41,9 +41,9 @@ pub const OOM_FAULT_INJECTION_ENV: &str = "ROCM_E2E_OOM_FAULT_INJECTION"; /// /// `1` means the lane runs on GPU hardware, so the `e2e-cucumber` harness fails /// before any scenario runs when its capability probe finds no usable AMD GPU, -/// and after the run when no `@requires-gpu` scenario executed. `0` means the -/// lane deliberately has no usable GPU (the WSL lanes, whose distro is left -/// without ROCm's WSL driver bridge). Unset or empty leaves the lane undeclared, +/// and after the run when no `@requires-gpu` (or `@requires-multi-gpu`) +/// scenario executed. `0` means the lane deliberately has no usable GPU (the +/// WSL lanes, whose distro is left without ROCm's WSL driver bridge). Unset or empty leaves the lane undeclared, /// as on the GitHub-hosted lanes and local runs; any other value is an error. /// /// Shared by the harness, which reads it, and xtask's workflow contract test, diff --git a/docs/architecture.md b/docs/architecture.md index 666d38f81..cf4d12f00 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -798,7 +798,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat | | [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat | | [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat | -| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no `@requires-gpu` scenario) executed. Also holds the decisions [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) takes around those checks, each unit-tested: reading the declaration, counting GPU scenarios, building the `platform.json` manifest that records a lane failure (`finish_run`), the final exit, and the exact escaped `::error::`/`::notice::` lines. The harness itself runs the probe and the scenarios, classifies their results, writes the files and exits; that glue has no test. | flat | +| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no GPU scenario: `@requires-gpu` or `@requires-multi-gpu`) executed. Also holds the decisions [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) takes around those checks, each unit-tested: reading the declaration, counting GPU scenarios, building the `platform.json` manifest that records a lane failure (`finish_run`), the final exit, and the exact escaped `::error::`/`::notice::` lines. The harness itself runs the probe and the scenarios, classifies their results, writes the files and exits; that glue has no test. | flat | | [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat | | [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat | | [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat | diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 52f0cab64..ca841e16c 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -173,9 +173,9 @@ pool). The last column is the lane's declaration of whether it has a GPU. On a lane that declares `1`, the harness fails before any scenario runs if its capability -probe finds no usable AMD GPU, and after the run if no `@requires-gpu` scenario -executed; on every lane it fails if the probe could not read the binary under -test or if no scenario executed at all. `nightly.yml`'s self-hosted lanes +probe finds no usable AMD GPU, and after the run if no `@requires-gpu` (or +`@requires-multi-gpu`) scenario executed; on every lane it fails if the probe +could not read the binary under test or if no scenario executed at all. `nightly.yml`'s self-hosted lanes declare the same way (`0` only for `e2e-wsl-nightly`), as does every lane of [`run-e2e-lane.sh`](../scripts/run-e2e-lane.sh). `every_self_hosted_lane_declares_whether_it_has_a_gpu` in [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) fails when a @@ -642,9 +642,9 @@ job, naming the lane, rather than silently dropping out of the grid; on a `continue-on-error: true` lane that is the only red signal for such a failure. The nightly report job applies the same check to every nightly lane. -A lane the harness failed after its run (no scenario executed, or no -`@requires-gpu` scenario on an `E2E_EXPECT_GPU=1` lane) records the reason in -its `platform.json`, and its row reads FAIL with the reason under "Needs +A lane the harness failed after its run (no scenario executed, or no GPU +scenario, `@requires-gpu` or `@requires-multi-gpu`, on an `E2E_EXPECT_GPU=1` +lane) records the reason in its `platform.json`, and its row reads FAIL with the reason under "Needs attention: platforms" (step summary and HTML report alike), even when every scenario it resolved was a skip. diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index 4c01a589c..c57d35440 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -127,8 +127,8 @@ suite left on the GPU is reclaimed first; the lane log shows it. Both always run, so a failing channel does not hide the other, and the lane fails if either did. The default command also declares that every lane has a GPU (`E2E_EXPECT_GPU`, default `1`), so a channel whose suite finds no usable AMD -GPU, or runs no `@requires-gpu` scenario, fails instead of passing with every -GPU scenario skipped; see "Lane verdict" in the +GPU, or runs no GPU scenario (`@requires-gpu` or `@requires-multi-gpu`), fails +instead of passing with every GPU scenario skipped; see "Lane verdict" in the [E2E README](../tests/e2e-cucumber/README.md). A lane starts cold, so its first steps fetch from the network: the build diff --git a/docs/testing.md b/docs/testing.md index ddef6f00d..a687cccd8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -971,7 +971,11 @@ reporting it green: `E2E_EXPECT_GPU` declares whether the host has a GPU. With `1`, the run also fails before any scenario if the probe found no usable AMD GPU, and after the -run if no `@requires-gpu` scenario executed. With `0`, it only notes that GPU +run if no `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU +scenarios the run's own configuration leaves out (`@nightly`, `@merge-queue`, +or anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved +but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and no +GPU scenario selected fails by design. With `0`, it only notes that GPU scenarios are not applicable (and that the declaration is stale if a GPU is found). Unset, as on a local run, it makes no GPU claim. Every self-hosted CI lane sets it; see "Lane verdict" in the [E2E README](../tests/e2e-cucumber/README.md) diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh index 08c3dad0c..4e2ee32a3 100755 --- a/scripts/run-e2e-lane.sh +++ b/scripts/run-e2e-lane.sh @@ -24,7 +24,9 @@ # LANE_OUT where reported files go (default /out) # E2E_EXPECT_GPU whether the lane has a GPU (default 1: every queue lane is # given one); the suite fails rather than passing with every -# GPU scenario skipped. See tests/e2e-cucumber/README.md. +# GPU scenario skipped. Set but empty, it leaves the lane +# undeclared, which switches that check off. See +# tests/e2e-cucumber/README.md. set -euo pipefail diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 16fa7429a..31bd6bd1d 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -125,7 +125,11 @@ scenarios: consolidated report shows it as EMPTY. - **After the run**, it fails when no scenario executed (for example a `-n` filter that matches nothing), and on an `E2E_EXPECT_GPU=1` lane when no - `@requires-gpu` scenario executed. The reason is also recorded as + `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU scenarios + the run's own configuration leaves out (`@nightly`, `@merge-queue`, or + anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved + but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and + no GPU scenario selected fails by design. The reason is also recorded as `lane_failure` in `platform.json`, so the consolidated report shows the lane as FAIL, even when every scenario it resolved was a skip, and gives the reason under "Needs attention: platforms" (in both the step summary and the HTML diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index aa8f9fb04..2d410397b 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -2387,6 +2387,30 @@ Local model engines ); } + /// Windows half of the test above: the deadline and the stderr kept from + /// before it. `cmd` is killed but the `ping` it started holds the pipe, so + /// the bytes reach the caller only through the shared buffer. The reaping + /// check needs `kill -0` and runs on Unix only. + #[cfg(windows)] + #[test] + fn a_probe_still_running_at_its_deadline_is_reported_with_its_stderr() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = shell_line("echo early-err 1>&2 & ping -n 10 127.0.0.1 >nul"); + let started = std::time::Instant::now(); + let result = run_probe_bounded( + &binary, + tmp.path(), + &arg_refs(&args), + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + ); + assert!(started.elapsed() < std::time::Duration::from_secs(10)); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail.trim(), "early-err"); + } + /// Stderr written after the kill still reaches the caller: the drain runs /// after a timeout too. A grandchild that inherited stderr writes `late` /// half a second after the 1 s deadline killed its parent, and the grace @@ -2408,28 +2432,62 @@ Local model engines assert_eq!(stderr_tail, "late"); } - /// Windows half of the test above: the deadline and the stderr kept from - /// before it. `cmd` is killed but the `ping` it started holds the pipe, so - /// the bytes reach the caller only through the shared buffer. The reaping - /// check needs `kill -0` and runs on Unix only. - #[cfg(windows)] + /// A child that reads stdin gets EOF at once rather than blocking until the + /// deadline: `output_within` replaces whatever stdin the command carried + /// with an empty one. The command is handed a pipe whose write end this + /// test holds open, so only that replacement lets the read finish (the + /// same arrangement as `a_child_that_reads_stdin_is_given_an_empty_one` in + /// rocm-core's `process/bounded.rs`). + /// + /// Like that sibling, it catches a missing stdin setting but does not pin + /// `null` against `inherit`: an `inherit` mutant passes whenever the test + /// process's own stdin is already at EOF. #[test] - fn a_probe_still_running_at_its_deadline_is_reported_with_its_stderr() { + fn a_probe_that_reads_stdin_is_given_an_empty_one() { let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); - let (binary, args) = shell_line("echo early-err 1>&2 & ping -n 10 127.0.0.1 >nul"); + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sh"); + command.args(["-c", "read line || echo stdin-eof"]); + command + }; + #[cfg(windows)] + let mut command = { + // `set /p` leaves the variable as it was when its read hits EOF. + let script = tmp.path().join("stdin.bat"); + std::fs::write( + &script, + "@echo off\r\nset \"LINE=stdin-eof\"\r\nset /p LINE=\r\necho %LINE%\r\n", + ) + .expect("write script"); + let mut command = std::process::Command::new("cmd"); + command.arg("/C").arg(&script); + command + }; + let (stdin_reader, stdin_writer) = std::io::pipe().expect("stdin pipe"); + command.stdin(stdin_reader); + let started = std::time::Instant::now(); - let result = run_probe_bounded( - &binary, - tmp.path(), - &arg_refs(&args), - std::time::Duration::from_secs(1), - std::time::Duration::from_secs(1), - ); - assert!(started.elapsed() < std::time::Duration::from_secs(10)); - let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { - panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + let finished = output_within( + command, + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(2), + ) + .expect("spawns"); + let elapsed = started.elapsed(); + // Dropped only now: closing the write end earlier would give the child + // EOF whatever stdin `output_within` set. + drop(stdin_writer); + drop(tmp); + + let Finished::Exited { output, .. } = finished else { + panic!("a child given an empty stdin must exit, not time out: {finished:?}"); }; - assert_eq!(stderr_tail.trim(), "early-err"); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "stdin-eof"); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the read must hit EOF at once, but took {elapsed:?}" + ); } /// A probe that exited 0 while a process it started still holds its diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 1a032e058..95bb00df5 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -805,7 +805,7 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, - driver_status: String::new(), + driver_status: "wsl_rocdxg_ready".into(), probe_error: None, }, // The same box without the passthrough: the gfx target is reported @@ -1844,8 +1844,6 @@ serve_timeout_secs = 90 let folded = HostCapability { case_sensitive_fs: false, dir_permissions_bind: true, - driver_status: String::new(), - probe_error: None, ..cap("wsl2") }; let both = decl(&[ @@ -1897,8 +1895,6 @@ serve_timeout_secs = 90 // scenario run and pass without testing anything. let as_root = HostCapability { dir_permissions_bind: false, - driver_status: String::new(), - probe_error: None, ..cap("wsl2") }; match resolve(&d, &as_root, &m, none) { @@ -1953,8 +1949,6 @@ serve_timeout_secs = 90 ); let host = HostCapability { dir_permissions_bind: false, - driver_status: String::new(), - probe_error: None, ..cap("strix-windows") }; assert_ne!(host.os_family, "linux", "fixture must fail the OS premise"); @@ -1991,8 +1985,6 @@ reason = "unrelated open bug" let folded = HostCapability { case_sensitive_fs: false, dir_permissions_bind: true, - driver_status: String::new(), - probe_error: None, ..cap("strix-ubuntu") }; assert!(matches!( diff --git a/tests/e2e-cucumber/src/lane_verdict.rs b/tests/e2e-cucumber/src/lane_verdict.rs index a2c63df93..9a909005a 100644 --- a/tests/e2e-cucumber/src/lane_verdict.rs +++ b/tests/e2e-cucumber/src/lane_verdict.rs @@ -8,8 +8,9 @@ //! resolves to skip never reach `report.json`, so a run in which every scenario //! was skipped, or none was selected, has no failure to report and passes. //! That is what a GPU lane does when its probe cannot read the binary under -//! test, or when the GPU is not visible: every `@requires-gpu` scenario -//! resolves to skip, and the lane is green without having tested a GPU. +//! test, or when the GPU is not visible: every GPU scenario (`@requires-gpu` or +//! `@requires-multi-gpu`) resolves to skip, and the lane is green without +//! having tested a GPU. //! //! Two checks close that gap, both pure so every row of their state matrix is a //! unit test: @@ -18,8 +19,8 @@ //! fatal on every lane, and so is a lane that declares a GPU (`1` in //! [`EXPECT_GPU_ENV`]) where the probe found none. //! - [`post_run_verdict`], after the run: a run that executed no scenario is -//! fatal on every lane, and so is a GPU-declaring lane on which no -//! `@requires-gpu` scenario executed. +//! fatal on every lane, and so is a GPU-declaring lane on which no GPU +//! scenario (`@requires-gpu` or `@requires-multi-gpu`) executed. //! //! The module also holds the decisions `tests/e2e.rs` takes around those two //! checks, so they are unit-tested too: [`preflight`] (the declaration plus @@ -122,8 +123,8 @@ pub fn preflight_verdict( verdict } -/// How many `@requires-gpu` scenarios the run resolved and how many of those -/// executed. +/// How many GPU scenarios (`@requires-gpu` or `@requires-multi-gpu`) the run +/// resolved and how many of those executed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct GpuScenarioCounts { /// Scenarios tagged `@requires-gpu` or `@requires-multi-gpu` that the @@ -160,13 +161,14 @@ pub fn post_run_verdict( match declaration { GpuDeclaration::Expected if gpu.executed == 0 => { verdict.fatal = Some(format!( - "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but no @requires-gpu scenario \ - executed ({} resolved, none ran)", + "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed ({} resolved, none ran)", gpu.resolved )); } GpuDeclaration::NotExpected => verdict.notes.push(format!( - "{} of {} @requires-gpu scenario(s) did not run on this lane ({EXPECT_GPU_ENV}=0)", + "{} of {} GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on this \ + lane ({EXPECT_GPU_ENV}=0)", gpu.resolved.saturating_sub(gpu.executed), gpu.resolved )), @@ -340,10 +342,9 @@ pub fn note_line(note: &str) -> String { #[cfg(test)] mod tests { use super::{ - EXPECT_GPU_ENV, GpuDeclaration, GpuScenarioCounts, Preflight, ResolvedRun, Verdict, - failure_lines, finish_run, gpu_scenario_counts, needs_gpu, note_line, - parse_gpu_declaration, post_run_verdict, preflight, preflight_verdict, run_failed, - workflow_command, + GpuDeclaration, GpuScenarioCounts, Preflight, ResolvedRun, Verdict, failure_lines, + finish_run, gpu_scenario_counts, needs_gpu, note_line, parse_gpu_declaration, + post_run_verdict, preflight, preflight_verdict, run_failed, workflow_command, }; use std::ffi::OsStr; @@ -357,11 +358,6 @@ mod tests { GpuDeclaration::Undeclared, ]; - #[test] - fn the_variable_name_is_the_one_the_workflows_set() { - assert_eq!(EXPECT_GPU_ENV, "E2E_EXPECT_GPU"); - } - #[test] fn only_one_zero_and_unset_or_empty_are_declarations() { assert_eq!(parse_gpu_declaration(None), Ok(GpuDeclaration::Undeclared)); @@ -534,8 +530,8 @@ mod tests { assert_eq!( verdict.fatal.as_deref(), Some( - "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu scenario \ - executed (7 resolved, none ran)" + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (7 resolved, none ran)" ) ); } @@ -557,8 +553,8 @@ mod tests { verdict, Verdict { fatal: Some( - "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu scenario \ - executed (0 resolved, none ran)" + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (0 resolved, none ran)" .to_owned() ), notes: vec![], @@ -607,7 +603,26 @@ mod tests { assert_eq!(verdict.fatal, None); assert_eq!( verdict.notes, - ["7 of 7 @requires-gpu scenario(s) did not run on this lane (E2E_EXPECT_GPU=0)"] + [ + "7 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \ + this lane (E2E_EXPECT_GPU=0)" + ] + ); + // Some ran: the note counts only the ones that did not. + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 3, + }), + GpuDeclaration::NotExpected, + ); + assert_eq!( + verdict.notes, + [ + "4 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \ + this lane (E2E_EXPECT_GPU=0)" + ] ); } @@ -776,8 +791,8 @@ mod tests { #[test] fn a_note_prints_as_an_escaped_notice() { assert_eq!( - note_line("7 of 7 @requires-gpu scenario(s) did not run (100%)"), - "::notice::E2E lane: 7 of 7 @requires-gpu scenario(s) did not run (100%25)" + note_line("7 of 7 GPU scenario(s) did not run (100%)"), + "::notice::E2E lane: 7 of 7 GPU scenario(s) did not run (100%25)" ); } @@ -891,8 +906,8 @@ mod tests { // Every cell is pass or skip, so only the recorded lane failure can // make this row FAIL. - let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu \ - scenario executed (1 resolved, none ran)"; + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (1 resolved, none ran)"; let (verdict, failed, mapping) = status_of(&resolved_with(&skip), &["examine-version"]); assert_eq!(verdict.fatal.as_deref(), Some(reason)); assert_eq!( diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index d3e74ccd3..554298d39 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1732,16 +1732,33 @@ jobs: ), ]; + /// Whether a step's `run:` body invokes the E2E suite itself + /// (`cargo xtask e2e`, not `e2e-prewarm` or `e2e-report`). + fn runs_the_e2e_suite(step: &str) -> bool { + run_block(step).is_some_and(|body| { + body.iter().any(|line| { + line.match_indices("cargo xtask e2e").any(|(at, m)| { + line[at + m.len()..] + .chars() + .next() + .is_none_or(char::is_whitespace) + }) + }) + }) + } + /// Every self-hosted E2E lane declares whether it has a GPU, so the harness /// can fail a GPU lane whose probe found none instead of passing it with - /// every `@requires-gpu` scenario skipped. + /// every GPU scenario (`@requires-gpu` or `@requires-multi-gpu`) skipped. /// /// Derived from the workflows like the GPU-preflight check above, so a new /// lane is held to it the day it lands. The declaration is read from the - /// job-level `env:` mapping only: a step-level one would leave the other - /// steps, and a reader of the job header, without it. A lane that runs the - /// suite inside WSL must also name the variable in every `WSLENV` it sets, - /// since `wsl.exe` passes nothing else into the guest. + /// job-level `env:` mapping: a step-level one would leave the other steps, + /// and a reader of the job header, without it, and one on the step that + /// runs the suite would override it there. A lane that runs the suite + /// inside WSL must name the variable in every `WSLENV` it sets, and in one + /// on the step that runs the suite, since `wsl.exe` passes nothing else + /// into the guest. #[test] fn every_self_hosted_lane_declares_whether_it_has_a_gpu() { let name = e2e_report::EXPECT_GPU_ENV; @@ -1772,23 +1789,58 @@ jobs: \"0\" (listed in NO_GPU_LANES)" ), } - let wslenvs: Vec<&str> = block + let why = NO_GPU_LANES + .iter() + .find(|(lane, _)| *lane == job) + .map_or_else(String::new, |(_, reason)| { + format!(" (a NO_GPU_LANES lane: {reason})") + }); + let names_it = |wslenv: &str| { + wslenv + .trim_matches(|c| c == '\'' || c == '"') + .split(':') + .any(|var| var.split('/').next() == Some(name)) + }; + for wslenv in block .lines() .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = ")) + { + assert!( + names_it(wslenv), + "{workflow} job `{job}`{why} sets WSLENV without {name}, so the suite \ + inside WSL reads the lane as undeclared: {wslenv}" + ); + } + // The step that runs the suite is where the declaration has + // to arrive: a step-level override there would replace the + // job's value, and a WSL step that names it in no WSLENV of + // its own leaves the guest without it, whatever other steps do. + let steps = job_steps(block) + .unwrap_or_else(|| panic!("{workflow} job `{job}` has no `steps:` list")); + let harness: Vec<&(String, String)> = steps + .iter() + .filter(|(_, step)| runs_the_e2e_suite(step)) .collect(); assert!( - !block.contains("Invoke-WslBash.ps1") || !wslenvs.is_empty(), - "{workflow} job `{job}` runs commands in WSL but sets no WSLENV, so \ - {name} never reaches the suite" + !harness.is_empty(), + "{workflow} job `{job}` has no step that runs `cargo xtask e2e`" ); - for wslenv in wslenvs { - let vars = wslenv.trim_matches(|c| c == '\'' || c == '"'); + for (step_name, step) in harness { assert!( - vars.split(':') - .any(|var| var.split('/').next() == Some(name)), - "{workflow} job `{job}` sets WSLENV without {name}, so the suite \ - inside WSL reads the lane as undeclared: {wslenv}" + !step_env(step).contains_key(name), + "{workflow} job `{job}`{why}: step `{step_name}` sets {name} itself, \ + overriding the job-level declaration" ); + if step.contains("Invoke-WslBash.ps1") { + assert!( + step.lines() + .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = ")) + .any(names_it), + "{workflow} job `{job}`{why}: step `{step_name}` runs the suite in \ + WSL but sets no WSLENV naming {name}, so the guest reads the lane \ + as undeclared" + ); + } } } } From 41d48d181bf05f3444dccc13f6f387b88074c43b Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 13:11:49 +0200 Subject: [PATCH 4/5] test(e2e): let the post-kill drain test fail (ROCMAI-388, ROCMAI-450) The late writer in the post-kill drain test inherited stdout as well as stderr, so the stdout wait already held the probe open until `late` was written, and the test stayed green with the stderr wait removed. Send the writer's stdout to /dev/null so only the stderr wait can collect it. Also correct two notes in the testing docs: passing an @id to -n selects nothing, and the run then fails with "0 scenarios executed" rather than passing silently. Refs: #383 Signed-off-by: Roman Inflianskas --- docs/testing.md | 6 ++++-- tests/e2e-cucumber/src/capability.rs | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index a687cccd8..67fbd61cf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4109,7 +4109,8 @@ to make the host match: masking every device with `HIP_VISIBLE_DEVICES=` makes the harness and the product agree the host has no usable GPU, which is what the mock lane has.) Note also that `-n` matches the scenario *name* (`serve-28 - Reuse succeeds on a GPU-less host`), not the `@id:` tag, so passing an id to it -silently selects nothing. +selects nothing and the run fails with `0 scenarios executed` (see +[E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)). - serve-28, `@id:serve-reuse-exempt-from-no-gpu-refusal` — a `serve` that matches an already-running vLLM service on a GPU-less host is not refused: it @@ -4260,7 +4261,8 @@ for the wrong reason instead of being reported as skipped. Do not select it by name on that route, nor any other `@requires-oom-fault-injection` scenario (`@id:serve-smoke-failure-summary` carries the same tag). Note also that `-n` matches the scenario *name* (`serve-41 - A launch that runs out of GPU memory ...`), not the `@id:` tag, so -passing an id to it silently selects nothing. +passing an id to it selects nothing and the run fails with +`0 scenarios executed` (see [E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)). - serve-41, `@id:serve-oom-launch-memory-guidance` — a launch that runs out of GPU memory blames this launch, names the knobs, and quotes the engine's own diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 2d410397b..a184bcadf 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -2412,9 +2412,10 @@ Local model engines } /// Stderr written after the kill still reaches the caller: the drain runs - /// after a timeout too. A grandchild that inherited stderr writes `late` - /// half a second after the 1 s deadline killed its parent, and the grace - /// leaves time to collect it. + /// after a timeout too. A grandchild holding only stderr (its stdout goes + /// to `/dev/null`) writes `late` half a second after the 1 s deadline + /// killed its parent. Stdout is closed by then, so only the wait on stderr + /// can collect it; the 10 s grace leaves time for that. #[cfg(unix)] #[test] fn stderr_written_after_the_kill_is_drained_into_the_timeout() { @@ -2422,7 +2423,10 @@ Local model engines let result = run_probe_bounded( std::ffi::OsStr::new("sh"), tmp.path(), - &["-c", "(sleep 1.5; echo late >&2) & exec sleep 30"], + &[ + "-c", + "(sleep 1.5; echo late >&2) >/dev/null & exec sleep 30", + ], std::time::Duration::from_secs(1), std::time::Duration::from_secs(10), ); From 2819c3304f81322119a748d517d4cb8dea8d5a11 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 17:56:38 +0200 Subject: [PATCH 5/5] docs(e2e): name report.json origin, drop counts (ROCMAI-388, ROCMAI-450) The probe-grace rationale in capability.rs's output_within doc, and the leaked-thread note under it, rested on "three" probes. They now state the invariant: each probe runs once per suite (the capability probes cached by host_capability, and collect_versions' version probe), not once per sample. The Cargo.toml note on the copied spawn-and-reap helpers likewise says "these copies" rather than counting them. The claim that a harness preflight refusal leaves an empty report.json that is still uploaded now names where that holds: run_suite in tests/e2e-cucumber/tests/e2e.rs creates report.json before it calls lane_verdict::preflight, and on every self-hosted lane that ran its suite the results upload (and the WSL copy-out) is gated on always(). This is said in the e2e-report test doc, the e2e README and both nightly.yml comments. Rewrapped the over-long lines in nightly.yml's missing-report comment, lane_verdict.rs's parse_gpu_declaration doc and e2e-report's EXPECT_GPU_ENV doc. Signed-off-by: Roman Inflianskas --- .github/workflows/nightly.yml | 29 +++++++++++++---------- crates/e2e-report/src/consolidated/mod.rs | 8 +++++-- crates/e2e-report/src/lib.rs | 5 ++-- tests/e2e-cucumber/Cargo.toml | 2 +- tests/e2e-cucumber/README.md | 9 +++++-- tests/e2e-cucumber/src/capability.rs | 10 ++++---- tests/e2e-cucumber/src/lane_verdict.rs | 5 ++-- 7 files changed, 43 insertions(+), 25 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a6b1608c5..04fd6acd2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1343,10 +1343,13 @@ jobs: # a failed step there does not turn the lane's check red. Only the # WSL lanes need this guard: every other lane writes report.json # directly in-job, with no guest-to-host copy step that can fail on - # its own. A harness preflight refusal is not one of those: it - # leaves an empty report.json that uploads fine, so the missing-report - # check does not catch it; its EMPTY row in the report is the signal - # (a post-run lane failure shows as a FAIL row with lane_failure). + # its own. A harness preflight refusal is not one of those: + # `run_suite` in tests/e2e-cucumber/tests/e2e.rs creates report.json + # empty before it calls `lane_verdict::preflight`, and the + # `if: always()` copy-out above carries it to the host, so it uploads + # fine and the missing-report check does not catch it; its EMPTY row + # in the report is the signal (a post-run lane failure shows as a + # FAIL row with lane_failure). if-no-files-found: error # The nightly lanes each uploaded a report nobody joined up, so reading a @@ -1523,14 +1526,16 @@ jobs: # A lane that dies before writing report.json (the GPU preflight step, a # build failure, a crash before results exist) uploads nothing. The - # harness's own preflight refusal is different: it leaves an empty - # report.json that does upload, and shows as an EMPTY row (a post-run - # lane failure as a FAIL row with lane_failure), not as missing. - # `discover()` can only render the reports that arrived, so a lane that - # uploads nothing would drop out of the grid with no row and no FAIL. On - # a `continue-on-error` lane nothing else is red either. So check it here: every lane that ran (any `needs` result but - # `skipped`, and not self-skipped as `stale`) must have uploaded its report - # artifact. This reads the run's + # harness's own preflight refusal is different: `run_suite` in + # tests/e2e-cucumber/tests/e2e.rs creates report.json empty before it + # calls `lane_verdict::preflight`, and each lane's `if: always()` upload + # step publishes it, so it shows as an EMPTY row (a post-run lane failure + # as a FAIL row with lane_failure), not as missing. `discover()` can only + # render the reports that arrived, so a lane that uploads nothing would + # drop out of the grid with no row and no FAIL. On a `continue-on-error` + # lane nothing else is red either. So check it here: every lane that ran + # (any `needs` result but `skipped`, and not self-skipped as `stale`) + # must have uploaded its report artifact. This reads the run's # artifact list rather than e2e-artifacts/, because download-artifact # flattens a single match into the root and loses its name. The list also # holds earlier attempts' artifacts, so a re-run that uploads nothing can diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 5c6a2cf3a..16de2c77b 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -949,8 +949,12 @@ mod tests { /// The artifact a lane uploads when the harness's PRE-run check refuses to /// run (a failed capability probe, a declared GPU the probe did not find, - /// an invalid `E2E_EXPECT_GPU`): the harness creates `report.json` empty - /// before probing and exits before writing `platform.json`. That lane must + /// an invalid `E2E_EXPECT_GPU`): `run_suite` in + /// `tests/e2e-cucumber/tests/e2e.rs` creates `report.json` empty before it + /// calls `lane_verdict::preflight`, and exits there before writing + /// `platform.json`; on every self-hosted lane that ran its suite, the + /// report upload step (and the WSL lanes' copy-out from the guest) is + /// gated on `always()`, so that empty file is what arrives. That lane must /// read as needing attention, never green. A POST-run lane-check failure /// writes `platform.json` with `lane_failure` instead; see the tests below. /// diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index b63ce8fed..448e9d394 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -43,8 +43,9 @@ pub const OOM_FAULT_INJECTION_ENV: &str = "ROCM_E2E_OOM_FAULT_INJECTION"; /// before any scenario runs when its capability probe finds no usable AMD GPU, /// and after the run when no `@requires-gpu` (or `@requires-multi-gpu`) /// scenario executed. `0` means the lane deliberately has no usable GPU (the -/// WSL lanes, whose distro is left without ROCm's WSL driver bridge). Unset or empty leaves the lane undeclared, -/// as on the GitHub-hosted lanes and local runs; any other value is an error. +/// WSL lanes, whose distro is left without ROCm's WSL driver bridge). Unset or +/// empty leaves the lane undeclared, as on the GitHub-hosted lanes and local +/// runs; any other value is an error. /// /// Shared by the harness, which reads it, and xtask's workflow contract test, /// which checks that every self-hosted E2E lane sets it, so the two cannot name diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index cbe30a22f..73050d327 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -94,7 +94,7 @@ shlex.workspace = true # `capability::output_within` cannot call `rocm_core::process::run_with_timeout`: # each repeats that helper's spawn, poll, kill and reap steps on `std` alone, # deliberately, and a normal dependency would be a new edge to review. A change -# to any one of the three should look at the other two. +# to any one of these copies should look at the others. rocm-core = { path = "../../crates/rocm-core" } # `test-util` pauses tokio's clock in the retry loops' unit tests # (`#[tokio::test(start_paused = true)]`), so their timing assertions read a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 31bd6bd1d..c68381db7 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -121,8 +121,13 @@ scenarios: open, held by a process it started (with the stderr tail so far). On a lane that sets `E2E_EXPECT_GPU=1` it also fails when the probe found no usable AMD GPU, naming the detected gfx target, WSL flag, driver status and visibility - mask it decided from. A run refused here writes no `platform.json`, and the - consolidated report shows it as EMPTY. + mask it decided from. A run refused here writes no `platform.json`; its + `report.json` is already there, empty, because `run_suite` in + [`tests/e2e.rs`](tests/e2e.rs) creates it before calling + `lane_verdict::preflight`, and on every self-hosted lane that ran its suite + the results upload step is gated on `always()` (`e2e-gpu`'s also skips a + run that found itself stale, which never reaches the suite), so the + consolidated report shows the lane as EMPTY. - **After the run**, it fails when no scenario executed (for example a `-n` filter that matches nothing), and on an `E2E_EXPECT_GPU=1` lane when no `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU scenarios diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index a184bcadf..1b0cf7f33 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -968,15 +968,17 @@ impl PipeReader { /// because the stderr written before the deadline is the only evidence of /// why `rocm examine` hung; /// - its grace is 5 s (`PROBE_DRAIN_GRACE`) rather than 2 s, because it runs -/// three times per suite (two capability probes and `collect_versions`) -/// rather than once per sample, so a generous wait for a loaded runner to -/// hand over the bytes costs little. +/// a handful of times per suite: each probe runs once (the capability +/// probes in `probe_host_capability`, cached by `host_capability`, and the +/// version probe in `collect_versions`, whose `rocm examine` repeats the +/// capability probe's), not once per sample, so a generous wait for a +/// loaded runner to hand over the bytes costs little. /// /// Only the direct child is killed: a process it started that inherited a pipe /// can keep that pipe open past the grace. The reader thread for that pipe is /// then left detached until the pipe closes, and the bytes it had read are /// used as they are. A leaked thread per stuck probe is accepted in a test -/// harness that runs three probes and exits. +/// harness that runs a handful of probes, each once, and exits. fn output_within( mut command: std::process::Command, timeout: std::time::Duration, diff --git a/tests/e2e-cucumber/src/lane_verdict.rs b/tests/e2e-cucumber/src/lane_verdict.rs index 9a909005a..9d58ef537 100644 --- a/tests/e2e-cucumber/src/lane_verdict.rs +++ b/tests/e2e-cucumber/src/lane_verdict.rs @@ -53,8 +53,9 @@ pub enum GpuDeclaration { /// Read [`EXPECT_GPU_ENV`]'s value. /// -/// Anything other than `1`, `0`, empty or unset is an error rather than a guess, since a misspelt declaration that read -/// as "undeclared" would switch the GPU check off without anyone noticing. +/// Anything other than `1`, `0`, empty or unset is an error rather than a +/// guess, since a misspelt declaration that read as "undeclared" would switch +/// the GPU check off without anyone noticing. pub fn parse_gpu_declaration(value: Option<&OsStr>) -> Result { match value.map(OsStr::to_str) { None | Some(Some("")) => Ok(GpuDeclaration::Undeclared),