,
+}
+
+/// Load one platform's `report.json` (and its sibling `platform.json`) the way
+/// the consolidated report does, and return its status.
+///
+/// Public so the harness that writes `platform.json` can test that what it
+/// writes is read back as intended, keeping producer and consumer tied.
+#[must_use]
+pub fn platform_status(label: &str, report_json: &Path) -> PlatformStatus {
+ let report = PlatformReport::load(label.to_owned(), report_json);
+ PlatformStatus {
+ ok: report.ok(),
+ status: report.status_text(),
+ lane_failure: report.lane_failure().map(str::to_owned),
+ }
+}
+
/// Parse every `(label, json_path)` input into a [`PlatformReport`], in input
/// order. The one place `report.json`/`platform.json`/`commands.jsonl` are
/// read from disk — [`Grid::build`] and [`scenario_reference`] consume the
@@ -897,6 +947,237 @@ mod tests {
assert_eq!(r.status_text(), "EMPTY");
}
+ /// The artifact a lane uploads when the harness's PRE-run check refuses to
+ /// run (a failed capability probe, a declared GPU the probe did not find,
+ /// an invalid `E2E_EXPECT_GPU`): `run_suite` in
+ /// `tests/e2e-cucumber/tests/e2e.rs` creates `report.json` empty before it
+ /// calls `lane_verdict::preflight`, and exits there before writing
+ /// `platform.json`; on every self-hosted lane that ran its suite, the
+ /// report upload step (and the WSL lanes' copy-out from the guest) is
+ /// gated on `always()`, so that empty file is what arrives. That lane must
+ /// read as needing attention, never green. A POST-run lane-check failure
+ /// writes `platform.json` with `lane_failure` instead; see the tests below.
+ ///
+ /// The report code already behaved this way before the harness could exit
+ /// early, so this is not a regression test; it pins that behaviour because
+ /// the early exit now relies on it to keep a refused lane off green in the grid.
+ #[test]
+ fn a_lane_that_stopped_before_any_scenario_is_not_ok() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let report = dir.path().join("report.json");
+ std::fs::write(&report, "").expect("write empty report.json");
+ let r = PlatformReport::load("e2e-gpu-report".into(), &report);
+ assert!(r.manifest.is_none(), "no platform.json was written");
+ assert!(!r.ok(), "an empty run must not count as healthy");
+ assert_eq!(r.status_text(), "EMPTY");
+ }
+
+ /// `platform.json` for a lane whose every resolved scenario was a skip,
+ /// with or without a recorded post-run lane failure.
+ fn all_skip_platform(lane_failure: Option<&str>) -> String {
+ let failure = lane_failure.map_or(String::new(), |reason| {
+ format!(r#""lane_failure": {},"#, serde_json::json!(reason))
+ });
+ format!(
+ r#"{{"platform_slug":"mi300x","capability":{{"effective_serve_engine":"vllm"}},{failure}
+ "expectations":[{{"id":"serve-a","feature":"Serving","expected":"skip"}}]}}"#
+ )
+ }
+
+ /// The regression: a GPU lane on which every scenario resolved to skip
+ /// was failed by the harness ("no @requires-gpu (or @requires-multi-gpu)
+ /// scenario executed"), yet
+ /// its row read PASS, since a skip cell is not a problem.
+ #[test]
+ fn an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason() {
+ let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \
+ @requires-multi-gpu) scenario executed (1 resolved, none ran)";
+ let (_dir, report) = write_platform("[]", &all_skip_platform(Some(reason)));
+ let r = PlatformReport::load("e2e-gpu-report".into(), &report);
+ assert!(!r.ok());
+ assert_eq!(r.status_text(), "FAIL");
+ assert_eq!(r.lane_failure(), Some(reason));
+ assert_eq!(
+ platform_status("e2e-gpu-report", &report),
+ PlatformStatus {
+ ok: false,
+ status: "FAIL",
+ lane_failure: Some(reason.to_owned()),
+ }
+ );
+ }
+
+ /// Unchanged: an all-skip lane the harness did not fail (an undeclared or
+ /// `E2E_EXPECT_GPU=0` lane) stays green.
+ #[test]
+ fn an_all_skip_lane_without_a_lane_failure_still_reads_pass() {
+ let (dir, report) = write_platform("[]", &all_skip_platform(None));
+ let r = PlatformReport::load("e2e-gpu-report".into(), &report);
+ assert!(r.ok());
+ assert_eq!(r.status_text(), "PASS");
+ assert_eq!(r.lane_failure(), None);
+
+ // ...and neither rendering shows any lane-failure element. (The legend
+ // and the explanation paragraph name the heading in prose, so the
+ // checks are on the elements themselves.)
+ let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]);
+ let md = markdown::consolidated_summary_markdown(&bundle);
+ assert!(!md.contains("### Needs attention: platforms"), "{md}");
+ assert!(!md.contains("**Lane check failed**"), "{md}");
+ let html_out = dir.path().join("out.html");
+ html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default())
+ .expect("html written");
+ let html = std::fs::read_to_string(&html_out).expect("html read");
+ assert!(
+ !html.contains("Needs attention: platforms
"),
+ "{html}"
+ );
+ assert!(
+ html.contains(r#"PASS | "#),
+ "the Status cell carries no tooltip:\n{html}"
+ );
+ assert!(!html.contains(r#""#), "{html}");
+ }
+
+ /// A `-n` filter that matched nothing: no expectations were resolved and
+ /// nothing ran, and the harness recorded "0 scenarios executed".
+ #[test]
+ fn a_zero_expectation_lane_reads_fail_only_with_a_recorded_failure() {
+ let failed =
+ r#"{"platform_slug":"mock","lane_failure":"0 scenarios executed","expectations":[]}"#;
+ let (_dir, report) = write_platform("[]", failed);
+ let r = PlatformReport::load("e2e-report".into(), &report);
+ assert!(!r.ok());
+ assert_eq!(r.status_text(), "FAIL");
+
+ let (_dir2, report) = write_platform("[]", r#"{"platform_slug":"mock","expectations":[]}"#);
+ let r = PlatformReport::load("e2e-report".into(), &report);
+ assert!(r.ok(), "no recorded failure keeps today's reading");
+ assert_eq!(r.status_text(), "PASS");
+ }
+
+ /// Both renderings name the reason, not just a FAIL.
+ #[test]
+ fn a_recorded_lane_failure_is_named_in_the_summary_and_the_html() {
+ // A newline, markup characters and a quote, so both the markdown's
+ // newline folding and the HTML escaping are exercised.
+ let reason = "0 scenarios executed\nsee & \"tail\"";
+ let html_reason = "0 scenarios executed\nsee <stderr> & "tail"";
+ let (dir, report) = write_platform("[]", &all_skip_platform(Some(reason)));
+ let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]);
+
+ let md = markdown::consolidated_summary_markdown(&bundle);
+ assert!(
+ md.contains(
+ "\n### Needs attention: platforms\n\n- **Lane check failed** in _MI300X \
+ Linux_: 0 scenarios executed see & \"tail\"\n"
+ ),
+ "{md}"
+ );
+ // A CRLF, or any run of whitespace, folds to one space too. Defensive:
+ // no current producer of `lane_failure` emits CR or LF.
+ let (_crlf_dir, crlf_report) =
+ write_platform("[]", &all_skip_platform(Some("first\r\n second")));
+ let crlf_md = markdown::consolidated_summary_markdown(&load_platform_reports(&[(
+ "e2e-gpu-report".to_owned(),
+ crlf_report,
+ )]));
+ assert!(crlf_md.contains("Linux_: first second\n"), "{crlf_md}");
+
+ let html_out = dir.path().join("out.html");
+ html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default())
+ .expect("html written");
+ let html = std::fs::read_to_string(&html_out).expect("html read");
+ // The Status cell's tooltip.
+ assert!(
+ html.contains(&format!(
+ r#"| FAIL | "#
+ )),
+ "status tooltip:\n{html}"
+ );
+ // The platform section's paragraph.
+ assert!(
+ html.contains(&format!(
+ r#"Lane check failed: {html_reason}
"#
+ )),
+ "lane failure paragraph:\n{html}"
+ );
+ // The platform-level attention list.
+ assert!(
+ html.contains(&format!(
+ r"Needs attention: platforms
- Lane check failed in
MI300X Linux: {html_reason}
"
+ )),
+ "attention list:\n{html}"
+ );
+ // ...styled, so it reads as a failure.
+ assert!(
+ html.contains(".lane-failure {"),
+ "CSS rule for .lane-failure"
+ );
+ // The legend says where the reason is.
+ assert!(
+ html.contains(
+ "or when the harness failed the lane after the run (for example, no scenario \
+ executed); the reason is listed under \u{201c}Needs attention: platforms\u{201d} \
+ and in the platform's section."
+ ),
+ "legend:\n{html}"
+ );
+ }
+
+ /// A lane on which scenarios ran and passed, but which the harness still
+ /// failed (on a GPU lane, no GPU scenario among them): its row
+ /// reads FAIL with a Fail count of 0, since no cell failed. Rows of exactly
+ /// this shape are what `total_e2e_failures` in xtask counts as failures.
+ #[test]
+ fn a_lane_that_ran_scenarios_and_was_failed_reads_fail_with_zero_failed_cells() {
+ let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \
+ @requires-multi-gpu) scenario executed (3 resolved, none ran)";
+ let platform = format!(
+ r#"{{"platform_slug":"mi300x","lane_failure":{},"expectations":[
+ {{"id":"examine-a","feature":"F","expected":"pass"}}]}}"#,
+ serde_json::json!(reason)
+ );
+ let (_dir, report) = write_platform(
+ &feature_json(&[(&["id:examine-a"], &["passed"])]),
+ &platform,
+ );
+ let r = PlatformReport::load("e2e-gpu-report".into(), &report);
+ assert_eq!(r.display_counts(), (1, 1, 0, 0, 0));
+ assert!(!r.ok());
+ assert_eq!(r.status_text(), "FAIL");
+ let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]);
+ let md = markdown::consolidated_summary_markdown(&bundle);
+ assert!(
+ md.contains("| MI300X | Linux | 1 | 1 | 0 | 0 | n/a | FAIL |"),
+ "{md}"
+ );
+ assert!(md.contains("| **Total** | | 1 | 1 | 0 | 0 | 0 | |"), "{md}");
+ }
+
+ /// Both attention lists can appear in one summary, under different
+ /// headings, each once.
+ #[test]
+ fn the_two_attention_lists_have_distinct_headings() {
+ let platform = r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed",
+ "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#;
+ let (_dir, report) =
+ write_platform(&feature_json(&[(&["id:examine-a"], &["failed"])]), platform);
+ let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]);
+ let md = markdown::consolidated_summary_markdown(&bundle);
+ assert_eq!(
+ md.matches("### Needs attention: platforms\n").count(),
+ 1,
+ "{md}"
+ );
+ assert_eq!(
+ md.matches("### Needs attention: scenarios\n").count(),
+ 1,
+ "{md}"
+ );
+ assert_eq!(md.matches("### Needs attention\n").count(), 0, "{md}");
+ }
+
#[test]
fn rendering_uses_the_bundle_not_the_artifacts_on_disk() {
// The render functions must work from what `load_platform_reports`
@@ -1765,8 +2046,8 @@ mod tests {
#[test]
fn renderers_take_the_pr_gating_phrases_from_the_shared_constants() {
// What this proves, and what it does not. The rendered text is pinned
- // verbatim by `summary_explanation_paragraphs_are_unchanged` (markdown)
- // and `legend_mock_and_other_platform_items_are_unchanged` (HTML), and
+ // verbatim by `summary_explanation_paragraphs_are_pinned` (markdown)
+ // and `legend_mock_and_other_platform_items_are_pinned` (HTML), and
// each of those also asserts that the rendered output contains
// `MOCK_GATES_PR` and `OTHER_LANES_NON_BLOCKING`; together they catch a
// renderer that hardcodes the words while a constant's value moves on.
diff --git a/crates/e2e-report/src/consolidated/sidecars.rs b/crates/e2e-report/src/consolidated/sidecars.rs
index b1ff32386..28de5088e 100644
--- a/crates/e2e-report/src/consolidated/sidecars.rs
+++ b/crates/e2e-report/src/consolidated/sidecars.rs
@@ -54,6 +54,10 @@ pub(super) struct PlatformManifest {
/// Component versions (OS/ROCm/vLLM/lemonade); absent in older artifacts.
#[serde(default)]
pub(super) versions: PlatformVersions,
+ /// Why the harness's post-run lane check failed the run (for example "0
+ /// scenarios executed"); absent when it passed, and in older artifacts.
+ #[serde(default)]
+ pub(super) lane_failure: Option,
#[serde(default)]
pub(super) expectations: Vec,
}
diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs
index 3eaae3947..448e9d394 100644
--- a/crates/e2e-report/src/lib.rs
+++ b/crates/e2e-report/src/lib.rs
@@ -17,8 +17,8 @@ mod single_report;
pub use consolidated::html::{RunMeta, generate_consolidated};
pub use consolidated::markdown::consolidated_summary_markdown;
pub use consolidated::{
- ADVISORY_QUEUE_MARKER, PlatformReports, artifact_name_for_platform_slug, load_platform_reports,
- queue_artifact_name, queue_lane_gpu,
+ ADVISORY_QUEUE_MARKER, PlatformReports, PlatformStatus, artifact_name_for_platform_slug,
+ load_platform_reports, platform_status, queue_artifact_name, queue_lane_gpu,
};
pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id};
pub use single_report::generate;
@@ -36,3 +36,18 @@ pub use single_report::generate;
/// typo previously would not fail to compile or fail a test, silently turning
/// `@requires-oom-fault-injection` into a skip (green suite, zero coverage).
pub const OOM_FAULT_INJECTION_ENV: &str = "ROCM_E2E_OOM_FAULT_INJECTION";
+
+/// Environment variable in which a CI lane declares whether it has a GPU.
+///
+/// `1` means the lane runs on GPU hardware, so the `e2e-cucumber` harness fails
+/// before any scenario runs when its capability probe finds no usable AMD GPU,
+/// and after the run when no `@requires-gpu` (or `@requires-multi-gpu`)
+/// scenario executed. `0` means the lane deliberately has no usable GPU (the
+/// WSL lanes, whose distro is left without ROCm's WSL driver bridge). Unset or
+/// empty leaves the lane undeclared, as on the GitHub-hosted lanes and local
+/// runs; any other value is an error.
+///
+/// Shared by the harness, which reads it, and xtask's workflow contract test,
+/// which checks that every self-hosted E2E lane sets it, so the two cannot name
+/// different variables.
+pub const EXPECT_GPU_ENV: &str = "E2E_EXPECT_GPU";
diff --git a/docs/architecture.md b/docs/architecture.md
index b929c3165..b30cf8fe8 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -634,7 +634,7 @@ HTML and markdown reporting for the cucumber E2E suite, consumed only by `xtask`
| [`lib.rs`](../crates/e2e-report/src/lib.rs) | Crate root: module declarations and the public API re-exports. | flat |
| [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) | The CLI command-coverage audit table: which `rocm` commands a platform's run exercised and whether they passed. | domain |
| [`components.rs`](../crates/e2e-report/src/components.rs) | Shared maud HTML fragments plus the CSS and timestamp helpers both report generators use. | flat |
-| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, plus the report text both renderers share. | domain |
+| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, `platform_status` (one platform's status, public so the harness can test the `platform.json` it writes), plus the report text both renderers share. | domain |
| [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) | Artifact-name identity: parses an artifact or directory name into platform, OS and channel, and owns the platform-label rule. | domain |
| [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) | The HTML renderer for the consolidated report. It does not render `PlatformVersions::provenance()`: the GFX target and package source appear only in the Markdown output (the wiki page and the step summary that shares its renderer). | domain |
| [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) | The markdown renderer, including the table piped into `$GITHUB_STEP_SUMMARY`. | domain |
@@ -788,7 +788,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr
| --- | --- | --- |
| [`src/lib.rs`](../tests/e2e-cucumber/src/lib.rs) | Library root for the test targets and the demo binary: declares the helper modules, re-exports `e2e-report` as `report`, and holds small shared helpers such as `cli_failure_report`. | flat |
| [`src/binary_path.rs`](../tests/e2e-cucumber/src/binary_path.rs) | Pins a relative `ROCM_CLI_BINARY` path before a scenario runs the child from another directory, and leaves a bare name for `PATH` to resolve. | flat |
-| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat |
+| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, records why either could not be read (`ProbeError`: not started, or could not be waited on, non-zero exit, no output, still running at its deadline (killed and reaped), or output left open after exit), keeping the stderr read so far, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat |
| [`src/chat_request.rs`](../tests/e2e-cucumber/src/chat_request.rs) | The chat round trip against a served endpoint (model discovery, then one chat completion), run on its own task so its timeout measures the server. | flat |
| [`src/dash_screen.rs`](../tests/e2e-cucumber/src/dash_screen.rs) | Pure matching over captured dashboard screens and config files, so the logic the dashboard scenarios depend on has unit tests. | flat |
| [`src/device_memory.rs`](../tests/e2e-cucumber/src/device_memory.rs) | Which device memory pool (VRAM, or GTT-backed memory on an APU) the drain-wait between serve scenarios judges, how it is sampled, and the free-memory floor. | flat |
@@ -800,6 +800,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr
| [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat |
| [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat |
| [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat |
+| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no GPU scenario: `@requires-gpu` or `@requires-multi-gpu`) executed. Also holds the decisions [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) takes around those checks, each unit-tested: reading the declaration, counting GPU scenarios, building the `platform.json` manifest that records a lane failure (`finish_run`), the final exit, and the exact escaped `::error::`/`::notice::` lines. The harness itself runs the probe and the scenarios, classifies their results, writes the files and exits; that glue has no test. | flat |
| [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat |
| [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat |
| [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat |
diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md
index 76834f785..6c8de6387 100644
--- a/docs/ci-hardware-testing.md
+++ b/docs/ci-hardware-testing.md
@@ -156,13 +156,13 @@ scenario to pass / xfail / skip for that host from its `@id` and
`@requires-*` tags, a capability probe, and `expectations.toml` — there is no
separate tier flag or tag filter to maintain.
-| Job | Workflow | Platform | Runner labels |
-|---|---|---|---|
-| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` |
-| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` |
-| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` |
-| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` |
-| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` |
+| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) |
+|---|---|---|---|---|
+| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | unset |
+| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | `1` |
+| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | `1` |
+| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `1` |
+| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `0` |
On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` is skipped; it still
runs on `push` and dispatch. Over 7 days 35% of its merge-queue jobs (98 of 281)
@@ -171,6 +171,17 @@ not wait for it. On a
PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo
pool).
+The last column is the lane's declaration of whether it has a GPU. On a lane
+that declares `1`, the harness fails before any scenario runs if its capability
+probe finds no usable AMD GPU, and after the run if no `@requires-gpu` (or
+`@requires-multi-gpu`) scenario executed; on every lane it fails if the probe
+could not read the binary under test or if no scenario executed at all. `nightly.yml`'s self-hosted lanes
+declare the same way (`0` only for `e2e-wsl-nightly`), as does every lane of
+[`run-e2e-lane.sh`](../scripts/run-e2e-lane.sh). `every_self_hosted_lane_declares_whether_it_has_a_gpu`
+in [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) fails when a
+self-hosted lane declares neither. See the [E2E README](../tests/e2e-cucumber/README.md)'s
+"Lane verdict".
+
`e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release,
nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact
names, so that lane proves the nightly package channel boots. It no longer runs
@@ -260,8 +271,8 @@ resolve to skip here, and known bugs resolve to xfail from
The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`,
and `e2e-wsl`) run on AMD GPU systems, so they
exercise host/GPU detection, engine `detect`/`capabilities`, and live serving
-scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as
-described below.
+scenarios that the mock job cannot. The WSL lane declares no usable GPU
+(`E2E_EXPECT_GPU=0`) and its GPU preflight is advisory, as described below.
`e2e-wsl` runs on the DevLab Dispatch pool. Its WSL2/Ubuntu-24.04 distro is
built fresh inside every job (see above), so there is nothing of its own for
@@ -275,9 +286,13 @@ preflight, toolchain bootstrap, shared-runtime pre-warm, then the full suite
with no hand filtering. It covers WSL host detection, the
Windows-to-WSL execution boundary, and whatever GPU access WSL exposes on that
machine. The GPU preflight is advisory here precisely because GPU-on-WSL is
-what the lane is proving out: where it is unavailable the capability probe
-resolves those scenarios to not-applicable and the rest of the suite still
-runs. Scenarios the product deliberately routes around on WSL carry
+what the lane is proving out. The lane declares `E2E_EXPECT_GPU=0` (forwarded
+into the guest through `WSLENV`), because its distro is deliberately left
+without the driver bridge (see "What the WSL distro needs"): the capability
+probe resolves `@requires-gpu` scenarios to not-applicable, the harness prints
+a note saying so rather than failing, and the rest of the suite still runs. If
+the probe finds a usable GPU anyway, the harness notes that the declaration is
+stale and runs the GPU scenarios. Scenarios the product deliberately routes around on WSL carry
`@requires-bare-metal`;
scenarios whose premise *is* a WSL host carry `@requires-wsl`, and this is the
only lane that runs them.
@@ -302,7 +317,9 @@ driver scenarios assert on the plan that command produces — a lane that
pre-installed it would hand those scenarios a host already in the state the
command under test exists to reach. That costs no coverage today, because
`rocm-smi` is absent too, so `@requires-gpu` scenarios resolve to skip either
-way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`).
+way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`). This is why
+the lane declares `E2E_EXPECT_GPU=0`; a lane that provisions the bridge
+declares `1` instead.
GPU coverage additionally needs ROCm's WSL passthrough to be complete —
`/dev/dxg` and dxcore alone are not enough, `librocdxg.so` must be present and
@@ -661,6 +678,12 @@ skipped itself as stale is the exception); on a
`continue-on-error: true` lane that is the only red signal for such a failure.
The nightly report job applies the same check to every nightly lane.
+A lane the harness failed after its run (no scenario executed, or no GPU
+scenario, `@requires-gpu` or `@requires-multi-gpu`, on an `E2E_EXPECT_GPU=1`
+lane) records the reason in its `platform.json`, and its row reads FAIL with the reason under "Needs
+attention: platforms" (step summary and HTML report alike), even when every
+scenario it resolved was a skip.
+
### Timeouts on the Strix lanes
Every Strix Halo lane raises two budgets rather than letting a slow host read
diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md
index e4fae4508..c57d35440 100644
--- a/docs/nightly-on-the-queue.md
+++ b/docs/nightly-on-the-queue.md
@@ -125,7 +125,11 @@ limit (see the run's inputs above). Each channel starts with the GPU preflight,
channel's job in `nightly.yml` does, so a model server the previous channel's
suite left on the GPU is reclaimed first; the lane log shows it. Both always
run, so a failing channel does not hide the other, and the lane fails if either
-did.
+did. The default command also declares that every lane has a GPU
+(`E2E_EXPECT_GPU`, default `1`), so a channel whose suite finds no usable AMD
+GPU, or runs no GPU scenario (`@requires-gpu` or `@requires-multi-gpu`), fails
+instead of passing with every GPU scenario skipped; see "Lane verdict" in the
+[E2E README](../tests/e2e-cucumber/README.md).
A lane starts cold, so its first steps fetch from the network: the build
packages, the rustup installer, the toolchain and the crates for the host
diff --git a/docs/testing.md b/docs/testing.md
index f2e113962..7d60b34f5 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -951,6 +951,45 @@ export ROCM_TEST_APK_REPOS="--repository http://dl-cdn.alpinelinux.org/alpine/v3
--repository http://dl-cdn.alpinelinux.org/alpine/v3.20/community"
```
+## E2E runs that test nothing fail
+
+`cargo xtask e2e` fails a run whose result could not mean anything, instead of
+reporting it green:
+
+- before any scenario runs, when the capability probe could not read the binary
+ under test: `rocm examine` or `rocm engines list` could not be started (check
+ `ROCM_CLI_BINARY`), or could not be waited on, exited non-zero, printed
+ nothing, was still running after `E2E_PROBE_TIMEOUT_SECS` (whole seconds ≥ 1;
+ anything else uses 120), or exited but left its output open. The post-run
+ version probe (`collect_versions`) uses the same deadline but fails quietly,
+ only blanking the version columns, and does not refuse the run;
+- after the run, when no scenario executed, which includes a
+ `cargo xtask e2e -- -n ` whose regex matches no scenario name. The
+ reason is written into `platform.json`, so `cargo xtask e2e-report` shows that
+ lane as FAIL with the reason under "Needs attention: platforms", in both its
+ markdown summary and its HTML report.
+
+`E2E_EXPECT_GPU` declares whether the host has a GPU. With `1`, the run also
+fails before any scenario if the probe found no usable AMD GPU, and after the
+run if no `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU
+scenarios the run's own configuration leaves out (`@nightly`, `@merge-queue`,
+or anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved
+but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and no
+GPU scenario selected fails by design. With `0`, it only notes that GPU
+scenarios are not applicable (and that the declaration is stale if a GPU is
+found). Unset, as on a local run, it makes no GPU claim. Every self-hosted CI
+lane sets it; see "Lane verdict" in the [E2E README](../tests/e2e-cucumber/README.md)
+and the Platforms table in [ci-hardware-testing.md](ci-hardware-testing.md). The
+checks are unit-tested in [lane_verdict.rs](../tests/e2e-cucumber/src/lane_verdict.rs)
+and, for each way a probe can fail, in
+[capability.rs](../tests/e2e-cucumber/src/capability.rs):
+
+```bash
+cargo test -p e2e-cucumber --lib lane_verdict
+cargo test -p e2e-cucumber --lib capability::tests
+cargo test -p xtask every_self_hosted_lane_declares_whether_it_has_a_gpu
+```
+
## CI test selection
On pull requests, CI does not test the whole workspace. The `test` job runs
@@ -4377,7 +4416,8 @@ to make the host match: masking every device with `HIP_VISIBLE_DEVICES=` makes
the harness and the product agree the host has no usable GPU, which is what the
mock lane has.) Note also that `-n` matches the scenario *name* (`serve-28 -
Reuse succeeds on a GPU-less host`), not the `@id:` tag, so passing an id to it
-silently selects nothing.
+selects nothing and the run fails with `0 scenarios executed` (see
+[E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)).
- serve-28, `@id:serve-reuse-exempt-from-no-gpu-refusal` — a `serve` that
matches an already-running vLLM service on a GPU-less host is not refused: it
@@ -4528,7 +4568,8 @@ for the wrong reason instead of being reported as skipped. Do not select it by
name on that route, nor any other `@requires-oom-fault-injection` scenario
(`@id:serve-smoke-failure-summary` carries the same tag). Note also that `-n` matches the scenario *name*
(`serve-41 - A launch that runs out of GPU memory ...`), not the `@id:` tag, so
-passing an id to it silently selects nothing.
+passing an id to it selects nothing and the run fails with
+`0 scenarios executed` (see [E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)).
- serve-41, `@id:serve-oom-launch-memory-guidance` — a launch that runs out of
GPU memory blames this launch, names the knobs, and quotes the engine's own
diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh
index 68e8569f8..4e2ee32a3 100755
--- a/scripts/run-e2e-lane.sh
+++ b/scripts/run-e2e-lane.sh
@@ -22,6 +22,11 @@
# both (default "release nightly")
# E2E_LANE_WORK where the build and runtimes go (default /var/tmp/e2e-lane)
# LANE_OUT where reported files go (default /out)
+# E2E_EXPECT_GPU whether the lane has a GPU (default 1: every queue lane is
+# given one); the suite fails rather than passing with every
+# GPU scenario skipped. Set but empty, it leaves the lane
+# undeclared, which switches that check off. See
+# tests/e2e-cucumber/README.md.
set -euo pipefail
@@ -39,6 +44,7 @@ if [[ -n "${CARGO_BUILD_TARGET:-}" ]]; then
fi
export E2E_INCLUDE_NIGHTLY="${E2E_INCLUDE_NIGHTLY-1}"
+export E2E_EXPECT_GPU="${E2E_EXPECT_GPU-1}"
export E2E_SERVE_TIMEOUT_SECS="${E2E_SERVE_TIMEOUT_SECS:-300}"
# The channels are checked before anything is built, so a typo costs nothing.
diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml
index 3efc130f5..73050d327 100644
--- a/tests/e2e-cucumber/Cargo.toml
+++ b/tests/e2e-cucumber/Cargo.toml
@@ -90,10 +90,11 @@ shlex.workspace = true
# itself, and `xtask check-crate-edges` exempts dev edges by design — so keeping
# it here needs no allowlist entry.
#
-# For the same reason the library's `device_memory::bounded_stdout` cannot call
-# `rocm_core::process::run_with_timeout`: it repeats that helper's spawn, poll,
-# kill and reap steps on `std` alone, deliberately, and a normal dependency
-# would be a new edge to review. A change to either one should look at the other.
+# For the same reason the library's `device_memory::bounded_stdout` and
+# `capability::output_within` cannot call `rocm_core::process::run_with_timeout`:
+# each repeats that helper's spawn, poll, kill and reap steps on `std` alone,
+# deliberately, and a normal dependency would be a new edge to review. A change
+# to any one of these copies should look at the others.
rocm-core = { path = "../../crates/rocm-core" }
# `test-util` pauses tokio's clock in the retry loops' unit tests
# (`#[tokio::test(start_paused = true)]`), so their timing assertions read a
diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md
index d871ab21d..c68381db7 100644
--- a/tests/e2e-cucumber/README.md
+++ b/tests/e2e-cucumber/README.md
@@ -64,7 +64,7 @@ runs the suite. Extra arguments after `--` are forwarded to the cucumber CLI.
# All features:
cargo xtask e2e
-# Filter by scenario name:
+# Filter by scenario name (bypasses capability gating; see below):
cargo xtask e2e -- -n "model name"
# Stop on first failure:
@@ -101,6 +101,57 @@ E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e
| `ROCM_CLI_CONFIG_DIR` | (temp dir) | Isolated config directory per scenario |
| `ROCM_CLI_DATA_DIR` | (temp dir) | Isolated data directory per scenario |
| `ROCM_CLI_CACHE_DIR` | (temp dir) | Isolated cache directory per scenario |
+| `E2E_PROBE_TIMEOUT_SECS` | 120 | How long each capability-probe command (`rocm examine`, `rocm engines list`) may run before it is killed and the run is refused: whole seconds ≥ 1; anything else uses 120. The post-run version probe (`collect_versions`) uses the same deadline but fails quietly, only blanking the version columns, and does not refuse the run |
+| `E2E_EXPECT_GPU` | unset (undeclared) | Whether the lane has a GPU: `1` = GPU expected, `0` = deliberately none, unset or empty = no claim; any other value fails the run. Set on every self-hosted CI lane; see "Lane verdict" below |
+
+### Lane verdict
+
+Scenarios that resolve to skip never reach `report.json`, so scenario results
+alone cannot tell a lane that tested its GPU from one on which every GPU
+scenario was skipped. The harness checks the run itself, before and after the
+scenarios:
+
+- **Before any scenario runs**, it fails when the capability probe could not
+ read the binary under test, naming the command and saying why:
+ `rocm examine` or `rocm engines list` could not be started (with the error),
+ or could not be waited on, exited non-zero (with the exit
+ status and the tail of its stderr), printed nothing, was still running after
+ `E2E_PROBE_TIMEOUT_SECS` (default 120; it is killed, and the message gives
+ the deadline and the stderr tail so far), or exited but left its output
+ open, held by a process it started (with the stderr tail so far). On a lane
+ that sets `E2E_EXPECT_GPU=1` it also fails when the probe found no usable AMD
+ GPU, naming the detected gfx target, WSL flag, driver status and visibility
+ mask it decided from. A run refused here writes no `platform.json`; its
+ `report.json` is already there, empty, because `run_suite` in
+ [`tests/e2e.rs`](tests/e2e.rs) creates it before calling
+ `lane_verdict::preflight`, and on every self-hosted lane that ran its suite
+ the results upload step is gated on `always()` (`e2e-gpu`'s also skips a
+ run that found itself stale, which never reaches the suite), so the
+ consolidated report shows the lane as EMPTY.
+- **After the run**, it fails when no scenario executed (for example a `-n`
+ filter that matches nothing), and on an `E2E_EXPECT_GPU=1` lane when no
+ `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU scenarios
+ the run's own configuration leaves out (`@nightly`, `@merge-queue`, or
+ anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved
+ but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and
+ no GPU scenario selected fails by design. The reason is also recorded as
+ `lane_failure` in `platform.json`, so the consolidated report shows the lane
+ as FAIL, even when every scenario it resolved was a skip, and gives the reason
+ under "Needs attention: platforms" (in both the step summary and the HTML
+ report) and in the lane's section of the HTML report.
+
+`E2E_EXPECT_GPU=0` is for a lane that deliberately has no usable GPU, such as
+the WSL lanes: the harness prints a note that `@requires-gpu` scenarios are not
+applicable, and a note that the declaration is stale if the probe finds a GPU
+anyway; neither fails the run. Both checks are in
+[`src/lane_verdict.rs`](src/lane_verdict.rs).
+
+A cucumber `-n`/`--name` or `--tags` filter on the command line replaces the
+harness's own scenario filter, so the scenarios it selects run without the
+capability gating, the opt-in sets and the expectation matrix described below,
+and the after-run GPU check does not apply. Use it to run a known scenario on a
+host that can satisfy it; use `E2E_ONLY_LIFECYCLE` with `E2E_ALSO_RUN_IDS` to
+select scenarios with the gating kept.
## Tags and per-scenario expectations
@@ -137,7 +188,7 @@ Scenarios carry stable-id and capability tags:
| Tag | Meaning |
|---|---|
| `@id:-` | Stable scenario id, prefixed with its feature's key. Keys the expectation matrix and the report grid; every scenario has one. |
-| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. |
+| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. On a lane that sets `E2E_EXPECT_GPU=1` a host with none fails the run instead; see "Lane verdict" above. |
| `@requires-multi-gpu` | Premise is a host with **more than one** AMD GPU present. Resolves to **skip** wherever the probed device count is not known to exceed one: a single-GPU host such as Strix Halo, and any host whose count could not be read (non-Linux, or WSL, which exposes neither a KFD topology nor an amdgpu DRM card). `@requires-gpu` cannot express this — it asks only whether *a* device is usable — so scenarios carry both. The count comes from the capability probe's `amd_gpu_count`, which mirrors the product's KFD+DRM rule rather than counting `amd-smi list` — the gate must ask the same "how many devices are present" question `--gpu` validation is built on, and `amd-smi` is only a best-effort fallback that may be absent or disagree. |
| `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. |
| `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. |
diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs
index d8857f43c..1b0cf7f33 100644
--- a/tests/e2e-cucumber/src/capability.rs
+++ b/tests/e2e-cucumber/src/capability.rs
@@ -16,7 +16,7 @@
//! changes engine support, so the unit tests below guard it.
//!
//! The AMD GPU *count* is re-implemented for the same reason (see
-//! [`probe_amd_gpu_count`]): no `rocm` command reports one, so a scenario whose
+//! `probe_amd_gpu_count`): no `rocm` command reports one, so a scenario whose
//! premise is a multi-GPU host has nothing else to gate on.
//!
//! `examine`'s `default_engine` now reports the host's real engine rather than the
@@ -28,10 +28,12 @@
use std::sync::OnceLock;
-/// Name of the `rocm` binary under test (mirrors the test binary's
-/// `rocm_binary()`): `ROCM_CLI_BINARY` or plain `rocm`.
-fn rocm_binary() -> String {
- std::env::var("ROCM_CLI_BINARY").unwrap_or_else(|_| "rocm".to_string())
+/// Name of the `rocm` binary under test: `ROCM_CLI_BINARY` or plain `rocm`,
+/// read through `env` so a test can supply it without a process-wide variable.
+/// Like the test binary's `rocm_binary()`, except that a value that is not
+/// UTF-8 is used as given here rather than replaced by `rocm`.
+fn rocm_binary(env: &impl Fn(&str) -> Option) -> std::ffi::OsString {
+ env("ROCM_CLI_BINARY").unwrap_or_else(|| "rocm".into())
}
/// Whether a container runtime is usable here.
@@ -132,11 +134,11 @@ pub struct HostCapability {
pub gfx_target: Option,
/// Whether an AMD GPU is present AND usable here, so `@requires-gpu`
/// scenarios can run. A real `detected_gfx_target`, plus a ready ROCm
- /// driver path on WSL — see [`host_has_usable_gpu`].
+ /// driver path on WSL — see `host_has_usable_gpu`.
pub has_amd_gpu: bool,
/// How many AMD GPUs are PRESENT on this host, before any visibility mask.
/// `None` where the count could not be determined (non-Linux, unreadable
- /// sysfs). Gates `@requires-multi-gpu`; see [`probe_amd_gpu_count`].
+ /// sysfs). Gates `@requires-multi-gpu`; see `probe_amd_gpu_count`.
pub amd_gpu_count: Option,
/// Engine adapters the binary reports as present. Both builtins are always
/// "built-in", so this is NOT the same as "can start here" — use
@@ -154,16 +156,27 @@ pub struct HostCapability {
/// exposes it), so `xtask e2e` reports it via the [`OOM_FAULT_INJECTION_ENV`]
/// environment variable — set only when xtask built the binary itself with
/// the feature, and absent for a prebuilt `ROCM_CLI_BINARY` (the self-hosted
- /// lanes' shipping release build). See [`probe_host_capability`].
+ /// lanes' shipping release build). See `probe_host_capability`.
pub oom_fault_injection: bool,
/// Whether two file names differing only in letter case are two files under
/// the temp root the scenarios' isolated roots are created in. Gates
- /// `@requires-case-sensitive-fs`; see [`probe_case_sensitive_fs`].
+ /// `@requires-case-sensitive-fs`; see `probe_case_sensitive_fs`.
pub case_sensitive_fs: bool,
/// Whether a directory made read-only refuses new files to the user
/// running the suite. Gates `@requires-dir-permissions`; see
- /// [`probe_dir_permissions_bind`].
+ /// `probe_dir_permissions_bind`.
pub dir_permissions_bind: bool,
+ /// `examine`'s `driver_status:` line, empty when it printed none. Only
+ /// decides GPU usability on WSL (see `host_has_usable_gpu`); kept so a
+ /// lane whose GPU went missing can say why.
+ pub driver_status: String,
+ /// Why the probe could not read the binary under test, if it could not:
+ /// the first failure of `rocm examine` or `rocm engines list` (see
+ /// [`ProbeError`]). When set, the rest of this struct holds the same
+ /// no-GPU values an empty `examine` would give, so callers never panic, and
+ /// the harness refuses to run (see `lane_verdict::preflight_verdict`).
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub probe_error: Option,
}
/// Environment variable `xtask e2e` sets to `1` when it compiled the binary
@@ -258,7 +271,11 @@ pub fn collect_versions(runtimes_dir: Option<&std::path::Path>) -> PlatformVersi
// OS distro from `examine` — ALWAYS collected (every platform, incl. mock and
// hosts without a shared runtime dir). Isolated throwaway root reads the host.
if let Ok(tmp) = tempfile::TempDir::with_prefix("rocm-e2e-ver-") {
- let examine = run_probe(tmp.path(), &["examine"]);
+ // Best-effort, unlike the capability probe: a failed `examine` here
+ // only leaves the report heading's OS blank. The capability probe ran
+ // the same command before any scenario and already refused the run if
+ // it failed.
+ let examine = run_probe(tmp.path(), &["examine"]).unwrap_or_default();
v.os = examine
.lines()
.find_map(|l| l.trim().strip_prefix("distro:"))
@@ -486,6 +503,29 @@ fn probe_host_capability() -> HostCapability {
// what the scenarios see.
let examine = run_probe(root, &["examine"]);
let engines = run_probe(root, &["engines", "list"]);
+ capability_from_probes(examine, engines, root)
+}
+
+/// Build a [`HostCapability`] from the two probes' results; `root` is the
+/// throwaway directory the filesystem probes write under.
+///
+/// Split from [`probe_host_capability`] so a test can hand it a failed probe
+/// without a binary to run.
+fn capability_from_probes(
+ examine: Result,
+ engines: Result,
+ root: &std::path::Path,
+) -> HostCapability {
+ // The first failure is the one reported; either one means the facts below
+ // were not read from the binary, so they fall back to the no-GPU values an
+ // empty dump parses to.
+ let probe_error = examine
+ .as_ref()
+ .err()
+ .or_else(|| engines.as_ref().err())
+ .map(ToString::to_string);
+ let examine = examine.unwrap_or_default();
+ let engines = engines.unwrap_or_default();
let ExamineFacts {
os_family,
@@ -516,6 +556,8 @@ fn probe_host_capability() -> HostCapability {
oom_fault_injection,
case_sensitive_fs,
dir_permissions_bind,
+ driver_status,
+ probe_error,
}
}
@@ -581,20 +623,455 @@ fn probe_case_sensitive_fs(dir: &std::path::Path) -> bool {
probe().unwrap_or(false)
}
-/// Run `rocm ` with an isolated config/data/cache root, returning stdout
-/// (empty string on any failure — the probe must never panic the suite).
-fn run_probe(root: &std::path::Path, args: &[&str]) -> String {
- let mut cmd = std::process::Command::new(rocm_binary());
+/// Why a probe of the binary under test produced nothing the harness can read.
+///
+/// Every variant is fatal to the run unless the caller ignores it, as
+/// `collect_versions` does: an unreadable probe parses as a host with no GPU,
+/// so every capability-gated scenario would resolve to skip and the run would
+/// pass without testing them. `command` is the binary and arguments as
+/// run, so the message names exactly what failed.
+#[derive(Debug)]
+pub enum ProbeError {
+ /// The binary could not be started at all (wrong path, not executable).
+ Spawn {
+ command: String,
+ source: std::io::Error,
+ },
+ /// It ran and exited non-zero: the CLI exits 0 for every finding, so this
+ /// means it could not examine the host at all.
+ Exit {
+ command: String,
+ status: std::process::ExitStatus,
+ stderr_tail: String,
+ },
+ /// It exited 0 but printed nothing on stdout.
+ EmptyOutput { command: String },
+ /// It was still running when the deadline ([`PROBE_TIMEOUT_ENV`]) passed,
+ /// so it was killed and reaped. `stderr_tail` is what it had written to
+ /// stderr by then.
+ Timeout {
+ command: String,
+ after: std::time::Duration,
+ stderr_tail: String,
+ },
+ /// It started, but waiting for it to exit failed.
+ Wait {
+ command: String,
+ source: std::io::Error,
+ },
+ /// It exited 0, but its stdout was still open `grace` later: a process it
+ /// started is holding the pipe, so what it printed cannot be read in full.
+ OutputNotClosed {
+ command: String,
+ grace: std::time::Duration,
+ stderr_tail: String,
+ },
+}
+
+impl std::fmt::Display for ProbeError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::Spawn { command, source } => write!(
+ f,
+ "could not start `{command}`: {source}. Set ROCM_CLI_BINARY to the rocm binary \
+ under test, or run the suite through `cargo xtask e2e`, which builds it and \
+ sets ROCM_CLI_BINARY"
+ ),
+ Self::Exit {
+ command,
+ status,
+ stderr_tail,
+ } => write!(
+ f,
+ "`{command}` exited with {status}\n{}",
+ crate::section("stderr (last lines)", stderr_tail)
+ ),
+ Self::Timeout {
+ command,
+ after,
+ stderr_tail,
+ } => write!(
+ f,
+ "`{command}` was still running after {after:?}, so it was killed; set \
+ {PROBE_TIMEOUT_ENV} to allow it longer\n{}",
+ crate::section("stderr (last lines)", stderr_tail)
+ ),
+ Self::Wait { command, source } => write!(
+ f,
+ "`{command}` was started, but waiting for it to exit failed: {source}"
+ ),
+ Self::OutputNotClosed {
+ command,
+ grace,
+ stderr_tail,
+ } => write!(
+ f,
+ "`{command}` exited 0, but its stdout was still open {grace:?} later (a \
+ process it started may be holding it), so what it printed cannot be read \
+ in full\n{}",
+ crate::section("stderr (last lines)", stderr_tail)
+ ),
+ Self::EmptyOutput { command } => {
+ write!(f, "`{command}` exited 0 but produced no output on stdout")
+ }
+ }
+ }
+}
+
+impl std::error::Error for ProbeError {
+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+ match self {
+ Self::Spawn { source, .. } | Self::Wait { source, .. } => Some(source),
+ Self::Exit { .. }
+ | Self::EmptyOutput { .. }
+ | Self::Timeout { .. }
+ | Self::OutputNotClosed { .. } => None,
+ }
+ }
+}
+
+/// How many trailing stderr lines a [`ProbeError`] keeps.
+const PROBE_STDERR_TAIL_LINES: usize = 20;
+
+/// The last [`PROBE_STDERR_TAIL_LINES`] lines of a probe's stderr.
+fn stderr_tail(stderr: &[u8]) -> String {
+ let stderr = String::from_utf8_lossy(stderr);
+ let lines: Vec<&str> = stderr.lines().collect();
+ lines[lines.len().saturating_sub(PROBE_STDERR_TAIL_LINES)..].join("\n")
+}
+
+/// Turn one probe's spawn result into its stdout, or the reason it has none.
+///
+/// Pure, so each outcome is unit-tested without a binary to run. Stdout that is
+/// only whitespace counts as empty: no `examine` field can be parsed from it.
+pub fn classify_probe_output(
+ command: &str,
+ result: std::io::Result,
+) -> Result {
+ let out = result.map_err(|source| ProbeError::Spawn {
+ command: command.to_owned(),
+ source,
+ })?;
+ if !out.status.success() {
+ return Err(ProbeError::Exit {
+ command: command.to_owned(),
+ status: out.status,
+ stderr_tail: stderr_tail(&out.stderr),
+ });
+ }
+ let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
+ if stdout.trim().is_empty() {
+ return Err(ProbeError::EmptyOutput {
+ command: command.to_owned(),
+ });
+ }
+ Ok(stdout)
+}
+
+/// How a bounded probe run ended (see `output_within`).
+#[derive(Debug)]
+pub enum Finished {
+ /// The child exited. `stdout_drained` is false when its stdout was still
+ /// open after the drain grace; `output` then holds what had been read by
+ /// then. Stderr is read the same way, but one left open does not fail the
+ /// probe: it carries only diagnostics.
+ Exited {
+ output: std::process::Output,
+ stdout_drained: bool,
+ },
+ /// The child was still running at the deadline and has been killed and
+ /// reaped; `stderr` is what it had written by then.
+ TimedOut { stderr: Vec },
+ /// Waiting for the child failed; it has been killed.
+ WaitFailed(std::io::Error),
+}
+
+/// [`classify_probe_output`] for a bounded run: a timeout and an output pipe
+/// left open are errors of their own, never read as "no output".
+pub fn classify_finished(
+ command: &str,
+ result: std::io::Result,
+ timeout: std::time::Duration,
+ grace: std::time::Duration,
+) -> Result {
+ match result {
+ Err(source) => classify_probe_output(command, Err(source)),
+ Ok(Finished::TimedOut { stderr }) => Err(ProbeError::Timeout {
+ command: command.to_owned(),
+ after: timeout,
+ stderr_tail: stderr_tail(&stderr),
+ }),
+ Ok(Finished::WaitFailed(source)) => Err(ProbeError::Wait {
+ command: command.to_owned(),
+ source,
+ }),
+ Ok(Finished::Exited {
+ output,
+ stdout_drained: false,
+ }) if output.status.success() => Err(ProbeError::OutputNotClosed {
+ command: command.to_owned(),
+ grace,
+ stderr_tail: stderr_tail(&output.stderr),
+ }),
+ Ok(Finished::Exited { output, .. }) => classify_probe_output(command, Ok(output)),
+ }
+}
+
+/// Run `rocm ` with an isolated config/data/cache root and return its
+/// stdout, or why there is none. Never panics: the caller decides whether a
+/// failure is fatal.
+fn run_probe(root: &std::path::Path, args: &[&str]) -> Result {
+ run_probe_using(root, args, |name| std::env::var_os(name))
+}
+
+/// `run_probe` with the environment lookup passed in.
+///
+/// The binary comes from `ROCM_CLI_BINARY` and the deadline from
+/// [`PROBE_TIMEOUT_ENV`], both read through `env`, so a test can drive both
+/// without a process-wide variable.
+pub fn run_probe_using(
+ root: &std::path::Path,
+ args: &[&str],
+ env: impl Fn(&str) -> Option,
+) -> Result {
+ run_probe_with(
+ &rocm_binary(&env),
+ root,
+ args,
+ probe_timeout_from_env(env(PROBE_TIMEOUT_ENV).as_deref()),
+ )
+}
+
+/// Environment variable that overrides how long one probe command may run,
+/// in whole seconds.
+pub const PROBE_TIMEOUT_ENV: &str = "E2E_PROBE_TIMEOUT_SECS";
+
+/// How long one probe command may run when [`PROBE_TIMEOUT_ENV`] is unset.
+///
+/// `rocm examine` normally answers in seconds; this is generous so a loaded
+/// runner does not fail a good probe, while a wedged one still fails the lane
+/// with a named reason long before the job's own timeout.
+pub const DEFAULT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(2);
+
+/// The probe deadline from [`PROBE_TIMEOUT_ENV`]'s value.
+///
+/// Accepts any whole number of seconds of at least 1, with surrounding
+/// whitespace ignored; a value too large to add to the current time is waited
+/// on without a deadline (see `output_within`). Anything else (unset, empty,
+/// zero, negative, a fraction, a non-number, a number past `u64`, or a value
+/// that is not UTF-8) gives [`DEFAULT_PROBE_TIMEOUT`] rather than an error.
+#[must_use]
+pub fn probe_timeout_from_env(value: Option<&std::ffi::OsStr>) -> std::time::Duration {
+ value
+ .and_then(std::ffi::OsStr::to_str)
+ .and_then(|v| v.trim().parse::().ok())
+ .filter(|secs| *secs >= 1)
+ .map_or(DEFAULT_PROBE_TIMEOUT, std::time::Duration::from_secs)
+}
+
+/// `run_probe` with the binary and deadline passed in, so a test can name
+/// them without setting process-wide environment variables.
+pub fn run_probe_with(
+ binary: &std::ffi::OsStr,
+ root: &std::path::Path,
+ args: &[&str],
+ timeout: std::time::Duration,
+) -> Result {
+ run_probe_bounded(binary, root, args, timeout, PROBE_DRAIN_GRACE)
+}
+
+/// [`run_probe_with`] with the drain grace passed in too, for tests.
+fn run_probe_bounded(
+ binary: &std::ffi::OsStr,
+ root: &std::path::Path,
+ args: &[&str],
+ timeout: std::time::Duration,
+ grace: std::time::Duration,
+) -> Result {
+ let mut cmd = std::process::Command::new(binary);
cmd.args(args);
cmd.env("ROCM_CLI_CONFIG_DIR", root.join("config"));
cmd.env("ROCM_CLI_DATA_DIR", root.join("data"));
cmd.env("ROCM_CLI_CACHE_DIR", root.join("cache"));
- match cmd.output() {
- Ok(out) => String::from_utf8_lossy(&out.stdout).into_owned(),
- Err(_) => String::new(),
+ let command = std::iter::once(binary.to_string_lossy().into_owned())
+ .chain(args.iter().map(|a| (*a).to_owned()))
+ .collect::>()
+ .join(" ");
+ classify_finished(&command, output_within(cmd, timeout, grace), timeout, grace)
+}
+
+/// How often [`output_within`] checks whether the child has exited.
+const PROBE_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50);
+
+/// How long [`output_within`] waits, after the child exited or was killed, for
+/// its output pipes to close. Only a process it started that inherited a pipe
+/// and outlives it can hold one open past this.
+const PROBE_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(5);
+
+/// A pipe being read on its own thread into a buffer the caller can read at any
+/// time, so bytes read before a timeout or a stuck drain are not lost.
+struct PipeReader {
+ bytes: std::sync::Arc>>,
+ done: std::sync::mpsc::Receiver<()>,
+}
+
+impl PipeReader {
+ fn spawn(pipe: Option) -> Self {
+ let bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
+ let (sender, done) = std::sync::mpsc::channel();
+ let sink = std::sync::Arc::clone(&bytes);
+ std::thread::spawn(move || {
+ if let Some(mut pipe) = pipe {
+ let mut chunk = [0u8; 8192];
+ loop {
+ match pipe.read(&mut chunk) {
+ Ok(0) | Err(_) => break,
+ Ok(n) => sink
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .extend_from_slice(&chunk[..n]),
+ }
+ }
+ }
+ let _ = sender.send(());
+ });
+ Self { bytes, done }
+ }
+
+ /// Wait until the pipe closes or `deadline` passes; whether it closed.
+ fn wait_closed(&self, deadline: std::time::Instant) -> bool {
+ let left = deadline.saturating_duration_since(std::time::Instant::now());
+ self.done.recv_timeout(left).is_ok()
+ }
+
+ /// What has been read so far.
+ fn snapshot(&self) -> Vec {
+ self.bytes
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .clone()
}
}
+/// `command.output()`, bounded by `timeout`: a child still running then is
+/// killed and reaped. Either way the output pipes get `grace` to close, and
+/// whatever was read by then is kept.
+///
+/// std-only, polling `try_wait`, so it behaves the same on Linux and Windows,
+/// with each output pipe read on its own thread so a chatty child cannot block
+/// on a full pipe. It repeats the spawn, poll, kill and reap steps of
+/// `device_memory::bounded_stdout`; a change to the kill or drain behaviour of
+/// one should look at the other. It differs in three ways:
+///
+/// - it keeps stderr and the exit status, which the probe reports;
+/// - it drains after a kill too, where `bounded_stdout` gives up at once,
+/// because the stderr written before the deadline is the only evidence of
+/// why `rocm examine` hung;
+/// - its grace is 5 s (`PROBE_DRAIN_GRACE`) rather than 2 s, because it runs
+/// a handful of times per suite: each probe runs once (the capability
+/// probes in `probe_host_capability`, cached by `host_capability`, and the
+/// version probe in `collect_versions`, whose `rocm examine` repeats the
+/// capability probe's), not once per sample, so a generous wait for a
+/// loaded runner to hand over the bytes costs little.
+///
+/// Only the direct child is killed: a process it started that inherited a pipe
+/// can keep that pipe open past the grace. The reader thread for that pipe is
+/// then left detached until the pipe closes, and the bytes it had read are
+/// used as they are. A leaked thread per stuck probe is accepted in a test
+/// harness that runs a handful of probes, each once, and exits.
+fn output_within(
+ mut command: std::process::Command,
+ timeout: std::time::Duration,
+ grace: std::time::Duration,
+) -> std::io::Result {
+ use std::process::Stdio;
+
+ let mut child = command
+ .stdin(Stdio::null())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()?;
+ let stdout = PipeReader::spawn(child.stdout.take());
+ let stderr = PipeReader::spawn(child.stderr.take());
+
+ // `None` only for a deadline too far off to represent: wait without one.
+ let deadline = std::time::Instant::now().checked_add(timeout);
+ let status = loop {
+ match child.try_wait() {
+ Ok(Some(status)) => break Some(status),
+ Ok(None) if deadline.is_none_or(|deadline| std::time::Instant::now() < deadline) => {
+ std::thread::sleep(PROBE_POLL_INTERVAL);
+ }
+ Ok(None) => {
+ let _ = child.kill();
+ let _ = child.wait();
+ break None;
+ }
+ Err(error) => {
+ let _ = child.kill();
+ let _ = child.wait();
+ return Ok(Finished::WaitFailed(error));
+ }
+ }
+ };
+ let drained_by = std::time::Instant::now() + grace;
+ let stdout_closed = stdout.wait_closed(drained_by);
+ // Waited for so the tail is complete when it can be, but not required.
+ let _ = stderr.wait_closed(drained_by);
+ Ok(match status {
+ None => Finished::TimedOut {
+ stderr: stderr.snapshot(),
+ },
+ Some(status) => Finished::Exited {
+ output: std::process::Output {
+ status,
+ stdout: stdout.snapshot(),
+ stderr: stderr.snapshot(),
+ },
+ stdout_drained: stdout_closed,
+ },
+ })
+}
+
+/// One line naming the probe facts that decide GPU usability, for a lane whose
+/// declared GPU the probe did not find.
+///
+/// Reads the visibility masks from the environment the way
+/// `host_has_usable_gpu` does; [`probe_facts_summary_with_mask`] is the pure
+/// form.
+#[must_use]
+pub fn probe_facts_summary(cap: &HostCapability) -> String {
+ let hip = std::env::var_os("HIP_VISIBLE_DEVICES");
+ let rocr = std::env::var_os("ROCR_VISIBLE_DEVICES");
+ probe_facts_summary_with_mask(cap, hip.as_deref(), rocr.as_deref())
+}
+
+/// Pure form of [`probe_facts_summary`]. Names the mask that is in force by the
+/// same precedence as `selected_visibility_mask`: HIP wins when set, even
+/// empty.
+#[must_use]
+pub fn probe_facts_summary_with_mask(
+ cap: &HostCapability,
+ hip: Option<&std::ffi::OsStr>,
+ rocr: Option<&std::ffi::OsStr>,
+) -> String {
+ let mask = match (hip, rocr) {
+ (Some(v), _) => format!("HIP_VISIBLE_DEVICES={:?}", v.to_string_lossy()),
+ (None, Some(v)) => format!("ROCR_VISIBLE_DEVICES={:?}", v.to_string_lossy()),
+ (None, None) => "no visibility mask".to_owned(),
+ };
+ let driver_status = if cap.driver_status.is_empty() {
+ ""
+ } else {
+ cap.driver_status.as_str()
+ };
+ format!(
+ "detected_gfx_target={}, wsl={}, driver_status={driver_status}, {mask}",
+ cap.gfx_target.as_deref().unwrap_or(""),
+ cap.is_wsl,
+ )
+}
+
/// The `examine` fields the capability probe reads.
struct ExamineFacts {
os_family: String,
@@ -992,6 +1469,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: false,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
};
assert!(strix.engine_available("lemonade"));
// vLLM adapter is "built-in" but cannot start on Windows / non-dcgpu.
@@ -1009,6 +1488,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: true,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
};
assert!(mi300x.engine_available("vllm"));
assert!(mi300x.engine_available("lemonade"));
@@ -1503,4 +1984,667 @@ Local model engines
std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o755)).expect("restore");
assert_eq!(probe_dir_permissions_bind(tmp.path()), refused);
}
+
+ /// An exit status that reports failure with code 1, built the way each
+ /// platform encodes it, so the non-zero-exit row runs on both CI gates.
+ fn failed_status() -> std::process::ExitStatus {
+ #[cfg(unix)]
+ {
+ use std::os::unix::process::ExitStatusExt;
+ // A wait status: the exit code sits in the second byte.
+ std::process::ExitStatus::from_raw(1 << 8)
+ }
+ #[cfg(windows)]
+ {
+ use std::os::windows::process::ExitStatusExt;
+ std::process::ExitStatus::from_raw(1)
+ }
+ }
+
+ fn succeeded_status() -> std::process::ExitStatus {
+ #[cfg(unix)]
+ {
+ use std::os::unix::process::ExitStatusExt;
+ std::process::ExitStatus::from_raw(0)
+ }
+ #[cfg(windows)]
+ {
+ use std::os::windows::process::ExitStatusExt;
+ std::process::ExitStatus::from_raw(0)
+ }
+ }
+
+ fn output(
+ status: std::process::ExitStatus,
+ stdout: &str,
+ stderr: &str,
+ ) -> std::process::Output {
+ std::process::Output {
+ status,
+ stdout: stdout.as_bytes().to_vec(),
+ stderr: stderr.as_bytes().to_vec(),
+ }
+ }
+
+ /// The regression: a probe that exits non-zero is an error even when it
+ /// printed something, rather than stdout parsed as a GPU-less host.
+ #[test]
+ fn a_probe_that_exits_non_zero_is_an_error_with_its_stderr_tail() {
+ let result = classify_probe_output(
+ "rocm examine",
+ Ok(output(
+ failed_status(),
+ "rocm examine\n os: linux\n",
+ "warning: early\nError: could not read the host\n",
+ )),
+ );
+ let Err(error @ ProbeError::Exit { .. }) = result else {
+ panic!("a non-zero exit must be ProbeError::Exit, got {result:?}");
+ };
+ assert_eq!(
+ error.to_string(),
+ format!(
+ "`rocm examine` exited with {}\n--- stderr (last lines) ---\n\
+ warning: early\nError: could not read the host",
+ failed_status()
+ )
+ );
+ }
+
+ #[test]
+ fn the_stderr_tail_keeps_only_the_last_lines() {
+ let stderr = (1..=30)
+ .map(|i| format!("line {i}"))
+ .collect::>()
+ .join("\n");
+ let Err(ProbeError::Exit { stderr_tail, .. }) =
+ classify_probe_output("rocm examine", Ok(output(failed_status(), "", &stderr)))
+ else {
+ panic!("a non-zero exit must be ProbeError::Exit");
+ };
+ let expected: Vec = (11..=30).map(|i| format!("line {i}")).collect();
+ assert_eq!(stderr_tail, expected.join("\n"));
+ }
+
+ #[test]
+ fn a_probe_that_exits_zero_with_output_returns_its_stdout() {
+ let stdout = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n";
+ assert_eq!(
+ classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, "")))
+ .expect("exit 0 with output is a reading"),
+ stdout
+ );
+ }
+
+ #[test]
+ fn a_probe_that_exits_zero_with_no_output_is_an_error() {
+ for stdout in ["", " \n\n"] {
+ let result =
+ classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, "")));
+ let Err(error @ ProbeError::EmptyOutput { .. }) = result else {
+ panic!("empty stdout {stdout:?} must be ProbeError::EmptyOutput, got {result:?}");
+ };
+ assert_eq!(
+ error.to_string(),
+ "`rocm examine` exited 0 but produced no output on stdout"
+ );
+ }
+ }
+
+ #[test]
+ fn a_probe_whose_binary_does_not_exist_is_a_spawn_error_naming_it() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let missing = tmp.path().join("no-such-rocm");
+ let result = run_probe_with(
+ missing.as_os_str(),
+ tmp.path(),
+ &["examine"],
+ DEFAULT_PROBE_TIMEOUT,
+ );
+ let Err(error @ ProbeError::Spawn { .. }) = result else {
+ panic!("a missing binary must be ProbeError::Spawn, got {result:?}");
+ };
+ let ProbeError::Spawn { command, source } = &error else {
+ unreachable!()
+ };
+ assert_eq!(command, &format!("{} examine", missing.display()));
+ assert_eq!(source.kind(), std::io::ErrorKind::NotFound);
+ assert_eq!(
+ error.to_string(),
+ format!(
+ "could not start `{} examine`: {source}. Set ROCM_CLI_BINARY to the rocm binary \
+ under test, or run the suite through `cargo xtask e2e`, which builds it and \
+ sets ROCM_CLI_BINARY",
+ missing.display()
+ )
+ );
+ }
+
+ const GPU_EXAMINE: &str = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n wsl: false\n driver_status: ok\n";
+
+ /// A failed probe records why, and leaves the struct at the no-GPU values
+ /// so no caller panics.
+ #[test]
+ fn a_failed_examine_records_the_error_and_falls_back_to_no_gpu() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let cap = capability_from_probes(
+ Err(ProbeError::EmptyOutput {
+ command: "rocm examine".to_owned(),
+ }),
+ Ok("lemonade\n".to_owned()),
+ tmp.path(),
+ );
+ assert_eq!(
+ cap.probe_error.as_deref(),
+ Some("`rocm examine` exited 0 but produced no output on stdout")
+ );
+ assert!(!cap.has_amd_gpu);
+ assert_eq!(cap.gfx_target, None);
+ assert_eq!(cap.platform_slug, "mock");
+ }
+
+ /// The `engines list fails, examine OK` row: the GPU facts were read, but
+ /// the probe still failed, so the run must not proceed on them.
+ #[test]
+ fn a_failed_engines_list_is_recorded_even_when_examine_succeeded() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let cap = capability_from_probes(
+ Ok(GPU_EXAMINE.to_owned()),
+ Err(ProbeError::Exit {
+ command: "rocm engines list".to_owned(),
+ status: failed_status(),
+ stderr_tail: "boom".to_owned(),
+ }),
+ tmp.path(),
+ );
+ assert_eq!(
+ cap.probe_error,
+ Some(format!(
+ "`rocm engines list` exited with {}\n--- stderr (last lines) ---\nboom",
+ failed_status()
+ ))
+ );
+ assert!(cap.available_engines.is_empty());
+ }
+
+ #[test]
+ fn the_first_failure_is_the_one_recorded() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let cap = capability_from_probes(
+ Err(ProbeError::EmptyOutput {
+ command: "rocm examine".to_owned(),
+ }),
+ Err(ProbeError::EmptyOutput {
+ command: "rocm engines list".to_owned(),
+ }),
+ tmp.path(),
+ );
+ assert_eq!(
+ cap.probe_error.as_deref(),
+ Some("`rocm examine` exited 0 but produced no output on stdout")
+ );
+ }
+
+ #[test]
+ fn successful_probes_record_no_error_and_keep_the_driver_status() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let cap = capability_from_probes(
+ Ok(GPU_EXAMINE.to_owned()),
+ Ok("lemonade\nvllm\n".to_owned()),
+ tmp.path(),
+ );
+ assert_eq!(cap.probe_error, None);
+ assert_eq!(cap.gfx_target.as_deref(), Some("gfx1151"));
+ assert_eq!(cap.driver_status, "ok");
+ // `probe_error` is left out of platform.json when there is none.
+ let json = serde_json::to_value(&cap).expect("serializes");
+ assert!(json.get("probe_error").is_none(), "{json}");
+ }
+
+ #[test]
+ fn the_probe_facts_summary_names_the_mask_in_force() {
+ let cap = HostCapability {
+ os_family: "linux".to_owned(),
+ is_wsl: true,
+ gfx_target: Some("gfx1151".to_owned()),
+ has_amd_gpu: false,
+ amd_gpu_count: None,
+ available_engines: vec![],
+ effective_serve_engine: "lemonade".to_owned(),
+ platform_slug: "mock".to_owned(),
+ oom_fault_injection: false,
+ case_sensitive_fs: true,
+ dir_permissions_bind: true,
+ driver_status: "wsl_rocdxg_missing".to_owned(),
+ probe_error: None,
+ };
+ let base = "detected_gfx_target=gfx1151, wsl=true, driver_status=wsl_rocdxg_missing";
+ assert_eq!(
+ probe_facts_summary_with_mask(&cap, None, None),
+ format!("{base}, no visibility mask")
+ );
+ assert_eq!(
+ probe_facts_summary_with_mask(
+ &cap,
+ Some(std::ffi::OsStr::new("")),
+ Some(std::ffi::OsStr::new("0"))
+ ),
+ format!("{base}, HIP_VISIBLE_DEVICES=\"\"")
+ );
+ assert_eq!(
+ probe_facts_summary_with_mask(&cap, None, Some(std::ffi::OsStr::new("1"))),
+ format!("{base}, ROCR_VISIBLE_DEVICES=\"1\"")
+ );
+ let none = HostCapability {
+ gfx_target: None,
+ is_wsl: false,
+ driver_status: String::new(),
+ ..cap
+ };
+ assert_eq!(
+ probe_facts_summary_with_mask(&none, None, None),
+ "detected_gfx_target=, wsl=false, driver_status=, no visibility mask"
+ );
+ }
+
+ /// A shell command line, run as `(binary, args)` on each CI gate.
+ fn shell_line(line: &str) -> (std::ffi::OsString, Vec) {
+ #[cfg(unix)]
+ {
+ ("sh".into(), vec!["-c".to_owned(), line.to_owned()])
+ }
+ #[cfg(windows)]
+ {
+ ("cmd".into(), vec!["/C".to_owned(), line.to_owned()])
+ }
+ }
+
+ fn arg_refs(args: &[String]) -> Vec<&str> {
+ args.iter().map(String::as_str).collect()
+ }
+
+ /// Prints `done` after about two seconds.
+ fn two_second_command() -> (std::ffi::OsString, Vec) {
+ #[cfg(unix)]
+ {
+ shell_line("sleep 2; echo done")
+ }
+ #[cfg(windows)]
+ {
+ shell_line("ping -n 3 127.0.0.1 >nul & echo done")
+ }
+ }
+
+ /// The env lookup `run_probe` would get, with only the given variables set.
+ fn env_of(
+ vars: Vec<(&'static str, std::ffi::OsString)>,
+ ) -> impl Fn(&str) -> Option {
+ move |name| {
+ vars.iter()
+ .find(|(key, _)| *key == name)
+ .map(|(_, value)| value.clone())
+ }
+ }
+
+ /// `E2E_PROBE_TIMEOUT_SECS` reaches the probe through the same lookup
+ /// `run_probe` uses (no process-wide variable), and the Timeout message's
+ /// advice holds: the same command that timed out at 1 second completes when
+ /// the variable allows it longer.
+ #[test]
+ fn the_probe_timeout_variable_sets_the_deadline_and_raising_it_clears_the_timeout() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let (binary, args) = two_second_command();
+ let command = format!("{} {}", binary.to_string_lossy(), args.join(" "));
+
+ let short = run_probe_using(
+ tmp.path(),
+ &arg_refs(&args),
+ env_of(vec![
+ ("ROCM_CLI_BINARY", binary.clone()),
+ (PROBE_TIMEOUT_ENV, "1".into()),
+ ]),
+ );
+ let Err(error @ ProbeError::Timeout { .. }) = short else {
+ panic!("a 1 s deadline on a 2 s command must time out, got {short:?}");
+ };
+ let ProbeError::Timeout { after, .. } = &error else {
+ unreachable!()
+ };
+ assert_eq!(*after, std::time::Duration::from_secs(1));
+ assert_eq!(
+ error.to_string(),
+ format!(
+ "`{command}` was still running after 1s, so it was killed; set \
+ E2E_PROBE_TIMEOUT_SECS to allow it longer\n--- stderr (last lines): (empty) ---"
+ )
+ );
+
+ // Following that advice clears the condition.
+ let longer = run_probe_using(
+ tmp.path(),
+ &arg_refs(&args),
+ env_of(vec![
+ ("ROCM_CLI_BINARY", binary),
+ (PROBE_TIMEOUT_ENV, "5".into()),
+ ]),
+ )
+ .expect("with 5 s allowed, the 2 s command completes");
+ assert_eq!(longer.trim(), "done");
+ }
+
+ /// The regression: a probe that never finishes must become an error at its
+ /// deadline, carrying what it had written to stderr, and be killed AND
+ /// reaped. A zombie still answers `kill -0`, so a failing `kill -0` shows
+ /// the child was reaped, not merely signalled.
+ #[cfg(unix)]
+ #[test]
+ fn a_probe_still_running_at_its_deadline_is_killed_reaped_and_keeps_its_stderr() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let pid_file = tmp.path().join("pid");
+ let script = format!(
+ "echo $$ > '{}'; echo early-err >&2; exec sleep 30",
+ pid_file.display()
+ );
+ let started = std::time::Instant::now();
+ let result = run_probe_bounded(
+ std::ffi::OsStr::new("sh"),
+ tmp.path(),
+ &["-c", &script],
+ std::time::Duration::from_secs(3),
+ std::time::Duration::from_secs(2),
+ );
+ assert!(
+ started.elapsed() < std::time::Duration::from_secs(30),
+ "returned after {:?}: it waited for the child instead of killing it",
+ started.elapsed()
+ );
+ let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else {
+ panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}");
+ };
+ assert_eq!(stderr_tail, "early-err");
+ // The shell creates the file and then writes the pid; give a loaded
+ // runner a moment to finish the write rather than read it half-done.
+ let mut pid = String::new();
+ for _ in 0..100 {
+ pid = std::fs::read_to_string(&pid_file).unwrap_or_default();
+ if !pid.trim().is_empty() {
+ break;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(50));
+ }
+ assert!(
+ !pid.trim().is_empty(),
+ "the child wrote its pid before the deadline"
+ );
+ let still_there = std::process::Command::new("kill")
+ .args(["-0", pid.trim()])
+ .stderr(std::process::Stdio::null())
+ .status()
+ .expect("kill runs")
+ .success();
+ assert!(
+ !still_there,
+ "child {} was left running or unreaped",
+ pid.trim()
+ );
+ }
+
+ /// Windows half of the test above: the deadline and the stderr kept from
+ /// before it. `cmd` is killed but the `ping` it started holds the pipe, so
+ /// the bytes reach the caller only through the shared buffer. The reaping
+ /// check needs `kill -0` and runs on Unix only.
+ #[cfg(windows)]
+ #[test]
+ fn a_probe_still_running_at_its_deadline_is_reported_with_its_stderr() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let (binary, args) = shell_line("echo early-err 1>&2 & ping -n 10 127.0.0.1 >nul");
+ let started = std::time::Instant::now();
+ let result = run_probe_bounded(
+ &binary,
+ tmp.path(),
+ &arg_refs(&args),
+ std::time::Duration::from_secs(1),
+ std::time::Duration::from_secs(1),
+ );
+ assert!(started.elapsed() < std::time::Duration::from_secs(10));
+ let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else {
+ panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}");
+ };
+ assert_eq!(stderr_tail.trim(), "early-err");
+ }
+
+ /// Stderr written after the kill still reaches the caller: the drain runs
+ /// after a timeout too. A grandchild holding only stderr (its stdout goes
+ /// to `/dev/null`) writes `late` half a second after the 1 s deadline
+ /// killed its parent. Stdout is closed by then, so only the wait on stderr
+ /// can collect it; the 10 s grace leaves time for that.
+ #[cfg(unix)]
+ #[test]
+ fn stderr_written_after_the_kill_is_drained_into_the_timeout() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let result = run_probe_bounded(
+ std::ffi::OsStr::new("sh"),
+ tmp.path(),
+ &[
+ "-c",
+ "(sleep 1.5; echo late >&2) >/dev/null & exec sleep 30",
+ ],
+ std::time::Duration::from_secs(1),
+ std::time::Duration::from_secs(10),
+ );
+ let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else {
+ panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}");
+ };
+ assert_eq!(stderr_tail, "late");
+ }
+
+ /// A child that reads stdin gets EOF at once rather than blocking until the
+ /// deadline: `output_within` replaces whatever stdin the command carried
+ /// with an empty one. The command is handed a pipe whose write end this
+ /// test holds open, so only that replacement lets the read finish (the
+ /// same arrangement as `a_child_that_reads_stdin_is_given_an_empty_one` in
+ /// rocm-core's `process/bounded.rs`).
+ ///
+ /// Like that sibling, it catches a missing stdin setting but does not pin
+ /// `null` against `inherit`: an `inherit` mutant passes whenever the test
+ /// process's own stdin is already at EOF.
+ #[test]
+ fn a_probe_that_reads_stdin_is_given_an_empty_one() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ #[cfg(unix)]
+ let mut command = {
+ let mut command = std::process::Command::new("sh");
+ command.args(["-c", "read line || echo stdin-eof"]);
+ command
+ };
+ #[cfg(windows)]
+ let mut command = {
+ // `set /p` leaves the variable as it was when its read hits EOF.
+ let script = tmp.path().join("stdin.bat");
+ std::fs::write(
+ &script,
+ "@echo off\r\nset \"LINE=stdin-eof\"\r\nset /p LINE=\r\necho %LINE%\r\n",
+ )
+ .expect("write script");
+ let mut command = std::process::Command::new("cmd");
+ command.arg("/C").arg(&script);
+ command
+ };
+ let (stdin_reader, stdin_writer) = std::io::pipe().expect("stdin pipe");
+ command.stdin(stdin_reader);
+
+ let started = std::time::Instant::now();
+ let finished = output_within(
+ command,
+ std::time::Duration::from_secs(5),
+ std::time::Duration::from_secs(2),
+ )
+ .expect("spawns");
+ let elapsed = started.elapsed();
+ // Dropped only now: closing the write end earlier would give the child
+ // EOF whatever stdin `output_within` set.
+ drop(stdin_writer);
+ drop(tmp);
+
+ let Finished::Exited { output, .. } = finished else {
+ panic!("a child given an empty stdin must exit, not time out: {finished:?}");
+ };
+ assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "stdin-eof");
+ assert!(
+ elapsed < std::time::Duration::from_secs(3),
+ "the read must hit EOF at once, but took {elapsed:?}"
+ );
+ }
+
+ /// A probe that exited 0 while a process it started still holds its
+ /// stdout is reported as such, not as "produced no output". Unix only: it
+ /// needs a background process that inherits the pipe, which `sh` gives
+ /// reliably; the classification itself is tested on both gates below.
+ #[cfg(unix)]
+ #[test]
+ fn a_probe_whose_output_stays_open_is_reported_as_that_not_as_no_output() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let result = run_probe_bounded(
+ std::ffi::OsStr::new("sh"),
+ tmp.path(),
+ &["-c", "echo real-output; echo warn >&2; sleep 3 &"],
+ std::time::Duration::from_secs(10),
+ std::time::Duration::from_millis(300),
+ );
+ let Err(error @ ProbeError::OutputNotClosed { .. }) = &result else {
+ panic!("an output left open must be ProbeError::OutputNotClosed, got {result:?}");
+ };
+ assert_eq!(
+ error.to_string(),
+ "`sh -c echo real-output; echo warn >&2; sleep 3 &` exited 0, but its stdout was \
+ still open 300ms later (a process it started may be holding it), so what it \
+ printed cannot be read in full\n--- stderr (last lines) ---\nwarn"
+ );
+ }
+
+ /// Only stdout carries the reading: a process that keeps just stderr open
+ /// does not fail the probe. Unix only, like the test above.
+ #[cfg(unix)]
+ #[test]
+ fn a_probe_whose_stderr_alone_stays_open_still_returns_its_stdout() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let result = run_probe_bounded(
+ std::ffi::OsStr::new("sh"),
+ tmp.path(),
+ &["-c", "echo real-output; sleep 3 >/dev/null &"],
+ std::time::Duration::from_secs(10),
+ std::time::Duration::from_millis(300),
+ );
+ assert_eq!(
+ result.expect("stdout closed, so a reading").trim(),
+ "real-output"
+ );
+ }
+
+ #[test]
+ fn an_undrained_exit_is_classified_by_its_status() {
+ let grace = std::time::Duration::from_secs(5);
+ let undrained = |status| {
+ Ok(Finished::Exited {
+ output: output(status, "partial", "err"),
+ stdout_drained: false,
+ })
+ };
+ let ok = classify_finished("rocm examine", undrained(succeeded_status()), grace, grace);
+ assert!(
+ matches!(&ok, Err(ProbeError::OutputNotClosed { stderr_tail, .. }) if stderr_tail == "err"),
+ "{ok:?}"
+ );
+ // A non-zero exit is reported as one, with what stderr had by then.
+ let failed = classify_finished("rocm examine", undrained(failed_status()), grace, grace);
+ assert!(
+ matches!(&failed, Err(ProbeError::Exit { stderr_tail, .. }) if stderr_tail == "err"),
+ "{failed:?}"
+ );
+ // A drained exit is classified as before.
+ let drained = classify_finished(
+ "rocm examine",
+ Ok(Finished::Exited {
+ output: output(succeeded_status(), "rocm examine\n", "partial err"),
+ stdout_drained: true,
+ }),
+ grace,
+ grace,
+ );
+ // Stdout closed but stderr may not have: the reading is used.
+ assert_eq!(drained.expect("a reading"), "rocm examine\n");
+ let wait_failed = classify_finished(
+ "rocm examine",
+ Ok(Finished::WaitFailed(std::io::Error::other("no child"))),
+ grace,
+ grace,
+ );
+ let Err(error @ ProbeError::Wait { .. }) = &wait_failed else {
+ panic!("a failed wait must be ProbeError::Wait, got {wait_failed:?}");
+ };
+ assert_eq!(
+ error.to_string(),
+ "`rocm examine` was started, but waiting for it to exit failed: no child"
+ );
+ let timed_out = classify_finished(
+ "rocm examine",
+ Ok(Finished::TimedOut {
+ stderr: b"a\nb\n".to_vec(),
+ }),
+ grace,
+ grace,
+ );
+ assert!(
+ matches!(&timed_out, Err(ProbeError::Timeout { stderr_tail, after, .. })
+ if stderr_tail == "a\nb" && *after == grace),
+ "{timed_out:?}"
+ );
+ }
+
+ #[test]
+ fn a_probe_that_finishes_in_time_returns_its_stdout() {
+ let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir");
+ let (binary, args) = shell_line("echo probe-ok");
+ let stdout = run_probe_with(&binary, tmp.path(), &arg_refs(&args), DEFAULT_PROBE_TIMEOUT)
+ .expect("a quick command is a reading");
+ assert_eq!(stdout.trim(), "probe-ok");
+ // A deadline too far off to add to the current time must not panic.
+ let stdout = run_probe_with(
+ &binary,
+ tmp.path(),
+ &arg_refs(&args),
+ std::time::Duration::MAX,
+ )
+ .expect("an unbounded deadline still reads");
+ assert_eq!(stdout.trim(), "probe-ok");
+ }
+
+ #[test]
+ fn the_probe_deadline_reads_whole_positive_seconds_else_the_default() {
+ assert_eq!(probe_timeout_from_env(None), DEFAULT_PROBE_TIMEOUT);
+ assert_eq!(DEFAULT_PROBE_TIMEOUT.as_secs(), 120);
+ for value in ["5", " 5 ", "5\n"] {
+ assert_eq!(
+ probe_timeout_from_env(Some(std::ffi::OsStr::new(value))),
+ std::time::Duration::from_secs(5),
+ "{value:?}"
+ );
+ }
+ // No upper bound: a huge value is honoured, and `output_within`'s
+ // `checked_add` keeps it from overflowing (see the `Duration::MAX` run
+ // in `a_probe_that_finishes_in_time_returns_its_stdout`).
+ for secs in [86_401, u64::MAX] {
+ assert_eq!(
+ probe_timeout_from_env(Some(std::ffi::OsStr::new(&secs.to_string()))),
+ std::time::Duration::from_secs(secs)
+ );
+ }
+ for value in ["", "0", "-3", "1.5", "soon", "18446744073709551616"] {
+ assert_eq!(
+ probe_timeout_from_env(Some(std::ffi::OsStr::new(value))),
+ DEFAULT_PROBE_TIMEOUT,
+ "{value:?}"
+ );
+ }
+ }
}
diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs
index fb3a513e6..0ff33664a 100644
--- a/tests/e2e-cucumber/src/device_memory.rs
+++ b/tests/e2e-cucumber/src/device_memory.rs
@@ -585,7 +585,8 @@ const SMI_DRAIN_GRACE: Duration = Duration::from_secs(2);
/// module is library code, and a normal dependency would be a new first-party
/// edge that `cargo xtask check-crate-edges` makes a reviewed addition. The two
/// can drift, so a change to either one's kill or drain behaviour should look at
-/// the other.
+/// the other. The same applies to `capability::output_within`, the capability
+/// probe's copy of these steps, which also keeps stderr and the exit status.
pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe {
let mut child = match command
.stdin(Stdio::null())
diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs
index fc44c822e..aa8408d0d 100644
--- a/tests/e2e-cucumber/src/expectation.rs
+++ b/tests/e2e-cucumber/src/expectation.rs
@@ -482,6 +482,12 @@ pub struct PlatformManifest<'a> {
pub capability: &'a HostCapability,
/// Component versions (OS/ROCm/vLLM/lemonade) for the report heading.
pub versions: crate::capability::PlatformVersions,
+ /// Why the post-run lane check failed the run
+ /// (`lane_verdict::post_run_verdict`), so the consolidated report shows the
+ /// lane as failed even when every scenario it resolved was a skip. Left out
+ /// when the check passed.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub lane_failure: Option,
pub expectations: Vec,
}
@@ -745,6 +751,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: true,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
},
"strix-ubuntu" => HostCapability {
os_family: "linux".into(),
@@ -758,6 +766,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: true,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
},
"strix-windows" => HostCapability {
os_family: "windows".into(),
@@ -771,6 +781,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: false,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
},
// A WSL2 dev box with the ROCm passthrough in place: `os_family` is
// `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes
@@ -793,6 +805,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: true,
dir_permissions_bind: true,
+ driver_status: "wsl_rocdxg_ready".into(),
+ probe_error: None,
},
// The same box without the passthrough: the gfx target is reported
// by the Windows-side driver but ROCm cannot reach it, so the probe
@@ -809,6 +823,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: true,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
},
// The hosted WSL runner before the ROCm passthrough is complete:
// Windows still reports the gfx target, but the CLI cannot use it.
@@ -824,6 +840,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: true,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
},
_ => HostCapability {
os_family: "other".into(),
@@ -837,6 +855,8 @@ mod tests {
oom_fault_injection: false,
case_sensitive_fs: true,
dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
},
}
}
@@ -2120,6 +2140,36 @@ reason = "unrelated open bug"
));
}
+ /// What the lane-verdict notes say about `@requires-gpu` scenarios (see
+ /// `lane_verdict::preflight_verdict`): `has_amd_gpu` alone decides their
+ /// skip, other gates aside. `resolve` takes no lane declaration, so
+ /// `E2E_EXPECT_GPU` does not enter into it.
+ #[test]
+ fn requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not() {
+ let m = Expectations::default();
+ let d = decl(&["id:gpu-premise", "requires-gpu"]);
+ let included = Included {
+ nightly: false,
+ lifecycle: false,
+ docker: false,
+ merge_queue: false,
+ };
+ let with_gpu = cap("strix-ubuntu");
+ assert!(with_gpu.has_amd_gpu);
+ assert_eq!(
+ resolve(&d, &with_gpu, &m, included),
+ Expectation::ExpectPass
+ );
+ let without_gpu = HostCapability {
+ has_amd_gpu: false,
+ ..with_gpu
+ };
+ assert!(matches!(
+ resolve(&d, &without_gpu, &m, included),
+ Expectation::Skip { .. }
+ ));
+ }
+
#[test]
fn non_gpu_scenario_always_runs() {
let m = Expectations::default();
diff --git a/tests/e2e-cucumber/src/lane_verdict.rs b/tests/e2e-cucumber/src/lane_verdict.rs
new file mode 100644
index 000000000..9d58ef537
--- /dev/null
+++ b/tests/e2e-cucumber/src/lane_verdict.rs
@@ -0,0 +1,961 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier: MIT
+
+//! Whether a run's outcome says anything about the lane it ran on.
+//!
+//! Scenario results alone cannot say that. Scenarios the capability probe
+//! resolves to skip never reach `report.json`, so a run in which every scenario
+//! was skipped, or none was selected, has no failure to report and passes.
+//! That is what a GPU lane does when its probe cannot read the binary under
+//! test, or when the GPU is not visible: every GPU scenario (`@requires-gpu` or
+//! `@requires-multi-gpu`) resolves to skip, and the lane is green without
+//! having tested a GPU.
+//!
+//! Two checks close that gap, both pure so every row of their state matrix is a
+//! unit test:
+//!
+//! - [`preflight_verdict`], before any scenario runs: a probe that failed is
+//! fatal on every lane, and so is a lane that declares a GPU (`1` in
+//! [`EXPECT_GPU_ENV`]) where the probe found none.
+//! - [`post_run_verdict`], after the run: a run that executed no scenario is
+//! fatal on every lane, and so is a GPU-declaring lane on which no GPU
+//! scenario (`@requires-gpu` or `@requires-multi-gpu`) executed.
+//!
+//! The module also holds the decisions `tests/e2e.rs` takes around those two
+//! checks, so they are unit-tested too: [`preflight`] (the declaration plus
+//! the pre-run verdict), [`needs_gpu`] and [`gpu_scenario_counts`] (the input to
+//! the post-run check), [`run_failed`] (the final exit, combining the
+//! harness's own XPASS and failure classification with the post-run verdict),
+//! [`finish_run`] (the post-run verdict and the `platform.json` manifest that
+//! records it), and [`failure_lines`] and [`note_line`] (the exact lines
+//! printed, with the annotation escaped by [`workflow_command`]). The harness
+//! itself still sets up the run, probes the host, reads its other environment
+//! variables, classifies the scenario results, writes the files, and does the
+//! printing and exiting; that glue has no test of its own.
+
+use std::ffi::OsStr;
+
+pub use e2e_report::EXPECT_GPU_ENV;
+
+/// What a lane says about its GPU in [`EXPECT_GPU_ENV`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GpuDeclaration {
+ /// `1`: the lane runs on GPU hardware, so a run that found or used none is
+ /// broken.
+ Expected,
+ /// `0`: the lane deliberately has no usable GPU, so `@requires-gpu`
+ /// scenarios are not applicable on it.
+ NotExpected,
+ /// Unset or empty: no claim either way (GitHub-hosted lanes, local runs).
+ Undeclared,
+}
+
+/// Read [`EXPECT_GPU_ENV`]'s value.
+///
+/// Anything other than `1`, `0`, empty or unset is an error rather than a
+/// guess, since a misspelt declaration that read as "undeclared" would switch
+/// the GPU check off without anyone noticing.
+pub fn parse_gpu_declaration(value: Option<&OsStr>) -> Result {
+ match value.map(OsStr::to_str) {
+ None | Some(Some("")) => Ok(GpuDeclaration::Undeclared),
+ Some(Some("1")) => Ok(GpuDeclaration::Expected),
+ Some(Some("0")) => Ok(GpuDeclaration::NotExpected),
+ Some(_) => Err(format!(
+ "invalid {EXPECT_GPU_ENV} value {:?}: use 1 (this lane has a GPU), 0 (it \
+ deliberately has none), or leave it unset",
+ value.unwrap_or_default().to_string_lossy()
+ )),
+ }
+}
+
+/// What a check concluded: a reason to fail the run, and lines to print either
+/// way.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Verdict {
+ /// Why the run must fail; `None` when it may proceed or pass.
+ pub fatal: Option,
+ /// Informational lines that do not change the outcome.
+ pub notes: Vec,
+}
+
+/// The verdict before any scenario runs, from the capability probe and the
+/// lane's declaration.
+///
+/// `probe_error` is [`crate::capability::HostCapability::probe_error`];
+/// `probe_facts` is the one-line summary of the facts the GPU decision was made
+/// from ([`crate::capability::probe_facts_summary`]), quoted when a declared GPU
+/// is missing.
+#[must_use]
+pub fn preflight_verdict(
+ probe_error: Option<&str>,
+ has_amd_gpu: bool,
+ probe_facts: &str,
+ declaration: GpuDeclaration,
+) -> Verdict {
+ let mut verdict = Verdict::default();
+ if let Some(error) = probe_error {
+ verdict.fatal = Some(format!(
+ "the host capability probe could not read the rocm binary under test, so every \
+ capability-gated scenario would resolve to skip and the run would pass without \
+ testing them: {error}"
+ ));
+ return verdict;
+ }
+ match (declaration, has_amd_gpu) {
+ (GpuDeclaration::Expected, false) => {
+ verdict.fatal = Some(format!(
+ "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but the capability probe found \
+ no usable AMD GPU ({probe_facts}), so every @requires-gpu scenario would \
+ resolve to skip"
+ ));
+ }
+ (GpuDeclaration::NotExpected, true) => verdict.notes.push(format!(
+ "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0) but the capability probe \
+ found one ({probe_facts}); the declaration is stale, and @requires-gpu scenarios \
+ will run"
+ )),
+ (GpuDeclaration::NotExpected, false) => verdict.notes.push(format!(
+ "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0); @requires-gpu scenarios \
+ are not applicable here"
+ )),
+ (GpuDeclaration::Expected, true) | (GpuDeclaration::Undeclared, _) => {}
+ }
+ verdict
+}
+
+/// How many GPU scenarios (`@requires-gpu` or `@requires-multi-gpu`) the run
+/// resolved and how many of those executed.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct GpuScenarioCounts {
+ /// Scenarios tagged `@requires-gpu` or `@requires-multi-gpu` that the
+ /// harness resolved, whether or not they ran.
+ pub resolved: usize,
+ /// Of those, how many reached `report.json`.
+ pub executed: usize,
+}
+
+/// The verdict after the run.
+///
+/// `executed` counts the scenarios that reached `report.json`. `gpu` is `None`
+/// when the harness's scenario filter never ran — a cucumber `--name`/`--tags`
+/// filter on the command line replaces it — so nothing is known about which
+/// scenarios needed a GPU and only the zero-executed check applies.
+#[must_use]
+pub fn post_run_verdict(
+ executed: usize,
+ gpu: Option,
+ declaration: GpuDeclaration,
+) -> Verdict {
+ let mut verdict = Verdict::default();
+ if executed == 0 {
+ verdict.fatal = Some(
+ "0 scenarios executed: a run that executes nothing tests nothing. Check the \
+ cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and E2E_ALSO_RUN_IDS"
+ .to_owned(),
+ );
+ return verdict;
+ }
+ let Some(gpu) = gpu else {
+ return verdict;
+ };
+ match declaration {
+ GpuDeclaration::Expected if gpu.executed == 0 => {
+ verdict.fatal = Some(format!(
+ "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but no @requires-gpu (or \
+ @requires-multi-gpu) scenario executed ({} resolved, none ran)",
+ gpu.resolved
+ ));
+ }
+ GpuDeclaration::NotExpected => verdict.notes.push(format!(
+ "{} of {} GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on this \
+ lane ({EXPECT_GPU_ENV}=0)",
+ gpu.resolved.saturating_sub(gpu.executed),
+ gpu.resolved
+ )),
+ GpuDeclaration::Expected | GpuDeclaration::Undeclared => {}
+ }
+ verdict
+}
+
+/// What the harness carries forward from the pre-run check when it passes.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Preflight {
+ /// The lane's parsed [`EXPECT_GPU_ENV`] declaration, for the post-run check.
+ pub declaration: GpuDeclaration,
+ /// Lines to print before the run.
+ pub notes: Vec,
+}
+
+/// The pre-run lane check, from the raw [`EXPECT_GPU_ENV`] value and the
+/// probe's results: `Err` with the reason when the run must not start.
+///
+/// An invalid declaration is reported before anything the probe found, since
+/// the verdict depends on the declaration. The harness only prints the notes
+/// or the refusal and exits.
+pub fn preflight(
+ declaration_value: Option<&OsStr>,
+ probe_error: Option<&str>,
+ has_amd_gpu: bool,
+ probe_facts: &str,
+) -> Result {
+ let declaration = parse_gpu_declaration(declaration_value)?;
+ let verdict = preflight_verdict(probe_error, has_amd_gpu, probe_facts, declaration);
+ match verdict.fatal {
+ Some(fatal) => Err(fatal),
+ None => Ok(Preflight {
+ declaration,
+ notes: verdict.notes,
+ }),
+ }
+}
+
+/// Whether a scenario counts toward the post-run GPU check.
+///
+/// It does when tagged `@requires-gpu` or `@requires-multi-gpu`. `@requires-gfx-target` does not
+/// count, since such a scenario reads the chip name without using the GPU.
+#[must_use]
+pub const fn needs_gpu(decl: &crate::expectation::ScenarioDecl) -> bool {
+ decl.requires_gpu || decl.requires_multi_gpu
+}
+
+/// Count the GPU scenarios the harness resolved and how many of them executed.
+///
+/// `resolved` is each scenario the harness's filter saw, as `(id, needs_gpu)`;
+/// `executed` is the id of each scenario in `report.json`. `None` when nothing
+/// was resolved: a cucumber `--name`/`--tags` filter replaced the harness's
+/// filter, so which scenarios need a GPU is unknown (see [`post_run_verdict`]).
+/// An executed id the harness did not resolve as a GPU scenario is not counted.
+pub fn gpu_scenario_counts<'a>(
+ resolved: impl IntoIterator- ,
+ executed: impl IntoIterator
- ,
+) -> Option {
+ let mut any_resolved = false;
+ let gpu_ids: std::collections::BTreeSet<&str> = resolved
+ .into_iter()
+ .inspect(|_| any_resolved = true)
+ .filter_map(|(id, gpu)| gpu.then_some(id))
+ .collect();
+ if !any_resolved {
+ return None;
+ }
+ Some(GpuScenarioCounts {
+ resolved: gpu_ids.len(),
+ executed: executed
+ .into_iter()
+ .filter(|id| gpu_ids.contains(id))
+ .count(),
+ })
+}
+
+/// Whether the run exits non-zero after reconciliation.
+///
+/// Any one of a stale XPASS, an unexpected failure (both counted by the
+/// harness) or a fatal post-run verdict is enough. Parse and hook errors exit
+/// earlier, in the harness.
+#[must_use]
+pub const fn run_failed(stale_xpass: usize, unexpected_fail: usize, post_run: &Verdict) -> bool {
+ stale_xpass > 0 || unexpected_fail > 0 || post_run.fatal.is_some()
+}
+
+/// A GitHub Actions workflow command (`::error::…`, `::notice::…`) carrying
+/// `message` as its data.
+///
+/// The data is escaped as the runner expects — `%` first, then CR and LF — so a
+/// probe's stderr tail can neither cut the annotation short at a newline nor
+/// have a literal `%0A` in it read back as one.
+#[must_use]
+pub fn workflow_command(command: &str, message: &str) -> String {
+ let data = message
+ .replace('%', "%25")
+ .replace('\r', "%0D")
+ .replace('\n', "%0A");
+ format!("::{command}::{data}")
+}
+
+/// One scenario the harness's filter resolved, as [`finish_run`] needs it.
+#[derive(Debug, Clone, Copy)]
+pub struct ResolvedRun<'s> {
+ pub id: &'s str,
+ pub feature: &'s str,
+ pub scenario: &'s str,
+ pub engine: &'s str,
+ pub expectation: &'s crate::expectation::Expectation,
+ /// [`needs_gpu`] of the scenario's tags.
+ pub needs_gpu: bool,
+}
+
+/// The post-run lane check, and the `platform.json` manifest that records its
+/// failure as `lane_failure` so the consolidated report shows the lane red.
+///
+/// `resolved` is every scenario the harness's filter saw (empty when a CLI
+/// filter replaced it); `executed` is the id of each scenario in
+/// `report.json`.
+#[must_use]
+pub fn finish_run<'a>(
+ cap: &'a crate::capability::HostCapability,
+ versions: crate::capability::PlatformVersions,
+ resolved: &[ResolvedRun<'_>],
+ executed: &[&str],
+ declaration: GpuDeclaration,
+) -> (Verdict, crate::expectation::PlatformManifest<'a>) {
+ let gpu = gpu_scenario_counts(
+ resolved.iter().map(|r| (r.id, r.needs_gpu)),
+ executed.iter().copied(),
+ );
+ let verdict = post_run_verdict(executed.len(), gpu, declaration);
+ let manifest = crate::expectation::PlatformManifest {
+ platform_slug: &cap.platform_slug,
+ capability: cap,
+ versions,
+ lane_failure: verdict.fatal.clone(),
+ expectations: resolved
+ .iter()
+ .map(|r| {
+ crate::expectation::ResolvedScenario::new(
+ r.id,
+ r.feature,
+ r.scenario,
+ r.engine,
+ r.expectation,
+ )
+ })
+ .collect(),
+ };
+ (verdict, manifest)
+}
+
+/// The lines the harness prints when a lane check fails: a GitHub `::error::`
+/// annotation, then the same message as a plain line for anyone reading the
+/// log outside Actions.
+#[must_use]
+pub fn failure_lines(message: &str) -> [String; 2] {
+ let line = format!("E2E lane check failed: {message}");
+ [workflow_command("error", &line), line]
+}
+
+/// The `::notice::` line the harness prints for a lane-check note.
+#[must_use]
+pub fn note_line(note: &str) -> String {
+ workflow_command("notice", &format!("E2E lane: {note}"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{
+ GpuDeclaration, GpuScenarioCounts, Preflight, ResolvedRun, Verdict, failure_lines,
+ finish_run, gpu_scenario_counts, needs_gpu, note_line, parse_gpu_declaration,
+ post_run_verdict, preflight, preflight_verdict, run_failed, workflow_command,
+ };
+ use std::ffi::OsStr;
+
+ const FACTS: &str =
+ "detected_gfx_target=, wsl=false, driver_status=, no visibility mask";
+ const PROBE_ERROR: &str = "`rocm examine` exited with exit status: 1";
+
+ const ALL: [GpuDeclaration; 3] = [
+ GpuDeclaration::Expected,
+ GpuDeclaration::NotExpected,
+ GpuDeclaration::Undeclared,
+ ];
+
+ #[test]
+ fn only_one_zero_and_unset_or_empty_are_declarations() {
+ assert_eq!(parse_gpu_declaration(None), Ok(GpuDeclaration::Undeclared));
+ assert_eq!(
+ parse_gpu_declaration(Some(OsStr::new(""))),
+ Ok(GpuDeclaration::Undeclared)
+ );
+ assert_eq!(
+ parse_gpu_declaration(Some(OsStr::new("1"))),
+ Ok(GpuDeclaration::Expected)
+ );
+ assert_eq!(
+ parse_gpu_declaration(Some(OsStr::new("0"))),
+ Ok(GpuDeclaration::NotExpected)
+ );
+ }
+
+ #[test]
+ fn a_misspelt_declaration_is_an_error_not_undeclared() {
+ for value in ["true", "yes", " 1", "2"] {
+ assert_eq!(
+ parse_gpu_declaration(Some(OsStr::new(value))),
+ Err(format!(
+ "invalid E2E_EXPECT_GPU value {value:?}: use 1 (this lane has a GPU), 0 (it \
+ deliberately has none), or leave it unset"
+ )),
+ );
+ }
+ }
+
+ // Row: probe failed (non-zero exit, spawn failure, empty output, or
+ // `engines list` failing after `examine` succeeded all reach here as one
+ // `probe_error`). Fatal under every declaration, GPU or not.
+ #[test]
+ fn a_failed_probe_is_fatal_under_every_declaration() {
+ for declaration in ALL {
+ for has_amd_gpu in [false, true] {
+ let verdict = preflight_verdict(Some(PROBE_ERROR), has_amd_gpu, FACTS, declaration);
+ assert_eq!(
+ verdict,
+ Verdict {
+ fatal: Some(format!(
+ "the host capability probe could not read the rocm binary under \
+ test, so every capability-gated scenario would resolve to skip and \
+ the run would pass without testing them: {PROBE_ERROR}"
+ )),
+ notes: vec![],
+ },
+ "{declaration:?}, gpu={has_amd_gpu}"
+ );
+ }
+ }
+ }
+
+ // Row: probe succeeded, no usable GPU, declared 1.
+ #[test]
+ fn a_declared_gpu_the_probe_did_not_find_is_fatal_and_names_the_facts() {
+ let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::Expected);
+ assert_eq!(
+ verdict.fatal.as_deref(),
+ Some(
+ "this lane declares a GPU (E2E_EXPECT_GPU=1) but the capability probe found no \
+ usable AMD GPU (detected_gfx_target=, wsl=false, driver_status=, \
+ no visibility mask), so every @requires-gpu scenario would resolve to skip"
+ )
+ );
+ assert!(verdict.notes.is_empty());
+ }
+
+ // Row: probe succeeded with a GPU, declared 1 — runs, nothing to say.
+ #[test]
+ fn a_declared_gpu_that_is_present_runs_without_comment() {
+ assert_eq!(
+ preflight_verdict(None, true, FACTS, GpuDeclaration::Expected),
+ Verdict::default()
+ );
+ }
+
+ // Row: probe succeeded with a GPU, declared 0 — runs, with a stale note.
+ #[test]
+ fn a_declared_absent_gpu_that_is_present_is_a_note_not_a_failure() {
+ let verdict = preflight_verdict(None, true, FACTS, GpuDeclaration::NotExpected);
+ assert_eq!(verdict.fatal, None);
+ // Pins the wording only. That @requires-gpu scenarios do run on a host
+ // whose probe found a GPU, whatever the declaration, is proved by
+ // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`.
+ assert_eq!(
+ verdict.notes,
+ [format!(
+ "this lane declares no usable GPU (E2E_EXPECT_GPU=0) but the capability probe \
+ found one ({FACTS}); the declaration is stale, and @requires-gpu scenarios will \
+ run"
+ )]
+ );
+ }
+
+ // Row: probe succeeded, no GPU, declared 0 — runs, says why GPU scenarios
+ // are not applicable.
+ #[test]
+ fn a_declared_absent_gpu_that_is_absent_runs_with_a_note() {
+ let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::NotExpected);
+ assert_eq!(verdict.fatal, None);
+ // Pins the wording only. That @requires-gpu scenarios resolve to skip
+ // (not applicable) where the probe found no GPU is proved by
+ // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`.
+ assert_eq!(
+ verdict.notes,
+ [
+ "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are \
+ not applicable here"
+ ]
+ );
+ }
+
+ // Rows: probe succeeded, undeclared — today's behaviour, GPU or not.
+ #[test]
+ fn an_undeclared_lane_runs_without_comment_either_way() {
+ for has_amd_gpu in [false, true] {
+ assert_eq!(
+ preflight_verdict(None, has_amd_gpu, FACTS, GpuDeclaration::Undeclared),
+ Verdict::default(),
+ "gpu={has_amd_gpu}"
+ );
+ }
+ }
+
+ // Post-run row: nothing executed — fatal under every declaration, whether
+ // or not the harness's own filter ran (a `-n` filter matching nothing is
+ // the `None` case).
+ #[test]
+ fn a_run_that_executed_nothing_is_fatal_under_every_declaration() {
+ let gpu_cases = [
+ None,
+ Some(GpuScenarioCounts {
+ resolved: 0,
+ executed: 0,
+ }),
+ Some(GpuScenarioCounts {
+ resolved: 4,
+ executed: 0,
+ }),
+ ];
+ for declaration in ALL {
+ for gpu in gpu_cases {
+ let verdict = post_run_verdict(0, gpu, declaration);
+ assert_eq!(
+ verdict.fatal.as_deref(),
+ Some(
+ "0 scenarios executed: a run that executes nothing tests nothing. Check \
+ the cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and \
+ E2E_ALSO_RUN_IDS"
+ ),
+ "{declaration:?}, {gpu:?}"
+ );
+ }
+ }
+ }
+
+ // Post-run row: declared 1, scenarios ran but no GPU one did.
+ #[test]
+ fn a_gpu_lane_on_which_no_gpu_scenario_ran_is_fatal() {
+ let verdict = post_run_verdict(
+ 12,
+ Some(GpuScenarioCounts {
+ resolved: 7,
+ executed: 0,
+ }),
+ GpuDeclaration::Expected,
+ );
+ assert_eq!(
+ verdict.fatal.as_deref(),
+ Some(
+ "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \
+ @requires-multi-gpu) scenario executed (7 resolved, none ran)"
+ )
+ );
+ }
+
+ // Post-run row: declared 1, scenarios ran, and the harness's filter ran but
+ // resolved no GPU scenario at all (none selected): still fatal, since the
+ // lane promised GPU coverage it did not get.
+ #[test]
+ fn a_gpu_lane_that_resolved_no_gpu_scenario_is_fatal() {
+ let verdict = post_run_verdict(
+ 5,
+ Some(GpuScenarioCounts {
+ resolved: 0,
+ executed: 0,
+ }),
+ GpuDeclaration::Expected,
+ );
+ assert_eq!(
+ verdict,
+ Verdict {
+ fatal: Some(
+ "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \
+ @requires-multi-gpu) scenario executed (0 resolved, none ran)"
+ .to_owned()
+ ),
+ notes: vec![],
+ }
+ );
+ }
+
+ #[test]
+ fn a_gpu_lane_on_which_a_gpu_scenario_ran_passes() {
+ assert_eq!(
+ post_run_verdict(
+ 12,
+ Some(GpuScenarioCounts {
+ resolved: 7,
+ executed: 1,
+ }),
+ GpuDeclaration::Expected,
+ ),
+ Verdict::default()
+ );
+ }
+
+ // Post-run row: a CLI filter bypassed the harness's filter, so which
+ // scenarios need a GPU is unknown — only the zero check applies.
+ #[test]
+ fn a_cli_filtered_run_is_not_held_to_the_gpu_check() {
+ for declaration in ALL {
+ assert_eq!(
+ post_run_verdict(3, None, declaration),
+ Verdict::default(),
+ "{declaration:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn a_no_gpu_lane_says_how_many_gpu_scenarios_did_not_run() {
+ let verdict = post_run_verdict(
+ 12,
+ Some(GpuScenarioCounts {
+ resolved: 7,
+ executed: 0,
+ }),
+ GpuDeclaration::NotExpected,
+ );
+ assert_eq!(verdict.fatal, None);
+ assert_eq!(
+ verdict.notes,
+ [
+ "7 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \
+ this lane (E2E_EXPECT_GPU=0)"
+ ]
+ );
+ // Some ran: the note counts only the ones that did not.
+ let verdict = post_run_verdict(
+ 12,
+ Some(GpuScenarioCounts {
+ resolved: 7,
+ executed: 3,
+ }),
+ GpuDeclaration::NotExpected,
+ );
+ assert_eq!(
+ verdict.notes,
+ [
+ "4 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \
+ this lane (E2E_EXPECT_GPU=0)"
+ ]
+ );
+ }
+
+ #[test]
+ fn an_undeclared_lane_is_not_held_to_the_gpu_check() {
+ assert_eq!(
+ post_run_verdict(
+ 12,
+ Some(GpuScenarioCounts {
+ resolved: 7,
+ executed: 0,
+ }),
+ GpuDeclaration::Undeclared,
+ ),
+ Verdict::default()
+ );
+ }
+
+ // --- the harness glue, as pure functions ---
+
+ #[test]
+ fn preflight_refuses_an_invalid_declaration_before_reading_the_probe() {
+ let invalid = Err(
+ "invalid E2E_EXPECT_GPU value \"yes\": use 1 (this lane has a GPU), \
+ 0 (it deliberately has none), or leave it unset"
+ .to_owned(),
+ );
+ // A probe that also failed, or found no GPU, must not displace the
+ // declaration error: the declaration is checked first.
+ for (probe_error, has_amd_gpu) in [(None, true), (Some(PROBE_ERROR), false), (None, false)]
+ {
+ assert_eq!(
+ preflight(Some(OsStr::new("yes")), probe_error, has_amd_gpu, FACTS),
+ invalid,
+ "probe_error={probe_error:?}, gpu={has_amd_gpu}"
+ );
+ }
+ }
+
+ #[test]
+ fn preflight_refuses_a_failed_probe_and_a_missing_declared_gpu() {
+ let refused = preflight(None, Some(PROBE_ERROR), true, FACTS).expect_err("probe failed");
+ assert!(refused.ends_with(PROBE_ERROR), "{refused}");
+ let refused =
+ preflight(Some(OsStr::new("1")), None, false, FACTS).expect_err("declared GPU missing");
+ assert!(
+ refused.starts_with("this lane declares a GPU (E2E_EXPECT_GPU=1)"),
+ "{refused}"
+ );
+ }
+
+ #[test]
+ fn preflight_carries_the_declaration_and_notes_into_the_run() {
+ assert_eq!(
+ preflight(Some(OsStr::new("0")), None, false, FACTS),
+ Ok(Preflight {
+ declaration: GpuDeclaration::NotExpected,
+ notes: vec![
+ "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu \
+ scenarios are not applicable here"
+ .to_owned()
+ ],
+ })
+ );
+ assert_eq!(
+ preflight(Some(OsStr::new("1")), None, true, FACTS),
+ Ok(Preflight {
+ declaration: GpuDeclaration::Expected,
+ notes: vec![],
+ })
+ );
+ }
+
+ #[test]
+ fn gpu_and_multi_gpu_scenarios_need_a_gpu_and_a_gfx_target_one_does_not() {
+ let decl = crate::expectation::ScenarioDecl::from_tags;
+ assert!(needs_gpu(&decl(&["id:a", "requires-gpu"])));
+ assert!(needs_gpu(&decl(&["id:b", "requires-multi-gpu"])));
+ assert!(!needs_gpu(&decl(&["id:c", "requires-gfx-target"])));
+ assert!(!needs_gpu(&decl(&["id:d"])));
+ }
+
+ #[test]
+ fn a_cli_filtered_run_resolved_nothing_so_has_no_gpu_counts() {
+ assert_eq!(gpu_scenario_counts([], ["serve-a", "serve-b"]), None);
+ }
+
+ #[test]
+ fn gpu_counts_take_resolved_from_the_filter_and_executed_from_the_report() {
+ let resolved = [
+ ("gpu-ran", true),
+ ("multi-gpu-only", true),
+ ("gpu-skipped", true),
+ ("cpu-ran", false),
+ ];
+ // `cpu-ran` executed but is not a GPU scenario, and `untagged` was
+ // never resolved: neither counts as an executed GPU scenario.
+ let executed = ["gpu-ran", "multi-gpu-only", "cpu-ran", "untagged"];
+ assert_eq!(
+ gpu_scenario_counts(resolved, executed),
+ Some(GpuScenarioCounts {
+ resolved: 3,
+ executed: 2,
+ })
+ );
+ }
+
+ #[test]
+ fn gpu_counts_with_nothing_executed_are_zero_not_none() {
+ assert_eq!(
+ gpu_scenario_counts([("gpu", true), ("cpu", false)], []),
+ Some(GpuScenarioCounts {
+ resolved: 1,
+ executed: 0,
+ })
+ );
+ assert_eq!(
+ gpu_scenario_counts([("cpu", false)], ["cpu"]),
+ Some(GpuScenarioCounts {
+ resolved: 0,
+ executed: 0,
+ })
+ );
+ }
+
+ #[test]
+ fn the_run_fails_on_any_one_of_its_three_reasons() {
+ let fatal = Verdict {
+ fatal: Some("0 scenarios executed".to_owned()),
+ notes: vec![],
+ };
+ let clean = Verdict::default();
+ assert!(!run_failed(0, 0, &clean));
+ assert!(run_failed(1, 0, &clean));
+ assert!(run_failed(0, 1, &clean));
+ assert!(run_failed(0, 0, &fatal));
+ // A note alone never fails the run.
+ let noted = Verdict {
+ fatal: None,
+ notes: vec!["stale".to_owned()],
+ };
+ assert!(!run_failed(0, 0, ¬ed));
+ }
+
+ #[test]
+ fn workflow_command_data_escapes_percent_cr_and_lf() {
+ assert_eq!(
+ workflow_command("error", "100% failed\r\nstderr: a%0Ab"),
+ "::error::100%25 failed%0D%0Astderr: a%250Ab"
+ );
+ assert_eq!(workflow_command("notice", "plain"), "::notice::plain");
+ }
+
+ #[test]
+ fn a_failure_prints_an_escaped_annotation_then_the_plain_message() {
+ assert_eq!(
+ failure_lines("probe failed\n--- stderr ---\n100% full"),
+ [
+ "::error::E2E lane check failed: probe failed%0A--- stderr ---%0A100%25 full"
+ .to_owned(),
+ "E2E lane check failed: probe failed\n--- stderr ---\n100% full".to_owned(),
+ ]
+ );
+ }
+
+ #[test]
+ fn a_note_prints_as_an_escaped_notice() {
+ assert_eq!(
+ note_line("7 of 7 GPU scenario(s) did not run (100%)"),
+ "::notice::E2E lane: 7 of 7 GPU scenario(s) did not run (100%25)"
+ );
+ }
+
+ /// `finish_run`'s manifest, written as the harness writes it and read back
+ /// with the consolidated report's own loader: the post-run lane failure
+ /// it records makes the lane red there, and a lane that passed the check
+ /// stays green. Ties the producer to the consumer through the function
+ /// `tests/e2e.rs` calls.
+ #[test]
+ fn finish_runs_manifest_reads_back_as_the_verdict_in_the_report() {
+ let cap = crate::capability::HostCapability {
+ os_family: "linux".into(),
+ is_wsl: false,
+ gfx_target: Some("gfx1151".into()),
+ has_amd_gpu: true,
+ amd_gpu_count: Some(1),
+ available_engines: vec!["lemonade".into()],
+ effective_serve_engine: "lemonade".into(),
+ platform_slug: "strix-halo-linux".into(),
+ oom_fault_injection: false,
+ case_sensitive_fs: true,
+ dir_permissions_bind: true,
+ driver_status: String::new(),
+ probe_error: None,
+ };
+ let pass = crate::expectation::Expectation::ExpectPass;
+ let skip = crate::expectation::Expectation::Skip {
+ reason: "no usable AMD GPU".to_owned(),
+ };
+ let resolved_with = |gpu_expectation| {
+ [
+ ResolvedRun {
+ id: "examine-version",
+ feature: "Examine",
+ scenario: "examine-01",
+ engine: "lemonade",
+ expectation: &pass,
+ needs_gpu: false,
+ },
+ ResolvedRun {
+ id: "serve-gpu",
+ feature: "Serving",
+ scenario: "serve-01",
+ engine: "vllm",
+ expectation: gpu_expectation,
+ needs_gpu: true,
+ },
+ ]
+ };
+ // Writes what the harness writes and reads it back: the verdict, the
+ // report's status, and `platform.json`'s id -> expected mapping.
+ // `executed` is also what report.json holds; every listed id passed.
+ let status_of = |resolved: &[ResolvedRun<'_>], executed: &[&str]| {
+ let (verdict, manifest) = finish_run(
+ &cap,
+ crate::capability::PlatformVersions::default(),
+ resolved,
+ executed,
+ GpuDeclaration::Expected,
+ );
+ let dir = tempfile::TempDir::with_prefix("rocm-e2e-manifest-").expect("temp dir");
+ let elements: Vec = executed
+ .iter()
+ .map(|id| {
+ format!(
+ r#"{{"name":"{id}","tags":[{{"name":"id:{id}"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"passed"}}}}]}}"#
+ )
+ })
+ .collect();
+ let report = dir.path().join("report.json");
+ std::fs::write(
+ &report,
+ format!(
+ r#"[{{"name":"F","uri":"f.feature","elements":[{}]}}]"#,
+ elements.join(",")
+ ),
+ )
+ .expect("report.json");
+ let platform = dir.path().join("platform.json");
+ std::fs::write(
+ &platform,
+ serde_json::to_string_pretty(&manifest).expect("serializes"),
+ )
+ .expect("platform.json");
+ let written: serde_json::Value =
+ serde_json::from_str(&std::fs::read_to_string(&platform).expect("read back"))
+ .expect("valid json");
+ let mapping: Vec> = written["expectations"]
+ .as_array()
+ .expect("an expectations array")
+ .iter()
+ .map(|e| {
+ ["id", "feature", "scenario", "effective_engine", "expected"]
+ .iter()
+ .map(|key| e[*key].as_str().unwrap_or_default().to_owned())
+ .collect()
+ })
+ .collect();
+ (
+ verdict,
+ e2e_report::platform_status("e2e-report", &report),
+ mapping,
+ )
+ };
+ // id, feature, scenario, effective_engine, expected — per expectation.
+ let mapping_of = |rows: &[[&str; 5]]| -> Vec> {
+ rows.iter()
+ .map(|row| row.iter().map(|field| (*field).to_owned()).collect())
+ .collect()
+ };
+
+ // Every cell is pass or skip, so only the recorded lane failure can
+ // make this row FAIL.
+ let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \
+ @requires-multi-gpu) scenario executed (1 resolved, none ran)";
+ let (verdict, failed, mapping) = status_of(&resolved_with(&skip), &["examine-version"]);
+ assert_eq!(verdict.fatal.as_deref(), Some(reason));
+ assert_eq!(
+ mapping,
+ mapping_of(&[
+ [
+ "examine-version",
+ "Examine",
+ "examine-01",
+ "lemonade",
+ "pass"
+ ],
+ ["serve-gpu", "Serving", "serve-01", "vllm", "skip"],
+ ])
+ );
+ assert_eq!(
+ failed,
+ e2e_report::PlatformStatus {
+ ok: false,
+ status: "FAIL",
+ lane_failure: Some(reason.to_owned()),
+ }
+ );
+
+ let (verdict, passed, mapping) =
+ status_of(&resolved_with(&pass), &["examine-version", "serve-gpu"]);
+ assert_eq!(verdict.fatal, None);
+ assert_eq!(
+ mapping,
+ mapping_of(&[
+ [
+ "examine-version",
+ "Examine",
+ "examine-01",
+ "lemonade",
+ "pass"
+ ],
+ ["serve-gpu", "Serving", "serve-01", "vllm", "pass"],
+ ])
+ );
+ assert_eq!(
+ passed,
+ e2e_report::PlatformStatus {
+ ok: true,
+ status: "PASS",
+ lane_failure: None,
+ }
+ );
+ }
+}
diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs
index 29e334583..1d10fe54b 100644
--- a/tests/e2e-cucumber/src/lib.rs
+++ b/tests/e2e-cucumber/src/lib.rs
@@ -12,6 +12,7 @@ pub mod fixture_binary;
pub mod harness_diagnostics;
pub mod http_server;
pub mod installer_fixture;
+pub mod lane_verdict;
pub mod lemonade_backend;
pub mod lemonade_diagnostics;
pub mod lemonade_protocol;
diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs
index 3c2cc9788..3dbb3d83b 100644
--- a/tests/e2e-cucumber/tests/e2e.rs
+++ b/tests/e2e-cucumber/tests/e2e.rs
@@ -263,6 +263,9 @@ struct Resolution {
engine: String,
feature: String,
scenario: String,
+ /// Tagged `@requires-gpu` or `@requires-multi-gpu`, for the post-run check
+ /// that a GPU-declaring lane executed at least one of them.
+ requires_gpu: bool,
}
/// Resolve a CI-provided shared-directory env var to a validated, existing path.
@@ -1499,6 +1502,7 @@ async fn run_suite() {
use e2e_cucumber::expectation::{
Expectation, ScenarioDecl, lifecycle_lane_includes, resolve, unmatched_also_run_ids,
};
+ use e2e_cucumber::lane_verdict;
use e2e_cucumber::monotonic_clock::MonotonicClockWriter;
let dir = results_dir();
@@ -1559,6 +1563,19 @@ async fn run_suite() {
cap.platform_slug, cap.os_family, cap.has_amd_gpu, cap.effective_serve_engine,
);
+ // Refuse to run when the result could not mean anything: a probe that could
+ // not read the binary, or a lane that declares a GPU the probe did not find,
+ // would resolve every GPU scenario to skip and pass. See `lane_verdict`.
+ let preflight = lane_verdict::preflight(
+ std::env::var_os(lane_verdict::EXPECT_GPU_ENV).as_deref(),
+ cap.probe_error.as_deref(),
+ cap.has_amd_gpu,
+ &e2e_cucumber::capability::probe_facts_summary(cap),
+ )
+ .unwrap_or_else(|fatal| lane_failure(&fatal));
+ print_lane_notes(&preflight.notes);
+ let gpu_declaration = preflight.declaration;
+
// Shared record of each scenario's resolved expectation, keyed by @id.
// Populated by `filter_run` (which sees every scenario, run or skipped) so
// the post-run evaluation and platform.json can reconcile by id — including
@@ -1681,6 +1698,7 @@ async fn run_suite() {
engine,
feature: feature.name.clone(),
scenario: scenario.name.clone(),
+ requires_gpu: lane_verdict::needs_gpu(&decl),
},
);
// Two scenarios sharing an `@id` would silently overwrite each
@@ -1736,31 +1754,41 @@ async fn run_suite() {
// OS is always collected (every platform, incl. mock); ROCm/vLLM/lemonade come
// from the shared runtimes tree CI provides (E2E_SHARED_RUNTIMES_DIR) — absent
// on mock (no runtime) and on local runs (per-scenario dirs already dropped),
- // where those render as "n/a". Best-effort.
+ // where those render as "n/a". Best-effort. Collected before the lane check
+ // so that, once decided, nothing stands between it and platform.json.
let shared = shared_runtimes_dir();
let versions = e2e_cucumber::capability::collect_versions(shared.as_deref());
+ // A run with nothing to fail passes; check it executed something, and on
+ // a GPU-declaring lane something that needed the GPU. `resolutions` is empty
+ // when a cucumber `--name`/`--tags` filter replaced `filter_run`, and then
+ // which scenarios need a GPU is unknown. The verdict goes into platform.json,
+ // so the consolidated report shows a failed lane check as red.
+ let resolved: Vec> = resolutions
+ .iter()
+ .map(|(id, r)| lane_verdict::ResolvedRun {
+ id,
+ feature: &r.feature,
+ scenario: &r.scenario,
+ engine: &r.engine,
+ expectation: &r.expectation,
+ needs_gpu: r.requires_gpu,
+ })
+ .collect();
+ let executed: Vec<&str> = actual.iter().map(|(id, _)| id.as_str()).collect();
+ let (post_run, manifest) =
+ lane_verdict::finish_run(cap, versions, &resolved, &executed, gpu_declaration);
+ print_lane_notes(&post_run.notes);
+
// Write the platform.json sidecar (probed capability + every resolution,
- // including skips) for the central report's expected-vs-actual grid.
- let manifest = e2e_cucumber::expectation::PlatformManifest {
- platform_slug: &cap.platform_slug,
- capability: cap,
- versions,
- expectations: resolutions
- .iter()
- .map(|(id, r)| {
- e2e_cucumber::expectation::ResolvedScenario::new(
- id,
- &r.feature,
- &r.scenario,
- &r.engine,
- &r.expectation,
- )
- })
- .collect(),
- };
- if let Ok(json) = serde_json::to_string_pretty(&manifest) {
- let _ = std::fs::write(dir.join("platform.json"), json);
+ // including skips, + any lane failure) for the central report's grid.
+ let written = serde_json::to_string_pretty(&manifest)
+ .map_err(|error| error.to_string())
+ .and_then(|json| {
+ std::fs::write(dir.join("platform.json"), json).map_err(|error| error.to_string())
+ });
+ if let Err(error) = written {
+ eprintln!("warning: could not write platform.json: {error}");
}
// Classify actual results against the matrix. A deterministic XPASS is
@@ -1805,7 +1833,7 @@ async fn run_suite() {
"XPASS (flaky, tolerated): '{x}' passed this run; the intermittent bug remains tracked."
);
}
- if !stale_xpass.is_empty() || !unexpected_fail.is_empty() {
+ if lane_verdict::run_failed(stale_xpass.len(), unexpected_fail.len(), &post_run) {
for x in &stale_xpass {
eprintln!(
"XPASS: '{x}' is expected to fail on this host but PASSED \u{2014} the bug appears \
@@ -1817,6 +1845,24 @@ async fn run_suite() {
"FAIL: '{f}' was expected to pass on this host but FAILED \u{2014} a regression."
);
}
+ if let Some(fatal) = &post_run.fatal {
+ lane_failure(fatal);
+ }
std::process::exit(1);
}
}
+
+/// Print a lane-check failure (see `lane_verdict::failure_lines`) and exit
+/// non-zero.
+fn lane_failure(message: &str) -> ! {
+ for line in e2e_cucumber::lane_verdict::failure_lines(message) {
+ eprintln!("{line}");
+ }
+ std::process::exit(1);
+}
+
+fn print_lane_notes(notes: &[String]) {
+ for note in notes {
+ eprintln!("{}", e2e_cucumber::lane_verdict::note_line(note));
+ }
+}
diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs
index 9d8d47c79..1c85f2e26 100644
--- a/xtask/src/release_readiness.rs
+++ b/xtask/src/release_readiness.rs
@@ -428,13 +428,13 @@ fn queue_absent_reason(markdown: &str) -> Option {
/// for every per-platform row that reports a non-`PASS` `Status` with a `Fail`
/// cell of `0`.
///
-/// The second part exists because an `EMPTY` platform (no `report.json`, or
-/// one that's empty/unparseable) contributes `0` to every numeric column —
-/// including `Fail` — so the Total row's own Fail cell has no record that the
-/// platform was ever there at all. A genuine `FAIL` status always has a
-/// nonzero own `Fail` cell (already folded into the Total row), so this only
-/// adds for platforms the Total row would otherwise read as having zero
-/// failures.
+/// The second part exists because two kinds of failed row contribute `0` to
+/// the `Fail` column, so the Total row's own Fail cell has no record of them:
+/// an `EMPTY` platform (no `report.json`, or one that's empty/unparseable),
+/// and a `FAIL` platform whose only problem is the harness's post-run lane
+/// check (`lane_failure` in its `platform.json`, e.g. "0 scenarios executed"),
+/// which fails no scenario cell. A `FAIL` row with a nonzero own `Fail` cell
+/// is already folded into the Total row and is not counted again.
///
/// A row for a shared-queue lane the queue does not block on (its platform
/// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the
@@ -795,6 +795,54 @@ mod tests {
assert_eq!(total_e2e_failures(md), Some(1));
}
+ /// A lane the harness failed after its run with no failed scenario cell
+ /// (every scenario passed or was skipped) reads `FAIL` with a `Fail` cell
+ /// of 0; the Total row cannot show it, so the row itself must count. One
+ /// that also has a failed cell is already in the Total row and is counted
+ /// once, not twice.
+ #[test]
+ fn total_e2e_failures_counts_a_lane_check_failure_with_no_failed_cells() {
+ // Real renderer output over a platform.json that records a lane
+ // failure, with the one scenario passing or failing.
+ let render = |step_status: &str| {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let report = dir.path().join("report.json");
+ std::fs::write(
+ &report,
+ format!(
+ r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s0","tags":[{{"name":"id:examine-a"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{step_status}"}}}}]}}]}}]"#
+ ),
+ )
+ .expect("write report");
+ std::fs::write(
+ dir.path().join("platform.json"),
+ r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed",
+ "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#,
+ )
+ .expect("write platform.json");
+ e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&[(
+ "e2e-gpu-report".to_string(),
+ report,
+ )]))
+ };
+
+ let md = render("passed");
+ assert!(
+ md.contains("| 1 | 1 | 0 | 0 | n/a | FAIL |")
+ && md.contains("| **Total** | | 1 | 1 | 0 |"),
+ "expected a FAIL row with a Fail cell of 0:\n{md}"
+ );
+ assert_eq!(total_e2e_failures(&md), Some(1), "{md}");
+
+ let md = render("failed");
+ assert!(
+ md.contains("| 1 | 0 | 1 | 0 | n/a | FAIL |")
+ && md.contains("| **Total** | | 1 | 0 | 1 |"),
+ "expected a FAIL row with a Fail cell of 1:\n{md}"
+ );
+ assert_eq!(total_e2e_failures(&md), Some(1), "{md}");
+ }
+
#[test]
fn total_e2e_failures_leaves_out_the_queue_lanes_the_queue_lets_fail() {
// Real producer output, so the marker the parser looks for is the one the
diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs
index a33154924..e4d25b65d 100644
--- a/xtask/src/workflow_contract.rs
+++ b/xtask/src/workflow_contract.rs
@@ -1718,6 +1718,150 @@ jobs:
}
}
+ /// Self-hosted E2E lanes that declare no usable GPU, each with the reason.
+ /// Every other self-hosted lane must declare one.
+ const NO_GPU_LANES: [(&str, &str); 2] = [
+ (
+ "e2e-wsl",
+ "the WSL distro is deliberately left without ROCm's WSL driver bridge, \
+ which `rocm install driver` installs under test",
+ ),
+ (
+ "e2e-wsl-nightly",
+ "the nightly twin of e2e-wsl, set up the same way",
+ ),
+ ];
+
+ /// Whether a step's `run:` body invokes the E2E suite itself
+ /// (`cargo xtask e2e`, not `e2e-prewarm` or `e2e-report`).
+ fn runs_the_e2e_suite(step: &str) -> bool {
+ run_block(step).is_some_and(|body| {
+ body.iter().any(|line| {
+ line.match_indices("cargo xtask e2e").any(|(at, m)| {
+ line[at + m.len()..]
+ .chars()
+ .next()
+ .is_none_or(char::is_whitespace)
+ })
+ })
+ })
+ }
+
+ /// Every self-hosted E2E lane declares whether it has a GPU, so the harness
+ /// can fail a GPU lane whose probe found none instead of passing it with
+ /// every GPU scenario (`@requires-gpu` or `@requires-multi-gpu`) skipped.
+ ///
+ /// Derived from the workflows like the GPU-preflight check above, so a new
+ /// lane is held to it the day it lands. The declaration is read from the
+ /// job-level `env:` mapping: a step-level one would leave the other steps,
+ /// and a reader of the job header, without it, and one on the step that
+ /// runs the suite would override it there. A lane that runs the suite
+ /// inside WSL must name the variable in every `WSLENV` it sets, and in one
+ /// on the step that runs the suite, since `wsl.exe` passes nothing else
+ /// into the guest.
+ #[test]
+ fn every_self_hosted_lane_declares_whether_it_has_a_gpu() {
+ let name = e2e_report::EXPECT_GPU_ENV;
+ let mut no_gpu_seen = Vec::new();
+ for (workflow, text) in self_hosted_workflows() {
+ for (job, _) in self_hosted_e2e_jobs(&text) {
+ let block = job_block(&text, &job);
+ let env = job_mapping(block, "env");
+ let exempt = NO_GPU_LANES.iter().any(|(lane, _)| *lane == job);
+ match env.get(name).map(String::as_str) {
+ Some("1") => assert!(
+ !exempt,
+ "{workflow} job `{job}` is listed in NO_GPU_LANES but declares \
+ {name}: \"1\"; drop it from the list"
+ ),
+ Some("0") => {
+ assert!(
+ exempt,
+ "{workflow} job `{job}` declares {name}: \"0\" but is not in \
+ NO_GPU_LANES; a self-hosted lane without a GPU needs its reason \
+ recorded there"
+ );
+ no_gpu_seen.push(job.clone());
+ }
+ other => panic!(
+ "{workflow} job `{job}` runs on self-hosted hardware but its job-level \
+ `env:` sets {name} to {other:?}; it must be \"1\" (a GPU lane) or \
+ \"0\" (listed in NO_GPU_LANES)"
+ ),
+ }
+ let why = NO_GPU_LANES
+ .iter()
+ .find(|(lane, _)| *lane == job)
+ .map_or_else(String::new, |(_, reason)| {
+ format!(" (a NO_GPU_LANES lane: {reason})")
+ });
+ let names_it = |wslenv: &str| {
+ wslenv
+ .trim_matches(|c| c == '\'' || c == '"')
+ .split(':')
+ .any(|var| var.split('/').next() == Some(name))
+ };
+ for wslenv in block
+ .lines()
+ .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = "))
+ {
+ assert!(
+ names_it(wslenv),
+ "{workflow} job `{job}`{why} sets WSLENV without {name}, so the suite \
+ inside WSL reads the lane as undeclared: {wslenv}"
+ );
+ }
+ // The step that runs the suite is where the declaration has
+ // to arrive: a step-level override there would replace the
+ // job's value, and a WSL step that names it in no WSLENV of
+ // its own leaves the guest without it, whatever other steps do.
+ let steps = job_steps(block)
+ .unwrap_or_else(|| panic!("{workflow} job `{job}` has no `steps:` list"));
+ let harness: Vec<&(String, String)> = steps
+ .iter()
+ .filter(|(_, step)| runs_the_e2e_suite(step))
+ .collect();
+ assert!(
+ !harness.is_empty(),
+ "{workflow} job `{job}` has no step that runs `cargo xtask e2e`"
+ );
+ for (step_name, step) in harness {
+ assert!(
+ !step_env(step).contains_key(name),
+ "{workflow} job `{job}`{why}: step `{step_name}` sets {name} itself, \
+ overriding the job-level declaration"
+ );
+ if step.contains("Invoke-WslBash.ps1") {
+ assert!(
+ step.lines()
+ .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = "))
+ .any(names_it),
+ "{workflow} job `{job}`{why}: step `{step_name}` runs the suite in \
+ WSL but sets no WSLENV naming {name}, so the guest reads the lane \
+ as undeclared"
+ );
+ }
+ }
+ }
+ }
+ for (lane, _) in NO_GPU_LANES {
+ assert!(
+ no_gpu_seen.iter().any(|seen| seen == lane),
+ "NO_GPU_LANES lists `{lane}`, which no self-hosted workflow declares \
+ {name}: \"0\"; drop the stale entry"
+ );
+ }
+ // The queue lanes run through this script, not a workflow job; every
+ // queue lane is given a GPU.
+ let script = std::fs::read_to_string(repo_root().join("scripts/run-e2e-lane.sh"))
+ .expect("reading the lane script");
+ let export = format!("export {name}=\"${{{name}-1}}\"");
+ assert!(
+ script.lines().any(|line| line.trim() == export),
+ "scripts/run-e2e-lane.sh must declare its lanes' GPU with `{export}`"
+ );
+ }
+
#[test]
fn only_e2e_gpu_may_carry_the_stale_gate_on_its_preflight() {
let workflow = |job: &str| {
@@ -5703,14 +5847,16 @@ fi
let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md"))
.expect("read hardware testing docs");
- let documented: Vec> =
- markdown_table_rows(&docs, "| Job | Workflow | Platform | Runner labels |")
- .into_iter()
- .filter(|row| {
- row.get(1)
- .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`")
- })
- .collect();
+ let documented: Vec> = markdown_table_rows(
+ &docs,
+ "| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) |",
+ )
+ .into_iter()
+ .filter(|row| {
+ row.get(1)
+ .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`")
+ })
+ .collect();
let documented_jobs: Vec = documented.iter().map(|row| row[0].clone()).collect();
let declared_jobs: Vec = lanes.iter().map(|(job, _)| format!("`{job}`")).collect();
@@ -5738,6 +5884,15 @@ fi
flattened_list_items(runs_on),
"the documented runner labels for `{job}` must match its actual runs-on"
);
+ let declared = job_mapping(job_block(&self_hosted, job), "env")
+ .get(e2e_report::EXPECT_GPU_ENV)
+ .map(|value| format!("`{value}`"));
+ assert_eq!(
+ row.get(4),
+ declared.as_ref(),
+ "the GPU declared cell for `{job}` must match its job-level {}",
+ e2e_report::EXPECT_GPU_ENV
+ );
}
// Derived from the same scan `every_uploaded_e2e_artifact_has_a_name_the_report_can_label`