diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index c0289afaf..3a7b995e2 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -270,6 +270,9 @@ jobs: contents: read pull-requests: read env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # Bound serve readiness below the 35-min job cap so a serve that never comes # ready fails the scenario with a real error instead of hanging until the # job is cancelled. 300s is ample for a real MI300X vLLM cold-start; @@ -625,6 +628,9 @@ jobs: - channel: nightly sdk_version: n-2 env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_SERVE_TIMEOUT_SECS: "300" # A pool host can be busy with other CI work between GPU preflight and # the TUI checks, so a frame that renders well inside the 30s default on @@ -779,6 +785,9 @@ jobs: ) continue-on-error: false env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # A pool host can be busy with other CI work between GPU preflight and # the TUI checks, so a frame that renders well inside the 30s default on # an idle runner can miss it. Raise the wait budget rather than let that @@ -841,8 +850,10 @@ jobs: # available so a wedged/held GPU fails fast instead of hanging to the cap. # BEST-EFFORT because the Windows ROCm GPU query tool isn't verified here — # if no rocm-smi is found we WARN and continue rather than false-fail a - # working runner. When rocm-smi IS present we poll free memory the same way - # as the Linux jobs and fail only after the ceiling. + # working runner; the harness's E2E_EXPECT_GPU=1 check is the backstop + # that fails the lane if `rocm examine` reports no usable GPU at all. + # When rocm-smi IS present we poll free memory the same way as the Linux + # jobs and fail only after the ceiling. # # WHAT THIS DOES NOT CATCH, and the Linux lanes now do. This is a # free-memory floor ONLY. A fixed 8 GiB against a whole ~62 GiB gfx1151 @@ -1130,6 +1141,13 @@ jobs: ) continue-on-error: false env: + # Declared GPU-less on purpose: the distro is left without ROCm's WSL + # driver bridge (see the "NO librocdxg provisioning step" note below), so + # the probe finds no usable GPU and @requires-gpu scenarios are not + # applicable. The harness reports that rather than failing on it; set "1" + # here if this lane ever provisions a GPU. Forwarded into the guest via + # WSLENV in the run step. + E2E_EXPECT_GPU: "0" # wsl.exe emits UTF-16 by default; without this its own error/status text # comes back mangled ("T h e f i l e c a n n o t ...") when captured # through PowerShell, which is exactly what a genuine wsl.exe error looks @@ -1406,12 +1424,14 @@ jobs: # today: @requires-gpu scenarios already resolve to skip here, since # `rocm-smi` is absent too (see the GPU preflight below). If this lane # ever needs a real GPU, provision it through the CLI as a separate, - # explicitly-not-under-test step rather than restoring the old script. + # explicitly-not-under-test step rather than restoring the old script, + # and flip the job's E2E_EXPECT_GPU to "1". # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU # access under WSL is exactly what this lane is proving out, so an absent - # rocm-smi is a reported condition, not a job failure: the capability probe - # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # rocm-smi is a reported condition, not a job failure: the lane declares + # E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not + # applicable instead of failing on them, and the rest still runs. A # GPU that IS present but stays held fails the job, since that would # corrupt the serve scenarios' results — though "held" means less here # than on the native lanes: this body keeps only the free-memory floor and @@ -1433,7 +1453,7 @@ jobs: MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" if ! command -v rocm-smi >/dev/null 2>&1; then - echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) @@ -1533,7 +1553,7 @@ jobs: sleep 5 done if [ "$saw_vram" -eq 0 ]; then - echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi if [ "$unmeasurable" -eq 1 ]; then @@ -1590,7 +1610,7 @@ jobs: # Forward the job-level E2E_* env vars into the guest: `wsl.exe` # does not share the calling process's environment unless a # variable is named in WSLENV. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' + $env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index da22f2344..04fd6acd2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -417,6 +417,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" # Include @nightly scenarios (the large-model serve). See src/expectation.rs. E2E_INCLUDE_NIGHTLY: "1" # Serve-readiness floor; a `@serve-timeout` tag lengthens it for the large @@ -531,6 +534,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -618,6 +624,9 @@ jobs: matrix: channel: [release, nightly] env: + # GPU lane: the E2E harness fails if its probe finds no usable AMD GPU + # or no GPU scenario runs (see tests/e2e-cucumber/README.md). + E2E_EXPECT_GPU: "1" E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -885,6 +894,13 @@ jobs: matrix: channel: [release, nightly] env: + # Declared GPU-less on purpose: the distro is left without ROCm's WSL + # driver bridge (see the "NO librocdxg provisioning step" note below), so + # the probe finds no usable GPU and @requires-gpu scenarios are not + # applicable. The harness reports that rather than failing on it; set "1" + # here if this lane ever provisions a GPU. Forwarded into the guest via + # WSLENV in the run step. + E2E_EXPECT_GPU: "0" # wsl.exe emits UTF-16 by default; without this its own error/status text # comes back mangled when captured through PowerShell. Matches the # ubuntu/WSL composite action's own env. @@ -1084,8 +1100,9 @@ jobs: # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU # access under WSL is exactly what this lane is proving out, so an absent - # rocm-smi is a reported condition, not a job failure: the capability probe - # then resolves @requires-gpu scenarios to skip and the rest still runs. A + # rocm-smi is a reported condition, not a job failure: the lane declares + # E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not + # applicable instead of failing on them, and the rest still runs. A # GPU that IS present but stays held fails the job, since that would # corrupt the serve scenarios' results. # @@ -1106,7 +1123,7 @@ jobs: MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" if ! command -v rocm-smi >/dev/null 2>&1; then - echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) @@ -1206,7 +1223,7 @@ jobs: sleep 5 done if [ "$saw_vram" -eq 0 ]; then - echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" + echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable" exit 0 fi if [ "$unmeasurable" -eq 1 ]; then @@ -1257,7 +1274,7 @@ jobs: shell: powershell run: | # See .github/scripts/Invoke-WslBash.ps1. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF' + $env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli @@ -1326,7 +1343,13 @@ jobs: # a failed step there does not turn the lane's check red. Only the # WSL lanes need this guard: every other lane writes report.json # directly in-job, with no guest-to-host copy step that can fail on - # its own. + # its own. A harness preflight refusal is not one of those: + # `run_suite` in tests/e2e-cucumber/tests/e2e.rs creates report.json + # empty before it calls `lane_verdict::preflight`, and the + # `if: always()` copy-out above carries it to the host, so it uploads + # fine and the missing-report check does not catch it; its EMPTY row + # in the report is the signal (a post-run lane failure shows as a + # FAIL row with lane_failure). if-no-files-found: error # The nightly lanes each uploaded a report nobody joined up, so reading a @@ -1501,13 +1524,18 @@ jobs: fi cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY" - # A lane that dies before writing report.json (preflight, build, a crash - # before results exist) uploads nothing, and `discover()` can only render - # the reports that arrived, so that lane would drop out of the grid with - # no row and no FAIL. On a `continue-on-error` lane nothing else is red - # either. So check it here: every lane that ran (any `needs` result but - # `skipped`, and not self-skipped as `stale`) must have uploaded its report - # artifact. This reads the run's + # A lane that dies before writing report.json (the GPU preflight step, a + # build failure, a crash before results exist) uploads nothing. The + # harness's own preflight refusal is different: `run_suite` in + # tests/e2e-cucumber/tests/e2e.rs creates report.json empty before it + # calls `lane_verdict::preflight`, and each lane's `if: always()` upload + # step publishes it, so it shows as an EMPTY row (a post-run lane failure + # as a FAIL row with lane_failure), not as missing. `discover()` can only + # render the reports that arrived, so a lane that uploads nothing would + # drop out of the grid with no row and no FAIL. On a `continue-on-error` + # lane nothing else is red either. So check it here: every lane that ran + # (any `needs` result but `skipped`, and not self-skipped as `stale`) + # must have uploaded its report artifact. This reads the run's # artifact list rather than e2e-artifacts/, because download-artifact # flattens a single match into the root and loses its name. The list also # holds earlier attempts' artifacts, so a re-run that uploads nothing can diff --git a/crates/e2e-report/src/components.rs b/crates/e2e-report/src/components.rs index 4015af617..ab008eb6a 100644 --- a/crates/e2e-report/src/components.rs +++ b/crates/e2e-report/src/components.rs @@ -215,6 +215,8 @@ const STYLE: &str = r#" .platform-row { display: flex; align-items: center; gap: 8px; cursor: pointer; font-size: 1.05rem; } .platform-name { font-weight: 700; flex: 1; } .empty-note { color: #999; font-style: italic; padding: 8px 0; } + .lane-failure { color: #c62828; background: #ffebee; border-left: 4px solid #c62828; + padding: 8px 12px; margin: 8px 0; white-space: pre-wrap; } .scenario { border: 1px solid #e0e0e0; border-top: none; padding: 12px; background: #fff; } .scenario:last-child { border-radius: 0 0 4px 4px; } .scenario-row { display: flex; align-items: center; gap: 8px; cursor: pointer; } diff --git a/crates/e2e-report/src/consolidated/html.rs b/crates/e2e-report/src/consolidated/html.rs index 6c4717152..042980bdf 100644 --- a/crates/e2e-report/src/consolidated/html.rs +++ b/crates/e2e-report/src/consolidated/html.rs @@ -14,8 +14,9 @@ use std::path::Path; use maud::{Markup, html}; use super::{ - CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, PlatformReport, - PlatformReports, platform_cell, + CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, + PLATFORM_ATTENTION_HEADING, PlatformReport, PlatformReports, SCENARIO_ATTENTION_HEADING, + platform_cell, }; use crate::components::{feature_group, now_utc, page_shell, stats_bar}; @@ -103,6 +104,7 @@ pub fn generate_consolidated( h2 { "Platforms" } (matrix_table(reports)) (legend()) + (lane_failures_html(reports)) (expectation_grid_html(&bundle.grid)) @@ -193,7 +195,7 @@ pub(super) fn expectation_grid_html(grid: &Grid) -> Markup { } } @if !problems.is_empty() { - h3 { "Needs attention" } + h3 { (SCENARIO_ATTENTION_HEADING) } ul { @for (platform_label, id, outcome, detail) in &problems { li { @@ -246,7 +248,8 @@ fn matrix_table(reports: &[PlatformReport]) -> Markup { td.num { (fail) } td.num { (skip) } td.num { @if xf > 0 || r.is_known_bugs { (xf) } @else { "n/a" } } - td class=(if r.ok() { "status-pass" } else { "status-fail" }) { + td class=(if r.ok() { "status-pass" } else { "status-fail" }) + title=[r.lane_failure()] { (r.status_text()) } td { (stats_bar(&r.stats)) } @@ -288,7 +291,10 @@ fn legend() -> Markup { " — PASS when the platform is in its expected state (every " "expect-pass scenario passed and every known bug failed as " "expected); FAIL on an unexpected failure or an XPASS (a known " - "bug that unexpectedly passed — remove its @expected-failure tag)." + "bug that unexpectedly passed — remove its @expected-failure tag), " + "or when the harness failed the lane after the run (for example, no " + "scenario executed); the reason is listed under \u{201c}" + (PLATFORM_ATTENTION_HEADING) "\u{201d} and in the platform's section." } li { b { "Xfail" } " — count of known-bug scenarios that failed as expected." } li { b { "Skip" } " — scenarios not applicable on this platform (not run)." } @@ -297,6 +303,26 @@ fn legend() -> Markup { } } +/// The platforms the harness failed after their run, with each reason: a +/// lane-check failure has no grid cell of its own (an all-skip lane has only +/// skip cells), so it is listed here. +fn lane_failures_html(reports: &[PlatformReport]) -> Markup { + let failed: Vec<(&str, &str)> = reports + .iter() + .filter_map(|r| r.lane_failure().map(|reason| (r.label.as_str(), reason))) + .collect(); + html! { + @if !failed.is_empty() { + h3 { (PLATFORM_ATTENTION_HEADING) } + ul { + @for (label, reason) in &failed { + li { b { "Lane check failed" } " in " code { (label) } ": " (reason) } + } + } + } + } +} + fn platform_section(report: &PlatformReport) -> Markup { let badge_class = if report.ok() { "badge-pass" @@ -310,6 +336,9 @@ fn platform_section(report: &PlatformReport) -> Markup { span.platform-name { (report.label) } span.elapsed { (report.stats.total) " scenarios" } } + @if let Some(reason) = report.lane_failure() { + p.lane-failure { b { "Lane check failed: " } (reason) } + } @if report.features.is_empty() { p.empty-note { "No report.json data for this platform." } } @else { @@ -328,7 +357,7 @@ mod tests { use crate::consolidated::{load_platform_reports, load_reports}; #[test] - fn legend_mock_and_other_platform_items_are_unchanged() { + fn legend_mock_and_other_platform_items_are_pinned() { // Pins the two gating items byte for byte, so building them from shared // fragments (see `MOCK_GATES_PR`) cannot change what the HTML // report's legend says. Text only: that Mock is the required check and @@ -365,12 +394,18 @@ mod tests { // now would be building the bundle's grid from `&[]` in // `load_platform_reports`. Give this one a platform.json via // write_platform and assert the grid heading actually renders. - let report = feature_json(&[(&["id:serve-a"], &["passed"])]); + // `serve-b` fails though expected to pass, so the grid's attention + // list renders too. + let report = feature_json(&[ + (&["id:serve-a"], &["passed"]), + (&["id:serve-b"], &["failed"]), + ]); let platform = r#"{ "platform_slug": "mock", "capability": {"effective_serve_engine": "none"}, "expectations": [ - {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"} + {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"}, + {"id":"serve-b","feature":"Serving","scenario":"serve-02 - second","expected":"pass"} ] }"#; let (_d, path) = write_platform(&report, platform); @@ -384,6 +419,10 @@ mod tests { html.contains("Expectation grid (scenario × platform)"), "grid heading missing from output:\n{html}" ); + assert!( + html.contains(&format!("

{SCENARIO_ATTENTION_HEADING}

")), + "scenario attention heading missing from output:\n{html}" + ); } #[test] diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs index 18203aae0..51024c330 100644 --- a/crates/e2e-report/src/consolidated/markdown.rs +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -11,8 +11,9 @@ //! crate root by `lib.rs`. use super::{ - CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, PlatformReports, - ScenarioReference, platform_cell, + CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, + PLATFORM_ATTENTION_HEADING, PlatformReports, SCENARIO_ATTENTION_HEADING, ScenarioReference, + platform_cell, }; use crate::command_coverage::command_coverage_markdown; @@ -113,13 +114,26 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ failed, or a known bug that unexpectedly passed (its tag is now stale). \ - **Status** is PASS unless a platform has any Fail.\n", + **Status** is PASS unless a platform has any Fail, or the harness failed \ + its lane check after the run (listed under {PLATFORM_ATTENTION_HEADING}).\n", ); - // Call out anything that needs a human: XPASS (fixed bug, stale tag) and + // Call out anything that needs a human: a failed post-run lane check + // (`lane_failure` in platform.json), XPASS (fixed bug, stale tag) and // untagged failures in a known-bugs run. let mut notes = Vec::new(); for r in reports { + if let Some(reason) = r.lane_failure() { + // Written by the harness's `finish_run` from `post_run_verdict`'s + // fixed strings, so not escaped; folded onto the bullet's line. + // Carrying a pre-run reason (which quotes probe stderr) into the + // report would need escaping here. + notes.push(format!( + "- **Lane check failed** in _{}_: {}", + r.label, + reason.split_whitespace().collect::>().join(" ") + )); + } for name in &r.xfail.xpass { notes.push(format!( "- **XPASS** in _{}_: `{}` is tagged `@expected-failure` but passed — remove the tag.", @@ -136,7 +150,7 @@ pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { } } if !notes.is_empty() { - out.push_str("\n### Needs attention\n\n"); + let _ = write!(out, "\n### {PLATFORM_ATTENTION_HEADING}\n\n"); for n in notes { out.push_str(&n); out.push('\n'); @@ -244,7 +258,7 @@ fn expectation_grid_markdown(grid: &Grid, scenarios: &ScenarioReference) -> Stri // Needs-attention list from reconciliation problems. let problems = grid.problems(); if !problems.is_empty() { - out.push_str("\n### Needs attention\n\n"); + let _ = write!(out, "\n### {SCENARIO_ATTENTION_HEADING}\n\n"); for (platform_label, id, outcome, detail) in problems { let bug = detail .and_then(|d| d.bug.as_deref()) @@ -360,12 +374,14 @@ mod tests { b.path().to_path_buf(), )]; let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("Needs attention")); - assert!(md.contains("XPASS")); + assert!( + md.contains("\n### Needs attention: platforms\n\n- **XPASS** in _"), + "the XPASS bullet must sit directly under the platform attention heading:\n{md}" + ); } #[test] - fn summary_explanation_paragraphs_are_unchanged() { + fn summary_explanation_paragraphs_are_pinned() { // Pins the explanatory text under the summary table byte for byte, so // building it from shared fragments (see `MOCK_GATES_PR`) cannot // change a character of what the step summary shows. This pins the text @@ -394,7 +410,12 @@ mod tests { reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ failed, or a known bug that unexpectedly passed (its tag is now stale). \ - **Status** is PASS unless a platform has any Fail.\n"; + **Status** is PASS unless a platform has any Fail, or the harness failed \ + its lane check after the run (listed under Needs attention: platforms).\n"; + // Pins the wording only. That a lane-check failure turns Status FAIL and + // is listed under "Needs attention: platforms" is proved by + // `consolidated::tests::an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason` + // and `consolidated::tests::a_recorded_lane_failure_is_named_in_the_summary_and_the_html`. assert!( md.contains(expected), "summary explanation text changed:\n{md}" @@ -410,6 +431,10 @@ mod tests { md.contains(OTHER_LANES_NON_BLOCKING), "summary must render OTHER_LANES_NON_BLOCKING ({OTHER_LANES_NON_BLOCKING:?}):\n{md}" ); + assert!( + md.contains(&format!("(listed under {PLATFORM_ATTENTION_HEADING})")), + "summary must render PLATFORM_ATTENTION_HEADING ({PLATFORM_ATTENTION_HEADING:?}):\n{md}" + ); } #[test] diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 9a51e9036..16de2c77b 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -194,6 +194,14 @@ const MOCK_GATES_PR: &str = "gates the PR"; /// See [`MOCK_GATES_PR`]. const OTHER_LANES_NON_BLOCKING: &str = "non-blocking"; +/// Heading of the platform-level attention list (lane-check failures; in the +/// markdown summary also stale `@expected-failure` tags and regressions). +/// Distinct from [`SCENARIO_ATTENTION_HEADING`], so a report with both lists +/// never shows two identical headings. Shared by both renderers. +const PLATFORM_ATTENTION_HEADING: &str = "Needs attention: platforms"; +/// Heading of the expectation grid's per-scenario attention list. +const SCENARIO_ATTENTION_HEADING: &str = "Needs attention: scenarios"; + /// One platform column of the reconciled (scenario-id × platform) grid. struct GridColumn { /// Platform identity from the manifest (e.g. "mi300x", "strix-halo", "mock"). @@ -571,12 +579,17 @@ struct ReconciledTally { skip: u32, /// Cells that need a human: unexpected-fail, XPASS, or ran-when-N/A. problems: u32, + /// The harness failed the run after it finished (`lane_failure` in + /// `platform.json`), e.g. because no scenario executed. No cell shows that: + /// a run whose every scenario resolved to skip has only skip cells. + lane_failed: bool, } impl ReconciledTally { - /// `true` when nothing needs attention (no unexpected-fail / XPASS / ran-when-NA). + /// `true` when nothing needs attention: no unexpected-fail / XPASS / + /// ran-when-NA cell, and no failed lane check. const fn ok(&self) -> bool { - self.problems == 0 + self.problems == 0 && !self.lane_failed } const fn status_text(&self) -> &'static str { @@ -592,7 +605,10 @@ fn reconciled_tally( manifest: &PlatformManifest, actual: &std::collections::HashMap, ) -> ReconciledTally { - let mut t = ReconciledTally::default(); + let mut t = ReconciledTally { + lane_failed: manifest.lane_failure.is_some(), + ..ReconciledTally::default() + }; for exp in &manifest.expectations { match CellOutcome::reconcile(&exp.expected, exp.flaky, actual.get(&exp.id).copied()) { CellOutcome::Pass => t.pass += 1, @@ -662,6 +678,12 @@ impl PlatformReport { } } + /// Why the harness's post-run lane check failed this run, from + /// `platform.json`; `None` when it passed or the artifact predates it. + fn lane_failure(&self) -> Option<&str> { + self.manifest.as_ref()?.lane_failure.as_deref() + } + /// The channel that governs this platform's rendering. See /// [`resolve_channel`]. fn effective_channel(&self) -> Option<&str> { @@ -713,9 +735,11 @@ impl PlatformReport { /// Display counts for the summary table, reconciliation-aware so the numbers /// agree with [`Self::status_text`]. Returns `(total, pass, fail, skip, /// xfail)`. With a `platform.json`, `fail` counts only cells needing - /// attention (unexpected-fail / XPASS / ran-when-NA) — a known bug failing as - /// expected lands in `xfail`, not `fail`. Without one, falls back to raw - /// junit stats. + /// attention (unexpected-fail / XPASS / ran-when-NA / no result) — a known + /// bug failing as expected lands in `xfail`, not `fail`. A post-run lane + /// failure (`lane_failure`) has no cell, so such a row can read `FAIL` + /// with a `fail` of 0. Without a `platform.json`, falls back to raw junit + /// stats. const fn display_counts(&self) -> (u32, u32, u32, u32, u32) { if let Some(t) = &self.tally { let total = t.pass + t.xfail + t.skip + t.problems; @@ -767,6 +791,32 @@ pub struct PlatformReports { scenarios: ScenarioReference, } +/// One platform's health as the consolidated report renders it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlatformStatus { + /// Whether the row is green. + pub ok: bool, + /// The Status cell: `PASS`, `FAIL` or `EMPTY`. + pub status: &'static str, + /// The recorded lane-check failure, if any. + pub lane_failure: Option, +} + +/// Load one platform's `report.json` (and its sibling `platform.json`) the way +/// the consolidated report does, and return its status. +/// +/// Public so the harness that writes `platform.json` can test that what it +/// writes is read back as intended, keeping producer and consumer tied. +#[must_use] +pub fn platform_status(label: &str, report_json: &Path) -> PlatformStatus { + let report = PlatformReport::load(label.to_owned(), report_json); + PlatformStatus { + ok: report.ok(), + status: report.status_text(), + lane_failure: report.lane_failure().map(str::to_owned), + } +} + /// Parse every `(label, json_path)` input into a [`PlatformReport`], in input /// order. The one place `report.json`/`platform.json`/`commands.jsonl` are /// read from disk — [`Grid::build`] and [`scenario_reference`] consume the @@ -897,6 +947,237 @@ mod tests { assert_eq!(r.status_text(), "EMPTY"); } + /// The artifact a lane uploads when the harness's PRE-run check refuses to + /// run (a failed capability probe, a declared GPU the probe did not find, + /// an invalid `E2E_EXPECT_GPU`): `run_suite` in + /// `tests/e2e-cucumber/tests/e2e.rs` creates `report.json` empty before it + /// calls `lane_verdict::preflight`, and exits there before writing + /// `platform.json`; on every self-hosted lane that ran its suite, the + /// report upload step (and the WSL lanes' copy-out from the guest) is + /// gated on `always()`, so that empty file is what arrives. That lane must + /// read as needing attention, never green. A POST-run lane-check failure + /// writes `platform.json` with `lane_failure` instead; see the tests below. + /// + /// The report code already behaved this way before the harness could exit + /// early, so this is not a regression test; it pins that behaviour because + /// the early exit now relies on it to keep a refused lane off green in the grid. + #[test] + fn a_lane_that_stopped_before_any_scenario_is_not_ok() { + let dir = tempfile::tempdir().expect("temp dir"); + let report = dir.path().join("report.json"); + std::fs::write(&report, "").expect("write empty report.json"); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(r.manifest.is_none(), "no platform.json was written"); + assert!(!r.ok(), "an empty run must not count as healthy"); + assert_eq!(r.status_text(), "EMPTY"); + } + + /// `platform.json` for a lane whose every resolved scenario was a skip, + /// with or without a recorded post-run lane failure. + fn all_skip_platform(lane_failure: Option<&str>) -> String { + let failure = lane_failure.map_or(String::new(), |reason| { + format!(r#""lane_failure": {},"#, serde_json::json!(reason)) + }); + format!( + r#"{{"platform_slug":"mi300x","capability":{{"effective_serve_engine":"vllm"}},{failure} + "expectations":[{{"id":"serve-a","feature":"Serving","expected":"skip"}}]}}"# + ) + } + + /// The regression: a GPU lane on which every scenario resolved to skip + /// was failed by the harness ("no @requires-gpu (or @requires-multi-gpu) + /// scenario executed"), yet + /// its row read PASS, since a skip cell is not a problem. + #[test] + fn an_all_skip_lane_the_harness_failed_reads_fail_with_its_reason() { + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (1 resolved, none ran)"; + let (_dir, report) = write_platform("[]", &all_skip_platform(Some(reason))); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + assert_eq!(r.lane_failure(), Some(reason)); + assert_eq!( + platform_status("e2e-gpu-report", &report), + PlatformStatus { + ok: false, + status: "FAIL", + lane_failure: Some(reason.to_owned()), + } + ); + } + + /// Unchanged: an all-skip lane the harness did not fail (an undeclared or + /// `E2E_EXPECT_GPU=0` lane) stays green. + #[test] + fn an_all_skip_lane_without_a_lane_failure_still_reads_pass() { + let (dir, report) = write_platform("[]", &all_skip_platform(None)); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert!(r.ok()); + assert_eq!(r.status_text(), "PASS"); + assert_eq!(r.lane_failure(), None); + + // ...and neither rendering shows any lane-failure element. (The legend + // and the explanation paragraph name the heading in prose, so the + // checks are on the elements themselves.) + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert!(!md.contains("### Needs attention: platforms"), "{md}"); + assert!(!md.contains("**Lane check failed**"), "{md}"); + let html_out = dir.path().join("out.html"); + html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default()) + .expect("html written"); + let html = std::fs::read_to_string(&html_out).expect("html read"); + assert!( + !html.contains("

Needs attention: platforms

"), + "{html}" + ); + assert!( + html.contains(r#"PASS"#), + "the Status cell carries no tooltip:\n{html}" + ); + assert!(!html.contains(r#"

"#), "{html}"); + } + + /// A `-n` filter that matched nothing: no expectations were resolved and + /// nothing ran, and the harness recorded "0 scenarios executed". + #[test] + fn a_zero_expectation_lane_reads_fail_only_with_a_recorded_failure() { + let failed = + r#"{"platform_slug":"mock","lane_failure":"0 scenarios executed","expectations":[]}"#; + let (_dir, report) = write_platform("[]", failed); + let r = PlatformReport::load("e2e-report".into(), &report); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + + let (_dir2, report) = write_platform("[]", r#"{"platform_slug":"mock","expectations":[]}"#); + let r = PlatformReport::load("e2e-report".into(), &report); + assert!(r.ok(), "no recorded failure keeps today's reading"); + assert_eq!(r.status_text(), "PASS"); + } + + /// Both renderings name the reason, not just a FAIL. + #[test] + fn a_recorded_lane_failure_is_named_in_the_summary_and_the_html() { + // A newline, markup characters and a quote, so both the markdown's + // newline folding and the HTML escaping are exercised. + let reason = "0 scenarios executed\nsee & \"tail\""; + let html_reason = "0 scenarios executed\nsee <stderr> & "tail""; + let (dir, report) = write_platform("[]", &all_skip_platform(Some(reason))); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + + let md = markdown::consolidated_summary_markdown(&bundle); + assert!( + md.contains( + "\n### Needs attention: platforms\n\n- **Lane check failed** in _MI300X \ + Linux_: 0 scenarios executed see & \"tail\"\n" + ), + "{md}" + ); + // A CRLF, or any run of whitespace, folds to one space too. Defensive: + // no current producer of `lane_failure` emits CR or LF. + let (_crlf_dir, crlf_report) = + write_platform("[]", &all_skip_platform(Some("first\r\n second"))); + let crlf_md = markdown::consolidated_summary_markdown(&load_platform_reports(&[( + "e2e-gpu-report".to_owned(), + crlf_report, + )])); + assert!(crlf_md.contains("Linux_: first second\n"), "{crlf_md}"); + + let html_out = dir.path().join("out.html"); + html::generate_consolidated(&bundle, &html_out, &html::RunMeta::default()) + .expect("html written"); + let html = std::fs::read_to_string(&html_out).expect("html read"); + // The Status cell's tooltip. + assert!( + html.contains(&format!( + r#"FAIL"# + )), + "status tooltip:\n{html}" + ); + // The platform section's paragraph. + assert!( + html.contains(&format!( + r#"

Lane check failed: {html_reason}

"# + )), + "lane failure paragraph:\n{html}" + ); + // The platform-level attention list. + assert!( + html.contains(&format!( + r"

Needs attention: platforms

  • Lane check failed in MI300X Linux: {html_reason}
" + )), + "attention list:\n{html}" + ); + // ...styled, so it reads as a failure. + assert!( + html.contains(".lane-failure {"), + "CSS rule for .lane-failure" + ); + // The legend says where the reason is. + assert!( + html.contains( + "or when the harness failed the lane after the run (for example, no scenario \ + executed); the reason is listed under \u{201c}Needs attention: platforms\u{201d} \ + and in the platform's section." + ), + "legend:\n{html}" + ); + } + + /// A lane on which scenarios ran and passed, but which the harness still + /// failed (on a GPU lane, no GPU scenario among them): its row + /// reads FAIL with a Fail count of 0, since no cell failed. Rows of exactly + /// this shape are what `total_e2e_failures` in xtask counts as failures. + #[test] + fn a_lane_that_ran_scenarios_and_was_failed_reads_fail_with_zero_failed_cells() { + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (3 resolved, none ran)"; + let platform = format!( + r#"{{"platform_slug":"mi300x","lane_failure":{},"expectations":[ + {{"id":"examine-a","feature":"F","expected":"pass"}}]}}"#, + serde_json::json!(reason) + ); + let (_dir, report) = write_platform( + &feature_json(&[(&["id:examine-a"], &["passed"])]), + &platform, + ); + let r = PlatformReport::load("e2e-gpu-report".into(), &report); + assert_eq!(r.display_counts(), (1, 1, 0, 0, 0)); + assert!(!r.ok()); + assert_eq!(r.status_text(), "FAIL"); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert!( + md.contains("| MI300X | Linux | 1 | 1 | 0 | 0 | n/a | FAIL |"), + "{md}" + ); + assert!(md.contains("| **Total** | | 1 | 1 | 0 | 0 | 0 | |"), "{md}"); + } + + /// Both attention lists can appear in one summary, under different + /// headings, each once. + #[test] + fn the_two_attention_lists_have_distinct_headings() { + let platform = r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed", + "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#; + let (_dir, report) = + write_platform(&feature_json(&[(&["id:examine-a"], &["failed"])]), platform); + let bundle = load_platform_reports(&[("e2e-gpu-report".to_owned(), report)]); + let md = markdown::consolidated_summary_markdown(&bundle); + assert_eq!( + md.matches("### Needs attention: platforms\n").count(), + 1, + "{md}" + ); + assert_eq!( + md.matches("### Needs attention: scenarios\n").count(), + 1, + "{md}" + ); + assert_eq!(md.matches("### Needs attention\n").count(), 0, "{md}"); + } + #[test] fn rendering_uses_the_bundle_not_the_artifacts_on_disk() { // The render functions must work from what `load_platform_reports` @@ -1765,8 +2046,8 @@ mod tests { #[test] fn renderers_take_the_pr_gating_phrases_from_the_shared_constants() { // What this proves, and what it does not. The rendered text is pinned - // verbatim by `summary_explanation_paragraphs_are_unchanged` (markdown) - // and `legend_mock_and_other_platform_items_are_unchanged` (HTML), and + // verbatim by `summary_explanation_paragraphs_are_pinned` (markdown) + // and `legend_mock_and_other_platform_items_are_pinned` (HTML), and // each of those also asserts that the rendered output contains // `MOCK_GATES_PR` and `OTHER_LANES_NON_BLOCKING`; together they catch a // renderer that hardcodes the words while a constant's value moves on. diff --git a/crates/e2e-report/src/consolidated/sidecars.rs b/crates/e2e-report/src/consolidated/sidecars.rs index b1ff32386..28de5088e 100644 --- a/crates/e2e-report/src/consolidated/sidecars.rs +++ b/crates/e2e-report/src/consolidated/sidecars.rs @@ -54,6 +54,10 @@ pub(super) struct PlatformManifest { /// Component versions (OS/ROCm/vLLM/lemonade); absent in older artifacts. #[serde(default)] pub(super) versions: PlatformVersions, + /// Why the harness's post-run lane check failed the run (for example "0 + /// scenarios executed"); absent when it passed, and in older artifacts. + #[serde(default)] + pub(super) lane_failure: Option, #[serde(default)] pub(super) expectations: Vec, } diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 3eaae3947..448e9d394 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -17,8 +17,8 @@ mod single_report; pub use consolidated::html::{RunMeta, generate_consolidated}; pub use consolidated::markdown::consolidated_summary_markdown; pub use consolidated::{ - ADVISORY_QUEUE_MARKER, PlatformReports, artifact_name_for_platform_slug, load_platform_reports, - queue_artifact_name, queue_lane_gpu, + ADVISORY_QUEUE_MARKER, PlatformReports, PlatformStatus, artifact_name_for_platform_slug, + load_platform_reports, platform_status, queue_artifact_name, queue_lane_gpu, }; pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id}; pub use single_report::generate; @@ -36,3 +36,18 @@ pub use single_report::generate; /// typo previously would not fail to compile or fail a test, silently turning /// `@requires-oom-fault-injection` into a skip (green suite, zero coverage). pub const OOM_FAULT_INJECTION_ENV: &str = "ROCM_E2E_OOM_FAULT_INJECTION"; + +/// Environment variable in which a CI lane declares whether it has a GPU. +/// +/// `1` means the lane runs on GPU hardware, so the `e2e-cucumber` harness fails +/// before any scenario runs when its capability probe finds no usable AMD GPU, +/// and after the run when no `@requires-gpu` (or `@requires-multi-gpu`) +/// scenario executed. `0` means the lane deliberately has no usable GPU (the +/// WSL lanes, whose distro is left without ROCm's WSL driver bridge). Unset or +/// empty leaves the lane undeclared, as on the GitHub-hosted lanes and local +/// runs; any other value is an error. +/// +/// Shared by the harness, which reads it, and xtask's workflow contract test, +/// which checks that every self-hosted E2E lane sets it, so the two cannot name +/// different variables. +pub const EXPECT_GPU_ENV: &str = "E2E_EXPECT_GPU"; diff --git a/docs/architecture.md b/docs/architecture.md index b929c3165..b30cf8fe8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -634,7 +634,7 @@ HTML and markdown reporting for the cucumber E2E suite, consumed only by `xtask` | [`lib.rs`](../crates/e2e-report/src/lib.rs) | Crate root: module declarations and the public API re-exports. | flat | | [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) | The CLI command-coverage audit table: which `rocm` commands a platform's run exercised and whether they passed. | domain | | [`components.rs`](../crates/e2e-report/src/components.rs) | Shared maud HTML fragments plus the CSS and timestamp helpers both report generators use. | flat | -| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, plus the report text both renderers share. | domain | +| [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) | The `PlatformReport` model, the reconciled scenario by platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, `platform_status` (one platform's status, public so the harness can test the `platform.json` it writes), plus the report text both renderers share. | domain | | [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) | Artifact-name identity: parses an artifact or directory name into platform, OS and channel, and owns the platform-label rule. | domain | | [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) | The HTML renderer for the consolidated report. It does not render `PlatformVersions::provenance()`: the GFX target and package source appear only in the Markdown output (the wiki page and the step summary that shares its renderer). | domain | | [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) | The markdown renderer, including the table piped into `$GITHUB_STEP_SUMMARY`. | domain | @@ -788,7 +788,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | --- | --- | --- | | [`src/lib.rs`](../tests/e2e-cucumber/src/lib.rs) | Library root for the test targets and the demo binary: declares the helper modules, re-exports `e2e-report` as `report`, and holds small shared helpers such as `cli_failure_report`. | flat | | [`src/binary_path.rs`](../tests/e2e-cucumber/src/binary_path.rs) | Pins a relative `ROCM_CLI_BINARY` path before a scenario runs the child from another directory, and leaves a bare name for `PATH` to resolve. | flat | -| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | +| [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, records why either could not be read (`ProbeError`: not started, or could not be waited on, non-zero exit, no output, still running at its deadline (killed and reaped), or output left open after exit), keeping the stderr read so far, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | | [`src/chat_request.rs`](../tests/e2e-cucumber/src/chat_request.rs) | The chat round trip against a served endpoint (model discovery, then one chat completion), run on its own task so its timeout measures the server. | flat | | [`src/dash_screen.rs`](../tests/e2e-cucumber/src/dash_screen.rs) | Pure matching over captured dashboard screens and config files, so the logic the dashboard scenarios depend on has unit tests. | flat | | [`src/device_memory.rs`](../tests/e2e-cucumber/src/device_memory.rs) | Which device memory pool (VRAM, or GTT-backed memory on an APU) the drain-wait between serve scenarios judges, how it is sampled, and the free-memory floor. | flat | @@ -800,6 +800,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat | | [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat | | [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat | +| [`src/lane_verdict.rs`](../tests/e2e-cucumber/src/lane_verdict.rs) | Lane verdict: fails a run before any scenario when the probe failed or an `E2E_EXPECT_GPU=1` lane has no usable GPU, and after the run when nothing (or, on such a lane, no GPU scenario: `@requires-gpu` or `@requires-multi-gpu`) executed. Also holds the decisions [`tests/e2e.rs`](../tests/e2e-cucumber/tests/e2e.rs) takes around those checks, each unit-tested: reading the declaration, counting GPU scenarios, building the `platform.json` manifest that records a lane failure (`finish_run`), the final exit, and the exact escaped `::error::`/`::notice::` lines. The harness itself runs the probe and the scenarios, classifies their results, writes the files and exits; that glue has no test. | flat | | [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat | | [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat | | [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat | diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 76834f785..6c8de6387 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -156,13 +156,13 @@ scenario to pass / xfail / skip for that host from its `@id` and `@requires-*` tags, a capability probe, and `expectations.toml` — there is no separate tier flag or tag filter to maintain. -| Job | Workflow | Platform | Runner labels | -|---|---|---|---| -| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | -| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | -| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | -| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | -| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | +| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) | +|---|---|---|---|---| +| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | unset | +| `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | `1` | +| `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | `1` | +| `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `1` | +| `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | `0` | On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` is skipped; it still runs on `push` and dispatch. Over 7 days 35% of its merge-queue jobs (98 of 281) @@ -171,6 +171,17 @@ not wait for it. On a PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo pool). +The last column is the lane's declaration of whether it has a GPU. On a lane +that declares `1`, the harness fails before any scenario runs if its capability +probe finds no usable AMD GPU, and after the run if no `@requires-gpu` (or +`@requires-multi-gpu`) scenario executed; on every lane it fails if the probe +could not read the binary under test or if no scenario executed at all. `nightly.yml`'s self-hosted lanes +declare the same way (`0` only for `e2e-wsl-nightly`), as does every lane of +[`run-e2e-lane.sh`](../scripts/run-e2e-lane.sh). `every_self_hosted_lane_declares_whether_it_has_a_gpu` +in [`workflow_contract.rs`](../xtask/src/workflow_contract.rs) fails when a +self-hosted lane declares neither. See the [E2E README](../tests/e2e-cucumber/README.md)'s +"Lane verdict". + `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact names, so that lane proves the nightly package channel boots. It no longer runs @@ -260,8 +271,8 @@ resolve to skip here, and known bugs resolve to xfail from The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, and `e2e-wsl`) run on AMD GPU systems, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving -scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as -described below. +scenarios that the mock job cannot. The WSL lane declares no usable GPU +(`E2E_EXPECT_GPU=0`) and its GPU preflight is advisory, as described below. `e2e-wsl` runs on the DevLab Dispatch pool. Its WSL2/Ubuntu-24.04 distro is built fresh inside every job (see above), so there is nothing of its own for @@ -275,9 +286,13 @@ preflight, toolchain bootstrap, shared-runtime pre-warm, then the full suite with no hand filtering. It covers WSL host detection, the Windows-to-WSL execution boundary, and whatever GPU access WSL exposes on that machine. The GPU preflight is advisory here precisely because GPU-on-WSL is -what the lane is proving out: where it is unavailable the capability probe -resolves those scenarios to not-applicable and the rest of the suite still -runs. Scenarios the product deliberately routes around on WSL carry +what the lane is proving out. The lane declares `E2E_EXPECT_GPU=0` (forwarded +into the guest through `WSLENV`), because its distro is deliberately left +without the driver bridge (see "What the WSL distro needs"): the capability +probe resolves `@requires-gpu` scenarios to not-applicable, the harness prints +a note saying so rather than failing, and the rest of the suite still runs. If +the probe finds a usable GPU anyway, the harness notes that the declaration is +stale and runs the GPU scenarios. Scenarios the product deliberately routes around on WSL carry `@requires-bare-metal`; scenarios whose premise *is* a WSL host carry `@requires-wsl`, and this is the only lane that runs them. @@ -302,7 +317,9 @@ driver scenarios assert on the plan that command produces — a lane that pre-installed it would hand those scenarios a host already in the state the command under test exists to reach. That costs no coverage today, because `rocm-smi` is absent too, so `@requires-gpu` scenarios resolve to skip either -way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`). +way (see `e2e-wsl`'s own step comments in `e2e-selfhosted.yml`). This is why +the lane declares `E2E_EXPECT_GPU=0`; a lane that provisions the bridge +declares `1` instead. GPU coverage additionally needs ROCm's WSL passthrough to be complete — `/dev/dxg` and dxcore alone are not enough, `librocdxg.so` must be present and @@ -661,6 +678,12 @@ skipped itself as stale is the exception); on a `continue-on-error: true` lane that is the only red signal for such a failure. The nightly report job applies the same check to every nightly lane. +A lane the harness failed after its run (no scenario executed, or no GPU +scenario, `@requires-gpu` or `@requires-multi-gpu`, on an `E2E_EXPECT_GPU=1` +lane) records the reason in its `platform.json`, and its row reads FAIL with the reason under "Needs +attention: platforms" (step summary and HTML report alike), even when every +scenario it resolved was a skip. + ### Timeouts on the Strix lanes Every Strix Halo lane raises two budgets rather than letting a slow host read diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index e4fae4508..c57d35440 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -125,7 +125,11 @@ limit (see the run's inputs above). Each channel starts with the GPU preflight, channel's job in `nightly.yml` does, so a model server the previous channel's suite left on the GPU is reclaimed first; the lane log shows it. Both always run, so a failing channel does not hide the other, and the lane fails if either -did. +did. The default command also declares that every lane has a GPU +(`E2E_EXPECT_GPU`, default `1`), so a channel whose suite finds no usable AMD +GPU, or runs no GPU scenario (`@requires-gpu` or `@requires-multi-gpu`), fails +instead of passing with every GPU scenario skipped; see "Lane verdict" in the +[E2E README](../tests/e2e-cucumber/README.md). A lane starts cold, so its first steps fetch from the network: the build packages, the rustup installer, the toolchain and the crates for the host diff --git a/docs/testing.md b/docs/testing.md index f2e113962..7d60b34f5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -951,6 +951,45 @@ export ROCM_TEST_APK_REPOS="--repository http://dl-cdn.alpinelinux.org/alpine/v3 --repository http://dl-cdn.alpinelinux.org/alpine/v3.20/community" ``` +## E2E runs that test nothing fail + +`cargo xtask e2e` fails a run whose result could not mean anything, instead of +reporting it green: + +- before any scenario runs, when the capability probe could not read the binary + under test: `rocm examine` or `rocm engines list` could not be started (check + `ROCM_CLI_BINARY`), or could not be waited on, exited non-zero, printed + nothing, was still running after `E2E_PROBE_TIMEOUT_SECS` (whole seconds ≥ 1; + anything else uses 120), or exited but left its output open. The post-run + version probe (`collect_versions`) uses the same deadline but fails quietly, + only blanking the version columns, and does not refuse the run; +- after the run, when no scenario executed, which includes a + `cargo xtask e2e -- -n ` whose regex matches no scenario name. The + reason is written into `platform.json`, so `cargo xtask e2e-report` shows that + lane as FAIL with the reason under "Needs attention: platforms", in both its + markdown summary and its HTML report. + +`E2E_EXPECT_GPU` declares whether the host has a GPU. With `1`, the run also +fails before any scenario if the probe found no usable AMD GPU, and after the +run if no `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU +scenarios the run's own configuration leaves out (`@nightly`, `@merge-queue`, +or anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved +but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and no +GPU scenario selected fails by design. With `0`, it only notes that GPU +scenarios are not applicable (and that the declaration is stale if a GPU is +found). Unset, as on a local run, it makes no GPU claim. Every self-hosted CI +lane sets it; see "Lane verdict" in the [E2E README](../tests/e2e-cucumber/README.md) +and the Platforms table in [ci-hardware-testing.md](ci-hardware-testing.md). The +checks are unit-tested in [lane_verdict.rs](../tests/e2e-cucumber/src/lane_verdict.rs) +and, for each way a probe can fail, in +[capability.rs](../tests/e2e-cucumber/src/capability.rs): + +```bash +cargo test -p e2e-cucumber --lib lane_verdict +cargo test -p e2e-cucumber --lib capability::tests +cargo test -p xtask every_self_hosted_lane_declares_whether_it_has_a_gpu +``` + ## CI test selection On pull requests, CI does not test the whole workspace. The `test` job runs @@ -4377,7 +4416,8 @@ to make the host match: masking every device with `HIP_VISIBLE_DEVICES=` makes the harness and the product agree the host has no usable GPU, which is what the mock lane has.) Note also that `-n` matches the scenario *name* (`serve-28 - Reuse succeeds on a GPU-less host`), not the `@id:` tag, so passing an id to it -silently selects nothing. +selects nothing and the run fails with `0 scenarios executed` (see +[E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)). - serve-28, `@id:serve-reuse-exempt-from-no-gpu-refusal` — a `serve` that matches an already-running vLLM service on a GPU-less host is not refused: it @@ -4528,7 +4568,8 @@ for the wrong reason instead of being reported as skipped. Do not select it by name on that route, nor any other `@requires-oom-fault-injection` scenario (`@id:serve-smoke-failure-summary` carries the same tag). Note also that `-n` matches the scenario *name* (`serve-41 - A launch that runs out of GPU memory ...`), not the `@id:` tag, so -passing an id to it silently selects nothing. +passing an id to it selects nothing and the run fails with +`0 scenarios executed` (see [E2E runs that test nothing fail](#e2e-runs-that-test-nothing-fail)). - serve-41, `@id:serve-oom-launch-memory-guidance` — a launch that runs out of GPU memory blames this launch, names the knobs, and quotes the engine's own diff --git a/scripts/run-e2e-lane.sh b/scripts/run-e2e-lane.sh index 68e8569f8..4e2ee32a3 100755 --- a/scripts/run-e2e-lane.sh +++ b/scripts/run-e2e-lane.sh @@ -22,6 +22,11 @@ # both (default "release nightly") # E2E_LANE_WORK where the build and runtimes go (default /var/tmp/e2e-lane) # LANE_OUT where reported files go (default /out) +# E2E_EXPECT_GPU whether the lane has a GPU (default 1: every queue lane is +# given one); the suite fails rather than passing with every +# GPU scenario skipped. Set but empty, it leaves the lane +# undeclared, which switches that check off. See +# tests/e2e-cucumber/README.md. set -euo pipefail @@ -39,6 +44,7 @@ if [[ -n "${CARGO_BUILD_TARGET:-}" ]]; then fi export E2E_INCLUDE_NIGHTLY="${E2E_INCLUDE_NIGHTLY-1}" +export E2E_EXPECT_GPU="${E2E_EXPECT_GPU-1}" export E2E_SERVE_TIMEOUT_SECS="${E2E_SERVE_TIMEOUT_SECS:-300}" # The channels are checked before anything is built, so a typo costs nothing. diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 3efc130f5..73050d327 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -90,10 +90,11 @@ shlex.workspace = true # itself, and `xtask check-crate-edges` exempts dev edges by design — so keeping # it here needs no allowlist entry. # -# For the same reason the library's `device_memory::bounded_stdout` cannot call -# `rocm_core::process::run_with_timeout`: it repeats that helper's spawn, poll, -# kill and reap steps on `std` alone, deliberately, and a normal dependency -# would be a new edge to review. A change to either one should look at the other. +# For the same reason the library's `device_memory::bounded_stdout` and +# `capability::output_within` cannot call `rocm_core::process::run_with_timeout`: +# each repeats that helper's spawn, poll, kill and reap steps on `std` alone, +# deliberately, and a normal dependency would be a new edge to review. A change +# to any one of these copies should look at the others. rocm-core = { path = "../../crates/rocm-core" } # `test-util` pauses tokio's clock in the retry loops' unit tests # (`#[tokio::test(start_paused = true)]`), so their timing assertions read a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d871ab21d..c68381db7 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -64,7 +64,7 @@ runs the suite. Extra arguments after `--` are forwarded to the cucumber CLI. # All features: cargo xtask e2e -# Filter by scenario name: +# Filter by scenario name (bypasses capability gating; see below): cargo xtask e2e -- -n "model name" # Stop on first failure: @@ -101,6 +101,57 @@ E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e | `ROCM_CLI_CONFIG_DIR` | (temp dir) | Isolated config directory per scenario | | `ROCM_CLI_DATA_DIR` | (temp dir) | Isolated data directory per scenario | | `ROCM_CLI_CACHE_DIR` | (temp dir) | Isolated cache directory per scenario | +| `E2E_PROBE_TIMEOUT_SECS` | 120 | How long each capability-probe command (`rocm examine`, `rocm engines list`) may run before it is killed and the run is refused: whole seconds ≥ 1; anything else uses 120. The post-run version probe (`collect_versions`) uses the same deadline but fails quietly, only blanking the version columns, and does not refuse the run | +| `E2E_EXPECT_GPU` | unset (undeclared) | Whether the lane has a GPU: `1` = GPU expected, `0` = deliberately none, unset or empty = no claim; any other value fails the run. Set on every self-hosted CI lane; see "Lane verdict" below | + +### Lane verdict + +Scenarios that resolve to skip never reach `report.json`, so scenario results +alone cannot tell a lane that tested its GPU from one on which every GPU +scenario was skipped. The harness checks the run itself, before and after the +scenarios: + +- **Before any scenario runs**, it fails when the capability probe could not + read the binary under test, naming the command and saying why: + `rocm examine` or `rocm engines list` could not be started (with the error), + or could not be waited on, exited non-zero (with the exit + status and the tail of its stderr), printed nothing, was still running after + `E2E_PROBE_TIMEOUT_SECS` (default 120; it is killed, and the message gives + the deadline and the stderr tail so far), or exited but left its output + open, held by a process it started (with the stderr tail so far). On a lane + that sets `E2E_EXPECT_GPU=1` it also fails when the probe found no usable AMD + GPU, naming the detected gfx target, WSL flag, driver status and visibility + mask it decided from. A run refused here writes no `platform.json`; its + `report.json` is already there, empty, because `run_suite` in + [`tests/e2e.rs`](tests/e2e.rs) creates it before calling + `lane_verdict::preflight`, and on every self-hosted lane that ran its suite + the results upload step is gated on `always()` (`e2e-gpu`'s also skips a + run that found itself stale, which never reaches the suite), so the + consolidated report shows the lane as EMPTY. +- **After the run**, it fails when no scenario executed (for example a `-n` + filter that matches nothing), and on an `E2E_EXPECT_GPU=1` lane when no + `@requires-gpu` (or `@requires-multi-gpu`) scenario executed. GPU scenarios + the run's own configuration leaves out (`@nightly`, `@merge-queue`, or + anything outside an `E2E_ONLY_LIFECYCLE` selection) still count as resolved + but never as executed, so `E2E_EXPECT_GPU=1` with `E2E_ONLY_LIFECYCLE=1` and + no GPU scenario selected fails by design. The reason is also recorded as + `lane_failure` in `platform.json`, so the consolidated report shows the lane + as FAIL, even when every scenario it resolved was a skip, and gives the reason + under "Needs attention: platforms" (in both the step summary and the HTML + report) and in the lane's section of the HTML report. + +`E2E_EXPECT_GPU=0` is for a lane that deliberately has no usable GPU, such as +the WSL lanes: the harness prints a note that `@requires-gpu` scenarios are not +applicable, and a note that the declaration is stale if the probe finds a GPU +anyway; neither fails the run. Both checks are in +[`src/lane_verdict.rs`](src/lane_verdict.rs). + +A cucumber `-n`/`--name` or `--tags` filter on the command line replaces the +harness's own scenario filter, so the scenarios it selects run without the +capability gating, the opt-in sets and the expectation matrix described below, +and the after-run GPU check does not apply. Use it to run a known scenario on a +host that can satisfy it; use `E2E_ONLY_LIFECYCLE` with `E2E_ALSO_RUN_IDS` to +select scenarios with the gating kept. ## Tags and per-scenario expectations @@ -137,7 +188,7 @@ Scenarios carry stable-id and capability tags: | Tag | Meaning | |---|---| | `@id:-` | Stable scenario id, prefixed with its feature's key. Keys the expectation matrix and the report grid; every scenario has one. | -| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. | +| `@requires-gpu` | Needs a usable AMD GPU. Resolves to **skip** (n/a) on a host with none — the mock job, or a WSL host whose ROCm passthrough is incomplete (`driver_status` other than `wsl_rocdxg_ready`), where the gfx target is reported but unreachable. On a lane that sets `E2E_EXPECT_GPU=1` a host with none fails the run instead; see "Lane verdict" above. | | `@requires-multi-gpu` | Premise is a host with **more than one** AMD GPU present. Resolves to **skip** wherever the probed device count is not known to exceed one: a single-GPU host such as Strix Halo, and any host whose count could not be read (non-Linux, or WSL, which exposes neither a KFD topology nor an amdgpu DRM card). `@requires-gpu` cannot express this — it asks only whether *a* device is usable — so scenarios carry both. The count comes from the capability probe's `amd_gpu_count`, which mirrors the product's KFD+DRM rule rather than counting `amd-smi list` — the gate must ask the same "how many devices are present" question `--gpu` validation is built on, and `amd-smi` is only a best-effort fallback that may be absent or disagree. | | `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. | | `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. | diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index d8857f43c..1b0cf7f33 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -16,7 +16,7 @@ //! changes engine support, so the unit tests below guard it. //! //! The AMD GPU *count* is re-implemented for the same reason (see -//! [`probe_amd_gpu_count`]): no `rocm` command reports one, so a scenario whose +//! `probe_amd_gpu_count`): no `rocm` command reports one, so a scenario whose //! premise is a multi-GPU host has nothing else to gate on. //! //! `examine`'s `default_engine` now reports the host's real engine rather than the @@ -28,10 +28,12 @@ use std::sync::OnceLock; -/// Name of the `rocm` binary under test (mirrors the test binary's -/// `rocm_binary()`): `ROCM_CLI_BINARY` or plain `rocm`. -fn rocm_binary() -> String { - std::env::var("ROCM_CLI_BINARY").unwrap_or_else(|_| "rocm".to_string()) +/// Name of the `rocm` binary under test: `ROCM_CLI_BINARY` or plain `rocm`, +/// read through `env` so a test can supply it without a process-wide variable. +/// Like the test binary's `rocm_binary()`, except that a value that is not +/// UTF-8 is used as given here rather than replaced by `rocm`. +fn rocm_binary(env: &impl Fn(&str) -> Option) -> std::ffi::OsString { + env("ROCM_CLI_BINARY").unwrap_or_else(|| "rocm".into()) } /// Whether a container runtime is usable here. @@ -132,11 +134,11 @@ pub struct HostCapability { pub gfx_target: Option, /// Whether an AMD GPU is present AND usable here, so `@requires-gpu` /// scenarios can run. A real `detected_gfx_target`, plus a ready ROCm - /// driver path on WSL — see [`host_has_usable_gpu`]. + /// driver path on WSL — see `host_has_usable_gpu`. pub has_amd_gpu: bool, /// How many AMD GPUs are PRESENT on this host, before any visibility mask. /// `None` where the count could not be determined (non-Linux, unreadable - /// sysfs). Gates `@requires-multi-gpu`; see [`probe_amd_gpu_count`]. + /// sysfs). Gates `@requires-multi-gpu`; see `probe_amd_gpu_count`. pub amd_gpu_count: Option, /// Engine adapters the binary reports as present. Both builtins are always /// "built-in", so this is NOT the same as "can start here" — use @@ -154,16 +156,27 @@ pub struct HostCapability { /// exposes it), so `xtask e2e` reports it via the [`OOM_FAULT_INJECTION_ENV`] /// environment variable — set only when xtask built the binary itself with /// the feature, and absent for a prebuilt `ROCM_CLI_BINARY` (the self-hosted - /// lanes' shipping release build). See [`probe_host_capability`]. + /// lanes' shipping release build). See `probe_host_capability`. pub oom_fault_injection: bool, /// Whether two file names differing only in letter case are two files under /// the temp root the scenarios' isolated roots are created in. Gates - /// `@requires-case-sensitive-fs`; see [`probe_case_sensitive_fs`]. + /// `@requires-case-sensitive-fs`; see `probe_case_sensitive_fs`. pub case_sensitive_fs: bool, /// Whether a directory made read-only refuses new files to the user /// running the suite. Gates `@requires-dir-permissions`; see - /// [`probe_dir_permissions_bind`]. + /// `probe_dir_permissions_bind`. pub dir_permissions_bind: bool, + /// `examine`'s `driver_status:` line, empty when it printed none. Only + /// decides GPU usability on WSL (see `host_has_usable_gpu`); kept so a + /// lane whose GPU went missing can say why. + pub driver_status: String, + /// Why the probe could not read the binary under test, if it could not: + /// the first failure of `rocm examine` or `rocm engines list` (see + /// [`ProbeError`]). When set, the rest of this struct holds the same + /// no-GPU values an empty `examine` would give, so callers never panic, and + /// the harness refuses to run (see `lane_verdict::preflight_verdict`). + #[serde(skip_serializing_if = "Option::is_none")] + pub probe_error: Option, } /// Environment variable `xtask e2e` sets to `1` when it compiled the binary @@ -258,7 +271,11 @@ pub fn collect_versions(runtimes_dir: Option<&std::path::Path>) -> PlatformVersi // OS distro from `examine` — ALWAYS collected (every platform, incl. mock and // hosts without a shared runtime dir). Isolated throwaway root reads the host. if let Ok(tmp) = tempfile::TempDir::with_prefix("rocm-e2e-ver-") { - let examine = run_probe(tmp.path(), &["examine"]); + // Best-effort, unlike the capability probe: a failed `examine` here + // only leaves the report heading's OS blank. The capability probe ran + // the same command before any scenario and already refused the run if + // it failed. + let examine = run_probe(tmp.path(), &["examine"]).unwrap_or_default(); v.os = examine .lines() .find_map(|l| l.trim().strip_prefix("distro:")) @@ -486,6 +503,29 @@ fn probe_host_capability() -> HostCapability { // what the scenarios see. let examine = run_probe(root, &["examine"]); let engines = run_probe(root, &["engines", "list"]); + capability_from_probes(examine, engines, root) +} + +/// Build a [`HostCapability`] from the two probes' results; `root` is the +/// throwaway directory the filesystem probes write under. +/// +/// Split from [`probe_host_capability`] so a test can hand it a failed probe +/// without a binary to run. +fn capability_from_probes( + examine: Result, + engines: Result, + root: &std::path::Path, +) -> HostCapability { + // The first failure is the one reported; either one means the facts below + // were not read from the binary, so they fall back to the no-GPU values an + // empty dump parses to. + let probe_error = examine + .as_ref() + .err() + .or_else(|| engines.as_ref().err()) + .map(ToString::to_string); + let examine = examine.unwrap_or_default(); + let engines = engines.unwrap_or_default(); let ExamineFacts { os_family, @@ -516,6 +556,8 @@ fn probe_host_capability() -> HostCapability { oom_fault_injection, case_sensitive_fs, dir_permissions_bind, + driver_status, + probe_error, } } @@ -581,20 +623,455 @@ fn probe_case_sensitive_fs(dir: &std::path::Path) -> bool { probe().unwrap_or(false) } -/// Run `rocm ` with an isolated config/data/cache root, returning stdout -/// (empty string on any failure — the probe must never panic the suite). -fn run_probe(root: &std::path::Path, args: &[&str]) -> String { - let mut cmd = std::process::Command::new(rocm_binary()); +/// Why a probe of the binary under test produced nothing the harness can read. +/// +/// Every variant is fatal to the run unless the caller ignores it, as +/// `collect_versions` does: an unreadable probe parses as a host with no GPU, +/// so every capability-gated scenario would resolve to skip and the run would +/// pass without testing them. `command` is the binary and arguments as +/// run, so the message names exactly what failed. +#[derive(Debug)] +pub enum ProbeError { + /// The binary could not be started at all (wrong path, not executable). + Spawn { + command: String, + source: std::io::Error, + }, + /// It ran and exited non-zero: the CLI exits 0 for every finding, so this + /// means it could not examine the host at all. + Exit { + command: String, + status: std::process::ExitStatus, + stderr_tail: String, + }, + /// It exited 0 but printed nothing on stdout. + EmptyOutput { command: String }, + /// It was still running when the deadline ([`PROBE_TIMEOUT_ENV`]) passed, + /// so it was killed and reaped. `stderr_tail` is what it had written to + /// stderr by then. + Timeout { + command: String, + after: std::time::Duration, + stderr_tail: String, + }, + /// It started, but waiting for it to exit failed. + Wait { + command: String, + source: std::io::Error, + }, + /// It exited 0, but its stdout was still open `grace` later: a process it + /// started is holding the pipe, so what it printed cannot be read in full. + OutputNotClosed { + command: String, + grace: std::time::Duration, + stderr_tail: String, + }, +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Spawn { command, source } => write!( + f, + "could not start `{command}`: {source}. Set ROCM_CLI_BINARY to the rocm binary \ + under test, or run the suite through `cargo xtask e2e`, which builds it and \ + sets ROCM_CLI_BINARY" + ), + Self::Exit { + command, + status, + stderr_tail, + } => write!( + f, + "`{command}` exited with {status}\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::Timeout { + command, + after, + stderr_tail, + } => write!( + f, + "`{command}` was still running after {after:?}, so it was killed; set \ + {PROBE_TIMEOUT_ENV} to allow it longer\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::Wait { command, source } => write!( + f, + "`{command}` was started, but waiting for it to exit failed: {source}" + ), + Self::OutputNotClosed { + command, + grace, + stderr_tail, + } => write!( + f, + "`{command}` exited 0, but its stdout was still open {grace:?} later (a \ + process it started may be holding it), so what it printed cannot be read \ + in full\n{}", + crate::section("stderr (last lines)", stderr_tail) + ), + Self::EmptyOutput { command } => { + write!(f, "`{command}` exited 0 but produced no output on stdout") + } + } + } +} + +impl std::error::Error for ProbeError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Spawn { source, .. } | Self::Wait { source, .. } => Some(source), + Self::Exit { .. } + | Self::EmptyOutput { .. } + | Self::Timeout { .. } + | Self::OutputNotClosed { .. } => None, + } + } +} + +/// How many trailing stderr lines a [`ProbeError`] keeps. +const PROBE_STDERR_TAIL_LINES: usize = 20; + +/// The last [`PROBE_STDERR_TAIL_LINES`] lines of a probe's stderr. +fn stderr_tail(stderr: &[u8]) -> String { + let stderr = String::from_utf8_lossy(stderr); + let lines: Vec<&str> = stderr.lines().collect(); + lines[lines.len().saturating_sub(PROBE_STDERR_TAIL_LINES)..].join("\n") +} + +/// Turn one probe's spawn result into its stdout, or the reason it has none. +/// +/// Pure, so each outcome is unit-tested without a binary to run. Stdout that is +/// only whitespace counts as empty: no `examine` field can be parsed from it. +pub fn classify_probe_output( + command: &str, + result: std::io::Result, +) -> Result { + let out = result.map_err(|source| ProbeError::Spawn { + command: command.to_owned(), + source, + })?; + if !out.status.success() { + return Err(ProbeError::Exit { + command: command.to_owned(), + status: out.status, + stderr_tail: stderr_tail(&out.stderr), + }); + } + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + if stdout.trim().is_empty() { + return Err(ProbeError::EmptyOutput { + command: command.to_owned(), + }); + } + Ok(stdout) +} + +/// How a bounded probe run ended (see `output_within`). +#[derive(Debug)] +pub enum Finished { + /// The child exited. `stdout_drained` is false when its stdout was still + /// open after the drain grace; `output` then holds what had been read by + /// then. Stderr is read the same way, but one left open does not fail the + /// probe: it carries only diagnostics. + Exited { + output: std::process::Output, + stdout_drained: bool, + }, + /// The child was still running at the deadline and has been killed and + /// reaped; `stderr` is what it had written by then. + TimedOut { stderr: Vec }, + /// Waiting for the child failed; it has been killed. + WaitFailed(std::io::Error), +} + +/// [`classify_probe_output`] for a bounded run: a timeout and an output pipe +/// left open are errors of their own, never read as "no output". +pub fn classify_finished( + command: &str, + result: std::io::Result, + timeout: std::time::Duration, + grace: std::time::Duration, +) -> Result { + match result { + Err(source) => classify_probe_output(command, Err(source)), + Ok(Finished::TimedOut { stderr }) => Err(ProbeError::Timeout { + command: command.to_owned(), + after: timeout, + stderr_tail: stderr_tail(&stderr), + }), + Ok(Finished::WaitFailed(source)) => Err(ProbeError::Wait { + command: command.to_owned(), + source, + }), + Ok(Finished::Exited { + output, + stdout_drained: false, + }) if output.status.success() => Err(ProbeError::OutputNotClosed { + command: command.to_owned(), + grace, + stderr_tail: stderr_tail(&output.stderr), + }), + Ok(Finished::Exited { output, .. }) => classify_probe_output(command, Ok(output)), + } +} + +/// Run `rocm ` with an isolated config/data/cache root and return its +/// stdout, or why there is none. Never panics: the caller decides whether a +/// failure is fatal. +fn run_probe(root: &std::path::Path, args: &[&str]) -> Result { + run_probe_using(root, args, |name| std::env::var_os(name)) +} + +/// `run_probe` with the environment lookup passed in. +/// +/// The binary comes from `ROCM_CLI_BINARY` and the deadline from +/// [`PROBE_TIMEOUT_ENV`], both read through `env`, so a test can drive both +/// without a process-wide variable. +pub fn run_probe_using( + root: &std::path::Path, + args: &[&str], + env: impl Fn(&str) -> Option, +) -> Result { + run_probe_with( + &rocm_binary(&env), + root, + args, + probe_timeout_from_env(env(PROBE_TIMEOUT_ENV).as_deref()), + ) +} + +/// Environment variable that overrides how long one probe command may run, +/// in whole seconds. +pub const PROBE_TIMEOUT_ENV: &str = "E2E_PROBE_TIMEOUT_SECS"; + +/// How long one probe command may run when [`PROBE_TIMEOUT_ENV`] is unset. +/// +/// `rocm examine` normally answers in seconds; this is generous so a loaded +/// runner does not fail a good probe, while a wedged one still fails the lane +/// with a named reason long before the job's own timeout. +pub const DEFAULT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(2); + +/// The probe deadline from [`PROBE_TIMEOUT_ENV`]'s value. +/// +/// Accepts any whole number of seconds of at least 1, with surrounding +/// whitespace ignored; a value too large to add to the current time is waited +/// on without a deadline (see `output_within`). Anything else (unset, empty, +/// zero, negative, a fraction, a non-number, a number past `u64`, or a value +/// that is not UTF-8) gives [`DEFAULT_PROBE_TIMEOUT`] rather than an error. +#[must_use] +pub fn probe_timeout_from_env(value: Option<&std::ffi::OsStr>) -> std::time::Duration { + value + .and_then(std::ffi::OsStr::to_str) + .and_then(|v| v.trim().parse::().ok()) + .filter(|secs| *secs >= 1) + .map_or(DEFAULT_PROBE_TIMEOUT, std::time::Duration::from_secs) +} + +/// `run_probe` with the binary and deadline passed in, so a test can name +/// them without setting process-wide environment variables. +pub fn run_probe_with( + binary: &std::ffi::OsStr, + root: &std::path::Path, + args: &[&str], + timeout: std::time::Duration, +) -> Result { + run_probe_bounded(binary, root, args, timeout, PROBE_DRAIN_GRACE) +} + +/// [`run_probe_with`] with the drain grace passed in too, for tests. +fn run_probe_bounded( + binary: &std::ffi::OsStr, + root: &std::path::Path, + args: &[&str], + timeout: std::time::Duration, + grace: std::time::Duration, +) -> Result { + let mut cmd = std::process::Command::new(binary); cmd.args(args); cmd.env("ROCM_CLI_CONFIG_DIR", root.join("config")); cmd.env("ROCM_CLI_DATA_DIR", root.join("data")); cmd.env("ROCM_CLI_CACHE_DIR", root.join("cache")); - match cmd.output() { - Ok(out) => String::from_utf8_lossy(&out.stdout).into_owned(), - Err(_) => String::new(), + let command = std::iter::once(binary.to_string_lossy().into_owned()) + .chain(args.iter().map(|a| (*a).to_owned())) + .collect::>() + .join(" "); + classify_finished(&command, output_within(cmd, timeout, grace), timeout, grace) +} + +/// How often [`output_within`] checks whether the child has exited. +const PROBE_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); + +/// How long [`output_within`] waits, after the child exited or was killed, for +/// its output pipes to close. Only a process it started that inherited a pipe +/// and outlives it can hold one open past this. +const PROBE_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(5); + +/// A pipe being read on its own thread into a buffer the caller can read at any +/// time, so bytes read before a timeout or a stuck drain are not lost. +struct PipeReader { + bytes: std::sync::Arc>>, + done: std::sync::mpsc::Receiver<()>, +} + +impl PipeReader { + fn spawn(pipe: Option) -> Self { + let bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let (sender, done) = std::sync::mpsc::channel(); + let sink = std::sync::Arc::clone(&bytes); + std::thread::spawn(move || { + if let Some(mut pipe) = pipe { + let mut chunk = [0u8; 8192]; + loop { + match pipe.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(n) => sink + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend_from_slice(&chunk[..n]), + } + } + } + let _ = sender.send(()); + }); + Self { bytes, done } + } + + /// Wait until the pipe closes or `deadline` passes; whether it closed. + fn wait_closed(&self, deadline: std::time::Instant) -> bool { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + self.done.recv_timeout(left).is_ok() + } + + /// What has been read so far. + fn snapshot(&self) -> Vec { + self.bytes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() } } +/// `command.output()`, bounded by `timeout`: a child still running then is +/// killed and reaped. Either way the output pipes get `grace` to close, and +/// whatever was read by then is kept. +/// +/// std-only, polling `try_wait`, so it behaves the same on Linux and Windows, +/// with each output pipe read on its own thread so a chatty child cannot block +/// on a full pipe. It repeats the spawn, poll, kill and reap steps of +/// `device_memory::bounded_stdout`; a change to the kill or drain behaviour of +/// one should look at the other. It differs in three ways: +/// +/// - it keeps stderr and the exit status, which the probe reports; +/// - it drains after a kill too, where `bounded_stdout` gives up at once, +/// because the stderr written before the deadline is the only evidence of +/// why `rocm examine` hung; +/// - its grace is 5 s (`PROBE_DRAIN_GRACE`) rather than 2 s, because it runs +/// a handful of times per suite: each probe runs once (the capability +/// probes in `probe_host_capability`, cached by `host_capability`, and the +/// version probe in `collect_versions`, whose `rocm examine` repeats the +/// capability probe's), not once per sample, so a generous wait for a +/// loaded runner to hand over the bytes costs little. +/// +/// Only the direct child is killed: a process it started that inherited a pipe +/// can keep that pipe open past the grace. The reader thread for that pipe is +/// then left detached until the pipe closes, and the bytes it had read are +/// used as they are. A leaked thread per stuck probe is accepted in a test +/// harness that runs a handful of probes, each once, and exits. +fn output_within( + mut command: std::process::Command, + timeout: std::time::Duration, + grace: std::time::Duration, +) -> std::io::Result { + use std::process::Stdio; + + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout = PipeReader::spawn(child.stdout.take()); + let stderr = PipeReader::spawn(child.stderr.take()); + + // `None` only for a deadline too far off to represent: wait without one. + let deadline = std::time::Instant::now().checked_add(timeout); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if deadline.is_none_or(|deadline| std::time::Instant::now() < deadline) => { + std::thread::sleep(PROBE_POLL_INTERVAL); + } + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Ok(Finished::WaitFailed(error)); + } + } + }; + let drained_by = std::time::Instant::now() + grace; + let stdout_closed = stdout.wait_closed(drained_by); + // Waited for so the tail is complete when it can be, but not required. + let _ = stderr.wait_closed(drained_by); + Ok(match status { + None => Finished::TimedOut { + stderr: stderr.snapshot(), + }, + Some(status) => Finished::Exited { + output: std::process::Output { + status, + stdout: stdout.snapshot(), + stderr: stderr.snapshot(), + }, + stdout_drained: stdout_closed, + }, + }) +} + +/// One line naming the probe facts that decide GPU usability, for a lane whose +/// declared GPU the probe did not find. +/// +/// Reads the visibility masks from the environment the way +/// `host_has_usable_gpu` does; [`probe_facts_summary_with_mask`] is the pure +/// form. +#[must_use] +pub fn probe_facts_summary(cap: &HostCapability) -> String { + let hip = std::env::var_os("HIP_VISIBLE_DEVICES"); + let rocr = std::env::var_os("ROCR_VISIBLE_DEVICES"); + probe_facts_summary_with_mask(cap, hip.as_deref(), rocr.as_deref()) +} + +/// Pure form of [`probe_facts_summary`]. Names the mask that is in force by the +/// same precedence as `selected_visibility_mask`: HIP wins when set, even +/// empty. +#[must_use] +pub fn probe_facts_summary_with_mask( + cap: &HostCapability, + hip: Option<&std::ffi::OsStr>, + rocr: Option<&std::ffi::OsStr>, +) -> String { + let mask = match (hip, rocr) { + (Some(v), _) => format!("HIP_VISIBLE_DEVICES={:?}", v.to_string_lossy()), + (None, Some(v)) => format!("ROCR_VISIBLE_DEVICES={:?}", v.to_string_lossy()), + (None, None) => "no visibility mask".to_owned(), + }; + let driver_status = if cap.driver_status.is_empty() { + "" + } else { + cap.driver_status.as_str() + }; + format!( + "detected_gfx_target={}, wsl={}, driver_status={driver_status}, {mask}", + cap.gfx_target.as_deref().unwrap_or(""), + cap.is_wsl, + ) +} + /// The `examine` fields the capability probe reads. struct ExamineFacts { os_family: String, @@ -992,6 +1469,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }; assert!(strix.engine_available("lemonade")); // vLLM adapter is "built-in" but cannot start on Windows / non-dcgpu. @@ -1009,6 +1488,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }; assert!(mi300x.engine_available("vllm")); assert!(mi300x.engine_available("lemonade")); @@ -1503,4 +1984,667 @@ Local model engines std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o755)).expect("restore"); assert_eq!(probe_dir_permissions_bind(tmp.path()), refused); } + + /// An exit status that reports failure with code 1, built the way each + /// platform encodes it, so the non-zero-exit row runs on both CI gates. + fn failed_status() -> std::process::ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + // A wait status: the exit code sits in the second byte. + std::process::ExitStatus::from_raw(1 << 8) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(1) + } + } + + fn succeeded_status() -> std::process::ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt; + std::process::ExitStatus::from_raw(0) + } + } + + fn output( + status: std::process::ExitStatus, + stdout: &str, + stderr: &str, + ) -> std::process::Output { + std::process::Output { + status, + stdout: stdout.as_bytes().to_vec(), + stderr: stderr.as_bytes().to_vec(), + } + } + + /// The regression: a probe that exits non-zero is an error even when it + /// printed something, rather than stdout parsed as a GPU-less host. + #[test] + fn a_probe_that_exits_non_zero_is_an_error_with_its_stderr_tail() { + let result = classify_probe_output( + "rocm examine", + Ok(output( + failed_status(), + "rocm examine\n os: linux\n", + "warning: early\nError: could not read the host\n", + )), + ); + let Err(error @ ProbeError::Exit { .. }) = result else { + panic!("a non-zero exit must be ProbeError::Exit, got {result:?}"); + }; + assert_eq!( + error.to_string(), + format!( + "`rocm examine` exited with {}\n--- stderr (last lines) ---\n\ + warning: early\nError: could not read the host", + failed_status() + ) + ); + } + + #[test] + fn the_stderr_tail_keeps_only_the_last_lines() { + let stderr = (1..=30) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + let Err(ProbeError::Exit { stderr_tail, .. }) = + classify_probe_output("rocm examine", Ok(output(failed_status(), "", &stderr))) + else { + panic!("a non-zero exit must be ProbeError::Exit"); + }; + let expected: Vec = (11..=30).map(|i| format!("line {i}")).collect(); + assert_eq!(stderr_tail, expected.join("\n")); + } + + #[test] + fn a_probe_that_exits_zero_with_output_returns_its_stdout() { + let stdout = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n"; + assert_eq!( + classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, ""))) + .expect("exit 0 with output is a reading"), + stdout + ); + } + + #[test] + fn a_probe_that_exits_zero_with_no_output_is_an_error() { + for stdout in ["", " \n\n"] { + let result = + classify_probe_output("rocm examine", Ok(output(succeeded_status(), stdout, ""))); + let Err(error @ ProbeError::EmptyOutput { .. }) = result else { + panic!("empty stdout {stdout:?} must be ProbeError::EmptyOutput, got {result:?}"); + }; + assert_eq!( + error.to_string(), + "`rocm examine` exited 0 but produced no output on stdout" + ); + } + } + + #[test] + fn a_probe_whose_binary_does_not_exist_is_a_spawn_error_naming_it() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let missing = tmp.path().join("no-such-rocm"); + let result = run_probe_with( + missing.as_os_str(), + tmp.path(), + &["examine"], + DEFAULT_PROBE_TIMEOUT, + ); + let Err(error @ ProbeError::Spawn { .. }) = result else { + panic!("a missing binary must be ProbeError::Spawn, got {result:?}"); + }; + let ProbeError::Spawn { command, source } = &error else { + unreachable!() + }; + assert_eq!(command, &format!("{} examine", missing.display())); + assert_eq!(source.kind(), std::io::ErrorKind::NotFound); + assert_eq!( + error.to_string(), + format!( + "could not start `{} examine`: {source}. Set ROCM_CLI_BINARY to the rocm binary \ + under test, or run the suite through `cargo xtask e2e`, which builds it and \ + sets ROCM_CLI_BINARY", + missing.display() + ) + ); + } + + const GPU_EXAMINE: &str = "rocm examine\n os: linux\n detected_gfx_target: gfx1151\n wsl: false\n driver_status: ok\n"; + + /// A failed probe records why, and leaves the struct at the no-GPU values + /// so no caller panics. + #[test] + fn a_failed_examine_records_the_error_and_falls_back_to_no_gpu() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Err(ProbeError::EmptyOutput { + command: "rocm examine".to_owned(), + }), + Ok("lemonade\n".to_owned()), + tmp.path(), + ); + assert_eq!( + cap.probe_error.as_deref(), + Some("`rocm examine` exited 0 but produced no output on stdout") + ); + assert!(!cap.has_amd_gpu); + assert_eq!(cap.gfx_target, None); + assert_eq!(cap.platform_slug, "mock"); + } + + /// The `engines list fails, examine OK` row: the GPU facts were read, but + /// the probe still failed, so the run must not proceed on them. + #[test] + fn a_failed_engines_list_is_recorded_even_when_examine_succeeded() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Ok(GPU_EXAMINE.to_owned()), + Err(ProbeError::Exit { + command: "rocm engines list".to_owned(), + status: failed_status(), + stderr_tail: "boom".to_owned(), + }), + tmp.path(), + ); + assert_eq!( + cap.probe_error, + Some(format!( + "`rocm engines list` exited with {}\n--- stderr (last lines) ---\nboom", + failed_status() + )) + ); + assert!(cap.available_engines.is_empty()); + } + + #[test] + fn the_first_failure_is_the_one_recorded() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Err(ProbeError::EmptyOutput { + command: "rocm examine".to_owned(), + }), + Err(ProbeError::EmptyOutput { + command: "rocm engines list".to_owned(), + }), + tmp.path(), + ); + assert_eq!( + cap.probe_error.as_deref(), + Some("`rocm examine` exited 0 but produced no output on stdout") + ); + } + + #[test] + fn successful_probes_record_no_error_and_keep_the_driver_status() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let cap = capability_from_probes( + Ok(GPU_EXAMINE.to_owned()), + Ok("lemonade\nvllm\n".to_owned()), + tmp.path(), + ); + assert_eq!(cap.probe_error, None); + assert_eq!(cap.gfx_target.as_deref(), Some("gfx1151")); + assert_eq!(cap.driver_status, "ok"); + // `probe_error` is left out of platform.json when there is none. + let json = serde_json::to_value(&cap).expect("serializes"); + assert!(json.get("probe_error").is_none(), "{json}"); + } + + #[test] + fn the_probe_facts_summary_names_the_mask_in_force() { + let cap = HostCapability { + os_family: "linux".to_owned(), + is_wsl: true, + gfx_target: Some("gfx1151".to_owned()), + has_amd_gpu: false, + amd_gpu_count: None, + available_engines: vec![], + effective_serve_engine: "lemonade".to_owned(), + platform_slug: "mock".to_owned(), + oom_fault_injection: false, + case_sensitive_fs: true, + dir_permissions_bind: true, + driver_status: "wsl_rocdxg_missing".to_owned(), + probe_error: None, + }; + let base = "detected_gfx_target=gfx1151, wsl=true, driver_status=wsl_rocdxg_missing"; + assert_eq!( + probe_facts_summary_with_mask(&cap, None, None), + format!("{base}, no visibility mask") + ); + assert_eq!( + probe_facts_summary_with_mask( + &cap, + Some(std::ffi::OsStr::new("")), + Some(std::ffi::OsStr::new("0")) + ), + format!("{base}, HIP_VISIBLE_DEVICES=\"\"") + ); + assert_eq!( + probe_facts_summary_with_mask(&cap, None, Some(std::ffi::OsStr::new("1"))), + format!("{base}, ROCR_VISIBLE_DEVICES=\"1\"") + ); + let none = HostCapability { + gfx_target: None, + is_wsl: false, + driver_status: String::new(), + ..cap + }; + assert_eq!( + probe_facts_summary_with_mask(&none, None, None), + "detected_gfx_target=, wsl=false, driver_status=, no visibility mask" + ); + } + + /// A shell command line, run as `(binary, args)` on each CI gate. + fn shell_line(line: &str) -> (std::ffi::OsString, Vec) { + #[cfg(unix)] + { + ("sh".into(), vec!["-c".to_owned(), line.to_owned()]) + } + #[cfg(windows)] + { + ("cmd".into(), vec!["/C".to_owned(), line.to_owned()]) + } + } + + fn arg_refs(args: &[String]) -> Vec<&str> { + args.iter().map(String::as_str).collect() + } + + /// Prints `done` after about two seconds. + fn two_second_command() -> (std::ffi::OsString, Vec) { + #[cfg(unix)] + { + shell_line("sleep 2; echo done") + } + #[cfg(windows)] + { + shell_line("ping -n 3 127.0.0.1 >nul & echo done") + } + } + + /// The env lookup `run_probe` would get, with only the given variables set. + fn env_of( + vars: Vec<(&'static str, std::ffi::OsString)>, + ) -> impl Fn(&str) -> Option { + move |name| { + vars.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone()) + } + } + + /// `E2E_PROBE_TIMEOUT_SECS` reaches the probe through the same lookup + /// `run_probe` uses (no process-wide variable), and the Timeout message's + /// advice holds: the same command that timed out at 1 second completes when + /// the variable allows it longer. + #[test] + fn the_probe_timeout_variable_sets_the_deadline_and_raising_it_clears_the_timeout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = two_second_command(); + let command = format!("{} {}", binary.to_string_lossy(), args.join(" ")); + + let short = run_probe_using( + tmp.path(), + &arg_refs(&args), + env_of(vec![ + ("ROCM_CLI_BINARY", binary.clone()), + (PROBE_TIMEOUT_ENV, "1".into()), + ]), + ); + let Err(error @ ProbeError::Timeout { .. }) = short else { + panic!("a 1 s deadline on a 2 s command must time out, got {short:?}"); + }; + let ProbeError::Timeout { after, .. } = &error else { + unreachable!() + }; + assert_eq!(*after, std::time::Duration::from_secs(1)); + assert_eq!( + error.to_string(), + format!( + "`{command}` was still running after 1s, so it was killed; set \ + E2E_PROBE_TIMEOUT_SECS to allow it longer\n--- stderr (last lines): (empty) ---" + ) + ); + + // Following that advice clears the condition. + let longer = run_probe_using( + tmp.path(), + &arg_refs(&args), + env_of(vec![ + ("ROCM_CLI_BINARY", binary), + (PROBE_TIMEOUT_ENV, "5".into()), + ]), + ) + .expect("with 5 s allowed, the 2 s command completes"); + assert_eq!(longer.trim(), "done"); + } + + /// The regression: a probe that never finishes must become an error at its + /// deadline, carrying what it had written to stderr, and be killed AND + /// reaped. A zombie still answers `kill -0`, so a failing `kill -0` shows + /// the child was reaped, not merely signalled. + #[cfg(unix)] + #[test] + fn a_probe_still_running_at_its_deadline_is_killed_reaped_and_keeps_its_stderr() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let pid_file = tmp.path().join("pid"); + let script = format!( + "echo $$ > '{}'; echo early-err >&2; exec sleep 30", + pid_file.display() + ); + let started = std::time::Instant::now(); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", &script], + std::time::Duration::from_secs(3), + std::time::Duration::from_secs(2), + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(30), + "returned after {:?}: it waited for the child instead of killing it", + started.elapsed() + ); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail, "early-err"); + // The shell creates the file and then writes the pid; give a loaded + // runner a moment to finish the write rather than read it half-done. + let mut pid = String::new(); + for _ in 0..100 { + pid = std::fs::read_to_string(&pid_file).unwrap_or_default(); + if !pid.trim().is_empty() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + assert!( + !pid.trim().is_empty(), + "the child wrote its pid before the deadline" + ); + let still_there = std::process::Command::new("kill") + .args(["-0", pid.trim()]) + .stderr(std::process::Stdio::null()) + .status() + .expect("kill runs") + .success(); + assert!( + !still_there, + "child {} was left running or unreaped", + pid.trim() + ); + } + + /// Windows half of the test above: the deadline and the stderr kept from + /// before it. `cmd` is killed but the `ping` it started holds the pipe, so + /// the bytes reach the caller only through the shared buffer. The reaping + /// check needs `kill -0` and runs on Unix only. + #[cfg(windows)] + #[test] + fn a_probe_still_running_at_its_deadline_is_reported_with_its_stderr() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = shell_line("echo early-err 1>&2 & ping -n 10 127.0.0.1 >nul"); + let started = std::time::Instant::now(); + let result = run_probe_bounded( + &binary, + tmp.path(), + &arg_refs(&args), + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + ); + assert!(started.elapsed() < std::time::Duration::from_secs(10)); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail.trim(), "early-err"); + } + + /// Stderr written after the kill still reaches the caller: the drain runs + /// after a timeout too. A grandchild holding only stderr (its stdout goes + /// to `/dev/null`) writes `late` half a second after the 1 s deadline + /// killed its parent. Stdout is closed by then, so only the wait on stderr + /// can collect it; the 10 s grace leaves time for that. + #[cfg(unix)] + #[test] + fn stderr_written_after_the_kill_is_drained_into_the_timeout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &[ + "-c", + "(sleep 1.5; echo late >&2) >/dev/null & exec sleep 30", + ], + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(10), + ); + let Err(ProbeError::Timeout { stderr_tail, .. }) = &result else { + panic!("a probe past its deadline must be ProbeError::Timeout, got {result:?}"); + }; + assert_eq!(stderr_tail, "late"); + } + + /// A child that reads stdin gets EOF at once rather than blocking until the + /// deadline: `output_within` replaces whatever stdin the command carried + /// with an empty one. The command is handed a pipe whose write end this + /// test holds open, so only that replacement lets the read finish (the + /// same arrangement as `a_child_that_reads_stdin_is_given_an_empty_one` in + /// rocm-core's `process/bounded.rs`). + /// + /// Like that sibling, it catches a missing stdin setting but does not pin + /// `null` against `inherit`: an `inherit` mutant passes whenever the test + /// process's own stdin is already at EOF. + #[test] + fn a_probe_that_reads_stdin_is_given_an_empty_one() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sh"); + command.args(["-c", "read line || echo stdin-eof"]); + command + }; + #[cfg(windows)] + let mut command = { + // `set /p` leaves the variable as it was when its read hits EOF. + let script = tmp.path().join("stdin.bat"); + std::fs::write( + &script, + "@echo off\r\nset \"LINE=stdin-eof\"\r\nset /p LINE=\r\necho %LINE%\r\n", + ) + .expect("write script"); + let mut command = std::process::Command::new("cmd"); + command.arg("/C").arg(&script); + command + }; + let (stdin_reader, stdin_writer) = std::io::pipe().expect("stdin pipe"); + command.stdin(stdin_reader); + + let started = std::time::Instant::now(); + let finished = output_within( + command, + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(2), + ) + .expect("spawns"); + let elapsed = started.elapsed(); + // Dropped only now: closing the write end earlier would give the child + // EOF whatever stdin `output_within` set. + drop(stdin_writer); + drop(tmp); + + let Finished::Exited { output, .. } = finished else { + panic!("a child given an empty stdin must exit, not time out: {finished:?}"); + }; + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "stdin-eof"); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the read must hit EOF at once, but took {elapsed:?}" + ); + } + + /// A probe that exited 0 while a process it started still holds its + /// stdout is reported as such, not as "produced no output". Unix only: it + /// needs a background process that inherits the pipe, which `sh` gives + /// reliably; the classification itself is tested on both gates below. + #[cfg(unix)] + #[test] + fn a_probe_whose_output_stays_open_is_reported_as_that_not_as_no_output() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", "echo real-output; echo warn >&2; sleep 3 &"], + std::time::Duration::from_secs(10), + std::time::Duration::from_millis(300), + ); + let Err(error @ ProbeError::OutputNotClosed { .. }) = &result else { + panic!("an output left open must be ProbeError::OutputNotClosed, got {result:?}"); + }; + assert_eq!( + error.to_string(), + "`sh -c echo real-output; echo warn >&2; sleep 3 &` exited 0, but its stdout was \ + still open 300ms later (a process it started may be holding it), so what it \ + printed cannot be read in full\n--- stderr (last lines) ---\nwarn" + ); + } + + /// Only stdout carries the reading: a process that keeps just stderr open + /// does not fail the probe. Unix only, like the test above. + #[cfg(unix)] + #[test] + fn a_probe_whose_stderr_alone_stays_open_still_returns_its_stdout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let result = run_probe_bounded( + std::ffi::OsStr::new("sh"), + tmp.path(), + &["-c", "echo real-output; sleep 3 >/dev/null &"], + std::time::Duration::from_secs(10), + std::time::Duration::from_millis(300), + ); + assert_eq!( + result.expect("stdout closed, so a reading").trim(), + "real-output" + ); + } + + #[test] + fn an_undrained_exit_is_classified_by_its_status() { + let grace = std::time::Duration::from_secs(5); + let undrained = |status| { + Ok(Finished::Exited { + output: output(status, "partial", "err"), + stdout_drained: false, + }) + }; + let ok = classify_finished("rocm examine", undrained(succeeded_status()), grace, grace); + assert!( + matches!(&ok, Err(ProbeError::OutputNotClosed { stderr_tail, .. }) if stderr_tail == "err"), + "{ok:?}" + ); + // A non-zero exit is reported as one, with what stderr had by then. + let failed = classify_finished("rocm examine", undrained(failed_status()), grace, grace); + assert!( + matches!(&failed, Err(ProbeError::Exit { stderr_tail, .. }) if stderr_tail == "err"), + "{failed:?}" + ); + // A drained exit is classified as before. + let drained = classify_finished( + "rocm examine", + Ok(Finished::Exited { + output: output(succeeded_status(), "rocm examine\n", "partial err"), + stdout_drained: true, + }), + grace, + grace, + ); + // Stdout closed but stderr may not have: the reading is used. + assert_eq!(drained.expect("a reading"), "rocm examine\n"); + let wait_failed = classify_finished( + "rocm examine", + Ok(Finished::WaitFailed(std::io::Error::other("no child"))), + grace, + grace, + ); + let Err(error @ ProbeError::Wait { .. }) = &wait_failed else { + panic!("a failed wait must be ProbeError::Wait, got {wait_failed:?}"); + }; + assert_eq!( + error.to_string(), + "`rocm examine` was started, but waiting for it to exit failed: no child" + ); + let timed_out = classify_finished( + "rocm examine", + Ok(Finished::TimedOut { + stderr: b"a\nb\n".to_vec(), + }), + grace, + grace, + ); + assert!( + matches!(&timed_out, Err(ProbeError::Timeout { stderr_tail, after, .. }) + if stderr_tail == "a\nb" && *after == grace), + "{timed_out:?}" + ); + } + + #[test] + fn a_probe_that_finishes_in_time_returns_its_stdout() { + let tmp = tempfile::TempDir::with_prefix("rocm-e2e-probe-test-").expect("temp dir"); + let (binary, args) = shell_line("echo probe-ok"); + let stdout = run_probe_with(&binary, tmp.path(), &arg_refs(&args), DEFAULT_PROBE_TIMEOUT) + .expect("a quick command is a reading"); + assert_eq!(stdout.trim(), "probe-ok"); + // A deadline too far off to add to the current time must not panic. + let stdout = run_probe_with( + &binary, + tmp.path(), + &arg_refs(&args), + std::time::Duration::MAX, + ) + .expect("an unbounded deadline still reads"); + assert_eq!(stdout.trim(), "probe-ok"); + } + + #[test] + fn the_probe_deadline_reads_whole_positive_seconds_else_the_default() { + assert_eq!(probe_timeout_from_env(None), DEFAULT_PROBE_TIMEOUT); + assert_eq!(DEFAULT_PROBE_TIMEOUT.as_secs(), 120); + for value in ["5", " 5 ", "5\n"] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(value))), + std::time::Duration::from_secs(5), + "{value:?}" + ); + } + // No upper bound: a huge value is honoured, and `output_within`'s + // `checked_add` keeps it from overflowing (see the `Duration::MAX` run + // in `a_probe_that_finishes_in_time_returns_its_stdout`). + for secs in [86_401, u64::MAX] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(&secs.to_string()))), + std::time::Duration::from_secs(secs) + ); + } + for value in ["", "0", "-3", "1.5", "soon", "18446744073709551616"] { + assert_eq!( + probe_timeout_from_env(Some(std::ffi::OsStr::new(value))), + DEFAULT_PROBE_TIMEOUT, + "{value:?}" + ); + } + } } diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index fb3a513e6..0ff33664a 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -585,7 +585,8 @@ const SMI_DRAIN_GRACE: Duration = Duration::from_secs(2); /// module is library code, and a normal dependency would be a new first-party /// edge that `cargo xtask check-crate-edges` makes a reviewed addition. The two /// can drift, so a change to either one's kill or drain behaviour should look at -/// the other. +/// the other. The same applies to `capability::output_within`, the capability +/// probe's copy of these steps, which also keeps stderr and the exit status. pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe { let mut child = match command .stdin(Stdio::null()) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index fc44c822e..aa8408d0d 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -482,6 +482,12 @@ pub struct PlatformManifest<'a> { pub capability: &'a HostCapability, /// Component versions (OS/ROCm/vLLM/lemonade) for the report heading. pub versions: crate::capability::PlatformVersions, + /// Why the post-run lane check failed the run + /// (`lane_verdict::post_run_verdict`), so the consolidated report shows the + /// lane as failed even when every scenario it resolved was a skip. Left out + /// when the check passed. + #[serde(skip_serializing_if = "Option::is_none")] + pub lane_failure: Option, pub expectations: Vec, } @@ -745,6 +751,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, "strix-ubuntu" => HostCapability { os_family: "linux".into(), @@ -758,6 +766,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, "strix-windows" => HostCapability { os_family: "windows".into(), @@ -771,6 +781,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: false, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, // A WSL2 dev box with the ROCm passthrough in place: `os_family` is // `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes @@ -793,6 +805,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: "wsl_rocdxg_ready".into(), + probe_error: None, }, // The same box without the passthrough: the gfx target is reported // by the Windows-side driver but ROCm cannot reach it, so the probe @@ -809,6 +823,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, // The hosted WSL runner before the ROCm passthrough is complete: // Windows still reports the gfx target, but the CLI cannot use it. @@ -824,6 +840,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, _ => HostCapability { os_family: "other".into(), @@ -837,6 +855,8 @@ mod tests { oom_fault_injection: false, case_sensitive_fs: true, dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, }, } } @@ -2120,6 +2140,36 @@ reason = "unrelated open bug" )); } + /// What the lane-verdict notes say about `@requires-gpu` scenarios (see + /// `lane_verdict::preflight_verdict`): `has_amd_gpu` alone decides their + /// skip, other gates aside. `resolve` takes no lane declaration, so + /// `E2E_EXPECT_GPU` does not enter into it. + #[test] + fn requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not() { + let m = Expectations::default(); + let d = decl(&["id:gpu-premise", "requires-gpu"]); + let included = Included { + nightly: false, + lifecycle: false, + docker: false, + merge_queue: false, + }; + let with_gpu = cap("strix-ubuntu"); + assert!(with_gpu.has_amd_gpu); + assert_eq!( + resolve(&d, &with_gpu, &m, included), + Expectation::ExpectPass + ); + let without_gpu = HostCapability { + has_amd_gpu: false, + ..with_gpu + }; + assert!(matches!( + resolve(&d, &without_gpu, &m, included), + Expectation::Skip { .. } + )); + } + #[test] fn non_gpu_scenario_always_runs() { let m = Expectations::default(); diff --git a/tests/e2e-cucumber/src/lane_verdict.rs b/tests/e2e-cucumber/src/lane_verdict.rs new file mode 100644 index 000000000..9d58ef537 --- /dev/null +++ b/tests/e2e-cucumber/src/lane_verdict.rs @@ -0,0 +1,961 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether a run's outcome says anything about the lane it ran on. +//! +//! Scenario results alone cannot say that. Scenarios the capability probe +//! resolves to skip never reach `report.json`, so a run in which every scenario +//! was skipped, or none was selected, has no failure to report and passes. +//! That is what a GPU lane does when its probe cannot read the binary under +//! test, or when the GPU is not visible: every GPU scenario (`@requires-gpu` or +//! `@requires-multi-gpu`) resolves to skip, and the lane is green without +//! having tested a GPU. +//! +//! Two checks close that gap, both pure so every row of their state matrix is a +//! unit test: +//! +//! - [`preflight_verdict`], before any scenario runs: a probe that failed is +//! fatal on every lane, and so is a lane that declares a GPU (`1` in +//! [`EXPECT_GPU_ENV`]) where the probe found none. +//! - [`post_run_verdict`], after the run: a run that executed no scenario is +//! fatal on every lane, and so is a GPU-declaring lane on which no GPU +//! scenario (`@requires-gpu` or `@requires-multi-gpu`) executed. +//! +//! The module also holds the decisions `tests/e2e.rs` takes around those two +//! checks, so they are unit-tested too: [`preflight`] (the declaration plus +//! the pre-run verdict), [`needs_gpu`] and [`gpu_scenario_counts`] (the input to +//! the post-run check), [`run_failed`] (the final exit, combining the +//! harness's own XPASS and failure classification with the post-run verdict), +//! [`finish_run`] (the post-run verdict and the `platform.json` manifest that +//! records it), and [`failure_lines`] and [`note_line`] (the exact lines +//! printed, with the annotation escaped by [`workflow_command`]). The harness +//! itself still sets up the run, probes the host, reads its other environment +//! variables, classifies the scenario results, writes the files, and does the +//! printing and exiting; that glue has no test of its own. + +use std::ffi::OsStr; + +pub use e2e_report::EXPECT_GPU_ENV; + +/// What a lane says about its GPU in [`EXPECT_GPU_ENV`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuDeclaration { + /// `1`: the lane runs on GPU hardware, so a run that found or used none is + /// broken. + Expected, + /// `0`: the lane deliberately has no usable GPU, so `@requires-gpu` + /// scenarios are not applicable on it. + NotExpected, + /// Unset or empty: no claim either way (GitHub-hosted lanes, local runs). + Undeclared, +} + +/// Read [`EXPECT_GPU_ENV`]'s value. +/// +/// Anything other than `1`, `0`, empty or unset is an error rather than a +/// guess, since a misspelt declaration that read as "undeclared" would switch +/// the GPU check off without anyone noticing. +pub fn parse_gpu_declaration(value: Option<&OsStr>) -> Result { + match value.map(OsStr::to_str) { + None | Some(Some("")) => Ok(GpuDeclaration::Undeclared), + Some(Some("1")) => Ok(GpuDeclaration::Expected), + Some(Some("0")) => Ok(GpuDeclaration::NotExpected), + Some(_) => Err(format!( + "invalid {EXPECT_GPU_ENV} value {:?}: use 1 (this lane has a GPU), 0 (it \ + deliberately has none), or leave it unset", + value.unwrap_or_default().to_string_lossy() + )), + } +} + +/// What a check concluded: a reason to fail the run, and lines to print either +/// way. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Verdict { + /// Why the run must fail; `None` when it may proceed or pass. + pub fatal: Option, + /// Informational lines that do not change the outcome. + pub notes: Vec, +} + +/// The verdict before any scenario runs, from the capability probe and the +/// lane's declaration. +/// +/// `probe_error` is [`crate::capability::HostCapability::probe_error`]; +/// `probe_facts` is the one-line summary of the facts the GPU decision was made +/// from ([`crate::capability::probe_facts_summary`]), quoted when a declared GPU +/// is missing. +#[must_use] +pub fn preflight_verdict( + probe_error: Option<&str>, + has_amd_gpu: bool, + probe_facts: &str, + declaration: GpuDeclaration, +) -> Verdict { + let mut verdict = Verdict::default(); + if let Some(error) = probe_error { + verdict.fatal = Some(format!( + "the host capability probe could not read the rocm binary under test, so every \ + capability-gated scenario would resolve to skip and the run would pass without \ + testing them: {error}" + )); + return verdict; + } + match (declaration, has_amd_gpu) { + (GpuDeclaration::Expected, false) => { + verdict.fatal = Some(format!( + "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but the capability probe found \ + no usable AMD GPU ({probe_facts}), so every @requires-gpu scenario would \ + resolve to skip" + )); + } + (GpuDeclaration::NotExpected, true) => verdict.notes.push(format!( + "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0) but the capability probe \ + found one ({probe_facts}); the declaration is stale, and @requires-gpu scenarios \ + will run" + )), + (GpuDeclaration::NotExpected, false) => verdict.notes.push(format!( + "this lane declares no usable GPU ({EXPECT_GPU_ENV}=0); @requires-gpu scenarios \ + are not applicable here" + )), + (GpuDeclaration::Expected, true) | (GpuDeclaration::Undeclared, _) => {} + } + verdict +} + +/// How many GPU scenarios (`@requires-gpu` or `@requires-multi-gpu`) the run +/// resolved and how many of those executed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GpuScenarioCounts { + /// Scenarios tagged `@requires-gpu` or `@requires-multi-gpu` that the + /// harness resolved, whether or not they ran. + pub resolved: usize, + /// Of those, how many reached `report.json`. + pub executed: usize, +} + +/// The verdict after the run. +/// +/// `executed` counts the scenarios that reached `report.json`. `gpu` is `None` +/// when the harness's scenario filter never ran — a cucumber `--name`/`--tags` +/// filter on the command line replaces it — so nothing is known about which +/// scenarios needed a GPU and only the zero-executed check applies. +#[must_use] +pub fn post_run_verdict( + executed: usize, + gpu: Option, + declaration: GpuDeclaration, +) -> Verdict { + let mut verdict = Verdict::default(); + if executed == 0 { + verdict.fatal = Some( + "0 scenarios executed: a run that executes nothing tests nothing. Check the \ + cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and E2E_ALSO_RUN_IDS" + .to_owned(), + ); + return verdict; + } + let Some(gpu) = gpu else { + return verdict; + }; + match declaration { + GpuDeclaration::Expected if gpu.executed == 0 => { + verdict.fatal = Some(format!( + "this lane declares a GPU ({EXPECT_GPU_ENV}=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed ({} resolved, none ran)", + gpu.resolved + )); + } + GpuDeclaration::NotExpected => verdict.notes.push(format!( + "{} of {} GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on this \ + lane ({EXPECT_GPU_ENV}=0)", + gpu.resolved.saturating_sub(gpu.executed), + gpu.resolved + )), + GpuDeclaration::Expected | GpuDeclaration::Undeclared => {} + } + verdict +} + +/// What the harness carries forward from the pre-run check when it passes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Preflight { + /// The lane's parsed [`EXPECT_GPU_ENV`] declaration, for the post-run check. + pub declaration: GpuDeclaration, + /// Lines to print before the run. + pub notes: Vec, +} + +/// The pre-run lane check, from the raw [`EXPECT_GPU_ENV`] value and the +/// probe's results: `Err` with the reason when the run must not start. +/// +/// An invalid declaration is reported before anything the probe found, since +/// the verdict depends on the declaration. The harness only prints the notes +/// or the refusal and exits. +pub fn preflight( + declaration_value: Option<&OsStr>, + probe_error: Option<&str>, + has_amd_gpu: bool, + probe_facts: &str, +) -> Result { + let declaration = parse_gpu_declaration(declaration_value)?; + let verdict = preflight_verdict(probe_error, has_amd_gpu, probe_facts, declaration); + match verdict.fatal { + Some(fatal) => Err(fatal), + None => Ok(Preflight { + declaration, + notes: verdict.notes, + }), + } +} + +/// Whether a scenario counts toward the post-run GPU check. +/// +/// It does when tagged `@requires-gpu` or `@requires-multi-gpu`. `@requires-gfx-target` does not +/// count, since such a scenario reads the chip name without using the GPU. +#[must_use] +pub const fn needs_gpu(decl: &crate::expectation::ScenarioDecl) -> bool { + decl.requires_gpu || decl.requires_multi_gpu +} + +/// Count the GPU scenarios the harness resolved and how many of them executed. +/// +/// `resolved` is each scenario the harness's filter saw, as `(id, needs_gpu)`; +/// `executed` is the id of each scenario in `report.json`. `None` when nothing +/// was resolved: a cucumber `--name`/`--tags` filter replaced the harness's +/// filter, so which scenarios need a GPU is unknown (see [`post_run_verdict`]). +/// An executed id the harness did not resolve as a GPU scenario is not counted. +pub fn gpu_scenario_counts<'a>( + resolved: impl IntoIterator, + executed: impl IntoIterator, +) -> Option { + let mut any_resolved = false; + let gpu_ids: std::collections::BTreeSet<&str> = resolved + .into_iter() + .inspect(|_| any_resolved = true) + .filter_map(|(id, gpu)| gpu.then_some(id)) + .collect(); + if !any_resolved { + return None; + } + Some(GpuScenarioCounts { + resolved: gpu_ids.len(), + executed: executed + .into_iter() + .filter(|id| gpu_ids.contains(id)) + .count(), + }) +} + +/// Whether the run exits non-zero after reconciliation. +/// +/// Any one of a stale XPASS, an unexpected failure (both counted by the +/// harness) or a fatal post-run verdict is enough. Parse and hook errors exit +/// earlier, in the harness. +#[must_use] +pub const fn run_failed(stale_xpass: usize, unexpected_fail: usize, post_run: &Verdict) -> bool { + stale_xpass > 0 || unexpected_fail > 0 || post_run.fatal.is_some() +} + +/// A GitHub Actions workflow command (`::error::…`, `::notice::…`) carrying +/// `message` as its data. +/// +/// The data is escaped as the runner expects — `%` first, then CR and LF — so a +/// probe's stderr tail can neither cut the annotation short at a newline nor +/// have a literal `%0A` in it read back as one. +#[must_use] +pub fn workflow_command(command: &str, message: &str) -> String { + let data = message + .replace('%', "%25") + .replace('\r', "%0D") + .replace('\n', "%0A"); + format!("::{command}::{data}") +} + +/// One scenario the harness's filter resolved, as [`finish_run`] needs it. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedRun<'s> { + pub id: &'s str, + pub feature: &'s str, + pub scenario: &'s str, + pub engine: &'s str, + pub expectation: &'s crate::expectation::Expectation, + /// [`needs_gpu`] of the scenario's tags. + pub needs_gpu: bool, +} + +/// The post-run lane check, and the `platform.json` manifest that records its +/// failure as `lane_failure` so the consolidated report shows the lane red. +/// +/// `resolved` is every scenario the harness's filter saw (empty when a CLI +/// filter replaced it); `executed` is the id of each scenario in +/// `report.json`. +#[must_use] +pub fn finish_run<'a>( + cap: &'a crate::capability::HostCapability, + versions: crate::capability::PlatformVersions, + resolved: &[ResolvedRun<'_>], + executed: &[&str], + declaration: GpuDeclaration, +) -> (Verdict, crate::expectation::PlatformManifest<'a>) { + let gpu = gpu_scenario_counts( + resolved.iter().map(|r| (r.id, r.needs_gpu)), + executed.iter().copied(), + ); + let verdict = post_run_verdict(executed.len(), gpu, declaration); + let manifest = crate::expectation::PlatformManifest { + platform_slug: &cap.platform_slug, + capability: cap, + versions, + lane_failure: verdict.fatal.clone(), + expectations: resolved + .iter() + .map(|r| { + crate::expectation::ResolvedScenario::new( + r.id, + r.feature, + r.scenario, + r.engine, + r.expectation, + ) + }) + .collect(), + }; + (verdict, manifest) +} + +/// The lines the harness prints when a lane check fails: a GitHub `::error::` +/// annotation, then the same message as a plain line for anyone reading the +/// log outside Actions. +#[must_use] +pub fn failure_lines(message: &str) -> [String; 2] { + let line = format!("E2E lane check failed: {message}"); + [workflow_command("error", &line), line] +} + +/// The `::notice::` line the harness prints for a lane-check note. +#[must_use] +pub fn note_line(note: &str) -> String { + workflow_command("notice", &format!("E2E lane: {note}")) +} + +#[cfg(test)] +mod tests { + use super::{ + GpuDeclaration, GpuScenarioCounts, Preflight, ResolvedRun, Verdict, failure_lines, + finish_run, gpu_scenario_counts, needs_gpu, note_line, parse_gpu_declaration, + post_run_verdict, preflight, preflight_verdict, run_failed, workflow_command, + }; + use std::ffi::OsStr; + + const FACTS: &str = + "detected_gfx_target=, wsl=false, driver_status=, no visibility mask"; + const PROBE_ERROR: &str = "`rocm examine` exited with exit status: 1"; + + const ALL: [GpuDeclaration; 3] = [ + GpuDeclaration::Expected, + GpuDeclaration::NotExpected, + GpuDeclaration::Undeclared, + ]; + + #[test] + fn only_one_zero_and_unset_or_empty_are_declarations() { + assert_eq!(parse_gpu_declaration(None), Ok(GpuDeclaration::Undeclared)); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new(""))), + Ok(GpuDeclaration::Undeclared) + ); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new("1"))), + Ok(GpuDeclaration::Expected) + ); + assert_eq!( + parse_gpu_declaration(Some(OsStr::new("0"))), + Ok(GpuDeclaration::NotExpected) + ); + } + + #[test] + fn a_misspelt_declaration_is_an_error_not_undeclared() { + for value in ["true", "yes", " 1", "2"] { + assert_eq!( + parse_gpu_declaration(Some(OsStr::new(value))), + Err(format!( + "invalid E2E_EXPECT_GPU value {value:?}: use 1 (this lane has a GPU), 0 (it \ + deliberately has none), or leave it unset" + )), + ); + } + } + + // Row: probe failed (non-zero exit, spawn failure, empty output, or + // `engines list` failing after `examine` succeeded all reach here as one + // `probe_error`). Fatal under every declaration, GPU or not. + #[test] + fn a_failed_probe_is_fatal_under_every_declaration() { + for declaration in ALL { + for has_amd_gpu in [false, true] { + let verdict = preflight_verdict(Some(PROBE_ERROR), has_amd_gpu, FACTS, declaration); + assert_eq!( + verdict, + Verdict { + fatal: Some(format!( + "the host capability probe could not read the rocm binary under \ + test, so every capability-gated scenario would resolve to skip and \ + the run would pass without testing them: {PROBE_ERROR}" + )), + notes: vec![], + }, + "{declaration:?}, gpu={has_amd_gpu}" + ); + } + } + } + + // Row: probe succeeded, no usable GPU, declared 1. + #[test] + fn a_declared_gpu_the_probe_did_not_find_is_fatal_and_names_the_facts() { + let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::Expected); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but the capability probe found no \ + usable AMD GPU (detected_gfx_target=, wsl=false, driver_status=, \ + no visibility mask), so every @requires-gpu scenario would resolve to skip" + ) + ); + assert!(verdict.notes.is_empty()); + } + + // Row: probe succeeded with a GPU, declared 1 — runs, nothing to say. + #[test] + fn a_declared_gpu_that_is_present_runs_without_comment() { + assert_eq!( + preflight_verdict(None, true, FACTS, GpuDeclaration::Expected), + Verdict::default() + ); + } + + // Row: probe succeeded with a GPU, declared 0 — runs, with a stale note. + #[test] + fn a_declared_absent_gpu_that_is_present_is_a_note_not_a_failure() { + let verdict = preflight_verdict(None, true, FACTS, GpuDeclaration::NotExpected); + assert_eq!(verdict.fatal, None); + // Pins the wording only. That @requires-gpu scenarios do run on a host + // whose probe found a GPU, whatever the declaration, is proved by + // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`. + assert_eq!( + verdict.notes, + [format!( + "this lane declares no usable GPU (E2E_EXPECT_GPU=0) but the capability probe \ + found one ({FACTS}); the declaration is stale, and @requires-gpu scenarios will \ + run" + )] + ); + } + + // Row: probe succeeded, no GPU, declared 0 — runs, says why GPU scenarios + // are not applicable. + #[test] + fn a_declared_absent_gpu_that_is_absent_runs_with_a_note() { + let verdict = preflight_verdict(None, false, FACTS, GpuDeclaration::NotExpected); + assert_eq!(verdict.fatal, None); + // Pins the wording only. That @requires-gpu scenarios resolve to skip + // (not applicable) where the probe found no GPU is proved by + // `expectation::tests::requires_gpu_runs_where_the_probe_found_a_gpu_and_skips_where_it_did_not`. + assert_eq!( + verdict.notes, + [ + "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are \ + not applicable here" + ] + ); + } + + // Rows: probe succeeded, undeclared — today's behaviour, GPU or not. + #[test] + fn an_undeclared_lane_runs_without_comment_either_way() { + for has_amd_gpu in [false, true] { + assert_eq!( + preflight_verdict(None, has_amd_gpu, FACTS, GpuDeclaration::Undeclared), + Verdict::default(), + "gpu={has_amd_gpu}" + ); + } + } + + // Post-run row: nothing executed — fatal under every declaration, whether + // or not the harness's own filter ran (a `-n` filter matching nothing is + // the `None` case). + #[test] + fn a_run_that_executed_nothing_is_fatal_under_every_declaration() { + let gpu_cases = [ + None, + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }), + Some(GpuScenarioCounts { + resolved: 4, + executed: 0, + }), + ]; + for declaration in ALL { + for gpu in gpu_cases { + let verdict = post_run_verdict(0, gpu, declaration); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "0 scenarios executed: a run that executes nothing tests nothing. Check \ + the cucumber --name/--tags filter, E2E_ONLY_LIFECYCLE and \ + E2E_ALSO_RUN_IDS" + ), + "{declaration:?}, {gpu:?}" + ); + } + } + } + + // Post-run row: declared 1, scenarios ran but no GPU one did. + #[test] + fn a_gpu_lane_on_which_no_gpu_scenario_ran_is_fatal() { + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::Expected, + ); + assert_eq!( + verdict.fatal.as_deref(), + Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (7 resolved, none ran)" + ) + ); + } + + // Post-run row: declared 1, scenarios ran, and the harness's filter ran but + // resolved no GPU scenario at all (none selected): still fatal, since the + // lane promised GPU coverage it did not get. + #[test] + fn a_gpu_lane_that_resolved_no_gpu_scenario_is_fatal() { + let verdict = post_run_verdict( + 5, + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }), + GpuDeclaration::Expected, + ); + assert_eq!( + verdict, + Verdict { + fatal: Some( + "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (0 resolved, none ran)" + .to_owned() + ), + notes: vec![], + } + ); + } + + #[test] + fn a_gpu_lane_on_which_a_gpu_scenario_ran_passes() { + assert_eq!( + post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 1, + }), + GpuDeclaration::Expected, + ), + Verdict::default() + ); + } + + // Post-run row: a CLI filter bypassed the harness's filter, so which + // scenarios need a GPU is unknown — only the zero check applies. + #[test] + fn a_cli_filtered_run_is_not_held_to_the_gpu_check() { + for declaration in ALL { + assert_eq!( + post_run_verdict(3, None, declaration), + Verdict::default(), + "{declaration:?}" + ); + } + } + + #[test] + fn a_no_gpu_lane_says_how_many_gpu_scenarios_did_not_run() { + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::NotExpected, + ); + assert_eq!(verdict.fatal, None); + assert_eq!( + verdict.notes, + [ + "7 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \ + this lane (E2E_EXPECT_GPU=0)" + ] + ); + // Some ran: the note counts only the ones that did not. + let verdict = post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 3, + }), + GpuDeclaration::NotExpected, + ); + assert_eq!( + verdict.notes, + [ + "4 of 7 GPU scenario(s) (@requires-gpu or @requires-multi-gpu) did not run on \ + this lane (E2E_EXPECT_GPU=0)" + ] + ); + } + + #[test] + fn an_undeclared_lane_is_not_held_to_the_gpu_check() { + assert_eq!( + post_run_verdict( + 12, + Some(GpuScenarioCounts { + resolved: 7, + executed: 0, + }), + GpuDeclaration::Undeclared, + ), + Verdict::default() + ); + } + + // --- the harness glue, as pure functions --- + + #[test] + fn preflight_refuses_an_invalid_declaration_before_reading_the_probe() { + let invalid = Err( + "invalid E2E_EXPECT_GPU value \"yes\": use 1 (this lane has a GPU), \ + 0 (it deliberately has none), or leave it unset" + .to_owned(), + ); + // A probe that also failed, or found no GPU, must not displace the + // declaration error: the declaration is checked first. + for (probe_error, has_amd_gpu) in [(None, true), (Some(PROBE_ERROR), false), (None, false)] + { + assert_eq!( + preflight(Some(OsStr::new("yes")), probe_error, has_amd_gpu, FACTS), + invalid, + "probe_error={probe_error:?}, gpu={has_amd_gpu}" + ); + } + } + + #[test] + fn preflight_refuses_a_failed_probe_and_a_missing_declared_gpu() { + let refused = preflight(None, Some(PROBE_ERROR), true, FACTS).expect_err("probe failed"); + assert!(refused.ends_with(PROBE_ERROR), "{refused}"); + let refused = + preflight(Some(OsStr::new("1")), None, false, FACTS).expect_err("declared GPU missing"); + assert!( + refused.starts_with("this lane declares a GPU (E2E_EXPECT_GPU=1)"), + "{refused}" + ); + } + + #[test] + fn preflight_carries_the_declaration_and_notes_into_the_run() { + assert_eq!( + preflight(Some(OsStr::new("0")), None, false, FACTS), + Ok(Preflight { + declaration: GpuDeclaration::NotExpected, + notes: vec![ + "this lane declares no usable GPU (E2E_EXPECT_GPU=0); @requires-gpu \ + scenarios are not applicable here" + .to_owned() + ], + }) + ); + assert_eq!( + preflight(Some(OsStr::new("1")), None, true, FACTS), + Ok(Preflight { + declaration: GpuDeclaration::Expected, + notes: vec![], + }) + ); + } + + #[test] + fn gpu_and_multi_gpu_scenarios_need_a_gpu_and_a_gfx_target_one_does_not() { + let decl = crate::expectation::ScenarioDecl::from_tags; + assert!(needs_gpu(&decl(&["id:a", "requires-gpu"]))); + assert!(needs_gpu(&decl(&["id:b", "requires-multi-gpu"]))); + assert!(!needs_gpu(&decl(&["id:c", "requires-gfx-target"]))); + assert!(!needs_gpu(&decl(&["id:d"]))); + } + + #[test] + fn a_cli_filtered_run_resolved_nothing_so_has_no_gpu_counts() { + assert_eq!(gpu_scenario_counts([], ["serve-a", "serve-b"]), None); + } + + #[test] + fn gpu_counts_take_resolved_from_the_filter_and_executed_from_the_report() { + let resolved = [ + ("gpu-ran", true), + ("multi-gpu-only", true), + ("gpu-skipped", true), + ("cpu-ran", false), + ]; + // `cpu-ran` executed but is not a GPU scenario, and `untagged` was + // never resolved: neither counts as an executed GPU scenario. + let executed = ["gpu-ran", "multi-gpu-only", "cpu-ran", "untagged"]; + assert_eq!( + gpu_scenario_counts(resolved, executed), + Some(GpuScenarioCounts { + resolved: 3, + executed: 2, + }) + ); + } + + #[test] + fn gpu_counts_with_nothing_executed_are_zero_not_none() { + assert_eq!( + gpu_scenario_counts([("gpu", true), ("cpu", false)], []), + Some(GpuScenarioCounts { + resolved: 1, + executed: 0, + }) + ); + assert_eq!( + gpu_scenario_counts([("cpu", false)], ["cpu"]), + Some(GpuScenarioCounts { + resolved: 0, + executed: 0, + }) + ); + } + + #[test] + fn the_run_fails_on_any_one_of_its_three_reasons() { + let fatal = Verdict { + fatal: Some("0 scenarios executed".to_owned()), + notes: vec![], + }; + let clean = Verdict::default(); + assert!(!run_failed(0, 0, &clean)); + assert!(run_failed(1, 0, &clean)); + assert!(run_failed(0, 1, &clean)); + assert!(run_failed(0, 0, &fatal)); + // A note alone never fails the run. + let noted = Verdict { + fatal: None, + notes: vec!["stale".to_owned()], + }; + assert!(!run_failed(0, 0, ¬ed)); + } + + #[test] + fn workflow_command_data_escapes_percent_cr_and_lf() { + assert_eq!( + workflow_command("error", "100% failed\r\nstderr: a%0Ab"), + "::error::100%25 failed%0D%0Astderr: a%250Ab" + ); + assert_eq!(workflow_command("notice", "plain"), "::notice::plain"); + } + + #[test] + fn a_failure_prints_an_escaped_annotation_then_the_plain_message() { + assert_eq!( + failure_lines("probe failed\n--- stderr ---\n100% full"), + [ + "::error::E2E lane check failed: probe failed%0A--- stderr ---%0A100%25 full" + .to_owned(), + "E2E lane check failed: probe failed\n--- stderr ---\n100% full".to_owned(), + ] + ); + } + + #[test] + fn a_note_prints_as_an_escaped_notice() { + assert_eq!( + note_line("7 of 7 GPU scenario(s) did not run (100%)"), + "::notice::E2E lane: 7 of 7 GPU scenario(s) did not run (100%25)" + ); + } + + /// `finish_run`'s manifest, written as the harness writes it and read back + /// with the consolidated report's own loader: the post-run lane failure + /// it records makes the lane red there, and a lane that passed the check + /// stays green. Ties the producer to the consumer through the function + /// `tests/e2e.rs` calls. + #[test] + fn finish_runs_manifest_reads_back_as_the_verdict_in_the_report() { + let cap = crate::capability::HostCapability { + os_family: "linux".into(), + is_wsl: false, + gfx_target: Some("gfx1151".into()), + has_amd_gpu: true, + amd_gpu_count: Some(1), + available_engines: vec!["lemonade".into()], + effective_serve_engine: "lemonade".into(), + platform_slug: "strix-halo-linux".into(), + oom_fault_injection: false, + case_sensitive_fs: true, + dir_permissions_bind: true, + driver_status: String::new(), + probe_error: None, + }; + let pass = crate::expectation::Expectation::ExpectPass; + let skip = crate::expectation::Expectation::Skip { + reason: "no usable AMD GPU".to_owned(), + }; + let resolved_with = |gpu_expectation| { + [ + ResolvedRun { + id: "examine-version", + feature: "Examine", + scenario: "examine-01", + engine: "lemonade", + expectation: &pass, + needs_gpu: false, + }, + ResolvedRun { + id: "serve-gpu", + feature: "Serving", + scenario: "serve-01", + engine: "vllm", + expectation: gpu_expectation, + needs_gpu: true, + }, + ] + }; + // Writes what the harness writes and reads it back: the verdict, the + // report's status, and `platform.json`'s id -> expected mapping. + // `executed` is also what report.json holds; every listed id passed. + let status_of = |resolved: &[ResolvedRun<'_>], executed: &[&str]| { + let (verdict, manifest) = finish_run( + &cap, + crate::capability::PlatformVersions::default(), + resolved, + executed, + GpuDeclaration::Expected, + ); + let dir = tempfile::TempDir::with_prefix("rocm-e2e-manifest-").expect("temp dir"); + let elements: Vec = executed + .iter() + .map(|id| { + format!( + r#"{{"name":"{id}","tags":[{{"name":"id:{id}"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"passed"}}}}]}}"# + ) + }) + .collect(); + let report = dir.path().join("report.json"); + std::fs::write( + &report, + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{}]}}]"#, + elements.join(",") + ), + ) + .expect("report.json"); + let platform = dir.path().join("platform.json"); + std::fs::write( + &platform, + serde_json::to_string_pretty(&manifest).expect("serializes"), + ) + .expect("platform.json"); + let written: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&platform).expect("read back")) + .expect("valid json"); + let mapping: Vec> = written["expectations"] + .as_array() + .expect("an expectations array") + .iter() + .map(|e| { + ["id", "feature", "scenario", "effective_engine", "expected"] + .iter() + .map(|key| e[*key].as_str().unwrap_or_default().to_owned()) + .collect() + }) + .collect(); + ( + verdict, + e2e_report::platform_status("e2e-report", &report), + mapping, + ) + }; + // id, feature, scenario, effective_engine, expected — per expectation. + let mapping_of = |rows: &[[&str; 5]]| -> Vec> { + rows.iter() + .map(|row| row.iter().map(|field| (*field).to_owned()).collect()) + .collect() + }; + + // Every cell is pass or skip, so only the recorded lane failure can + // make this row FAIL. + let reason = "this lane declares a GPU (E2E_EXPECT_GPU=1) but no @requires-gpu (or \ + @requires-multi-gpu) scenario executed (1 resolved, none ran)"; + let (verdict, failed, mapping) = status_of(&resolved_with(&skip), &["examine-version"]); + assert_eq!(verdict.fatal.as_deref(), Some(reason)); + assert_eq!( + mapping, + mapping_of(&[ + [ + "examine-version", + "Examine", + "examine-01", + "lemonade", + "pass" + ], + ["serve-gpu", "Serving", "serve-01", "vllm", "skip"], + ]) + ); + assert_eq!( + failed, + e2e_report::PlatformStatus { + ok: false, + status: "FAIL", + lane_failure: Some(reason.to_owned()), + } + ); + + let (verdict, passed, mapping) = + status_of(&resolved_with(&pass), &["examine-version", "serve-gpu"]); + assert_eq!(verdict.fatal, None); + assert_eq!( + mapping, + mapping_of(&[ + [ + "examine-version", + "Examine", + "examine-01", + "lemonade", + "pass" + ], + ["serve-gpu", "Serving", "serve-01", "vllm", "pass"], + ]) + ); + assert_eq!( + passed, + e2e_report::PlatformStatus { + ok: true, + status: "PASS", + lane_failure: None, + } + ); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 29e334583..1d10fe54b 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -12,6 +12,7 @@ pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lane_verdict; pub mod lemonade_backend; pub mod lemonade_diagnostics; pub mod lemonade_protocol; diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 3c2cc9788..3dbb3d83b 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -263,6 +263,9 @@ struct Resolution { engine: String, feature: String, scenario: String, + /// Tagged `@requires-gpu` or `@requires-multi-gpu`, for the post-run check + /// that a GPU-declaring lane executed at least one of them. + requires_gpu: bool, } /// Resolve a CI-provided shared-directory env var to a validated, existing path. @@ -1499,6 +1502,7 @@ async fn run_suite() { use e2e_cucumber::expectation::{ Expectation, ScenarioDecl, lifecycle_lane_includes, resolve, unmatched_also_run_ids, }; + use e2e_cucumber::lane_verdict; use e2e_cucumber::monotonic_clock::MonotonicClockWriter; let dir = results_dir(); @@ -1559,6 +1563,19 @@ async fn run_suite() { cap.platform_slug, cap.os_family, cap.has_amd_gpu, cap.effective_serve_engine, ); + // Refuse to run when the result could not mean anything: a probe that could + // not read the binary, or a lane that declares a GPU the probe did not find, + // would resolve every GPU scenario to skip and pass. See `lane_verdict`. + let preflight = lane_verdict::preflight( + std::env::var_os(lane_verdict::EXPECT_GPU_ENV).as_deref(), + cap.probe_error.as_deref(), + cap.has_amd_gpu, + &e2e_cucumber::capability::probe_facts_summary(cap), + ) + .unwrap_or_else(|fatal| lane_failure(&fatal)); + print_lane_notes(&preflight.notes); + let gpu_declaration = preflight.declaration; + // Shared record of each scenario's resolved expectation, keyed by @id. // Populated by `filter_run` (which sees every scenario, run or skipped) so // the post-run evaluation and platform.json can reconcile by id — including @@ -1681,6 +1698,7 @@ async fn run_suite() { engine, feature: feature.name.clone(), scenario: scenario.name.clone(), + requires_gpu: lane_verdict::needs_gpu(&decl), }, ); // Two scenarios sharing an `@id` would silently overwrite each @@ -1736,31 +1754,41 @@ async fn run_suite() { // OS is always collected (every platform, incl. mock); ROCm/vLLM/lemonade come // from the shared runtimes tree CI provides (E2E_SHARED_RUNTIMES_DIR) — absent // on mock (no runtime) and on local runs (per-scenario dirs already dropped), - // where those render as "n/a". Best-effort. + // where those render as "n/a". Best-effort. Collected before the lane check + // so that, once decided, nothing stands between it and platform.json. let shared = shared_runtimes_dir(); let versions = e2e_cucumber::capability::collect_versions(shared.as_deref()); + // A run with nothing to fail passes; check it executed something, and on + // a GPU-declaring lane something that needed the GPU. `resolutions` is empty + // when a cucumber `--name`/`--tags` filter replaced `filter_run`, and then + // which scenarios need a GPU is unknown. The verdict goes into platform.json, + // so the consolidated report shows a failed lane check as red. + let resolved: Vec> = resolutions + .iter() + .map(|(id, r)| lane_verdict::ResolvedRun { + id, + feature: &r.feature, + scenario: &r.scenario, + engine: &r.engine, + expectation: &r.expectation, + needs_gpu: r.requires_gpu, + }) + .collect(); + let executed: Vec<&str> = actual.iter().map(|(id, _)| id.as_str()).collect(); + let (post_run, manifest) = + lane_verdict::finish_run(cap, versions, &resolved, &executed, gpu_declaration); + print_lane_notes(&post_run.notes); + // Write the platform.json sidecar (probed capability + every resolution, - // including skips) for the central report's expected-vs-actual grid. - let manifest = e2e_cucumber::expectation::PlatformManifest { - platform_slug: &cap.platform_slug, - capability: cap, - versions, - expectations: resolutions - .iter() - .map(|(id, r)| { - e2e_cucumber::expectation::ResolvedScenario::new( - id, - &r.feature, - &r.scenario, - &r.engine, - &r.expectation, - ) - }) - .collect(), - }; - if let Ok(json) = serde_json::to_string_pretty(&manifest) { - let _ = std::fs::write(dir.join("platform.json"), json); + // including skips, + any lane failure) for the central report's grid. + let written = serde_json::to_string_pretty(&manifest) + .map_err(|error| error.to_string()) + .and_then(|json| { + std::fs::write(dir.join("platform.json"), json).map_err(|error| error.to_string()) + }); + if let Err(error) = written { + eprintln!("warning: could not write platform.json: {error}"); } // Classify actual results against the matrix. A deterministic XPASS is @@ -1805,7 +1833,7 @@ async fn run_suite() { "XPASS (flaky, tolerated): '{x}' passed this run; the intermittent bug remains tracked." ); } - if !stale_xpass.is_empty() || !unexpected_fail.is_empty() { + if lane_verdict::run_failed(stale_xpass.len(), unexpected_fail.len(), &post_run) { for x in &stale_xpass { eprintln!( "XPASS: '{x}' is expected to fail on this host but PASSED \u{2014} the bug appears \ @@ -1817,6 +1845,24 @@ async fn run_suite() { "FAIL: '{f}' was expected to pass on this host but FAILED \u{2014} a regression." ); } + if let Some(fatal) = &post_run.fatal { + lane_failure(fatal); + } std::process::exit(1); } } + +/// Print a lane-check failure (see `lane_verdict::failure_lines`) and exit +/// non-zero. +fn lane_failure(message: &str) -> ! { + for line in e2e_cucumber::lane_verdict::failure_lines(message) { + eprintln!("{line}"); + } + std::process::exit(1); +} + +fn print_lane_notes(notes: &[String]) { + for note in notes { + eprintln!("{}", e2e_cucumber::lane_verdict::note_line(note)); + } +} diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index 9d8d47c79..1c85f2e26 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -428,13 +428,13 @@ fn queue_absent_reason(markdown: &str) -> Option { /// for every per-platform row that reports a non-`PASS` `Status` with a `Fail` /// cell of `0`. /// -/// The second part exists because an `EMPTY` platform (no `report.json`, or -/// one that's empty/unparseable) contributes `0` to every numeric column — -/// including `Fail` — so the Total row's own Fail cell has no record that the -/// platform was ever there at all. A genuine `FAIL` status always has a -/// nonzero own `Fail` cell (already folded into the Total row), so this only -/// adds for platforms the Total row would otherwise read as having zero -/// failures. +/// The second part exists because two kinds of failed row contribute `0` to +/// the `Fail` column, so the Total row's own Fail cell has no record of them: +/// an `EMPTY` platform (no `report.json`, or one that's empty/unparseable), +/// and a `FAIL` platform whose only problem is the harness's post-run lane +/// check (`lane_failure` in its `platform.json`, e.g. "0 scenarios executed"), +/// which fails no scenario cell. A `FAIL` row with a nonzero own `Fail` cell +/// is already folded into the Total row and is not counted again. /// /// A row for a shared-queue lane the queue does not block on (its platform /// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the @@ -795,6 +795,54 @@ mod tests { assert_eq!(total_e2e_failures(md), Some(1)); } + /// A lane the harness failed after its run with no failed scenario cell + /// (every scenario passed or was skipped) reads `FAIL` with a `Fail` cell + /// of 0; the Total row cannot show it, so the row itself must count. One + /// that also has a failed cell is already in the Total row and is counted + /// once, not twice. + #[test] + fn total_e2e_failures_counts_a_lane_check_failure_with_no_failed_cells() { + // Real renderer output over a platform.json that records a lane + // failure, with the one scenario passing or failing. + let render = |step_status: &str| { + let dir = tempfile::tempdir().expect("tempdir"); + let report = dir.path().join("report.json"); + std::fs::write( + &report, + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s0","tags":[{{"name":"id:examine-a"}}],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{step_status}"}}}}]}}]}}]"# + ), + ) + .expect("write report"); + std::fs::write( + dir.path().join("platform.json"), + r#"{"platform_slug":"mi300x","lane_failure":"0 scenarios executed", + "expectations":[{"id":"examine-a","feature":"F","expected":"pass"}]}"#, + ) + .expect("write platform.json"); + e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&[( + "e2e-gpu-report".to_string(), + report, + )])) + }; + + let md = render("passed"); + assert!( + md.contains("| 1 | 1 | 0 | 0 | n/a | FAIL |") + && md.contains("| **Total** | | 1 | 1 | 0 |"), + "expected a FAIL row with a Fail cell of 0:\n{md}" + ); + assert_eq!(total_e2e_failures(&md), Some(1), "{md}"); + + let md = render("failed"); + assert!( + md.contains("| 1 | 0 | 1 | 0 | n/a | FAIL |") + && md.contains("| **Total** | | 1 | 0 | 1 |"), + "expected a FAIL row with a Fail cell of 1:\n{md}" + ); + assert_eq!(total_e2e_failures(&md), Some(1), "{md}"); + } + #[test] fn total_e2e_failures_leaves_out_the_queue_lanes_the_queue_lets_fail() { // Real producer output, so the marker the parser looks for is the one the diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index a33154924..e4d25b65d 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1718,6 +1718,150 @@ jobs: } } + /// Self-hosted E2E lanes that declare no usable GPU, each with the reason. + /// Every other self-hosted lane must declare one. + const NO_GPU_LANES: [(&str, &str); 2] = [ + ( + "e2e-wsl", + "the WSL distro is deliberately left without ROCm's WSL driver bridge, \ + which `rocm install driver` installs under test", + ), + ( + "e2e-wsl-nightly", + "the nightly twin of e2e-wsl, set up the same way", + ), + ]; + + /// Whether a step's `run:` body invokes the E2E suite itself + /// (`cargo xtask e2e`, not `e2e-prewarm` or `e2e-report`). + fn runs_the_e2e_suite(step: &str) -> bool { + run_block(step).is_some_and(|body| { + body.iter().any(|line| { + line.match_indices("cargo xtask e2e").any(|(at, m)| { + line[at + m.len()..] + .chars() + .next() + .is_none_or(char::is_whitespace) + }) + }) + }) + } + + /// Every self-hosted E2E lane declares whether it has a GPU, so the harness + /// can fail a GPU lane whose probe found none instead of passing it with + /// every GPU scenario (`@requires-gpu` or `@requires-multi-gpu`) skipped. + /// + /// Derived from the workflows like the GPU-preflight check above, so a new + /// lane is held to it the day it lands. The declaration is read from the + /// job-level `env:` mapping: a step-level one would leave the other steps, + /// and a reader of the job header, without it, and one on the step that + /// runs the suite would override it there. A lane that runs the suite + /// inside WSL must name the variable in every `WSLENV` it sets, and in one + /// on the step that runs the suite, since `wsl.exe` passes nothing else + /// into the guest. + #[test] + fn every_self_hosted_lane_declares_whether_it_has_a_gpu() { + let name = e2e_report::EXPECT_GPU_ENV; + let mut no_gpu_seen = Vec::new(); + for (workflow, text) in self_hosted_workflows() { + for (job, _) in self_hosted_e2e_jobs(&text) { + let block = job_block(&text, &job); + let env = job_mapping(block, "env"); + let exempt = NO_GPU_LANES.iter().any(|(lane, _)| *lane == job); + match env.get(name).map(String::as_str) { + Some("1") => assert!( + !exempt, + "{workflow} job `{job}` is listed in NO_GPU_LANES but declares \ + {name}: \"1\"; drop it from the list" + ), + Some("0") => { + assert!( + exempt, + "{workflow} job `{job}` declares {name}: \"0\" but is not in \ + NO_GPU_LANES; a self-hosted lane without a GPU needs its reason \ + recorded there" + ); + no_gpu_seen.push(job.clone()); + } + other => panic!( + "{workflow} job `{job}` runs on self-hosted hardware but its job-level \ + `env:` sets {name} to {other:?}; it must be \"1\" (a GPU lane) or \ + \"0\" (listed in NO_GPU_LANES)" + ), + } + let why = NO_GPU_LANES + .iter() + .find(|(lane, _)| *lane == job) + .map_or_else(String::new, |(_, reason)| { + format!(" (a NO_GPU_LANES lane: {reason})") + }); + let names_it = |wslenv: &str| { + wslenv + .trim_matches(|c| c == '\'' || c == '"') + .split(':') + .any(|var| var.split('/').next() == Some(name)) + }; + for wslenv in block + .lines() + .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = ")) + { + assert!( + names_it(wslenv), + "{workflow} job `{job}`{why} sets WSLENV without {name}, so the suite \ + inside WSL reads the lane as undeclared: {wslenv}" + ); + } + // The step that runs the suite is where the declaration has + // to arrive: a step-level override there would replace the + // job's value, and a WSL step that names it in no WSLENV of + // its own leaves the guest without it, whatever other steps do. + let steps = job_steps(block) + .unwrap_or_else(|| panic!("{workflow} job `{job}` has no `steps:` list")); + let harness: Vec<&(String, String)> = steps + .iter() + .filter(|(_, step)| runs_the_e2e_suite(step)) + .collect(); + assert!( + !harness.is_empty(), + "{workflow} job `{job}` has no step that runs `cargo xtask e2e`" + ); + for (step_name, step) in harness { + assert!( + !step_env(step).contains_key(name), + "{workflow} job `{job}`{why}: step `{step_name}` sets {name} itself, \ + overriding the job-level declaration" + ); + if step.contains("Invoke-WslBash.ps1") { + assert!( + step.lines() + .filter_map(|line| line.trim().strip_prefix("$env:WSLENV = ")) + .any(names_it), + "{workflow} job `{job}`{why}: step `{step_name}` runs the suite in \ + WSL but sets no WSLENV naming {name}, so the guest reads the lane \ + as undeclared" + ); + } + } + } + } + for (lane, _) in NO_GPU_LANES { + assert!( + no_gpu_seen.iter().any(|seen| seen == lane), + "NO_GPU_LANES lists `{lane}`, which no self-hosted workflow declares \ + {name}: \"0\"; drop the stale entry" + ); + } + // The queue lanes run through this script, not a workflow job; every + // queue lane is given a GPU. + let script = std::fs::read_to_string(repo_root().join("scripts/run-e2e-lane.sh")) + .expect("reading the lane script"); + let export = format!("export {name}=\"${{{name}-1}}\""); + assert!( + script.lines().any(|line| line.trim() == export), + "scripts/run-e2e-lane.sh must declare its lanes' GPU with `{export}`" + ); + } + #[test] fn only_e2e_gpu_may_carry_the_stale_gate_on_its_preflight() { let workflow = |job: &str| { @@ -5703,14 +5847,16 @@ fi let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) .expect("read hardware testing docs"); - let documented: Vec> = - markdown_table_rows(&docs, "| Job | Workflow | Platform | Runner labels |") - .into_iter() - .filter(|row| { - row.get(1) - .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") - }) - .collect(); + let documented: Vec> = markdown_table_rows( + &docs, + "| Job | Workflow | Platform | Runner labels | GPU declared (`E2E_EXPECT_GPU`) |", + ) + .into_iter() + .filter(|row| { + row.get(1) + .is_some_and(|workflow| workflow == "`e2e-selfhosted.yml`") + }) + .collect(); let documented_jobs: Vec = documented.iter().map(|row| row[0].clone()).collect(); let declared_jobs: Vec = lanes.iter().map(|(job, _)| format!("`{job}`")).collect(); @@ -5738,6 +5884,15 @@ fi flattened_list_items(runs_on), "the documented runner labels for `{job}` must match its actual runs-on" ); + let declared = job_mapping(job_block(&self_hosted, job), "env") + .get(e2e_report::EXPECT_GPU_ENV) + .map(|value| format!("`{value}`")); + assert_eq!( + row.get(4), + declared.as_ref(), + "the GPU declared cell for `{job}` must match its job-level {}", + e2e_report::EXPECT_GPU_ENV + ); } // Derived from the same scan `every_uploaded_e2e_artifact_has_a_name_the_report_can_label`