Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions .github/workflows/e2e-selfhosted.yml
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,9 @@ jobs:
contents: read
pull-requests: read
env:
# GPU lane: the E2E harness fails if its probe finds no usable AMD GPU
# or no GPU scenario runs (see tests/e2e-cucumber/README.md).
E2E_EXPECT_GPU: "1"
# Bound serve readiness below the 35-min job cap so a serve that never comes
# ready fails the scenario with a real error instead of hanging until the
# job is cancelled. 300s is ample for a real MI300X vLLM cold-start;
Expand Down Expand Up @@ -625,6 +628,9 @@ jobs:
- channel: nightly
sdk_version: n-2
env:
# GPU lane: the E2E harness fails if its probe finds no usable AMD GPU
# or no GPU scenario runs (see tests/e2e-cucumber/README.md).
E2E_EXPECT_GPU: "1"
E2E_SERVE_TIMEOUT_SECS: "300"
# A pool host can be busy with other CI work between GPU preflight and
# the TUI checks, so a frame that renders well inside the 30s default on
Expand Down Expand Up @@ -779,6 +785,9 @@ jobs:
)
continue-on-error: false
env:
# GPU lane: the E2E harness fails if its probe finds no usable AMD GPU
# or no GPU scenario runs (see tests/e2e-cucumber/README.md).
E2E_EXPECT_GPU: "1"
# A pool host can be busy with other CI work between GPU preflight and
# the TUI checks, so a frame that renders well inside the 30s default on
# an idle runner can miss it. Raise the wait budget rather than let that
Expand Down Expand Up @@ -841,8 +850,10 @@ jobs:
# available so a wedged/held GPU fails fast instead of hanging to the cap.
# BEST-EFFORT because the Windows ROCm GPU query tool isn't verified here —
# if no rocm-smi is found we WARN and continue rather than false-fail a
# working runner. When rocm-smi IS present we poll free memory the same way
# as the Linux jobs and fail only after the ceiling.
# working runner; the harness's E2E_EXPECT_GPU=1 check is the backstop
# that fails the lane if `rocm examine` reports no usable GPU at all.
# When rocm-smi IS present we poll free memory the same way as the Linux
# jobs and fail only after the ceiling.
#
# WHAT THIS DOES NOT CATCH, and the Linux lanes now do. This is a
# free-memory floor ONLY. A fixed 8 GiB against a whole ~62 GiB gfx1151
Expand Down Expand Up @@ -1130,6 +1141,13 @@ jobs:
)
continue-on-error: false
env:
# Declared GPU-less on purpose: the distro is left without ROCm's WSL
# driver bridge (see the "NO librocdxg provisioning step" note below), so
# the probe finds no usable GPU and @requires-gpu scenarios are not
# applicable. The harness reports that rather than failing on it; set "1"
# here if this lane ever provisions a GPU. Forwarded into the guest via
# WSLENV in the run step.
E2E_EXPECT_GPU: "0"
# wsl.exe emits UTF-16 by default; without this its own error/status text
# comes back mangled ("T h e f i l e c a n n o t ...") when captured
# through PowerShell, which is exactly what a genuine wsl.exe error looks
Expand Down Expand Up @@ -1406,12 +1424,14 @@ jobs:
# today: @requires-gpu scenarios already resolve to skip here, since
# `rocm-smi` is absent too (see the GPU preflight below). If this lane
# ever needs a real GPU, provision it through the CLI as a separate,
# explicitly-not-under-test step rather than restoring the old script.
# explicitly-not-under-test step rather than restoring the old script,
# and flip the job's E2E_EXPECT_GPU to "1".

# GPU preflight, bounded like the native lanes — but ADVISORY here. GPU
# access under WSL is exactly what this lane is proving out, so an absent
# rocm-smi is a reported condition, not a job failure: the capability probe
# then resolves @requires-gpu scenarios to skip and the rest still runs. A
# rocm-smi is a reported condition, not a job failure: the lane declares
# E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not
# applicable instead of failing on them, and the rest still runs. A
# GPU that IS present but stays held fails the job, since that would
# corrupt the serve scenarios' results — though "held" means less here
# than on the native lanes: this body keeps only the free-memory floor and
Expand All @@ -1433,7 +1453,7 @@ jobs:
MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}"
CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}"
if ! command -v rocm-smi >/dev/null 2>&1; then
echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip"
echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable"
exit 0
fi
min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 ))
Expand Down Expand Up @@ -1533,7 +1553,7 @@ jobs:
sleep 5
done
if [ "$saw_vram" -eq 0 ]; then
echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip"
echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable"
exit 0
fi
if [ "$unmeasurable" -eq 1 ]; then
Expand Down Expand Up @@ -1590,7 +1610,7 @@ jobs:
# Forward the job-level E2E_* env vars into the guest: `wsl.exe`
# does not share the calling process's environment unless a
# variable is named in WSLENV.
$env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER'
$env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER'
& "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @'
export PATH="$HOME/.cargo/bin:$PATH"
cd /root/work/rocm-cli
Expand Down
54 changes: 41 additions & 13 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,9 @@ jobs:
matrix:
channel: [release, nightly]
env:
# GPU lane: the E2E harness fails if its probe finds no usable AMD GPU
# or no GPU scenario runs (see tests/e2e-cucumber/README.md).
E2E_EXPECT_GPU: "1"
# Include @nightly scenarios (the large-model serve). See src/expectation.rs.
E2E_INCLUDE_NIGHTLY: "1"
# Serve-readiness floor; a `@serve-timeout` tag lengthens it for the large
Expand Down Expand Up @@ -531,6 +534,9 @@ jobs:
matrix:
channel: [release, nightly]
env:
# GPU lane: the E2E harness fails if its probe finds no usable AMD GPU
# or no GPU scenario runs (see tests/e2e-cucumber/README.md).
E2E_EXPECT_GPU: "1"
E2E_INCLUDE_NIGHTLY: "1"
E2E_SERVE_TIMEOUT_SECS: "300"
E2E_TUI_TIMEOUT_SECS: "90"
Expand Down Expand Up @@ -618,6 +624,9 @@ jobs:
matrix:
channel: [release, nightly]
env:
# GPU lane: the E2E harness fails if its probe finds no usable AMD GPU
# or no GPU scenario runs (see tests/e2e-cucumber/README.md).
E2E_EXPECT_GPU: "1"
E2E_INCLUDE_NIGHTLY: "1"
E2E_SERVE_TIMEOUT_SECS: "300"
E2E_TUI_TIMEOUT_SECS: "90"
Expand Down Expand Up @@ -885,6 +894,13 @@ jobs:
matrix:
channel: [release, nightly]
env:
# Declared GPU-less on purpose: the distro is left without ROCm's WSL
# driver bridge (see the "NO librocdxg provisioning step" note below), so
# the probe finds no usable GPU and @requires-gpu scenarios are not
# applicable. The harness reports that rather than failing on it; set "1"
# here if this lane ever provisions a GPU. Forwarded into the guest via
# WSLENV in the run step.
E2E_EXPECT_GPU: "0"
# wsl.exe emits UTF-16 by default; without this its own error/status text
# comes back mangled when captured through PowerShell. Matches the
# ubuntu/WSL composite action's own env.
Expand Down Expand Up @@ -1084,8 +1100,9 @@ jobs:

# GPU preflight, bounded like the native lanes — but ADVISORY here. GPU
# access under WSL is exactly what this lane is proving out, so an absent
# rocm-smi is a reported condition, not a job failure: the capability probe
# then resolves @requires-gpu scenarios to skip and the rest still runs. A
# rocm-smi is a reported condition, not a job failure: the lane declares
# E2E_EXPECT_GPU=0, so the harness reports @requires-gpu scenarios as not
# applicable instead of failing on them, and the rest still runs. A
# GPU that IS present but stays held fails the job, since that would
# corrupt the serve scenarios' results.
#
Expand All @@ -1106,7 +1123,7 @@ jobs:
MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}"
CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}"
if ! command -v rocm-smi >/dev/null 2>&1; then
echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip"
echo "::warning::rocm-smi not found in this WSL distro — expected on this lane (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable"
exit 0
fi
min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 ))
Expand Down Expand Up @@ -1206,7 +1223,7 @@ jobs:
sleep 5
done
if [ "$saw_vram" -eq 0 ]; then
echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip"
echo "::warning::rocm-smi reported no VRAM figures under WSL — this lane declares no GPU (E2E_EXPECT_GPU=0); @requires-gpu scenarios are not applicable"
exit 0
fi
if [ "$unmeasurable" -eq 1 ]; then
Expand Down Expand Up @@ -1257,7 +1274,7 @@ jobs:
shell: powershell
run: |
# See .github/scripts/Invoke-WslBash.ps1.
$env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF'
$env:WSLENV = 'E2E_EXPECT_GPU:E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:ROCM_CLI_RELEASE_REF'
& "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @'
export PATH="$HOME/.cargo/bin:$PATH"
cd /root/work/rocm-cli
Expand Down Expand Up @@ -1326,7 +1343,13 @@ jobs:
# a failed step there does not turn the lane's check red. Only the
# WSL lanes need this guard: every other lane writes report.json
# directly in-job, with no guest-to-host copy step that can fail on
# its own.
# its own. A harness preflight refusal is not one of those:
# `run_suite` in tests/e2e-cucumber/tests/e2e.rs creates report.json
# empty before it calls `lane_verdict::preflight`, and the
# `if: always()` copy-out above carries it to the host, so it uploads
# fine and the missing-report check does not catch it; its EMPTY row
# in the report is the signal (a post-run lane failure shows as a
# FAIL row with lane_failure).
if-no-files-found: error

# The nightly lanes each uploaded a report nobody joined up, so reading a
Expand Down Expand Up @@ -1501,13 +1524,18 @@ jobs:
fi
cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY"

# A lane that dies before writing report.json (preflight, build, a crash
# before results exist) uploads nothing, and `discover()` can only render
# the reports that arrived, so that lane would drop out of the grid with
# no row and no FAIL. On a `continue-on-error` lane nothing else is red
# either. So check it here: every lane that ran (any `needs` result but
# `skipped`, and not self-skipped as `stale`) must have uploaded its report
# artifact. This reads the run's
# A lane that dies before writing report.json (the GPU preflight step, a
# build failure, a crash before results exist) uploads nothing. The
# harness's own preflight refusal is different: `run_suite` in
# tests/e2e-cucumber/tests/e2e.rs creates report.json empty before it
# calls `lane_verdict::preflight`, and each lane's `if: always()` upload
# step publishes it, so it shows as an EMPTY row (a post-run lane failure
# as a FAIL row with lane_failure), not as missing. `discover()` can only
# render the reports that arrived, so a lane that uploads nothing would
# drop out of the grid with no row and no FAIL. On a `continue-on-error`
# lane nothing else is red either. So check it here: every lane that ran
# (any `needs` result but `skipped`, and not self-skipped as `stale`)
# must have uploaded its report artifact. This reads the run's
# artifact list rather than e2e-artifacts/, because download-artifact
# flattens a single match into the root and loses its name. The list also
# holds earlier attempts' artifacts, so a re-run that uploads nothing can
Expand Down
2 changes: 2 additions & 0 deletions crates/e2e-report/src/components.rs
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,8 @@ const STYLE: &str = r#"
.platform-row { display: flex; align-items: center; gap: 8px; cursor: pointer; font-size: 1.05rem; }
.platform-name { font-weight: 700; flex: 1; }
.empty-note { color: #999; font-style: italic; padding: 8px 0; }
.lane-failure { color: #c62828; background: #ffebee; border-left: 4px solid #c62828;
padding: 8px 12px; margin: 8px 0; white-space: pre-wrap; }
.scenario { border: 1px solid #e0e0e0; border-top: none; padding: 12px; background: #fff; }
.scenario:last-child { border-radius: 0 0 4px 4px; }
.scenario-row { display: flex; align-items: center; gap: 8px; cursor: pointer; }
Expand Down
55 changes: 47 additions & 8 deletions crates/e2e-report/src/consolidated/html.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ use std::path::Path;
use maud::{Markup, html};

use super::{
CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING, PlatformReport,
PlatformReports, platform_cell,
CellOutcome, Grid, LEGEND_ENTRIES, MOCK_GATES_PR, OTHER_LANES_NON_BLOCKING,
PLATFORM_ATTENTION_HEADING, PlatformReport, PlatformReports, SCENARIO_ATTENTION_HEADING,
platform_cell,
};
use crate::components::{feature_group, now_utc, page_shell, stats_bar};

Expand Down Expand Up @@ -103,6 +104,7 @@ pub fn generate_consolidated(
h2 { "Platforms" }
(matrix_table(reports))
(legend())
(lane_failures_html(reports))

(expectation_grid_html(&bundle.grid))

Expand Down Expand Up @@ -193,7 +195,7 @@ pub(super) fn expectation_grid_html(grid: &Grid) -> Markup {
}
}
@if !problems.is_empty() {
h3 { "Needs attention" }
h3 { (SCENARIO_ATTENTION_HEADING) }
ul {
@for (platform_label, id, outcome, detail) in &problems {
li {
Expand Down Expand Up @@ -246,7 +248,8 @@ fn matrix_table(reports: &[PlatformReport]) -> Markup {
td.num { (fail) }
td.num { (skip) }
td.num { @if xf > 0 || r.is_known_bugs { (xf) } @else { "n/a" } }
td class=(if r.ok() { "status-pass" } else { "status-fail" }) {
td class=(if r.ok() { "status-pass" } else { "status-fail" })
title=[r.lane_failure()] {
(r.status_text())
}
td { (stats_bar(&r.stats)) }
Expand Down Expand Up @@ -288,7 +291,10 @@ fn legend() -> Markup {
" — PASS when the platform is in its expected state (every "
"expect-pass scenario passed and every known bug failed as "
"expected); FAIL on an unexpected failure or an XPASS (a known "
"bug that unexpectedly passed — remove its @expected-failure tag)."
"bug that unexpectedly passed — remove its @expected-failure tag), "
"or when the harness failed the lane after the run (for example, no "
"scenario executed); the reason is listed under \u{201c}"
(PLATFORM_ATTENTION_HEADING) "\u{201d} and in the platform's section."
}
li { b { "Xfail" } " — count of known-bug scenarios that failed as expected." }
li { b { "Skip" } " — scenarios not applicable on this platform (not run)." }
Expand All @@ -297,6 +303,26 @@ fn legend() -> Markup {
}
}

/// The platforms the harness failed after their run, with each reason: a
/// lane-check failure has no grid cell of its own (an all-skip lane has only
/// skip cells), so it is listed here.
fn lane_failures_html(reports: &[PlatformReport]) -> Markup {
let failed: Vec<(&str, &str)> = reports
.iter()
.filter_map(|r| r.lane_failure().map(|reason| (r.label.as_str(), reason)))
.collect();
html! {
@if !failed.is_empty() {
h3 { (PLATFORM_ATTENTION_HEADING) }
ul {
@for (label, reason) in &failed {
li { b { "Lane check failed" } " in " code { (label) } ": " (reason) }
}
}
}
}
}

fn platform_section(report: &PlatformReport) -> Markup {
let badge_class = if report.ok() {
"badge-pass"
Expand All @@ -310,6 +336,9 @@ fn platform_section(report: &PlatformReport) -> Markup {
span.platform-name { (report.label) }
span.elapsed { (report.stats.total) " scenarios" }
}
@if let Some(reason) = report.lane_failure() {
p.lane-failure { b { "Lane check failed: " } (reason) }
}
@if report.features.is_empty() {
p.empty-note { "No report.json data for this platform." }
} @else {
Expand All @@ -328,7 +357,7 @@ mod tests {
use crate::consolidated::{load_platform_reports, load_reports};

#[test]
fn legend_mock_and_other_platform_items_are_unchanged() {
fn legend_mock_and_other_platform_items_are_pinned() {
// Pins the two gating items byte for byte, so building them from shared
// fragments (see `MOCK_GATES_PR`) cannot change what the HTML
// report's legend says. Text only: that Mock is the required check and
Expand Down Expand Up @@ -365,12 +394,18 @@ mod tests {
// now would be building the bundle's grid from `&[]` in
// `load_platform_reports`. Give this one a platform.json via
// write_platform and assert the grid heading actually renders.
let report = feature_json(&[(&["id:serve-a"], &["passed"])]);
// `serve-b` fails though expected to pass, so the grid's attention
// list renders too.
let report = feature_json(&[
(&["id:serve-a"], &["passed"]),
(&["id:serve-b"], &["failed"]),
]);
let platform = r#"{
"platform_slug": "mock",
"capability": {"effective_serve_engine": "none"},
"expectations": [
{"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"}
{"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"},
{"id":"serve-b","feature":"Serving","scenario":"serve-02 - second","expected":"pass"}
]
}"#;
let (_d, path) = write_platform(&report, platform);
Expand All @@ -384,6 +419,10 @@ mod tests {
html.contains("Expectation grid (scenario × platform)"),
"grid heading missing from output:\n{html}"
);
assert!(
html.contains(&format!("<h3>{SCENARIO_ATTENTION_HEADING}</h3>")),
"scenario attention heading missing from output:\n{html}"
);
}

#[test]
Expand Down
Loading
Loading