From 888baab852e40ba681af154133e156392214de36 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 3 Sep 2026 13:55:50 +0000 Subject: [PATCH 1/5] fix(e2e): report gfx950 as its own platform, not as MI300X `platform_hardware_slug` matched the `-dcgpu` suffix, but two distinct data-center families normalize to that suffix: `gfx94X-dcgpu` and `gfx950-dcgpu`. A gfx950 host therefore reported `platform_slug: "mi300x"`, so a flattened single-artifact report download would file its results in the MI300X column of the cross-platform grid rather than its own. Match the family instead of the suffix. Verified on hardware: rocm-smi reports `GFX VER gfx950` for the part this affects. No behaviour change for gfx94x, which still slugs as mi300x. Signed-off-by: Tomas Saaristola --- tests/e2e-cucumber/src/capability.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 90bc4f2bb4..251e19aae5 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -532,7 +532,13 @@ fn derive_platform_slug( fn platform_hardware_slug(gfx_target: &str) -> String { let family = normalize_family(gfx_target); - if family.ends_with("-dcgpu") { + // Two distinct data-center parts normalize to a `-dcgpu` family, so match the + // family rather than the suffix: a suffix test reports gfx950 hardware as + // `mi300x`, which would file its results in the MI300X column of the report + // grid instead of its own. + if family == "gfx950-dcgpu" { + "mi350p".to_owned() + } else if family.ends_with("-dcgpu") { "mi300x".to_owned() } else if family.starts_with("gfx115") { "strix-halo".to_owned() @@ -799,6 +805,13 @@ Local model engines derive_platform_slug(true, Some("gfx942"), "linux", false), "mi300x" ); + // gfx950 normalizes to a `-dcgpu` family like gfx94x does, but it is a + // different part with its own lane and report column — it must not be + // slugged as mi300x. + assert_eq!( + derive_platform_slug(true, Some("gfx950"), "linux", false), + "mi350p" + ); // Strix Halo: same gfx1151 silicon on both OSes → distinct slugs so the // report grid gets a column per platform, not a collision. assert_eq!( From 417a528287a142608bb3e80f99410127b293cfe1 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 3 Sep 2026 13:55:50 +0000 Subject: [PATCH 2/5] ci(e2e): add an MI350P self-hosted E2E lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds per-PR and nightly lanes on an MI350P (gfx950) runner, so the suite covers a second Instinct generation alongside the MI300X lane. Modelled on the existing lanes, with two deliberate choices: - runs-on [self-hosted, linux, mi350p]. The label names the hardware rather than reusing the generic amd-gpu, because GitHub matches a job to any runner whose labels are a superset — a shared label would let the two Instinct generations pick up each other's work. - The GPU preflight floor matches the other Instinct lane rather than scaling with the card's 144GB. It only has to catch a leftover serve holding the card while letting a clean host pass on the first poll. Non-blocking (continue-on-error) like every other self-hosted lane, wired into both consolidated reports, and documented in the hardware-testing contract so the derived guards stay green. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 129 ++++++++++++++++++++++++++- .github/workflows/nightly.yml | 90 +++++++++++++++++++ crates/e2e-report/src/lib.rs | 2 + docs/ci-hardware-testing.md | 33 +++---- xtask/src/e2e_report.rs | 3 + 5 files changed, 241 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 9cf95bf19a..a22e17c0e8 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -34,7 +34,7 @@ on: type: choice default: all # Each value also selectable on its own; `all` covers every lane below. - options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, rad3] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, rad3, mi350p] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -958,6 +958,132 @@ jobs: name: e2e-gpu-rad3-report path: tests/e2e-cucumber/results/ + e2e-gpu-mi350p: + name: E2E tests (MI350P) + # 90min: matches the other Instinct lane — same collapsed suite. + timeout-minutes: 90 + # `mi350p` names the hardware rather than reusing the generic `amd-gpu`: + # GitHub matches a job to any runner whose labels are a superset, so a shared + # label would let this card pick up MI300X-targeted work and vice versa. + runs-on: [self-hosted, linux, mi350p] + needs: [changes] + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.serve == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'mi350p')) + ) + continue-on-error: true + env: + E2E_SERVE_TIMEOUT_SECS: "300" + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + # Full serve matrix at the merge-queue gate, like every other lane (see e2e-gpu). + E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The pod is usually fresh (the runner scales to zero between jobs), but the + # work volume is not, and two jobs can share one pod's life. Same reclaim as + # e2e-gpu, scoped to e2e leftovers only. + - name: Reclaim GPU from stray E2E processes + run: | + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # Bounded wait, as in e2e-gpu. The pod is admitted for exactly one card, so + # rocm-smi sees that card alone. The floor matches the other Instinct lane + # rather than scaling with the 144GB capacity: it only has to be high enough + # to catch a leftover serve still holding the card, and low enough that a + # clean pod passes on the first poll. + - name: GPU preflight (bounded wait for an available GPU) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + exit 1 + + # cache: false for the same reason as e2e-gpu — the build cache is kept in + # CARGO_TARGET_DIR on the work volume, not in GitHub's cache service. + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + + - name: Run E2E tests on MI350P + run: | + # $RUNNER_WORKSPACE is on a volume that survives the runner being scaled + # away, so the cargo target dir and the shared caches are still warm on + # the next job even though the container is not. + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # A SEPARATE volume is mounted at exactly this path by the runner's + # deployment. Keeping the ~23GB uv cache off the work volume is + # deliberate; if you change this path, change that deployment too. + export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Pre-warm one shared runtime in place before the suite. Not an + # optimization — `install sdk` bakes absolute paths into the runtime + # manifest, so installing anywhere temporary breaks every later serve. + # See e2e-gpu for the full rationale. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # See the e2e-gpu lane for why the e2e-test-hooks feature must match + # what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" + + # Pre-warm once, serially, in place (no mv/symlink), and refresh it when + # the channel index has moved on — the tree is a cache, not a one-shot. + # See the e2e-gpu lane and `xtask e2e-prewarm`. + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ + cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" + + # Optional scenario-name filter for a scoped manual dispatch — a cheap + # way to exercise this lane against a single scenario without the full suite. + NAME_FILTER="${{ github.event.inputs.name_filter }}" + if [ -n "$NAME_FILTER" ]; then + echo "name filter active: $NAME_FILTER" + cargo xtask e2e -- --name "$NAME_FILTER" + else + cargo xtask e2e + fi + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-mi350p-report + path: tests/e2e-cucumber/results/ + # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -975,6 +1101,7 @@ jobs: - e2e-gpu-strix-windows - e2e-wsl - e2e-gpu-rad3 + - e2e-gpu-mi350p # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 373e8ff411..2a18416cf0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -549,6 +549,95 @@ jobs: name: e2e-gpu-rad3-report path: tests/e2e-cucumber/results/ + e2e-gpu-nightly-mi350p: + name: E2E tests (MI350P, incl. nightly-only) + timeout-minutes: 90 + runs-on: [self-hosted, linux, mi350p] + continue-on-error: true + env: + E2E_INCLUDE_NIGHTLY: "1" + E2E_SERVE_TIMEOUT_SECS: "300" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Reclaim GPU from stray E2E processes + run: | + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # Bounded wait for a free GPU. Floor matches the per-PR MI350P lane and the + # other Instinct lane — high enough to catch a leftover serve, low enough + # that a clean pod passes on the first poll. + - name: GPU preflight (bounded wait for an available GPU) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + exit 1 + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + + - name: Run full E2E on MI350P (incl. nightly-only) + run: | + # $RUNNER_WORKSPACE is on a volume that survives the runner being scaled + # away, so the cargo target dir and shared caches stay warm. + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Separate volume mounted at exactly this path by the runner's + # deployment; if you change this path, change that deployment too. + export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" + + # Pre-warm once, serially, in place (no mv/symlink), and refresh it when + # the channel index has published a newer runtime — the tree is a cache, + # not a one-shot. See `xtask e2e-prewarm`. + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ + cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" + + cargo xtask e2e + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-mi350p-report + path: tests/e2e-cucumber/results/ + # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the # runner's storage and runtime setup aligned with ci.yml's proven Strix Ubuntu job. @@ -910,6 +999,7 @@ jobs: needs: - e2e-gpu-nightly - e2e-gpu-nightly-rad3 + - e2e-gpu-nightly-mi350p - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows - e2e-wsl-nightly diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index db0c4ac957..a89fcd9fbd 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -413,6 +413,7 @@ fn parse_descriptor(name: &str) -> Descriptor { "" | "report" => ("Mock", "Linux"), "gpu" => ("MI300X", "Linux"), "gpu-rad3" => ("R9700", "Linux"), + "gpu-mi350p" => ("MI350P", "Linux"), "gpu-strix-ubuntu" => ("Strix Halo", "Ubuntu"), "gpu-strix-windows" => ("Strix Halo", "Windows"), // Same silicon again, third host boundary: an Ubuntu distro under WSL2 on @@ -2219,6 +2220,7 @@ mod tests { ("e2e-report", "Mock", "Linux"), ("e2e-gpu-report", "MI300X", "Linux"), ("e2e-gpu-rad3-report", "R9700", "Linux"), + ("e2e-gpu-mi350p-report", "MI350P", "Linux"), ("e2e-gpu-strix-ubuntu-report", "Strix Halo", "Ubuntu"), ("e2e-gpu-strix-windows-report", "Strix Halo", "Windows"), // Must not fall through to `fallback_descriptor`, which would render diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 7942a434f9..4a0a5fca50 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -37,6 +37,7 @@ separate tier flag or tag filter to maintain. | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on native Windows 11 | self-hosted `[self-hosted, windows, strix-halo, native]` | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, linux, strix-halo, wsl]` | | `e2e-gpu-rad3` | `e2e-selfhosted.yml` | Radeon AI PRO R9700 (gfx1201) on Linux | self-hosted `[self-hosted, linux, r9700]` | +| `e2e-gpu-mi350p` | `e2e-selfhosted.yml` | MI350P (AMD Instinct, gfx950) on Linux | self-hosted `[self-hosted, linux, mi350p]` | The Strix Halo lanes pin the extra `native` label because two Linux runners share the `strix-halo` label (a native host and a WSL host) and the jobs' @@ -55,10 +56,10 @@ named for. resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. -The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-rad3`) run on AMD GPU systems, so they exercise -host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios -that the mock job cannot. GPU availability is advisory in the WSL lane, as +The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, +`e2e-wsl`, `e2e-gpu-rad3`, and `e2e-gpu-mi350p`) run on AMD GPU systems, so they +exercise host/GPU detection, engine `detect`/`capabilities`, and live serving +scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as described below. `e2e-wsl` runs on an Ubuntu distro hosted in WSL2 on the Strix Halo Windows box @@ -100,10 +101,11 @@ reports — including partial or failed runs — by scenario id into one HTML re and GitHub step summary. The lane artifacts are named canonically (`e2e-report`, `e2e-gpu-report`, -`e2e-gpu-rad3-report`, `e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`, -`e2e-gpu-strix-wsl-report`) in every workflow, because the report derives each -platform's name and OS from the artifact name. An unrecognised name renders as a -guessed platform on Linux, which would report a Windows lane as Linux; `xtask`'s +`e2e-gpu-rad3-report`, `e2e-gpu-mi350p-report`, `e2e-gpu-strix-ubuntu-report`, +`e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-report`) in every workflow, +because the report derives each platform's name and OS from the artifact name. +An unrecognised name renders as a guessed platform on Linux, which would report +a Windows lane as Linux; `xtask`'s `every_uploaded_e2e_artifact_has_a_name_the_report_can_label` guards against it. ## Triggers @@ -128,11 +130,11 @@ They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: - `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, - `strix-wsl`, `rad3`) — which self-hosted job(s) to run. `app-dev-gpu` maps to - `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to - `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, and `rad3` to - `e2e-gpu-rad3`. (The mock lane has its own `platform` input on `ci.yml`; it is - not part of this workflow.) + `strix-wsl`, `rad3`, `mi350p`) — which self-hosted job(s) to run. `app-dev-gpu` + maps to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to + `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, `rad3` to + `e2e-gpu-rad3`, and `mi350p` to `e2e-gpu-mi350p`. (The mock lane has its own + `platform` input on `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a single scenario instead of the full suite. Empty runs everything applicable @@ -214,8 +216,9 @@ the pre-warm block is duplicated across multiple jobs in two shells; ## Blocking vs. non-blocking The self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-rad3` — all run with `continue-on-error: true`, so a -hardware failure that RUNS never gates a PR merge. Their results still surface +`e2e-gpu-strix-windows`, `e2e-wsl`, `e2e-gpu-rad3`, and `e2e-gpu-mi350p` — all run with +`continue-on-error: true`, so a hardware failure that RUNS never gates a PR +merge. Their results still surface in the self-hosted consolidated report for visibility. ### Timeouts on the shared Strix box diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 3a70f99cdc..6fda5e74ae 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -132,6 +132,7 @@ fn label_for_root_report(dir: &Path) -> String { Some("mock") => "e2e-report".to_owned(), Some("mi300x") => "e2e-gpu-report".to_owned(), Some("gfx1201") => "e2e-gpu-rad3-report".to_owned(), + Some("mi350p") => "e2e-gpu-mi350p-report".to_owned(), Some("strix-halo-linux") => "e2e-gpu-strix-ubuntu-report".to_owned(), Some("strix-halo-windows") => "e2e-gpu-strix-windows-report".to_owned(), // This workflow is statically pinned to the Strix WSL runner. A bare @@ -188,6 +189,7 @@ mod tests { "e2e-report", "e2e-gpu-report", "e2e-gpu-rad3-report", + "e2e-gpu-mi350p-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-windows-report", "e2e-gpu-strix-wsl-report", @@ -300,6 +302,7 @@ mod tests { for (slug, expected) in [ ("mi300x", "e2e-gpu-report"), ("gfx1201", "e2e-gpu-rad3-report"), + ("mi350p", "e2e-gpu-mi350p-report"), ("strix-halo-linux", "e2e-gpu-strix-ubuntu-report"), ("strix-halo-windows", "e2e-gpu-strix-windows-report"), ("strix-halo-wsl", "e2e-gpu-strix-wsl-report"), From 5cf4f0f311c86b4af1fb89dc61fe7b6d0311f498 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 8 Sep 2026 10:38:20 +0000 Subject: [PATCH 3/5] docs(e2e): derive the README lane lists from the workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README's nightly sentence was guarded by a literal string match against a copy of that sentence in the test source, so the guard and the doc could go stale together while staying green — which is what happened when the R9700 lane landed, and would have happened again here. Its own failure message ("must identify every nightly job platform") had already stopped being true. Derive it instead, the way the four assertions above it already derive from the YAML: the sentence now names the nightly job ids in backticks and the guard compares them against self_hosted_e2e_jobs(nightly.yml). The CI job table in the same file was unguarded and had drifted further — missing e2e-gpu-rad3 as well as the new lane. Add both rows and derive that table too, against e2e-selfhosted.yml. Both new assertions were confirmed to fail with a lane removed from the doc. Signed-off-by: Tomas Saaristola --- tests/e2e-cucumber/README.md | 16 +++++---- xtask/src/workflow_contract.rs | 62 +++++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 12a21bca67..5b9b6b7b13 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -172,6 +172,8 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | +| `e2e-gpu-rad3` | `e2e-selfhosted.yml` | Radeon R9700 (self-hosted) | no | +| `e2e-gpu-mi350p` | `e2e-selfhosted.yml` | MI350P (self-hosted) | no | The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow @@ -182,12 +184,14 @@ Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to skip there. -The nightly workflow runs non-blocking jobs — MI300X, Radeon R9700, and Strix -Halo on Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`, then -consolidates them into the same cross-platform grid. The -shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on MI300X and -the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint -through Lemonade on Strix Halo. +The nightly workflow covers the same hardware as the table above, as +non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-rad3`, +`e2e-gpu-nightly-mi350p`, `e2e-gpu-nightly-strix`, +`e2e-gpu-nightly-strix-windows`, `e2e-wsl-nightly`) with +`E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform +grid. The shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on +MI300X and the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` +checkpoint through Lemonade on Strix Halo. Use the self-hosted E2E workflow dispatch to run either model independently on a ref (the GPU platform / `include_nightly` / `name_filter` inputs live on diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 05b8bc87dc..5199feace0 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1006,14 +1006,68 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" ); } + // The README's own lane lists were the one hand-copied hole in this + // otherwise derived net: the nightly sentence was asserted by literal + // string match against a copy in this test source, so both could go + // stale together while the guard stayed green — which is exactly what + // happened when the R9700 lane landed. Derive both from the YAML. + // + // Whitespace is normalized first because these are prose sentences and + // a table, wrapped for readability; the lane lists must survive a + // reflow that does not change what the doc says. let readme = std::fs::read_to_string(repo_root().join("tests/e2e-cucumber/README.md")) .expect("read E2E README"); + let readme_flat = normalized_whitespace(&readme); + + let nightly = read_workflow("nightly.yml"); + let nightly_lanes = self_hosted_e2e_jobs(&nightly); assert!( - normalized_whitespace(&readme).contains( - "The nightly workflow runs non-blocking jobs — MI300X, Radeon R9700, and Strix Halo on Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`" - ), - "E2E README must identify every nightly job platform" + !nightly_lanes.is_empty(), + "expected at least one self-hosted job in nightly.yml (extractor sanity check)" + ); + let nightly_ids: Vec = nightly_lanes.iter().map(|(job, _)| job.clone()).collect(); + assert_eq!( + backticked_list_between(&readme_flat, "as non-blocking lanes (", ") with"), + nightly_ids, + "the E2E README must name every nightly self-hosted lane, in workflow order" + ); + + // The README's CI job table is the per-PR view: the blocking mock job + // from ci.yml, then one row per self-hosted lane. Only the self-hosted + // rows are derivable, so the mock row is matched by workflow and the + // rest are compared against e2e-selfhosted.yml. + let readme_rows = markdown_table_rows(&readme, "| Job | Workflow | Platform | Blocking |"); + let (mock_rows, self_hosted_rows): (Vec<_>, Vec<_>) = readme_rows + .into_iter() + .partition(|row| row.get(1).is_some_and(|workflow| workflow == "`ci.yml`")); + assert_eq!( + mock_rows + .iter() + .map(|row| row[0].clone()) + .collect::>(), + vec!["`e2e`".to_owned()], + "the README CI job table must carry exactly one blocking mock row" + ); + assert_eq!( + self_hosted_rows + .iter() + .map(|row| row[0].clone()) + .collect::>(), + lanes + .iter() + .map(|(job, _)| format!("`{job}`")) + .collect::>(), + "the README CI job table must have one row per self-hosted job in \ + e2e-selfhosted.yml, in workflow order" ); + for row in &self_hosted_rows { + assert_eq!( + row[3], "no", + "self-hosted lane `{}` is continue-on-error, so the README must not \ + document it as blocking", + row[0] + ); + } } #[test] From c446667811a92ce76565e14c479ad6d37be66a2e Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 8 Sep 2026 10:38:26 +0000 Subject: [PATCH 4/5] docs(e2e-report): stop enumerating platforms in the report legend Both the Markdown step summary and the HTML legend named the self-hosted platforms by hand ("MI300X / Strix Halo"). R9700 was already missing and MI350P makes three lanes the legend implicitly disclaims while the report displays their results. Since the list is hand-maintained prose with nothing guarding it, stop enumerating rather than adding a name per lane: the report's own rows and columns are the list. Signed-off-by: Tomas Saaristola --- crates/e2e-report/src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index a89fcd9fbd..5d0d3e157e 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -1299,8 +1299,10 @@ pub fn consolidated_summary_markdown(inputs: &[(String, PathBuf)]) -> String { forwarding — with no GPU, no model download, and no engine process, so it \ runs on a GitHub-hosted runner. It **gates the PR**: it runs on every push, \ and if it fails the PR's required check goes red and the PR cannot merge. \ - **MI300X / Strix Halo** run on real self-hosted GPU hardware with real \ - engines. They are **non-blocking**: they still run and are reported here, but \ + **Every other platform** is a real self-hosted GPU host running real \ + engines — the rows below are the list, so naming them here would only go \ + stale as lanes are added. They are **non-blocking**: they still run and \ + are reported here, but \ a failure does NOT block the PR from merging (the hardware/runners are still \ being proven out, so their results are informational rather than a merge \ gate).\n\n\ @@ -1936,7 +1938,7 @@ fn legend() -> Markup { "and gates the PR." } li { - b { "MI300X / Strix Halo" } + b { "Every other platform" } " — real self-hosted GPU hardware; non-blocking while proven out." } li { From fd0ceaf5a6835c3fc47b2680cb33abb997d3b5b0 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 8 Sep 2026 10:38:32 +0000 Subject: [PATCH 5/5] test(e2e): cover every gfx form that reaches the slug mapping The slug assertions pinned only the raw gfx942/gfx950 targets, leaving the branch this PR restructured partly uncovered. Table-drive them so the family-label forms (gfx94X-dcgpu, gfx950-dcgpu), which take normalize_family's early -dcgpu return, and the suffixed real-world form (gfx950:sramecc+:xnack-), which depends on its starts_with, are pinned too. The suffixed case is the one with no other guard: tightening that starts_with to an == would send a real gfx950 host back into the mi300x column, and the probe emitting a bare gfx950 today is what hides it. Confirmed the new table fails under exactly that change. Signed-off-by: Tomas Saaristola --- tests/e2e-cucumber/src/capability.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 251e19aae5..3b01a5d64b 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -801,17 +801,28 @@ Local model engines #[test] fn platform_slug_derivation() { assert_eq!(derive_platform_slug(false, None, "other", false), "mock"); - assert_eq!( - derive_platform_slug(true, Some("gfx942"), "linux", false), - "mi300x" - ); // gfx950 normalizes to a `-dcgpu` family like gfx94x does, but it is a // different part with its own lane and report column — it must not be // slugged as mi300x. - assert_eq!( - derive_platform_slug(true, Some("gfx950"), "linux", false), - "mi350p" - ); + // + // Cover every form that reaches `platform_hardware_slug`, not just the + // bare target the probe happens to emit today: the family label takes + // `normalize_family`'s early `-dcgpu` return, while the suffixed form + // depends on its `starts_with` — an `==` "tidy-up" there would silently + // send a real gfx950 host back into the `mi300x` column. + for (gfx_target, expected) in [ + ("gfx950", "mi350p"), + ("gfx950-dcgpu", "mi350p"), + ("gfx950:sramecc+:xnack-", "mi350p"), + ("gfx942", "mi300x"), + ("gfx94X-dcgpu", "mi300x"), + ] { + assert_eq!( + derive_platform_slug(true, Some(gfx_target), "linux", false), + expected, + "gfx target `{gfx_target}` must slug as `{expected}`" + ); + } // Strix Halo: same gfx1151 silicon on both OSes → distinct slugs so the // report grid gets a column per platform, not a collision. assert_eq!(