From c1d5dd8af62638e85ad45ea56395aeeacbfc4acf Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Tue, 10 Nov 2026 13:12:47 +0000 Subject: [PATCH 01/16] ci: run pull requests on one Linux and one Windows GPU node A PR now starts the MI300X lane (the only lane where vLLM is the effective engine) and the Strix Halo Windows pool lanes (native + WSL2). The Strix Halo Ubuntu and W7900 lanes skip pull_request but keep push/merge_group/dispatch. nightly.yml's Strix Halo Linux lane targeted the offline static `native` host and never ran; point it at the DevLab Dispatch pool the per-PR lane uses and pin that in the workflow contract test. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 15 +++++++++++++++ .github/workflows/nightly.yml | 28 ++++++++++------------------ docs/ci-hardware-testing.md | 8 ++++++++ xtask/src/workflow_contract.rs | 23 +++++++++++++++++------ 4 files changed, 50 insertions(+), 24 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 964216e06..02d6fb730 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -9,6 +9,12 @@ name: E2E self-hosted # THIS workflow's supersession, never the required checks in ci.yml. Only # pull_request runs are superseded (see `concurrency:` below). # +# Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, +# MI300X: the only lane where vLLM is the effective engine) and one Windows GPU +# node (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix +# Ubuntu and W7900 lanes keep their push / merge_group / dispatch triggers and +# run nightly (nightly.yml); they just no longer start on pull_request. +# # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not # because of `continue-on-error`, which is `false` on every lane but @@ -470,11 +476,16 @@ jobs: runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. + # Not on pull_request: a PR gets exactly one Linux GPU node (MI300X, the only + # lane where vLLM runs) and one Windows node (the Strix Halo pool). This lane + # still runs on push/merge_group/dispatch and nightly (nightly.yml, + # e2e-gpu-nightly-strix, same pool labels). if: >- always() && needs.changes.result == 'success' && ( (github.event_name != 'workflow_dispatch' + && github.event_name != 'pull_request' && needs.changes.outputs.serve == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')) @@ -1599,11 +1610,15 @@ jobs: fail-fast: false matrix: sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} + # Not on pull_request: one runner on borrowed quota, 58-70% of its PR jobs + # never started before being superseded. Still runs on push/merge_group/ + # dispatch and nightly (nightly.yml, e2e-gpu-nightly-w7900). if: >- always() && needs.changes.result == 'success' && ( (github.event_name != 'workflow_dispatch' + && github.event_name != 'pull_request' && needs.changes.outputs.serve == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'w7900')) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7e6bd1292..cd09a1f72 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -768,23 +768,18 @@ jobs: # it, so the "Report whether Docker is reachable" step logs the answer. If it # does not, no nightly lane runs AIM serve and a pool-based one is needed. timeout-minutes: 120 - # `native` disambiguates this runner from any host that might carry a bare - # `wsl` label alongside `strix-halo` (the static WSL runner this repo used - # before the WSL lanes moved to the ephemeral pool) -- the paths below - # exist only on the native host, whether or not that other runner remains - # registered. - runs-on: [self-hosted, linux, strix-halo, native] + # The DevLab Dispatch pool, identical to the per-PR e2e-gpu-strix-ubuntu + # lane in e2e-selfhosted.yml (pinned by workflow_contract.rs). This lane used + # to target the static `native` host, which is offline: every nightly sat + # queued until the 12h run cap and the lane never ran. Pool hosts are cold, + # so there are no host-specific HOME or cache-dir overrides. + runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] continue-on-error: true strategy: fail-fast: false matrix: channel: [release, nightly] env: - HOME: /home/ubuntu/actions-runner/e2e-home - CARGO_HOME: /home/ubuntu/actions-runner/.cargo - RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup - TMPDIR: /home/ubuntu/actions-runner/tmp - PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -793,11 +788,8 @@ jobs: with: persist-credentials: false - - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs - run: | - mkdir -p "$HOME" "$TMPDIR" "$PIP_CACHE_DIR" - echo "prepared" - bash scripts/reclaim-gpu.sh + - name: Reclaim GPU from stray E2E processes + run: bash scripts/reclaim-gpu.sh # Bounded wait for a free GPU, then an assertion that no leaked E2E engine # process survives. gfx1151 shares system memory, so the floor is smaller @@ -815,11 +807,11 @@ jobs: - name: Ensure Rust toolchain run: | - if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | sh -s -- -y --no-modify-path --default-toolchain none fi - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" # serve-26/27 (AIM) need Docker reachable on this host; without it they skip # silently. Say which it is, so a nightly that never ran them is visible. diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 10768bdd7..e75155149 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -44,6 +44,14 @@ R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only rung instead of being removed outright. +A pull request now runs on exactly one Linux GPU node and one Windows GPU +node: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and +the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo +Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer +start on `pull_request`; they keep their `push`/`merge_group`/dispatch +triggers and run nightly (`e2e-gpu-nightly-strix` on the same DevLab Dispatch +pool labels, `e2e-gpu-nightly-w7900`). + [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length of a job, and publishes the result; diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 291d45029..9c1cf2b0e 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1620,6 +1620,10 @@ jobs: for (per_pr_job, nightly_job) in [ ("e2e-wsl", "e2e-wsl-nightly"), ("e2e-gpu-strix-windows", "e2e-gpu-nightly-strix-windows"), + // Off the PR path since the two-node PR scope, so the nightly is where + // this lane runs on a schedule: it must not drift back to the + // offline static `native` host. + ("e2e-gpu-strix-ubuntu", "e2e-gpu-nightly-strix"), ] { let per_pr = runs_on_values(job_block(&sh, per_pr_job)); let nightly_values = runs_on_values(job_block(&nightly, nightly_job)); @@ -7679,12 +7683,7 @@ jobs: !job_block(&sh, "e2e-gpu").contains("E2E_MERGE_QUEUE"), "e2e-gpu never runs in the merge queue, so it must not opt into @merge-queue scenarios" ); - for job in [ - "e2e-gpu-strix-ubuntu", - "e2e-gpu-strix-windows", - "e2e-wsl", - "e2e-gpu-w7900", - ] { + for job in ["e2e-gpu-strix-windows", "e2e-wsl"] { // Only the dispatch split may look at the event: any other event // condition could silently keep the lane out of the merge queue. let gate = normalized_whitespace(&gate(job)); @@ -7693,6 +7692,18 @@ jobs: "`{job}` must keep running in the merge queue (gate: `{gate}`)" ); } + // A pull request runs one Linux node (MI300X) and one Windows node (the + // Strix Halo pool). These two lanes are off the pull_request path but must + // still run in the merge queue, so the gate may name `pull_request` and + // nothing else about the event. + for job in ["e2e-gpu-strix-ubuntu", "e2e-gpu-w7900"] { + let gate = normalized_whitespace(&gate(job)); + assert!( + gate.contains("github.event_name != 'pull_request'") + && !gate.contains("merge_group"), + "`{job}` must skip pull_request but keep running in the merge queue (gate: `{gate}`)" + ); + } } #[test] From bfd8ce8602401cd6b4aaf45b342f77360655ebea Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 00:37:42 +0000 Subject: [PATCH 02/16] docs: say which per-PR lanes skip pull_request Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index e75155149..369ee7e5b 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,7 +36,7 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below (`e2e-gpu-strix-ubuntu` runs both its `[release, nightly]` channel legs per PR, ROCMAI-125) | No — absent from the required-status-check list | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` (see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | | Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The same per-PR lanes | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | @@ -393,7 +393,7 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and ## Triggers The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, -`pull_request`, and `merge_group` (every GPU lane except MI300X, which is not +`pull_request` (not `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and `merge_group` (every GPU lane except MI300X, which is not required for merging and so is skipped in the merge queue rather than left queued on limited runner capacity after its PR has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request From 640940249340f930eb15ef9cdbb1b7e1924aaf0c Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 12:16:27 +0000 Subject: [PATCH 03/16] docs: release-candidate gate runs the full self-hosted matrix The row said 'the same per-PR lanes', which now means the narrowed three-lane set. A push to release/** still runs all five lanes. Also drop the paragraph that said the Ubuntu nightly lane still pins the offline `native` host. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 369ee7e5b..9e42c1544 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -38,7 +38,7 @@ differs is when each runs and what hardware it covers. |---|---|---|---|---| | Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` (see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The same per-PR lanes | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which only skip `pull_request` | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only @@ -148,22 +148,15 @@ cloning the checkout into the guest's native filesystem to avoid building against the slow DrvFs mount. The distro is deliberately left without ROCm's WSL driver bridge — see "What the WSL distro needs" below. -This migration is scoped to the per-PR lanes in this table, plus the nightly -WSL and Windows Strix lanes below — not every nightly lane. `nightly.yml`'s -Ubuntu Strix lane (`e2e-gpu-nightly-strix`) still pins the `native` label and -runs on the same static, always-on host these per-PR lanes moved off of — -that label still means what it always did there: `native` disambiguates this -host from any runner that might carry a bare `wsl` label alongside -`strix-halo` (the static WSL runner this repo used before the WSL lanes below -moved to the ephemeral pool). Whether that runner remains registered is not -something this migration changes either way, since nothing here targets a -`wsl`-labeled runner anymore. `nightly.yml`'s `e2e-wsl-nightly` and -`e2e-gpu-nightly-strix-windows`, however, both moved onto the same DevLab -Dispatch pool as the per-PR `e2e-wsl`/`e2e-gpu-strix-windows` lanes above, for -the same reason: neither needs a dedicated `native`- or `wsl`-labeled runner, -so there is nothing pool-incompatible about either. Moving the remaining -`native` Ubuntu nightly lane onto the pool as well is a separate, not-yet-made -decision. +The Strix Halo lanes in this table run on the DevLab Dispatch pool, and so do +their nightly counterparts: `nightly.yml`'s `e2e-gpu-nightly-strix` (Ubuntu), +`e2e-gpu-nightly-strix-windows` and `e2e-wsl-nightly` all use the same +`runs-on` labels as `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows` and +`e2e-wsl` (pinned by `nightly_devlab_dispatch_lanes_share_the_per_pr_pool_labels` +in `xtask/src/workflow_contract.rs`). The Ubuntu nightly lane used to pin the +static `native` label; that host is offline, so the lane sat queued until the +run cap and never ran. Whether a `native`- or `wsl`-labeled runner remains +registered no longer matters: nothing targets one. Every label in a `runs-on` must narrow the pool to one kind of hardware. In particular the MI300X lane pins `mi300x` rather than `amd-gpu`: `amd-gpu` is From 6235d9fb8a4984b01678084f1c6bf68836146236 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 12:40:24 +0000 Subject: [PATCH 04/16] ci: pin the whole per-PR gate; guard the nightly Strix job; fix stale coverage docs - Contract test asserts the full if: gate of e2e-gpu-strix-ubuntu and e2e-gpu-w7900, so the pull_request exclusion must sit in the non-dispatch branch and nothing else about the event may change. - New test: the nightly Strix Ubuntu job sets no static-host paths and reads no CARGO_HOME/RUSTUP_HOME/TMPDIR/PIP_CACHE_DIR it does not set. - Docs: the cucumber README, testing.md, the nightly-channel paragraph and the Triggers paragraph no longer claim Strix Ubuntu / W7900 run per PR. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 18 ++++++----- docs/testing.md | 4 +-- scripts/reclaim-gpu.sh | 7 +++-- tests/e2e-cucumber/README.md | 4 +++ xtask/src/workflow_contract.rs | 55 +++++++++++++++++++++++++++++----- 5 files changed, 69 insertions(+), 19 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 9e42c1544..b506fac7f 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -128,12 +128,16 @@ separate tier flag or tag filter to maintain. | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | AMD Radeon PRO W7900 (gfx1100, 48GB) on Linux | self-hosted `[self-hosted, linux, w7900]` | +On `pull_request`, `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` are skipped; the +other lanes above run (one Linux node, MI300X, and the Windows Strix Halo pool). +Both still run on `push`, `merge_group`, dispatch and nightly. + `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, -nightly]` axis — two concurrent legs on the same job, channel-suffixed -artifact names — so the per-PR smoke gate proves the nightly channel boots on -at least one platform without adding the axis (and its wall-clock cost) to -every per-PR lane. This mirrors `nightly.yml`'s channel matrix from -ROCMAI-429, applied to a single per-PR lane rather than all of them. +nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact +names, so that lane proves the nightly package channel boots. It no longer runs +on `pull_request`, so on a PR no lane exercises the nightly channel; it runs on +`push`, `merge_group`, dispatch and in `nightly.yml`'s channel matrix +(ROCMAI-429). All three Strix Halo lanes run on the AMD Ryzen DevLab Dispatch pool: a fresh runner is registered per job and destroyed after, opt-in only via the @@ -386,8 +390,8 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and ## Triggers The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, -`pull_request` (not `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and `merge_group` (every GPU lane except MI300X, which is not -required for merging and so is skipped in the merge queue rather than left +`pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and +`merge_group` (every GPU lane except MI300X, which is not required for merging and so is skipped in the merge queue rather than left queued on limited runner capacity after its PR has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a diff --git a/docs/testing.md b/docs/testing.md index 2127f5255..74ebc74be 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1473,8 +1473,8 @@ install. `serve-26` and `serve-27` (`features/model_serving.feature`, the AIM serve and restart canaries) are `@nightly` for cost: each pulls the AIM image and a 9B checkpoint on a cold runner and may wait up to 1200s for readiness, which on the -per-PR Strix Ubuntu job (about 25 minutes without them, 45 minute cap) used up -the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable +Strix Ubuntu job when it still ran per PR (about 25 minutes without them, 45 +minute cap) used up the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` is set on such a host: the nightly Strix Halo lane, or an `include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then gets a 105 minute diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 90ca4b192..0db3a3b10 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -46,9 +46,10 @@ set -euo pipefail # The scenario root stays ABSOLUTE, and follows TMPDIR. # # Scenario dirs come from `TempDir::with_prefix("rocm-e2e-")`, which builds on -# `std::env::temp_dir()` and so follows TMPDIR; the nightly Strix lane redirects -# it to `$HOME/actions-runner/tmp`. A bare `/tmp/rocm-e2e` kept matching there -# only by accident — that directory is itself named `tmp`, so the substring +# `std::env::temp_dir()` and so follows TMPDIR; a lane on a host that redirects +# it (the nightly Strix lane did, to `$HOME/actions-runner/tmp`, before it moved +# to the pool) is the case this handles. A bare `/tmp/rocm-e2e` kept matching +# there only by accident — that directory is itself named `tmp`, so the substring # still appeared — and would silently stop matching under any redirect not # named `tmp`. Deriving the redirected form fixes that for ANY target. # diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index f3e9ea9bd..3e9ff2249 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -181,6 +181,10 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | +`e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux and one +Windows GPU node per PR); they still run on `push`, `merge_group`, dispatch and +nightly. + The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow consolidates its own platforms: `ci.yml`'s `e2e-report` the mock platform, diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 9c1cf2b0e..7092dc398 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7694,14 +7694,55 @@ jobs: } // A pull request runs one Linux node (MI300X) and one Windows node (the // Strix Halo pool). These two lanes are off the pull_request path but must - // still run in the merge queue, so the gate may name `pull_request` and - // nothing else about the event. - for job in ["e2e-gpu-strix-ubuntu", "e2e-gpu-w7900"] { - let gate = normalized_whitespace(&gate(job)); + // still run on push and in the merge queue. Pin the WHOLE gate, not a + // substring: the `pull_request` exclusion must sit in the non-dispatch + // branch (so a dispatch can still select the lane) and nothing else about + // the event may be named, so a gate that dropped push/merge_group some + // other way fails here too. + for (job, platform) in [ + ("e2e-gpu-strix-ubuntu", "strix-ubuntu"), + ("e2e-gpu-w7900", "w7900"), + ] { + let expected = format!( + "always() && needs.changes.result == 'success' && ( \ + (github.event_name != 'workflow_dispatch' \ + && github.event_name != 'pull_request' \ + && needs.changes.outputs.serve == 'true') \ + || (github.event_name == 'workflow_dispatch' \ + && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" + ); + assert_eq!( + normalized_whitespace(&gate(job)), + normalized_whitespace(&expected), + "`{job}` must skip pull_request in its non-dispatch branch and keep running \ + on push, merge_group and dispatch" + ); + } + } + + /// The nightly Strix Ubuntu lane moved from a static host (HOME, CARGO_HOME, + /// RUSTUP_HOME, TMPDIR and PIP_CACHE_DIR pointed at `/home/ubuntu/actions-runner`) + /// to the DevLab Dispatch pool. A pool host has none of those paths, so the job + /// must neither set them nor read one it does not set: an unset variable is + /// empty, not defaulted, and a step that built a path from it would write to `/`. + #[test] + fn nightly_strix_ubuntu_job_holds_no_static_host_paths() { + let nightly = read_workflow("nightly.yml"); + let block = job_block(&nightly, "e2e-gpu-nightly-strix"); + let env = job_mapping(block, "env"); + let uncommented: String = block + .lines() + .filter(|line| !line.trim_start().starts_with('#')) + .collect::>() + .join("\n"); + assert!( + !uncommented.contains("actions-runner"), + "the pool host has no /home/ubuntu/actions-runner tree" + ); + for var in ["CARGO_HOME", "RUSTUP_HOME", "TMPDIR", "PIP_CACHE_DIR"] { assert!( - gate.contains("github.event_name != 'pull_request'") - && !gate.contains("merge_group"), - "`{job}` must skip pull_request but keep running in the merge queue (gate: `{gate}`)" + env.contains_key(var) || !uncommented.contains(var), + "e2e-gpu-nightly-strix reads {var} but no longer sets it" ); } } From c88d7732841b86b3f85a56995c1f1958e7a9b892 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 13:07:06 +0000 Subject: [PATCH 05/16] ci: tighten the nightly Strix path guard; fix remaining per-PR Strix/W7900 claims - nightly_strix_ubuntu_job_holds_no_static_host_paths now also rejects any $HOME read other than the $HOME/.cargo/bin Rust bootstrap. - Docs and workflow comments no longer call the Strix Ubuntu / W7900 lanes per-PR; reflow two overlong prose lines. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 2 +- .github/workflows/nightly.yml | 6 +++--- docs/ci-hardware-testing.md | 7 ++++--- docs/testing.md | 8 ++++---- xtask/src/workflow_contract.rs | 18 ++++++++++++++++-- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 02d6fb730..78d4f8c56 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -497,7 +497,7 @@ jobs: channel: [release, nightly] # See `e2e-gpu` (ROCMAI-430): layered alongside the pre-existing # `channel` dimension above, not replacing it — this lane is the only - # per-PR job with two independent matrix axes. + # job with two independent matrix axes. sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # The nightly channel only carries the latest (pre-)release builds, so # it cannot resolve the n-1/n-2 released versions the matrix pins diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cd09a1f72..30b9dbcf6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -671,9 +671,9 @@ jobs: name: e2e-gpu-mi350p-${{ matrix.channel }}-report path: tests/e2e-cucumber/results/ - # W7900 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-w7900 lane - # in e2e-selfhosted.yml (w7900 label, shared uv cache, GPU preflight), with - # @nightly scenarios enabled. Keep the two in step: the report crate asserts + # W7900 counterpart to e2e-gpu-nightly. Mirrors the e2e-gpu-w7900 lane (which + # no longer runs on pull_request) in e2e-selfhosted.yml (w7900 label, shared + # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step: the report crate asserts # the nightly lanes publish the same platforms as the per-PR lanes. e2e-gpu-nightly-w7900: name: E2E tests (W7900, incl. nightly-only) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index b506fac7f..da0449518 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -376,7 +376,7 @@ run there; WSL reports `os_family` `"linux"` and stays vLLM-eligible); otherwise `vllm` is only preferred for the `*-dcgpu` families and `gfx906`/`gfx908`/`gfx90a`. Consequences for the lanes above: -- Strix Halo is `gfx1151` on every OS, so all three per-PR Strix lanes +- Strix Halo is `gfx1151` on every OS, so all three Strix lanes (Ubuntu, Windows, WSL2) and their nightly counterparts are lemonade-only — no matrix value turns a Strix lane into a vLLM lane. - `e2e-gpu` (MI300X) is the only per-PR lane where vLLM is the effective @@ -391,8 +391,9 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, `pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and -`merge_group` (every GPU lane except MI300X, which is not required for merging and so is skipped in the merge queue rather than left -queued on limited runner capacity after its PR has merged) when the workflow's +`merge_group` (every GPU lane except MI300X, which is not required for merging +and so is skipped in the merge queue rather than left queued on limited runner +capacity after its PR has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy diff --git a/docs/testing.md b/docs/testing.md index 74ebc74be..96e02cc6f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1474,10 +1474,10 @@ install. restart canaries) are `@nightly` for cost: each pulls the AIM image and a 9B checkpoint on a cold runner and may wait up to 1200s for readiness, which on the Strix Ubuntu job when it still ran per PR (about 25 minutes without them, 45 -minute cap) used up the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable -and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` -is set on such a host: the nightly Strix Halo lane, or an `include_nightly` -dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then gets a 105 minute +minute cap) used up the whole budget. They still need `@requires-engine:aim`, +i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where +`E2E_INCLUDE_NIGHTLY=1` is set on such a host: the nightly Strix Halo lane, or an +`include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that they stay reachable when the opt-in is set. diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 7092dc398..95637e3f3 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7723,8 +7723,10 @@ jobs: /// The nightly Strix Ubuntu lane moved from a static host (HOME, CARGO_HOME, /// RUSTUP_HOME, TMPDIR and PIP_CACHE_DIR pointed at `/home/ubuntu/actions-runner`) /// to the DevLab Dispatch pool. A pool host has none of those paths, so the job - /// must neither set them nor read one it does not set: an unset variable is - /// empty, not defaulted, and a step that built a path from it would write to `/`. + /// must not name the static tree, must not read one of those four variables it + /// does not set (an unset variable is empty, not defaulted), and may read + /// `$HOME` only for the Rust bootstrap under `$HOME/.cargo/bin`. This is a + /// textual guard over the job's non-comment lines, not proof the job runs. #[test] fn nightly_strix_ubuntu_job_holds_no_static_host_paths() { let nightly = read_workflow("nightly.yml"); @@ -7745,6 +7747,18 @@ jobs: "e2e-gpu-nightly-strix reads {var} but no longer sets it" ); } + for (index, _) in uncommented + .match_indices("$HOME") + .chain(uncommented.match_indices("${HOME")) + { + let rest = &uncommented[index..]; + assert!( + rest.starts_with("$HOME/.cargo/bin"), + "e2e-gpu-nightly-strix reads $HOME outside the Rust bootstrap, which a pool \ + host does not provision: `{}`", + rest.lines().next().unwrap_or_default().trim() + ); + } } #[test] From 9eb279926276def30243855829f74324f1380cdf Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 14:28:47 +0000 Subject: [PATCH 06/16] ci: drop W7900 from merge_group; fix the last per-PR Strix claims - e2e-gpu-w7900 no longer runs in the merge queue (one runner on borrowed quota, the queue does not wait for it); it still runs on push, dispatch and nightly. The contract test pins each lane's whole gate and that the W7900 job holds no E2E_MERGE_QUEUE opt-in. - Reword the nightly Strix comment that called e2e-gpu-strix-ubuntu per-PR, and two comment-only leftovers in xtask and the cucumber crate. - The nightly path guard matches whole words, so HF_TMPDIR is not TMPDIR. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 18 +++++++----- .github/workflows/nightly.yml | 5 ++-- docs/ci-hardware-testing.md | 22 +++++++------- tests/e2e-cucumber/README.md | 4 +-- tests/e2e-cucumber/src/expectation.rs | 4 +-- xtask/src/e2e_report.rs | 2 +- xtask/src/workflow_contract.rs | 42 ++++++++++++++++++--------- 7 files changed, 59 insertions(+), 38 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 78d4f8c56..578d5303c 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -12,8 +12,9 @@ name: E2E self-hosted # Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, # MI300X: the only lane where vLLM is the effective engine) and one Windows GPU # node (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix -# Ubuntu and W7900 lanes keep their push / merge_group / dispatch triggers and -# run nightly (nightly.yml); they just no longer start on pull_request. +# Ubuntu lane keeps its push / merge_group / dispatch triggers and the W7900 +# lane its push / dispatch triggers (no merge_group); both run nightly +# (nightly.yml) and neither starts on pull_request. # # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not @@ -116,7 +117,7 @@ jobs: outputs: heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} # Narrow GPU-serve gate; forced true off-PR so the merge queue runs the - # full matrix (every lane but MI300X, which skips merge_group) and its + # full matrix (every lane but MI300X and W7900, which skip merge_group) and its # required checks are never starved. serve: ${{ steps.filter.outputs.serve || steps.all.outputs.forced }} # ROCMAI-430: the three most recent rows of the official release list @@ -1610,15 +1611,18 @@ jobs: fail-fast: false matrix: sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} - # Not on pull_request: one runner on borrowed quota, 58-70% of its PR jobs - # never started before being superseded. Still runs on push/merge_group/ - # dispatch and nightly (nightly.yml, e2e-gpu-nightly-w7900). + # Not on pull_request or merge_group: one runner on borrowed quota, 58-70% of + # its PR jobs never started before being superseded, and a merge_group run + # would queue behind it the same way (the queue does not wait for this + # check). Still runs on push (main, release/**), dispatch and nightly + # (nightly.yml, e2e-gpu-nightly-w7900). if: >- always() && needs.changes.result == 'success' && ( (github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request' + && github.event_name != 'merge_group' && needs.changes.outputs.serve == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'w7900')) @@ -1631,8 +1635,6 @@ jobs: env: E2E_SERVE_TIMEOUT_SECS: "300" E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - # Full serve matrix at the merge-queue gate. - E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 30b9dbcf6..452d7bd2f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -768,8 +768,9 @@ jobs: # it, so the "Report whether Docker is reachable" step logs the answer. If it # does not, no nightly lane runs AIM serve and a pool-based one is needed. timeout-minutes: 120 - # The DevLab Dispatch pool, identical to the per-PR e2e-gpu-strix-ubuntu - # lane in e2e-selfhosted.yml (pinned by workflow_contract.rs). This lane used + # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in + # e2e-selfhosted.yml, which runs on push, merge_group and dispatch but not on + # pull_request (pinned by workflow_contract.rs). This lane used # to target the static `native` host, which is offline: every nightly sat # queued until the 12h run cap and the lane never ran. Pool hosts are cold, # so there are no host-specific HOME or cache-dir overrides. diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index da0449518..7c0f1e602 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,9 +36,9 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` (see below) | No — absent from the required-status-check list | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` (and W7900 also `merge_group`; see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which only skip `pull_request` | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` (W7900 also `merge_group`, but a `release/**` push runs both) | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only @@ -48,9 +48,10 @@ A pull request now runs on exactly one Linux GPU node and one Windows GPU node: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer -start on `pull_request`; they keep their `push`/`merge_group`/dispatch -triggers and run nightly (`e2e-gpu-nightly-strix` on the same DevLab Dispatch -pool labels, `e2e-gpu-nightly-w7900`). +start on `pull_request`; they keep their `push`/dispatch triggers (Strix +Ubuntu also `merge_group`; W7900 does not run in the merge queue) and run +nightly (`e2e-gpu-nightly-strix` on the same DevLab Dispatch pool labels, +`e2e-gpu-nightly-w7900`). [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length @@ -130,7 +131,8 @@ separate tier flag or tag filter to maintain. On `pull_request`, `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` are skipped; the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo pool). -Both still run on `push`, `merge_group`, dispatch and nightly. +Both still run on `push`, dispatch and nightly; Strix Ubuntu also on `merge_group`, +W7900 does not (a single runner on borrowed quota would sit behind the queue). `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact @@ -391,10 +393,10 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, `pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and -`merge_group` (every GPU lane except MI300X, which is not required for merging -and so is skipped in the merge queue rather than left queued on limited runner -capacity after its PR has merged) when the workflow's -own `changes` job's `serve` path filter is `true`. A new push to a pull request +`merge_group` (every GPU lane except MI300X and W7900, which are not required for +merging and so are skipped in the merge queue rather than left queued on limited runner +capacity after its PR has merged) when the workflow's own `changes` job's +`serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 3e9ff2249..9e0675f7c 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -182,8 +182,8 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux and one -Windows GPU node per PR); they still run on `push`, `merge_group`, dispatch and -nightly. +Windows GPU node per PR); they still run on `push`, dispatch and nightly, and +`e2e-gpu-strix-ubuntu` also on `merge_group` (`e2e-gpu-w7900` does not). The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 52736ff4c..4fea9d034 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1196,8 +1196,8 @@ serve_timeout_secs = 90 #[test] fn aim_serve_scenarios_stay_off_the_per_pr_lanes() { // serve-26/27 pull the AIM image and a 9B checkpoint on a cold runner and - // may each wait out a 1200s readiness cap. Run per-PR on the Strix Ubuntu - // pool they pushed the job (~25 min otherwise) into its 45 min cap on every + // may each wait out a 1200s readiness cap. Run on every PR on the Strix + // Ubuntu pool (before that lane left the PR path) they pushed the job (~25 min otherwise) into its 45 min cap on every // run. They must resolve to Skip unless the nightly opt-in is set, and must // still be reachable (not Skip for the nightly reason) when it is. let m = Expectations::default(); diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index bfd9db279..720fd4405 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -142,7 +142,7 @@ fn label_for_root_report(dir: &Path) -> String { /// Strips the literal `${{ matrix.channel }}` template segment a channel /// matrix inserts into its artifact names — nightly's six GPU lanes and, per -/// ROCMAI-125, `e2e-gpu-strix-ubuntu`'s per-PR `[release, nightly]` leg — so +/// ROCMAI-125, `e2e-gpu-strix-ubuntu`'s `[release, nightly]` leg — so /// a name like `e2e-gpu-${{ matrix.channel }}-report` compares as its /// canonical `e2e-gpu-report` base. This scan is text-based over the raw /// workflow source and does not simulate matrix expansion, so the template diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 95637e3f3..fe23b0829 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7693,20 +7693,25 @@ jobs: ); } // A pull request runs one Linux node (MI300X) and one Windows node (the - // Strix Halo pool). These two lanes are off the pull_request path but must - // still run on push and in the merge queue. Pin the WHOLE gate, not a - // substring: the `pull_request` exclusion must sit in the non-dispatch - // branch (so a dispatch can still select the lane) and nothing else about - // the event may be named, so a gate that dropped push/merge_group some - // other way fails here too. - for (job, platform) in [ - ("e2e-gpu-strix-ubuntu", "strix-ubuntu"), - ("e2e-gpu-w7900", "w7900"), + // Strix Halo pool). Strix Ubuntu is off the pull_request path but must + // still run on push and in the merge queue; W7900 is off pull_request AND + // merge_group (one runner on borrowed quota, the queue does not wait for + // it) but still runs on push and dispatch. Pin the WHOLE gate, not a + // substring: the event exclusions must sit in the non-dispatch branch (so + // a dispatch can still select the lane) and nothing else about the event + // may be named, so a gate that dropped push some other way fails too. + for (job, platform, excluded) in [ + ("e2e-gpu-strix-ubuntu", "strix-ubuntu", ""), + ( + "e2e-gpu-w7900", + "w7900", + "&& github.event_name != 'merge_group' ", + ), ] { let expected = format!( "always() && needs.changes.result == 'success' && ( \ (github.event_name != 'workflow_dispatch' \ - && github.event_name != 'pull_request' \ + && github.event_name != 'pull_request' {excluded}\ && needs.changes.outputs.serve == 'true') \ || (github.event_name == 'workflow_dispatch' \ && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" @@ -7714,10 +7719,14 @@ jobs: assert_eq!( normalized_whitespace(&gate(job)), normalized_whitespace(&expected), - "`{job}` must skip pull_request in its non-dispatch branch and keep running \ - on push, merge_group and dispatch" + "`{job}` has the wrong event gate (pull_request, plus merge_group for W7900, \ + excluded in the non-dispatch branch only)" ); } + assert!( + !job_block(&sh, "e2e-gpu-w7900").contains("E2E_MERGE_QUEUE"), + "e2e-gpu-w7900 never runs in the merge queue, so it must not opt into @merge-queue scenarios" + ); } /// The nightly Strix Ubuntu lane moved from a static host (HOME, CARGO_HOME, @@ -7741,9 +7750,16 @@ jobs: !uncommented.contains("actions-runner"), "the pool host has no /home/ubuntu/actions-runner tree" ); + // Whole-word match, so a future `HF_TMPDIR` is not read as `TMPDIR`. + let is_word = |c: char| c.is_ascii_alphanumeric() || c == '_'; for var in ["CARGO_HOME", "RUSTUP_HOME", "TMPDIR", "PIP_CACHE_DIR"] { + let read = uncommented.match_indices(var).any(|(index, _)| { + let before = uncommented[..index].chars().next_back(); + let after = uncommented[index + var.len()..].chars().next(); + !before.is_some_and(is_word) && !after.is_some_and(is_word) + }); assert!( - env.contains_key(var) || !uncommented.contains(var), + env.contains_key(var) || !read, "e2e-gpu-nightly-strix reads {var} but no longer sets it" ); } From d68476e70acc0e8932f362c2b857201f4b4afeaf Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 14:43:55 +0000 Subject: [PATCH 07/16] ci: drop e2e-gpu-strix-ubuntu from merge_group Over 7 days 35% of its merge-queue jobs (98 of 281) were cancelled at the 45 min cap. The check is not required and the queue does not wait for it. It still runs on push (main, release/**), dispatch and nightly. The contract test pins both lanes' whole gate and that neither job opts into @merge-queue scenarios; docs updated. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 19 ++++++------- docs/ci-hardware-testing.md | 30 +++++++++++---------- tests/e2e-cucumber/README.md | 4 +-- xtask/src/workflow_contract.rs | 40 +++++++++++++--------------- 4 files changed, 47 insertions(+), 46 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 578d5303c..d13ab446c 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -12,9 +12,8 @@ name: E2E self-hosted # Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, # MI300X: the only lane where vLLM is the effective engine) and one Windows GPU # node (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix -# Ubuntu lane keeps its push / merge_group / dispatch triggers and the W7900 -# lane its push / dispatch triggers (no merge_group); both run nightly -# (nightly.yml) and neither starts on pull_request. +# Ubuntu and W7900 lanes keep their push / dispatch triggers and run nightly +# (nightly.yml); neither starts on pull_request or merge_group. # # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not @@ -117,7 +116,8 @@ jobs: outputs: heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} # Narrow GPU-serve gate; forced true off-PR so the merge queue runs the - # full matrix (every lane but MI300X and W7900, which skip merge_group) and its + # full matrix (every lane but MI300X, Strix Ubuntu and W7900, which skip + # merge_group) and its # required checks are never starved. serve: ${{ steps.filter.outputs.serve || steps.all.outputs.forced }} # ROCMAI-430: the three most recent rows of the official release list @@ -478,15 +478,18 @@ jobs: needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. # Not on pull_request: a PR gets exactly one Linux GPU node (MI300X, the only - # lane where vLLM runs) and one Windows node (the Strix Halo pool). This lane - # still runs on push/merge_group/dispatch and nightly (nightly.yml, - # e2e-gpu-nightly-strix, same pool labels). + # lane where vLLM runs) and one Windows node (the Strix Halo pool). Not on + # merge_group either: over 7 days 35% of its merge-queue jobs (98 of 281) were + # cancelled at the 45 min cap, and the queue does not wait for this + # non-required check. Still runs on push (main, release/**), dispatch and + # nightly (nightly.yml, e2e-gpu-nightly-strix, same pool labels). if: >- always() && needs.changes.result == 'success' && ( (github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request' + && github.event_name != 'merge_group' && needs.changes.outputs.serve == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')) @@ -520,8 +523,6 @@ jobs: # Match the MI300X dispatch path: opt into the platform-adaptive large-model # scenario only when the manual include_nightly input is enabled. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - # Heavy @merge-queue serves run only in the merge queue. - E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 7c0f1e602..902c2737b 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -48,10 +48,9 @@ A pull request now runs on exactly one Linux GPU node and one Windows GPU node: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer -start on `pull_request`; they keep their `push`/dispatch triggers (Strix -Ubuntu also `merge_group`; W7900 does not run in the merge queue) and run -nightly (`e2e-gpu-nightly-strix` on the same DevLab Dispatch pool labels, -`e2e-gpu-nightly-w7900`). +start on `pull_request` or in the merge queue; they keep their +`push`/dispatch triggers and run nightly (`e2e-gpu-nightly-strix` on the same +DevLab Dispatch pool labels, `e2e-gpu-nightly-w7900`). [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length @@ -129,17 +128,19 @@ separate tier flag or tag filter to maintain. | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | AMD Radeon PRO W7900 (gfx1100, 48GB) on Linux | self-hosted `[self-hosted, linux, w7900]` | -On `pull_request`, `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` are skipped; the -other lanes above run (one Linux node, MI300X, and the Windows Strix Halo pool). -Both still run on `push`, dispatch and nightly; Strix Ubuntu also on `merge_group`, -W7900 does not (a single runner on borrowed quota would sit behind the queue). +On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` +are skipped; both still run on `push`, dispatch and nightly. In the merge queue +W7900 is one runner on borrowed quota that would sit behind the queue, and over +7 days 35% of Strix Ubuntu's merge-queue jobs (98 of 281) were cancelled at the +45 min cap; neither check is required and the queue does not wait for it. On a +PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo +pool). `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact names, so that lane proves the nightly package channel boots. It no longer runs on `pull_request`, so on a PR no lane exercises the nightly channel; it runs on -`push`, `merge_group`, dispatch and in `nightly.yml`'s channel matrix -(ROCMAI-429). +`push`, dispatch and in `nightly.yml`'s channel matrix (ROCMAI-429). All three Strix Halo lanes run on the AMD Ryzen DevLab Dispatch pool: a fresh runner is registered per job and destroyed after, opt-in only via the @@ -393,9 +394,10 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, `pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and -`merge_group` (every GPU lane except MI300X and W7900, which are not required for -merging and so are skipped in the merge queue rather than left queued on limited runner -capacity after its PR has merged) when the workflow's own `changes` job's +`merge_group` (the Windows Strix Halo lanes only: MI300X, Strix Ubuntu and +W7900 are not required for merging and so are skipped in the merge queue rather +than left queued on limited runner capacity or hung at their cap after its PR +has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy @@ -441,7 +443,7 @@ plus broad-dependency safety nets), **not** a blanket `**/*.rs`. So a Rust chang that cannot affect serving — e.g. a dashboard-only or unrelated-crate PR — skips the heavy GPU matrix, while compile coverage for every crate still runs on `ci.yml`'s always-on build/test lanes. Off `pull_request` (push/merge_group) the -filter is forced `true`, so the full matrix (minus MI300X in the merge queue) +filter is forced `true`, so the full matrix (minus MI300X, Strix Ubuntu and W7900 in the merge queue) always runs there. Unlike the pre-split layout the GPU jobs do **not** gate on the hosted `build-and-test` job — cross-workflow `needs` is not possible, so each GPU job builds the `rocm` diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 9e0675f7c..3ea8d8e27 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -182,8 +182,8 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux and one -Windows GPU node per PR); they still run on `push`, dispatch and nightly, and -`e2e-gpu-strix-ubuntu` also on `merge_group` (`e2e-gpu-w7900` does not). +Windows GPU node per PR); they also skip `merge_group` and still run on `push`, +dispatch and nightly. The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index fe23b0829..d73e67e47 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7693,25 +7693,23 @@ jobs: ); } // A pull request runs one Linux node (MI300X) and one Windows node (the - // Strix Halo pool). Strix Ubuntu is off the pull_request path but must - // still run on push and in the merge queue; W7900 is off pull_request AND - // merge_group (one runner on borrowed quota, the queue does not wait for - // it) but still runs on push and dispatch. Pin the WHOLE gate, not a - // substring: the event exclusions must sit in the non-dispatch branch (so - // a dispatch can still select the lane) and nothing else about the event - // may be named, so a gate that dropped push some other way fails too. - for (job, platform, excluded) in [ - ("e2e-gpu-strix-ubuntu", "strix-ubuntu", ""), - ( - "e2e-gpu-w7900", - "w7900", - "&& github.event_name != 'merge_group' ", - ), + // Strix Halo pool). Strix Ubuntu and W7900 are off pull_request AND + // merge_group (W7900 is one runner on borrowed quota; 35% of Strix + // Ubuntu's merge-queue jobs hit the 45 min cap; neither check is required + // and the queue does not wait for it) but still run on push and dispatch. + // Pin the WHOLE gate, not a substring: the event exclusions must sit in the + // non-dispatch branch (so a dispatch can still select the lane) and nothing + // else about the event may be named, so a gate that dropped push some other + // way fails too. + for (job, platform) in [ + ("e2e-gpu-strix-ubuntu", "strix-ubuntu"), + ("e2e-gpu-w7900", "w7900"), ] { let expected = format!( "always() && needs.changes.result == 'success' && ( \ (github.event_name != 'workflow_dispatch' \ - && github.event_name != 'pull_request' {excluded}\ + && github.event_name != 'pull_request' \ + && github.event_name != 'merge_group' \ && needs.changes.outputs.serve == 'true') \ || (github.event_name == 'workflow_dispatch' \ && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" @@ -7719,14 +7717,14 @@ jobs: assert_eq!( normalized_whitespace(&gate(job)), normalized_whitespace(&expected), - "`{job}` has the wrong event gate (pull_request, plus merge_group for W7900, \ - excluded in the non-dispatch branch only)" + "`{job}` has the wrong event gate (pull_request and merge_group excluded in \ + the non-dispatch branch only)" + ); + assert!( + !job_block(&sh, job).contains("E2E_MERGE_QUEUE"), + "{job} never runs in the merge queue, so it must not opt into @merge-queue scenarios" ); } - assert!( - !job_block(&sh, "e2e-gpu-w7900").contains("E2E_MERGE_QUEUE"), - "e2e-gpu-w7900 never runs in the merge queue, so it must not opt into @merge-queue scenarios" - ); } /// The nightly Strix Ubuntu lane moved from a static host (HOME, CARGO_HOME, From e7a3ce061f79d758d8d3a0fc1928730c8dde0498 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 14:51:35 +0000 Subject: [PATCH 08/16] docs, ci: make every merge_group claim match the workflow Strix Ubuntu and W7900 skip pull_request and merge_group and run on push, dispatch and nightly. Fix the nightly.yml comment and the ci-hardware-testing.md rows that said otherwise, state that the merge queue has no native Linux GPU lane (Windows Strix and WSL2 only), and wrap the comment lines this PR left long. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 13 +++++++------ docs/ci-hardware-testing.md | 19 +++++++++++-------- docs/testing.md | 4 ++-- scripts/reclaim-gpu.sh | 6 +++--- tests/e2e-cucumber/README.md | 3 ++- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 452d7bd2f..f735e2bf0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -673,8 +673,9 @@ jobs: # W7900 counterpart to e2e-gpu-nightly. Mirrors the e2e-gpu-w7900 lane (which # no longer runs on pull_request) in e2e-selfhosted.yml (w7900 label, shared - # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step: the report crate asserts - # the nightly lanes publish the same platforms as the per-PR lanes. + # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in + # step: the report crate asserts the nightly lanes publish the same platforms + # as the lanes in e2e-selfhosted.yml. e2e-gpu-nightly-w7900: name: E2E tests (W7900, incl. nightly-only) timeout-minutes: 90 @@ -769,10 +770,10 @@ jobs: # does not, no nightly lane runs AIM serve and a pool-based one is needed. timeout-minutes: 120 # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in - # e2e-selfhosted.yml, which runs on push, merge_group and dispatch but not on - # pull_request (pinned by workflow_contract.rs). This lane used - # to target the static `native` host, which is offline: every nightly sat - # queued until the 12h run cap and the lane never ran. Pool hosts are cold, + # e2e-selfhosted.yml, which runs on push and dispatch but not on pull_request + # or merge_group (pinned by workflow_contract.rs). This lane used to target + # the static `native` host, which is offline: every nightly sat queued until + # the 12h run cap and the lane never ran. Pool hosts are cold, # so there are no host-specific HOME or cache-dir overrides. runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] continue-on-error: true diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 902c2737b..a7ff5545a 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,9 +36,9 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` (and W7900 also `merge_group`; see below) | No — absent from the required-status-check list | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` (W7900 also `merge_group`, but a `release/**` push runs both) | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` but run on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only @@ -402,10 +402,13 @@ has merged) when the workflow's own `changes` job's cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM -lane: they run only on the other lanes, and the only vLLM serve before a change -lands is the per-PR canary (`serve-05`), which runs when the `serve` filter -matches. `push` fires on `main` and on any `release/**` branch, so pushing to -a release branch runs the full self-hosted matrix ahead of cutting the `v*` tag that `release.yml` builds/publishes from — this is the +lane: they run only on the Windows Strix Halo lanes, and the only vLLM serve +before a change lands is the per-PR canary (`serve-05`), which runs when the +`serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X, +Strix Ubuntu and W7900 all skip `merge_group`, so it runs only Windows Strix and +WSL2. `push` fires on `main` and on any `release/**` branch, so pushing to a +release branch runs the full self-hosted matrix (all five lanes) ahead of +cutting the `v*` tag that `release.yml` builds/publishes from — this is the release-candidate regression gate; `pull_request` fires against any base branch. Merging or closing a pull request also starts a run (`closed` is a listed `pull_request` activity type) that joins that PR's group and runs only a @@ -443,8 +446,8 @@ plus broad-dependency safety nets), **not** a blanket `**/*.rs`. So a Rust chang that cannot affect serving — e.g. a dashboard-only or unrelated-crate PR — skips the heavy GPU matrix, while compile coverage for every crate still runs on `ci.yml`'s always-on build/test lanes. Off `pull_request` (push/merge_group) the -filter is forced `true`, so the full matrix (minus MI300X, Strix Ubuntu and W7900 in the merge queue) -always runs there. Unlike the +filter is forced `true`, so the full matrix (minus MI300X, Strix Ubuntu and +W7900 in the merge queue) always runs there. Unlike the pre-split layout the GPU jobs do **not** gate on the hosted `build-and-test` job — cross-workflow `needs` is not possible, so each GPU job builds the `rocm` binary itself as its first real step (a broken build fails that job fast and diff --git a/docs/testing.md b/docs/testing.md index 96e02cc6f..2ad4b23cb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1477,8 +1477,8 @@ Strix Ubuntu job when it still ran per PR (about 25 minutes without them, 45 minute cap) used up the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` is set on such a host: the nightly Strix Halo lane, or an -`include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then gets a 105 minute -cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that +`include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then +gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that they stay reachable when the opt-in is set. Caveat: it is unconfirmed that the nightly Strix Halo host has Docker (nothing in diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 0db3a3b10..73e1f2df0 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -49,9 +49,9 @@ set -euo pipefail # `std::env::temp_dir()` and so follows TMPDIR; a lane on a host that redirects # it (the nightly Strix lane did, to `$HOME/actions-runner/tmp`, before it moved # to the pool) is the case this handles. A bare `/tmp/rocm-e2e` kept matching -# there only by accident — that directory is itself named `tmp`, so the substring -# still appeared — and would silently stop matching under any redirect not -# named `tmp`. Deriving the redirected form fixes that for ANY target. +# there only by accident — that directory is itself named `tmp`, so the +# substring still appeared — and would silently stop matching under any +# redirect not named `tmp`. Deriving the redirected form fixes that for ANY target. # # WHAT THE ANCHORING DOES AND DOES NOT BUY. Roots are matched as unanchored # substrings of the WHOLE command line, so any root also matches a process that diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 3ea8d8e27..085443335 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -183,7 +183,8 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux and one Windows GPU node per PR); they also skip `merge_group` and still run on `push`, -dispatch and nightly. +dispatch and nightly. The merge queue therefore has no native Linux GPU lane +(MI300X skips it too); it runs only Windows Strix and WSL2. The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow From 66d2bc3ef3a226abf14d663b008d626aeb5efe2f Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 15:30:45 +0000 Subject: [PATCH 09/16] docs, tests: fix stale merge-queue and per-PR claims, re-wrap two lines The merge_queue field doc said the serve matrix runs on every lane but MI300X; only Windows Strix and WSL2 run @merge-queue serves now. The AIM serve comment in model_serving.feature still called Strix Ubuntu a per-PR job. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 2 +- docs/testing.md | 8 ++++---- tests/e2e-cucumber/features/model_serving.feature | 9 +++++---- tests/e2e-cucumber/src/expectation.rs | 11 +++++++---- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index a7ff5545a..ea32d6009 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -394,7 +394,7 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, `pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and -`merge_group` (the Windows Strix Halo lanes only: MI300X, Strix Ubuntu and +`merge_group` (the Windows Strix Halo and WSL2 lanes only: MI300X, Strix Ubuntu and W7900 are not required for merging and so are skipped in the merge queue rather than left queued on limited runner capacity or hung at their cap after its PR has merged) when the workflow's own `changes` job's diff --git a/docs/testing.md b/docs/testing.md index 2ad4b23cb..5aaf4bf57 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1473,13 +1473,13 @@ install. `serve-26` and `serve-27` (`features/model_serving.feature`, the AIM serve and restart canaries) are `@nightly` for cost: each pulls the AIM image and a 9B checkpoint on a cold runner and may wait up to 1200s for readiness, which on the -Strix Ubuntu job when it still ran per PR (about 25 minutes without them, 45 -minute cap) used up the whole budget. They still need `@requires-engine:aim`, +Strix Ubuntu job, back when it ran on pull requests (about 25 minutes without +them, 45 minute cap) used up the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` is set on such a host: the nightly Strix Halo lane, or an `include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then -gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that -they stay reachable when the opt-in is set. +gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins +the tagging and that they stay reachable when the opt-in is set. Caveat: it is unconfirmed that the nightly Strix Halo host has Docker (nothing in `nightly.yml` installs or checks it). Without it these two scenarios skip on diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 1786d007e..15ecb725f 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -359,10 +359,11 @@ Feature: Model serving # verified on (see `HostCapability::engine_available`'s "aim" arm — Radeon # gfx110X/120X and Strix Halo gfx1151, not Instinct); a Strix Halo # self-hosted runner with Docker is where this actually runs. - # `@nightly` keeps both AIM scenarios off the per-PR lanes: each one pulls the - # image and a 9B checkpoint on a cold runner and may wait out the full - # 1200s readiness cap, so the pair on the per-PR Strix Ubuntu job (~25 min - # without them, 45 min cap) ended every run at the cap. They run only when + # `@nightly` keeps both AIM scenarios off the pull-request lanes: each one + # pulls the image and a 9B checkpoint on a cold runner and may wait out the + # full 1200s readiness cap, so the pair on the Strix Ubuntu job (~25 min + # without them, 45 min cap) ended every run at the cap. That job now runs + # only on push, dispatch and nightly, never on pull_request. They run only when # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. # Unconfirmed: whether the nightly Strix Halo host has Docker; without it both diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 4fea9d034..c6bc0c815 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -174,7 +174,9 @@ pub struct ScenarioDecl { /// per-engine canary on the fast path, so it is skipped on ordinary per-PR / /// on-demand runs and only runs in the merge queue, which opts in via /// `E2E_MERGE_QUEUE`. Keeps the PR feedback loop short while still exercising - /// the full serve matrix (on every lane but MI300X) before a change lands. + /// the heavy serves before a change lands. Only the Windows Strix and WSL2 + /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, W7900, + /// Strix Ubuntu) do not. pub merge_queue: bool, } @@ -1197,9 +1199,10 @@ serve_timeout_secs = 90 fn aim_serve_scenarios_stay_off_the_per_pr_lanes() { // serve-26/27 pull the AIM image and a 9B checkpoint on a cold runner and // may each wait out a 1200s readiness cap. Run on every PR on the Strix - // Ubuntu pool (before that lane left the PR path) they pushed the job (~25 min otherwise) into its 45 min cap on every - // run. They must resolve to Skip unless the nightly opt-in is set, and must - // still be reachable (not Skip for the nightly reason) when it is. + // Ubuntu pool (before that lane left the PR path) they pushed the job + // (~25 min otherwise) into its 45 min cap on every run. They must + // resolve to Skip unless the nightly opt-in is set, and must still be + // reachable (not Skip for the nightly reason) when it is. let m = Expectations::default(); for prefix in ["serve-26 -", "serve-27 -"] { let d = ScenarioDecl::from_tags(&model_serving_tags(prefix)); From 0b2d5dc69963d00986d500daf921cff411e9116c Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 09:41:33 +0000 Subject: [PATCH 10/16] ci: pin the PR-skipping lanes in #1032's closed-PR if table After rebasing onto #1032, closing_a_pr_supersedes_its_self_hosted_runs pinned the strix-ubuntu and w7900 gates without the pull_request and merge_group exclusions this PR adds. Pin them with the exclusions. Signed-off-by: Tomas Saaristola --- xtask/src/workflow_contract.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index d73e67e47..eba14f796 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7559,18 +7559,32 @@ jobs: // did, while a prefix check still passed. A new job has no entry here // and fails until it is added, which is the point: someone must decide // how it behaves on `closed`. - let lane_if = |extra: &str, platform: &str| { + // `extra` gates the whole lane; `inner` sits in the non-dispatch branch, + // where a lane that skips `pull_request` keeps the exclusion (see + // `mi300x_lane_is_skipped_in_the_merge_queue`). + let lane_if = |extra: &str, inner: &str, platform: &str| { format!( "always() && needs.changes.result == 'success' && {extra}\ - ( (github.event_name != 'workflow_dispatch' \ - && needs.changes.outputs.serve == 'true') \ + ( (github.event_name != 'workflow_dispatch' && {inner}\ + needs.changes.outputs.serve == 'true') \ || (github.event_name == 'workflow_dispatch' \ && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" ) }; let lane = |job, extra: &str, platform: &str| PinnedIf { job, - condition: lane_if(extra, platform), + condition: lane_if(extra, "", platform), + matrix_lane: true, + }; + // Strix Ubuntu and W7900 are off `pull_request` and `merge_group`. + let off_pr_lane = |job, platform: &str| PinnedIf { + job, + condition: lane_if( + "", + "github.event_name != 'pull_request' \ + && github.event_name != 'merge_group' && ", + platform, + ), matrix_lane: true, }; let expected = vec![ @@ -7579,10 +7593,10 @@ jobs: "github.event_name != 'merge_group' && ", "app-dev-gpu", ), - lane("e2e-gpu-strix-ubuntu", "", "strix-ubuntu"), + off_pr_lane("e2e-gpu-strix-ubuntu", "strix-ubuntu"), lane("e2e-gpu-strix-windows", "", "strix-windows"), lane("e2e-wsl", "", "strix-wsl"), - lane("e2e-gpu-w7900", "", "w7900"), + off_pr_lane("e2e-gpu-w7900", "w7900"), // Skips because the `serve` output of a skipped `changes` is empty. PinnedIf { job: "e2e-report", From bfde20190052c79a3c36caf3879a93b2f05dab65 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 10:00:24 +0000 Subject: [PATCH 11/16] docs, ci: drop the stale Docker 'unconfirmed' and per-PR budget claims The nightly Strix lane is on the pool where Docker was reached (serve-26 passed on both legs); say so, and note serve-27's stop failure. The include_nightly budget comment no longer calls the push-run budget per-PR. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 2 +- .github/workflows/nightly.yml | 12 +++++++----- docs/testing.md | 13 +++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index d13ab446c..c38f818b8 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -472,7 +472,7 @@ jobs: # ratio the 35 was chosen for. # An include_nightly dispatch also runs the AIM serve scenarios (serve-26/27, # @nightly, up to 1200s readiness each on a cold pull) and the large-model - # serve, which cannot fit the per-PR 45min budget. + # serve, which cannot fit the 45min budget of an ordinary push run. timeout-minutes: ${{ inputs.include_nightly && 105 || 45 }} runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] needs: [changes] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f735e2bf0..35d3434ff 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -763,11 +763,13 @@ jobs: # Strix Ubuntu job. e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) - # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits when - # this host has Docker; on a host without it they skip and cost nothing. - # Whether it has Docker is unconfirmed: nothing in this job installs or checks - # it, so the "Report whether Docker is reachable" step logs the answer. If it - # does not, no nightly lane runs AIM serve and a pool-based one is needed. + # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits; on a + # host without Docker they skip and cost nothing. Docker is reachable on this + # pool: a dispatch of the same lane (e2e-selfhosted.yml run 43025895270) passed + # serve-26 on both channel legs. serve-27 fails there at stopping the AIM + # service (STOP FAILED rc=1) and has no xfail row yet, so expect it red in the + # nightly report; the job is continue-on-error. The "Report whether Docker is + # reachable" step still logs the answer each run. timeout-minutes: 120 # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in # e2e-selfhosted.yml, which runs on push and dispatch but not on pull_request diff --git a/docs/testing.md b/docs/testing.md index 5aaf4bf57..6f3f27b21 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1481,12 +1481,13 @@ i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that they stay reachable when the opt-in is set. -Caveat: it is unconfirmed that the nightly Strix Halo host has Docker (nothing in -`nightly.yml` installs or checks it). Without it these two scenarios skip on -every nightly run, and only a manual `include_nightly` dispatch on a host with -Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker -is reachable ("Report whether Docker is reachable"); check that line before -relying on this coverage. +Docker is reachable on the pool the nightly Strix Halo lane runs on: an +`include_nightly` dispatch of the same lane (`e2e-selfhosted.yml` run +43025895270) passed `serve-26` on both channel legs. `serve-27` fails there at +stopping the AIM service (`STOP FAILED rc=1`) and has no xfail row yet, so expect +it red in the nightly report; the lane is `continue-on-error`. The +`e2e-gpu-nightly-strix` job still logs whether Docker is reachable ("Report +whether Docker is reachable"), and on a host without it both scenarios skip. ### Lemonade managed-serve failure evidence From 31584664a706694ff1be4170a8d051ac19448089 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 10:22:23 +0000 Subject: [PATCH 12/16] tests: say Docker is reachable on the nightly Strix pool, and serve-27's stop failure The model_serving.feature AIM comment still called Docker on the nightly Strix Halo host unconfirmed; dispatch run 43025895270 reached it (serve-26 passed on both legs) and serve-27 fails at stopping the service. Signed-off-by: Tomas Saaristola --- tests/e2e-cucumber/features/model_serving.feature | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 15ecb725f..9c1105997 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -366,8 +366,12 @@ Feature: Model serving # only on push, dispatch and nightly, never on pull_request. They run only when # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. - # Unconfirmed: whether the nightly Strix Halo host has Docker; without it both - # skip every night (the nightly job logs whether Docker is reachable). + # Docker is reachable on the pool host the nightly Strix Halo lane runs on: a + # dispatch of the same lane (e2e-selfhosted.yml run 43025895270) passed + # serve-26 on both channel legs, so it does not skip there. serve-27 fails + # there at stopping the AIM service ("STOP FAILED (rc=1) ... Local server stop + # not confirmed"). On a host without Docker both skip (the nightly job logs + # whether Docker is reachable). # `@serve-timeout:1200` # widens both the harness's readiness poll and the CLI's own # ROCM_CLI_AIM_READY_TIMEOUT_SECS (see isolate_cmd) — a cold-start HF weight From b7d8f464cced45576072a713ccca0db3577618b1 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 10:46:38 +0000 Subject: [PATCH 13/16] tests: declare serve-aim-lifecycle-parity an expected failure on Strix Halo Ubuntu Its stop step fails with 'Local server stop not confirmed' (tracked in rocm-cli#1058), so the nightly lane reports it as xfail instead of an unexpected regression. Point the nightly.yml, docs and feature comments at it. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 7 ++++--- docs/testing.md | 5 +++-- tests/e2e-cucumber/expectations.toml | 9 +++++++++ tests/e2e-cucumber/features/model_serving.feature | 5 +++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 35d3434ff..2377207d2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -767,9 +767,10 @@ jobs: # host without Docker they skip and cost nothing. Docker is reachable on this # pool: a dispatch of the same lane (e2e-selfhosted.yml run 43025895270) passed # serve-26 on both channel legs. serve-27 fails there at stopping the AIM - # service (STOP FAILED rc=1) and has no xfail row yet, so expect it red in the - # nightly report; the job is continue-on-error. The "Report whether Docker is - # reachable" step still logs the answer each run. + # service (STOP FAILED rc=1); it is an expected failure in expectations.toml + # (rocm-cli#1058), so the nightly reports it as xfail, and an XPASS means the + # row is stale. The "Report whether Docker is reachable" step still logs the + # answer each run. timeout-minutes: 120 # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in # e2e-selfhosted.yml, which runs on push and dispatch but not on pull_request diff --git a/docs/testing.md b/docs/testing.md index 6f3f27b21..b75186269 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1484,8 +1484,9 @@ the tagging and that they stay reachable when the opt-in is set. Docker is reachable on the pool the nightly Strix Halo lane runs on: an `include_nightly` dispatch of the same lane (`e2e-selfhosted.yml` run 43025895270) passed `serve-26` on both channel legs. `serve-27` fails there at -stopping the AIM service (`STOP FAILED rc=1`) and has no xfail row yet, so expect -it red in the nightly report; the lane is `continue-on-error`. The +stopping the AIM service (`STOP FAILED rc=1`); it is an xfail row in +`expectations.toml` (rocm-cli#1058), so the nightly reports it as an expected +failure and a pass is an XPASS, meaning the row is stale. The `e2e-gpu-nightly-strix` job still logs whether Docker is reachable ("Report whether Docker is reachable"), and on a host without it both scenarios skip. diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index a50f1229f..9b5a34370 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -253,3 +253,12 @@ reason = "Lemonade 11.5.1 fetches TheRock only from repo.amd.com/rocm/tarball/, when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" reason = "The Windows large-model serve crashes in the SDK's Tensile library on the first inference request; the pinned ROCm 7.13 backend runs against the 7.14.1 SDK because Lemonade 11.5.1 cannot align it (suspected cause, not reproduced off Windows)." + +# serve-26 (serve-aim-inference) passes on the same host; only the stop step of +# the lifecycle scenario fails. Scoped to Linux Strix Halo, the only place it has +# been seen (run 43025895270, both channel legs). A host without Docker skips the +# scenario before this row applies. +[["serve-aim-lifecycle-parity"]] +when = { os = "linux", therock_family = "gfx1151*", has_amd_gpu = true } +bug = "#1058" +reason = "Stopping the AIM service on Strix Halo Ubuntu is not confirmed: `rocm stop` reports `STOP FAILED (rc=1) Local server stop not confirmed`." diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 9c1105997..c0cf01ffe 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -370,8 +370,9 @@ Feature: Model serving # dispatch of the same lane (e2e-selfhosted.yml run 43025895270) passed # serve-26 on both channel legs, so it does not skip there. serve-27 fails # there at stopping the AIM service ("STOP FAILED (rc=1) ... Local server stop - # not confirmed"). On a host without Docker both skip (the nightly job logs - # whether Docker is reachable). + # not confirmed"; an xfail row in expectations.toml, rocm-cli#1058). On a + # host without Docker both skip (the nightly job logs whether Docker is + # reachable). # `@serve-timeout:1200` # widens both the harness's readiness poll and the CLI's own # ROCM_CLI_AIM_READY_TIMEOUT_SECS (see isolate_cmd) — a cold-start HF weight From b2f23df36e10dc680a485f5cf44e2ac48f6dc452 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 10:50:37 +0000 Subject: [PATCH 14/16] docs, ci: reference rocm-cli#1058 instead of pinning a run ID and failure text; say Windows pool The serve-27 evidence lives in the tracking issue, so nightly.yml, docs/testing.md, model_serving.feature and expectations.toml no longer quote a run ID or the failure string. A PR runs two Windows-pool jobs (Windows and WSL2), so say one Windows GPU pool, not node. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 4 ++-- .github/workflows/nightly.yml | 10 +++------- docs/ci-hardware-testing.md | 2 +- docs/testing.md | 13 +++++-------- tests/e2e-cucumber/README.md | 4 ++-- tests/e2e-cucumber/expectations.toml | 8 +++----- tests/e2e-cucumber/features/model_serving.feature | 10 +++------- xtask/src/workflow_contract.rs | 2 +- 8 files changed, 20 insertions(+), 33 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index c38f818b8..a06b7b8f5 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -11,7 +11,7 @@ name: E2E self-hosted # # Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, # MI300X: the only lane where vLLM is the effective engine) and one Windows GPU -# node (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix +# pool (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix # Ubuntu and W7900 lanes keep their push / dispatch triggers and run nightly # (nightly.yml); neither starts on pull_request or merge_group. # @@ -478,7 +478,7 @@ jobs: needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. # Not on pull_request: a PR gets exactly one Linux GPU node (MI300X, the only - # lane where vLLM runs) and one Windows node (the Strix Halo pool). Not on + # lane where vLLM runs) and one Windows pool (Strix Halo). Not on # merge_group either: over 7 days 35% of its merge-queue jobs (98 of 281) were # cancelled at the 45 min cap, and the queue does not wait for this # non-required check. Still runs on push (main, release/**), dispatch and diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2377207d2..5399e19e9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -764,13 +764,9 @@ jobs: e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits; on a - # host without Docker they skip and cost nothing. Docker is reachable on this - # pool: a dispatch of the same lane (e2e-selfhosted.yml run 43025895270) passed - # serve-26 on both channel legs. serve-27 fails there at stopping the AIM - # service (STOP FAILED rc=1); it is an expected failure in expectations.toml - # (rocm-cli#1058), so the nightly reports it as xfail, and an XPASS means the - # row is stale. The "Report whether Docker is reachable" step still logs the - # answer each run. + # host without Docker they skip and cost nothing. The "Report whether Docker + # is reachable" step logs the answer each run. serve-27's known failure on + # this pool is an xfail row in expectations.toml (rocm-cli#1058). timeout-minutes: 120 # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in # e2e-selfhosted.yml, which runs on push and dispatch but not on pull_request diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index ea32d6009..50b932251 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -45,7 +45,7 @@ covered, but off every PR's critical path, so they moved to the nightly-only rung instead of being removed outright. A pull request now runs on exactly one Linux GPU node and one Windows GPU -node: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and +pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer start on `pull_request` or in the merge queue; they keep their diff --git a/docs/testing.md b/docs/testing.md index b75186269..a0b13b8f7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1481,14 +1481,11 @@ i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that they stay reachable when the opt-in is set. -Docker is reachable on the pool the nightly Strix Halo lane runs on: an -`include_nightly` dispatch of the same lane (`e2e-selfhosted.yml` run -43025895270) passed `serve-26` on both channel legs. `serve-27` fails there at -stopping the AIM service (`STOP FAILED rc=1`); it is an xfail row in -`expectations.toml` (rocm-cli#1058), so the nightly reports it as an expected -failure and a pass is an XPASS, meaning the row is stale. The -`e2e-gpu-nightly-strix` job still logs whether Docker is reachable ("Report -whether Docker is reachable"), and on a host without it both scenarios skip. +Both scenarios skip on a host without Docker; the `e2e-gpu-nightly-strix` job +logs whether Docker is reachable ("Report whether Docker is reachable"). +`serve-27`'s known failure on the Strix Halo pool is an xfail row in +`expectations.toml`, tracked in rocm-cli#1058; a pass there is an XPASS, which +means the row is stale. ### Lemonade managed-serve failure evidence diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 085443335..f24f136c6 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -181,8 +181,8 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | -`e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux and one -Windows GPU node per PR); they also skip `merge_group` and still run on `push`, +`e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux GPU +node and one Windows GPU pool per PR); they also skip `merge_group` and still run on `push`, dispatch and nightly. The merge queue therefore has no native Linux GPU lane (MI300X skips it too); it runs only Windows Strix and WSL2. diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 9b5a34370..277b12a51 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -254,11 +254,9 @@ when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" reason = "The Windows large-model serve crashes in the SDK's Tensile library on the first inference request; the pinned ROCm 7.13 backend runs against the 7.14.1 SDK because Lemonade 11.5.1 cannot align it (suspected cause, not reproduced off Windows)." -# serve-26 (serve-aim-inference) passes on the same host; only the stop step of -# the lifecycle scenario fails. Scoped to Linux Strix Halo, the only place it has -# been seen (run 43025895270, both channel legs). A host without Docker skips the -# scenario before this row applies. +# Scoped to Linux Strix Halo, the only place it has been seen; evidence is in the +# tracking issue. A host without Docker skips the scenario before this row applies. [["serve-aim-lifecycle-parity"]] when = { os = "linux", therock_family = "gfx1151*", has_amd_gpu = true } bug = "#1058" -reason = "Stopping the AIM service on Strix Halo Ubuntu is not confirmed: `rocm stop` reports `STOP FAILED (rc=1) Local server stop not confirmed`." +reason = "Stopping the AIM service on Strix Halo Ubuntu is not confirmed." diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index c0cf01ffe..549a4f662 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -366,13 +366,9 @@ Feature: Model serving # only on push, dispatch and nightly, never on pull_request. They run only when # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. - # Docker is reachable on the pool host the nightly Strix Halo lane runs on: a - # dispatch of the same lane (e2e-selfhosted.yml run 43025895270) passed - # serve-26 on both channel legs, so it does not skip there. serve-27 fails - # there at stopping the AIM service ("STOP FAILED (rc=1) ... Local server stop - # not confirmed"; an xfail row in expectations.toml, rocm-cli#1058). On a - # host without Docker both skip (the nightly job logs whether Docker is - # reachable). + # On a host without Docker both skip (the nightly job logs whether Docker is + # reachable). serve-27's known failure on Strix Halo Ubuntu is an xfail row in + # expectations.toml, tracked in rocm-cli#1058. # `@serve-timeout:1200` # widens both the harness's readiness poll and the CLI's own # ROCM_CLI_AIM_READY_TIMEOUT_SECS (see isolate_cmd) — a cold-start HF weight diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index eba14f796..c9a2cc1c4 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7706,7 +7706,7 @@ jobs: "`{job}` must keep running in the merge queue (gate: `{gate}`)" ); } - // A pull request runs one Linux node (MI300X) and one Windows node (the + // A pull request runs one Linux node (MI300X) and one Windows pool (the // Strix Halo pool). Strix Ubuntu and W7900 are off pull_request AND // merge_group (W7900 is one runner on borrowed quota; 35% of Strix // Ubuntu's merge-queue jobs hit the 45 min cap; neither check is required From c91e228faeb568941bcee73693c0a15ff3b1c54d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 11:02:17 +0000 Subject: [PATCH 15/16] tests, docs: keep WSL2 out of the serve-27 xfail row; fix two stale ci-hardware-testing claims The row matched os=linux, which WSL2 also reports on gfx1151, so a WSL host that gained Docker and passed the scenario would XPASS. Add is_wsl = false. Reword the e2e-report coverage line and the pre-warmed runtime paragraph, which still named per-PR platforms that no longer all run per PR and the static native hosts. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 7 ++++--- tests/e2e-cucumber/expectations.toml | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 50b932251..02af4ab03 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -250,7 +250,8 @@ not applicable, instead of failing them on a premise the host cannot meet. Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` covers the mock platform; -`e2e-selfhosted.yml`'s `e2e-report` covers the per-PR GPU platforms; +`e2e-selfhosted.yml`'s `e2e-report` covers its self-hosted GPU platforms (a +pull request runs only the MI300X and Windows Strix Halo pool lanes among them); `nightly.yml`'s `e2e-report-nightly` covers a strict superset of those platforms (it also runs `e2e-gpu-rad3` and `e2e-gpu-mi350p`, demoted from per-PR to nightly-only per ROCMAI-125) with the `@nightly` scenarios @@ -488,8 +489,8 @@ gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu Nearly every GPU serve scenario points its `data/runtimes` at one shared, pre-warmed managed runtime tree (`E2E_SHARED_RUNTIMES_DIR`), so a multi-GiB `rocm install sdk` happens once per runner instead of once per scenario. The tree -lives on the runner's workspace — persistent across runs on the long-lived -`native`-labeled hosts, rebuilt by the prewarm step below on each ephemeral +lives on the runner's workspace — persistent across runs on a long-lived +runner, rebuilt by the prewarm step below on each ephemeral DevLab Dispatch pool run — survives `git clean`, and is namespaced by source-layout generation (`e2e-prewarm-multi-arch-v2`) so a branch using a new package layout cannot poison the cache consumed by code that only understands the diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 277b12a51..58b75dae7 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -254,9 +254,12 @@ when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" reason = "The Windows large-model serve crashes in the SDK's Tensile library on the first inference request; the pinned ROCm 7.13 backend runs against the 7.14.1 SDK because Lemonade 11.5.1 cannot align it (suspected cause, not reproduced off Windows)." -# Scoped to Linux Strix Halo, the only place it has been seen; evidence is in the -# tracking issue. A host without Docker skips the scenario before this row applies. +# Scoped to native Linux Strix Halo, the only place it has been seen (evidence is +# in the tracking issue); `is_wsl = false` keeps WSL2, which also reports +# `os = "linux"` on gfx1151, out of it, so a WSL host that gains Docker and passes +# the scenario is not an XPASS. A host without Docker skips the scenario before +# this row applies. [["serve-aim-lifecycle-parity"]] -when = { os = "linux", therock_family = "gfx1151*", has_amd_gpu = true } +when = { os = "linux", is_wsl = false, therock_family = "gfx1151*", has_amd_gpu = true } bug = "#1058" reason = "Stopping the AIM service on Strix Halo Ubuntu is not confirmed." From ee855d72be0a9b7e48c6eae45756e24fab6bd389 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 11:39:58 +0000 Subject: [PATCH 16/16] ci, docs: pin that e2e-wsl and Windows Strix set E2E_MERGE_QUEUE; say WSL2 runs the merge-queue serves The Triggers paragraph said the heavy serves run only on the Windows Strix Halo lanes while other docs said WSL2 too. The workflow sets E2E_MERGE_QUEUE on both e2e-gpu-strix-windows and e2e-wsl, so reword the paragraph and pin both in the contract test. Add a one-line symptom to the serve-27 xfail reason. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 3 ++- tests/e2e-cucumber/expectations.toml | 2 +- xtask/src/workflow_contract.rs | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 02af4ab03..46c9c47ad 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -403,7 +403,8 @@ has merged) when the workflow's own `changes` job's cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM -lane: they run only on the Windows Strix Halo lanes, and the only vLLM serve +lane: they run only on the Windows Strix Halo and WSL2 lanes (both set +`E2E_MERGE_QUEUE` for `merge_group`), and the only vLLM serve before a change lands is the per-PR canary (`serve-05`), which runs when the `serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X, Strix Ubuntu and W7900 all skip `merge_group`, so it runs only Windows Strix and diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 58b75dae7..dcf773659 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -262,4 +262,4 @@ reason = "The Windows large-model serve crashes in the SDK's Tensile library on [["serve-aim-lifecycle-parity"]] when = { os = "linux", is_wsl = false, therock_family = "gfx1151*", has_amd_gpu = true } bug = "#1058" -reason = "Stopping the AIM service on Strix Halo Ubuntu is not confirmed." +reason = "Stopping a running AIM service on Strix Halo Ubuntu reports `Local server stop not confirmed` (exit 1); the serve itself works." diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index c9a2cc1c4..25ffb5794 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7705,6 +7705,16 @@ jobs: !gate.contains("merge_group") && !gate.contains("'pull_request'"), "`{job}` must keep running in the merge queue (gate: `{gate}`)" ); + // And it must opt into the heavy serves there, for merge_group only, + // so the docs' "Windows Strix Halo and WSL2 run the merge-queue + // serves" stays true of the workflow. + assert_eq!( + job_mapping(job_block(&sh, job), "env") + .get("E2E_MERGE_QUEUE") + .map(String::as_str), + Some("${{ github.event_name == 'merge_group' && '1' || '' }}"), + "`{job}` must set E2E_MERGE_QUEUE for merge_group events only" + ); } // A pull request runs one Linux node (MI300X) and one Windows pool (the // Strix Halo pool). Strix Ubuntu and W7900 are off pull_request AND