From f487d2088ed8bded30ecea77dfe226dba75fd0d5 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 13:08:22 +0000 Subject: [PATCH 1/3] ci: remove the W7900 lane from e2e-selfhosted The W7900 job targets a runner label nothing else uses, and it has not run on pull_request or merge_group since the one-Linux-node change. Its push, release and dispatch runs sat queued behind the single runner. Remove the job, its dispatch choice, its report artifacts and the consolidated-report entries, and update the workflow contract and docs. W7900 hardware is covered by the queue. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 164 ++------------------------ docs/ci-hardware-testing.md | 53 ++++----- tests/e2e-cucumber/README.md | 7 +- tests/e2e-cucumber/src/expectation.rs | 2 +- xtask/src/e2e_report.rs | 2 - xtask/src/workflow_contract.rs | 66 +++++------ 6 files changed, 66 insertions(+), 228 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index a06b7b8f5..008d214c1 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -12,14 +12,14 @@ name: E2E self-hosted # Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, # MI300X: the only lane where vLLM is the effective engine) and one Windows GPU # pool (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix -# Ubuntu and W7900 lanes keep their push / dispatch triggers and run nightly -# (nightly.yml); neither starts on pull_request or merge_group. +# Ubuntu lane keeps its push / dispatch triggers; it does not start on +# pull_request or merge_group. There is no W7900 lane: that hardware is covered +# by the queue (docs/nightly-on-the-queue.md). # # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not -# because of `continue-on-error`, which is `false` on every lane but -# one, so a real hardware regression shows red instead of always green. The -# exception is `e2e-gpu-w7900`, `true` only while that hardware is proven out. PRs #334, #335 and #343 all +# because of `continue-on-error`, which is `false` on every lane, so a real +# hardware regression shows red instead of always green. PRs #334, #335 and #343 all # landed while `E2E tests (Strix Halo, Ubuntu)` reported `cancelled` on the # merge_group ref, which a required check under ALLGREEN would have blocked; # renaming a lane is therefore safe. @@ -41,7 +41,7 @@ on: # supersedes that PR's earlier run. It runs only the no-op `pr-closed` job; # every GPU lane skips because `changes` skips. This frees runners only where # the cancel completes: a job queued for a runner that is not picking it up - # (W7900 while its one runner is busy) may not finish cancelling, so this + # (a lane whose runner is busy) may not finish cancelling, so this # does NOT clear such queued runs. types: [opened, synchronize, reopened, closed] merge_group: @@ -57,7 +57,7 @@ on: type: choice default: all # Each value also selectable on its own; `all` covers every lane below. - options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, w7900] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -116,7 +116,7 @@ jobs: outputs: heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} # Narrow GPU-serve gate; forced true off-PR so the merge queue runs the - # full matrix (every lane but MI300X, Strix Ubuntu and W7900, which skip + # full matrix (every lane but MI300X and Strix Ubuntu, which skip # merge_group) and its # required checks are never starved. serve: ${{ steps.filter.outputs.serve || steps.all.outputs.forced }} @@ -204,7 +204,7 @@ jobs: # Exists only to start a run in the PR's concurrency group when the PR is # merged or closed: that run cancels the PR's older run. A GPU job of that run # which is queued for a runner that is not picking it up may not finish - # cancelling (observed on W7900-queued runs). + # cancelling (observed on queued runs). pr-closed: name: Supersede runs of a closed PR if: github.event_name == 'pull_request' && github.event.action == 'closed' @@ -1597,150 +1597,6 @@ jobs: path: tests/e2e-cucumber/results/ if-no-files-found: error - e2e-gpu-w7900: - name: E2E tests (W7900, ${{ matrix.sdk_version }}) - # 90min: matches every other Linux GPU lane — same collapsed suite. The card - # is smaller than the Instinct parts, so do not tighten this below their cap. - timeout-minutes: 90 - # `w7900` names the hardware rather than reusing the generic `amd-gpu`: - # GitHub matches a job to any runner whose labels are a superset, so a shared - # label would let this workstation card pick up Instinct-targeted work and - # vice versa. - runs-on: [self-hosted, linux, w7900] - needs: [changes] - strategy: - fail-fast: false - matrix: - sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} - # Not on pull_request or merge_group: one runner on borrowed quota, 58-70% of - # its PR jobs never started before being superseded, and a merge_group run - # would queue behind it the same way (the queue does not wait for this - # check). Still runs on push (main, release/**), dispatch and nightly - # (nightly.yml, e2e-gpu-nightly-w7900). - if: >- - always() - && needs.changes.result == 'success' - && ( - (github.event_name != 'workflow_dispatch' - && github.event_name != 'pull_request' - && github.event_name != 'merge_group' - && needs.changes.outputs.serve == 'true') - || (github.event_name == 'workflow_dispatch' - && (inputs.platform == 'all' || inputs.platform == 'w7900')) - ) - # Non-blocking while this hardware is proven out, as the MI350P lane is. - # Flip to `false` once it has a track record — that is what the older lanes - # run with, so a real regression on this silicon shows red instead of - # always green. - continue-on-error: true - env: - E2E_SERVE_TIMEOUT_SECS: "300" - E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait, as in e2e-gpu, then an assertion that no leaked E2E engine - # process survives. One card per job, so rocm-smi sees exactly the GPU this - # lane runs on. The floor stays at the shared 16 GiB default, which is a - # flat constant rather than a proportion of capacity: it only has to be - # high enough to catch a leftover serve still holding the card, and low - # enough that a clean host passes on the first poll. e2e-gpu uses the same - # 16 GiB against a very different VRAM size for exactly that reason — do - # NOT scale it down for a smaller part. On a 48GB card that floor only - # trips past ~32 GiB held, so the floor alone would miss a leftover serve - # holding less; the process gate is what catches it (#446). The floor is - # kept for the case the process gate cannot see: a non-E2E tenant - # exhausting the host. See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - # cache: false for the same reason as e2e-gpu — the build cache is kept in - # CARGO_TARGET_DIR on the work volume, not in GitHub's cache service. - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run E2E tests on W7900 - run: | - # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the - # cargo target dir and the shared caches are still warm on the next one. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the directory the runner's uv-cache-gc - # initContainer bounds — see the e2e-gpu lane for both mechanisms and - # why a cache one level deeper keeps the links but loses the floor. - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - # Pre-warm one shared runtime in place before the suite. Not an - # optimization — `install sdk` bakes absolute paths into the runtime - # manifest, so installing anywhere temporary breaks every later serve. - # See e2e-gpu for the full rationale. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane for why the e2e-test-hooks feature is the - # floor a pre-built binary must carry (xtask builds a superset). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has moved on — the tree is a cache, not a one-shot. - # See the e2e-gpu lane and `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" \ - --release-label "${MATRIX_SDK_VERSION}" - - # Optional scenario-name filter for a scoped manual dispatch — a cheap - # way to exercise this lane against a single scenario without the full suite. - NAME_FILTER="${GITHUB_EVENT_INPUTS_NAME_FILTER}" - if [ -n "$NAME_FILTER" ]; then - echo "name filter active: $NAME_FILTER" - cargo xtask e2e -- --name "$NAME_FILTER" - else - cargo xtask e2e - fi - env: - MATRIX_SDK_VERSION: ${{ matrix.sdk_version }} - GITHUB_EVENT_INPUTS_NAME_FILTER: ${{ github.event.inputs.name_filter }} - - - name: Upload E2E report (current) - if: always() && matrix.sdk_version == 'current' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-report - path: tests/e2e-cucumber/results/ - - - name: Upload E2E report (n-1) - if: always() && matrix.sdk_version == 'n-1' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-n-1-report - path: tests/e2e-cucumber/results/ - - - name: Upload E2E report (n-2) - if: always() && matrix.sdk_version == 'n-2' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-n-2-report - path: tests/e2e-cucumber/results/ - # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -1761,7 +1617,6 @@ jobs: - e2e-gpu-strix-ubuntu - e2e-gpu-strix-windows - e2e-wsl - - e2e-gpu-w7900 # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was @@ -1833,7 +1688,6 @@ jobs: e2e-gpu-strix-ubuntu=e2e-gpu-strix-ubuntu-release-report,e2e-gpu-strix-ubuntu-nightly-report e2e-gpu-strix-windows=e2e-gpu-strix-windows-report e2e-wsl=e2e-gpu-strix-wsl-report - e2e-gpu-w7900=e2e-gpu-w7900-report run: | uploaded=$(gh api --paginate \ "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \ diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 46c9c47ad..4ba2b8432 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,9 +36,9 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` but run on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only @@ -47,10 +47,9 @@ rung instead of being removed outright. A pull request now runs on exactly one Linux GPU node and one Windows GPU pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo -Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer -start on `pull_request` or in the merge queue; they keep their -`push`/dispatch triggers and run nightly (`e2e-gpu-nightly-strix` on the same -DevLab Dispatch pool labels, `e2e-gpu-nightly-w7900`). +Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in +the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane +in `e2e-selfhosted.yml` any more: that hardware is covered by the queue (below). [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length @@ -126,13 +125,11 @@ separate tier flag or tag filter to maintain. | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | -| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | AMD Radeon PRO W7900 (gfx1100, 48GB) on Linux | self-hosted `[self-hosted, linux, w7900]` | -On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` -are skipped; both still run on `push`, dispatch and nightly. In the merge queue -W7900 is one runner on borrowed quota that would sit behind the queue, and over -7 days 35% of Strix Ubuntu's merge-queue jobs (98 of 281) were cancelled at the -45 min cap; neither check is required and the queue does not wait for it. On a +On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` is skipped; it still +runs on `push` and dispatch. Over 7 days 35% of its merge-queue jobs (98 of 281) +were cancelled at the 45 min cap; the check is not required and the queue does +not wait for it. On a PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo pool). @@ -190,7 +187,7 @@ resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, -`e2e-wsl`, and `e2e-gpu-w7900`) run on AMD GPU systems, so they +and `e2e-wsl`) run on AMD GPU systems, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as described below. @@ -330,7 +327,6 @@ The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, `e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`, `e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, -`e2e-gpu-w7900-n-1-report`, `e2e-gpu-w7900-n-2-report`, `e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs @@ -394,9 +390,9 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and ## Triggers The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, -`pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and -`merge_group` (the Windows Strix Halo and WSL2 lanes only: MI300X, Strix Ubuntu and -W7900 are not required for merging and so are skipped in the merge queue rather +`pull_request` (all lanes but `e2e-gpu-strix-ubuntu`), and +`merge_group` (the Windows Strix Halo and WSL2 lanes only: MI300X and Strix Ubuntu +are not required for merging and so are skipped in the merge queue rather than left queued on limited runner capacity or hung at their cap after its PR has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request @@ -406,10 +402,10 @@ group of their own, so none is cancelled or replaced by another. The heavy lane: they run only on the Windows Strix Halo and WSL2 lanes (both set `E2E_MERGE_QUEUE` for `merge_group`), and the only vLLM serve before a change lands is the per-PR canary (`serve-05`), which runs when the -`serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X, -Strix Ubuntu and W7900 all skip `merge_group`, so it runs only Windows Strix and +`serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X +and Strix Ubuntu both skip `merge_group`, so it runs only Windows Strix and WSL2. `push` fires on `main` and on any `release/**` branch, so pushing to a -release branch runs the full self-hosted matrix (all five lanes) ahead of +release branch runs the full self-hosted matrix (all four lanes) ahead of cutting the `v*` tag that `release.yml` builds/publishes from — this is the release-candidate regression gate; `pull_request` fires against any base branch. Merging or closing a pull request also starts a run (`closed` is a @@ -448,8 +444,8 @@ plus broad-dependency safety nets), **not** a blanket `**/*.rs`. So a Rust chang that cannot affect serving — e.g. a dashboard-only or unrelated-crate PR — skips the heavy GPU matrix, while compile coverage for every crate still runs on `ci.yml`'s always-on build/test lanes. Off `pull_request` (push/merge_group) the -filter is forced `true`, so the full matrix (minus MI300X, Strix Ubuntu and -W7900 in the merge queue) always runs there. Unlike the +filter is forced `true`, so the full matrix (minus MI300X and Strix Ubuntu +in the merge queue) always runs there. Unlike the pre-split layout the GPU jobs do **not** gate on the hosted `build-and-test` job — cross-workflow `needs` is not possible, so each GPU job builds the `rocm` binary itself as its first real step (a broken build fails that job fast and @@ -466,10 +462,9 @@ They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: - `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, - `strix-wsl`, `w7900`) — which self-hosted job(s) to run. `app-dev-gpu` + `strix-wsl`) — which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to - `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, and `w7900` to - `e2e-gpu-w7900`. (The mock lane has its own `platform` input on `ci.yml`; it + `e2e-gpu-strix-windows`, and `strix-wsl` to `e2e-wsl`. (The mock lane has its own `platform` input on `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a @@ -574,13 +569,11 @@ side-by-side runtime only when no match is found. See ## Blocking vs. non-blocking The self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-w7900` — all run with +`e2e-gpu-strix-windows`, and `e2e-wsl` — all run with check names that are absent from branch protection's required-status-check list (see below), so a hardware failure never gates a PR merge no matter how -it reports. Four of them run with `continue-on-error: false`, so a real -regression shows red instead of always green; `e2e-gpu-w7900` runs with -`continue-on-error: true` while that hardware is being proven out, and should -be flipped to `false` once it has a track record. Their results also surface in +it reports. They run with `continue-on-error: false`, so a real +regression shows red instead of always green. Their results also surface in the self-hosted consolidated report for visibility. A lane that ran but uploaded no report (it died before writing its results) fails the consolidated report job, naming the lane, rather than silently dropping out of the grid; on a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index f24f136c6..d005021bc 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -179,11 +179,10 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | -| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | -`e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux GPU -node and one Windows GPU pool per PR); they also skip `merge_group` and still run on `push`, -dispatch and nightly. The merge queue therefore has no native Linux GPU lane +`e2e-gpu-strix-ubuntu` skips `pull_request` (one Linux GPU +node and one Windows GPU pool per PR); it also skips `merge_group` and still runs on `push` +and dispatch. The merge queue therefore has no native Linux GPU lane (MI300X skips it too); it runs only Windows Strix and WSL2. The blocking mock job passes when every applicable scenario is pass-or-xfail with diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index c6bc0c815..9ebdca6c6 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -175,7 +175,7 @@ pub struct ScenarioDecl { /// on-demand runs and only runs in the merge queue, which opts in via /// `E2E_MERGE_QUEUE`. Keeps the PR feedback loop short while still exercising /// the heavy serves before a change lands. Only the Windows Strix and WSL2 - /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, W7900, + /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, /// Strix Ubuntu) do not. pub merge_queue: bool, } diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 720fd4405..63fa727cc 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -207,8 +207,6 @@ mod tests { "e2e-gpu-rad3-report", "e2e-gpu-mi350p-report", "e2e-gpu-w7900-report", - "e2e-gpu-w7900-n-1-report", - "e2e-gpu-w7900-n-2-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-ubuntu-n-1-report", "e2e-gpu-strix-ubuntu-n-2-report", diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 3686819b3..2a7adc479 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2872,8 +2872,8 @@ jobs: } } assert_eq!( - delegating, 8, - "expected the eight native Linux preflight lanes to delegate to \ + delegating, 7, + "expected the seven native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2941,8 +2941,8 @@ jobs: } assert_eq!( bodies.len(), - 8, - "expected eight native Linux preflight bodies to compare, found {} — \ + 7, + "expected seven native Linux preflight bodies to compare, found {} — \ if a lane was added or removed, update this count", bodies.len() ); @@ -4301,8 +4301,8 @@ fi } } assert_eq!( - delegating, 8, - "expected the 8 native Linux preflight lanes to delegate to \ + delegating, 7, + "expected the 7 native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); assert_eq!( @@ -7576,7 +7576,7 @@ jobs: condition: lane_if(extra, "", platform), matrix_lane: true, }; - // Strix Ubuntu and W7900 are off `pull_request` and `merge_group`. + // Strix Ubuntu is off `pull_request` and `merge_group`. let off_pr_lane = |job, platform: &str| PinnedIf { job, condition: lane_if( @@ -7596,7 +7596,6 @@ jobs: off_pr_lane("e2e-gpu-strix-ubuntu", "strix-ubuntu"), lane("e2e-gpu-strix-windows", "", "strix-windows"), lane("e2e-wsl", "", "strix-wsl"), - off_pr_lane("e2e-gpu-w7900", "w7900"), // Skips because the `serve` output of a skipped `changes` is empty. PinnedIf { job: "e2e-report", @@ -7717,38 +7716,33 @@ jobs: ); } // A pull request runs one Linux node (MI300X) and one Windows pool (the - // Strix Halo pool). Strix Ubuntu and W7900 are off pull_request AND - // merge_group (W7900 is one runner on borrowed quota; 35% of Strix - // Ubuntu's merge-queue jobs hit the 45 min cap; neither check is required - // and the queue does not wait for it) but still run on push and dispatch. + // Strix Halo pool). Strix Ubuntu is off pull_request AND merge_group (35% + // of its merge-queue jobs hit the 45 min cap; the check is not required + // and the queue does not wait for it) but still runs on push and dispatch. // Pin the WHOLE gate, not a substring: the event exclusions must sit in the // non-dispatch branch (so a dispatch can still select the lane) and nothing // else about the event may be named, so a gate that dropped push some other // way fails too. - for (job, platform) in [ - ("e2e-gpu-strix-ubuntu", "strix-ubuntu"), - ("e2e-gpu-w7900", "w7900"), - ] { - let expected = format!( - "always() && needs.changes.result == 'success' && ( \ - (github.event_name != 'workflow_dispatch' \ - && github.event_name != 'pull_request' \ - && github.event_name != 'merge_group' \ - && needs.changes.outputs.serve == 'true') \ - || (github.event_name == 'workflow_dispatch' \ - && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" - ); - assert_eq!( - normalized_whitespace(&gate(job)), - normalized_whitespace(&expected), - "`{job}` has the wrong event gate (pull_request and merge_group excluded in \ - the non-dispatch branch only)" - ); - assert!( - !job_block(&sh, job).contains("E2E_MERGE_QUEUE"), - "{job} never runs in the merge queue, so it must not opt into @merge-queue scenarios" - ); - } + let (job, platform) = ("e2e-gpu-strix-ubuntu", "strix-ubuntu"); + let expected = format!( + "always() && needs.changes.result == 'success' && ( \ + (github.event_name != 'workflow_dispatch' \ + && github.event_name != 'pull_request' \ + && github.event_name != 'merge_group' \ + && needs.changes.outputs.serve == 'true') \ + || (github.event_name == 'workflow_dispatch' \ + && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" + ); + assert_eq!( + normalized_whitespace(&gate(job)), + normalized_whitespace(&expected), + "`{job}` has the wrong event gate (pull_request and merge_group excluded in \ + the non-dispatch branch only)" + ); + assert!( + !job_block(&sh, job).contains("E2E_MERGE_QUEUE"), + "{job} never runs in the merge queue, so it must not opt into @merge-queue scenarios" + ); } /// The nightly Strix Ubuntu lane moved from a static host (HOME, CARGO_HOME, From 1c99c06c8a1bd7078c643c5285a3a32e414c2e6d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 11:27:46 +0000 Subject: [PATCH 2/3] docs: scope the W7900 report artifact to nightly.yml e2e-selfhosted.yml has no W7900 lane any more, so e2e-gpu-w7900-report is uploaded only by nightly.yml, like the rad3 and mi350p reports. Say so next to them, and bring the lane-count comments in line with the seven lanes the assertions now pin. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 7 ++++--- xtask/src/workflow_contract.rs | 18 +++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 4ba2b8432..8698dfe49 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -331,9 +331,10 @@ The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `rad3`/`mi350p` are nightly-only -lanes (ROCMAI-125) and don't run that matrix, so they have no `-n-1`/`-n-2` -entries. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900` +are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any +more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they +don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 2a7adc479..f0183849b 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2833,7 +2833,7 @@ jobs: #[test] fn every_native_linux_preflight_delegates_to_the_shared_script() { // What the deduplication (#294) actually bought, and the only thing - // keeping the next #446 from landing eight times: the eight native Linux + // keeping the next #446 from landing seven times: the seven native Linux // preflights are now one script plus a floor. The drift test below can no // longer catch a block left behind, because a block left behind would // carry its own body and simply not be compared against anything. @@ -2885,7 +2885,7 @@ jobs: // where re-inlining drift grows back: a lane that keeps the delegation // and adds a line of its own above it — one more `pkill -f`, a lane-local // retry, a second floor — satisfies `step.contains(SHARED_PREFLIGHT)` - // and the floor check and leaves that test green, while the eight bodies + // and the floor check and leaves that test green, while the seven bodies // are no longer one body. That is how the original six inline copies // diverged from their PowerShell mirrors: nothing compared them to each // other, only to a substring. @@ -2894,8 +2894,8 @@ jobs: // not_drift` and `both_wsl_lanes_settle_the_clock_before_running_the_ // suite` pin their pairs, and then pin the shape they agree on: a single // `bash