diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index a06b7b8f5..96b63f1b9 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -12,14 +12,15 @@ name: E2E self-hosted # Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, # MI300X: the only lane where vLLM is the effective engine) and one Windows GPU # pool (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix -# Ubuntu and W7900 lanes keep their push / dispatch triggers and run nightly -# (nightly.yml); neither starts on pull_request or merge_group. +# Ubuntu lane keeps its push / dispatch triggers; it does not start on +# pull_request or merge_group. There is no W7900 lane here: W7900 is covered only +# by the scheduled queue nightly's `w7900-dual` lane (docs/nightly-on-the-queue.md), +# not on push, release/** or dispatch. # # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not -# because of `continue-on-error`, which is `false` on every lane but -# one, so a real hardware regression shows red instead of always green. The -# exception is `e2e-gpu-w7900`, `true` only while that hardware is proven out. PRs #334, #335 and #343 all +# because of `continue-on-error`, which is `false` on every lane, so a real +# hardware regression shows red instead of always green. PRs #334, #335 and #343 all # landed while `E2E tests (Strix Halo, Ubuntu)` reported `cancelled` on the # merge_group ref, which a required check under ALLGREEN would have blocked; # renaming a lane is therefore safe. @@ -41,7 +42,7 @@ on: # supersedes that PR's earlier run. It runs only the no-op `pr-closed` job; # every GPU lane skips because `changes` skips. This frees runners only where # the cancel completes: a job queued for a runner that is not picking it up - # (W7900 while its one runner is busy) may not finish cancelling, so this + # (a lane whose runner is busy) may not finish cancelling, so this # does NOT clear such queued runs. types: [opened, synchronize, reopened, closed] merge_group: @@ -57,7 +58,7 @@ on: type: choice default: all # Each value also selectable on its own; `all` covers every lane below. - options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, w7900] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -116,7 +117,7 @@ jobs: outputs: heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} # Narrow GPU-serve gate; forced true off-PR so the merge queue runs the - # full matrix (every lane but MI300X, Strix Ubuntu and W7900, which skip + # full matrix (every lane but MI300X and Strix Ubuntu, which skip # merge_group) and its # required checks are never starved. serve: ${{ steps.filter.outputs.serve || steps.all.outputs.forced }} @@ -204,7 +205,7 @@ jobs: # Exists only to start a run in the PR's concurrency group when the PR is # merged or closed: that run cancels the PR's older run. A GPU job of that run # which is queued for a runner that is not picking it up may not finish - # cancelling (observed on W7900-queued runs). + # cancelling (observed on queued runs). pr-closed: name: Supersede runs of a closed PR if: github.event_name == 'pull_request' && github.event.action == 'closed' @@ -1597,150 +1598,6 @@ jobs: path: tests/e2e-cucumber/results/ if-no-files-found: error - e2e-gpu-w7900: - name: E2E tests (W7900, ${{ matrix.sdk_version }}) - # 90min: matches every other Linux GPU lane — same collapsed suite. The card - # is smaller than the Instinct parts, so do not tighten this below their cap. - timeout-minutes: 90 - # `w7900` names the hardware rather than reusing the generic `amd-gpu`: - # GitHub matches a job to any runner whose labels are a superset, so a shared - # label would let this workstation card pick up Instinct-targeted work and - # vice versa. - runs-on: [self-hosted, linux, w7900] - needs: [changes] - strategy: - fail-fast: false - matrix: - sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} - # Not on pull_request or merge_group: one runner on borrowed quota, 58-70% of - # its PR jobs never started before being superseded, and a merge_group run - # would queue behind it the same way (the queue does not wait for this - # check). Still runs on push (main, release/**), dispatch and nightly - # (nightly.yml, e2e-gpu-nightly-w7900). - if: >- - always() - && needs.changes.result == 'success' - && ( - (github.event_name != 'workflow_dispatch' - && github.event_name != 'pull_request' - && github.event_name != 'merge_group' - && needs.changes.outputs.serve == 'true') - || (github.event_name == 'workflow_dispatch' - && (inputs.platform == 'all' || inputs.platform == 'w7900')) - ) - # Non-blocking while this hardware is proven out, as the MI350P lane is. - # Flip to `false` once it has a track record — that is what the older lanes - # run with, so a real regression on this silicon shows red instead of - # always green. - continue-on-error: true - env: - E2E_SERVE_TIMEOUT_SECS: "300" - E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait, as in e2e-gpu, then an assertion that no leaked E2E engine - # process survives. One card per job, so rocm-smi sees exactly the GPU this - # lane runs on. The floor stays at the shared 16 GiB default, which is a - # flat constant rather than a proportion of capacity: it only has to be - # high enough to catch a leftover serve still holding the card, and low - # enough that a clean host passes on the first poll. e2e-gpu uses the same - # 16 GiB against a very different VRAM size for exactly that reason — do - # NOT scale it down for a smaller part. On a 48GB card that floor only - # trips past ~32 GiB held, so the floor alone would miss a leftover serve - # holding less; the process gate is what catches it (#446). The floor is - # kept for the case the process gate cannot see: a non-E2E tenant - # exhausting the host. See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - # cache: false for the same reason as e2e-gpu — the build cache is kept in - # CARGO_TARGET_DIR on the work volume, not in GitHub's cache service. - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run E2E tests on W7900 - run: | - # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the - # cargo target dir and the shared caches are still warm on the next one. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the directory the runner's uv-cache-gc - # initContainer bounds — see the e2e-gpu lane for both mechanisms and - # why a cache one level deeper keeps the links but loses the floor. - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - # Pre-warm one shared runtime in place before the suite. Not an - # optimization — `install sdk` bakes absolute paths into the runtime - # manifest, so installing anywhere temporary breaks every later serve. - # See e2e-gpu for the full rationale. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane for why the e2e-test-hooks feature is the - # floor a pre-built binary must carry (xtask builds a superset). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has moved on — the tree is a cache, not a one-shot. - # See the e2e-gpu lane and `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" \ - --release-label "${MATRIX_SDK_VERSION}" - - # Optional scenario-name filter for a scoped manual dispatch — a cheap - # way to exercise this lane against a single scenario without the full suite. - NAME_FILTER="${GITHUB_EVENT_INPUTS_NAME_FILTER}" - if [ -n "$NAME_FILTER" ]; then - echo "name filter active: $NAME_FILTER" - cargo xtask e2e -- --name "$NAME_FILTER" - else - cargo xtask e2e - fi - env: - MATRIX_SDK_VERSION: ${{ matrix.sdk_version }} - GITHUB_EVENT_INPUTS_NAME_FILTER: ${{ github.event.inputs.name_filter }} - - - name: Upload E2E report (current) - if: always() && matrix.sdk_version == 'current' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-report - path: tests/e2e-cucumber/results/ - - - name: Upload E2E report (n-1) - if: always() && matrix.sdk_version == 'n-1' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-n-1-report - path: tests/e2e-cucumber/results/ - - - name: Upload E2E report (n-2) - if: always() && matrix.sdk_version == 'n-2' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-n-2-report - path: tests/e2e-cucumber/results/ - # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -1761,7 +1618,6 @@ jobs: - e2e-gpu-strix-ubuntu - e2e-gpu-strix-windows - e2e-wsl - - e2e-gpu-w7900 # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was @@ -1833,7 +1689,6 @@ jobs: e2e-gpu-strix-ubuntu=e2e-gpu-strix-ubuntu-release-report,e2e-gpu-strix-ubuntu-nightly-report e2e-gpu-strix-windows=e2e-gpu-strix-windows-report e2e-wsl=e2e-gpu-strix-wsl-report - e2e-gpu-w7900=e2e-gpu-w7900-report run: | uploaded=$(gh api --paginate \ "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \ diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 46c9c47ad..82f89390b 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,9 +36,9 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` but run on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only @@ -47,10 +47,12 @@ rung instead of being removed outright. A pull request now runs on exactly one Linux GPU node and one Windows GPU pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo -Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer -start on `pull_request` or in the merge queue; they keep their -`push`/dispatch triggers and run nightly (`e2e-gpu-nightly-strix` on the same -DevLab Dispatch pool labels, `e2e-gpu-nightly-w7900`). +Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in +the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane +in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue +nightly's `w7900-dual` lane (below; advisory), not on `push`, `release/**` or +dispatch. Its first run, the 11-13 queue nightly +([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), got capacity at once and finished `Done`, exit 0. [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length @@ -126,13 +128,11 @@ separate tier flag or tag filter to maintain. | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | -| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | AMD Radeon PRO W7900 (gfx1100, 48GB) on Linux | self-hosted `[self-hosted, linux, w7900]` | -On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` -are skipped; both still run on `push`, dispatch and nightly. In the merge queue -W7900 is one runner on borrowed quota that would sit behind the queue, and over -7 days 35% of Strix Ubuntu's merge-queue jobs (98 of 281) were cancelled at the -45 min cap; neither check is required and the queue does not wait for it. On a +On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` is skipped; it still +runs on `push` and dispatch. Over 7 days 35% of its merge-queue jobs (98 of 281) +were cancelled at the 45 min cap; the check is not required and the queue does +not wait for it. On a PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo pool). @@ -190,7 +190,7 @@ resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, -`e2e-wsl`, and `e2e-gpu-w7900`) run on AMD GPU systems, so they +and `e2e-wsl`) run on AMD GPU systems, so they exercise host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as described below. @@ -330,14 +330,14 @@ The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, `e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`, `e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, -`e2e-gpu-w7900-n-1-report`, `e2e-gpu-w7900-n-2-report`, `e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `rad3`/`mi350p` are nightly-only -lanes (ROCMAI-125) and don't run that matrix, so they have no `-n-1`/`-n-2` -entries. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900` +are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any +more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they +don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is @@ -394,9 +394,9 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and ## Triggers The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, -`pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and -`merge_group` (the Windows Strix Halo and WSL2 lanes only: MI300X, Strix Ubuntu and -W7900 are not required for merging and so are skipped in the merge queue rather +`pull_request` (all lanes but `e2e-gpu-strix-ubuntu`), and +`merge_group` (the Windows Strix Halo and WSL2 lanes only: MI300X and Strix Ubuntu +are not required for merging and so are skipped in the merge queue rather than left queued on limited runner capacity or hung at their cap after its PR has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request @@ -406,10 +406,10 @@ group of their own, so none is cancelled or replaced by another. The heavy lane: they run only on the Windows Strix Halo and WSL2 lanes (both set `E2E_MERGE_QUEUE` for `merge_group`), and the only vLLM serve before a change lands is the per-PR canary (`serve-05`), which runs when the -`serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X, -Strix Ubuntu and W7900 all skip `merge_group`, so it runs only Windows Strix and +`serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X +and Strix Ubuntu both skip `merge_group`, so it runs only Windows Strix and WSL2. `push` fires on `main` and on any `release/**` branch, so pushing to a -release branch runs the full self-hosted matrix (all five lanes) ahead of +release branch runs the full self-hosted matrix (all four lanes) ahead of cutting the `v*` tag that `release.yml` builds/publishes from — this is the release-candidate regression gate; `pull_request` fires against any base branch. Merging or closing a pull request also starts a run (`closed` is a @@ -448,8 +448,8 @@ plus broad-dependency safety nets), **not** a blanket `**/*.rs`. So a Rust chang that cannot affect serving — e.g. a dashboard-only or unrelated-crate PR — skips the heavy GPU matrix, while compile coverage for every crate still runs on `ci.yml`'s always-on build/test lanes. Off `pull_request` (push/merge_group) the -filter is forced `true`, so the full matrix (minus MI300X, Strix Ubuntu and -W7900 in the merge queue) always runs there. Unlike the +filter is forced `true`, so the full matrix (minus MI300X and Strix Ubuntu +in the merge queue) always runs there. Unlike the pre-split layout the GPU jobs do **not** gate on the hosted `build-and-test` job — cross-workflow `needs` is not possible, so each GPU job builds the `rocm` binary itself as its first real step (a broken build fails that job fast and @@ -466,10 +466,9 @@ They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: - `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, - `strix-wsl`, `w7900`) — which self-hosted job(s) to run. `app-dev-gpu` + `strix-wsl`) — which self-hosted job(s) to run. `app-dev-gpu` maps to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to - `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, and `w7900` to - `e2e-gpu-w7900`. (The mock lane has its own `platform` input on `ci.yml`; it + `e2e-gpu-strix-windows`, and `strix-wsl` to `e2e-wsl`. (The mock lane has its own `platform` input on `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a @@ -574,13 +573,11 @@ side-by-side runtime only when no match is found. See ## Blocking vs. non-blocking The self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-w7900` — all run with +`e2e-gpu-strix-windows`, and `e2e-wsl` — all run with check names that are absent from branch protection's required-status-check list (see below), so a hardware failure never gates a PR merge no matter how -it reports. Four of them run with `continue-on-error: false`, so a real -regression shows red instead of always green; `e2e-gpu-w7900` runs with -`continue-on-error: true` while that hardware is being proven out, and should -be flipped to `false` once it has a track record. Their results also surface in +it reports. They run with `continue-on-error: false`, so a real +regression shows red instead of always green. Their results also surface in the self-hosted consolidated report for visibility. A lane that ran but uploaded no report (it died before writing its results) fails the consolidated report job, naming the lane, rather than silently dropping out of the grid; on a diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index deb7090ab..c8a3bdbb6 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -22,6 +22,12 @@ GPU jobs of [`nightly.yml`](../.github/workflows/nightly.yml) do (see way those jobs' results are: the queue runs on the same infrastructure and under the same access as the self-hosted runners, and neither signs what it reports. +The lane set is whatever GPU models the queue has. The 11-13 run +([43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)) discovered `mi300x`, `mi325x`, `mi350p`, `mi350x`, +`mi355x`, `r9700s` and `w7900-dual`; `w7900-dual` (W7900) is advisory, and it is +the only scheduled coverage of that hardware, since `e2e-selfhosted.yml` no +longer has a W7900 lane. + **The remaining gap is native Windows and WSL.** The queue runs Linux pods, so the native Windows and WSL lanes stay on their dedicated runners, in `nightly.yml` and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); nothing diff --git a/docs/testing.md b/docs/testing.md index 7e4e853b3..127d45c6b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3066,10 +3066,11 @@ The user-visible side is `serve-55` (`@id:serve-small-assistant-short-name`): `rocm serve qwen-small` with no `--engine`, which must reach `ready` and answer. It carries `@requires-gpu` and deliberately no `@requires-engine`, because the engine is the CLI's to pick here — so the lane that exercises the Lemonade half -is whichever GPU lane defaults to Lemonade (the Strix Halo and W7900 lanes) and -the vLLM half falls to the Instinct lane. Those lanes do run on pull requests +is whichever GPU lane defaults to Lemonade (the Strix Halo lanes, and W7900 on +the queue nightly only) and the vLLM half falls to the Instinct lane. Of those, +MI300X and the Strix Halo Windows and WSL2 lanes run on pull requests ([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when the `serve` -path filter matches) but none of their check names is a required status check, +path filter matches; W7900 has no lane there any more) but none of their check names is a required status check, so the two unit tests above are the blocking per-PR coverage. The scenario carries no [expectations.toml](../tests/e2e-cucumber/expectations.toml) row, unlike its managed-Lemonade siblings on the Linux Strix lane: whether it shares diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index f24f136c6..d005021bc 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -179,11 +179,10 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | -| `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | -`e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux GPU -node and one Windows GPU pool per PR); they also skip `merge_group` and still run on `push`, -dispatch and nightly. The merge queue therefore has no native Linux GPU lane +`e2e-gpu-strix-ubuntu` skips `pull_request` (one Linux GPU +node and one Windows GPU pool per PR); it also skips `merge_group` and still runs on `push` +and dispatch. The merge queue therefore has no native Linux GPU lane (MI300X skips it too); it runs only Windows Strix and WSL2. The blocking mock job passes when every applicable scenario is pass-or-xfail with diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index c6bc0c815..9ebdca6c6 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -175,7 +175,7 @@ pub struct ScenarioDecl { /// on-demand runs and only runs in the merge queue, which opts in via /// `E2E_MERGE_QUEUE`. Keeps the PR feedback loop short while still exercising /// the heavy serves before a change lands. Only the Windows Strix and WSL2 - /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, W7900, + /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, /// Strix Ubuntu) do not. pub merge_queue: bool, } diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 720fd4405..63fa727cc 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -207,8 +207,6 @@ mod tests { "e2e-gpu-rad3-report", "e2e-gpu-mi350p-report", "e2e-gpu-w7900-report", - "e2e-gpu-w7900-n-1-report", - "e2e-gpu-w7900-n-2-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-ubuntu-n-1-report", "e2e-gpu-strix-ubuntu-n-2-report", diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 3686819b3..f0183849b 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2833,7 +2833,7 @@ jobs: #[test] fn every_native_linux_preflight_delegates_to_the_shared_script() { // What the deduplication (#294) actually bought, and the only thing - // keeping the next #446 from landing eight times: the eight native Linux + // keeping the next #446 from landing seven times: the seven native Linux // preflights are now one script plus a floor. The drift test below can no // longer catch a block left behind, because a block left behind would // carry its own body and simply not be compared against anything. @@ -2872,8 +2872,8 @@ jobs: } } assert_eq!( - delegating, 8, - "expected the eight native Linux preflight lanes to delegate to \ + delegating, 7, + "expected the seven native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2885,7 +2885,7 @@ jobs: // where re-inlining drift grows back: a lane that keeps the delegation // and adds a line of its own above it — one more `pkill -f`, a lane-local // retry, a second floor — satisfies `step.contains(SHARED_PREFLIGHT)` - // and the floor check and leaves that test green, while the eight bodies + // and the floor check and leaves that test green, while the seven bodies // are no longer one body. That is how the original six inline copies // diverged from their PowerShell mirrors: nothing compared them to each // other, only to a substring. @@ -2894,8 +2894,8 @@ jobs: // not_drift` and `both_wsl_lanes_settle_the_clock_before_running_the_ // suite` pin their pairs, and then pin the shape they agree on: a single // `bash