diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 964216e06..a06b7b8f5 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -9,6 +9,12 @@ name: E2E self-hosted # THIS workflow's supersession, never the required checks in ci.yml. Only # pull_request runs are superseded (see `concurrency:` below). # +# Per-PR scope: a pull request runs exactly one Linux GPU node (`e2e-gpu`, +# MI300X: the only lane where vLLM is the effective engine) and one Windows GPU +# pool (the Strix Halo pool: `e2e-gpu-strix-windows` and `e2e-wsl`). The Strix +# Ubuntu and W7900 lanes keep their push / dispatch triggers and run nightly +# (nightly.yml); neither starts on pull_request or merge_group. +# # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not # because of `continue-on-error`, which is `false` on every lane but @@ -110,7 +116,8 @@ jobs: outputs: heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }} # Narrow GPU-serve gate; forced true off-PR so the merge queue runs the - # full matrix (every lane but MI300X, which skips merge_group) and its + # full matrix (every lane but MI300X, Strix Ubuntu and W7900, which skip + # merge_group) and its # required checks are never starved. serve: ${{ steps.filter.outputs.serve || steps.all.outputs.forced }} # ROCMAI-430: the three most recent rows of the official release list @@ -465,16 +472,24 @@ jobs: # ratio the 35 was chosen for. # An include_nightly dispatch also runs the AIM serve scenarios (serve-26/27, # @nightly, up to 1200s readiness each on a cold pull) and the large-model - # serve, which cannot fit the per-PR 45min budget. + # serve, which cannot fit the 45min budget of an ordinary push run. timeout-minutes: ${{ inputs.include_nightly && 105 || 45 }} runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. + # Not on pull_request: a PR gets exactly one Linux GPU node (MI300X, the only + # lane where vLLM runs) and one Windows pool (Strix Halo). Not on + # merge_group either: over 7 days 35% of its merge-queue jobs (98 of 281) were + # cancelled at the 45 min cap, and the queue does not wait for this + # non-required check. Still runs on push (main, release/**), dispatch and + # nightly (nightly.yml, e2e-gpu-nightly-strix, same pool labels). if: >- always() && needs.changes.result == 'success' && ( (github.event_name != 'workflow_dispatch' + && github.event_name != 'pull_request' + && github.event_name != 'merge_group' && needs.changes.outputs.serve == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')) @@ -486,7 +501,7 @@ jobs: channel: [release, nightly] # See `e2e-gpu` (ROCMAI-430): layered alongside the pre-existing # `channel` dimension above, not replacing it — this lane is the only - # per-PR job with two independent matrix axes. + # job with two independent matrix axes. sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # The nightly channel only carries the latest (pre-)release builds, so # it cannot resolve the n-1/n-2 released versions the matrix pins @@ -508,8 +523,6 @@ jobs: # Match the MI300X dispatch path: opt into the platform-adaptive large-model # scenario only when the manual include_nightly input is enabled. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - # Heavy @merge-queue serves run only in the merge queue. - E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -1599,11 +1612,18 @@ jobs: fail-fast: false matrix: sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} + # Not on pull_request or merge_group: one runner on borrowed quota, 58-70% of + # its PR jobs never started before being superseded, and a merge_group run + # would queue behind it the same way (the queue does not wait for this + # check). Still runs on push (main, release/**), dispatch and nightly + # (nightly.yml, e2e-gpu-nightly-w7900). if: >- always() && needs.changes.result == 'success' && ( (github.event_name != 'workflow_dispatch' + && github.event_name != 'pull_request' + && github.event_name != 'merge_group' && needs.changes.outputs.serve == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'w7900')) @@ -1616,8 +1636,6 @@ jobs: env: E2E_SERVE_TIMEOUT_SECS: "300" E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" - # Full serve matrix at the merge-queue gate. - E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7e6bd1292..5399e19e9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -671,10 +671,11 @@ jobs: name: e2e-gpu-mi350p-${{ matrix.channel }}-report path: tests/e2e-cucumber/results/ - # W7900 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-w7900 lane - # in e2e-selfhosted.yml (w7900 label, shared uv cache, GPU preflight), with - # @nightly scenarios enabled. Keep the two in step: the report crate asserts - # the nightly lanes publish the same platforms as the per-PR lanes. + # W7900 counterpart to e2e-gpu-nightly. Mirrors the e2e-gpu-w7900 lane (which + # no longer runs on pull_request) in e2e-selfhosted.yml (w7900 label, shared + # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in + # step: the report crate asserts the nightly lanes publish the same platforms + # as the lanes in e2e-selfhosted.yml. e2e-gpu-nightly-w7900: name: E2E tests (W7900, incl. nightly-only) timeout-minutes: 90 @@ -762,29 +763,24 @@ jobs: # Strix Ubuntu job. e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) - # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits when - # this host has Docker; on a host without it they skip and cost nothing. - # Whether it has Docker is unconfirmed: nothing in this job installs or checks - # it, so the "Report whether Docker is reachable" step logs the answer. If it - # does not, no nightly lane runs AIM serve and a pool-based one is needed. + # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits; on a + # host without Docker they skip and cost nothing. The "Report whether Docker + # is reachable" step logs the answer each run. serve-27's known failure on + # this pool is an xfail row in expectations.toml (rocm-cli#1058). timeout-minutes: 120 - # `native` disambiguates this runner from any host that might carry a bare - # `wsl` label alongside `strix-halo` (the static WSL runner this repo used - # before the WSL lanes moved to the ephemeral pool) -- the paths below - # exist only on the native host, whether or not that other runner remains - # registered. - runs-on: [self-hosted, linux, strix-halo, native] + # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in + # e2e-selfhosted.yml, which runs on push and dispatch but not on pull_request + # or merge_group (pinned by workflow_contract.rs). This lane used to target + # the static `native` host, which is offline: every nightly sat queued until + # the 12h run cap and the lane never ran. Pool hosts are cold, + # so there are no host-specific HOME or cache-dir overrides. + runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] continue-on-error: true strategy: fail-fast: false matrix: channel: [release, nightly] env: - HOME: /home/ubuntu/actions-runner/e2e-home - CARGO_HOME: /home/ubuntu/actions-runner/.cargo - RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup - TMPDIR: /home/ubuntu/actions-runner/tmp - PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache E2E_INCLUDE_NIGHTLY: "1" E2E_SERVE_TIMEOUT_SECS: "300" E2E_TUI_TIMEOUT_SECS: "90" @@ -793,11 +789,8 @@ jobs: with: persist-credentials: false - - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs - run: | - mkdir -p "$HOME" "$TMPDIR" "$PIP_CACHE_DIR" - echo "prepared" - bash scripts/reclaim-gpu.sh + - name: Reclaim GPU from stray E2E processes + run: bash scripts/reclaim-gpu.sh # Bounded wait for a free GPU, then an assertion that no leaked E2E engine # process survives. gfx1151 shares system memory, so the floor is smaller @@ -815,11 +808,11 @@ jobs: - name: Ensure Rust toolchain run: | - if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then + if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | sh -s -- -y --no-modify-path --default-toolchain none fi - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" # serve-26/27 (AIM) need Docker reachable on this host; without it they skip # silently. Say which it is, so a nightly that never ran them is visible. diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 10768bdd7..46c9c47ad 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,14 +36,22 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below (`e2e-gpu-strix-ubuntu` runs both its `[release, nightly]` channel legs per PR, ROCMAI-125) | No — absent from the required-status-check list | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | | Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The same per-PR lanes | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` but run on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still covered, but off every PR's critical path, so they moved to the nightly-only rung instead of being removed outright. +A pull request now runs on exactly one Linux GPU node and one Windows GPU +pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and +the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo +Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer +start on `pull_request` or in the merge queue; they keep their +`push`/dispatch triggers and run nightly (`e2e-gpu-nightly-strix` on the same +DevLab Dispatch pool labels, `e2e-gpu-nightly-w7900`). + [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length of a job, and publishes the result; @@ -120,12 +128,19 @@ separate tier flag or tag filter to maintain. | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | AMD Radeon PRO W7900 (gfx1100, 48GB) on Linux | self-hosted `[self-hosted, linux, w7900]` | +On `pull_request` and `merge_group`, `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` +are skipped; both still run on `push`, dispatch and nightly. In the merge queue +W7900 is one runner on borrowed quota that would sit behind the queue, and over +7 days 35% of Strix Ubuntu's merge-queue jobs (98 of 281) were cancelled at the +45 min cap; neither check is required and the queue does not wait for it. On a +PR the other lanes above run (one Linux node, MI300X, and the Windows Strix Halo +pool). + `e2e-gpu-strix-ubuntu` additionally runs a `strategy.matrix.channel: [release, -nightly]` axis — two concurrent legs on the same job, channel-suffixed -artifact names — so the per-PR smoke gate proves the nightly channel boots on -at least one platform without adding the axis (and its wall-clock cost) to -every per-PR lane. This mirrors `nightly.yml`'s channel matrix from -ROCMAI-429, applied to a single per-PR lane rather than all of them. +nightly]` axis: two concurrent legs on the same job, channel-suffixed artifact +names, so that lane proves the nightly package channel boots. It no longer runs +on `pull_request`, so on a PR no lane exercises the nightly channel; it runs on +`push`, dispatch and in `nightly.yml`'s channel matrix (ROCMAI-429). All three Strix Halo lanes run on the AMD Ryzen DevLab Dispatch pool: a fresh runner is registered per job and destroyed after, opt-in only via the @@ -140,22 +155,15 @@ cloning the checkout into the guest's native filesystem to avoid building against the slow DrvFs mount. The distro is deliberately left without ROCm's WSL driver bridge — see "What the WSL distro needs" below. -This migration is scoped to the per-PR lanes in this table, plus the nightly -WSL and Windows Strix lanes below — not every nightly lane. `nightly.yml`'s -Ubuntu Strix lane (`e2e-gpu-nightly-strix`) still pins the `native` label and -runs on the same static, always-on host these per-PR lanes moved off of — -that label still means what it always did there: `native` disambiguates this -host from any runner that might carry a bare `wsl` label alongside -`strix-halo` (the static WSL runner this repo used before the WSL lanes below -moved to the ephemeral pool). Whether that runner remains registered is not -something this migration changes either way, since nothing here targets a -`wsl`-labeled runner anymore. `nightly.yml`'s `e2e-wsl-nightly` and -`e2e-gpu-nightly-strix-windows`, however, both moved onto the same DevLab -Dispatch pool as the per-PR `e2e-wsl`/`e2e-gpu-strix-windows` lanes above, for -the same reason: neither needs a dedicated `native`- or `wsl`-labeled runner, -so there is nothing pool-incompatible about either. Moving the remaining -`native` Ubuntu nightly lane onto the pool as well is a separate, not-yet-made -decision. +The Strix Halo lanes in this table run on the DevLab Dispatch pool, and so do +their nightly counterparts: `nightly.yml`'s `e2e-gpu-nightly-strix` (Ubuntu), +`e2e-gpu-nightly-strix-windows` and `e2e-wsl-nightly` all use the same +`runs-on` labels as `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows` and +`e2e-wsl` (pinned by `nightly_devlab_dispatch_lanes_share_the_per_pr_pool_labels` +in `xtask/src/workflow_contract.rs`). The Ubuntu nightly lane used to pin the +static `native` label; that host is offline, so the lane sat queued until the +run cap and never ran. Whether a `native`- or `wsl`-labeled runner remains +registered no longer matters: nothing targets one. Every label in a `runs-on` must narrow the pool to one kind of hardware. In particular the MI300X lane pins `mi300x` rather than `amd-gpu`: `amd-gpu` is @@ -242,7 +250,8 @@ not applicable, instead of failing them on a premise the host cannot meet. Each workflow has its own consolidated report job. `ci.yml`'s `e2e-report` covers the mock platform; -`e2e-selfhosted.yml`'s `e2e-report` covers the per-PR GPU platforms; +`e2e-selfhosted.yml`'s `e2e-report` covers its self-hosted GPU platforms (a +pull request runs only the MI300X and Windows Strix Halo pool lanes among them); `nightly.yml`'s `e2e-report-nightly` covers a strict superset of those platforms (it also runs `e2e-gpu-rad3` and `e2e-gpu-mi350p`, demoted from per-PR to nightly-only per ROCMAI-125) with the `@nightly` scenarios @@ -371,7 +380,7 @@ run there; WSL reports `os_family` `"linux"` and stays vLLM-eligible); otherwise `vllm` is only preferred for the `*-dcgpu` families and `gfx906`/`gfx908`/`gfx90a`. Consequences for the lanes above: -- Strix Halo is `gfx1151` on every OS, so all three per-PR Strix lanes +- Strix Halo is `gfx1151` on every OS, so all three Strix lanes (Ubuntu, Windows, WSL2) and their nightly counterparts are lemonade-only — no matrix value turns a Strix lane into a vLLM lane. - `e2e-gpu` (MI300X) is the only per-PR lane where vLLM is the effective @@ -385,17 +394,23 @@ otherwise `vllm` is only preferred for the `*-dcgpu` families and ## Triggers The GPU jobs (in `e2e-selfhosted.yml`) run automatically on `push`, -`pull_request`, and `merge_group` (every GPU lane except MI300X, which is not -required for merging and so is skipped in the merge queue rather than left -queued on limited runner capacity after its PR has merged) when the workflow's -own `changes` job's `serve` path filter is `true`. A new push to a pull request +`pull_request` (all lanes but `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`), and +`merge_group` (the Windows Strix Halo and WSL2 lanes only: MI300X, Strix Ubuntu and +W7900 are not required for merging and so are skipped in the merge queue rather +than left queued on limited runner capacity or hung at their cap after its PR +has merged) when the workflow's own `changes` job's +`serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM -lane: they run only on the other lanes, and the only vLLM serve before a change -lands is the per-PR canary (`serve-05`), which runs when the `serve` filter -matches. `push` fires on `main` and on any `release/**` branch, so pushing to -a release branch runs the full self-hosted matrix ahead of cutting the `v*` tag that `release.yml` builds/publishes from — this is the +lane: they run only on the Windows Strix Halo and WSL2 lanes (both set +`E2E_MERGE_QUEUE` for `merge_group`), and the only vLLM serve +before a change lands is the per-PR canary (`serve-05`), which runs when the +`serve` filter matches. **The merge queue has no native Linux GPU lane**: MI300X, +Strix Ubuntu and W7900 all skip `merge_group`, so it runs only Windows Strix and +WSL2. `push` fires on `main` and on any `release/**` branch, so pushing to a +release branch runs the full self-hosted matrix (all five lanes) ahead of +cutting the `v*` tag that `release.yml` builds/publishes from — this is the release-candidate regression gate; `pull_request` fires against any base branch. Merging or closing a pull request also starts a run (`closed` is a listed `pull_request` activity type) that joins that PR's group and runs only a @@ -433,8 +448,8 @@ plus broad-dependency safety nets), **not** a blanket `**/*.rs`. So a Rust chang that cannot affect serving — e.g. a dashboard-only or unrelated-crate PR — skips the heavy GPU matrix, while compile coverage for every crate still runs on `ci.yml`'s always-on build/test lanes. Off `pull_request` (push/merge_group) the -filter is forced `true`, so the full matrix (minus MI300X in the merge queue) -always runs there. Unlike the +filter is forced `true`, so the full matrix (minus MI300X, Strix Ubuntu and +W7900 in the merge queue) always runs there. Unlike the pre-split layout the GPU jobs do **not** gate on the hosted `build-and-test` job — cross-workflow `needs` is not possible, so each GPU job builds the `rocm` binary itself as its first real step (a broken build fails that job fast and @@ -475,8 +490,8 @@ gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu Nearly every GPU serve scenario points its `data/runtimes` at one shared, pre-warmed managed runtime tree (`E2E_SHARED_RUNTIMES_DIR`), so a multi-GiB `rocm install sdk` happens once per runner instead of once per scenario. The tree -lives on the runner's workspace — persistent across runs on the long-lived -`native`-labeled hosts, rebuilt by the prewarm step below on each ephemeral +lives on the runner's workspace — persistent across runs on a long-lived +runner, rebuilt by the prewarm step below on each ephemeral DevLab Dispatch pool run — survives `git clean`, and is namespaced by source-layout generation (`e2e-prewarm-multi-arch-v2`) so a branch using a new package layout cannot poison the cache consumed by code that only understands the diff --git a/docs/testing.md b/docs/testing.md index 2127f5255..a0b13b8f7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1473,20 +1473,19 @@ install. `serve-26` and `serve-27` (`features/model_serving.feature`, the AIM serve and restart canaries) are `@nightly` for cost: each pulls the AIM image and a 9B checkpoint on a cold runner and may wait up to 1200s for readiness, which on the -per-PR Strix Ubuntu job (about 25 minutes without them, 45 minute cap) used up -the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable -and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` -is set on such a host: the nightly Strix Halo lane, or an `include_nightly` -dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then gets a 105 minute -cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that -they stay reachable when the opt-in is set. - -Caveat: it is unconfirmed that the nightly Strix Halo host has Docker (nothing in -`nightly.yml` installs or checks it). Without it these two scenarios skip on -every nightly run, and only a manual `include_nightly` dispatch on a host with -Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker -is reachable ("Report whether Docker is reachable"); check that line before -relying on this coverage. +Strix Ubuntu job, back when it ran on pull requests (about 25 minutes without +them, 45 minute cap) used up the whole budget. They still need `@requires-engine:aim`, +i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where +`E2E_INCLUDE_NIGHTLY=1` is set on such a host: the nightly Strix Halo lane, or an +`include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then +gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins +the tagging and that they stay reachable when the opt-in is set. + +Both scenarios skip on a host without Docker; the `e2e-gpu-nightly-strix` job +logs whether Docker is reachable ("Report whether Docker is reachable"). +`serve-27`'s known failure on the Strix Halo pool is an xfail row in +`expectations.toml`, tracked in rocm-cli#1058; a pass there is an XPASS, which +means the row is stale. ### Lemonade managed-serve failure evidence diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 90ca4b192..73e1f2df0 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -46,11 +46,12 @@ set -euo pipefail # The scenario root stays ABSOLUTE, and follows TMPDIR. # # Scenario dirs come from `TempDir::with_prefix("rocm-e2e-")`, which builds on -# `std::env::temp_dir()` and so follows TMPDIR; the nightly Strix lane redirects -# it to `$HOME/actions-runner/tmp`. A bare `/tmp/rocm-e2e` kept matching there -# only by accident — that directory is itself named `tmp`, so the substring -# still appeared — and would silently stop matching under any redirect not -# named `tmp`. Deriving the redirected form fixes that for ANY target. +# `std::env::temp_dir()` and so follows TMPDIR; a lane on a host that redirects +# it (the nightly Strix lane did, to `$HOME/actions-runner/tmp`, before it moved +# to the pool) is the case this handles. A bare `/tmp/rocm-e2e` kept matching +# there only by accident — that directory is itself named `tmp`, so the +# substring still appeared — and would silently stop matching under any +# redirect not named `tmp`. Deriving the redirected form fixes that for ANY target. # # WHAT THE ANCHORING DOES AND DOES NOT BUY. Roots are matched as unanchored # substrings of the WHOLE command line, so any root also matches a process that diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index f3e9ea9bd..f24f136c6 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -181,6 +181,11 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | | `e2e-gpu-w7900` | `e2e-selfhosted.yml` | Radeon PRO W7900 (self-hosted) | no | +`e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900` skip `pull_request` (one Linux GPU +node and one Windows GPU pool per PR); they also skip `merge_group` and still run on `push`, +dispatch and nightly. The merge queue therefore has no native Linux GPU lane +(MI300X skips it too); it runs only Windows Strix and WSL2. + The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow consolidates its own platforms: `ci.yml`'s `e2e-report` the mock platform, diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index a50f1229f..dcf773659 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -253,3 +253,13 @@ reason = "Lemonade 11.5.1 fetches TheRock only from repo.amd.com/rocm/tarball/, when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" reason = "The Windows large-model serve crashes in the SDK's Tensile library on the first inference request; the pinned ROCm 7.13 backend runs against the 7.14.1 SDK because Lemonade 11.5.1 cannot align it (suspected cause, not reproduced off Windows)." + +# Scoped to native Linux Strix Halo, the only place it has been seen (evidence is +# in the tracking issue); `is_wsl = false` keeps WSL2, which also reports +# `os = "linux"` on gfx1151, out of it, so a WSL host that gains Docker and passes +# the scenario is not an XPASS. A host without Docker skips the scenario before +# this row applies. +[["serve-aim-lifecycle-parity"]] +when = { os = "linux", is_wsl = false, therock_family = "gfx1151*", has_amd_gpu = true } +bug = "#1058" +reason = "Stopping a running AIM service on Strix Halo Ubuntu reports `Local server stop not confirmed` (exit 1); the serve itself works." diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 1786d007e..549a4f662 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -359,14 +359,16 @@ Feature: Model serving # verified on (see `HostCapability::engine_available`'s "aim" arm — Radeon # gfx110X/120X and Strix Halo gfx1151, not Instinct); a Strix Halo # self-hosted runner with Docker is where this actually runs. - # `@nightly` keeps both AIM scenarios off the per-PR lanes: each one pulls the - # image and a 9B checkpoint on a cold runner and may wait out the full - # 1200s readiness cap, so the pair on the per-PR Strix Ubuntu job (~25 min - # without them, 45 min cap) ended every run at the cap. They run only when + # `@nightly` keeps both AIM scenarios off the pull-request lanes: each one + # pulls the image and a 9B checkpoint on a cold runner and may wait out the + # full 1200s readiness cap, so the pair on the Strix Ubuntu job (~25 min + # without them, 45 min cap) ended every run at the cap. That job now runs + # only on push, dispatch and nightly, never on pull_request. They run only when # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. - # Unconfirmed: whether the nightly Strix Halo host has Docker; without it both - # skip every night (the nightly job logs whether Docker is reachable). + # On a host without Docker both skip (the nightly job logs whether Docker is + # reachable). serve-27's known failure on Strix Halo Ubuntu is an xfail row in + # expectations.toml, tracked in rocm-cli#1058. # `@serve-timeout:1200` # widens both the harness's readiness poll and the CLI's own # ROCM_CLI_AIM_READY_TIMEOUT_SECS (see isolate_cmd) — a cold-start HF weight diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 52736ff4c..c6bc0c815 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -174,7 +174,9 @@ pub struct ScenarioDecl { /// per-engine canary on the fast path, so it is skipped on ordinary per-PR / /// on-demand runs and only runs in the merge queue, which opts in via /// `E2E_MERGE_QUEUE`. Keeps the PR feedback loop short while still exercising - /// the full serve matrix (on every lane but MI300X) before a change lands. + /// the heavy serves before a change lands. Only the Windows Strix and WSL2 + /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, W7900, + /// Strix Ubuntu) do not. pub merge_queue: bool, } @@ -1196,10 +1198,11 @@ serve_timeout_secs = 90 #[test] fn aim_serve_scenarios_stay_off_the_per_pr_lanes() { // serve-26/27 pull the AIM image and a 9B checkpoint on a cold runner and - // may each wait out a 1200s readiness cap. Run per-PR on the Strix Ubuntu - // pool they pushed the job (~25 min otherwise) into its 45 min cap on every - // run. They must resolve to Skip unless the nightly opt-in is set, and must - // still be reachable (not Skip for the nightly reason) when it is. + // may each wait out a 1200s readiness cap. Run on every PR on the Strix + // Ubuntu pool (before that lane left the PR path) they pushed the job + // (~25 min otherwise) into its 45 min cap on every run. They must + // resolve to Skip unless the nightly opt-in is set, and must still be + // reachable (not Skip for the nightly reason) when it is. let m = Expectations::default(); for prefix in ["serve-26 -", "serve-27 -"] { let d = ScenarioDecl::from_tags(&model_serving_tags(prefix)); diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index bfd9db279..720fd4405 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -142,7 +142,7 @@ fn label_for_root_report(dir: &Path) -> String { /// Strips the literal `${{ matrix.channel }}` template segment a channel /// matrix inserts into its artifact names — nightly's six GPU lanes and, per -/// ROCMAI-125, `e2e-gpu-strix-ubuntu`'s per-PR `[release, nightly]` leg — so +/// ROCMAI-125, `e2e-gpu-strix-ubuntu`'s `[release, nightly]` leg — so /// a name like `e2e-gpu-${{ matrix.channel }}-report` compares as its /// canonical `e2e-gpu-report` base. This scan is text-based over the raw /// workflow source and does not simulate matrix expansion, so the template diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 291d45029..25ffb5794 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1620,6 +1620,10 @@ jobs: for (per_pr_job, nightly_job) in [ ("e2e-wsl", "e2e-wsl-nightly"), ("e2e-gpu-strix-windows", "e2e-gpu-nightly-strix-windows"), + // Off the PR path since the two-node PR scope, so the nightly is where + // this lane runs on a schedule: it must not drift back to the + // offline static `native` host. + ("e2e-gpu-strix-ubuntu", "e2e-gpu-nightly-strix"), ] { let per_pr = runs_on_values(job_block(&sh, per_pr_job)); let nightly_values = runs_on_values(job_block(&nightly, nightly_job)); @@ -7555,18 +7559,32 @@ jobs: // did, while a prefix check still passed. A new job has no entry here // and fails until it is added, which is the point: someone must decide // how it behaves on `closed`. - let lane_if = |extra: &str, platform: &str| { + // `extra` gates the whole lane; `inner` sits in the non-dispatch branch, + // where a lane that skips `pull_request` keeps the exclusion (see + // `mi300x_lane_is_skipped_in_the_merge_queue`). + let lane_if = |extra: &str, inner: &str, platform: &str| { format!( "always() && needs.changes.result == 'success' && {extra}\ - ( (github.event_name != 'workflow_dispatch' \ - && needs.changes.outputs.serve == 'true') \ + ( (github.event_name != 'workflow_dispatch' && {inner}\ + needs.changes.outputs.serve == 'true') \ || (github.event_name == 'workflow_dispatch' \ && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" ) }; let lane = |job, extra: &str, platform: &str| PinnedIf { job, - condition: lane_if(extra, platform), + condition: lane_if(extra, "", platform), + matrix_lane: true, + }; + // Strix Ubuntu and W7900 are off `pull_request` and `merge_group`. + let off_pr_lane = |job, platform: &str| PinnedIf { + job, + condition: lane_if( + "", + "github.event_name != 'pull_request' \ + && github.event_name != 'merge_group' && ", + platform, + ), matrix_lane: true, }; let expected = vec![ @@ -7575,10 +7593,10 @@ jobs: "github.event_name != 'merge_group' && ", "app-dev-gpu", ), - lane("e2e-gpu-strix-ubuntu", "", "strix-ubuntu"), + off_pr_lane("e2e-gpu-strix-ubuntu", "strix-ubuntu"), lane("e2e-gpu-strix-windows", "", "strix-windows"), lane("e2e-wsl", "", "strix-wsl"), - lane("e2e-gpu-w7900", "", "w7900"), + off_pr_lane("e2e-gpu-w7900", "w7900"), // Skips because the `serve` output of a skipped `changes` is empty. PinnedIf { job: "e2e-report", @@ -7679,12 +7697,7 @@ jobs: !job_block(&sh, "e2e-gpu").contains("E2E_MERGE_QUEUE"), "e2e-gpu never runs in the merge queue, so it must not opt into @merge-queue scenarios" ); - for job in [ - "e2e-gpu-strix-ubuntu", - "e2e-gpu-strix-windows", - "e2e-wsl", - "e2e-gpu-w7900", - ] { + for job in ["e2e-gpu-strix-windows", "e2e-wsl"] { // Only the dispatch split may look at the event: any other event // condition could silently keep the lane out of the merge queue. let gate = normalized_whitespace(&gate(job)); @@ -7692,6 +7705,97 @@ jobs: !gate.contains("merge_group") && !gate.contains("'pull_request'"), "`{job}` must keep running in the merge queue (gate: `{gate}`)" ); + // And it must opt into the heavy serves there, for merge_group only, + // so the docs' "Windows Strix Halo and WSL2 run the merge-queue + // serves" stays true of the workflow. + assert_eq!( + job_mapping(job_block(&sh, job), "env") + .get("E2E_MERGE_QUEUE") + .map(String::as_str), + Some("${{ github.event_name == 'merge_group' && '1' || '' }}"), + "`{job}` must set E2E_MERGE_QUEUE for merge_group events only" + ); + } + // A pull request runs one Linux node (MI300X) and one Windows pool (the + // Strix Halo pool). Strix Ubuntu and W7900 are off pull_request AND + // merge_group (W7900 is one runner on borrowed quota; 35% of Strix + // Ubuntu's merge-queue jobs hit the 45 min cap; neither check is required + // and the queue does not wait for it) but still run on push and dispatch. + // Pin the WHOLE gate, not a substring: the event exclusions must sit in the + // non-dispatch branch (so a dispatch can still select the lane) and nothing + // else about the event may be named, so a gate that dropped push some other + // way fails too. + for (job, platform) in [ + ("e2e-gpu-strix-ubuntu", "strix-ubuntu"), + ("e2e-gpu-w7900", "w7900"), + ] { + let expected = format!( + "always() && needs.changes.result == 'success' && ( \ + (github.event_name != 'workflow_dispatch' \ + && github.event_name != 'pull_request' \ + && github.event_name != 'merge_group' \ + && needs.changes.outputs.serve == 'true') \ + || (github.event_name == 'workflow_dispatch' \ + && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" + ); + assert_eq!( + normalized_whitespace(&gate(job)), + normalized_whitespace(&expected), + "`{job}` has the wrong event gate (pull_request and merge_group excluded in \ + the non-dispatch branch only)" + ); + assert!( + !job_block(&sh, job).contains("E2E_MERGE_QUEUE"), + "{job} never runs in the merge queue, so it must not opt into @merge-queue scenarios" + ); + } + } + + /// The nightly Strix Ubuntu lane moved from a static host (HOME, CARGO_HOME, + /// RUSTUP_HOME, TMPDIR and PIP_CACHE_DIR pointed at `/home/ubuntu/actions-runner`) + /// to the DevLab Dispatch pool. A pool host has none of those paths, so the job + /// must not name the static tree, must not read one of those four variables it + /// does not set (an unset variable is empty, not defaulted), and may read + /// `$HOME` only for the Rust bootstrap under `$HOME/.cargo/bin`. This is a + /// textual guard over the job's non-comment lines, not proof the job runs. + #[test] + fn nightly_strix_ubuntu_job_holds_no_static_host_paths() { + let nightly = read_workflow("nightly.yml"); + let block = job_block(&nightly, "e2e-gpu-nightly-strix"); + let env = job_mapping(block, "env"); + let uncommented: String = block + .lines() + .filter(|line| !line.trim_start().starts_with('#')) + .collect::>() + .join("\n"); + assert!( + !uncommented.contains("actions-runner"), + "the pool host has no /home/ubuntu/actions-runner tree" + ); + // Whole-word match, so a future `HF_TMPDIR` is not read as `TMPDIR`. + let is_word = |c: char| c.is_ascii_alphanumeric() || c == '_'; + for var in ["CARGO_HOME", "RUSTUP_HOME", "TMPDIR", "PIP_CACHE_DIR"] { + let read = uncommented.match_indices(var).any(|(index, _)| { + let before = uncommented[..index].chars().next_back(); + let after = uncommented[index + var.len()..].chars().next(); + !before.is_some_and(is_word) && !after.is_some_and(is_word) + }); + assert!( + env.contains_key(var) || !read, + "e2e-gpu-nightly-strix reads {var} but no longer sets it" + ); + } + for (index, _) in uncommented + .match_indices("$HOME") + .chain(uncommented.match_indices("${HOME")) + { + let rest = &uncommented[index..]; + assert!( + rest.starts_with("$HOME/.cargo/bin"), + "e2e-gpu-nightly-strix reads $HOME outside the Rust bootstrap, which a pool \ + host does not provision: `{}`", + rest.lines().next().unwrap_or_default().trim() + ); } }