diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8296d6af2..fb02a1ded 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,8 +71,9 @@ env: # with no timeout — holds a runner for six hours and, in the merge queue, stalls # every queued PR behind it with no signal that anything is wrong. The values here # are deliberately loose (roughly 4x the observed p100 for most jobs, which is -# under 3 min; the two Windows jobs are the exception, since they run for many -# minutes on a cold pull-request cache and keep a 45 min limit) so they never fire +# under 3 min; the two Windows build jobs are the exception, since they run for +# many minutes on a cold pull-request cache and keep a 45 min limit, and so is the +# hosted Windows E2E job, which also runs the suite and keeps 60) so they never fire # on a merely slow run; they exist to convert a silent multi-hour stall into a # prompt, legible failure. Raise a value if a job legitimately grows into it. jobs: @@ -1000,8 +1001,8 @@ jobs: # every run, on a lane that determines when CI goes green. # # Deliberately NOT built with `rocm/e2e-test-hooks`, unlike the prebuilt - # lanes in e2e-selfhosted.yml / nightly.yml. Those run the full suite, - # whose scripted failure seams are compiled out without the feature. + # lanes in e2e-selfhosted.yml / nightly.yml. Those run scenarios whose + # scripted failure seams are compiled out without the feature. # `E2E_ONLY_LIFECYCLE` restricts this lane to @lifecycle scenarios, none # of which touch a seam — and this lane packages and installs the binary # through the real installer, so it should ship exactly what a release @@ -1016,8 +1017,8 @@ jobs: # (e2e-selfhosted.yml) is optional and not a required check. They are not # tagged `@lifecycle` themselves — they are not release packaging/ # install/uninstall scenarios, and tagging them so would drop them out of - # nightly.yml's and e2e-selfhosted.yml's full-suite runs, neither of - # which sets `E2E_INCLUDE_LIFECYCLE`. + # nightly.yml's and e2e-selfhosted.yml's runs, neither of which sets + # `E2E_INCLUDE_LIFECYCLE`. # # serve-33/serve-34/serve-35 (the native-Windows vLLM refusal for `serve` and # `engines install`) are folded in the same way: `@requires-os:windows`, @@ -1990,10 +1991,9 @@ jobs: # e2e-selfhosted.yml: the artifact-name contract in `xtask::e2e_report` # reads `name: e2e-*` lines literally. Artifact names are unique per run. # The 24.04 leg keeps `e2e-report`, which the consolidated report below - # picks up through its `*-report` glob. The 26.04 leg's artifact is named - # outside both that glob and the `e2e-` namespace: the report has one - # mock platform, and two inputs labelled as it would collide. It is - # uploaded for inspection only. + # downloads by that exact name. The 26.04 leg's artifact is named outside + # the `e2e-` namespace: the report has one mock Linux platform, and two + # inputs labelled as it would collide. It is uploaded for inspection only. - name: Upload E2E report if: always() && steps.gate.outputs.run == 'true' && matrix.os == 'ubuntu-24.04' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -2008,6 +2008,145 @@ jobs: name: ubuntu-26.04-e2e-results path: tests/e2e-cucumber/results/ + # The same suite on a GitHub-hosted Windows runner. The self-hosted GPU lanes + # (e2e-selfhosted.yml) run only the scenarios that need their GPU + # (E2E_GPU_ONLY), so this is where everything else meets native Windows — + # path handling, the Windows probes, the PowerShell surfaces. Built with + # `rocm/e2e-test-hooks` by `cargo xtask e2e`, like the Linux lane above and + # unlike windows-build-and-test's lifecycle run, which must ship exactly what + # a release ships. Scenarios on a simulated machine are `@requires-os:linux` + # and skip here: the Windows probes ask the registry, not a filesystem. + e2e-windows: + name: E2E tests (Windows) + timeout-minutes: 60 + runs-on: windows-latest + # Only `changes`: unlike the Linux `e2e` job this does not wait for the + # Linux `build-and-test`, which tells it nothing about the Windows build and + # would put that job's duration in front of this one's. + needs: [changes] + # Same compile cache as windows-build-and-test (see build-and-test for the + # design: Azure backend, written only by push and merge_group), under its + # own prefix — this build carries `rocm/e2e-test-hooks`, so its entries + # would never match that job's anyway. + env: + CARGO_INCREMENTAL: "0" + RUSTC_WRAPPER: sccache + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.CACHE_AZURE_CONNECTION_STRING }} + SCCACHE_AZURE_BLOB_CONTAINER: cache + SCCACHE_AZURE_KEY_PREFIX: rocm-cli/e2e-windows + SCCACHE_AZURE_RW_MODE: ${{ (github.event_name == 'push' || github.event_name == 'merge_group') && 'READ_WRITE' || 'READ_ONLY' }} + if: >- + always() + && needs.changes.result == 'success' + && (github.event_name != 'workflow_dispatch' + || inputs.platform == 'all' || inputs.platform == 'mock') + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Whether the real work runs this time: on a dispatch, or when the change + # is heavy (`merge_group` forces heavy). The job itself always runs, so the + # check is always reported — required checks that never report stall the + # merge queue. + - name: Decide whether to run + id: gate + shell: bash + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ] \ + || [ "${{ needs.changes.outputs.heavy }}" = "true" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + # Installs sccache. Runs BEFORE setup-rust-toolchain, whose rust-cache step + # runs `cargo metadata` and honours RUSTC_WRAPPER. `version` is pinned so + # the compiler wrapper does not float with the latest sccache release. + # continue-on-error for the same reason as build-and-test: a failed install + # is handled by the probe, not by failing the job. + - name: Set up sccache + if: steps.gate.outputs.run == 'true' + continue-on-error: true + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + with: + version: v0.17.0 + + # Keep the remote backend only if it answers — see the equivalent step in + # build-and-test for why this is a probe, and why failure clears the Azure + # variables rather than the wrapper. + - name: Probe sccache cache backend + id: sccache_probe + if: steps.gate.outputs.run == 'true' + shell: pwsh + run: | + # This script decides whether to use the cache; a non-zero status from + # the commands it runs is an answer, not a job failure. Pin the native + # error preference off so a future runner default cannot turn a failed + # probe into a failed step. + $PSNativeCommandUseErrorActionPreference = $false + # Every Out-File below passes -Encoding utf8 explicitly. PowerShell 7 + # (`shell: pwsh`) already defaults to UTF-8, but Windows PowerShell 5.1 + # defaults to UTF-16LE, which the runner cannot parse — so an edit that + # switched this step to `shell: powershell` would silently corrupt the + # environment file rather than fail visibly. + function Disable-Backend { + "SCCACHE_AZURE_CONNECTION_STRING=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "SCCACHE_AZURE_BLOB_CONTAINER=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + } + if (-not (Get-Command sccache -ErrorAction SilentlyContinue)) { + Write-Output "::warning::sccache is not installed; compiling without it." + "RUSTC_WRAPPER=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + # Nothing will read the credential now; don't leave it in the job env. + Disable-Backend + exit 0 + } + # sccache is the compiler wrapper from here on, remote backend or not, + # so there are statistics worth reporting either way. + "active=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + if (-not $env:SCCACHE_AZURE_CONNECTION_STRING) { + Write-Output "::notice::No cache credential available (expected on forks); compiling without a shared cache." + Disable-Backend + exit 0 + } + sccache --start-server + if ($LASTEXITCODE -ne 0) { + Write-Output "::warning::sccache could not reach the cache backend; compiling without a shared cache." + # Leave no half-started server behind for the cargo steps to find. + sccache --stop-server *> $null + Disable-Backend + } + exit 0 + + - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 + if: steps.gate.outputs.run == 'true' + with: + cache-on-failure: true + + - name: Run E2E tests + if: steps.gate.outputs.run == 'true' + shell: pwsh + run: cargo xtask e2e + + - name: sccache stats + if: always() && steps.sccache_probe.outputs.active == 'true' + shell: pwsh + run: sccache --show-stats + + # Its own grid in the run Summary; see e2e-report for why the consolidated + # (required) report does not wait for this lane. + - name: Report summary + if: always() && steps.gate.outputs.run == 'true' + shell: pwsh + run: cargo xtask e2e-report --artifacts-dir tests/e2e-cucumber/results --html-out tests/e2e-cucumber/results/consolidated.html >> $env:GITHUB_STEP_SUMMARY + + - name: Upload E2E report + if: always() && steps.gate.outputs.run == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: e2e-windows-report + path: tests/e2e-cucumber/results/ + # Consolidate the mock (GitHub-hosted) platform's report into the cross-platform # report: a platform × tier matrix in the run Summary plus a single merged HTML # artifact. `if: always()` so a failing platform still appears; runs on @@ -2015,8 +2154,11 @@ jobs: # # The self-hosted GPU platforms are consolidated by e2e-selfhosted.yml's own # report job (they moved there so an offline runner can't stall this workflow's - # required checks). This job only depends on the mock `e2e` lane; the `*-report` - # glob still auto-includes any new GitHub-hosted platform added here. + # required checks). This job only depends on the mock `e2e` lane, and downloads + # only its artifact. `e2e-windows` summarizes itself instead: waiting on it here + # would put a cold Windows release build in front of this REQUIRED check, and + # matching it by glob would include it or not depending on which job finished + # first. e2e-report: name: E2E consolidated report runs-on: ubuntu-24.04 # one LTS: only merges report.json files @@ -2043,9 +2185,9 @@ jobs: # built against one Ubuntu release's glibc is never restored on another. cache-key: ubuntu-24.04 - # Pull every e2e artifact matching the glob. `xtask e2e-report` turns each - # into one labeled platform and the `*-report` glob makes new platforms - # appear automatically. Layout note: download-artifact@v8 extracts a match + # Pull the mock lane's artifact (named exactly; see the job comment for why + # not a glob). `xtask e2e-report` turns it into one labeled platform. + # Layout note: download-artifact@v8 extracts a match # into e2e-artifacts// when several match, but flattens # straight into e2e-artifacts/ when EXACTLY ONE matches (its source picks # the root path when `artifacts.length === 1`). After the ci.yml ⇄ @@ -2055,7 +2197,7 @@ jobs: - name: Download all E2E reports uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: '*-report' + pattern: 'e2e-report' path: e2e-artifacts - name: Build consolidated report + step summary diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 6ab781edc..8b8693bed 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -264,9 +264,22 @@ jobs: # workflow sets this unconditionally; here it lets a scoped dispatch confirm # a single nightly scenario (e.g. the 27B serve) without the full nightly run. E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + # On a pull request and in the merge queue (which this lane skips; the + # lanes that run there share this expression), of the scenarios that need + # a real GPU only the `@gpu-smoke` canaries (and, in the queue, the + # @merge-queue serves) run. Every GPU scenario that only needs the CLI to + # see a GPU runs on a simulated machine on the hosted `E2E tests` lane. Off + # those events the narrowing does not apply (the @nightly and @merge-queue + # gates still do). Keeps the GPU lanes, which set merge latency, short. + E2E_GPU_SMOKE_ONLY: "${{ (github.event_name == 'merge_group' || github.event_name == 'pull_request') && '1' || '' }}" # A real GPU lane: run the `@requires-real-gpu` scenarios that a simulated # host cannot stand in for. See HardwareMode in src/expectation.rs. E2E_HARDWARE: real + # ...and ONLY those, on every event: everything that needs no GPU runs on + # the GitHub-hosted Linux and Windows lanes in ci.yml (`E2E tests`, + # `E2E tests (Windows)`), so a GPU runner's time is not spent on it. See + # restrict_to_lane in src/expectation.rs. + E2E_GPU_ONLY: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -529,6 +542,7 @@ jobs: E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" # See the e2e-gpu lane. E2E_HARDWARE: real + E2E_GPU_ONLY: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -688,8 +702,11 @@ jobs: E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" # Heavy @merge-queue serves run only in the merge queue. E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" + # GPU smoke test only on pull requests and in the merge queue; see the e2e-gpu lane. + E2E_GPU_SMOKE_ONLY: "${{ (github.event_name == 'merge_group' || github.event_name == 'pull_request') && '1' || '' }}" # See the e2e-gpu lane. E2E_HARDWARE: real + E2E_GPU_ONLY: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -993,9 +1010,12 @@ jobs: # WSL2 works on these hosts; pin `--id Microsoft.WSL` in the install step so # a failed install can't report green; expect ~5min/job for distro install # since distros are unregistered between jobs by design; static TA runners - # are not needed for WSL coverage. Same suite as every other platform — - # scenarios the host cannot satisfy resolve to skip from the capability - # probe, so nothing is filtered out here. + # are not needed for WSL coverage. Unlike the GPU lanes, it sets no + # E2E_GPU_ONLY: scenarios the host cannot satisfy resolve to skip from the + # capability probe, the usual tag gates (@nightly, @merge-queue, @lifecycle, + # @requires-docker) apply as on every lane, and on a pull request and in the + # merge queue its real-GPU scenarios narrow to the smoke set + # (E2E_GPU_SMOKE_ONLY). Nothing is filtered out by hand. e2e-wsl: name: E2E tests (Strix Halo, WSL2, ${{ matrix.sdk_version }}) # 120min: like the native pool lanes, every job here is cold (fresh distro, @@ -1043,6 +1063,8 @@ jobs: E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" # Heavy @merge-queue serves run only in the merge queue. E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" + # GPU smoke test only on pull requests and in the merge queue; see the e2e-gpu lane. + E2E_GPU_SMOKE_ONLY: "${{ (github.event_name == 'merge_group' || github.event_name == 'pull_request') && '1' || '' }}" # See the e2e-gpu lane. E2E_HARDWARE: real # apps/rocm/build.rs falls back to GitHub's own GITHUB_HEAD_REF/ @@ -1490,7 +1512,7 @@ jobs: # Forward the job-level E2E_* env vars into the guest: `wsl.exe` # does not share the calling process's environment unless a # variable is named in WSLENV. - $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:E2E_HARDWARE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' + $env:WSLENV = 'E2E_SERVE_TIMEOUT_SECS:E2E_TUI_TIMEOUT_SECS:E2E_INCLUDE_NIGHTLY:E2E_MERGE_QUEUE:E2E_GPU_SMOKE_ONLY:E2E_HARDWARE:ROCM_CLI_RELEASE_REF:MATRIX_SDK_VERSION:GITHUB_EVENT_INPUTS_NAME_FILTER' & "$env:GITHUB_WORKSPACE\.github\scripts\Invoke-WslBash.ps1" -PipeFail -Script @' export PATH="$HOME/.cargo/bin:$PATH" cd /root/work/rocm-cli diff --git a/crates/e2e-report/src/consolidated/descriptor.rs b/crates/e2e-report/src/consolidated/descriptor.rs index 7291dd2cb..a330a41c4 100644 --- a/crates/e2e-report/src/consolidated/descriptor.rs +++ b/crates/e2e-report/src/consolidated/descriptor.rs @@ -128,6 +128,13 @@ const KNOWN_PLATFORMS: &[KnownPlatform] = &[ os: "Linux", slugs: &["mock"], }, + // The GitHub-hosted Windows lane: no GPU, native Windows. + KnownPlatform { + artifact: "e2e-windows-report", + platform: "Mock", + os: "Windows", + slugs: &["mock-windows"], + }, KnownPlatform { artifact: "e2e-gpu-report", platform: "MI300X", @@ -275,6 +282,7 @@ mod tests { fn parse_descriptor_maps_known_artifacts() { for (name, platform, os) in [ ("e2e-report", "Mock", "Linux"), + ("e2e-windows-report", "Mock", "Windows"), ("e2e-gpu-report", "MI300X", "Linux"), ("e2e-gpu-rad3-report", "R9700", "Linux"), ("e2e-gpu-mi350p-report", "MI350P", "Linux"), @@ -404,6 +412,7 @@ mod tests { // values the harness writes and the artifact names the workflows upload. for (slug, artifact) in [ ("mock", "e2e-report"), + ("mock-windows", "e2e-windows-report"), ("mi300x", "e2e-gpu-report"), ("gfx1201", "e2e-gpu-rad3-report"), ("mi350p", "e2e-gpu-mi350p-report"), diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 880555cc8..545231771 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -124,6 +124,7 @@ separate tier flag or tag filter to maintain. | Job | Workflow | Platform | Runner labels | |---|---|---|---| | `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | +| `e2e-windows` | `ci.yml` | Mock (no GPU) on Windows | GitHub-hosted `windows-latest` | | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | @@ -191,12 +192,24 @@ resolve to skip here, and known bugs resolve to xfail from every scenario that plants a simulated machine (a WSL2 distribution, a bare-metal host, several Instinct GPUs; see [testing.md](testing.md)), so those premises no longer need the lane whose hardware they describe. +`e2e-windows` runs the same suite on GitHub-hosted Windows, the platform where +the non-GPU scenarios meet the Windows probes and paths; it is advisory until +it has a record of being green. The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, -and `e2e-wsl`) run on AMD GPU systems, so they -exercise host/GPU detection, engine `detect`/`capabilities`, and live serving -scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as -described below. +and `e2e-wsl`) run on AMD GPU systems. All but `e2e-wsl` run ONLY the scenarios +that need a real GPU (`E2E_GPU_ONLY=1`): live serving, SDK installs, and +detection on the real machine. Everything else runs on the two hosted jobs +above, so a GPU runner's time is not spent on it. On a pull request and in the +merge queue, the lanes that run there narrow their real-GPU scenarios further +to the `@gpu-smoke` canaries (plus, in the queue, the `@merge-queue` serves). +Off those events the narrowing does not apply, and a lane takes every GPU +scenario its other gates allow (`@nightly` scenarios still need +`E2E_INCLUDE_NIGHTLY`, `@merge-queue` serves `E2E_MERGE_QUEUE`). `e2e-wsl` sets +no `E2E_GPU_ONLY`, so it runs the whole non-GPU suite inside a real WSL2 +distribution. GPU access in the WSL lane is advisory and probed per run, as +described below; where it is unavailable, its real-GPU scenarios resolve to +skip. `e2e-wsl` runs on the DevLab Dispatch pool. Its WSL2/Ubuntu-24.04 distro is built fresh inside every job (see above), so there is nothing of its own for @@ -205,9 +218,12 @@ GPU from stray E2E processes` step, retained from when that lane ran on a static host rather than added for anything specific to the pool; both lanes now pin the same `devlab-dispatch` label, so whether that step still earns its place there is a separate, open question, not something this paragraph -answers. Beyond that, `e2e-wsl` otherwise mirrors the Linux lane: GPU -preflight, toolchain bootstrap, shared-runtime pre-warm, then the full suite -with no hand filtering. It covers WSL host detection, the +answers. Beyond that, `e2e-wsl` follows the Linux GPU lane's steps: GPU +preflight, toolchain bootstrap, shared-runtime pre-warm, then the suite. +Unlike that lane it sets no `E2E_GPU_ONLY`, so it runs every scenario the host +can satisfy and the usual tag gates (`@nightly`, `@merge-queue`, `@lifecycle`, +`@requires-docker`) allow, smoke narrowing on a pull request and in the queue +apart, with no hand filtering. It covers WSL host detection, the Windows-to-WSL execution boundary, and whatever GPU access WSL exposes on that machine. The GPU preflight is advisory here precisely because GPU-on-WSL is what the lane is proving out: where it is unavailable the capability probe @@ -333,7 +349,7 @@ The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, `e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`, `e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, -`e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and +`e2e-gpu-w7900-report`, `e2e-report`, `e2e-windows-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg diff --git a/docs/testing.md b/docs/testing.md index 71e1e1655..0ec267493 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -984,6 +984,19 @@ names it on any other line. To run them locally on a GPU host: E2E_HARDWARE=real cargo xtask e2e ``` +The self-hosted GPU lanes run only the scenarios that need their GPU +(`E2E_GPU_ONLY=1`); everything else runs on the GitHub-hosted `E2E tests` +(Linux) and `E2E tests (Windows)` jobs, and inside a real WSL2 distribution on +the WSL lane. On a pull request and in the merge queue the lanes that run +there, the WSL lane included, narrow their real-GPU scenarios further, to the +cheap per-engine `@gpu-smoke` canaries (and, in the queue, the +`@merge-queue` serves) with `E2E_GPU_SMOKE_ONLY=1`. Like `E2E_HARDWARE`, both +switches abort the run on a value they do not recognise (anything but `1`, +`0`, empty or unset). Off those events the +narrowing does not apply, and a lane takes every GPU scenario its other gates +allow (`@nightly` scenarios still need `E2E_INCLUDE_NIGHTLY`, `@merge-queue` +serves `E2E_MERGE_QUEUE`). + A scenario describes its machine with a Given step — `a machine with an AMD Instinct GPU`, `a machine with two AMD Instinct GPUs`, `a machine with eight AMD Instinct GPUs`, `a bare-metal Linux @@ -3367,8 +3380,11 @@ is whichever GPU lane defaults to Lemonade (the Strix Halo lanes, and W7900 on the queue nightly only) and the vLLM half falls to the Instinct lane. Of those, MI300X and the Strix Halo Windows and WSL2 lanes run on pull requests ([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when the `serve` -path filter matches; W7900 has no lane there any more) but none of their check names is a required status check, -so the two unit tests above are the blocking per-PR coverage. The scenario +path filter matches; W7900 has no lane there any more), but there they narrow to +the `@gpu-smoke` canaries, so the scenario does not run on a pull request, and +none of their check names is a required status check either. The two unit tests +above are the blocking per-PR coverage; the scenario itself runs on pushes, +dispatches and nightly. The scenario carries no [expectations.toml](../tests/e2e-cucumber/expectations.toml) row, unlike its managed-Lemonade siblings on the Linux Strix lane: whether it shares their known serve failure there has not been observed, and a row claiming a @@ -4039,7 +4055,10 @@ silently selects nothing. Lemonade service for an *unrelated* model does not soften the no-GPU refusal: serving a different model is refused before any engine starts, and nothing is prepared or downloaded. Lemonade specifically, because it manages its own - runtime and so is the engine whose preparation is user-visible. + runtime and so is the engine whose preparation is user-visible. It is + `@requires-os:linux`: native Windows has no GPU probe that can say "none", so + a GPU-required launch there is passed through rather than refused, and on the + GPU-less hosted Windows lane it launched a real serve. It carries a second premise — a configured setup venv the runtime registry has not yet recorded — which makes the refusal's *ordering* observable. `rocm serve` resolves its runtime selection below the no-GPU bail deliberately, diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 2236fe166..965e8dbac 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -157,6 +157,8 @@ Scenarios carry stable-id and capability tags: | `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch that runs out of GPU memory so the OOM guidance can be verified without a GPU. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | +| `@merge-queue` | A heavy real-GPU serve (default-engine endpoint, inference, readiness, port reservation) that runs only in the merge queue, which opts in with `E2E_MERGE_QUEUE=1`; on the pull request the `@gpu-smoke` canaries cover per-engine serve and inference, but not default-engine selection, the readiness contract or port reservation. | +| `@gpu-smoke` | A cheap per-engine real-GPU canary. On a pull request and in the merge queue the self-hosted lanes that run there set `E2E_GPU_SMOKE_ONLY=1`, under which these (and, in the queue, the `@merge-queue` serves) are the only real-GPU scenarios that run; every other one resolves to **skip**, with the reason recorded. Scenarios that need no GPU never run on the lanes that set `E2E_GPU_ONLY=1` (every lane in `e2e-selfhosted.yml` but WSL; the nightly lanes set neither switch), which leave them to the hosted Linux and Windows jobs. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios and the native-Windows vLLM refusal scenarios serve-33/34/35. | Known bugs are **not** tagged in the `.feature` files — they live in @@ -176,14 +178,27 @@ serve does not compete with the first for device memory, and the failure quotes the service log tail plus the device's free-VRAM state, which is where the engine's own reason for the stall is recorded. -CI runs one job per platform, each executing the full suite. The mock job lives -in the `CI` workflow (`ci.yml`); the self-hosted GPU jobs live in a separate +CI runs one job per platform. The two GitHub-hosted jobs, `e2e` on Linux and +`e2e-windows` on Windows, run every scenario that needs no GPU on every pull +request — on Linux including every GPU scenario that plants a simulated +machine. The self-hosted GPU jobs run only the scenarios that need their GPU +(`E2E_GPU_ONLY=1`), and on a pull request and in the merge queue only the +`@gpu-smoke` canaries of those (plus, in the queue, the `@merge-queue` serves). +Both switches take `1` to turn on and unset, empty or `0` to stay off; any other +value aborts the run rather than running the wrong share of the suite. Off those +events the narrowing does not apply, and a job takes every GPU +scenario its other gates allow (`@nightly` scenarios still need +`E2E_INCLUDE_NIGHTLY`, `@merge-queue` serves `E2E_MERGE_QUEUE`). The WSL job +sets no `E2E_GPU_ONLY`: it runs the whole non-GPU suite inside a real WSL2 +distribution; its GPU access is advisory and probed per run, and where it is +unavailable its real-GPU scenarios skip. The mock job lives in the `CI` workflow (`ci.yml`); the self-hosted GPU jobs live in a separate `E2E self-hosted` workflow (`e2e-selfhosted.yml`) so a job queued on an offline self-hosted runner can never stall `ci.yml`'s merge-required checks: | Job | Workflow | Platform | Blocking | |---|---|---|---| | `e2e` | `ci.yml` | Mock (no GPU, GitHub-hosted) | yes | +| `e2e-windows` | `ci.yml` | Mock / Windows (no GPU, GitHub-hosted) | no | | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (self-hosted) | no | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index 40ac9699e..ba582706d 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -58,9 +58,12 @@ Feature: Chat and endpoint detection When the user lists running services Then the served model endpoint is listed - # Runs on every lane: on a real-hardware run (`E2E_HARDWARE=real`) on a GPU host - # `a model is served in the background` does a real `rocm serve`; otherwise, - # including a simulated run on a GPU box, it's backed by MockServer. The + # Runs on every lane that runs the non-GPU suite (the per-PR and push GPU + # lanes set E2E_GPU_ONLY and skip it). On a real-hardware run + # (`E2E_HARDWARE=real`) on a host with a usable GPU — the nightly GPU lanes, + # and `e2e-wsl` when its GPU probe passes — `a model is served in the + # background` does a real `rocm serve`; otherwise, including a simulated run + # on a GPU box, it's backed by MockServer. The # assertion (a tools-bearing request is accepted) is engine-agnostic, so no GPU # is required — dropping @requires-gpu gave this per-PR mock-lane coverage. @id:chat-tool-definitions-accepted @@ -70,9 +73,10 @@ Feature: Chat and endpoint detection When a chat request with tool definitions is sent Then the chat response is successful - # Runs on every lane (see chat-06): real serve on a real-hardware GPU run, - # MockServer otherwise. Asserts only that a served model returns a non-empty - # reply, which is engine-agnostic — real generation is covered by the + # Runs on every lane that runs the non-GPU suite (see chat-06): real serve on + # a real-hardware run with a usable GPU, MockServer otherwise. Asserts only + # that a served model returns a non-empty reply, which is engine-agnostic — + # real generation is covered by the # @requires-real-gpu serve-*-inference scenarios. @id:chat-end-to-end-local-model Scenario: chat-07 - End-to-end chat through a locally served model diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 9bc658d4a..acc0bcd2b 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -41,11 +41,12 @@ Feature: Model serving # vLLM serve + inference (safetensors model). Engine coverage: vLLM. This is the # deliberate vLLM half of a per-engine pair with `serve-lemonade-inference` # below, so it stays pinned to vLLM (the slug names the engine). It is also the - # vLLM per-PR canary: one real vLLM serve runs on every PR so a broken serve is - # caught before merge, while the heavier `@merge-queue` serves + # vLLM per-PR canary (`@gpu-smoke`): one real vLLM serve runs on every PR so a + # broken serve is caught before merge, while the heavier `@merge-queue` serves # (`serve-default-engine-working-endpoint`, `serve-default-engine-inference`, - # and `serve-readiness-contract`) run only in the merge queue (not on MI300X). - @id:serve-vllm-inference @requires-real-gpu @requires-engine:vllm + # `serve-readiness-contract` and the three port-reservation serves) run only + # in the merge queue (not on MI300X). + @id:serve-vllm-inference @requires-real-gpu @requires-engine:vllm @gpu-smoke Scenario: serve-05 - A served model responds to inference requests on vLLM Given a managed runtime is active And a model is being served on GPU @@ -69,9 +70,9 @@ Feature: Model serving And the response identifies the correct model # Lemonade serve + inference (GGUF model). Engine coverage: Lemonade. The - # lemonade per-PR canary: one real lemonade serve runs on every PR (the - # counterpart to the vLLM canary above). - @id:serve-lemonade-inference @requires-real-gpu @requires-engine:lemonade + # lemonade per-PR canary (`@gpu-smoke`): one real lemonade serve runs on every + # PR (the counterpart to the vLLM canary above). + @id:serve-lemonade-inference @requires-real-gpu @requires-engine:lemonade @gpu-smoke Scenario: serve-07 - A model served on lemonade responds to inference requests Given a managed runtime is active And a GGUF model is being served on lemonade @@ -84,7 +85,7 @@ Feature: Model serving # catalog entries can also serve directly on Linux when their metadata is # simple enough; other registered entries use Lemonade's router. Both canaries # use the same small checkpoint, sharing the cache without another download. - @id:serve-hf-checkpoint-inference @requires-real-gpu @requires-engine:lemonade + @id:serve-hf-checkpoint-inference @requires-real-gpu @requires-engine:lemonade @gpu-smoke Scenario: serve-08 - A canonical Hugging Face checkpoint serves and responds to inference Given a managed runtime is active And a canonical Hugging Face GGUF checkpoint is being served on lemonade @@ -489,7 +490,14 @@ Feature: Model serving # the bail left all three refusal scenarios green. Planting a configured setup # venv that the registry has not yet recorded is what makes the write reachable, # and the last Then is what refuses it. - @id:serve-unrelated-live-service-still-fails-fast @requires-no-gpu + # + # Linux only: the refusal needs a probe that can say "no usable GPU", and + # `rocm_core::usable_amd_gpu_indices` answers "unknown" off Linux, so a + # GPU-required launch is passed through on native Windows and this premise + # cannot hold there. On the GPU-less hosted Windows lane it launched a real + # Lemonade serve instead of refusing (ROCMAI-1059). Drop the tag when Windows + # has a probe. + @id:serve-unrelated-live-service-still-fails-fast @requires-no-gpu @requires-os:linux Scenario: serve-29 - A live service for another model does not soften the no-GPU refusal Given a live managed Lemonade serve for an unrelated model And setup names an unregistered runtime folder diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 3da571111..b39216bc2 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -137,7 +137,8 @@ pub struct HostCapability { /// What `serve` picks with no `--engine` on this host (re-implemented rule). pub effective_serve_engine: String, /// Stable platform identity derived from hardware, not from an artifact name: - /// "mock" (no AMD GPU), else the family/target (e.g. "mi300x", "strix-halo"). + /// "mock" / "mock-windows" (no AMD GPU), else the family/target (e.g. + /// "mi300x", "strix-halo"). pub platform_slug: String, /// Whether the `rocm` binary under test carries the test-only /// `e2e-oom-fault-injection` hook, so `@requires-oom-fault-injection` @@ -725,7 +726,8 @@ fn parse_engines_list(text: &str) -> Vec { /// Stable platform identity from hardware and host environment. WSL is a /// distinct platform even without a GPU, so its report never collides with the -/// ordinary hosted mock column. Otherwise no AMD GPU → "mock"; GPU hosts use a +/// ordinary hosted mock column. Otherwise no AMD GPU → "mock" ("mock-windows" +/// on Windows, the hosted Windows lane's own column); GPU hosts use a /// coarse slug from the gfx family (data-center → "mi300x"; Strix gfx115x → /// "strix-halo"), falling back to the normalized family. The OS is appended for /// families that ship on more than one OS (Strix Halo runs both Ubuntu and @@ -744,7 +746,12 @@ fn derive_platform_slug( }; } if !has_amd_gpu { - return "mock".to_owned(); + // The hosted Windows lane has no GPU either; give it its own column. + return if os_normalized(os_family) == "windows" { + "mock-windows".to_owned() + } else { + "mock".to_owned() + }; } match gfx_target { Some(t) => { @@ -1037,6 +1044,11 @@ Local model engines #[test] fn platform_slug_derivation() { assert_eq!(derive_platform_slug(false, None, "other", false), "mock"); + // The hosted Windows lane must not collide with the hosted Linux one. + assert_eq!( + derive_platform_slug(false, None, "windows", false), + "mock-windows" + ); // gfx950 normalizes to a `-dcgpu` family like gfx94x does, but it is a // different part with its own lane and report column — it must not be // slugged as mi300x. diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index c2461b5bc..5f7a4c3c8 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -38,6 +38,7 @@ const SERVE_TIMEOUT_PREFIX: &str = "serve-timeout:"; const NIGHTLY_TAG: &str = "nightly"; const LIFECYCLE_TAG: &str = "lifecycle"; const MERGE_QUEUE_TAG: &str = "merge-queue"; +const GPU_SMOKE_TAG: &str = "gpu-smoke"; // `@serial` deliberately has no entry here, and `from_tags` below silently // ignores it like any other unrecognized tag: it isn't an expectation- @@ -240,6 +241,10 @@ pub struct ScenarioDecl { /// lanes run in the merge queue; the native Linux GPU lanes (MI300X, /// Strix Ubuntu) do not. pub merge_queue: bool, + /// `@gpu-smoke`: one of the cheap per-engine real-GPU canaries that still + /// run on a pull request and in the merge queue, where a GPU lane runs no + /// other real-GPU scenario (`E2E_GPU_SMOKE_ONLY`, see [`restrict_to_lane`]). + pub gpu_smoke: bool, } impl ScenarioDecl { @@ -273,6 +278,7 @@ impl ScenarioDecl { let mut nightly = false; let mut lifecycle = false; let mut merge_queue = false; + let mut gpu_smoke = false; for tag in tags { let tag = tag .as_ref() @@ -315,6 +321,8 @@ impl ScenarioDecl { lifecycle = true; } else if tag == MERGE_QUEUE_TAG { merge_queue = true; + } else if tag == GPU_SMOKE_TAG { + gpu_smoke = true; } } Self { @@ -335,6 +343,7 @@ impl ScenarioDecl { nightly, lifecycle, merge_queue, + gpu_smoke, } } @@ -796,6 +805,131 @@ pub fn unmatched_also_run_ids<'a>(extra_ids: &'a [String], known_ids: &[String]) .collect() } +/// Which part of the suite a lane runs, beyond what the host can run at all. +/// +/// Both default to `false`: a lane runs everything its host can. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct LaneSelection { + /// `E2E_GPU_ONLY`: a self-hosted GPU lane runs only the scenarios that need + /// the real hardware it has. Everything else runs on the GitHub-hosted + /// Linux and Windows lanes, where a GPU runner's time is not spent on it. + pub gpu_only: bool, + /// `E2E_GPU_SMOKE_ONLY`: on a pull request and in the merge queue, a GPU + /// lane's real-GPU scenarios narrow further to the `@gpu-smoke` canaries + /// (and, in the queue, the `@merge-queue` serves). + pub smoke_only: bool, +} + +impl LaneSelection { + /// The variable a self-hosted GPU lane sets to run only its GPU scenarios. + pub const GPU_ONLY_ENV: &'static str = "E2E_GPU_ONLY"; + /// The variable a lane sets on a pull request and in the merge queue to + /// narrow its real-GPU scenarios to the smoke test. + pub const SMOKE_ONLY_ENV: &'static str = "E2E_GPU_SMOKE_ONLY"; + + /// Parse the two switches; each is off when unset, empty or `0` (the + /// workflows' expression yields `''` off its events) and on when `1`. + /// + /// Anything else is an error rather than a fallback, as for + /// [`HardwareMode::parse`]: a lane that wrote `true` would otherwise run + /// the full suite on a GPU runner and still go green. + pub fn parse(gpu_only: Option<&str>, smoke_only: Option<&str>) -> Result { + let flag = |name: &str, value: Option<&str>| match value.map(str::trim) { + None | Some("" | "0") => Ok(false), + Some("1") => Ok(true), + Some(other) => Err(format!( + "{name}={other:?} is not a switch; use `1` or leave it unset" + )), + }; + Ok(Self { + gpu_only: flag(Self::GPU_ONLY_ENV, gpu_only)?, + smoke_only: flag(Self::SMOKE_ONLY_ENV, smoke_only)?, + }) + } + + /// The selection this process runs with, read from [`Self::GPU_ONLY_ENV`] + /// and [`Self::SMOKE_ONLY_ENV`]. + /// + /// # Panics + /// + /// On a value [`LaneSelection::parse`] rejects, including one that is not + /// valid Unicode: a lane must never run a share of the suite it did not + /// ask for. + #[must_use] + pub fn from_env() -> Self { + Self::from_vars(|name| std::env::var_os(name)) + } + + /// [`LaneSelection::from_env`] over any source of variables, so the name + /// each switch is read from can be tested without touching the process + /// environment. + fn from_vars(read: impl Fn(&str) -> Option) -> Self { + let read = |name: &str| { + read(name).map(|value| { + value.into_string().unwrap_or_else(|raw| { + panic!( + "{name}={} is not valid Unicode, so it is not a switch; use `1` or \ + leave it unset", + raw.display() + ) + }) + }) + }; + Self::parse( + read(Self::GPU_ONLY_ENV).as_deref(), + read(Self::SMOKE_ONLY_ENV).as_deref(), + ) + .unwrap_or_else(|err| panic!("{err}")) + } +} + +/// Whether a scenario needs the real GPU of the machine it runs on — a usable +/// device (`@requires-gpu` or `@requires-real-gpu`, see +/// [`ScenarioDecl::needs_gpu`]), or a detected chip name — as opposed to +/// running anywhere, simulated machines included. +/// +/// Built on `needs_gpu` rather than `requires_gpu`, so a declaration carrying +/// only `requires_real_gpu` still counts: left out, it would be skipped by +/// every GPU lane and run on none. +const fn needs_real_gpu(decl: &ScenarioDecl) -> bool { + decl.needs_gpu() || decl.requires_gfx_target +} + +/// Narrow a scenario's resolution to what this lane runs (see [`LaneSelection`]). +/// +/// A scenario outside the lane's share resolves to `Skip` with the reason +/// recorded, so the report shows it as not applicable on that lane rather than +/// as a result that never arrived. An expectation that already resolved to +/// `Skip` keeps its own, more precise reason. +#[must_use] +pub fn restrict_to_lane( + decl: &ScenarioDecl, + expectation: Expectation, + lane: LaneSelection, +) -> Expectation { + if matches!(expectation, Expectation::Skip { .. }) { + return expectation; + } + let real_gpu = needs_real_gpu(decl); + if lane.gpu_only && !real_gpu { + return Expectation::Skip { + reason: "this GPU lane runs only scenarios that need its GPU (E2E_GPU_ONLY); \ + the rest run on the GitHub-hosted Linux and Windows lanes" + .to_owned(), + }; + } + if lane.smoke_only && real_gpu && !decl.gpu_smoke && !decl.merge_queue { + return Expectation::Skip { + reason: "on a pull request and in the merge queue this lane runs only the \ + @gpu-smoke canaries (and, in the queue, the @merge-queue serves) of \ + the scenarios that need a GPU (E2E_GPU_SMOKE_ONLY); other events run \ + the rest" + .to_owned(), + }; + } + expectation +} + /// Tiny glob: `*` matches any run of chars. Used only for `therock_family`. fn glob_match(pattern: &str, text: &str) -> bool { let pattern = pattern.to_ascii_lowercase(); @@ -2785,4 +2919,180 @@ flaky = true } } } + + #[test] + fn a_gpu_lane_runs_only_its_gpu_scenarios_narrowed_to_the_smoke_test_on_prs() { + let xfail = || Expectation::ExpectXfail { + bug: "B-1".to_owned(), + reason: "known".to_owned(), + flaky: false, + }; + let canary = decl(&["id:serve-vllm", "requires-real-gpu", "gpu-smoke"]); + let queue_serve = decl(&["id:serve-default", "requires-real-gpu", "merge-queue"]); + let heavy = decl(&["id:runtime-install", "requires-real-gpu"]); + let gfx = decl(&["id:runtime-resolve", "requires-gfx-target"]); + let no_gpu = decl(&["id:examine-version"]); + let simulated = decl(&["id:examine-detects-wsl", "requires-os:linux"]); + let skip_reason = |e: Expectation| match e { + Expectation::Skip { reason } => reason, + other => panic!("expected skip, got {other:?}"), + }; + + let gpu_lane = LaneSelection { + gpu_only: true, + smoke_only: false, + }; + // A GPU lane leaves everything that needs no GPU, simulated machines + // included, to the hosted lanes — and says so. + for d in [&no_gpu, &simulated] { + assert!( + skip_reason(restrict_to_lane(d, Expectation::ExpectPass, gpu_lane)) + .contains("E2E_GPU_ONLY") + ); + } + // It keeps every scenario that needs its hardware, resolution intact. + for d in [&canary, &queue_serve, &heavy, &gfx] { + assert_eq!(restrict_to_lane(d, xfail(), gpu_lane), xfail()); + } + + let pr_gpu_lane = LaneSelection { + gpu_only: true, + smoke_only: true, + }; + // On a pull request only the canaries and the queue serves remain. + assert_eq!(restrict_to_lane(&canary, xfail(), pr_gpu_lane), xfail()); + assert_eq!( + restrict_to_lane(&queue_serve, Expectation::ExpectPass, pr_gpu_lane), + Expectation::ExpectPass + ); + for d in [&heavy, &gfx] { + assert!( + skip_reason(restrict_to_lane(d, xfail(), pr_gpu_lane)) + .contains("E2E_GPU_SMOKE_ONLY") + ); + } + + // A lane that sets only the smoke switch (the WSL lane, whose GPU + // passthrough is advisory) keeps its non-GPU scenarios. + let smoke_only = LaneSelection { + gpu_only: false, + smoke_only: true, + }; + assert_eq!( + restrict_to_lane(&no_gpu, Expectation::ExpectPass, smoke_only), + Expectation::ExpectPass + ); + // A skip that already happened keeps its own, more precise reason. + let skip = Expectation::Skip { + reason: "requires os 'linux'".to_owned(), + }; + assert_eq!(restrict_to_lane(&no_gpu, skip.clone(), pr_gpu_lane), skip); + // A lane with no selection changes nothing. + assert_eq!( + restrict_to_lane(&heavy, Expectation::ExpectPass, LaneSelection::default()), + Expectation::ExpectPass + ); + } + + /// The switches accept exactly what the workflows write — `1` on, `''` off + /// their events — and refuse anything else instead of ignoring it. + #[test] + fn lane_selection_reads_its_switches_strictly() { + assert_eq!( + LaneSelection::parse(None, None), + Ok(LaneSelection::default()) + ); + assert_eq!( + LaneSelection::parse(Some("1"), Some("")), + Ok(LaneSelection { + gpu_only: true, + smoke_only: false, + }) + ); + assert_eq!( + LaneSelection::parse(Some("0"), Some("1")), + Ok(LaneSelection { + gpu_only: false, + smoke_only: true, + }) + ); + let err = LaneSelection::parse(Some("true"), None).expect_err("`true` is not a switch"); + assert!(err.contains("E2E_GPU_ONLY"), "{err}"); + let err = LaneSelection::parse(None, Some("yes")).expect_err("`yes` is not a switch"); + assert!(err.contains("E2E_GPU_SMOKE_ONLY"), "{err}"); + // A valid first switch does not hide a bad second one. + let err = LaneSelection::parse(Some("1"), Some("on")).expect_err("`on` is not a switch"); + assert!(err.contains("E2E_GPU_SMOKE_ONLY"), "{err}"); + // Surrounding whitespace from a YAML value is not an error. + assert_eq!( + LaneSelection::parse(Some(" 1 "), Some(" ")), + Ok(LaneSelection { + gpu_only: true, + smoke_only: false, + }) + ); + } + + /// Each switch is read from its own variable: a swapped or misspelt name + /// would leave a lane running a share of the suite it did not set. + #[test] + fn lane_selection_reads_each_switch_from_its_own_variable() { + let vars = |gpu_only: Option<&'static str>, smoke_only: Option<&'static str>| { + move |name: &str| match name { + "E2E_GPU_ONLY" => gpu_only.map(std::ffi::OsString::from), + "E2E_GPU_SMOKE_ONLY" => smoke_only.map(std::ffi::OsString::from), + other => panic!("read an unexpected variable {other}"), + } + }; + assert_eq!( + LaneSelection::from_vars(vars(Some("1"), None)), + LaneSelection { + gpu_only: true, + smoke_only: false, + } + ); + assert_eq!( + LaneSelection::from_vars(vars(None, Some("1"))), + LaneSelection { + gpu_only: false, + smoke_only: true, + } + ); + } + + #[test] + #[should_panic(expected = "E2E_GPU_ONLY=\"true\" is not a switch")] + fn lane_selection_aborts_on_a_switch_it_does_not_recognise() { + let _ = LaneSelection::from_vars(|name| { + (name == "E2E_GPU_ONLY").then(|| std::ffi::OsString::from("true")) + }); + } + + #[cfg(unix)] + #[test] + #[should_panic(expected = "is not valid Unicode")] + fn lane_selection_aborts_on_a_switch_that_is_not_unicode() { + use std::os::unix::ffi::OsStringExt as _; + let _ = LaneSelection::from_vars(|name| { + (name == "E2E_GPU_SMOKE_ONLY").then(|| std::ffi::OsString::from_vec(vec![0x31, 0xff])) + }); + } + + /// Like the GPU gate, the lane split reads the implication itself rather + /// than trusting `from_tags` to have set `requires_gpu` too: a declaration + /// carrying only `requires_real_gpu` would otherwise be skipped by every + /// GPU lane as needing no GPU, and run on no lane at all. + #[test] + fn requires_real_gpu_alone_stays_on_a_gpu_lane() { + let mut d = decl(&["id:x", "requires-real-gpu"]); + d.requires_gpu = false; + let gpu_lane = LaneSelection { + gpu_only: true, + smoke_only: false, + }; + assert_eq!( + restrict_to_lane(&d, Expectation::ExpectPass, gpu_lane), + Expectation::ExpectPass + ); + } } diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 49ca40887..027be5d24 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1643,6 +1643,12 @@ async fn run_suite() { // per-engine canary covers them on the PR fast path); set by // e2e-selfhosted.yml on the `merge_group` event. let include_merge_queue = std::env::var_os("E2E_MERGE_QUEUE").is_some_and(|v| v == "1"); + // Which share of the suite this lane runs: a self-hosted GPU lane runs only + // what needs its GPU, narrowed to the `@gpu-smoke` canaries on a pull + // request; see `restrict_to_lane`. Selected inside the filter, like + // `E2E_ONLY_LIFECYCLE`, so expectation resolution still runs. + // An unrecognised value aborts the run rather than running the wrong share. + let lane = e2e_cucumber::expectation::LaneSelection::from_env(); // `@requires-real-gpu` scenarios run only where a lane on real hardware opts // in; everywhere else GPU behaviour is exercised against a simulated host. // An unrecognised value aborts the run instead of silently skipping them. @@ -1768,6 +1774,8 @@ async fn run_suite() { hardware, }, ); + let expectation = + e2e_cucumber::expectation::restrict_to_lane(&decl, expectation, lane); let run = (!only_lifecycle || lifecycle_lane_includes(&decl, &also_run_ids)) && !matches!(expectation, Expectation::Skip { .. }); if let Some(id) = &decl.id { diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index e68f209ec..77c995ff2 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1228,8 +1228,10 @@ async fn setup_background_model(world: &mut E2eWorld) { // never assert on real generation — only that a served model answers. So it // doesn't need a real GPU serve: on a real-hardware run with an AMD GPU we // still exercise the real `rocm serve --managed` path (extra real coverage - // where hardware exists), but everywhere else — a no-GPU host, or a simulated - // run on a GPU box — we back it with the in-process MockServer + a planted + // on the nightly GPU lanes, and on `e2e-wsl` when its GPU probe passes; the + // per-PR and push GPU lanes set E2E_GPU_ONLY and skip these untagged + // scenarios), but everywhere else — a no-GPU host, or a simulated run on a + // GPU box — we back it with the in-process MockServer + a planted // managed-service record. This lets the scenarios run on the GitHub-hosted // mock lane every PR, with no coverage loss. Real inference stays covered by // the `@requires-real-gpu` serve-*-inference scenarios. diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 63fa727cc..0bbac5a5c 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -201,6 +201,7 @@ mod tests { /// entries. const CANONICAL_REPORT_ARTIFACTS: &[&str] = &[ "e2e-report", + "e2e-windows-report", "e2e-gpu-report", "e2e-gpu-n-1-report", "e2e-gpu-n-2-report", @@ -370,6 +371,7 @@ mod tests { ("strix-halo-windows", "e2e-gpu-strix-windows-report"), ("strix-halo-wsl", "e2e-gpu-strix-wsl-report"), ("mock", "e2e-report"), + ("mock-windows", "e2e-windows-report"), ] { let tmp = tempfile::tempdir().expect("tempdir"); let root = tmp.path(); diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index dfd3b2dbc..66ce762f4 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1556,24 +1556,127 @@ jobs: ); } - #[test] - fn self_hosted_wsl_enables_merge_queue_scenarios_only_for_merge_group() { + /// On a pull request and in the merge queue every self-hosted lane that runs + /// there narrows its real-GPU scenarios to the `@gpu-smoke` canaries, while + /// the heavy `@merge-queue` serves stay opted in for the merge queue alone. + /// Pushes, dispatches and nightly set neither, so no narrowing applies there. + #[test] + fn self_hosted_gpu_lanes_run_only_the_gpu_smoke_test_on_prs_and_in_the_merge_queue() { + const SMOKE_ONLY: &str = "${{ (github.event_name == 'merge_group' || github.event_name == 'pull_request') && '1' || '' }}"; + const QUEUE_ONLY: &str = "${{ github.event_name == 'merge_group' && '1' || '' }}"; let sh = read_workflow("e2e-selfhosted.yml"); - let wsl = job_block(&sh, "e2e-wsl"); - let env = job_mapping(wsl, "env"); - assert_eq!( - env.get("E2E_MERGE_QUEUE").map(String::as_str), - Some("${{ github.event_name == 'merge_group' && '1' || '' }}"), - "e2e-wsl's job-level env must opt into @merge-queue scenarios for merge_group only" - ); + // (lane, runs on pull_request, runs on merge_group). MI300X is skipped in + // the merge queue (`mi300x_lane_is_skipped_in_the_merge_queue`); Strix + // Ubuntu runs on neither event, only on push, dispatch and nightly. + // There is no W7900 lane in this workflow any more. The gate below is + // checked against each job's own `if:` so this table cannot drift from + // it; the gates themselves are pinned by + // `mi300x_lane_is_skipped_in_the_merge_queue` (in full for Strix + // Ubuntu). + for (lane, on_pr, in_queue) in [ + ("e2e-gpu", true, false), + ("e2e-gpu-strix-ubuntu", false, false), + ("e2e-gpu-strix-windows", true, true), + ("e2e-wsl", true, true), + ] { + let block = job_block(&sh, lane); + let gate = { + let start = block + .find(" if: >-\n") + .unwrap_or_else(|| panic!("`{lane}` has a folded `if:` gate")); + normalized_whitespace( + &block[start..] + .lines() + .skip(1) + .take_while(|line| indent_of(line) > 4) + .map(strip_comment) + .collect::>() + .join(" "), + ) + }; + assert_eq!( + !gate.contains("github.event_name != 'pull_request'"), + on_pr, + "{lane}: the table says it runs on pull_request={on_pr}, its `if:` disagrees" + ); + assert_eq!( + !gate.contains("github.event_name != 'merge_group'"), + in_queue, + "{lane}: the table says it runs on merge_group={in_queue}, its `if:` disagrees" + ); + let env = job_mapping(block, "env"); + // A lane that never runs on either event would only ever see the + // smoke switch unset; carrying it there reads as if it did. + assert_eq!( + env.get("E2E_GPU_SMOKE_ONLY").map(String::as_str), + (on_pr || in_queue).then_some(SMOKE_ONLY), + "{lane} must narrow its real-GPU scenarios to the smoke test on pull \ + requests and in the merge queue, and nowhere else" + ); + assert_eq!( + env.get("E2E_MERGE_QUEUE").map(String::as_str), + in_queue.then_some(QUEUE_ONLY), + "{lane} must opt into the @merge-queue serves for merge_group only" + ); + // A GPU lane leaves everything that needs no GPU to the hosted Linux + // and Windows lanes. The WSL lane's GPU access is advisory and + // probed per run, so it is the one self-hosted lane that must keep + // running those. + let gpu_only = env.get("E2E_GPU_ONLY").map(String::as_str); + if lane == "e2e-wsl" { + assert_eq!( + gpu_only, None, + "e2e-wsl's GPU is advisory, so it must keep running the non-GPU suite" + ); + } else { + assert_eq!( + gpu_only, + Some("1"), + "{lane} must run only its GPU scenarios" + ); + } + } let nightly = read_workflow("nightly.yml"); let nightly_wsl = job_block(&nightly, "e2e-wsl-nightly"); let nightly_env = job_mapping(nightly_wsl, "env"); - assert!( - !nightly_env.contains_key("E2E_MERGE_QUEUE"), - "the nightly WSL job cannot receive merge_group events and must not opt into @merge-queue scenarios" - ); + for key in ["E2E_MERGE_QUEUE", "E2E_GPU_SMOKE_ONLY"] { + assert!( + !nightly_env.contains_key(key), + "the nightly WSL job takes no narrowing switch and must not set {key}" + ); + } + } + + /// The WSL lane runs the suite inside the distro, which sees only the + /// variables WSLENV names. One left out there is silently unset: dropping + /// `E2E_GPU_SMOKE_ONLY` would run the full GPU suite on every pull request, + /// dropping `E2E_HARDWARE` would skip every real-GPU scenario, and neither + /// would ever turn the lane red. + #[test] + fn wsl_lanes_forward_every_e2e_variable_into_the_distro() { + for (workflow, job) in [ + ("e2e-selfhosted.yml", "e2e-wsl"), + ("nightly.yml", "e2e-wsl-nightly"), + ] { + let text = read_workflow(workflow); + let block = job_block(&text, job); + let forwarded: Vec<&str> = block + .lines() + .find_map(|line| line.trim().strip_prefix("$env:WSLENV = '")) + .and_then(|rest| rest.strip_suffix('\'')) + .unwrap_or_else(|| panic!("{workflow} job {job} sets no WSLENV")) + .split(':') + .collect(); + for key in job_mapping(block, "env").keys() { + if key.starts_with("E2E_") { + assert!( + forwarded.contains(&key.as_str()), + "{workflow} job {job} sets {key} but does not forward it through WSLENV" + ); + } + } + } } #[test] @@ -5402,10 +5505,11 @@ fi "the E2E README must name every nightly self-hosted lane, in workflow order" ); - // The README's CI job table is the per-PR view: the blocking mock job - // from ci.yml, then one row per self-hosted lane. Only the self-hosted - // rows are derivable, so the mock row is matched by workflow and the - // rest are compared against e2e-selfhosted.yml. + // The README's CI job table is the per-PR view: the two GitHub-hosted + // jobs from ci.yml (the blocking Linux mock job and the Windows one), + // then one row per self-hosted lane. Only the self-hosted rows are + // derivable, so the hosted rows are matched by workflow and the rest + // are compared against e2e-selfhosted.yml. let readme_rows = markdown_table_rows(&readme, "| Job | Workflow | Platform | Blocking |"); let (mock_rows, self_hosted_rows): (Vec<_>, Vec<_>) = readme_rows .into_iter() @@ -5415,8 +5519,8 @@ fi .iter() .map(|row| row[0].clone()) .collect::>(), - vec!["`e2e`".to_owned()], - "the README CI job table must carry exactly one blocking mock row" + vec!["`e2e`".to_owned(), "`e2e-windows`".to_owned()], + "the README CI job table must carry exactly the two hosted E2E rows" ); assert_eq!( self_hosted_rows @@ -6043,8 +6147,10 @@ fi /// Jobs that are NOT part of the Windows build-and-test lane even though they /// are not on a Linux runner: each is a check of its own, so the aggregator - /// does not (and must not have to) wait on it. - const JOBS_OUTSIDE_THE_WINDOWS_LANE: &[&str] = &["powershell-lint"]; + /// does not (and must not have to) wait on it. `e2e-windows` (`E2E tests + /// (Windows)`) is advisory until it has a green record, and is then made a + /// required check under its own name, not through the aggregator. + const JOBS_OUTSIDE_THE_WINDOWS_LANE: &[&str] = &["powershell-lint", "e2e-windows"]; /// Every job in the Windows lane, other than the `windows-build-and-test` /// aggregator: any job not shown to run on `ubuntu-*` runners only, bar the