From 99c6d7d8f1dc211fffb184ea6633e152962813ca Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 12 Nov 2026 11:25:10 +0000 Subject: [PATCH 1/9] ci: retire the self-hosted nightly lanes that never get a runner The R9700, MI350P and W7900 nightly jobs have sat queued for hours and been cancelled since at least 11-09. Remove them from nightly.yml and from the consolidated report that waited on them. These lanes now run on the queue. The MI300X nightly and the Strix Halo nightlies (Ubuntu, Windows, WSL2, all on the Strix pool since #1018) are unchanged. Update the workflow contract tests and the docs to match. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 257 --------------------------------- docs/ci-hardware-testing.md | 34 ++--- docs/nightly-on-the-queue.md | 19 ++- tests/e2e-cucumber/README.md | 18 +-- xtask/src/e2e_report.rs | 15 +- xtask/src/workflow_contract.rs | 19 +-- 6 files changed, 57 insertions(+), 305 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5399e19e9..fc58216ac 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -506,257 +506,6 @@ jobs: name: e2e-gpu-${{ matrix.channel }}-report path: tests/e2e-cucumber/results/ - # rad3 R9700 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-rad3 - # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, shared uv - # cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step: - # the report crate asserts the nightly lanes publish the same platforms as the - # per-PR lanes. - e2e-gpu-nightly-rad3: - name: E2E tests (rad3 R9700, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, r9700] - continue-on-error: true - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. The floor stays at the shared 16 GiB default, a flat - # constant rather than a proportion of capacity — high enough to catch a - # leftover serve, low enough that a clean pod passes on the first poll; do - # NOT scale it down for a smaller part. On a 32GB card it still only trips - # past 16 GiB held, so the floor alone would miss a leftover serve holding - # less than that; the process gate is what catches it (#446). Same as the - # per-PR rad3 lane. See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on rad3 (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on the runner-work PVC, which survives the pod - # being scaled away, so the cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm` (EAI-8057). - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-rad3-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - - e2e-gpu-nightly-mi350p: - name: E2E tests (MI350P, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, mi350p] - continue-on-error: true - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. Floor matches the per-PR MI350P lane and the other - # Instinct lane — low enough that a clean pod passes on the first poll, - # and against a pool this size it only trips near exhaustion, so the - # process gate is what actually catches a leftover serve (#446). See - # .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on MI350P (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on a volume that survives the runner being scaled - # away, so the cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-mi350p-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - - # W7900 counterpart to e2e-gpu-nightly. Mirrors the e2e-gpu-w7900 lane (which - # no longer runs on pull_request) in e2e-selfhosted.yml (w7900 label, shared - # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in - # step: the report crate asserts the nightly lanes publish the same platforms - # as the lanes in e2e-selfhosted.yml. - e2e-gpu-nightly-w7900: - name: E2E tests (W7900, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, w7900] - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - continue-on-error: true - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. The floor stays at the shared 16 GiB default, which is - # a flat constant rather than a proportion of capacity: it only has to be - # high enough to catch a leftover serve still holding the card, and low - # enough that a clean host passes on the first poll. Same as the per-PR - # W7900 lane — do NOT scale it down for a smaller part. On a 48GB card it - # only trips past ~32 GiB held, so the process gate is what catches a - # leftover serve holding less (#446). See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on W7900 (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the - # cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the # runner's storage and runtime setup aligned with e2e-selfhosted.yml's proven @@ -1596,9 +1345,6 @@ jobs: actions: read needs: - e2e-gpu-nightly - - e2e-gpu-nightly-rad3 - - e2e-gpu-nightly-mi350p - - e2e-gpu-nightly-w7900 - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows - e2e-wsl-nightly @@ -1667,9 +1413,6 @@ jobs: NEEDS: ${{ toJSON(needs) }} LANE_REPORTS: | e2e-gpu-nightly=e2e-gpu-release-report,e2e-gpu-nightly-report - e2e-gpu-nightly-rad3=e2e-gpu-rad3-release-report,e2e-gpu-rad3-nightly-report - e2e-gpu-nightly-mi350p=e2e-gpu-mi350p-release-report,e2e-gpu-mi350p-nightly-report - e2e-gpu-nightly-w7900=e2e-gpu-w7900-release-report,e2e-gpu-w7900-nightly-report e2e-gpu-nightly-strix=e2e-gpu-strix-ubuntu-release-report,e2e-gpu-strix-ubuntu-nightly-report e2e-gpu-nightly-strix-windows=e2e-gpu-strix-windows-release-report,e2e-gpu-strix-windows-nightly-report e2e-wsl-nightly=e2e-gpu-strix-wsl-release-report,e2e-gpu-strix-wsl-nightly-report diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 46c9c47ad..fe041fbd5 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -37,28 +37,30 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| | Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | -| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | +| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | MI300X (`e2e-gpu-nightly`), Strix Halo Ubuntu (`e2e-gpu-nightly-strix`), Windows (`e2e-gpu-nightly-strix-windows`) and WSL2 (`e2e-wsl-nightly`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | | Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` but run on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | -R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still -covered, but off every PR's critical path, so they moved to the nightly-only -rung instead of being removed outright. +R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125) and moved to +the nightly-only rung. Their self-hosted nightly jobs, and W7900's, have since +been removed from `nightly.yml`: they never got a runner. The queue (below) is +the scheduled run of R9700 (`r9700s`), MI350P (`mi350p`) and W7900 +(`w7900-dual`). The decommissioning of the runners themselves is separate. A pull request now runs on exactly one Linux GPU node and one Windows GPU pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer start on `pull_request` or in the merge queue; they keep their -`push`/dispatch triggers and run nightly (`e2e-gpu-nightly-strix` on the same -DevLab Dispatch pool labels, `e2e-gpu-nightly-w7900`). +`push`/dispatch triggers; Strix Ubuntu also runs nightly (`e2e-gpu-nightly-strix` +on the same DevLab Dispatch pool labels). [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length of a job, and publishes the result; [`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) publishes a finished run again. They cover the Linux GPU nightly on both the release and -the nightly channel; native Windows and WSL stay on their dedicated runners, -since the queue runs Linux pods. See +the nightly channel; the Strix Halo lanes stay on the Strix pool, since the +queue has no Strix lane. See [nightly-on-the-queue.md](nightly-on-the-queue.md). ## Nightly stream canary @@ -253,9 +255,9 @@ covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers its self-hosted GPU platforms (a pull request runs only the MI300X and Windows Strix Halo pool lanes among them); `nightly.yml`'s `e2e-report-nightly` covers a strict superset of those -platforms (it also runs `e2e-gpu-rad3` and `e2e-gpu-mi350p`, demoted from -per-PR to nightly-only per ROCMAI-125) with the `@nightly` scenarios -included. Each joins its platforms' +platforms (it adds MI300X's `e2e-gpu-nightly` and the Strix Halo Ubuntu, +Windows and WSL2 nightly lanes) with the `@nightly` scenarios included. The R9700, MI350P and +W7900 nightlies run on the queue and are not in this report. Each joins its platforms' reports — including partial or failed runs — by scenario id into one HTML report and GitHub step summary. Since ROCMAI-429 the report keys each column by `(platform_slug, channel)` rather than platform alone, so `nightly.yml`'s @@ -323,8 +325,7 @@ unlike `publish-e2e-wiki`, it still runs (and publishes an explicit outright, instead of leaving the wiki showing a stale verdict with no sign it's stale. -The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, -`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`, `e2e-gpu-rad3-report`, +The lane artifacts are named canonically (`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`, `e2e-gpu-report`, `e2e-gpu-strix-ubuntu-n-1-report`, `e2e-gpu-strix-ubuntu-n-2-report`, `e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, @@ -335,9 +336,7 @@ The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `rad3`/`mi350p` are nightly-only -lanes (ROCMAI-125) and don't run that matrix, so they have no `-n-1`/`-n-2` -entries. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is @@ -438,8 +437,7 @@ combinations on that lane: the `nightly` channel is excluded from `n-1`/`n-2`, because it only carries the latest (pre-)release builds and cannot resolve the pinned released versions (`rocm install sdk` fails with "no mutually compatible TheRock ... versions"); those versions are covered by the `release` channel. -`rad3`/`mi350p` are nightly-only (above) and don't run this matrix -at all. + `serve` is narrower than `heavy`: it matches only paths that can change serve *behaviour* or the GPU E2E harness (the engines, the diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index bae5d09d9..8917a6a12 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -22,10 +22,10 @@ GPU jobs of [`nightly.yml`](../.github/workflows/nightly.yml) do (see way those jobs' results are: the queue runs on the same infrastructure and under the same access as the self-hosted runners, and neither signs what it reports. -**The remaining gap is native Windows and WSL.** The queue runs Linux pods, so -the native Windows and WSL lanes stay on their dedicated runners, in `nightly.yml` -and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); nothing -here replaces them. Which checks are required before merging is a repository +**The remaining gap is Strix Halo.** The queue has no Strix lane and runs Linux +pods, so the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool, +in `nightly.yml` and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); +nothing here replaces them. Which checks are required before merging is a repository setting, outside this page. This page is the **contract** between this repository and the queue side: what @@ -123,10 +123,13 @@ pre-warm's package downloads and the suite's model downloads are left to the tools doing them, so a model that downloads more slowly than its scenario waits fails that scenario. -The queue's results are an additional signal, not a replacement: the lanes -write only into the queue artifact, and the `e2e-gpu--report` -artifacts that `nightly.yml`'s consolidated report reads are not produced from -them. +The queue is the only scheduled run of the R9700, MI350P and W7900 lanes: their +self-hosted nightly jobs were removed from `nightly.yml`. The MI300X nightly +job is still there, and the queue's `mi300x` lane is an additional signal to it. +The lanes write only into the queue artifact, and the +`e2e-gpu--report` artifacts that `nightly.yml`'s consolidated report +reads are not produced from them, so the consolidated report and the support +matrix no longer have rows for R9700, MI350P or W7900 Linux. ## Output diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index f24f136c6..7be740aa5 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -195,18 +195,16 @@ Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to skip there. -The nightly workflow covers a strict superset of the hardware in the table -above, as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-rad3`, -`e2e-gpu-nightly-mi350p`, `e2e-gpu-nightly-w7900`, `e2e-gpu-nightly-strix`, +The nightly workflow runs the MI300X and the Strix Halo hardware from the +table above as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-strix`, `e2e-gpu-nightly-strix-windows`, `e2e-wsl-nightly`) with `E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform -grid. `e2e-gpu-nightly-rad3` and `e2e-gpu-nightly-mi350p` have no per-PR -counterpart — both were demoted from per-PR to nightly-only per ROCMAI-125 — -so nightly is where their regression coverage now lives. The shared -large-model scenario follows each lane's serve engine: it serves -`Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), and the -`unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified on -Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). +grid. R9700, MI350P and W7900 no longer have a self-hosted nightly lane in +`nightly.yml`: they run on the queue ([nightly-on-the-queue.md](../../docs/nightly-on-the-queue.md)), +whose results are not in that grid. The shared large-model scenario follows each lane's serve engine: it +serves `Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), +and the `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified +on Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). Use the self-hosted E2E workflow dispatch to run either model independently on a ref (the GPU platform / `include_nightly` / `name_filter` inputs live on diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 720fd4405..d17b9bf1e 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -274,6 +274,10 @@ mod tests { #[test] fn the_nightly_lanes_publish_the_same_platforms_as_the_per_pr_lanes() { + // W7900's self-hosted nightly job was removed because it never got a + // runner; its nightly runs on the queue, which does not report into this + // grid. Its per-PR-workflow lane may still exist. + const NO_SELF_HOSTED_NIGHTLY: [&str; 1] = ["e2e-gpu-w7900-report"]; // The nightly workflow exists to run *more* scenarios on *at least* // the per-PR platforms, plus lanes the per-PR smoke gate deliberately // skips (e.g. rad3, mi350p -- ROCMAI-125). If a per-PR platform ever @@ -325,9 +329,18 @@ mod tests { "per-PR lane scan found nothing to check" ); assert!( - per_pr.iter().all(|p| nightly.contains(p)), + per_pr + .iter() + .filter(|p| !NO_SELF_HOSTED_NIGHTLY.contains(&p.as_str())) + .all(|p| nightly.contains(p)), "every per-PR platform must also run nightly: per_pr={per_pr:?} nightly={nightly:?}" ); + assert!( + NO_SELF_HOSTED_NIGHTLY + .iter() + .all(|p| !nightly.iter().any(|q| q == p)), + "NO_SELF_HOSTED_NIGHTLY names a lane that runs nightly again: drop it from the list" + ); } #[test] diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 87a407ca5..26a4bdb8e 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2872,8 +2872,8 @@ jobs: } } assert_eq!( - delegating, 8, - "expected the eight native Linux preflight lanes to delegate to \ + delegating, 5, + "expected the five native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2941,8 +2941,8 @@ jobs: } assert_eq!( bodies.len(), - 8, - "expected eight native Linux preflight bodies to compare, found {} — \ + 5, + "expected five native Linux preflight bodies to compare, found {} — \ if a lane was added or removed, update this count", bodies.len() ); @@ -4301,8 +4301,8 @@ fi } } assert_eq!( - delegating, 8, - "expected the 8 native Linux preflight lanes to delegate to \ + delegating, 5, + "expected the 5 native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); assert_eq!( @@ -4852,11 +4852,8 @@ fi } for lane in [ "e2e-gpu-nightly", - "e2e-gpu-nightly-rad3", - "e2e-gpu-nightly-mi350p", "e2e-gpu-nightly-strix", "e2e-gpu-nightly-strix-windows", - "e2e-gpu-nightly-w7900", "e2e-wsl-nightly", ] { assert!( @@ -9659,8 +9656,8 @@ permissions: let flat = doc.split_whitespace().collect::>().join(" "); for needed in [ "both the `release` and the `nightly` channel", - "The remaining gap is native Windows and WSL.", - "the native Windows and WSL lanes stay on their dedicated runners", + "The remaining gap is Strix Halo.", + "the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool", "`E2E_CHANNELS`", "`/out//results`", ] { From 036287efbd2d16d5fd455238bdcdfeb5107b6f06 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 16:19:04 +0200 Subject: [PATCH 2/9] docs,test: take main's wording where #1066 and this change touch the same lines Preparation for merging main into this branch without conflicts; the next commits restore this change's own wording on top of the merge. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 21 +++++++++++++-------- docs/nightly-on-the-queue.md | 14 ++++++++++---- xtask/src/workflow_contract.rs | 12 ++++++------ 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index fe041fbd5..eb982db7b 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -36,9 +36,9 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| -| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | -| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | MI300X (`e2e-gpu-nightly`), Strix Halo Ubuntu (`e2e-gpu-nightly-strix`), Windows (`e2e-gpu-nightly-strix-windows`) and WSL2 (`e2e-wsl-nightly`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | -| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu` and `e2e-gpu-w7900`, which skip `pull_request` and `merge_group` but run on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | +| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | +| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | +| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125) and moved to the nightly-only rung. Their self-hosted nightly jobs, and W7900's, have since @@ -49,10 +49,12 @@ the scheduled run of R9700 (`r9700s`), MI350P (`mi350p`) and W7900 A pull request now runs on exactly one Linux GPU node and one Windows GPU pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo -Ubuntu (`e2e-gpu-strix-ubuntu`) and W7900 (`e2e-gpu-w7900`) lanes no longer -start on `pull_request` or in the merge queue; they keep their -`push`/dispatch triggers; Strix Ubuntu also runs nightly (`e2e-gpu-nightly-strix` -on the same DevLab Dispatch pool labels). +Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in +the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane +in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue +nightly's `w7900-dual` lane (below; advisory), not on `push`, `release/**` or +dispatch. Its first run, the 11-13 queue nightly +([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), got capacity at once and finished `Done`, exit 0. [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length @@ -336,7 +338,10 @@ The lane artifacts are named canonically (`e2e-gpu-n-1-report`, `e2e-gpu-n-2-rep `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900` +are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any +more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they +don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index 8917a6a12..052ffba52 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -22,10 +22,16 @@ GPU jobs of [`nightly.yml`](../.github/workflows/nightly.yml) do (see way those jobs' results are: the queue runs on the same infrastructure and under the same access as the self-hosted runners, and neither signs what it reports. -**The remaining gap is Strix Halo.** The queue has no Strix lane and runs Linux -pods, so the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool, -in `nightly.yml` and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); -nothing here replaces them. Which checks are required before merging is a repository +The lane set is whatever GPU models the queue has. The 11-13 run +([43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)) discovered `mi300x`, `mi325x`, `mi350p`, `mi350x`, +`mi355x`, `r9700s` and `w7900-dual`; `w7900-dual` (W7900) is advisory, and it is +the only scheduled coverage of that hardware, since `e2e-selfhosted.yml` no +longer has a W7900 lane. + +**The remaining gap is native Windows and WSL.** The queue runs Linux pods, so +the native Windows and WSL lanes stay on their dedicated runners, in `nightly.yml` +and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); nothing +here replaces them. Which checks are required before merging is a repository setting, outside this page. This page is the **contract** between this repository and the queue side: what diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 26a4bdb8e..435e3f851 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2872,8 +2872,8 @@ jobs: } } assert_eq!( - delegating, 5, - "expected the five native Linux preflight lanes to delegate to \ + delegating, 7, + "expected the seven native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2941,8 +2941,8 @@ jobs: } assert_eq!( bodies.len(), - 5, - "expected five native Linux preflight bodies to compare, found {} — \ + 7, + "expected seven native Linux preflight bodies to compare, found {} — \ if a lane was added or removed, update this count", bodies.len() ); @@ -4301,8 +4301,8 @@ fi } } assert_eq!( - delegating, 5, - "expected the 5 native Linux preflight lanes to delegate to \ + delegating, 7, + "expected the 7 native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); assert_eq!( From 92eadae700d3d5a2975b1735420727cafe24d65e Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 16:20:48 +0200 Subject: [PATCH 3/9] feat(xtask): add e2e-queue-import to turn a queue run into nightly report inputs Reads a finished run of the shared GPU queue (its per-lane results artifact and the run's job log) and writes each lane's report files where `cargo xtask e2e-report` reads them, named for the lane and the queue's gate, plus a status section for the top of the support matrix. Wired in by the following commits. Signed-off-by: Tomas Saaristola --- xtask/src/e2e_queue_import.rs | 931 ++++++++++++++++++++++++++++++++++ 1 file changed, 931 insertions(+) create mode 100644 xtask/src/e2e_queue_import.rs diff --git a/xtask/src/e2e_queue_import.rs b/xtask/src/e2e_queue_import.rs new file mode 100644 index 000000000..a5f0bc459 --- /dev/null +++ b/xtask/src/e2e_queue_import.rs @@ -0,0 +1,931 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Bring a finished run of the shared GPU queue into the nightly's consolidated +//! E2E report, so the support matrix and the release-readiness verdict derived +//! from it keep covering the GPUs the queue tests. +//! +//! `nightly.yml`'s `e2e-report-nightly` merges one `report.json` per +//! `e2e-*-report` artifact. The queue publishes a different layout: one +//! directory per lane in its `queue-run-results` artifact, holding +//! `/results/{report.json,platform.json,commands.jsonl,...}` as +//! `scripts/run-e2e-lane.sh` writes them (docs/nightly-on-the-queue.md). This +//! turns the one into the other: +//! +//! * every lane's reports are copied to `/e2e-queue-[-advisory]--report/`, +//! named so `e2e_report` labels them by lane and gate. Only the three files the +//! report reads are copied; `lane.log`, the JUnit and HTML output and the +//! harness diagnostics stay where they are. +//! * the report's `platform_slug` is replaced by the lane's name and channel. The +//! slug is what the report keys a grid column by, and queue lanes of different +//! models report one slug (an MI325X lane probes as `mi300x`, an MI355X as +//! `mi350p`), which would fold them, and the self-hosted MI300X lane, into one +//! column. The channel is there because a lane's manifest names the channel of +//! the runtime it found, which need not be the one the lane was told to run. +//! * which lanes exist and which of them the queue blocks on is read from the +//! run's lane matrix in its job log, the only place either is published. A lane +//! the queue marks advisory is labelled so, and `release-readiness` leaves it +//! out of its verdict. Anything this cannot read counts as blocking. +//! * a lane that reported nothing, or that did not finish cleanly and lacks a +//! channel, gets an empty report for it, which the report shows as `EMPTY` and +//! `release-readiness` counts as a failure for a blocking lane: a lane without +//! a result is not a pass. +//! * a short status table of every lane is written for the top of the page. +//! +//! Nothing is imported unless the caller found a run: with `--absent ` +//! only the status section is written, saying so, and the nightly report goes on +//! without queue lanes. The repository is public, so everything written is either +//! copied from the queue's own scanned files or built from fields this checks +//! against a strict pattern first. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use regex::Regex; + +/// The channels `scripts/run-e2e-lane.sh` runs, in the order it runs them. +const CHANNELS: [&str; 2] = ["release", "nightly"]; + +/// The files of a lane's `/results` the consolidated report reads. +const REPORT_FILES: [&str; 3] = ["report.json", "platform.json", "commands.jsonl"]; + +#[derive(clap::Args)] +pub struct Args { + /// Directory the queue run's `queue-run-results` artifact was downloaded + /// into: one subdirectory per lane. + #[arg(long, requires = "run_url", conflicts_with = "absent")] + queue_dir: Option, + /// The queue run's job log, which prints the branch the run tested and its + /// lane matrix. Without it, or when it names a branch other than `main`, + /// nothing is imported. + #[arg(long, requires = "queue_dir")] + log: Option, + /// The queue run's URL, linked from the status section. + #[arg(long)] + run_url: Option, + /// When the queue run started, as the API reports it. + #[arg(long, requires = "queue_dir")] + run_started: Option, + /// Import nothing and say why: no queue run was found, or its results could + /// not be fetched. + #[arg( + long, + conflicts_with = "queue_dir", + required_unless_present = "queue_dir" + )] + absent: Option, + /// Where the per-lane report directories go: the directory + /// `xtask e2e-report` reads. + #[arg(long)] + artifacts_dir: PathBuf, + /// Where to write the status section, for the top of the support matrix. + #[arg(long)] + section_out: PathBuf, +} + +pub fn run(args: &Args) -> Result<()> { + let section = match (&args.queue_dir, &args.absent) { + (Some(queue_dir), _) => { + let log = args.log.as_ref().and_then(|path| { + std::fs::read_to_string(path) + .map_err(|e| eprintln!("warning: queue job log {}: {e}", path.display())) + .ok() + }); + let log = parse_log(log.as_deref()); + if log.branch.as_deref() == Some("main") { + // A log that yielded no lanes is as good as none for the lanes' + // state: every lane is then read as blocking. + let have_matrix = !log.lanes.is_empty(); + let plan = plan(&log, scan(queue_dir)?, have_matrix); + write_reports(&plan, queue_dir, &args.artifacts_dir)?; + render_section( + &plan, + args.run_url.as_deref(), + args.run_started.as_deref(), + have_matrix, + ) + } else { + // The run is of the workflow on `main`, but may have been asked + // to test another branch, which is no night's result for `main`. + // The log is the only place that says which, so without it + // nothing is imported. + render_absent( + "the newest queue run did not test main, or its job log could not be read \ + to tell", + ) + } + } + (None, Some(reason)) => render_absent(reason), + (None, None) => bail!("one of --queue-dir and --absent is required"), + }; + if let Some(parent) = args.section_out.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + std::fs::write(&args.section_out, section) + .with_context(|| format!("writing {}", args.section_out.display()))?; + Ok(()) +} + +/// What the queue run's job log says about the run. +#[derive(Debug, Default, PartialEq, Eq)] +struct QueueLog { + /// The branch the run tested and the commit every lane tested, from the + /// report's first line. The branch is not the one the workflow ran on: a run + /// started from `main` can be asked to test another branch. + branch: Option, + commit: Option, + lanes: Vec, +} + +/// One row of the lane matrix the queue prints. +#[derive(Debug, PartialEq, Eq)] +struct MatrixRow { + lane: String, + /// The lane's phase (`Done`, `Failed`, ...), when the row had the usual shape. + phase: Option, + /// The lane command's exit status, or `-` when it never ran. + exit: Option, + /// Whether the queue lets this lane fail the run. Anything but `advisory` is + /// read as blocking. + blocking: bool, +} + +impl MatrixRow { + /// A lane the queue reports as finished with exit status zero. + fn is_clean(&self) -> bool { + self.phase.as_deref() == Some("Done") && self.exit.as_deref() == Some("0") + } +} + +/// A lane name is a path segment and a word in an artifact name: lowercase +/// letters, digits and dashes only. +/// +/// A name ending in a word the artifact names use for something else +/// (`-advisory`, `-release`, `-nightly`) is refused too: it would be read back +/// as a gate or a channel. +fn is_lane_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 40 + && !name.starts_with('-') + && !["-advisory", "-release", "-nightly"] + .iter() + .any(|suffix| name.ends_with(suffix)) + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// Read the lane matrix and the tested commit out of the run's job log. +/// +/// The publisher prints its report with a `| ` prefix on every line, which the +/// runner puts a timestamp in front of. Only the fields that match a strict +/// pattern are kept; a line that does not is ignored, so a log that is not what +/// was expected yields no lanes rather than text to publish. +fn parse_log(log: Option<&str>) -> QueueLog { + let mut out = QueueLog::default(); + let Some(log) = log else { return out }; + let tested = Regex::new(r"^rocm-cli (\S+) at ([0-9a-f]{40})$").expect("static pattern"); + let mut in_matrix = false; + let mut unnamed = 0; + for line in log.lines() { + let line = line.trim_start_matches('\u{feff}'); + // ` | `: the part after the first `| `. + let Some((_, body)) = line.split_once("| ") else { + // A blank report line is `|` alone; either way the matrix is over. + if line.trim_end().ends_with('|') { + in_matrix = false; + } + continue; + }; + let body = body.trim_end(); + if let Some(caps) = tested.captures(body) { + out.branch = Some(caps[1].to_owned()); + out.commit = Some(caps[2].to_owned()); + } + let tokens: Vec<&str> = body.split_whitespace().collect(); + if tokens.first() == Some(&"LANE") && tokens.last() == Some(&"GATE") { + in_matrix = true; + continue; + } + if !in_matrix { + continue; + } + if body.is_empty() || tokens.first() == Some(&"SUMMARY:") { + in_matrix = false; + continue; + } + let (Some(lane), Some(gate)) = (tokens.first(), tokens.last()) else { + continue; + }; + if out.lanes.iter().any(|row| row.lane == *lane) { + continue; + } + // A row whose lane cannot be named safely still counts: as a lane that + // reported nothing, and blocking, since its gate cannot be trusted either. + if !is_lane_name(lane) { + unnamed += 1; + out.lanes.push(MatrixRow { + lane: format!("unnamed-{unnamed}"), + phase: None, + exit: None, + blocking: true, + }); + continue; + } + // LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE + let shaped = tokens.len() == 7; + let phase = shaped + .then(|| tokens[2]) + .filter(|p| p.bytes().all(|b| b.is_ascii_alphabetic())) + .map(str::to_owned); + let exit = shaped + .then(|| tokens[3]) + .filter(|e| *e == "-" || (e.len() <= 4 && e.bytes().all(|b| b.is_ascii_digit()))) + .map(str::to_owned); + out.lanes.push(MatrixRow { + lane: (*lane).to_owned(), + phase, + exit, + blocking: *gate != "advisory", + }); + } + out +} + +/// The channels with a `results/report.json` under each lane directory of the +/// downloaded artifact. A directory (or the final file) that is a symlink is not +/// followed. A lane directory with a name that is not a lane name is kept as an +/// `unnamed-N` lane with no reports, so that it shows as a lane without a result +/// instead of dropping out unseen. +fn scan(queue_dir: &Path) -> Result>> { + let mut lanes = BTreeMap::new(); + let mut unnamed = 0; + let entries = std::fs::read_dir(queue_dir) + .with_context(|| format!("reading the queue results in {}", queue_dir.display()))?; + for entry in entries { + let entry = entry.context("reading a queue results entry")?; + if !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let Some(lane) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if !is_lane_name(&lane) { + eprintln!("warning: a queue lane directory has an unexpected name; shown as unnamed"); + unnamed += 1; + lanes.insert(format!("unnamed-{unnamed}"), Vec::new()); + continue; + } + let channels = CHANNELS + .into_iter() + .filter(|channel| regular_file(&entry.path().join(channel).join("results/report.json"))) + .collect(); + lanes.insert(lane, channels); + } + Ok(lanes) +} + +fn regular_file(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file()) +} + +/// What one lane contributes to the report. +#[derive(Debug, PartialEq, Eq)] +struct LanePlan { + lane: String, + blocking: bool, + phase: Option, + exit: Option, + /// Channels whose reports are copied. + reported: Vec<&'static str>, + /// Channels (or, for `None`, the lane as a whole) an empty report stands in for. + placeholders: Vec>, +} + +#[derive(Debug, PartialEq, Eq)] +struct Plan { + commit: Option, + lanes: Vec, +} + +/// Decide, lane by lane, what goes into the report. `have_matrix` says whether the +/// lane matrix could be read at all: without it nothing is known of a lane's +/// state beyond its files. +fn plan(log: &QueueLog, found: BTreeMap>, have_matrix: bool) -> Plan { + let mut names: Vec<&String> = found + .keys() + .chain(log.lanes.iter().map(|r| &r.lane)) + .collect(); + names.sort(); + names.dedup(); + let mut lanes = Vec::new(); + for name in names { + let row = log.lanes.iter().find(|r| &r.lane == name); + let reported = found.get(name).cloned().unwrap_or_default(); + let clean = row.is_some_and(MatrixRow::is_clean); + let mut placeholders = Vec::new(); + if reported.is_empty() { + // Nothing came back from this lane, whatever it was told to run. + placeholders.push(None); + } else if have_matrix && !clean { + // It did not finish cleanly, and a channel it should have run left + // no report. A lane that finished cleanly may have been told to run + // one channel only. + placeholders.extend( + CHANNELS + .into_iter() + .filter(|channel| !reported.contains(channel)) + .map(Some), + ); + } + lanes.push(LanePlan { + lane: name.clone(), + blocking: row.is_none_or(|r| r.blocking), + phase: row.and_then(|r| r.phase.clone()), + exit: row.and_then(|r| r.exit.clone()), + reported, + placeholders, + }); + } + Plan { + commit: log.commit.clone(), + lanes, + } +} + +/// Write each lane's report directories under `artifacts_dir`. +fn write_reports(plan: &Plan, queue_dir: &Path, artifacts_dir: &Path) -> Result<()> { + for lane in &plan.lanes { + for channel in &lane.reported { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + Some(channel), + )); + std::fs::create_dir_all(&dest) + .with_context(|| format!("creating {}", dest.display()))?; + let src = queue_dir.join(&lane.lane).join(channel).join("results"); + for file in REPORT_FILES { + let from = src.join(file); + if !regular_file(&from) { + continue; + } + if file == "platform.json" { + // A manifest that does not parse is left out: the report then + // names the lane as having no usable manifest. + let text = std::fs::read_to_string(&from) + .with_context(|| format!("reading {}", from.display()))?; + if let Some(text) = with_lane_slug(&text, &lane.lane, channel) { + std::fs::write(dest.join(file), text) + .with_context(|| format!("writing {}", dest.join(file).display()))?; + } + } else { + std::fs::copy(&from, dest.join(file)) + .with_context(|| format!("copying {}", from.display()))?; + } + } + } + for channel in &lane.placeholders { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + *channel, + )); + std::fs::create_dir_all(&dest) + .with_context(|| format!("creating {}", dest.display()))?; + // No scenarios at all: the report shows the lane as EMPTY. + std::fs::write(dest.join("report.json"), "[]\n") + .with_context(|| format!("writing {}", dest.join("report.json").display()))?; + } + } + Ok(()) +} + +/// `platform.json` with its `platform_slug` (and the copy under `capability`) +/// set to the lane's name and channel, or `None` when it is not a JSON object. +fn with_lane_slug(text: &str, lane: &str, channel: &str) -> Option { + let mut manifest: serde_json::Value = serde_json::from_str(text).ok()?; + let slug = serde_json::Value::String(format!("queue-{lane}-{channel}")); + let object = manifest.as_object_mut()?; + object.insert("platform_slug".to_owned(), slug.clone()); + if let Some(capability) = object + .get_mut("capability") + .and_then(serde_json::Value::as_object_mut) + { + capability.insert("platform_slug".to_owned(), slug); + } + serde_json::to_string_pretty(&manifest).ok() +} + +/// Text that fits in a table cell, from a reason this repository wrote itself. +fn one_line(text: &str) -> String { + text.chars() + .map(|c| { + if c.is_ascii_graphic() || c == ' ' { + c + } else { + ' ' + } + }) + .filter(|c| *c != '|' && *c != '`') + .take(200) + .collect::() + .trim() + .to_owned() +} + +const SECTION_HEADING: &str = "## Queue lanes\n\n"; + +/// The section for a nightly with no queue results to import. +fn render_absent(reason: &str) -> String { + format!( + "{SECTION_HEADING}_No queue lanes are in this report: {}. The GPUs the shared queue \ + tests are not shown tonight._\n\n", + one_line(reason) + ) +} + +fn is_run_url(url: &str) -> bool { + Regex::new(r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+$") + .expect("static pattern") + .is_match(url) +} + +/// The status section: which run was imported, and one row per lane. +fn render_section( + plan: &Plan, + run_url: Option<&str>, + run_started: Option<&str>, + have_matrix: bool, +) -> String { + let mut out = String::from(SECTION_HEADING); + let mut source = String::from("Latest finished run of the shared GPU queue on `main`"); + if let Some(url) = run_url.filter(|u| is_run_url(u)) { + let id = url.rsplit('/').next().unwrap_or_default(); + let _ = write!(source, ": [run {id}]({url})"); + } + if let Some(started) = run_started.filter(|s| { + Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$") + .expect("static pattern") + .is_match(s) + }) { + let _ = write!(source, ", started {started}"); + } + if let Some(commit) = &plan.commit { + let _ = write!(source, ", tested commit `{}`", &commit[..12]); + } + let _ = writeln!(out, "{source}.\n"); + if !have_matrix { + out.push_str( + "_The run's lane matrix could not be read, so every lane is shown as blocking._\n\n", + ); + } + out.push_str("| Lane | GPU | Gate | Phase | Exit | Channels reported |\n"); + out.push_str("|---|---|---|---|---|---|\n"); + for lane in &plan.lanes { + let channels = if lane.reported.is_empty() { + "none".to_owned() + } else { + lane.reported.join(", ") + }; + let _ = writeln!( + out, + "| {} | {} | {} | {} | {} | {channels} |", + lane.lane, + e2e_report::queue_lane_gpu(&lane.lane), + if lane.blocking { + "blocking" + } else { + "advisory" + }, + lane.phase.as_deref().unwrap_or("unknown"), + lane.exit.as_deref().unwrap_or("unknown"), + ); + } + let _ = write!( + out, + "\nThe queue's own gate decides which lanes count: a lane it marks advisory is labelled \ + `{}` below and is left out of the release-readiness verdict, and a blocking lane that \ + reported no result is shown as `EMPTY` and counts as a failure there.\n\n", + e2e_report::ADVISORY_QUEUE_MARKER + ); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The lane matrix of the 2026-11-13 scheduled run, as its public job log + /// printed it. + const LOG: &str = "\u{feff}2026-11-13T02:22:14.0268419Z started gh-43351651832-1; waiting for it to finish +2026-11-13T03:45:26.1317642Z gh-43351651832-1 still running (10 min, 0 of 7 lane(s) finished) +2026-11-13T03:45:26.1323838Z request gh-43351651832-1, finished 2026-11-13T03:45:26Z +2026-11-13T03:45:26.1361153Z | image: \"rocm/dev-ubuntu-24.04:7.2.4\" +2026-11-13T03:45:26.1361751Z | blocking: \"mi300x,r9700s\" +2026-11-13T03:45:26.1362299Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351 +2026-11-13T03:45:26.1362503Z | lanes: mi300x mi325x mi350p mi350x mi355x r9700s w7900-dual +2026-11-13T03:45:26.1362739Z | +2026-11-13T03:45:26.1362910Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE +2026-11-13T03:45:26.1363201Z | mi300x rocm-cli-gh-43351651832-1-mi300x Failed 1 Exported 0m block +2026-11-13T03:45:26.1363511Z | mi325x rocm-cli-gh-43351651832-1-mi325x Failed 1 Exported 0m advisory +2026-11-13T03:45:26.1364685Z | r9700s rocm-cli-gh-43351651832-1-r9700s Failed - - 0m block +2026-11-13T03:45:26.1364991Z | w7900-dual rocm-cli-gh-43351651832-1-w7900-dual Done 0 Exported 0m advisory +2026-11-13T03:45:26.1365442Z | +2026-11-13T03:45:26.1365634Z | SUMMARY: 7 lane(s), 1 passed, 2 failed, 0 incomplete, 0 missing, 4 advisory +2026-11-13T03:45:26.1365895Z | lane mi300x: 7 file(s) collected +2026-11-13T03:45:26.1365895Z | mi999x not a row after the matrix Failed 1 Exported 0m block +"; + + fn row(lane: &str, phase: &str, exit: &str, blocking: bool) -> MatrixRow { + MatrixRow { + lane: lane.to_owned(), + phase: Some(phase.to_owned()), + exit: Some(exit.to_owned()), + blocking, + } + } + + #[test] + fn the_matrix_and_the_tested_commit_are_read_from_the_job_log() { + let log = parse_log(Some(LOG)); + assert_eq!(log.branch.as_deref(), Some("main")); + assert_eq!( + log.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + assert_eq!( + log.lanes, + vec![ + row("mi300x", "Failed", "1", true), + row("mi325x", "Failed", "1", false), + row("r9700s", "Failed", "-", true), + row("w7900-dual", "Done", "0", false), + ], + "a line after the matrix is not a lane" + ); + } + + #[test] + fn a_log_that_is_not_the_report_yields_nothing_to_publish() { + assert_eq!(parse_log(None), QueueLog::default()); + assert_eq!( + parse_log(Some( + "ERR_ACCESS_DENIED\nLANE GATE\nmi300x x\n" + )), + QueueLog::default() + ); + // A row of an unexpected shape keeps its lane and its gate, and nothing else. + let odd = "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE GATE\n\ + 2026-11-13T03:45:26Z | mi300x some workload Pending now advisory\n"; + assert_eq!( + parse_log(Some(odd)).lanes, + vec![MatrixRow { + lane: "mi300x".to_owned(), + phase: None, + exit: None, + blocking: false, + }] + ); + // Text that is not a lane name is never kept, whatever the row says; the + // lane is counted, as blocking. + let hostile = "2026-11-13T03:45:26Z | LANE GATE\n\ + 2026-11-13T03:45:26Z | ../../etc/passwd advisory\n\ + 2026-11-13T03:45:26Z | Uppercase block\n"; + let lanes = parse_log(Some(hostile)).lanes; + assert_eq!( + lanes.iter().map(|l| l.lane.as_str()).collect::>(), + ["unnamed-1", "unnamed-2"] + ); + assert!(lanes.iter().all(|l| l.blocking)); + } + + fn found(entries: &[(&str, &[&'static str])]) -> BTreeMap> { + entries + .iter() + .map(|(lane, channels)| ((*lane).to_owned(), channels.to_vec())) + .collect() + } + + #[test] + fn a_blocking_lane_with_no_result_gets_an_empty_report_and_an_advisory_one_stays_advisory() { + let log = parse_log(Some(LOG)); + let plan = plan( + &log, + found(&[ + ("mi300x", &["release"]), + ("mi325x", &["release", "nightly"]), + ("w7900-dual", &["release", "nightly"]), + ]), + true, + ); + let by_lane = |name: &str| { + plan.lanes + .iter() + .find(|l| l.lane == name) + .expect("lane planned") + }; + // Failed, and the nightly channel left nothing: one stands in for it. + assert_eq!(by_lane("mi300x").placeholders, vec![Some("nightly")]); + assert!(by_lane("mi300x").blocking); + // Nothing at all came back from r9700s, which the queue never listed a file for. + assert_eq!(by_lane("r9700s").placeholders, vec![None]); + assert!(by_lane("r9700s").reported.is_empty()); + // Advisory stays advisory, and a lane with every channel needs no stand-in. + assert!(!by_lane("mi325x").blocking); + assert!(by_lane("mi325x").placeholders.is_empty()); + assert!(by_lane("w7900-dual").placeholders.is_empty()); + assert_eq!( + plan.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + } + + #[test] + fn a_lane_that_finished_cleanly_may_have_run_one_channel_only() { + let log = QueueLog { + branch: Some("main".to_owned()), + commit: None, + lanes: vec![row("mi300x", "Done", "0", true)], + }; + let plan = plan(&log, found(&[("mi300x", &["release"])]), true); + assert!(plan.lanes[0].placeholders.is_empty()); + } + + #[test] + fn without_the_matrix_every_lane_is_blocking_and_only_a_lane_with_no_files_is_missing() { + let plan = plan( + &QueueLog::default(), + found(&[("mi325x", &["release"]), ("r9700s", &[])]), + false, + ); + assert!(plan.lanes.iter().all(|l| l.blocking)); + assert!(plan.lanes[0].placeholders.is_empty()); + assert_eq!(plan.lanes[1].placeholders, vec![None]); + } + + #[test] + fn the_lane_and_channel_replace_the_slug_that_two_lanes_share() { + let manifest = r#"{"platform_slug":"mi300x","capability":{"platform_slug":"mi300x", + "gfx_target":"gfx942"},"versions":{"channel":"release"},"expectations":[]}"#; + let rewritten = with_lane_slug(manifest, "mi325x", "nightly").expect("an object"); + let value: serde_json::Value = serde_json::from_str(&rewritten).expect("json"); + assert_eq!(value["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["gfx_target"], "gfx942"); + assert_eq!(value["versions"]["channel"], "release"); + assert_eq!(with_lane_slug("[1]", "mi325x", "release"), None); + assert_eq!(with_lane_slug("not json", "mi325x", "release"), None); + } + + fn write(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, text).expect("write"); + } + + /// A queue artifact as the run above published it, cut down to two lanes. + fn queue_artifact(dir: &Path) { + for (lane, slug, channels) in [ + ("mi300x", "mi300x", &["release"][..]), + ("mi325x", "mi300x", &["release", "nightly"][..]), + ] { + write(&dir.join(lane).join("lane.log"), "the lane's own log\n"); + for channel in channels { + let results = dir.join(lane).join(channel).join("results"); + write(&results.join("report.json"), "[]\n"); + write(&results.join("commands.jsonl"), ""); + write(&results.join("junit.xml"), ""); + write(&results.join("harness-diagnostics.log"), "diagnostics"); + write( + &results.join("platform.json"), + &format!( + r#"{{"platform_slug":"{slug}","versions":{{"channel":"{channel}"}},"expectations":[]}}"# + ), + ); + } + } + write(&dir.join("r9700s").join("lane.log"), "no files came back\n"); + } + + #[test] + fn the_import_writes_only_what_the_report_reads_under_names_that_say_lane_and_gate() { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let log = queue.path().join("job.log"); + // The log sits beside the lanes here; a stray file is no lane. + write(&log, LOG); + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("consolidated/queue-lanes.md"); + let args = Args { + queue_dir: Some(queue.path().to_owned()), + log: Some(log), + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/43351651832".to_owned()), + run_started: Some("2026-11-13T02:21:58Z".to_owned()), + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }; + run(&args).expect("import"); + + let mut dirs: Vec = std::fs::read_dir(out.path().join("e2e-artifacts")) + .expect("artifacts") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + dirs.sort(); + assert_eq!( + dirs, + [ + "e2e-queue-mi300x-nightly-report", + "e2e-queue-mi300x-release-report", + "e2e-queue-mi325x-advisory-nightly-report", + "e2e-queue-mi325x-advisory-release-report", + "e2e-queue-r9700s-report", + // Listed in the log, with no files at all: an empty stand-in. + "e2e-queue-w7900-dual-advisory-report", + ] + ); + let copied = out + .path() + .join("e2e-artifacts/e2e-queue-mi325x-advisory-release-report"); + let mut files: Vec = std::fs::read_dir(&copied) + .expect("lane dir") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + files.sort(); + assert_eq!(files, ["commands.jsonl", "platform.json", "report.json"]); + let manifest = std::fs::read_to_string(copied.join("platform.json")).expect("manifest"); + assert!(manifest.contains("queue-mi325x-release"), "{manifest}"); + assert_eq!( + std::fs::read_to_string( + out.path() + .join("e2e-artifacts/e2e-queue-r9700s-report/report.json") + ) + .expect("placeholder"), + "[]\n" + ); + + let section = std::fs::read_to_string(section).expect("section"); + assert!(section.starts_with("## Queue lanes\n"), "{section}"); + assert!( + section.contains( + "[run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832), \ + started 2026-11-13T02:21:58Z, tested commit `691834a04540`" + ), + "{section}" + ); + assert!( + section.contains("| mi325x | MI325X | advisory | Failed | 1 | release, nightly |"), + "{section}" + ); + assert!( + section.contains("| r9700s | R9700 | blocking | Failed | - | none |"), + "{section}" + ); + // Nothing of the lane's own log or the workload names reaches the page. + assert!(!section.contains("rocm-cli-gh-"), "{section}"); + assert!(!section.contains("lane's own log"), "{section}"); + } + + #[test] + fn an_absent_queue_run_is_a_section_that_says_so_and_nothing_else() { + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: None, + log: None, + run_url: None, + run_started: None, + absent: Some("no queue run finished on main in the last 24 hours | `x`\n".to_owned()), + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("an absent run is not an error"); + assert!(!out.path().join("e2e-artifacts").exists()); + let text = std::fs::read_to_string(section).expect("section"); + assert_eq!( + text, + "## Queue lanes\n\n_No queue lanes are in this report: no queue run finished on \ + main in the last 24 hours x. The GPUs the shared queue tests are not shown \ + tonight._\n\n" + ); + } + + fn import_with_log(log_text: Option<&str>) -> (tempfile::TempDir, String) { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let out = tempfile::tempdir().expect("tempdir"); + let log = log_text.map(|text| { + let path = out.path().join("job.log"); + write(&path, text); + path + }); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: Some(queue.path().to_owned()), + log, + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/1".to_owned()), + run_started: None, + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("a log that cannot be used is not an error"); + let text = std::fs::read_to_string(section).expect("section"); + (out, text) + } + + #[test] + fn a_run_that_tested_another_branch_or_whose_log_is_unreadable_is_not_imported() { + // A run started from main can be asked to test any branch of the repository. + let other = LOG.replace("rocm-cli main at", "rocm-cli some-branch at"); + for log in [ + Some(other.as_str()), + // What the egress proxy answers when it refuses the request. + Some("ERR_ACCESS_DENIED\n"), + None, + ] { + let (out, text) = import_with_log(log); + assert!( + text.contains("did not test main, or its job log could not be read"), + "{text}" + ); + assert!(!text.contains("| Lane |"), "{text}"); + assert!(!out.path().join("e2e-artifacts").exists()); + } + } + + #[test] + fn a_log_without_a_matrix_leaves_every_lane_blocking_and_says_so() { + let (out, text) = import_with_log(Some( + "2026-11-13T03:45:26Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351\n", + )); + assert!(text.contains("every lane is shown as blocking"), "{text}"); + assert!(!text.contains("advisory |"), "{text}"); + // The lane with no files is still a missing result; the others are not + // second-guessed about a channel they never ran. + let arts = out.path().join("e2e-artifacts"); + assert!(arts.join("e2e-queue-r9700s-report").is_dir()); + assert!(!arts.join("e2e-queue-mi300x-nightly-report").exists()); + } + + #[test] + fn a_lane_that_cannot_be_named_still_shows_as_a_blocking_lane_with_no_result() { + for name in [ + "Mi300X", + "mi300x_2", + "x-advisory", + "x-release", + "x-nightly", + "-x", + ] { + assert!(!is_lane_name(name), "{name}"); + } + let log = parse_log(Some( + "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE\n\ + 2026-11-13T03:45:26Z | x-advisory w Done 0 Exported 0m advisory\n", + )); + assert_eq!( + log.lanes, + vec![MatrixRow { + lane: "unnamed-1".to_owned(), + phase: None, + exit: None, + blocking: true, + }] + ); + // The same for a directory of the artifact. + let queue = tempfile::tempdir().expect("tempdir"); + write(&queue.path().join("mi300x_2/lane.log"), "log\n"); + write( + &queue.path().join("mi300x/release/results/report.json"), + "[]", + ); + let found = scan(queue.path()).expect("scan"); + assert_eq!(found.get("unnamed-1"), Some(&Vec::new())); + assert_eq!(found.get("mi300x"), Some(&vec!["release"])); + let plan = plan(&QueueLog::default(), found, false); + let unnamed = plan + .lanes + .iter() + .find(|l| l.lane == "unnamed-1") + .expect("lane"); + assert!(unnamed.blocking); + assert_eq!(unnamed.placeholders, vec![None]); + } + + #[test] + fn a_run_url_that_is_not_a_run_is_not_linked() { + let plan = Plan { + commit: None, + lanes: Vec::new(), + }; + let text = render_section(&plan, Some("https://example.com/x)[y]("), None, true); + assert!(!text.contains("example.com"), "{text}"); + } +} From 61c2d057a16244810ba786cd24aaad583f93d701 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 16:21:47 +0200 Subject: [PATCH 4/9] feat(e2e-report): label queue lanes by lane and gate, and keep advisory ones out of the verdict The e2e report names a lane of the shared GPU queue by the lane (a queue MI300X lane is not the self-hosted one) and marks one the queue does not block on as advisory. release-readiness leaves advisory rows out of its failure count and still counts a blocking lane that came back with no result. xtask gains the `e2e-queue-import` command, and the W7900 exemption in the per-PR/nightly platform check goes, since #1066 removed the per-PR W7900 lane it exempted. Signed-off-by: Tomas Saaristola --- .../e2e-report/src/consolidated/descriptor.rs | 94 +++++++++++++++++++ crates/e2e-report/src/consolidated/mod.rs | 4 +- crates/e2e-report/src/lib.rs | 5 +- docs/architecture.md | 1 + xtask/src/e2e_report.rs | 15 +-- xtask/src/main.rs | 8 ++ xtask/src/release_readiness.rs | 88 +++++++++++++---- 7 files changed, 182 insertions(+), 33 deletions(-) diff --git a/crates/e2e-report/src/consolidated/descriptor.rs b/crates/e2e-report/src/consolidated/descriptor.rs index 7291dd2cb..1185546de 100644 --- a/crates/e2e-report/src/consolidated/descriptor.rs +++ b/crates/e2e-report/src/consolidated/descriptor.rs @@ -48,6 +48,11 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { }, }; + // A lane of the shared GPU queue (see `queue_descriptor`). + if let Some(lane) = core.strip_prefix("queue-") { + return queue_descriptor(lane, channel_from_name); + } + // Legacy `-known-bugs` suffix (retained only as a stable secondary sort key). let (core, known_bugs) = match core.strip_suffix("known-bugs") { Some(rest) => (rest.trim_end_matches('-'), true), @@ -86,6 +91,68 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { } } +/// The marker an advisory queue lane carries in its platform label. +/// +/// A reader of the rendered report, and `xtask release-readiness`, which reads +/// that rendering, tell a lane the queue lets fail from one it does not by it. +pub const ADVISORY_QUEUE_MARKER: &str = "(queue, advisory)"; + +/// The GPU each shared-queue lane stands for, by the lane's name. The queue +/// discovers its lanes from its fleet, so a lane missing from this table is not +/// an error: it renders under its own name in capitals. +const QUEUE_LANE_GPUS: &[(&str, &str)] = &[ + ("mi300x", "MI300X"), + ("mi325x", "MI325X"), + ("mi350p", "MI350P"), + ("mi350x", "MI350X"), + ("mi355x", "MI355X"), + ("r9700s", "R9700"), + ("w7900-dual", "W7900"), +]; + +/// The GPU a shared-queue lane stands for (`w7900-dual` is a W7900), or the +/// lane's own name in capitals for a lane this table does not know yet. +pub fn queue_lane_gpu(lane: &str) -> String { + QUEUE_LANE_GPUS + .iter() + .find(|(name, _)| *name == lane) + .map_or_else(|| lane.to_uppercase(), |(_, gpu)| (*gpu).to_owned()) +} + +/// The artifact name `xtask e2e-queue-import` gives a queue lane's report. +/// +/// For one channel, or none for a lane that reported no channel at all. The lane +/// is named, not the `platform_slug` in its `platform.json`: queue lanes of +/// different models report the same slug (an MI325X lane probes as `mi300x`, an +/// MI355X as `mi350p`), so the slug cannot say which lane a report came from. +/// A lane the queue does not block on carries the `-advisory` suffix. +pub fn queue_artifact_name(lane: &str, advisory: bool, channel: Option<&str>) -> String { + let advisory = if advisory { "-advisory" } else { "" }; + let channel = channel.map_or_else(String::new, |c| format!("-{c}")); + format!("e2e-queue-{lane}{advisory}{channel}-report") +} + +/// Parse the core of a queue lane's artifact name (`mi325x`, `mi325x-advisory`; +/// see [`queue_artifact_name`]). Always Linux: the queue runs Linux pods. +fn queue_descriptor(lane: &str, channel_from_name: Option<&'static str>) -> Descriptor { + let (lane, advisory) = match lane.strip_suffix("-advisory") { + Some(rest) => (rest, true), + None => (lane, false), + }; + let gpu = queue_lane_gpu(lane); + let queue = if advisory { + ADVISORY_QUEUE_MARKER + } else { + "(queue)" + }; + Descriptor { + platform: format!("{gpu} {queue}"), + os: "Linux".to_owned(), + known_bugs: false, + channel_from_name, + } +} + /// The identity-bearing core of an artifact name: `e2e-` prefix and `-report` /// suffix stripped, each relative to the prior result (not `name`), so /// `e2e-report` correctly reduces to `report`, not back to itself. @@ -328,6 +395,33 @@ mod tests { } } + #[test] + fn parse_descriptor_names_each_queue_lane_and_its_gate() { + // Every lane name is a path segment of a queue artifact, so the table is + // what turns `w7900-dual` into the GPU a reader knows. A lane the table + // lacks keeps its own name rather than being guessed at. + for (lane, gpu) in QUEUE_LANE_GPUS { + let blocking = parse_descriptor(&queue_artifact_name(lane, false, Some("release"))); + assert_eq!( + (blocking.platform.as_str(), blocking.os.as_str()), + (format!("{gpu} (queue)").as_str(), "Linux"), + "{lane}" + ); + assert_eq!(blocking.channel_from_name, Some("release")); + let advisory = parse_descriptor(&queue_artifact_name(lane, true, Some("nightly"))); + assert_eq!(advisory.platform, format!("{gpu} {ADVISORY_QUEUE_MARKER}")); + assert_eq!(advisory.channel_from_name, Some("nightly")); + } + let new_lane = parse_descriptor(&queue_artifact_name("mi400x", true, None)); + assert_eq!(new_lane.platform, "MI400X (queue, advisory)"); + assert_eq!(new_lane.channel_from_name, None); + // A queue MI300X lane is not the self-hosted MI300X lane. + assert_ne!( + parse_descriptor("e2e-queue-mi300x-report").platform, + parse_descriptor("e2e-gpu-report").platform + ); + } + #[test] fn parse_descriptor_distinguishes_sdk_version_matrix_legs() { // The sdk_version matrix (ROCMAI-430) uploads non-`current` legs under a diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 911385029..9a51e9036 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -24,7 +24,9 @@ mod sidecars; #[cfg(test)] mod test_support; -pub use descriptor::artifact_name_for_platform_slug; +pub use descriptor::{ + ADVISORY_QUEUE_MARKER, artifact_name_for_platform_slug, queue_artifact_name, queue_lane_gpu, +}; use descriptor::{Descriptor, full_platform_label, parse_descriptor, with_paren_suffix}; use sidecars::{ CommandRecord, ManifestExpectation, PlatformManifest, PlatformVersions, parse_commands, diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 73a702e74..3eaae3947 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -16,7 +16,10 @@ mod single_report; pub use consolidated::html::{RunMeta, generate_consolidated}; pub use consolidated::markdown::consolidated_summary_markdown; -pub use consolidated::{PlatformReports, artifact_name_for_platform_slug, load_platform_reports}; +pub use consolidated::{ + ADVISORY_QUEUE_MARKER, PlatformReports, artifact_name_for_platform_slug, load_platform_reports, + queue_artifact_name, queue_lane_gpu, +}; pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id}; pub use single_report::generate; diff --git a/docs/architecture.md b/docs/architecture.md index a303a91f8..f0f8f00d9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -738,6 +738,7 @@ Repository tasks, run through the workspace alias `cargo xtask `: relea | [`demos.rs`](../xtask/src/demos.rs) | `cargo xtask demos`: builds the demo binaries, starts the mock environment before VHS runs, and renders the documentation GIFs. | flat | | [`e2e.rs`](../xtask/src/e2e.rs) | `cargo xtask e2e`: builds release `rocm` and `rocmd` with the E2E test features and runs the cucumber suite, forwarding extra arguments to cucumber. | flat | | [`e2e_prewarm.rs`](../xtask/src/e2e_prewarm.rs) | `cargo xtask e2e-prewarm`: prepares the shared managed runtime the GPU E2E lanes serve against, refreshing it when the channel index publishes something newer or the required packages differ. | flat | +| [`e2e_queue_import.rs`](../xtask/src/e2e_queue_import.rs) | `cargo xtask e2e-queue-import`: puts a finished shared-queue run's per-lane reports where `e2e-report` reads them, named for the lane and the queue's gate, and writes the queue lanes' status section for the top of the support matrix. | flat | | [`e2e_report.rs`](../xtask/src/e2e_report.rs) | `cargo xtask e2e-report`: merges per-platform `report.json` files into one HTML report and a step-summary matrix, one platform per artifact subdirectory. | flat | | [`engine_stdio_client_contract.rs`](../xtask/src/engine_stdio_client_contract.rs) | Source-scan test that only the shared stdio client in [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) puts a request on an engine child's stdin: a file that both pipes a child's stdin and names an engine envelope is reported unless it is a reviewed `ALLOWED` entry, and a stale entry fails too. | flat | | [`env_mutation_contract.rs`](../xtask/src/env_mutation_contract.rs) | Source-scan test that every test which mutates the process environment serializes itself. | flat | diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 7316f1e79..63fa727cc 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -272,10 +272,6 @@ mod tests { #[test] fn the_nightly_lanes_publish_the_same_platforms_as_the_per_pr_lanes() { - // W7900's self-hosted nightly job was removed because it never got a - // runner; its nightly runs on the queue, which does not report into this - // grid. Its per-PR-workflow lane may still exist. - const NO_SELF_HOSTED_NIGHTLY: [&str; 1] = ["e2e-gpu-w7900-report"]; // The nightly workflow exists to run *more* scenarios on *at least* // the per-PR platforms, plus lanes the per-PR smoke gate deliberately // skips (e.g. rad3, mi350p -- ROCMAI-125). If a per-PR platform ever @@ -327,18 +323,9 @@ mod tests { "per-PR lane scan found nothing to check" ); assert!( - per_pr - .iter() - .filter(|p| !NO_SELF_HOSTED_NIGHTLY.contains(&p.as_str())) - .all(|p| nightly.contains(p)), + per_pr.iter().all(|p| nightly.contains(p)), "every per-PR platform must also run nightly: per_pr={per_pr:?} nightly={nightly:?}" ); - assert!( - NO_SELF_HOSTED_NIGHTLY - .iter() - .all(|p| !nightly.iter().any(|q| q == p)), - "NO_SELF_HOSTED_NIGHTLY names a lane that runs nightly again: drop it from the list" - ); } #[test] diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e3db2768f..4a32095aa 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,6 +22,7 @@ mod dash_text_edit_contract; mod demos; mod e2e; mod e2e_prewarm; +mod e2e_queue_import; mod e2e_report; mod engine_stdio_client_contract; mod env_mutation_contract; @@ -331,6 +332,12 @@ pub(crate) enum Command { #[arg(long)] html_out: PathBuf, }, + /// Bring a finished run of the shared GPU queue into the nightly's E2E + /// report: write each lane's reports where `e2e-report` reads them, named + /// for the lane and for whether the queue blocks on it, and write a status + /// section for the top of the support matrix. With `--absent`, import + /// nothing and say why. See docs/nightly-on-the-queue.md. + E2eQueueImport(e2e_queue_import::Args), /// Count received Doctor reports from an export of the report mailbox. /// /// Reads `.eml` files (one message each) and raw `.json` report bodies, @@ -497,6 +504,7 @@ fn run() -> Result<()> { artifacts_dir, html_out, } => e2e_report::run(&artifacts_dir, &html_out)?, + Command::E2eQueueImport(args) => e2e_queue_import::run(&args)?, Command::ReportRollup(args) => report_rollup::run(&args)?, Command::ReleaseReadiness { since_tag, diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index c3060f557..3cdd834df 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -407,6 +407,11 @@ fn excerpt(text: &str, max_chars: usize) -> String { /// adds for platforms the Total row would otherwise read as having zero /// failures. /// +/// A row for a shared-queue lane the queue does not block on (its platform +/// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the +/// queue lets that lane fail, so the verdict does too. Its `Fail` cell is +/// taken back out of the Total row, which sums every row. +/// /// Parsing the already-rendered rows, rather than re-deriving per-platform /// pass/fail here, keeps this in lockstep with that crate's known-bugs/xfail /// handling instead of duplicating it. @@ -431,24 +436,31 @@ fn total_e2e_failures(markdown: &str) -> Option { .nth(5) .and_then(|cell| cell.trim().parse::().ok())?; - let silently_empty_platforms = lines[header_idx + 1..total_idx] - .iter() - .filter(|l| { - let t = l.trim_start(); - t.starts_with('|') && !t.starts_with("|---") - }) - .filter(|l| { - let cells: Vec<&str> = l.split('|').collect(); - let status = cells.get(8).map(|c| c.trim()); - let fail = cells - .get(5) - .and_then(|c| c.trim().parse::().ok()) - .unwrap_or(0); - !matches!(status, Some("PASS") | None) && fail == 0 - }) - .count() as u64; + let mut advisory_fails = 0; + let mut silently_empty_platforms = 0; + for line in lines[header_idx + 1..total_idx].iter().filter(|l| { + let t = l.trim_start(); + t.starts_with('|') && !t.starts_with("|---") + }) { + let cells: Vec<&str> = line.split('|').collect(); + let fail = cells + .get(5) + .and_then(|c| c.trim().parse::().ok()) + .unwrap_or(0); + if cells + .get(1) + .is_some_and(|c| c.contains(e2e_report::ADVISORY_QUEUE_MARKER)) + { + advisory_fails += fail; + continue; + } + let status = cells.get(8).map(|c| c.trim()); + if !matches!(status, Some("PASS") | None) && fail == 0 { + silently_empty_platforms += 1; + } + } - Some(total_row_fails + silently_empty_platforms) + Some(total_row_fails.saturating_sub(advisory_fails) + silently_empty_platforms) } /// Neutralize markdown syntax in untrusted text (GitHub issue titles) before @@ -741,6 +753,48 @@ mod tests { assert_eq!(total_e2e_failures(md), Some(1)); } + #[test] + fn total_e2e_failures_leaves_out_the_queue_lanes_the_queue_lets_fail() { + // Real producer output, so the marker the parser looks for is the one the + // report writes: a failing advisory lane and an empty advisory lane add + // nothing, while the same lanes marked blocking add one failure and one + // silent failure. + fn report(status: &str) -> String { + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s","tags":[],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{status}"}}}}]}}]}}]"# + ) + } + let render = |advisory: bool| { + let dir = tempfile::tempdir().expect("tempdir"); + let write = |name: &str, text: &str| { + let path = dir.path().join(name); + std::fs::create_dir_all(&path).expect("mkdir"); + std::fs::write(path.join("report.json"), text).expect("write report"); + (name.to_owned(), path.join("report.json")) + }; + let failing = e2e_report::queue_artifact_name("mi325x", advisory, Some("release")); + let empty = e2e_report::queue_artifact_name("r9700s", advisory, None); + let inputs = vec![ + write("e2e-gpu-report", &report("passed")), + write(&failing, &report("failed")), + write(&empty, "[]"), + ]; + e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&inputs)) + }; + let advisory = render(true); + assert!( + advisory.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{advisory}" + ); + assert_eq!(total_e2e_failures(&advisory), Some(0), "{advisory}"); + let blocking = render(false); + assert!( + !blocking.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{blocking}" + ); + assert_eq!(total_e2e_failures(&blocking), Some(2), "{blocking}"); + } + #[test] fn total_e2e_failures_ignores_the_expectation_grid_and_command_coverage_tables() { // Regression: the scan used to run over every `|`-prefixed line in the From 2e20fbadd1ae36ee43e7b5ade0d5b53508a0fda1 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 16:23:53 +0200 Subject: [PATCH 5/9] ci: keep the queue's GPUs in the support matrix and release readiness Retiring the self-hosted R9700, MI350P and W7900 nightly lanes would drop them from the E2E-Support-Matrix and Release-Readiness wiki pages, and MI325X, MI350X and MI355X, which only the queue runs, would never appear. e2e-report-nightly now fetches the newest queue-run-results artifact of a main run that finished in the last 24 hours (and its job log, for the tested branch and each lane's gate), imports it, and opens the matrix with a Queue lanes table. A missing, late or malformed queue run never fails the report: the page says so instead. The workflow contract pins the new steps and the native Linux preflight lane counts for this change plus #1066 (four). The docs describe the queue import, link the queue page instead of "(below; advisory)", and fix the "Of those" wording in testing.md. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 91 +++++++++++++++++++++++ docs/ci-hardware-testing.md | 40 ++++++---- docs/nightly-on-the-queue.md | 79 ++++++++++++++++++-- docs/testing.md | 11 +-- tests/e2e-cucumber/README.md | 2 +- xtask/src/workflow_contract.rs | 131 +++++++++++++++++++++++++++++++-- 6 files changed, 322 insertions(+), 32 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fc58216ac..3ddce9b2e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1378,12 +1378,103 @@ jobs: pattern: '*-report' path: e2e-artifacts + # The shared GPU queue's nightly (docs/nightly-on-the-queue.md) runs R9700, + # MI350P, W7900 and the Instinct lanes of the queue's fleet on its own + # schedule, and its results are published as one `queue-run-results` + # artifact of that run, not as `*-report` artifacts of this one. So fetch + # the newest such artifact of a run on main that finished, and started, in + # the last 24 hours (a run still going, or one from an earlier night, is + # not today's), and the job log of that run, which is the only place the + # queue prints which branch it tested and which lanes it blocks on. The + # artifact is found by its name, not through the workflow that makes it: + # that workflow runs on a runner no other workflow may name or depend on. + # A conclusion of `failure` is no reason to skip it: a red lane fails the + # queue's run and the artifact is still uploaded. This step never fails the + # job and never stops the report: when there is nothing to import it + # records why, and the next step says so at the top of the page instead of + # leaving the queue's GPUs out without a word. + - name: Fetch the latest queue results + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + mkdir -p queue-results + absent() { + echo "$1" > queue-results/absent.txt + echo "::warning::No queue lanes in this report: $1" + exit 0 + } + repo_id=$(gh api "repos/${GITHUB_REPOSITORY}" --jq .id) || + absent "the repository could not be read" + since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) + found=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=queue-run-results&per_page=50" | + jq -r --arg since "$since" --argjson repo "$repo_id" ' + [.artifacts[] + | select(.expired == false) + | select(.workflow_run.head_branch == "main" and .workflow_run.head_repository_id == $repo) + | select(.created_at >= $since)] + | sort_by(.created_at) | last // empty | .workflow_run.id') || + absent "the queue results could not be listed" + [ -n "$found" ] || + absent "no queue run on main finished in the last 24 hours (a run still going is not imported)" + # The run of main's workflow can have been asked to test another branch, + # and a run that started a day ago is not tonight's. + started=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${found}" --jq .run_started_at) || + absent "when the newest queue run started could not be read" + started_at=$(date -u -d "$started" +%s 2>/dev/null) && + [ "$started_at" -ge "$(date -u -d '24 hours ago' +%s)" ] || + absent "the newest queue run started more than 24 hours ago" + echo "$started" > queue-results/started.txt + gh run download "$found" --repo "$GITHUB_REPOSITORY" --name queue-run-results --dir queue-results/results || + absent "the queue results of run $found could not be downloaded" + # The log names the branch the run tested and which lanes the queue + # blocks on. Without it nothing is imported. + job=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${found}/jobs" --jq '.jobs[0].id') && + gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log || + rm -f queue-results/job.log + echo "$found" > queue-results/run-id.txt + + # Turn the run's lanes into reports of this job's own kind, and write the + # status section for the top of the matrix (`cargo xtask e2e-queue-import`). + # A lane the queue marks advisory is labelled so and does not count towards + # the release-readiness verdict; a blocking lane with no result does. + - name: Add the queue lanes to the report + continue-on-error: true + run: | + set -uo pipefail + mkdir -p queue-results + if [ -s queue-results/run-id.txt ] && [ -d queue-results/results ]; then + run_id=$(cat queue-results/run-id.txt) + args=(--queue-dir queue-results/results + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${run_id}" + --run-started "$(cat queue-results/started.txt 2>/dev/null)") + [ -f queue-results/job.log ] && args+=(--log queue-results/job.log) + else + args=(--absent "$(cat queue-results/absent.txt 2>/dev/null || echo 'fetching the queue results did not complete')") + fi + cargo xtask e2e-queue-import "${args[@]}" \ + --artifacts-dir e2e-artifacts \ + --section-out queue-results/queue-lanes.md || { + # Half an import is worse than none, and none is said at the top of the page. + rm -rf e2e-artifacts/e2e-queue-* queue-results/queue-lanes.md + echo "::warning::The queue lanes could not be added to the report." + cargo xtask e2e-queue-import --absent "the queue lanes could not be imported" \ + --artifacts-dir e2e-artifacts --section-out queue-results/queue-lanes.md || true + } + + # The queue lanes' status leads the page. The matrix keeps its own first + # table, the one `release-readiness` reads its verdict from. - name: Build consolidated report + step summary run: | mkdir -p consolidated cargo xtask e2e-report \ --artifacts-dir e2e-artifacts \ --html-out consolidated/index.html > consolidated/support-matrix.md + if [ -f queue-results/queue-lanes.md ]; then + cat queue-results/queue-lanes.md consolidated/support-matrix.md > "$RUNNER_TEMP/with-queue.md" + mv "$RUNNER_TEMP/with-queue.md" consolidated/support-matrix.md + fi cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY" # A lane that dies before writing report.json (preflight, build, a crash diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 2fa17a819..f50d77f5d 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -37,7 +37,7 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| | Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | -| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | +| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | MI300X (`e2e-gpu-nightly`), Strix Halo Ubuntu (`e2e-gpu-nightly-strix`), Windows (`e2e-gpu-nightly-strix-windows`) and WSL2 (`e2e-wsl-nightly`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | | Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125) and moved to @@ -50,11 +50,14 @@ A pull request now runs on exactly one Linux GPU node and one Windows GPU pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in -the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane -in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue -nightly's `w7900-dual` lane (below; advisory), not on `push`, `release/**` or -dispatch. Its first run, the 11-13 queue nightly -([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), got capacity at once and finished `Done`, exit 0. +the merge queue; it keeps its `push`/dispatch triggers and also runs nightly +(`e2e-gpu-nightly-strix` on the same DevLab Dispatch pool labels). There is no +W7900 lane in `e2e-selfhosted.yml` any more: W7900 is covered only by the +scheduled queue nightly's `w7900-dual` lane (see +[nightly-on-the-queue.md](nightly-on-the-queue.md)), not on `push`, `release/**` +or dispatch. Its first run, the 11-13 queue nightly +([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), +got capacity at once and passed (`Done`, exit 0). [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length @@ -256,14 +259,27 @@ covers the mock platform; pull request runs only the MI300X and Windows Strix Halo pool lanes among them); `nightly.yml`'s `e2e-report-nightly` covers a strict superset of those platforms (it adds MI300X's `e2e-gpu-nightly` and the Strix Halo Ubuntu, -Windows and WSL2 nightly lanes) with the `@nightly` scenarios included. The R9700, MI350P and -W7900 nightlies run on the queue and are not in this report. Each joins its platforms' +Windows and WSL2 nightly lanes) with the `@nightly` scenarios included. The +R9700, MI350P and W7900 nightlies run on the queue, whose lanes the report reads +separately (next paragraph). Each joins its platforms' reports — including partial or failed runs — by scenario id into one HTML report and GitHub step summary. Since ROCMAI-429 the report keys each column by `(platform_slug, channel)` rather than platform alone, so `nightly.yml`'s release/nightly channel matrix renders two columns per platform instead of one channel's result silently overwriting the other. +`e2e-report-nightly` also reads the shared queue's nightly (below), which runs +R9700, MI350P, W7900 and the Instinct lanes of the queue's fleet. It fetches the +`queue-run-results` artifact of the newest queue run that tested `main` and +finished, and started, in the last 24 hours, and `cargo xtask e2e-queue-import` +puts each lane's reports among the others, as `MI325X (queue) Linux (release)` +and the like. A lane the queue does not block on is labelled `(queue, advisory)` +and does not count towards the release-readiness verdict; a blocking lane that +came back with no result is shown as `EMPTY` and does. The page opens with a +`Queue lanes` table. A missing or late queue run never fails the report: the +page says no queue lanes are in it. See +[The support matrix and the wiki](nightly-on-the-queue.md#the-support-matrix-and-the-wiki). + On `main`, the `publish-e2e-wiki` job in `nightly.yml` also copies the consolidated nightly matrix to the `E2E-Support-Matrix` page of the repository wiki, overwriting it each night (the page header names the run, commit and @@ -330,15 +346,11 @@ The lane artifacts are named canonically (`e2e-gpu-n-1-report`, `e2e-gpu-n-2-rep `e2e-gpu-strix-ubuntu-n-2-report`, `e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, `e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`, -`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, -`e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and +`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, `e2e-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900` -are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any -more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they -don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index 57364bad1..04ad16f0b 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -28,10 +28,10 @@ The lane set is whatever GPU models the queue has. The 11-13 run the only scheduled coverage of that hardware, since `e2e-selfhosted.yml` no longer has a W7900 lane. -**The remaining gap is native Windows and WSL.** The queue runs Linux pods, so -the native Windows and WSL lanes stay on their dedicated runners, in `nightly.yml` -and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); nothing -here replaces them. Which checks are required before merging is a repository +**The remaining gap is Strix Halo.** The queue has no Strix lane and runs Linux +pods, so the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool, +in `nightly.yml` and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); +nothing here replaces them. Which checks are required before merging is a repository setting, outside this page. This page is the **contract** between this repository and the queue side: what @@ -140,8 +140,10 @@ self-hosted nightly jobs were removed from `nightly.yml`. The MI300X nightly job is still there, and the queue's `mi300x` lane is an additional signal to it. The lanes write only into the queue artifact, and the `e2e-gpu--report` artifacts that `nightly.yml`'s consolidated report -reads are not produced from them, so the consolidated report and the support -matrix no longer have rows for R9700, MI350P or W7900 Linux. +reads are not produced from them. The report fetches the queue's artifact +itself instead, so the support matrix, the `E2E-Support-Matrix` and +`Release-Readiness` wiki pages and the release-readiness verdict keep every +GPU the queue tests: see [The support matrix and the wiki](#the-support-matrix-and-the-wiki). ## Output @@ -195,6 +197,71 @@ that line is what counts: a blocking lane without a result fails the run. result is not a pass, but they point at queue capacity or infrastructure rather than at a regression. +## The support matrix and the wiki + +`nightly.yml`'s `e2e-report-nightly` job builds the consolidated support +matrix, which `publish-e2e-wiki` publishes as the `E2E-Support-Matrix` wiki page +and `release-readiness-nightly` reads its GO/NO-GO verdict from (see +[ci-hardware-testing.md](ci-hardware-testing.md)). It also reads the queue's +lanes, from the queue's own results, so retiring a self-hosted nightly lane +that the queue covers drops no GPU from either page, and a GPU that only the +queue has (MI325X, MI350X, MI355X) is on them. + +- **Which run.** The newest `queue-run-results` artifact of a run on this + repository's `main` that finished, and started, in the last 24 hours, + whatever the run's conclusion was: a red lane fails the run, and its artifact + is still uploaded. The job finds the artifact by name, not through the + workflow that makes it, because no other workflow may name that one (see + below). The run also has to have tested `main`: a run started from `main` can + be asked to test another branch (the `branch` input), and the job log names + the branch it did. `queue-run.yml` starts at 02:17 UTC and the nightly + starts at 06:00 UTC, its report job after its GPU lanes have finished, so a + run that took under about three and a half hours is in the night's report. A + run that is still going, that started on an earlier day or that tested + another branch is not imported, and neither is a run published again by + `nightly-queue.yml`. +- **Which lanes, and whether each blocks.** The artifact holds the lanes' + files and nothing else about the run, so the job reads the run's job log, whose + lane matrix is the only place the queue prints which lanes it ran and which of + them can fail the run (the `GATE` column). Only the tested branch and commit + and each lane's name, phase, exit status and gate are taken from it, each + checked against a strict pattern first. A lane whose name is not a plain + lowercase word (letters, digits and dashes, and not ending in `-advisory`, + `-release` or `-nightly`) is shown as `unnamed-1` and so on, as a blocking + lane with no result. Without the log nothing is imported. +- **What is imported.** For each lane and channel, + `//results/{report.json,platform.json,commands.jsonl}`, the + three files the report reads, as an artifact named + `e2e-queue-[-advisory]--report`. The `lane.log`, the JUnit and + HTML output and the harness diagnostics are not copied, and nothing the + queue's scan withheld is brought back. The lane's `platform_slug` is replaced + by `queue--`: the slug is the report's column identity, and + queue lanes of different models report the same one (an MI325X lane probes as + `mi300x`, an MI350X or MI355X lane as `mi350p`). The lane command therefore + has to keep writing those files to `/out//results`, as + [`scripts/run-e2e-lane.sh`](../scripts/run-e2e-lane.sh) does. +- **How it reads.** A queue row is labelled `MI325X (queue) Linux (release)`, + so it is not confused with the self-hosted MI300X lane, and a lane the queue + does not block on is `MI325X (queue, advisory) Linux (release)`. The page + opens with a `Queue lanes` table: one row per lane the queue ran, with its + gate, phase, exit status and the channels that reported. +- **What counts towards the verdict.** The same as for the queue's own run. A + blocking lane's failures are in the `E2E` failure count. An advisory lane's are + shown but left out of it. A lane that came back with no result, or whose + channel is missing after it did not finish cleanly, is shown as `EMPTY` and + counts for a blocking lane, because a blocking lane without a result is not + a pass. +- **When there is nothing to import.** No run in the window, a run of another + branch, an artifact or log that cannot be downloaded or an import that fails + never fails the nightly report: the page opens with a `Queue lanes` note + saying that no queue lanes are in it tonight and why, and the rest is built as + before. The verdict does not take that absence for a failure, so read the note + before reading a `GO`. + +What is public follows from the queue's own rules: the artifact is public and +already scanned, only three of its files are used, and the log is the public job +log. + ## What the workflow requires - a self-hosted Linux runner registered with the `queue-report` label alone, diff --git a/docs/testing.md b/docs/testing.md index 127d45c6b..31af227a3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3067,11 +3067,12 @@ The user-visible side is `serve-55` (`@id:serve-small-assistant-short-name`): It carries `@requires-gpu` and deliberately no `@requires-engine`, because the engine is the CLI's to pick here — so the lane that exercises the Lemonade half is whichever GPU lane defaults to Lemonade (the Strix Halo lanes, and W7900 on -the queue nightly only) and the vLLM half falls to the Instinct lane. Of those, -MI300X and the Strix Halo Windows and WSL2 lanes run on pull requests -([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when the `serve` -path filter matches; W7900 has no lane there any more) but none of their check names is a required status check, -so the two unit tests above are the blocking per-PR coverage. The scenario +the queue nightly only) and the vLLM half falls to the Instinct lane. Of all +these lanes only MI300X and the Strix Halo Windows and WSL2 lanes run on pull +requests ([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when +the `serve` path filter matches; W7900 has no per-PR lane any more and runs only +on the queue nightly), and none of their check names is a required status +check, so the two unit tests above are the blocking per-PR coverage. The scenario carries no [expectations.toml](../tests/e2e-cucumber/expectations.toml) row, unlike its managed-Lemonade siblings on the Linux Strix lane: whether it shares their known serve failure there has not been observed, and a row claiming a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index cbbe6a7e1..8723a3c01 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -200,7 +200,7 @@ table above as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-strix`, `E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform grid. R9700, MI350P and W7900 no longer have a self-hosted nightly lane in `nightly.yml`: they run on the queue ([nightly-on-the-queue.md](../../docs/nightly-on-the-queue.md)), -whose results are not in that grid. The shared large-model scenario follows each lane's serve engine: it +and the nightly report adds the queue's lanes to that grid. The shared large-model scenario follows each lane's serve engine: it serves `Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), and the `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified on Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 21882a975..c93a4898b 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2872,8 +2872,8 @@ jobs: } } assert_eq!( - delegating, 7, - "expected the seven native Linux preflight lanes to delegate to \ + delegating, 4, + "expected the four native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2941,8 +2941,8 @@ jobs: } assert_eq!( bodies.len(), - 7, - "expected seven native Linux preflight bodies to compare, found {} — \ + 4, + "expected four native Linux preflight bodies to compare, found {} — \ if a lane was added or removed, update this count", bodies.len() ); @@ -4301,8 +4301,8 @@ fi } } assert_eq!( - delegating, 7, - "expected the 7 native Linux preflight lanes to delegate to \ + delegating, 4, + "expected the 4 native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); assert_eq!( @@ -4399,6 +4399,125 @@ fi } } + /// The nightly report carries the shared queue's GPUs into the matrix, and + /// so into the wiki and the release-readiness verdict, from the queue's own + /// results: the newest finished run on `main`, found by its artifact's name + /// (no workflow may name or wait on the queue's, see + /// `assert_queue_workflow_holds_no_credential`). Neither step may fail the + /// job: a missing or late run is said so at the top of the page. + #[test] + fn nightly_report_imports_the_latest_queue_run_without_depending_on_it() { + let nightly = read_workflow("nightly.yml"); + let job = job_block(&nightly, "e2e-report-nightly"); + let steps = job_steps(job).expect("the report job has steps"); + let position = |prefix: &str| { + steps + .iter() + .position(|(name, _)| name.starts_with(prefix)) + .unwrap_or_else(|| panic!("e2e-report-nightly has a `{prefix}` step")) + }; + let (fetch, add, build) = ( + position("Fetch the latest queue results"), + position("Add the queue lanes to the report"), + position("Build consolidated report"), + ); + assert!( + fetch < add && add < build, + "the queue lanes must be fetched, then put among the reports, before the report is built" + ); + for at in [fetch, add] { + assert_eq!( + step_key(&steps[at].1, "continue-on-error"), + "true", + "`{}` must not be able to fail the nightly report: a missing or late queue run \ + is shown as missing, not as a red report", + steps[at].0 + ); + } + + let fetch = normalized_whitespace(&steps[fetch].1); + for needed in [ + // By artifact name, from a run on this repository's main: the + // workflow that makes it may not be named here. + "actions/artifacts?name=queue-run-results", + ".workflow_run.head_branch == \"main\"", + ".workflow_run.head_repository_id == $repo", + ".expired == false", + // Today's run: not yesterday's, whatever its conclusion was. + "date -u -d '24 hours ago'", + ".created_at >= $since", + "gh run download \"$found\"", + "--name queue-run-results", + // A run that started a day ago is not tonight's, whenever it ended. + "[ \"$started_at\" -ge \"$(date -u -d '24 hours ago' +%s)\" ]", + // The log says which branch the run tested and which lanes block. + "/logs\" > queue-results/job.log", + ] { + assert!( + fetch.contains(needed), + "the queue fetch step must contain `{needed}`:\n{fetch}" + ); + } + assert!( + !fetch.contains("conclusion"), + "a red queue run is still a run: its artifact is uploaded whatever it concluded" + ); + // No credential beyond the job's own read-only token. + let fetch_step = &steps[position("Fetch the latest queue results")].1; + let env = step_env(fetch_step); + assert_eq!( + env.get("GH_TOKEN").map(String::as_str), + Some("${{ github.token }}"), + "{env:?}" + ); + assert_eq!(env.len(), 1, "{env:?}"); + assert!(!fetch_step.contains("secrets."), "{fetch_step}"); + let permissions = job_mapping(job, "permissions"); + assert_eq!( + permissions.get("actions").map(String::as_str), + Some("read"), + "listing and downloading the queue's artifact needs actions: read" + ); + assert_eq!( + permissions.get("contents").map(String::as_str), + Some("read") + ); + assert_eq!(permissions.len(), 2, "{permissions:?}"); + + // Both outcomes go through the importer, which writes the status section. + let add = normalized_whitespace(&steps[position("Add the queue lanes to the report")].1); + for needed in [ + "cargo xtask e2e-queue-import", + "--queue-dir queue-results/results", + "--absent", + "--artifacts-dir e2e-artifacts", + "--section-out queue-results/queue-lanes.md", + // An import that fails leaves no half of one behind, and says so. + "rm -rf e2e-artifacts/e2e-queue-*", + "--absent \"the queue lanes could not be imported\"", + ] { + assert!( + add.contains(needed), + "the queue import step must contain `{needed}`:\n{add}" + ); + } + // The section leads the page; the matrix keeps its own first table. + let build = normalized_whitespace(&steps[build].1); + assert!( + build.contains( + "cat queue-results/queue-lanes.md consolidated/support-matrix.md > \ + \"$RUNNER_TEMP/with-queue.md\"" + ), + "the queue section must be put ahead of the matrix:\n{build}" + ); + // A queue run is never something the report job waits for. + let needs = job_scalar(job, "needs"); + assert!( + !needs.contains("queue"), + "e2e-report-nightly must not depend on the queue's workflows: {needs}" + ); + } + #[test] fn nightly_publishes_the_consolidated_matrix_to_the_wiki() { let nightly = read_workflow("nightly.yml"); From fda8a4c59a251f6b6f327d103226ae7e0206c811 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 16 Nov 2026 16:41:40 +0000 Subject: [PATCH 6/9] ci: say when the queue lanes are missing from the readiness page, and skip other-branch runs Review of the queue import found two gaps against "no GPU row goes missing": - When no queue run was imported, the Release-Readiness table still read "0 failure(s) across all platforms" and GO, and the absence was noted only on the E2E-Support-Matrix page. release-readiness now reads the matrix's absent note and marks the E2E row and adds a warning under the table. The verdict itself is unchanged: absence is still not a failure. - The fetch step took the single newest queue-run-results artifact and then dropped the whole import when its log said another branch, so a later dispatch of another branch hid a valid run of main. It now walks the five newest runs of the last 24 hours and takes the first that tested main. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 45 ++++++++++++++---------- docs/nightly-on-the-queue.md | 10 ++++-- xtask/src/e2e_queue_import.rs | 7 ++-- xtask/src/release_readiness.rs | 62 ++++++++++++++++++++++++++++++++-- xtask/src/workflow_contract.rs | 4 ++- 5 files changed, 102 insertions(+), 26 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3ddce9b2e..7b22f7ccc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1408,31 +1408,40 @@ jobs: repo_id=$(gh api "repos/${GITHUB_REPOSITORY}" --jq .id) || absent "the repository could not be read" since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) - found=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=queue-run-results&per_page=50" | + # Newest first: a run asked to test another branch (a `queue-run.yml` + # dispatch runs main's workflow too) must not hide an older run of main. + candidates=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=queue-run-results&per_page=50" | jq -r --arg since "$since" --argjson repo "$repo_id" ' [.artifacts[] | select(.expired == false) | select(.workflow_run.head_branch == "main" and .workflow_run.head_repository_id == $repo) | select(.created_at >= $since)] - | sort_by(.created_at) | last // empty | .workflow_run.id') || + | sort_by(.created_at) | reverse | .[:5][] | .workflow_run.id') || absent "the queue results could not be listed" - [ -n "$found" ] || + [ -n "$candidates" ] || absent "no queue run on main finished in the last 24 hours (a run still going is not imported)" - # The run of main's workflow can have been asked to test another branch, - # and a run that started a day ago is not tonight's. - started=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${found}" --jq .run_started_at) || - absent "when the newest queue run started could not be read" - started_at=$(date -u -d "$started" +%s 2>/dev/null) && - [ "$started_at" -ge "$(date -u -d '24 hours ago' +%s)" ] || - absent "the newest queue run started more than 24 hours ago" - echo "$started" > queue-results/started.txt - gh run download "$found" --repo "$GITHUB_REPOSITORY" --name queue-run-results --dir queue-results/results || - absent "the queue results of run $found could not be downloaded" - # The log names the branch the run tested and which lanes the queue - # blocks on. Without it nothing is imported. - job=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${found}/jobs" --jq '.jobs[0].id') && - gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log || - rm -f queue-results/job.log + found="" + for candidate in $candidates; do + rm -rf queue-results/results queue-results/job.log + # A run that started a day ago is not tonight's. + started=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}" --jq .run_started_at) || continue + started_at=$(date -u -d "$started" +%s 2>/dev/null) && + [ "$started_at" -ge "$(date -u -d '24 hours ago' +%s)" ] || continue + gh run download "$candidate" --repo "$GITHUB_REPOSITORY" --name queue-run-results --dir queue-results/results || continue + # The log names the branch the run tested and which lanes the queue + # blocks on; a run that tested another branch is skipped, not the end + # of the search. `e2e-queue-import` checks the log again. + job=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}/jobs" --jq '.jobs[0].id') && + gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log && + grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log || continue + found="$candidate" + echo "$started" > queue-results/started.txt + break + done + if [ -z "$found" ]; then + rm -rf queue-results/results queue-results/job.log + absent "none of the newest queue runs of the last 24 hours tested main with a readable log and results" + fi echo "$found" > queue-results/run-id.txt # Turn the run's lanes into reports of this job's own kind, and write the diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index 04ad16f0b..91bcbb1b0 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -219,7 +219,9 @@ queue has (MI325X, MI350X, MI355X) is on them. run that took under about three and a half hours is in the night's report. A run that is still going, that started on an earlier day or that tested another branch is not imported, and neither is a run published again by - `nightly-queue.yml`. + `nightly-queue.yml`. The job looks at the five newest runs of the window and + takes the first that tested `main`, so a run of another branch does not hide + an earlier run of `main`. - **Which lanes, and whether each blocks.** The artifact holds the lanes' files and nothing else about the run, so the job reads the run's job log, whose lane matrix is the only place the queue prints which lanes it ran and which of @@ -255,8 +257,10 @@ queue has (MI325X, MI350X, MI355X) is on them. branch, an artifact or log that cannot be downloaded or an import that fails never fails the nightly report: the page opens with a `Queue lanes` note saying that no queue lanes are in it tonight and why, and the rest is built as - before. The verdict does not take that absence for a failure, so read the note - before reading a `GO`. + before. The verdict does not take that absence for a failure, so the + `Release-Readiness` table says it too: its `E2E` row is marked "queue lanes + missing" and a warning under the table names the reason. Read it before + reading a `GO`: that `GO` does not cover the queue's GPUs. What is public follows from the queue's own rules: the artifact is public and already scanned, only three of its files are used, and the log is the public job diff --git a/xtask/src/e2e_queue_import.rs b/xtask/src/e2e_queue_import.rs index a5f0bc459..1408f0719 100644 --- a/xtask/src/e2e_queue_import.rs +++ b/xtask/src/e2e_queue_import.rs @@ -441,12 +441,15 @@ fn one_line(text: &str) -> String { } const SECTION_HEADING: &str = "## Queue lanes\n\n"; +/// How the "nothing imported" note starts; `release-readiness` looks for it in +/// the matrix so the verdict page can say the queue lanes are missing too. +pub(crate) const ABSENT_LEAD: &str = "_No queue lanes are in this report: "; +pub(crate) const ABSENT_TAIL: &str = ". The GPUs the shared queue tests are not shown tonight._"; /// The section for a nightly with no queue results to import. fn render_absent(reason: &str) -> String { format!( - "{SECTION_HEADING}_No queue lanes are in this report: {}. The GPUs the shared queue \ - tests are not shown tonight._\n\n", + "{SECTION_HEADING}{ABSENT_LEAD}{}{ABSENT_TAIL}\n\n", one_line(reason) ) } diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index 3cdd834df..0b124b243 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -72,6 +72,11 @@ pub struct Facts { /// Whether the nightly or Windows build gate was skipped (a legitimate /// no-op-night outcome, not a failure) rather than actually run. pub build_skipped: bool, + /// Why the shared queue's GPU lanes are not in tonight's E2E report, when + /// the matrix says so. The verdict does not count that absence as a + /// failure, so the table has to say it: the E2E row covers only the lanes + /// that are in the report. + pub queue_absent: Option, pub commits: CommitBuckets, pub known_issues: Vec, } @@ -93,7 +98,7 @@ pub fn run( .with_context(|| format!("reading e2e report {}", e2e_report.display()))?; let e2e_fail_total = total_e2e_failures(&e2e_text); - let facts = assemble_facts( + let mut facts = assemble_facts( since_tag, commits, known_issues, @@ -102,6 +107,7 @@ pub fn run( windows_result, e2e_report_job_result, ); + facts.queue_absent = queue_absent_reason(&e2e_text); // Trailing newline so the file ends like a text file. write_parented(out, &format!("{}\n", serde_json::to_string_pretty(&facts)?))?; @@ -164,6 +170,7 @@ fn assemble_facts( failing_gates, e2e_fail_total, build_skipped: nightly_result == "skipped" || windows_result == "skipped", + queue_absent: None, commits, known_issues, } @@ -415,6 +422,16 @@ fn excerpt(text: &str, max_chars: usize) -> String { /// Parsing the already-rendered rows, rather than re-deriving per-platform /// pass/fail here, keeps this in lockstep with that crate's known-bugs/xfail /// handling instead of duplicating it. +/// The reason the matrix gives for having no queue lanes, if it says so +/// (`e2e_queue_import::render_absent`). +fn queue_absent_reason(markdown: &str) -> Option { + use crate::e2e_queue_import::{ABSENT_LEAD, ABSENT_TAIL}; + markdown.lines().find_map(|line| { + let rest = line.strip_prefix(ABSENT_LEAD)?; + Some(rest.strip_suffix(ABSENT_TAIL).unwrap_or(rest).to_string()) + }) +} + fn total_e2e_failures(markdown: &str) -> Option { // Scope strictly to the summary table's own row range — the document // keeps going after it with an expectation grid and a command-coverage @@ -497,10 +514,13 @@ pub(crate) fn render_table(facts: &Facts) -> String { // the header below so `publish-release-readiness-wiki`'s guard (which // greps for it) only trips on a truly empty/degenerate table. let e2e_failing = facts.failing_gates.iter().any(|g| g.starts_with("E2E")); - let e2e_evidence = match facts.e2e_fail_total { + let mut e2e_evidence = match facts.e2e_fail_total { Some(n) => format!("{n} failure(s) across all platforms"), None => "evidence unreadable".to_string(), }; + if facts.queue_absent.is_some() { + e2e_evidence.push_str(" in the report (queue lanes missing, see below)"); + } out.push_str("| Area | Evidence | Status |\n|---|---|:--:|\n"); let _ = writeln!( @@ -521,6 +541,16 @@ pub(crate) fn render_table(facts: &Facts) -> String { ); out.push('\n'); + if let Some(reason) = &facts.queue_absent { + let _ = writeln!( + out, + "⚠️ **The shared queue's GPU lanes are not in tonight's report** ({}). The E2E row \ + above does not cover them, so a GO here does not cover R9700, MI350P, W7900 or the \ + queue's blocking lanes.\n", + escape_markdown(reason) + ); + } + // The status table above only has room for a glyph per row, which is // ambiguous on its own (e.g. a cancelled E2E lane renders as // "0 failure(s)" with a red ❌, which looks contradictory). Spell out @@ -851,6 +881,34 @@ mod tests { ); } + #[test] + fn the_table_says_when_the_queue_lanes_are_missing() { + let md = format!( + "## Queue lanes\n\n{}{}{}\n\n", + crate::e2e_queue_import::ABSENT_LEAD, + "no queue run on main finished in the last 24 hours", + crate::e2e_queue_import::ABSENT_TAIL + ); + let reason = queue_absent_reason(&md).expect("the note is recognised"); + assert_eq!(reason, "no queue run on main finished in the last 24 hours"); + assert_eq!(queue_absent_reason("| Total | 0 |\n"), None); + let mut facts = assemble_facts( + "v1.0.0", + CommitBuckets::default(), + vec![], + Some(0), + "success", + "success", + "success", + ); + facts.queue_absent = Some(reason); + let table = render_table(&facts); + assert!(table.contains("queue lanes missing"), "{table}"); + assert!(table.contains("not in tonight's report"), "{table}"); + // Not a failure: the verdict stays as the other gates decide it. + assert_eq!(facts.verdict, Verdict::Go); + } + #[test] fn assemble_facts_is_go_when_everything_succeeds() { let facts = assemble_facts( diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index c93a4898b..f502be5f1 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -4446,7 +4446,7 @@ fi // Today's run: not yesterday's, whatever its conclusion was. "date -u -d '24 hours ago'", ".created_at >= $since", - "gh run download \"$found\"", + "gh run download \"$candidate\"", "--name queue-run-results", // A run that started a day ago is not tonight's, whenever it ended. "[ \"$started_at\" -ge \"$(date -u -d '24 hours ago' +%s)\" ]", @@ -4749,6 +4749,7 @@ fi failing_gates: vec!["E2E (report unreadable)".to_string()], e2e_fail_total: None, build_skipped: false, + queue_absent: None, commits: CommitBuckets::default(), known_issues: vec![], }); @@ -4768,6 +4769,7 @@ fi failing_gates: vec![], e2e_fail_total: Some(0), build_skipped: false, + queue_absent: None, commits: CommitBuckets::default(), known_issues: vec![], }); From d6d85e60fe1ad5304ab10ab2967cfa4ab553889d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 16 Nov 2026 17:07:09 +0000 Subject: [PATCH 7/9] ci: address review on the queue import and readiness doc - release_readiness: move queue_absent_reason above the total_e2e_failures doc block so each function keeps its own rustdoc. - nightly: import only a completed queue run (any conclusion), and scan every job of the run for the lane matrix instead of assuming the first. - docs: neutral pool wording, drop the leftover blank line. - workflow_contract: pin the new fetch-step guards. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 22 ++++++++++++++++++---- docs/ci-hardware-testing.md | 3 +-- xtask/src/release_readiness.rs | 20 ++++++++++---------- xtask/src/workflow_contract.rs | 8 ++++++-- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7b22f7ccc..7aca0e072 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1424,16 +1424,30 @@ jobs: for candidate in $candidates; do rm -rf queue-results/results queue-results/job.log # A run that started a day ago is not tonight's. - started=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}" --jq .run_started_at) || continue + run_info=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}" --jq '[.status, .run_started_at] | @tsv') || continue + # A finished run only, red or green: a run still going has a + # partial job log. + [ "${run_info%%$'\t'*}" = "completed" ] || continue + started="${run_info#*$'\t'}" started_at=$(date -u -d "$started" +%s 2>/dev/null) && [ "$started_at" -ge "$(date -u -d '24 hours ago' +%s)" ] || continue gh run download "$candidate" --repo "$GITHUB_REPOSITORY" --name queue-run-results --dir queue-results/results || continue # The log names the branch the run tested and which lanes the queue # blocks on; a run that tested another branch is skipped, not the end # of the search. `e2e-queue-import` checks the log again. - job=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}/jobs" --jq '.jobs[0].id') && - gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log && - grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log || continue + # The run may have several jobs: scan them all for the one that + # prints the lane matrix. + jobs=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}/jobs?per_page=100" --jq '.jobs[].id') || continue + : > queue-results/job.log + for job in $jobs; do + gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log.part 2>/dev/null || continue + if grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log.part; then + mv queue-results/job.log.part queue-results/job.log + break + fi + done + rm -f queue-results/job.log.part + grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log || continue found="$candidate" echo "$started" > queue-results/started.txt break diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index f50d77f5d..ea2b5c480 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -51,7 +51,7 @@ pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in the merge queue; it keeps its `push`/dispatch triggers and also runs nightly -(`e2e-gpu-nightly-strix` on the same DevLab Dispatch pool labels). There is no +(`e2e-gpu-nightly-strix` on the same shared Strix pool labels). There is no W7900 lane in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue nightly's `w7900-dual` lane (see [nightly-on-the-queue.md](nightly-on-the-queue.md)), not on `push`, `release/**` @@ -452,7 +452,6 @@ because it only carries the latest (pre-)release builds and cannot resolve the pinned released versions (`rocm install sdk` fails with "no mutually compatible TheRock ... versions"); those versions are covered by the `release` channel. - `serve` is narrower than `heavy`: it matches only paths that can change serve *behaviour* or the GPU E2E harness (the engines, the serve code path in `apps/rocm`/`apps/rocmd`, `rocm-core`, the e2e-cucumber crate, diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index 0b124b243..a901474fb 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -400,6 +400,16 @@ fn excerpt(text: &str, max_chars: usize) -> String { } } +/// The reason the matrix gives for having no queue lanes, if it says so +/// (`e2e_queue_import::render_absent`). +fn queue_absent_reason(markdown: &str) -> Option { + use crate::e2e_queue_import::{ABSENT_LEAD, ABSENT_TAIL}; + markdown.lines().find_map(|line| { + let rest = line.strip_prefix(ABSENT_LEAD)?; + Some(rest.strip_suffix(ABSENT_TAIL).unwrap_or(rest).to_string()) + }) +} + /// Sum of the `Fail` column across the consolidated E2E markdown's `**Total**` /// row (`crates/e2e-report`'s `consolidated_summary_markdown` format: `| /// Platform | OS | Total | Pass | Fail | Skip | Xfail | Status |`), plus one @@ -422,16 +432,6 @@ fn excerpt(text: &str, max_chars: usize) -> String { /// Parsing the already-rendered rows, rather than re-deriving per-platform /// pass/fail here, keeps this in lockstep with that crate's known-bugs/xfail /// handling instead of duplicating it. -/// The reason the matrix gives for having no queue lanes, if it says so -/// (`e2e_queue_import::render_absent`). -fn queue_absent_reason(markdown: &str) -> Option { - use crate::e2e_queue_import::{ABSENT_LEAD, ABSENT_TAIL}; - markdown.lines().find_map(|line| { - let rest = line.strip_prefix(ABSENT_LEAD)?; - Some(rest.strip_suffix(ABSENT_TAIL).unwrap_or(rest).to_string()) - }) -} - fn total_e2e_failures(markdown: &str) -> Option { // Scope strictly to the summary table's own row range — the document // keeps going after it with an expectation grid and a command-coverage diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f502be5f1..17378b85c 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -4450,8 +4450,12 @@ fi "--name queue-run-results", // A run that started a day ago is not tonight's, whenever it ended. "[ \"$started_at\" -ge \"$(date -u -d '24 hours ago' +%s)\" ]", - // The log says which branch the run tested and which lanes block. - "/logs\" > queue-results/job.log", + // Only a finished run: a run still going has a partial log. + "[ \"${run_info%%$'\\t'*}\" = \"completed\" ]", + // The log says which branch the run tested and which lanes block; + // the run may have several jobs, so every one is scanned. + "actions/runs/${candidate}/jobs?per_page=100", + "/logs\" > queue-results/job.log.part", ] { assert!( fetch.contains(needed), From 4684933598479b7a579e357626d95f10984fadf5 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 16 Nov 2026 17:34:40 +0000 Subject: [PATCH 8/9] ci: keep unnamed queue rows and directories apart; tidy the importer - An unnamed matrix row is now unnamed-row-N and an unnamed artifact directory unnamed-dir-N, so plan() can no longer merge two unrelated lanes into one. - Compile the three static regexes once (LazyLock), stop shadowing plan(), and share one make_dir helper in write_reports. Signed-off-by: Tomas Saaristola --- xtask/src/e2e_queue_import.rs | 54 ++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/xtask/src/e2e_queue_import.rs b/xtask/src/e2e_queue_import.rs index 1408f0719..849d3a715 100644 --- a/xtask/src/e2e_queue_import.rs +++ b/xtask/src/e2e_queue_import.rs @@ -42,6 +42,7 @@ use std::collections::BTreeMap; use std::fmt::Write as _; use std::path::{Path, PathBuf}; +use std::sync::LazyLock; use anyhow::{Context, Result, bail}; use regex::Regex; @@ -99,10 +100,10 @@ pub fn run(args: &Args) -> Result<()> { // A log that yielded no lanes is as good as none for the lanes' // state: every lane is then read as blocking. let have_matrix = !log.lanes.is_empty(); - let plan = plan(&log, scan(queue_dir)?, have_matrix); - write_reports(&plan, queue_dir, &args.artifacts_dir)?; + let lanes_plan = plan(&log, scan(queue_dir)?, have_matrix); + write_reports(&lanes_plan, queue_dir, &args.artifacts_dir)?; render_section( - &plan, + &lanes_plan, args.run_url.as_deref(), args.run_started.as_deref(), have_matrix, @@ -188,9 +189,12 @@ fn is_lane_name(name: &str) -> bool { /// pattern are kept; a line that does not is ignored, so a log that is not what /// was expected yields no lanes rather than text to publish. fn parse_log(log: Option<&str>) -> QueueLog { + static TESTED: LazyLock = LazyLock::new(|| { + Regex::new(r"^rocm-cli (\S+) at ([0-9a-f]{40})$").expect("static pattern") + }); let mut out = QueueLog::default(); let Some(log) = log else { return out }; - let tested = Regex::new(r"^rocm-cli (\S+) at ([0-9a-f]{40})$").expect("static pattern"); + let tested = &*TESTED; let mut in_matrix = false; let mut unnamed = 0; for line in log.lines() { @@ -231,7 +235,7 @@ fn parse_log(log: Option<&str>) -> QueueLog { if !is_lane_name(lane) { unnamed += 1; out.lanes.push(MatrixRow { - lane: format!("unnamed-{unnamed}"), + lane: format!("unnamed-row-{unnamed}"), phase: None, exit: None, blocking: true, @@ -261,7 +265,8 @@ fn parse_log(log: Option<&str>) -> QueueLog { /// The channels with a `results/report.json` under each lane directory of the /// downloaded artifact. A directory (or the final file) that is a symlink is not /// followed. A lane directory with a name that is not a lane name is kept as an -/// `unnamed-N` lane with no reports, so that it shows as a lane without a result +/// `unnamed-dir-N` lane with no reports (a matrix row is `unnamed-row-N`: the two +/// cannot be told to be the same lane, so they are never merged), so that it shows as a lane without a result /// instead of dropping out unseen. fn scan(queue_dir: &Path) -> Result>> { let mut lanes = BTreeMap::new(); @@ -279,7 +284,7 @@ fn scan(queue_dir: &Path) -> Result>> { if !is_lane_name(&lane) { eprintln!("warning: a queue lane directory has an unexpected name; shown as unnamed"); unnamed += 1; - lanes.insert(format!("unnamed-{unnamed}"), Vec::new()); + lanes.insert(format!("unnamed-dir-{unnamed}"), Vec::new()); continue; } let channels = CHANNELS @@ -360,6 +365,10 @@ fn plan(log: &QueueLog, found: BTreeMap>, have_matrix: } /// Write each lane's report directories under `artifacts_dir`. +fn make_dir(dir: &Path) -> Result<()> { + std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display())) +} + fn write_reports(plan: &Plan, queue_dir: &Path, artifacts_dir: &Path) -> Result<()> { for lane in &plan.lanes { for channel in &lane.reported { @@ -368,8 +377,7 @@ fn write_reports(plan: &Plan, queue_dir: &Path, artifacts_dir: &Path) -> Result< !lane.blocking, Some(channel), )); - std::fs::create_dir_all(&dest) - .with_context(|| format!("creating {}", dest.display()))?; + make_dir(&dest)?; let src = queue_dir.join(&lane.lane).join(channel).join("results"); for file in REPORT_FILES { let from = src.join(file); @@ -397,8 +405,7 @@ fn write_reports(plan: &Plan, queue_dir: &Path, artifacts_dir: &Path) -> Result< !lane.blocking, *channel, )); - std::fs::create_dir_all(&dest) - .with_context(|| format!("creating {}", dest.display()))?; + make_dir(&dest)?; // No scenarios at all: the report shows the lane as EMPTY. std::fs::write(dest.join("report.json"), "[]\n") .with_context(|| format!("writing {}", dest.join("report.json").display()))?; @@ -455,9 +462,11 @@ fn render_absent(reason: &str) -> String { } fn is_run_url(url: &str) -> bool { - Regex::new(r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+$") - .expect("static pattern") - .is_match(url) + static RUN_URL: LazyLock = LazyLock::new(|| { + Regex::new(r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+$") + .expect("static pattern") + }); + RUN_URL.is_match(url) } /// The status section: which run was imported, and one row per lane. @@ -467,17 +476,16 @@ fn render_section( run_started: Option<&str>, have_matrix: bool, ) -> String { + static STARTED: LazyLock = LazyLock::new(|| { + Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$").expect("static pattern") + }); let mut out = String::from(SECTION_HEADING); let mut source = String::from("Latest finished run of the shared GPU queue on `main`"); if let Some(url) = run_url.filter(|u| is_run_url(u)) { let id = url.rsplit('/').next().unwrap_or_default(); let _ = write!(source, ": [run {id}]({url})"); } - if let Some(started) = run_started.filter(|s| { - Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$") - .expect("static pattern") - .is_match(s) - }) { + if let Some(started) = run_started.filter(|s| STARTED.is_match(s)) { let _ = write!(source, ", started {started}"); } if let Some(commit) = &plan.commit { @@ -604,7 +612,7 @@ mod tests { let lanes = parse_log(Some(hostile)).lanes; assert_eq!( lanes.iter().map(|l| l.lane.as_str()).collect::>(), - ["unnamed-1", "unnamed-2"] + ["unnamed-row-1", "unnamed-row-2"] ); assert!(lanes.iter().all(|l| l.blocking)); } @@ -896,7 +904,7 @@ mod tests { assert_eq!( log.lanes, vec![MatrixRow { - lane: "unnamed-1".to_owned(), + lane: "unnamed-row-1".to_owned(), phase: None, exit: None, blocking: true, @@ -910,13 +918,13 @@ mod tests { "[]", ); let found = scan(queue.path()).expect("scan"); - assert_eq!(found.get("unnamed-1"), Some(&Vec::new())); + assert_eq!(found.get("unnamed-dir-1"), Some(&Vec::new())); assert_eq!(found.get("mi300x"), Some(&vec!["release"])); let plan = plan(&QueueLog::default(), found, false); let unnamed = plan .lanes .iter() - .find(|l| l.lane == "unnamed-1") + .find(|l| l.lane == "unnamed-dir-1") .expect("lane"); assert!(unnamed.blocking); assert_eq!(unnamed.placeholders, vec![None]); From 7225add8d1e42cba90ed2937932818183dc1a24d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 16 Nov 2026 18:08:43 +0000 Subject: [PATCH 9/9] ci: a missing queue run makes the release-readiness verdict NO-GO A queue that never ran is not a pass. When the nightly report carries the 'no queue lanes' note, the verdict gets the failing gate 'queue lanes absent: ' and reads NO-GO, instead of GO plus a warning. Tests and the docs line follow. Signed-off-by: Tomas Saaristola --- docs/nightly-on-the-queue.md | 8 ++++---- xtask/src/release_readiness.rs | 37 +++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index 91bcbb1b0..e4fae4508 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -257,10 +257,10 @@ queue has (MI325X, MI350X, MI355X) is on them. branch, an artifact or log that cannot be downloaded or an import that fails never fails the nightly report: the page opens with a `Queue lanes` note saying that no queue lanes are in it tonight and why, and the rest is built as - before. The verdict does not take that absence for a failure, so the - `Release-Readiness` table says it too: its `E2E` row is marked "queue lanes - missing" and a warning under the table names the reason. Read it before - reading a `GO`: that `GO` does not cover the queue's GPUs. + before. A queue that never ran is not a pass, so the + release-readiness verdict is `NO-GO` with the failing gate `queue lanes + absent: `; its `E2E` row is marked "queue lanes missing" and a warning + under the table names the reason. What is public follows from the queue's own rules: the artifact is public and already scanned, only three of its files are used, and the log is the public job diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index a901474fb..9d8d47c79 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -107,7 +107,7 @@ pub fn run( windows_result, e2e_report_job_result, ); - facts.queue_absent = queue_absent_reason(&e2e_text); + facts.note_queue_absent(queue_absent_reason(&e2e_text)); // Trailing newline so the file ends like a text file. write_parented(out, &format!("{}\n", serde_json::to_string_pretty(&facts)?))?; @@ -176,6 +176,18 @@ fn assemble_facts( } } +impl Facts { + /// Record that the queue's lanes are not in the report. A queue that never + /// ran is not a pass, so this is a failing gate: the verdict is NO-GO. + fn note_queue_absent(&mut self, reason: Option) { + let Some(reason) = reason else { return }; + self.failing_gates + .push(format!("queue lanes absent: {reason}")); + self.verdict = Verdict::NoGo; + self.queue_absent = Some(reason); + } +} + fn write_parented(path: &Path, contents: &str) -> Result<()> { if let Some(parent) = path.parent() && !parent.as_os_str().is_empty() @@ -545,8 +557,8 @@ pub(crate) fn render_table(facts: &Facts) -> String { let _ = writeln!( out, "⚠️ **The shared queue's GPU lanes are not in tonight's report** ({}). The E2E row \ - above does not cover them, so a GO here does not cover R9700, MI350P, W7900 or the \ - queue's blocking lanes.\n", + above does not cover R9700, MI350P, W7900 or the queue's blocking lanes, and a queue \ + that did not run is not a pass: the verdict is NO-GO.\n", escape_markdown(reason) ); } @@ -901,11 +913,26 @@ mod tests { "success", "success", ); - facts.queue_absent = Some(reason); + assert_eq!(facts.verdict, Verdict::Go); + facts.note_queue_absent(Some(reason.clone())); let table = render_table(&facts); assert!(table.contains("queue lanes missing"), "{table}"); assert!(table.contains("not in tonight's report"), "{table}"); - // Not a failure: the verdict stays as the other gates decide it. + // A queue that never ran is not a pass. + assert_eq!(facts.verdict, Verdict::NoGo); + assert_eq!( + facts.failing_gates, + [format!("queue lanes absent: {reason}")] + ); + assert!(table.contains("NO-GO"), "{table}"); + assert!( + table.contains("queue lanes absent: no queue run"), + "{table}" + ); + // Nothing absent, nothing changed. + facts.verdict = Verdict::Go; + facts.failing_gates.clear(); + facts.note_queue_absent(None); assert_eq!(facts.verdict, Verdict::Go); }