diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5399e19e9..7aca0e072 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -506,257 +506,6 @@ jobs: name: e2e-gpu-${{ matrix.channel }}-report path: tests/e2e-cucumber/results/ - # rad3 R9700 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-rad3 - # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, shared uv - # cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step: - # the report crate asserts the nightly lanes publish the same platforms as the - # per-PR lanes. - e2e-gpu-nightly-rad3: - name: E2E tests (rad3 R9700, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, r9700] - continue-on-error: true - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. The floor stays at the shared 16 GiB default, a flat - # constant rather than a proportion of capacity — high enough to catch a - # leftover serve, low enough that a clean pod passes on the first poll; do - # NOT scale it down for a smaller part. On a 32GB card it still only trips - # past 16 GiB held, so the floor alone would miss a leftover serve holding - # less than that; the process gate is what catches it (#446). Same as the - # per-PR rad3 lane. See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on rad3 (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on the runner-work PVC, which survives the pod - # being scaled away, so the cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm` (EAI-8057). - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-rad3-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - - e2e-gpu-nightly-mi350p: - name: E2E tests (MI350P, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, mi350p] - continue-on-error: true - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. Floor matches the per-PR MI350P lane and the other - # Instinct lane — low enough that a clean pod passes on the first poll, - # and against a pool this size it only trips near exhaustion, so the - # process gate is what actually catches a leftover serve (#446). See - # .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on MI350P (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on a volume that survives the runner being scaled - # away, so the cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-mi350p-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - - # W7900 counterpart to e2e-gpu-nightly. Mirrors the e2e-gpu-w7900 lane (which - # no longer runs on pull_request) in e2e-selfhosted.yml (w7900 label, shared - # uv cache, GPU preflight), with @nightly scenarios enabled. Keep the two in - # step: the report crate asserts the nightly lanes publish the same platforms - # as the lanes in e2e-selfhosted.yml. - e2e-gpu-nightly-w7900: - name: E2E tests (W7900, incl. nightly-only) - timeout-minutes: 90 - runs-on: [self-hosted, linux, w7900] - strategy: - fail-fast: false - matrix: - channel: [release, nightly] - continue-on-error: true - env: - E2E_INCLUDE_NIGHTLY: "1" - E2E_SERVE_TIMEOUT_SECS: "300" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Reclaim the GPU before running: a serve leaked by a killed or - # timed-out prior run (its Drop teardown never executed) keeps an engine - # process on the card and starves this job's serves. Scoped to E2E-owned - # trees: a manual /workload serve is matched only if its command line - # also names an E2E root — see the exposure documented in - # scripts/reclaim-gpu.sh. - - name: Reclaim GPU from stray E2E processes - run: bash scripts/reclaim-gpu.sh - - # Bounded wait for a free GPU, then an assertion that no leaked E2E engine - # process survives. The floor stays at the shared 16 GiB default, which is - # a flat constant rather than a proportion of capacity: it only has to be - # high enough to catch a leftover serve still holding the card, and low - # enough that a clean host passes on the first poll. Same as the per-PR - # W7900 lane — do NOT scale it down for a smaller part. On a 48GB card it - # only trips past ~32 GiB held, so the process gate is what catches a - # leftover serve holding less (#446). See .github/scripts/gpu-preflight.sh. - - name: GPU preflight (bounded wait for an available GPU) - env: - GPU_PREFLIGHT_MIN_FREE_GIB: 16 - run: | - bash .github/scripts/gpu-preflight.sh - - - uses: actions-rust-lang/setup-rust-toolchain@a188e68d0fd2bd36ad51dccb2b07ba6e75105f30 # v2.0.1 - with: - cache: false - - - name: Run full E2E on W7900 (incl. nightly-only) - run: | - # $RUNNER_WORKSPACE is on a volume that outlives any single job, so the - # cargo target dir and shared caches stay warm. - export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" - export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same mount as the runtimes it populates, so uv hardlinks into them - # instead of copying, and inside the uv-cache-gc initContainer's floor - # (see e2e-selfhosted.yml e2e-gpu). - runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" - export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" - export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" - - # See the e2e-gpu lane in e2e-selfhosted.yml for why the - # e2e-test-hooks feature is the floor a pre-built binary must carry - # (xtask builds a superset: it also enables e2e-oom-fault-injection). - cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks - export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" - export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" - - # Pre-warm once, serially, in place (no mv/symlink), and refresh it when - # the channel index has published a newer runtime — the tree is a cache, - # not a one-shot. See `xtask e2e-prewarm`. - HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ - UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ - cargo xtask e2e-prewarm --channel ${{ matrix.channel }} --prewarm-dir "$prewarm" - - cargo xtask e2e - - - name: Upload E2E report - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: e2e-gpu-w7900-${{ matrix.channel }}-report - path: tests/e2e-cucumber/results/ - # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the # runner's storage and runtime setup aligned with e2e-selfhosted.yml's proven @@ -1596,9 +1345,6 @@ jobs: actions: read needs: - e2e-gpu-nightly - - e2e-gpu-nightly-rad3 - - e2e-gpu-nightly-mi350p - - e2e-gpu-nightly-w7900 - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows - e2e-wsl-nightly @@ -1632,12 +1378,126 @@ jobs: pattern: '*-report' path: e2e-artifacts + # The shared GPU queue's nightly (docs/nightly-on-the-queue.md) runs R9700, + # MI350P, W7900 and the Instinct lanes of the queue's fleet on its own + # schedule, and its results are published as one `queue-run-results` + # artifact of that run, not as `*-report` artifacts of this one. So fetch + # the newest such artifact of a run on main that finished, and started, in + # the last 24 hours (a run still going, or one from an earlier night, is + # not today's), and the job log of that run, which is the only place the + # queue prints which branch it tested and which lanes it blocks on. The + # artifact is found by its name, not through the workflow that makes it: + # that workflow runs on a runner no other workflow may name or depend on. + # A conclusion of `failure` is no reason to skip it: a red lane fails the + # queue's run and the artifact is still uploaded. This step never fails the + # job and never stops the report: when there is nothing to import it + # records why, and the next step says so at the top of the page instead of + # leaving the queue's GPUs out without a word. + - name: Fetch the latest queue results + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + mkdir -p queue-results + absent() { + echo "$1" > queue-results/absent.txt + echo "::warning::No queue lanes in this report: $1" + exit 0 + } + repo_id=$(gh api "repos/${GITHUB_REPOSITORY}" --jq .id) || + absent "the repository could not be read" + since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) + # Newest first: a run asked to test another branch (a `queue-run.yml` + # dispatch runs main's workflow too) must not hide an older run of main. + candidates=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=queue-run-results&per_page=50" | + jq -r --arg since "$since" --argjson repo "$repo_id" ' + [.artifacts[] + | select(.expired == false) + | select(.workflow_run.head_branch == "main" and .workflow_run.head_repository_id == $repo) + | select(.created_at >= $since)] + | sort_by(.created_at) | reverse | .[:5][] | .workflow_run.id') || + absent "the queue results could not be listed" + [ -n "$candidates" ] || + absent "no queue run on main finished in the last 24 hours (a run still going is not imported)" + found="" + for candidate in $candidates; do + rm -rf queue-results/results queue-results/job.log + # A run that started a day ago is not tonight's. + run_info=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}" --jq '[.status, .run_started_at] | @tsv') || continue + # A finished run only, red or green: a run still going has a + # partial job log. + [ "${run_info%%$'\t'*}" = "completed" ] || continue + started="${run_info#*$'\t'}" + started_at=$(date -u -d "$started" +%s 2>/dev/null) && + [ "$started_at" -ge "$(date -u -d '24 hours ago' +%s)" ] || continue + gh run download "$candidate" --repo "$GITHUB_REPOSITORY" --name queue-run-results --dir queue-results/results || continue + # The log names the branch the run tested and which lanes the queue + # blocks on; a run that tested another branch is skipped, not the end + # of the search. `e2e-queue-import` checks the log again. + # The run may have several jobs: scan them all for the one that + # prints the lane matrix. + jobs=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}/jobs?per_page=100" --jq '.jobs[].id') || continue + : > queue-results/job.log + for job in $jobs; do + gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log.part 2>/dev/null || continue + if grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log.part; then + mv queue-results/job.log.part queue-results/job.log + break + fi + done + rm -f queue-results/job.log.part + grep -aqE 'rocm-cli main at [0-9a-f]{40}' queue-results/job.log || continue + found="$candidate" + echo "$started" > queue-results/started.txt + break + done + if [ -z "$found" ]; then + rm -rf queue-results/results queue-results/job.log + absent "none of the newest queue runs of the last 24 hours tested main with a readable log and results" + fi + echo "$found" > queue-results/run-id.txt + + # Turn the run's lanes into reports of this job's own kind, and write the + # status section for the top of the matrix (`cargo xtask e2e-queue-import`). + # A lane the queue marks advisory is labelled so and does not count towards + # the release-readiness verdict; a blocking lane with no result does. + - name: Add the queue lanes to the report + continue-on-error: true + run: | + set -uo pipefail + mkdir -p queue-results + if [ -s queue-results/run-id.txt ] && [ -d queue-results/results ]; then + run_id=$(cat queue-results/run-id.txt) + args=(--queue-dir queue-results/results + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${run_id}" + --run-started "$(cat queue-results/started.txt 2>/dev/null)") + [ -f queue-results/job.log ] && args+=(--log queue-results/job.log) + else + args=(--absent "$(cat queue-results/absent.txt 2>/dev/null || echo 'fetching the queue results did not complete')") + fi + cargo xtask e2e-queue-import "${args[@]}" \ + --artifacts-dir e2e-artifacts \ + --section-out queue-results/queue-lanes.md || { + # Half an import is worse than none, and none is said at the top of the page. + rm -rf e2e-artifacts/e2e-queue-* queue-results/queue-lanes.md + echo "::warning::The queue lanes could not be added to the report." + cargo xtask e2e-queue-import --absent "the queue lanes could not be imported" \ + --artifacts-dir e2e-artifacts --section-out queue-results/queue-lanes.md || true + } + + # The queue lanes' status leads the page. The matrix keeps its own first + # table, the one `release-readiness` reads its verdict from. - name: Build consolidated report + step summary run: | mkdir -p consolidated cargo xtask e2e-report \ --artifacts-dir e2e-artifacts \ --html-out consolidated/index.html > consolidated/support-matrix.md + if [ -f queue-results/queue-lanes.md ]; then + cat queue-results/queue-lanes.md consolidated/support-matrix.md > "$RUNNER_TEMP/with-queue.md" + mv "$RUNNER_TEMP/with-queue.md" consolidated/support-matrix.md + fi cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY" # A lane that dies before writing report.json (preflight, build, a crash @@ -1667,9 +1527,6 @@ jobs: NEEDS: ${{ toJSON(needs) }} LANE_REPORTS: | e2e-gpu-nightly=e2e-gpu-release-report,e2e-gpu-nightly-report - e2e-gpu-nightly-rad3=e2e-gpu-rad3-release-report,e2e-gpu-rad3-nightly-report - e2e-gpu-nightly-mi350p=e2e-gpu-mi350p-release-report,e2e-gpu-mi350p-nightly-report - e2e-gpu-nightly-w7900=e2e-gpu-w7900-release-report,e2e-gpu-w7900-nightly-report e2e-gpu-nightly-strix=e2e-gpu-strix-ubuntu-release-report,e2e-gpu-strix-ubuntu-nightly-report e2e-gpu-nightly-strix-windows=e2e-gpu-strix-windows-release-report,e2e-gpu-strix-windows-nightly-report e2e-wsl-nightly=e2e-gpu-strix-wsl-release-report,e2e-gpu-strix-wsl-nightly-report diff --git a/crates/e2e-report/src/consolidated/descriptor.rs b/crates/e2e-report/src/consolidated/descriptor.rs index 7291dd2cb..1185546de 100644 --- a/crates/e2e-report/src/consolidated/descriptor.rs +++ b/crates/e2e-report/src/consolidated/descriptor.rs @@ -48,6 +48,11 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { }, }; + // A lane of the shared GPU queue (see `queue_descriptor`). + if let Some(lane) = core.strip_prefix("queue-") { + return queue_descriptor(lane, channel_from_name); + } + // Legacy `-known-bugs` suffix (retained only as a stable secondary sort key). let (core, known_bugs) = match core.strip_suffix("known-bugs") { Some(rest) => (rest.trim_end_matches('-'), true), @@ -86,6 +91,68 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { } } +/// The marker an advisory queue lane carries in its platform label. +/// +/// A reader of the rendered report, and `xtask release-readiness`, which reads +/// that rendering, tell a lane the queue lets fail from one it does not by it. +pub const ADVISORY_QUEUE_MARKER: &str = "(queue, advisory)"; + +/// The GPU each shared-queue lane stands for, by the lane's name. The queue +/// discovers its lanes from its fleet, so a lane missing from this table is not +/// an error: it renders under its own name in capitals. +const QUEUE_LANE_GPUS: &[(&str, &str)] = &[ + ("mi300x", "MI300X"), + ("mi325x", "MI325X"), + ("mi350p", "MI350P"), + ("mi350x", "MI350X"), + ("mi355x", "MI355X"), + ("r9700s", "R9700"), + ("w7900-dual", "W7900"), +]; + +/// The GPU a shared-queue lane stands for (`w7900-dual` is a W7900), or the +/// lane's own name in capitals for a lane this table does not know yet. +pub fn queue_lane_gpu(lane: &str) -> String { + QUEUE_LANE_GPUS + .iter() + .find(|(name, _)| *name == lane) + .map_or_else(|| lane.to_uppercase(), |(_, gpu)| (*gpu).to_owned()) +} + +/// The artifact name `xtask e2e-queue-import` gives a queue lane's report. +/// +/// For one channel, or none for a lane that reported no channel at all. The lane +/// is named, not the `platform_slug` in its `platform.json`: queue lanes of +/// different models report the same slug (an MI325X lane probes as `mi300x`, an +/// MI355X as `mi350p`), so the slug cannot say which lane a report came from. +/// A lane the queue does not block on carries the `-advisory` suffix. +pub fn queue_artifact_name(lane: &str, advisory: bool, channel: Option<&str>) -> String { + let advisory = if advisory { "-advisory" } else { "" }; + let channel = channel.map_or_else(String::new, |c| format!("-{c}")); + format!("e2e-queue-{lane}{advisory}{channel}-report") +} + +/// Parse the core of a queue lane's artifact name (`mi325x`, `mi325x-advisory`; +/// see [`queue_artifact_name`]). Always Linux: the queue runs Linux pods. +fn queue_descriptor(lane: &str, channel_from_name: Option<&'static str>) -> Descriptor { + let (lane, advisory) = match lane.strip_suffix("-advisory") { + Some(rest) => (rest, true), + None => (lane, false), + }; + let gpu = queue_lane_gpu(lane); + let queue = if advisory { + ADVISORY_QUEUE_MARKER + } else { + "(queue)" + }; + Descriptor { + platform: format!("{gpu} {queue}"), + os: "Linux".to_owned(), + known_bugs: false, + channel_from_name, + } +} + /// The identity-bearing core of an artifact name: `e2e-` prefix and `-report` /// suffix stripped, each relative to the prior result (not `name`), so /// `e2e-report` correctly reduces to `report`, not back to itself. @@ -328,6 +395,33 @@ mod tests { } } + #[test] + fn parse_descriptor_names_each_queue_lane_and_its_gate() { + // Every lane name is a path segment of a queue artifact, so the table is + // what turns `w7900-dual` into the GPU a reader knows. A lane the table + // lacks keeps its own name rather than being guessed at. + for (lane, gpu) in QUEUE_LANE_GPUS { + let blocking = parse_descriptor(&queue_artifact_name(lane, false, Some("release"))); + assert_eq!( + (blocking.platform.as_str(), blocking.os.as_str()), + (format!("{gpu} (queue)").as_str(), "Linux"), + "{lane}" + ); + assert_eq!(blocking.channel_from_name, Some("release")); + let advisory = parse_descriptor(&queue_artifact_name(lane, true, Some("nightly"))); + assert_eq!(advisory.platform, format!("{gpu} {ADVISORY_QUEUE_MARKER}")); + assert_eq!(advisory.channel_from_name, Some("nightly")); + } + let new_lane = parse_descriptor(&queue_artifact_name("mi400x", true, None)); + assert_eq!(new_lane.platform, "MI400X (queue, advisory)"); + assert_eq!(new_lane.channel_from_name, None); + // A queue MI300X lane is not the self-hosted MI300X lane. + assert_ne!( + parse_descriptor("e2e-queue-mi300x-report").platform, + parse_descriptor("e2e-gpu-report").platform + ); + } + #[test] fn parse_descriptor_distinguishes_sdk_version_matrix_legs() { // The sdk_version matrix (ROCMAI-430) uploads non-`current` legs under a diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 911385029..9a51e9036 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -24,7 +24,9 @@ mod sidecars; #[cfg(test)] mod test_support; -pub use descriptor::artifact_name_for_platform_slug; +pub use descriptor::{ + ADVISORY_QUEUE_MARKER, artifact_name_for_platform_slug, queue_artifact_name, queue_lane_gpu, +}; use descriptor::{Descriptor, full_platform_label, parse_descriptor, with_paren_suffix}; use sidecars::{ CommandRecord, ManifestExpectation, PlatformManifest, PlatformVersions, parse_commands, diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 73a702e74..3eaae3947 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -16,7 +16,10 @@ mod single_report; pub use consolidated::html::{RunMeta, generate_consolidated}; pub use consolidated::markdown::consolidated_summary_markdown; -pub use consolidated::{PlatformReports, artifact_name_for_platform_slug, load_platform_reports}; +pub use consolidated::{ + ADVISORY_QUEUE_MARKER, PlatformReports, artifact_name_for_platform_slug, load_platform_reports, + queue_artifact_name, queue_lane_gpu, +}; pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id}; pub use single_report::generate; diff --git a/docs/architecture.md b/docs/architecture.md index a303a91f8..f0f8f00d9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -738,6 +738,7 @@ Repository tasks, run through the workspace alias `cargo xtask `: relea | [`demos.rs`](../xtask/src/demos.rs) | `cargo xtask demos`: builds the demo binaries, starts the mock environment before VHS runs, and renders the documentation GIFs. | flat | | [`e2e.rs`](../xtask/src/e2e.rs) | `cargo xtask e2e`: builds release `rocm` and `rocmd` with the E2E test features and runs the cucumber suite, forwarding extra arguments to cucumber. | flat | | [`e2e_prewarm.rs`](../xtask/src/e2e_prewarm.rs) | `cargo xtask e2e-prewarm`: prepares the shared managed runtime the GPU E2E lanes serve against, refreshing it when the channel index publishes something newer or the required packages differ. | flat | +| [`e2e_queue_import.rs`](../xtask/src/e2e_queue_import.rs) | `cargo xtask e2e-queue-import`: puts a finished shared-queue run's per-lane reports where `e2e-report` reads them, named for the lane and the queue's gate, and writes the queue lanes' status section for the top of the support matrix. | flat | | [`e2e_report.rs`](../xtask/src/e2e_report.rs) | `cargo xtask e2e-report`: merges per-platform `report.json` files into one HTML report and a step-summary matrix, one platform per artifact subdirectory. | flat | | [`engine_stdio_client_contract.rs`](../xtask/src/engine_stdio_client_contract.rs) | Source-scan test that only the shared stdio client in [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) puts a request on an engine child's stdin: a file that both pipes a child's stdin and names an engine envelope is reported unless it is a reviewed `ALLOWED` entry, and a stale entry fails too. | flat | | [`env_mutation_contract.rs`](../xtask/src/env_mutation_contract.rs) | Source-scan test that every test which mutates the process environment serializes itself. | flat | diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 82f89390b..ea2b5c480 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -37,30 +37,35 @@ differs is when each runs and what hardware it covers. | Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? | |---|---|---|---|---| | Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list | -| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | +| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | MI300X (`e2e-gpu-nightly`), Strix Halo Ubuntu (`e2e-gpu-nightly-strix`), Windows (`e2e-gpu-nightly-strix-windows`) and WSL2 (`e2e-wsl-nightly`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event | | Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block | -R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still -covered, but off every PR's critical path, so they moved to the nightly-only -rung instead of being removed outright. +R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125) and moved to +the nightly-only rung. Their self-hosted nightly jobs, and W7900's, have since +been removed from `nightly.yml`: they never got a runner. The queue (below) is +the scheduled run of R9700 (`r9700s`), MI350P (`mi350p`) and W7900 +(`w7900-dual`). The decommissioning of the runners themselves is separate. A pull request now runs on exactly one Linux GPU node and one Windows GPU pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in -the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane -in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue -nightly's `w7900-dual` lane (below; advisory), not on `push`, `release/**` or -dispatch. Its first run, the 11-13 queue nightly -([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), got capacity at once and finished `Done`, exit 0. +the merge queue; it keeps its `push`/dispatch triggers and also runs nightly +(`e2e-gpu-nightly-strix` on the same shared Strix pool labels). There is no +W7900 lane in `e2e-selfhosted.yml` any more: W7900 is covered only by the +scheduled queue nightly's `w7900-dual` lane (see +[nightly-on-the-queue.md](nightly-on-the-queue.md)), not on `push`, `release/**` +or dispatch. Its first run, the 11-13 queue nightly +([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), +got capacity at once and passed (`Done`, exit 0). [`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a shared GPU queue, one per GPU model, so no runner holds a GPU for the length of a job, and publishes the result; [`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) publishes a finished run again. They cover the Linux GPU nightly on both the release and -the nightly channel; native Windows and WSL stay on their dedicated runners, -since the queue runs Linux pods. See +the nightly channel; the Strix Halo lanes stay on the Strix pool, since the +queue has no Strix lane. See [nightly-on-the-queue.md](nightly-on-the-queue.md). ## Nightly stream canary @@ -253,15 +258,28 @@ covers the mock platform; `e2e-selfhosted.yml`'s `e2e-report` covers its self-hosted GPU platforms (a pull request runs only the MI300X and Windows Strix Halo pool lanes among them); `nightly.yml`'s `e2e-report-nightly` covers a strict superset of those -platforms (it also runs `e2e-gpu-rad3` and `e2e-gpu-mi350p`, demoted from -per-PR to nightly-only per ROCMAI-125) with the `@nightly` scenarios -included. Each joins its platforms' +platforms (it adds MI300X's `e2e-gpu-nightly` and the Strix Halo Ubuntu, +Windows and WSL2 nightly lanes) with the `@nightly` scenarios included. The +R9700, MI350P and W7900 nightlies run on the queue, whose lanes the report reads +separately (next paragraph). Each joins its platforms' reports — including partial or failed runs — by scenario id into one HTML report and GitHub step summary. Since ROCMAI-429 the report keys each column by `(platform_slug, channel)` rather than platform alone, so `nightly.yml`'s release/nightly channel matrix renders two columns per platform instead of one channel's result silently overwriting the other. +`e2e-report-nightly` also reads the shared queue's nightly (below), which runs +R9700, MI350P, W7900 and the Instinct lanes of the queue's fleet. It fetches the +`queue-run-results` artifact of the newest queue run that tested `main` and +finished, and started, in the last 24 hours, and `cargo xtask e2e-queue-import` +puts each lane's reports among the others, as `MI325X (queue) Linux (release)` +and the like. A lane the queue does not block on is labelled `(queue, advisory)` +and does not count towards the release-readiness verdict; a blocking lane that +came back with no result is shown as `EMPTY` and does. The page opens with a +`Queue lanes` table. A missing or late queue run never fails the report: the +page says no queue lanes are in it. See +[The support matrix and the wiki](nightly-on-the-queue.md#the-support-matrix-and-the-wiki). + On `main`, the `publish-e2e-wiki` job in `nightly.yml` also copies the consolidated nightly matrix to the `E2E-Support-Matrix` page of the repository wiki, overwriting it each night (the page header names the run, commit and @@ -323,21 +341,16 @@ unlike `publish-e2e-wiki`, it still runs (and publishes an explicit outright, instead of leaving the wiki showing a stale verdict with no sign it's stale. -The lane artifacts are named canonically (`e2e-gpu-mi350p-report`, -`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`, `e2e-gpu-rad3-report`, +The lane artifacts are named canonically (`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`, `e2e-gpu-report`, `e2e-gpu-strix-ubuntu-n-1-report`, `e2e-gpu-strix-ubuntu-n-2-report`, `e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`, `e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`, -`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, -`e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and +`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, `e2e-report`) in `ci.yml` and `e2e-selfhosted.yml`, because the report derives each platform's name and OS from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg -artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900` -are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any -more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they -don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel: +artifact keeps its pre-matrix name unchanged. `nightly.yml`'s self-hosted lanes run a `channel: [release, nightly]` matrix and append the channel as the final segment before the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report strips that segment before matching, so the platform/OS derivation is @@ -438,8 +451,6 @@ combinations on that lane: the `nightly` channel is excluded from `n-1`/`n-2`, because it only carries the latest (pre-)release builds and cannot resolve the pinned released versions (`rocm install sdk` fails with "no mutually compatible TheRock ... versions"); those versions are covered by the `release` channel. -`rad3`/`mi350p` are nightly-only (above) and don't run this matrix -at all. `serve` is narrower than `heavy`: it matches only paths that can change serve *behaviour* or the GPU E2E harness (the engines, the diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index c8a3bdbb6..91bcbb1b0 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -28,10 +28,10 @@ The lane set is whatever GPU models the queue has. The 11-13 run the only scheduled coverage of that hardware, since `e2e-selfhosted.yml` no longer has a W7900 lane. -**The remaining gap is native Windows and WSL.** The queue runs Linux pods, so -the native Windows and WSL lanes stay on their dedicated runners, in `nightly.yml` -and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); nothing -here replaces them. Which checks are required before merging is a repository +**The remaining gap is Strix Halo.** The queue has no Strix lane and runs Linux +pods, so the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool, +in `nightly.yml` and [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml); +nothing here replaces them. Which checks are required before merging is a repository setting, outside this page. This page is the **contract** between this repository and the queue side: what @@ -135,10 +135,15 @@ model downloads are left to the tools doing them, so a model that downloads more slowly than its scenario waits fails that scenario. The lane fetches only the host's crates, so it refuses to start with `CARGO_BUILD_TARGET` set. -The queue's results are an additional signal, not a replacement: the lanes -write only into the queue artifact, and the `e2e-gpu--report` -artifacts that `nightly.yml`'s consolidated report reads are not produced from -them. +The queue is the only scheduled run of the R9700, MI350P and W7900 lanes: their +self-hosted nightly jobs were removed from `nightly.yml`. The MI300X nightly +job is still there, and the queue's `mi300x` lane is an additional signal to it. +The lanes write only into the queue artifact, and the +`e2e-gpu--report` artifacts that `nightly.yml`'s consolidated report +reads are not produced from them. The report fetches the queue's artifact +itself instead, so the support matrix, the `E2E-Support-Matrix` and +`Release-Readiness` wiki pages and the release-readiness verdict keep every +GPU the queue tests: see [The support matrix and the wiki](#the-support-matrix-and-the-wiki). ## Output @@ -192,6 +197,75 @@ that line is what counts: a blocking lane without a result fails the run. result is not a pass, but they point at queue capacity or infrastructure rather than at a regression. +## The support matrix and the wiki + +`nightly.yml`'s `e2e-report-nightly` job builds the consolidated support +matrix, which `publish-e2e-wiki` publishes as the `E2E-Support-Matrix` wiki page +and `release-readiness-nightly` reads its GO/NO-GO verdict from (see +[ci-hardware-testing.md](ci-hardware-testing.md)). It also reads the queue's +lanes, from the queue's own results, so retiring a self-hosted nightly lane +that the queue covers drops no GPU from either page, and a GPU that only the +queue has (MI325X, MI350X, MI355X) is on them. + +- **Which run.** The newest `queue-run-results` artifact of a run on this + repository's `main` that finished, and started, in the last 24 hours, + whatever the run's conclusion was: a red lane fails the run, and its artifact + is still uploaded. The job finds the artifact by name, not through the + workflow that makes it, because no other workflow may name that one (see + below). The run also has to have tested `main`: a run started from `main` can + be asked to test another branch (the `branch` input), and the job log names + the branch it did. `queue-run.yml` starts at 02:17 UTC and the nightly + starts at 06:00 UTC, its report job after its GPU lanes have finished, so a + run that took under about three and a half hours is in the night's report. A + run that is still going, that started on an earlier day or that tested + another branch is not imported, and neither is a run published again by + `nightly-queue.yml`. The job looks at the five newest runs of the window and + takes the first that tested `main`, so a run of another branch does not hide + an earlier run of `main`. +- **Which lanes, and whether each blocks.** The artifact holds the lanes' + files and nothing else about the run, so the job reads the run's job log, whose + lane matrix is the only place the queue prints which lanes it ran and which of + them can fail the run (the `GATE` column). Only the tested branch and commit + and each lane's name, phase, exit status and gate are taken from it, each + checked against a strict pattern first. A lane whose name is not a plain + lowercase word (letters, digits and dashes, and not ending in `-advisory`, + `-release` or `-nightly`) is shown as `unnamed-1` and so on, as a blocking + lane with no result. Without the log nothing is imported. +- **What is imported.** For each lane and channel, + `//results/{report.json,platform.json,commands.jsonl}`, the + three files the report reads, as an artifact named + `e2e-queue-[-advisory]--report`. The `lane.log`, the JUnit and + HTML output and the harness diagnostics are not copied, and nothing the + queue's scan withheld is brought back. The lane's `platform_slug` is replaced + by `queue--`: the slug is the report's column identity, and + queue lanes of different models report the same one (an MI325X lane probes as + `mi300x`, an MI350X or MI355X lane as `mi350p`). The lane command therefore + has to keep writing those files to `/out//results`, as + [`scripts/run-e2e-lane.sh`](../scripts/run-e2e-lane.sh) does. +- **How it reads.** A queue row is labelled `MI325X (queue) Linux (release)`, + so it is not confused with the self-hosted MI300X lane, and a lane the queue + does not block on is `MI325X (queue, advisory) Linux (release)`. The page + opens with a `Queue lanes` table: one row per lane the queue ran, with its + gate, phase, exit status and the channels that reported. +- **What counts towards the verdict.** The same as for the queue's own run. A + blocking lane's failures are in the `E2E` failure count. An advisory lane's are + shown but left out of it. A lane that came back with no result, or whose + channel is missing after it did not finish cleanly, is shown as `EMPTY` and + counts for a blocking lane, because a blocking lane without a result is not + a pass. +- **When there is nothing to import.** No run in the window, a run of another + branch, an artifact or log that cannot be downloaded or an import that fails + never fails the nightly report: the page opens with a `Queue lanes` note + saying that no queue lanes are in it tonight and why, and the rest is built as + before. The verdict does not take that absence for a failure, so the + `Release-Readiness` table says it too: its `E2E` row is marked "queue lanes + missing" and a warning under the table names the reason. Read it before + reading a `GO`: that `GO` does not cover the queue's GPUs. + +What is public follows from the queue's own rules: the artifact is public and +already scanned, only three of its files are used, and the log is the public job +log. + ## What the workflow requires - a self-hosted Linux runner registered with the `queue-report` label alone, diff --git a/docs/testing.md b/docs/testing.md index 127d45c6b..31af227a3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3067,11 +3067,12 @@ The user-visible side is `serve-55` (`@id:serve-small-assistant-short-name`): It carries `@requires-gpu` and deliberately no `@requires-engine`, because the engine is the CLI's to pick here — so the lane that exercises the Lemonade half is whichever GPU lane defaults to Lemonade (the Strix Halo lanes, and W7900 on -the queue nightly only) and the vLLM half falls to the Instinct lane. Of those, -MI300X and the Strix Halo Windows and WSL2 lanes run on pull requests -([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when the `serve` -path filter matches; W7900 has no lane there any more) but none of their check names is a required status check, -so the two unit tests above are the blocking per-PR coverage. The scenario +the queue nightly only) and the vLLM half falls to the Instinct lane. Of all +these lanes only MI300X and the Strix Halo Windows and WSL2 lanes run on pull +requests ([e2e-selfhosted.yml](../.github/workflows/e2e-selfhosted.yml), when +the `serve` path filter matches; W7900 has no per-PR lane any more and runs only +on the queue nightly), and none of their check names is a required status +check, so the two unit tests above are the blocking per-PR coverage. The scenario carries no [expectations.toml](../tests/e2e-cucumber/expectations.toml) row, unlike its managed-Lemonade siblings on the Linux Strix lane: whether it shares their known serve failure there has not been observed, and a row claiming a diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d005021bc..8723a3c01 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -194,18 +194,16 @@ Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to skip there. -The nightly workflow covers a strict superset of the hardware in the table -above, as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-rad3`, -`e2e-gpu-nightly-mi350p`, `e2e-gpu-nightly-w7900`, `e2e-gpu-nightly-strix`, +The nightly workflow runs the MI300X and the Strix Halo hardware from the +table above as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-strix`, `e2e-gpu-nightly-strix-windows`, `e2e-wsl-nightly`) with `E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform -grid. `e2e-gpu-nightly-rad3` and `e2e-gpu-nightly-mi350p` have no per-PR -counterpart — both were demoted from per-PR to nightly-only per ROCMAI-125 — -so nightly is where their regression coverage now lives. The shared -large-model scenario follows each lane's serve engine: it serves -`Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), and the -`unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified on -Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). +grid. R9700, MI350P and W7900 no longer have a self-hosted nightly lane in +`nightly.yml`: they run on the queue ([nightly-on-the-queue.md](../../docs/nightly-on-the-queue.md)), +and the nightly report adds the queue's lanes to that grid. The shared large-model scenario follows each lane's serve engine: it +serves `Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), +and the `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint (hardware-verified +on Strix Halo) through Lemonade on every other lane (Strix Halo, R9700, W7900). Use the self-hosted E2E workflow dispatch to run either model independently on a ref (the GPU platform / `include_nightly` / `name_filter` inputs live on diff --git a/xtask/src/e2e_queue_import.rs b/xtask/src/e2e_queue_import.rs new file mode 100644 index 000000000..849d3a715 --- /dev/null +++ b/xtask/src/e2e_queue_import.rs @@ -0,0 +1,942 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Bring a finished run of the shared GPU queue into the nightly's consolidated +//! E2E report, so the support matrix and the release-readiness verdict derived +//! from it keep covering the GPUs the queue tests. +//! +//! `nightly.yml`'s `e2e-report-nightly` merges one `report.json` per +//! `e2e-*-report` artifact. The queue publishes a different layout: one +//! directory per lane in its `queue-run-results` artifact, holding +//! `/results/{report.json,platform.json,commands.jsonl,...}` as +//! `scripts/run-e2e-lane.sh` writes them (docs/nightly-on-the-queue.md). This +//! turns the one into the other: +//! +//! * every lane's reports are copied to `/e2e-queue-[-advisory]--report/`, +//! named so `e2e_report` labels them by lane and gate. Only the three files the +//! report reads are copied; `lane.log`, the JUnit and HTML output and the +//! harness diagnostics stay where they are. +//! * the report's `platform_slug` is replaced by the lane's name and channel. The +//! slug is what the report keys a grid column by, and queue lanes of different +//! models report one slug (an MI325X lane probes as `mi300x`, an MI355X as +//! `mi350p`), which would fold them, and the self-hosted MI300X lane, into one +//! column. The channel is there because a lane's manifest names the channel of +//! the runtime it found, which need not be the one the lane was told to run. +//! * which lanes exist and which of them the queue blocks on is read from the +//! run's lane matrix in its job log, the only place either is published. A lane +//! the queue marks advisory is labelled so, and `release-readiness` leaves it +//! out of its verdict. Anything this cannot read counts as blocking. +//! * a lane that reported nothing, or that did not finish cleanly and lacks a +//! channel, gets an empty report for it, which the report shows as `EMPTY` and +//! `release-readiness` counts as a failure for a blocking lane: a lane without +//! a result is not a pass. +//! * a short status table of every lane is written for the top of the page. +//! +//! Nothing is imported unless the caller found a run: with `--absent ` +//! only the status section is written, saying so, and the nightly report goes on +//! without queue lanes. The repository is public, so everything written is either +//! copied from the queue's own scanned files or built from fields this checks +//! against a strict pattern first. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; + +use anyhow::{Context, Result, bail}; +use regex::Regex; + +/// The channels `scripts/run-e2e-lane.sh` runs, in the order it runs them. +const CHANNELS: [&str; 2] = ["release", "nightly"]; + +/// The files of a lane's `/results` the consolidated report reads. +const REPORT_FILES: [&str; 3] = ["report.json", "platform.json", "commands.jsonl"]; + +#[derive(clap::Args)] +pub struct Args { + /// Directory the queue run's `queue-run-results` artifact was downloaded + /// into: one subdirectory per lane. + #[arg(long, requires = "run_url", conflicts_with = "absent")] + queue_dir: Option, + /// The queue run's job log, which prints the branch the run tested and its + /// lane matrix. Without it, or when it names a branch other than `main`, + /// nothing is imported. + #[arg(long, requires = "queue_dir")] + log: Option, + /// The queue run's URL, linked from the status section. + #[arg(long)] + run_url: Option, + /// When the queue run started, as the API reports it. + #[arg(long, requires = "queue_dir")] + run_started: Option, + /// Import nothing and say why: no queue run was found, or its results could + /// not be fetched. + #[arg( + long, + conflicts_with = "queue_dir", + required_unless_present = "queue_dir" + )] + absent: Option, + /// Where the per-lane report directories go: the directory + /// `xtask e2e-report` reads. + #[arg(long)] + artifacts_dir: PathBuf, + /// Where to write the status section, for the top of the support matrix. + #[arg(long)] + section_out: PathBuf, +} + +pub fn run(args: &Args) -> Result<()> { + let section = match (&args.queue_dir, &args.absent) { + (Some(queue_dir), _) => { + let log = args.log.as_ref().and_then(|path| { + std::fs::read_to_string(path) + .map_err(|e| eprintln!("warning: queue job log {}: {e}", path.display())) + .ok() + }); + let log = parse_log(log.as_deref()); + if log.branch.as_deref() == Some("main") { + // A log that yielded no lanes is as good as none for the lanes' + // state: every lane is then read as blocking. + let have_matrix = !log.lanes.is_empty(); + let lanes_plan = plan(&log, scan(queue_dir)?, have_matrix); + write_reports(&lanes_plan, queue_dir, &args.artifacts_dir)?; + render_section( + &lanes_plan, + args.run_url.as_deref(), + args.run_started.as_deref(), + have_matrix, + ) + } else { + // The run is of the workflow on `main`, but may have been asked + // to test another branch, which is no night's result for `main`. + // The log is the only place that says which, so without it + // nothing is imported. + render_absent( + "the newest queue run did not test main, or its job log could not be read \ + to tell", + ) + } + } + (None, Some(reason)) => render_absent(reason), + (None, None) => bail!("one of --queue-dir and --absent is required"), + }; + if let Some(parent) = args.section_out.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + std::fs::write(&args.section_out, section) + .with_context(|| format!("writing {}", args.section_out.display()))?; + Ok(()) +} + +/// What the queue run's job log says about the run. +#[derive(Debug, Default, PartialEq, Eq)] +struct QueueLog { + /// The branch the run tested and the commit every lane tested, from the + /// report's first line. The branch is not the one the workflow ran on: a run + /// started from `main` can be asked to test another branch. + branch: Option, + commit: Option, + lanes: Vec, +} + +/// One row of the lane matrix the queue prints. +#[derive(Debug, PartialEq, Eq)] +struct MatrixRow { + lane: String, + /// The lane's phase (`Done`, `Failed`, ...), when the row had the usual shape. + phase: Option, + /// The lane command's exit status, or `-` when it never ran. + exit: Option, + /// Whether the queue lets this lane fail the run. Anything but `advisory` is + /// read as blocking. + blocking: bool, +} + +impl MatrixRow { + /// A lane the queue reports as finished with exit status zero. + fn is_clean(&self) -> bool { + self.phase.as_deref() == Some("Done") && self.exit.as_deref() == Some("0") + } +} + +/// A lane name is a path segment and a word in an artifact name: lowercase +/// letters, digits and dashes only. +/// +/// A name ending in a word the artifact names use for something else +/// (`-advisory`, `-release`, `-nightly`) is refused too: it would be read back +/// as a gate or a channel. +fn is_lane_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 40 + && !name.starts_with('-') + && !["-advisory", "-release", "-nightly"] + .iter() + .any(|suffix| name.ends_with(suffix)) + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// Read the lane matrix and the tested commit out of the run's job log. +/// +/// The publisher prints its report with a `| ` prefix on every line, which the +/// runner puts a timestamp in front of. Only the fields that match a strict +/// pattern are kept; a line that does not is ignored, so a log that is not what +/// was expected yields no lanes rather than text to publish. +fn parse_log(log: Option<&str>) -> QueueLog { + static TESTED: LazyLock = LazyLock::new(|| { + Regex::new(r"^rocm-cli (\S+) at ([0-9a-f]{40})$").expect("static pattern") + }); + let mut out = QueueLog::default(); + let Some(log) = log else { return out }; + let tested = &*TESTED; + let mut in_matrix = false; + let mut unnamed = 0; + for line in log.lines() { + let line = line.trim_start_matches('\u{feff}'); + // ` | `: the part after the first `| `. + let Some((_, body)) = line.split_once("| ") else { + // A blank report line is `|` alone; either way the matrix is over. + if line.trim_end().ends_with('|') { + in_matrix = false; + } + continue; + }; + let body = body.trim_end(); + if let Some(caps) = tested.captures(body) { + out.branch = Some(caps[1].to_owned()); + out.commit = Some(caps[2].to_owned()); + } + let tokens: Vec<&str> = body.split_whitespace().collect(); + if tokens.first() == Some(&"LANE") && tokens.last() == Some(&"GATE") { + in_matrix = true; + continue; + } + if !in_matrix { + continue; + } + if body.is_empty() || tokens.first() == Some(&"SUMMARY:") { + in_matrix = false; + continue; + } + let (Some(lane), Some(gate)) = (tokens.first(), tokens.last()) else { + continue; + }; + if out.lanes.iter().any(|row| row.lane == *lane) { + continue; + } + // A row whose lane cannot be named safely still counts: as a lane that + // reported nothing, and blocking, since its gate cannot be trusted either. + if !is_lane_name(lane) { + unnamed += 1; + out.lanes.push(MatrixRow { + lane: format!("unnamed-row-{unnamed}"), + phase: None, + exit: None, + blocking: true, + }); + continue; + } + // LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE + let shaped = tokens.len() == 7; + let phase = shaped + .then(|| tokens[2]) + .filter(|p| p.bytes().all(|b| b.is_ascii_alphabetic())) + .map(str::to_owned); + let exit = shaped + .then(|| tokens[3]) + .filter(|e| *e == "-" || (e.len() <= 4 && e.bytes().all(|b| b.is_ascii_digit()))) + .map(str::to_owned); + out.lanes.push(MatrixRow { + lane: (*lane).to_owned(), + phase, + exit, + blocking: *gate != "advisory", + }); + } + out +} + +/// The channels with a `results/report.json` under each lane directory of the +/// downloaded artifact. A directory (or the final file) that is a symlink is not +/// followed. A lane directory with a name that is not a lane name is kept as an +/// `unnamed-dir-N` lane with no reports (a matrix row is `unnamed-row-N`: the two +/// cannot be told to be the same lane, so they are never merged), so that it shows as a lane without a result +/// instead of dropping out unseen. +fn scan(queue_dir: &Path) -> Result>> { + let mut lanes = BTreeMap::new(); + let mut unnamed = 0; + let entries = std::fs::read_dir(queue_dir) + .with_context(|| format!("reading the queue results in {}", queue_dir.display()))?; + for entry in entries { + let entry = entry.context("reading a queue results entry")?; + if !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let Some(lane) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if !is_lane_name(&lane) { + eprintln!("warning: a queue lane directory has an unexpected name; shown as unnamed"); + unnamed += 1; + lanes.insert(format!("unnamed-dir-{unnamed}"), Vec::new()); + continue; + } + let channels = CHANNELS + .into_iter() + .filter(|channel| regular_file(&entry.path().join(channel).join("results/report.json"))) + .collect(); + lanes.insert(lane, channels); + } + Ok(lanes) +} + +fn regular_file(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file()) +} + +/// What one lane contributes to the report. +#[derive(Debug, PartialEq, Eq)] +struct LanePlan { + lane: String, + blocking: bool, + phase: Option, + exit: Option, + /// Channels whose reports are copied. + reported: Vec<&'static str>, + /// Channels (or, for `None`, the lane as a whole) an empty report stands in for. + placeholders: Vec>, +} + +#[derive(Debug, PartialEq, Eq)] +struct Plan { + commit: Option, + lanes: Vec, +} + +/// Decide, lane by lane, what goes into the report. `have_matrix` says whether the +/// lane matrix could be read at all: without it nothing is known of a lane's +/// state beyond its files. +fn plan(log: &QueueLog, found: BTreeMap>, have_matrix: bool) -> Plan { + let mut names: Vec<&String> = found + .keys() + .chain(log.lanes.iter().map(|r| &r.lane)) + .collect(); + names.sort(); + names.dedup(); + let mut lanes = Vec::new(); + for name in names { + let row = log.lanes.iter().find(|r| &r.lane == name); + let reported = found.get(name).cloned().unwrap_or_default(); + let clean = row.is_some_and(MatrixRow::is_clean); + let mut placeholders = Vec::new(); + if reported.is_empty() { + // Nothing came back from this lane, whatever it was told to run. + placeholders.push(None); + } else if have_matrix && !clean { + // It did not finish cleanly, and a channel it should have run left + // no report. A lane that finished cleanly may have been told to run + // one channel only. + placeholders.extend( + CHANNELS + .into_iter() + .filter(|channel| !reported.contains(channel)) + .map(Some), + ); + } + lanes.push(LanePlan { + lane: name.clone(), + blocking: row.is_none_or(|r| r.blocking), + phase: row.and_then(|r| r.phase.clone()), + exit: row.and_then(|r| r.exit.clone()), + reported, + placeholders, + }); + } + Plan { + commit: log.commit.clone(), + lanes, + } +} + +/// Write each lane's report directories under `artifacts_dir`. +fn make_dir(dir: &Path) -> Result<()> { + std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display())) +} + +fn write_reports(plan: &Plan, queue_dir: &Path, artifacts_dir: &Path) -> Result<()> { + for lane in &plan.lanes { + for channel in &lane.reported { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + Some(channel), + )); + make_dir(&dest)?; + let src = queue_dir.join(&lane.lane).join(channel).join("results"); + for file in REPORT_FILES { + let from = src.join(file); + if !regular_file(&from) { + continue; + } + if file == "platform.json" { + // A manifest that does not parse is left out: the report then + // names the lane as having no usable manifest. + let text = std::fs::read_to_string(&from) + .with_context(|| format!("reading {}", from.display()))?; + if let Some(text) = with_lane_slug(&text, &lane.lane, channel) { + std::fs::write(dest.join(file), text) + .with_context(|| format!("writing {}", dest.join(file).display()))?; + } + } else { + std::fs::copy(&from, dest.join(file)) + .with_context(|| format!("copying {}", from.display()))?; + } + } + } + for channel in &lane.placeholders { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + *channel, + )); + make_dir(&dest)?; + // No scenarios at all: the report shows the lane as EMPTY. + std::fs::write(dest.join("report.json"), "[]\n") + .with_context(|| format!("writing {}", dest.join("report.json").display()))?; + } + } + Ok(()) +} + +/// `platform.json` with its `platform_slug` (and the copy under `capability`) +/// set to the lane's name and channel, or `None` when it is not a JSON object. +fn with_lane_slug(text: &str, lane: &str, channel: &str) -> Option { + let mut manifest: serde_json::Value = serde_json::from_str(text).ok()?; + let slug = serde_json::Value::String(format!("queue-{lane}-{channel}")); + let object = manifest.as_object_mut()?; + object.insert("platform_slug".to_owned(), slug.clone()); + if let Some(capability) = object + .get_mut("capability") + .and_then(serde_json::Value::as_object_mut) + { + capability.insert("platform_slug".to_owned(), slug); + } + serde_json::to_string_pretty(&manifest).ok() +} + +/// Text that fits in a table cell, from a reason this repository wrote itself. +fn one_line(text: &str) -> String { + text.chars() + .map(|c| { + if c.is_ascii_graphic() || c == ' ' { + c + } else { + ' ' + } + }) + .filter(|c| *c != '|' && *c != '`') + .take(200) + .collect::() + .trim() + .to_owned() +} + +const SECTION_HEADING: &str = "## Queue lanes\n\n"; +/// How the "nothing imported" note starts; `release-readiness` looks for it in +/// the matrix so the verdict page can say the queue lanes are missing too. +pub(crate) const ABSENT_LEAD: &str = "_No queue lanes are in this report: "; +pub(crate) const ABSENT_TAIL: &str = ". The GPUs the shared queue tests are not shown tonight._"; + +/// The section for a nightly with no queue results to import. +fn render_absent(reason: &str) -> String { + format!( + "{SECTION_HEADING}{ABSENT_LEAD}{}{ABSENT_TAIL}\n\n", + one_line(reason) + ) +} + +fn is_run_url(url: &str) -> bool { + static RUN_URL: LazyLock = LazyLock::new(|| { + Regex::new(r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+$") + .expect("static pattern") + }); + RUN_URL.is_match(url) +} + +/// The status section: which run was imported, and one row per lane. +fn render_section( + plan: &Plan, + run_url: Option<&str>, + run_started: Option<&str>, + have_matrix: bool, +) -> String { + static STARTED: LazyLock = LazyLock::new(|| { + Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$").expect("static pattern") + }); + let mut out = String::from(SECTION_HEADING); + let mut source = String::from("Latest finished run of the shared GPU queue on `main`"); + if let Some(url) = run_url.filter(|u| is_run_url(u)) { + let id = url.rsplit('/').next().unwrap_or_default(); + let _ = write!(source, ": [run {id}]({url})"); + } + if let Some(started) = run_started.filter(|s| STARTED.is_match(s)) { + let _ = write!(source, ", started {started}"); + } + if let Some(commit) = &plan.commit { + let _ = write!(source, ", tested commit `{}`", &commit[..12]); + } + let _ = writeln!(out, "{source}.\n"); + if !have_matrix { + out.push_str( + "_The run's lane matrix could not be read, so every lane is shown as blocking._\n\n", + ); + } + out.push_str("| Lane | GPU | Gate | Phase | Exit | Channels reported |\n"); + out.push_str("|---|---|---|---|---|---|\n"); + for lane in &plan.lanes { + let channels = if lane.reported.is_empty() { + "none".to_owned() + } else { + lane.reported.join(", ") + }; + let _ = writeln!( + out, + "| {} | {} | {} | {} | {} | {channels} |", + lane.lane, + e2e_report::queue_lane_gpu(&lane.lane), + if lane.blocking { + "blocking" + } else { + "advisory" + }, + lane.phase.as_deref().unwrap_or("unknown"), + lane.exit.as_deref().unwrap_or("unknown"), + ); + } + let _ = write!( + out, + "\nThe queue's own gate decides which lanes count: a lane it marks advisory is labelled \ + `{}` below and is left out of the release-readiness verdict, and a blocking lane that \ + reported no result is shown as `EMPTY` and counts as a failure there.\n\n", + e2e_report::ADVISORY_QUEUE_MARKER + ); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The lane matrix of the 2026-11-13 scheduled run, as its public job log + /// printed it. + const LOG: &str = "\u{feff}2026-11-13T02:22:14.0268419Z started gh-43351651832-1; waiting for it to finish +2026-11-13T03:45:26.1317642Z gh-43351651832-1 still running (10 min, 0 of 7 lane(s) finished) +2026-11-13T03:45:26.1323838Z request gh-43351651832-1, finished 2026-11-13T03:45:26Z +2026-11-13T03:45:26.1361153Z | image: \"rocm/dev-ubuntu-24.04:7.2.4\" +2026-11-13T03:45:26.1361751Z | blocking: \"mi300x,r9700s\" +2026-11-13T03:45:26.1362299Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351 +2026-11-13T03:45:26.1362503Z | lanes: mi300x mi325x mi350p mi350x mi355x r9700s w7900-dual +2026-11-13T03:45:26.1362739Z | +2026-11-13T03:45:26.1362910Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE +2026-11-13T03:45:26.1363201Z | mi300x rocm-cli-gh-43351651832-1-mi300x Failed 1 Exported 0m block +2026-11-13T03:45:26.1363511Z | mi325x rocm-cli-gh-43351651832-1-mi325x Failed 1 Exported 0m advisory +2026-11-13T03:45:26.1364685Z | r9700s rocm-cli-gh-43351651832-1-r9700s Failed - - 0m block +2026-11-13T03:45:26.1364991Z | w7900-dual rocm-cli-gh-43351651832-1-w7900-dual Done 0 Exported 0m advisory +2026-11-13T03:45:26.1365442Z | +2026-11-13T03:45:26.1365634Z | SUMMARY: 7 lane(s), 1 passed, 2 failed, 0 incomplete, 0 missing, 4 advisory +2026-11-13T03:45:26.1365895Z | lane mi300x: 7 file(s) collected +2026-11-13T03:45:26.1365895Z | mi999x not a row after the matrix Failed 1 Exported 0m block +"; + + fn row(lane: &str, phase: &str, exit: &str, blocking: bool) -> MatrixRow { + MatrixRow { + lane: lane.to_owned(), + phase: Some(phase.to_owned()), + exit: Some(exit.to_owned()), + blocking, + } + } + + #[test] + fn the_matrix_and_the_tested_commit_are_read_from_the_job_log() { + let log = parse_log(Some(LOG)); + assert_eq!(log.branch.as_deref(), Some("main")); + assert_eq!( + log.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + assert_eq!( + log.lanes, + vec![ + row("mi300x", "Failed", "1", true), + row("mi325x", "Failed", "1", false), + row("r9700s", "Failed", "-", true), + row("w7900-dual", "Done", "0", false), + ], + "a line after the matrix is not a lane" + ); + } + + #[test] + fn a_log_that_is_not_the_report_yields_nothing_to_publish() { + assert_eq!(parse_log(None), QueueLog::default()); + assert_eq!( + parse_log(Some( + "ERR_ACCESS_DENIED\nLANE GATE\nmi300x x\n" + )), + QueueLog::default() + ); + // A row of an unexpected shape keeps its lane and its gate, and nothing else. + let odd = "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE GATE\n\ + 2026-11-13T03:45:26Z | mi300x some workload Pending now advisory\n"; + assert_eq!( + parse_log(Some(odd)).lanes, + vec![MatrixRow { + lane: "mi300x".to_owned(), + phase: None, + exit: None, + blocking: false, + }] + ); + // Text that is not a lane name is never kept, whatever the row says; the + // lane is counted, as blocking. + let hostile = "2026-11-13T03:45:26Z | LANE GATE\n\ + 2026-11-13T03:45:26Z | ../../etc/passwd advisory\n\ + 2026-11-13T03:45:26Z | Uppercase block\n"; + let lanes = parse_log(Some(hostile)).lanes; + assert_eq!( + lanes.iter().map(|l| l.lane.as_str()).collect::>(), + ["unnamed-row-1", "unnamed-row-2"] + ); + assert!(lanes.iter().all(|l| l.blocking)); + } + + fn found(entries: &[(&str, &[&'static str])]) -> BTreeMap> { + entries + .iter() + .map(|(lane, channels)| ((*lane).to_owned(), channels.to_vec())) + .collect() + } + + #[test] + fn a_blocking_lane_with_no_result_gets_an_empty_report_and_an_advisory_one_stays_advisory() { + let log = parse_log(Some(LOG)); + let plan = plan( + &log, + found(&[ + ("mi300x", &["release"]), + ("mi325x", &["release", "nightly"]), + ("w7900-dual", &["release", "nightly"]), + ]), + true, + ); + let by_lane = |name: &str| { + plan.lanes + .iter() + .find(|l| l.lane == name) + .expect("lane planned") + }; + // Failed, and the nightly channel left nothing: one stands in for it. + assert_eq!(by_lane("mi300x").placeholders, vec![Some("nightly")]); + assert!(by_lane("mi300x").blocking); + // Nothing at all came back from r9700s, which the queue never listed a file for. + assert_eq!(by_lane("r9700s").placeholders, vec![None]); + assert!(by_lane("r9700s").reported.is_empty()); + // Advisory stays advisory, and a lane with every channel needs no stand-in. + assert!(!by_lane("mi325x").blocking); + assert!(by_lane("mi325x").placeholders.is_empty()); + assert!(by_lane("w7900-dual").placeholders.is_empty()); + assert_eq!( + plan.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + } + + #[test] + fn a_lane_that_finished_cleanly_may_have_run_one_channel_only() { + let log = QueueLog { + branch: Some("main".to_owned()), + commit: None, + lanes: vec![row("mi300x", "Done", "0", true)], + }; + let plan = plan(&log, found(&[("mi300x", &["release"])]), true); + assert!(plan.lanes[0].placeholders.is_empty()); + } + + #[test] + fn without_the_matrix_every_lane_is_blocking_and_only_a_lane_with_no_files_is_missing() { + let plan = plan( + &QueueLog::default(), + found(&[("mi325x", &["release"]), ("r9700s", &[])]), + false, + ); + assert!(plan.lanes.iter().all(|l| l.blocking)); + assert!(plan.lanes[0].placeholders.is_empty()); + assert_eq!(plan.lanes[1].placeholders, vec![None]); + } + + #[test] + fn the_lane_and_channel_replace_the_slug_that_two_lanes_share() { + let manifest = r#"{"platform_slug":"mi300x","capability":{"platform_slug":"mi300x", + "gfx_target":"gfx942"},"versions":{"channel":"release"},"expectations":[]}"#; + let rewritten = with_lane_slug(manifest, "mi325x", "nightly").expect("an object"); + let value: serde_json::Value = serde_json::from_str(&rewritten).expect("json"); + assert_eq!(value["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["gfx_target"], "gfx942"); + assert_eq!(value["versions"]["channel"], "release"); + assert_eq!(with_lane_slug("[1]", "mi325x", "release"), None); + assert_eq!(with_lane_slug("not json", "mi325x", "release"), None); + } + + fn write(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, text).expect("write"); + } + + /// A queue artifact as the run above published it, cut down to two lanes. + fn queue_artifact(dir: &Path) { + for (lane, slug, channels) in [ + ("mi300x", "mi300x", &["release"][..]), + ("mi325x", "mi300x", &["release", "nightly"][..]), + ] { + write(&dir.join(lane).join("lane.log"), "the lane's own log\n"); + for channel in channels { + let results = dir.join(lane).join(channel).join("results"); + write(&results.join("report.json"), "[]\n"); + write(&results.join("commands.jsonl"), ""); + write(&results.join("junit.xml"), ""); + write(&results.join("harness-diagnostics.log"), "diagnostics"); + write( + &results.join("platform.json"), + &format!( + r#"{{"platform_slug":"{slug}","versions":{{"channel":"{channel}"}},"expectations":[]}}"# + ), + ); + } + } + write(&dir.join("r9700s").join("lane.log"), "no files came back\n"); + } + + #[test] + fn the_import_writes_only_what_the_report_reads_under_names_that_say_lane_and_gate() { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let log = queue.path().join("job.log"); + // The log sits beside the lanes here; a stray file is no lane. + write(&log, LOG); + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("consolidated/queue-lanes.md"); + let args = Args { + queue_dir: Some(queue.path().to_owned()), + log: Some(log), + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/43351651832".to_owned()), + run_started: Some("2026-11-13T02:21:58Z".to_owned()), + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }; + run(&args).expect("import"); + + let mut dirs: Vec = std::fs::read_dir(out.path().join("e2e-artifacts")) + .expect("artifacts") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + dirs.sort(); + assert_eq!( + dirs, + [ + "e2e-queue-mi300x-nightly-report", + "e2e-queue-mi300x-release-report", + "e2e-queue-mi325x-advisory-nightly-report", + "e2e-queue-mi325x-advisory-release-report", + "e2e-queue-r9700s-report", + // Listed in the log, with no files at all: an empty stand-in. + "e2e-queue-w7900-dual-advisory-report", + ] + ); + let copied = out + .path() + .join("e2e-artifacts/e2e-queue-mi325x-advisory-release-report"); + let mut files: Vec = std::fs::read_dir(&copied) + .expect("lane dir") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + files.sort(); + assert_eq!(files, ["commands.jsonl", "platform.json", "report.json"]); + let manifest = std::fs::read_to_string(copied.join("platform.json")).expect("manifest"); + assert!(manifest.contains("queue-mi325x-release"), "{manifest}"); + assert_eq!( + std::fs::read_to_string( + out.path() + .join("e2e-artifacts/e2e-queue-r9700s-report/report.json") + ) + .expect("placeholder"), + "[]\n" + ); + + let section = std::fs::read_to_string(section).expect("section"); + assert!(section.starts_with("## Queue lanes\n"), "{section}"); + assert!( + section.contains( + "[run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832), \ + started 2026-11-13T02:21:58Z, tested commit `691834a04540`" + ), + "{section}" + ); + assert!( + section.contains("| mi325x | MI325X | advisory | Failed | 1 | release, nightly |"), + "{section}" + ); + assert!( + section.contains("| r9700s | R9700 | blocking | Failed | - | none |"), + "{section}" + ); + // Nothing of the lane's own log or the workload names reaches the page. + assert!(!section.contains("rocm-cli-gh-"), "{section}"); + assert!(!section.contains("lane's own log"), "{section}"); + } + + #[test] + fn an_absent_queue_run_is_a_section_that_says_so_and_nothing_else() { + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: None, + log: None, + run_url: None, + run_started: None, + absent: Some("no queue run finished on main in the last 24 hours | `x`\n".to_owned()), + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("an absent run is not an error"); + assert!(!out.path().join("e2e-artifacts").exists()); + let text = std::fs::read_to_string(section).expect("section"); + assert_eq!( + text, + "## Queue lanes\n\n_No queue lanes are in this report: no queue run finished on \ + main in the last 24 hours x. The GPUs the shared queue tests are not shown \ + tonight._\n\n" + ); + } + + fn import_with_log(log_text: Option<&str>) -> (tempfile::TempDir, String) { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let out = tempfile::tempdir().expect("tempdir"); + let log = log_text.map(|text| { + let path = out.path().join("job.log"); + write(&path, text); + path + }); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: Some(queue.path().to_owned()), + log, + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/1".to_owned()), + run_started: None, + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("a log that cannot be used is not an error"); + let text = std::fs::read_to_string(section).expect("section"); + (out, text) + } + + #[test] + fn a_run_that_tested_another_branch_or_whose_log_is_unreadable_is_not_imported() { + // A run started from main can be asked to test any branch of the repository. + let other = LOG.replace("rocm-cli main at", "rocm-cli some-branch at"); + for log in [ + Some(other.as_str()), + // What the egress proxy answers when it refuses the request. + Some("ERR_ACCESS_DENIED\n"), + None, + ] { + let (out, text) = import_with_log(log); + assert!( + text.contains("did not test main, or its job log could not be read"), + "{text}" + ); + assert!(!text.contains("| Lane |"), "{text}"); + assert!(!out.path().join("e2e-artifacts").exists()); + } + } + + #[test] + fn a_log_without_a_matrix_leaves_every_lane_blocking_and_says_so() { + let (out, text) = import_with_log(Some( + "2026-11-13T03:45:26Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351\n", + )); + assert!(text.contains("every lane is shown as blocking"), "{text}"); + assert!(!text.contains("advisory |"), "{text}"); + // The lane with no files is still a missing result; the others are not + // second-guessed about a channel they never ran. + let arts = out.path().join("e2e-artifacts"); + assert!(arts.join("e2e-queue-r9700s-report").is_dir()); + assert!(!arts.join("e2e-queue-mi300x-nightly-report").exists()); + } + + #[test] + fn a_lane_that_cannot_be_named_still_shows_as_a_blocking_lane_with_no_result() { + for name in [ + "Mi300X", + "mi300x_2", + "x-advisory", + "x-release", + "x-nightly", + "-x", + ] { + assert!(!is_lane_name(name), "{name}"); + } + let log = parse_log(Some( + "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE\n\ + 2026-11-13T03:45:26Z | x-advisory w Done 0 Exported 0m advisory\n", + )); + assert_eq!( + log.lanes, + vec![MatrixRow { + lane: "unnamed-row-1".to_owned(), + phase: None, + exit: None, + blocking: true, + }] + ); + // The same for a directory of the artifact. + let queue = tempfile::tempdir().expect("tempdir"); + write(&queue.path().join("mi300x_2/lane.log"), "log\n"); + write( + &queue.path().join("mi300x/release/results/report.json"), + "[]", + ); + let found = scan(queue.path()).expect("scan"); + assert_eq!(found.get("unnamed-dir-1"), Some(&Vec::new())); + assert_eq!(found.get("mi300x"), Some(&vec!["release"])); + let plan = plan(&QueueLog::default(), found, false); + let unnamed = plan + .lanes + .iter() + .find(|l| l.lane == "unnamed-dir-1") + .expect("lane"); + assert!(unnamed.blocking); + assert_eq!(unnamed.placeholders, vec![None]); + } + + #[test] + fn a_run_url_that_is_not_a_run_is_not_linked() { + let plan = Plan { + commit: None, + lanes: Vec::new(), + }; + let text = render_section(&plan, Some("https://example.com/x)[y]("), None, true); + assert!(!text.contains("example.com"), "{text}"); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e3db2768f..4a32095aa 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,6 +22,7 @@ mod dash_text_edit_contract; mod demos; mod e2e; mod e2e_prewarm; +mod e2e_queue_import; mod e2e_report; mod engine_stdio_client_contract; mod env_mutation_contract; @@ -331,6 +332,12 @@ pub(crate) enum Command { #[arg(long)] html_out: PathBuf, }, + /// Bring a finished run of the shared GPU queue into the nightly's E2E + /// report: write each lane's reports where `e2e-report` reads them, named + /// for the lane and for whether the queue blocks on it, and write a status + /// section for the top of the support matrix. With `--absent`, import + /// nothing and say why. See docs/nightly-on-the-queue.md. + E2eQueueImport(e2e_queue_import::Args), /// Count received Doctor reports from an export of the report mailbox. /// /// Reads `.eml` files (one message each) and raw `.json` report bodies, @@ -497,6 +504,7 @@ fn run() -> Result<()> { artifacts_dir, html_out, } => e2e_report::run(&artifacts_dir, &html_out)?, + Command::E2eQueueImport(args) => e2e_queue_import::run(&args)?, Command::ReportRollup(args) => report_rollup::run(&args)?, Command::ReleaseReadiness { since_tag, diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index c3060f557..a901474fb 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -72,6 +72,11 @@ pub struct Facts { /// Whether the nightly or Windows build gate was skipped (a legitimate /// no-op-night outcome, not a failure) rather than actually run. pub build_skipped: bool, + /// Why the shared queue's GPU lanes are not in tonight's E2E report, when + /// the matrix says so. The verdict does not count that absence as a + /// failure, so the table has to say it: the E2E row covers only the lanes + /// that are in the report. + pub queue_absent: Option, pub commits: CommitBuckets, pub known_issues: Vec, } @@ -93,7 +98,7 @@ pub fn run( .with_context(|| format!("reading e2e report {}", e2e_report.display()))?; let e2e_fail_total = total_e2e_failures(&e2e_text); - let facts = assemble_facts( + let mut facts = assemble_facts( since_tag, commits, known_issues, @@ -102,6 +107,7 @@ pub fn run( windows_result, e2e_report_job_result, ); + facts.queue_absent = queue_absent_reason(&e2e_text); // Trailing newline so the file ends like a text file. write_parented(out, &format!("{}\n", serde_json::to_string_pretty(&facts)?))?; @@ -164,6 +170,7 @@ fn assemble_facts( failing_gates, e2e_fail_total, build_skipped: nightly_result == "skipped" || windows_result == "skipped", + queue_absent: None, commits, known_issues, } @@ -393,6 +400,16 @@ fn excerpt(text: &str, max_chars: usize) -> String { } } +/// The reason the matrix gives for having no queue lanes, if it says so +/// (`e2e_queue_import::render_absent`). +fn queue_absent_reason(markdown: &str) -> Option { + use crate::e2e_queue_import::{ABSENT_LEAD, ABSENT_TAIL}; + markdown.lines().find_map(|line| { + let rest = line.strip_prefix(ABSENT_LEAD)?; + Some(rest.strip_suffix(ABSENT_TAIL).unwrap_or(rest).to_string()) + }) +} + /// Sum of the `Fail` column across the consolidated E2E markdown's `**Total**` /// row (`crates/e2e-report`'s `consolidated_summary_markdown` format: `| /// Platform | OS | Total | Pass | Fail | Skip | Xfail | Status |`), plus one @@ -407,6 +424,11 @@ fn excerpt(text: &str, max_chars: usize) -> String { /// adds for platforms the Total row would otherwise read as having zero /// failures. /// +/// A row for a shared-queue lane the queue does not block on (its platform +/// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the +/// queue lets that lane fail, so the verdict does too. Its `Fail` cell is +/// taken back out of the Total row, which sums every row. +/// /// Parsing the already-rendered rows, rather than re-deriving per-platform /// pass/fail here, keeps this in lockstep with that crate's known-bugs/xfail /// handling instead of duplicating it. @@ -431,24 +453,31 @@ fn total_e2e_failures(markdown: &str) -> Option { .nth(5) .and_then(|cell| cell.trim().parse::().ok())?; - let silently_empty_platforms = lines[header_idx + 1..total_idx] - .iter() - .filter(|l| { - let t = l.trim_start(); - t.starts_with('|') && !t.starts_with("|---") - }) - .filter(|l| { - let cells: Vec<&str> = l.split('|').collect(); - let status = cells.get(8).map(|c| c.trim()); - let fail = cells - .get(5) - .and_then(|c| c.trim().parse::().ok()) - .unwrap_or(0); - !matches!(status, Some("PASS") | None) && fail == 0 - }) - .count() as u64; + let mut advisory_fails = 0; + let mut silently_empty_platforms = 0; + for line in lines[header_idx + 1..total_idx].iter().filter(|l| { + let t = l.trim_start(); + t.starts_with('|') && !t.starts_with("|---") + }) { + let cells: Vec<&str> = line.split('|').collect(); + let fail = cells + .get(5) + .and_then(|c| c.trim().parse::().ok()) + .unwrap_or(0); + if cells + .get(1) + .is_some_and(|c| c.contains(e2e_report::ADVISORY_QUEUE_MARKER)) + { + advisory_fails += fail; + continue; + } + let status = cells.get(8).map(|c| c.trim()); + if !matches!(status, Some("PASS") | None) && fail == 0 { + silently_empty_platforms += 1; + } + } - Some(total_row_fails + silently_empty_platforms) + Some(total_row_fails.saturating_sub(advisory_fails) + silently_empty_platforms) } /// Neutralize markdown syntax in untrusted text (GitHub issue titles) before @@ -485,10 +514,13 @@ pub(crate) fn render_table(facts: &Facts) -> String { // the header below so `publish-release-readiness-wiki`'s guard (which // greps for it) only trips on a truly empty/degenerate table. let e2e_failing = facts.failing_gates.iter().any(|g| g.starts_with("E2E")); - let e2e_evidence = match facts.e2e_fail_total { + let mut e2e_evidence = match facts.e2e_fail_total { Some(n) => format!("{n} failure(s) across all platforms"), None => "evidence unreadable".to_string(), }; + if facts.queue_absent.is_some() { + e2e_evidence.push_str(" in the report (queue lanes missing, see below)"); + } out.push_str("| Area | Evidence | Status |\n|---|---|:--:|\n"); let _ = writeln!( @@ -509,6 +541,16 @@ pub(crate) fn render_table(facts: &Facts) -> String { ); out.push('\n'); + if let Some(reason) = &facts.queue_absent { + let _ = writeln!( + out, + "⚠️ **The shared queue's GPU lanes are not in tonight's report** ({}). The E2E row \ + above does not cover them, so a GO here does not cover R9700, MI350P, W7900 or the \ + queue's blocking lanes.\n", + escape_markdown(reason) + ); + } + // The status table above only has room for a glyph per row, which is // ambiguous on its own (e.g. a cancelled E2E lane renders as // "0 failure(s)" with a red ❌, which looks contradictory). Spell out @@ -741,6 +783,48 @@ mod tests { assert_eq!(total_e2e_failures(md), Some(1)); } + #[test] + fn total_e2e_failures_leaves_out_the_queue_lanes_the_queue_lets_fail() { + // Real producer output, so the marker the parser looks for is the one the + // report writes: a failing advisory lane and an empty advisory lane add + // nothing, while the same lanes marked blocking add one failure and one + // silent failure. + fn report(status: &str) -> String { + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s","tags":[],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{status}"}}}}]}}]}}]"# + ) + } + let render = |advisory: bool| { + let dir = tempfile::tempdir().expect("tempdir"); + let write = |name: &str, text: &str| { + let path = dir.path().join(name); + std::fs::create_dir_all(&path).expect("mkdir"); + std::fs::write(path.join("report.json"), text).expect("write report"); + (name.to_owned(), path.join("report.json")) + }; + let failing = e2e_report::queue_artifact_name("mi325x", advisory, Some("release")); + let empty = e2e_report::queue_artifact_name("r9700s", advisory, None); + let inputs = vec![ + write("e2e-gpu-report", &report("passed")), + write(&failing, &report("failed")), + write(&empty, "[]"), + ]; + e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&inputs)) + }; + let advisory = render(true); + assert!( + advisory.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{advisory}" + ); + assert_eq!(total_e2e_failures(&advisory), Some(0), "{advisory}"); + let blocking = render(false); + assert!( + !blocking.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{blocking}" + ); + assert_eq!(total_e2e_failures(&blocking), Some(2), "{blocking}"); + } + #[test] fn total_e2e_failures_ignores_the_expectation_grid_and_command_coverage_tables() { // Regression: the scan used to run over every `|`-prefixed line in the @@ -797,6 +881,34 @@ mod tests { ); } + #[test] + fn the_table_says_when_the_queue_lanes_are_missing() { + let md = format!( + "## Queue lanes\n\n{}{}{}\n\n", + crate::e2e_queue_import::ABSENT_LEAD, + "no queue run on main finished in the last 24 hours", + crate::e2e_queue_import::ABSENT_TAIL + ); + let reason = queue_absent_reason(&md).expect("the note is recognised"); + assert_eq!(reason, "no queue run on main finished in the last 24 hours"); + assert_eq!(queue_absent_reason("| Total | 0 |\n"), None); + let mut facts = assemble_facts( + "v1.0.0", + CommitBuckets::default(), + vec![], + Some(0), + "success", + "success", + "success", + ); + facts.queue_absent = Some(reason); + let table = render_table(&facts); + assert!(table.contains("queue lanes missing"), "{table}"); + assert!(table.contains("not in tonight's report"), "{table}"); + // Not a failure: the verdict stays as the other gates decide it. + assert_eq!(facts.verdict, Verdict::Go); + } + #[test] fn assemble_facts_is_go_when_everything_succeeds() { let facts = assemble_facts( diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f0183849b..17378b85c 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2872,8 +2872,8 @@ jobs: } } assert_eq!( - delegating, 7, - "expected the seven native Linux preflight lanes to delegate to \ + delegating, 4, + "expected the four native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); } @@ -2941,8 +2941,8 @@ jobs: } assert_eq!( bodies.len(), - 7, - "expected seven native Linux preflight bodies to compare, found {} — \ + 4, + "expected four native Linux preflight bodies to compare, found {} — \ if a lane was added or removed, update this count", bodies.len() ); @@ -4301,8 +4301,8 @@ fi } } assert_eq!( - delegating, 7, - "expected the 7 native Linux preflight lanes to delegate to \ + delegating, 4, + "expected the 4 native Linux preflight lanes to delegate to \ {SHARED_PREFLIGHT} — if a lane was added or removed, update this count" ); assert_eq!( @@ -4399,6 +4399,129 @@ fi } } + /// The nightly report carries the shared queue's GPUs into the matrix, and + /// so into the wiki and the release-readiness verdict, from the queue's own + /// results: the newest finished run on `main`, found by its artifact's name + /// (no workflow may name or wait on the queue's, see + /// `assert_queue_workflow_holds_no_credential`). Neither step may fail the + /// job: a missing or late run is said so at the top of the page. + #[test] + fn nightly_report_imports_the_latest_queue_run_without_depending_on_it() { + let nightly = read_workflow("nightly.yml"); + let job = job_block(&nightly, "e2e-report-nightly"); + let steps = job_steps(job).expect("the report job has steps"); + let position = |prefix: &str| { + steps + .iter() + .position(|(name, _)| name.starts_with(prefix)) + .unwrap_or_else(|| panic!("e2e-report-nightly has a `{prefix}` step")) + }; + let (fetch, add, build) = ( + position("Fetch the latest queue results"), + position("Add the queue lanes to the report"), + position("Build consolidated report"), + ); + assert!( + fetch < add && add < build, + "the queue lanes must be fetched, then put among the reports, before the report is built" + ); + for at in [fetch, add] { + assert_eq!( + step_key(&steps[at].1, "continue-on-error"), + "true", + "`{}` must not be able to fail the nightly report: a missing or late queue run \ + is shown as missing, not as a red report", + steps[at].0 + ); + } + + let fetch = normalized_whitespace(&steps[fetch].1); + for needed in [ + // By artifact name, from a run on this repository's main: the + // workflow that makes it may not be named here. + "actions/artifacts?name=queue-run-results", + ".workflow_run.head_branch == \"main\"", + ".workflow_run.head_repository_id == $repo", + ".expired == false", + // Today's run: not yesterday's, whatever its conclusion was. + "date -u -d '24 hours ago'", + ".created_at >= $since", + "gh run download \"$candidate\"", + "--name queue-run-results", + // A run that started a day ago is not tonight's, whenever it ended. + "[ \"$started_at\" -ge \"$(date -u -d '24 hours ago' +%s)\" ]", + // Only a finished run: a run still going has a partial log. + "[ \"${run_info%%$'\\t'*}\" = \"completed\" ]", + // The log says which branch the run tested and which lanes block; + // the run may have several jobs, so every one is scanned. + "actions/runs/${candidate}/jobs?per_page=100", + "/logs\" > queue-results/job.log.part", + ] { + assert!( + fetch.contains(needed), + "the queue fetch step must contain `{needed}`:\n{fetch}" + ); + } + assert!( + !fetch.contains("conclusion"), + "a red queue run is still a run: its artifact is uploaded whatever it concluded" + ); + // No credential beyond the job's own read-only token. + let fetch_step = &steps[position("Fetch the latest queue results")].1; + let env = step_env(fetch_step); + assert_eq!( + env.get("GH_TOKEN").map(String::as_str), + Some("${{ github.token }}"), + "{env:?}" + ); + assert_eq!(env.len(), 1, "{env:?}"); + assert!(!fetch_step.contains("secrets."), "{fetch_step}"); + let permissions = job_mapping(job, "permissions"); + assert_eq!( + permissions.get("actions").map(String::as_str), + Some("read"), + "listing and downloading the queue's artifact needs actions: read" + ); + assert_eq!( + permissions.get("contents").map(String::as_str), + Some("read") + ); + assert_eq!(permissions.len(), 2, "{permissions:?}"); + + // Both outcomes go through the importer, which writes the status section. + let add = normalized_whitespace(&steps[position("Add the queue lanes to the report")].1); + for needed in [ + "cargo xtask e2e-queue-import", + "--queue-dir queue-results/results", + "--absent", + "--artifacts-dir e2e-artifacts", + "--section-out queue-results/queue-lanes.md", + // An import that fails leaves no half of one behind, and says so. + "rm -rf e2e-artifacts/e2e-queue-*", + "--absent \"the queue lanes could not be imported\"", + ] { + assert!( + add.contains(needed), + "the queue import step must contain `{needed}`:\n{add}" + ); + } + // The section leads the page; the matrix keeps its own first table. + let build = normalized_whitespace(&steps[build].1); + assert!( + build.contains( + "cat queue-results/queue-lanes.md consolidated/support-matrix.md > \ + \"$RUNNER_TEMP/with-queue.md\"" + ), + "the queue section must be put ahead of the matrix:\n{build}" + ); + // A queue run is never something the report job waits for. + let needs = job_scalar(job, "needs"); + assert!( + !needs.contains("queue"), + "e2e-report-nightly must not depend on the queue's workflows: {needs}" + ); + } + #[test] fn nightly_publishes_the_consolidated_matrix_to_the_wiki() { let nightly = read_workflow("nightly.yml"); @@ -4630,6 +4753,7 @@ fi failing_gates: vec!["E2E (report unreadable)".to_string()], e2e_fail_total: None, build_skipped: false, + queue_absent: None, commits: CommitBuckets::default(), known_issues: vec![], }); @@ -4649,6 +4773,7 @@ fi failing_gates: vec![], e2e_fail_total: Some(0), build_skipped: false, + queue_absent: None, commits: CommitBuckets::default(), known_issues: vec![], }); @@ -4852,11 +4977,8 @@ fi } for lane in [ "e2e-gpu-nightly", - "e2e-gpu-nightly-rad3", - "e2e-gpu-nightly-mi350p", "e2e-gpu-nightly-strix", "e2e-gpu-nightly-strix-windows", - "e2e-gpu-nightly-w7900", "e2e-wsl-nightly", ] { assert!( @@ -9653,8 +9775,8 @@ permissions: let flat = doc.split_whitespace().collect::>().join(" "); for needed in [ "both the `release` and the `nightly` channel", - "The remaining gap is native Windows and WSL.", - "the native Windows and WSL lanes stay on their dedicated runners", + "The remaining gap is Strix Halo.", + "the Strix Halo nightlies (Ubuntu, Windows and WSL) stay on the shared Strix pool", "`E2E_CHANNELS`", "`/out//results`", ] {