From 201e22692d2f4b0aaca0e927d014c52d81dca901 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 10:43:44 +0000 Subject: [PATCH 1/9] ci: supersede a PR's self-hosted E2E runs when the PR is merged or closed The E2E self-hosted workflow supersedes a PR's older run only when a newer push starts a run in the PR's concurrency group. A merged or closed PR starts none, so its queued MI300X/W7900 jobs stay in the queue and hold a runner for a head nobody will look at again (about half of the 18 queued MI300X jobs on 2026-11-11 were of this kind). List `closed` among the pull_request types. The run it starts joins the PR's group, so cancel-in-progress cancels the earlier run. `changes` skips on `closed`, which skips every lane, leaving only a no-op `pr-closed` job. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 18 +++++++++++++++ xtask/src/workflow_contract.rs | 34 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 398f9ded6..5d31e85d3 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -31,6 +31,10 @@ on: # Mirror ci.yml: run on PRs against ANY base branch so a stacked PR still gets # the GPU lanes. Affected-crate/heavy selection diffs against github.base_ref. pull_request: + # `closed` (merge OR close) joins the PR's concurrency group below, so it + # supersedes that PR's queued/running GPU runs. It runs only the no-op + # `pr-closed` job; every GPU lane skips because `changes` skips. + types: [opened, synchronize, reopened, closed] merge_group: branches: [main] types: [checks_requested] @@ -93,6 +97,8 @@ jobs: # forced true, matching ci.yml. changes: runs-on: ubuntu-24.04 # one LTS: path filter only, nothing here depends on the image + # Skipped on `closed`: every lane gates on `needs.changes.result == 'success'`. + if: github.event_name != 'pull_request' || github.event.action != 'closed' permissions: contents: read pull-requests: read @@ -183,6 +189,18 @@ jobs: echo 'matrix=["current"]' >> "$GITHUB_OUTPUT" fi + # Exists only to start a run in the PR's concurrency group when the PR is + # merged or closed: that run cancels the PR's older run, and with it any GPU + # job still queued for a head nobody will look at again. + pr-closed: + name: Supersede runs of a closed PR + if: github.event_name == 'pull_request' && github.event.action == 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - run: echo "PR closed; its earlier E2E runs are superseded by this run." + e2e-gpu: name: E2E tests (MI300X, ${{ matrix.sdk_version }}) timeout-minutes: 90 diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 12a0726bd..13c034b8b 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7416,6 +7416,40 @@ jobs: ); } + #[test] + fn closing_a_pr_supersedes_its_self_hosted_runs() { + // A merged or closed PR's queued GPU run is dead work that still takes a + // runner. `closed` is not a default `pull_request` activity type, so it + // must be listed; the run it starts joins the PR's group (same key as the + // opened/synchronize runs) and so cancels the earlier one. It must run + // nothing but the no-op job: `changes` skips, and with it every lane. + let sh = read_workflow("e2e-selfhosted.yml"); + let on = normalized_whitespace( + &top_level_block(&sh, "on") + .lines() + .map(strip_comment) + .collect::>() + .join("\n"), + ); + assert!( + on.contains("types: [opened, synchronize, reopened, closed]"), + "pull_request must list `closed` next to the default types, got `{on}`" + ); + assert!( + sh.contains( + "if: github.event_name != 'pull_request' || github.event.action != 'closed'" + ), + "the `changes` job must skip on `closed`, or every GPU lane would run for a closed PR" + ); + assert!( + sh.contains(" pr-closed:\n") + && sh.contains( + "github.event_name == 'pull_request' && github.event.action == 'closed'" + ), + "a no-op `pr-closed` job must exist so the closed run has something to run" + ); + } + #[test] fn mi300x_lane_is_skipped_in_the_merge_queue() { // The MI300X check is not required for merging, so a merge_group run's From 867fb95e1815467b5eb16062933a169842d33ecc Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 11:15:22 +0000 Subject: [PATCH 2/9] docs: say that closing a PR supersedes its self-hosted E2E run Note in the Triggers section that a PR's closed event starts a no-op run in the PR's concurrency group, and that a run whose job is still queued for a runner may not finish cancelling. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index e5185829d..a315b7a59 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -357,7 +357,12 @@ required for merging and so is skipped in the merge queue rather than left queued on limited runner capacity after its PR has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a -group of their own, so none is cancelled or replaced by another. The heavy +group of their own, so none is cancelled or replaced by another. Merging or +closing a pull request also starts a run (`closed` is a listed `pull_request` +activity type) that joins that PR's group and runs only a no-op job, so it +supersedes the PR's earlier run and frees the queue of dead work. A run whose +job is still queued for a runner may not finish cancelling, so it can keep +its place in the queue until the runner picks the job up. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM lane: they run only on the other lanes, and the only vLLM serve before a change lands is the per-PR canary (`serve-05`), which runs when the `serve` filter From 57227cc05d74cd8e0257cd7b9b6fa65e6234b2cc Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 12:01:35 +0000 Subject: [PATCH 3/9] test(ci): scope the closed-PR contract test to job blocks Read each assertion from one job's comment-free block, so prose or another job cannot satisfy it, pin the whole `if` of `changes` and `pr-closed` with the latter's runner and empty permissions, and assert that every other job (including e2e-report) skips when `changes` skips. Signed-off-by: Tomas Saaristola --- xtask/src/workflow_contract.rs | 95 ++++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 9 deletions(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 13c034b8b..9bda34ee7 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7416,6 +7416,34 @@ jobs: ); } + /// A job's block with comments removed, so a match cannot come from prose. + fn uncommented_job_block(text: &str, job: &str) -> String { + job_block(text, job) + .lines() + .map(strip_comment) + .collect::>() + .join("\n") + } + + /// The job-level `if:` of an already uncommented job block, with a folded + /// (`>-`) condition joined onto one line. Empty when the job has none. + fn job_level_if(block: &str) -> String { + let mut lines = block.lines(); + let Some(first) = lines.find(|l| l.starts_with(" if:")) else { + return String::new(); + }; + let mut parts = vec![first.trim_start_matches(" if:").trim().to_owned()]; + // Continuation lines are indented deeper than the 4-space job keys. + parts.extend( + lines + .take_while(|l| l.starts_with(" ")) + .map(|l| l.trim().to_owned()), + ); + normalized_whitespace(&parts.join(" ")) + .trim_start_matches(">- ") + .to_owned() + } + #[test] fn closing_a_pr_supersedes_its_self_hosted_runs() { // A merged or closed PR's queued GPU run is dead work that still takes a @@ -7423,6 +7451,8 @@ jobs: // must be listed; the run it starts joins the PR's group (same key as the // opened/synchronize runs) and so cancels the earlier one. It must run // nothing but the no-op job: `changes` skips, and with it every lane. + // Every assertion reads one job's comment-free block, so prose or another + // job cannot satisfy it. let sh = read_workflow("e2e-selfhosted.yml"); let on = normalized_whitespace( &top_level_block(&sh, "on") @@ -7435,19 +7465,66 @@ jobs: on.contains("types: [opened, synchronize, reopened, closed]"), "pull_request must list `closed` next to the default types, got `{on}`" ); - assert!( - sh.contains( - "if: github.event_name != 'pull_request' || github.event.action != 'closed'" - ), + + let changes = uncommented_job_block(&sh, "changes"); + assert_eq!( + job_level_if(&changes), + "github.event_name != 'pull_request' || github.event.action != 'closed'", "the `changes` job must skip on `closed`, or every GPU lane would run for a closed PR" ); + + let closed = uncommented_job_block(&sh, "pr-closed"); + assert_eq!( + job_level_if(&closed), + "github.event_name == 'pull_request' && github.event.action == 'closed'", + "`pr-closed` must run on `closed` and on nothing else" + ); assert!( - sh.contains(" pr-closed:\n") - && sh.contains( - "github.event_name == 'pull_request' && github.event.action == 'closed'" - ), - "a no-op `pr-closed` job must exist so the closed run has something to run" + closed.contains(" runs-on: ubuntu-24.04\n"), + "`pr-closed` must run on a GitHub-hosted runner, never queue for a GPU:\n{closed}" + ); + assert!( + closed.contains(" permissions: {}\n"), + "`pr-closed` must hold no token permissions:\n{closed}" + ); + for forbidden in ["uses:", "needs:", "secrets", "checkout"] { + assert!( + !closed.contains(forbidden), + "`pr-closed` is a no-op and must not contain `{forbidden}`:\n{closed}" + ); + } + + // Every other job must skip when `changes` skips: either its own `if` + // requires `changes` to have succeeded (the lanes) or it keys off a + // `changes` output, which is empty for a skipped job (`e2e-report`). + let jobs = top_level_block(&sh, "jobs"); + let ids: Vec<&str> = jobs + .lines() + .filter(|l| l.starts_with(" ") && !l.starts_with(" ")) + .filter_map(|l| l.trim().strip_suffix(':')) + .filter(|id| { + !id.is_empty() + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + }) + .collect(); + assert!( + ids.contains(&"e2e-report"), + "extractor sanity check: {ids:?}" ); + for id in ids + .iter() + .filter(|id| !["changes", "pr-closed"].contains(id)) + { + let condition = job_level_if(&uncommented_job_block(&sh, id)); + assert!( + condition.contains("needs.changes.result == 'success'") + || condition.contains("needs.changes.outputs.serve == 'true'"), + "job `{id}` must skip when `changes` is skipped (a closed PR); its `if` is \ + `{condition}`" + ); + } } #[test] From c4ee13df8a9d4705a9cd00960ec4abbc37b4ce0c Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 12:01:35 +0000 Subject: [PATCH 4/9] docs(ci): state that a closed-PR run does not clear queued GPU runs Reword the workflow comments and the Triggers paragraph to the same hedge: a job queued for a runner that is not picking it up may not finish cancelling, so the closed run frees runners only where the cancel completes. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 12 ++++++++---- docs/ci-hardware-testing.md | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 5d31e85d3..e274eb97f 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -32,8 +32,11 @@ on: # the GPU lanes. Affected-crate/heavy selection diffs against github.base_ref. pull_request: # `closed` (merge OR close) joins the PR's concurrency group below, so it - # supersedes that PR's queued/running GPU runs. It runs only the no-op - # `pr-closed` job; every GPU lane skips because `changes` skips. + # supersedes that PR's earlier run. It runs only the no-op `pr-closed` job; + # every GPU lane skips because `changes` skips. This frees runners only where + # the cancel completes: a job queued for a runner that is not picking it up + # (W7900 while its one runner is busy) may not finish cancelling, so this + # does NOT clear such queued runs. types: [opened, synchronize, reopened, closed] merge_group: branches: [main] @@ -190,8 +193,9 @@ jobs: fi # Exists only to start a run in the PR's concurrency group when the PR is - # merged or closed: that run cancels the PR's older run, and with it any GPU - # job still queued for a head nobody will look at again. + # merged or closed: that run cancels the PR's older run. A GPU job of that run + # which is queued for a runner that is not picking it up may not finish + # cancelling (observed on W7900-queued runs). pr-closed: name: Supersede runs of a closed PR if: github.event_name == 'pull_request' && github.event.action == 'closed' diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index a315b7a59..9b4da9202 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -360,7 +360,7 @@ cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. Merging or closing a pull request also starts a run (`closed` is a listed `pull_request` activity type) that joins that PR's group and runs only a no-op job, so it -supersedes the PR's earlier run and frees the queue of dead work. A run whose +supersedes the PR's earlier run and frees runners where the cancel completes. A run whose job is still queued for a runner may not finish cancelling, so it can keep its place in the queue until the runner picks the job up. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM From 30175ac19a5baa2240321bc9a37b55c51947993d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 14:22:09 +0000 Subject: [PATCH 5/9] ci: keep a closed PR's run evaluable and pin the lanes' skip gate Fall back to ["current"] when changes' matrix output is empty (changes is skipped on closed), so no lane's strategy evaluates fromJSON of an empty string. Require every lane's if to open with always() && needs.changes.result == 'success', share the comment-stripping helper, and re-wrap the docs paragraph. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 10 +++--- docs/ci-hardware-testing.md | 6 ++-- xtask/src/workflow_contract.rs | 49 +++++++++++++++++++--------- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index e274eb97f..0f02b9245 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -221,7 +221,7 @@ jobs: # single-element `["current"]` matrix, so the ordinary PR/merge-queue path # runs exactly one job, unchanged from before this matrix existed. matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} # No build-and-test gate (cross-workflow needs is unavailable): the job builds # the rocm binary itself, and this check isn't in the required list (see the # file header), so a broken build here still can't block a merge; ci.yml's @@ -485,7 +485,7 @@ jobs: # See `e2e-gpu` (ROCMAI-430): layered alongside the pre-existing # `channel` dimension above, not replacing it — this lane is the only # per-PR job with two independent matrix axes. - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} # The nightly channel only carries the latest (pre-)release builds, so # it cannot resolve the n-1/n-2 released versions the matrix pins # (`rocm install sdk` fails with "no mutually compatible TheRock ... @@ -640,7 +640,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-windows. if: >- always() @@ -988,7 +988,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-wsl. Gated on # `serve` like the sibling lanes rather than the broader `heavy`: the # consolidated report gates on `serve` too, so a heavy-but-not-serve change @@ -1596,7 +1596,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} if: >- always() && needs.changes.result == 'success' diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 9b4da9202..4684dc7da 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -360,9 +360,9 @@ cancels that PR's older run; push, `merge_group` and manual runs each get a group of their own, so none is cancelled or replaced by another. Merging or closing a pull request also starts a run (`closed` is a listed `pull_request` activity type) that joins that PR's group and runs only a no-op job, so it -supersedes the PR's earlier run and frees runners where the cancel completes. A run whose -job is still queued for a runner may not finish cancelling, so it can keep -its place in the queue until the runner picks the job up. The heavy +supersedes the PR's earlier run and frees runners where the cancel completes. +A run whose job is still queued for a runner may not finish cancelling, so it +can keep its place in the queue until the runner picks the job up. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM lane: they run only on the other lanes, and the only vLLM serve before a change lands is the per-PR canary (`serve-05`), which runs when the `serve` filter diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 9bda34ee7..744afaceb 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7416,15 +7416,19 @@ jobs: ); } - /// A job's block with comments removed, so a match cannot come from prose. - fn uncommented_job_block(text: &str, job: &str) -> String { - job_block(text, job) - .lines() + /// `text` with every comment removed, so a match cannot come from prose. + fn yaml_without_comments(text: &str) -> String { + text.lines() .map(strip_comment) .collect::>() .join("\n") } + /// A job's block with comments removed. + fn uncommented_job_block(text: &str, job: &str) -> String { + yaml_without_comments(job_block(text, job)) + } + /// The job-level `if:` of an already uncommented job block, with a folded /// (`>-`) condition joined onto one line. Empty when the job has none. fn job_level_if(block: &str) -> String { @@ -7454,13 +7458,7 @@ jobs: // Every assertion reads one job's comment-free block, so prose or another // job cannot satisfy it. let sh = read_workflow("e2e-selfhosted.yml"); - let on = normalized_whitespace( - &top_level_block(&sh, "on") - .lines() - .map(strip_comment) - .collect::>() - .join("\n"), - ); + let on = normalized_whitespace(&yaml_without_comments(&top_level_block(&sh, "on"))); assert!( on.contains("types: [opened, synchronize, reopened, closed]"), "pull_request must list `closed` next to the default types, got `{on}`" @@ -7517,12 +7515,31 @@ jobs: .iter() .filter(|id| !["changes", "pr-closed"].contains(id)) { - let condition = job_level_if(&uncommented_job_block(&sh, id)); + let block = uncommented_job_block(&sh, id); + let condition = job_level_if(&block); + if *id == "e2e-report" { + // Skips because the `serve` output of a skipped `changes` is empty. + assert!( + condition.starts_with("always() && (needs.changes.outputs.serve == 'true'"), + "`e2e-report` must open with `always() && (needs.changes.outputs.serve == \ + 'true'`, so it skips on `closed`; its `if` is `{condition}`" + ); + continue; + } + // A lane's `if` must START with the `changes` gate: an `||` arm or a + // reordered operand would make it run when `changes` is skipped. + assert!( + condition.starts_with("always() && needs.changes.result == 'success'"), + "lane `{id}` must open with `always() && needs.changes.result == 'success'`, \ + so it skips on `closed`; its `if` is `{condition}`" + ); + // `changes` is skipped on `closed`, so its matrix output is empty; + // the fallback keeps the strategy evaluable. assert!( - condition.contains("needs.changes.result == 'success'") - || condition.contains("needs.changes.outputs.serve == 'true'"), - "job `{id}` must skip when `changes` is skipped (a closed PR); its `if` is \ - `{condition}`" + block.contains( + "fromJSON(needs.changes.outputs.sdk_version_matrix || '[\"current\"]')" + ), + "lane `{id}` must fall back to `[\"current\"]` when the matrix output is empty:\n{block}" ); } } From 4cabe6554cf0e8d28e398595e73349e2683a8aec Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 15:41:26 +0000 Subject: [PATCH 6/9] test(ci): pin each lane's whole if, reuse job_ids, reorder the docs A lane's if was checked by prefix, so an `|| true` appended to it (`&&` binds tighter than `||`) would still pass while defeating the skip on closed. Pin every lane's and e2e-report's full condition with assert_eq!, and fail on a job with no pinned condition. Take the job ids from job_ids, and move the closed-PR sentences to the end of the Triggers paragraph so the 'therefore' sentence follows the premise it refers to. Signed-off-by: Tomas Saaristola --- docs/ci-hardware-testing.md | 14 ++--- xtask/src/workflow_contract.rs | 99 ++++++++++++++++++++-------------- 2 files changed, 66 insertions(+), 47 deletions(-) diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 4684dc7da..5d34089d1 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -357,19 +357,19 @@ required for merging and so is skipped in the merge queue rather than left queued on limited runner capacity after its PR has merged) when the workflow's own `changes` job's `serve` path filter is `true`. A new push to a pull request cancels that PR's older run; push, `merge_group` and manual runs each get a -group of their own, so none is cancelled or replaced by another. Merging or -closing a pull request also starts a run (`closed` is a listed `pull_request` -activity type) that joins that PR's group and runs only a no-op job, so it -supersedes the PR's earlier run and frees runners where the cancel completes. -A run whose job is still queued for a runner may not finish cancelling, so it -can keep its place in the queue until the runner picks the job up. The heavy +group of their own, so none is cancelled or replaced by another. The heavy `@merge-queue` serves therefore no longer run on MI300X, the only per-PR vLLM lane: they run only on the other lanes, and the only vLLM serve before a change lands is the per-PR canary (`serve-05`), which runs when the `serve` filter matches. `push` fires on `main` and on any `release/**` branch, so pushing to a release branch runs the full self-hosted matrix ahead of cutting the `v*` tag that `release.yml` builds/publishes from — this is the release-candidate regression gate; `pull_request` fires against any base -branch. +branch. Merging or closing a pull request also starts a run (`closed` is a +listed `pull_request` activity type) that joins that PR's group and runs only a +no-op job, so it supersedes the PR's earlier run and frees runners where the +cancel completes. A run whose job is still queued for a runner may not finish +cancelling, so it can keep its place in the queue until the runner picks the +job up. On a release-branch push, every self-hosted GPU job additionally expands into a `strategy.matrix.sdk_version` of `[current, n-1, n-2]` (ROCMAI-430) — three diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 744afaceb..38b317703 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7492,54 +7492,73 @@ jobs: ); } - // Every other job must skip when `changes` skips: either its own `if` - // requires `changes` to have succeeded (the lanes) or it keys off a - // `changes` output, which is empty for a skipped job (`e2e-report`). - let jobs = top_level_block(&sh, "jobs"); - let ids: Vec<&str> = jobs - .lines() - .filter(|l| l.starts_with(" ") && !l.starts_with(" ")) - .filter_map(|l| l.trim().strip_suffix(':')) - .filter(|id| { - !id.is_empty() - && id - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') - }) - .collect(); - assert!( - ids.contains(&"e2e-report"), - "extractor sanity check: {ids:?}" - ); + // Every other job must skip when `changes` skips. Each `if` is pinned in + // full, not by prefix: an `||` arm appended to a lane's condition (as + // `&&` binds tighter than `||`) would make it true whatever `changes` + // did, while a prefix check still passed. A new job has no entry here + // and fails until it is added, which is the point: someone must decide + // how it behaves on `closed`. + let lane_if = |extra: &str, platform: &str| { + format!( + "always() && needs.changes.result == 'success' && {extra}\ + ( (github.event_name != 'workflow_dispatch' \ + && needs.changes.outputs.serve == 'true') \ + || (github.event_name == 'workflow_dispatch' \ + && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" + ) + }; + let expected: Vec<(&str, String)> = vec![ + ( + "e2e-gpu", + lane_if("github.event_name != 'merge_group' && ", "app-dev-gpu"), + ), + ("e2e-gpu-strix-ubuntu", lane_if("", "strix-ubuntu")), + ("e2e-gpu-strix-windows", lane_if("", "strix-windows")), + ("e2e-wsl", lane_if("", "strix-wsl")), + ("e2e-gpu-w7900", lane_if("", "w7900")), + // Skips because the `serve` output of a skipped `changes` is empty. + ( + "e2e-report", + "always() && (needs.changes.outputs.serve == 'true' \ + || github.event_name == 'workflow_dispatch')" + .to_owned(), + ), + ]; + let ids = job_ids(&sh); for id in ids .iter() - .filter(|id| !["changes", "pr-closed"].contains(id)) + .filter(|id| !["changes", "pr-closed"].contains(&id.as_str())) { + let (_, want) = expected + .iter() + .find(|(name, _)| name == id) + .unwrap_or_else(|| { + panic!( + "job `{id}` has no pinned `if`: add it, deciding how it behaves on `closed`" + ) + }); let block = uncommented_job_block(&sh, id); - let condition = job_level_if(&block); - if *id == "e2e-report" { - // Skips because the `serve` output of a skipped `changes` is empty. + assert_eq!( + &job_level_if(&block), + want, + "job `{id}` must skip when `changes` is skipped (a closed PR)" + ); + if *id != "e2e-report" { + // `changes` is skipped on `closed`, so its matrix output is empty; + // the fallback keeps the strategy evaluable. assert!( - condition.starts_with("always() && (needs.changes.outputs.serve == 'true'"), - "`e2e-report` must open with `always() && (needs.changes.outputs.serve == \ - 'true'`, so it skips on `closed`; its `if` is `{condition}`" + block.contains( + "fromJSON(needs.changes.outputs.sdk_version_matrix || '[\"current\"]')" + ), + "lane `{id}` must fall back to `[\"current\"]` when the matrix output is \ + empty:\n{block}" ); - continue; } - // A lane's `if` must START with the `changes` gate: an `||` arm or a - // reordered operand would make it run when `changes` is skipped. - assert!( - condition.starts_with("always() && needs.changes.result == 'success'"), - "lane `{id}` must open with `always() && needs.changes.result == 'success'`, \ - so it skips on `closed`; its `if` is `{condition}`" - ); - // `changes` is skipped on `closed`, so its matrix output is empty; - // the fallback keeps the strategy evaluable. + } + for (name, _) in &expected { assert!( - block.contains( - "fromJSON(needs.changes.outputs.sdk_version_matrix || '[\"current\"]')" - ), - "lane `{id}` must fall back to `[\"current\"]` when the matrix output is empty:\n{block}" + ids.iter().any(|id| id == name), + "pinned job `{name}` is gone" ); } } From a8760362eb7134eeb54ae125874bf2b621462c0c Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 15:48:20 +0000 Subject: [PATCH 7/9] ci: use the matrix placeholder only when changes was skipped Read each lane's matrix as fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix). On a closed PR changes is skipped, so the placeholder applies; if changes succeeded but stopped writing the output, fromJSON('') still fails instead of quietly running current. Pin that expression per entry in the contract test, and only for jobs pinned as matrix lanes, so a non-matrix helper job is not forced to carry it. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 10 +++---- xtask/src/workflow_contract.rs | 44 ++++++++++++++++++---------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 0f02b9245..eaaede47c 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -221,7 +221,7 @@ jobs: # single-element `["current"]` matrix, so the ordinary PR/merge-queue path # runs exactly one job, unchanged from before this matrix existed. matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # No build-and-test gate (cross-workflow needs is unavailable): the job builds # the rocm binary itself, and this check isn't in the required list (see the # file header), so a broken build here still can't block a merge; ci.yml's @@ -485,7 +485,7 @@ jobs: # See `e2e-gpu` (ROCMAI-430): layered alongside the pre-existing # `channel` dimension above, not replacing it — this lane is the only # per-PR job with two independent matrix axes. - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # The nightly channel only carries the latest (pre-)release builds, so # it cannot resolve the n-1/n-2 released versions the matrix pins # (`rocm install sdk` fails with "no mutually compatible TheRock ... @@ -640,7 +640,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-windows. if: >- always() @@ -988,7 +988,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-wsl. Gated on # `serve` like the sibling lanes rather than the broader `heavy`: the # consolidated report gates on `serve` too, so a heavy-but-not-serve change @@ -1596,7 +1596,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix || '["current"]') }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} if: >- always() && needs.changes.result == 'success' diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 38b317703..698ced2de 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7498,6 +7498,8 @@ jobs: // did, while a prefix check still passed. A new job has no entry here // and fails until it is added, which is the point: someone must decide // how it behaves on `closed`. + const MATRIX_EXPR: &str = "fromJSON(needs.changes.result == 'skipped' && '[\"current\"]' \ + || needs.changes.outputs.sdk_version_matrix)"; let lane_if = |extra: &str, platform: &str| { format!( "always() && needs.changes.result == 'success' && {extra}\ @@ -7507,21 +7509,23 @@ jobs: && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" ) }; - let expected: Vec<(&str, String)> = vec![ + let expected: Vec<(&str, String, bool)> = vec![ ( "e2e-gpu", lane_if("github.event_name != 'merge_group' && ", "app-dev-gpu"), + true, ), - ("e2e-gpu-strix-ubuntu", lane_if("", "strix-ubuntu")), - ("e2e-gpu-strix-windows", lane_if("", "strix-windows")), - ("e2e-wsl", lane_if("", "strix-wsl")), - ("e2e-gpu-w7900", lane_if("", "w7900")), + ("e2e-gpu-strix-ubuntu", lane_if("", "strix-ubuntu"), true), + ("e2e-gpu-strix-windows", lane_if("", "strix-windows"), true), + ("e2e-wsl", lane_if("", "strix-wsl"), true), + ("e2e-gpu-w7900", lane_if("", "w7900"), true), // Skips because the `serve` output of a skipped `changes` is empty. ( "e2e-report", "always() && (needs.changes.outputs.serve == 'true' \ || github.event_name == 'workflow_dispatch')" .to_owned(), + false, ), ]; let ids = job_ids(&sh); @@ -7529,9 +7533,9 @@ jobs: .iter() .filter(|id| !["changes", "pr-closed"].contains(&id.as_str())) { - let (_, want) = expected + let (_, want, matrix) = expected .iter() - .find(|(name, _)| name == id) + .find(|(name, _, _)| name == id) .unwrap_or_else(|| { panic!( "job `{id}` has no pinned `if`: add it, deciding how it behaves on `closed`" @@ -7543,19 +7547,27 @@ jobs: want, "job `{id}` must skip when `changes` is skipped (a closed PR)" ); - if *id != "e2e-report" { - // `changes` is skipped on `closed`, so its matrix output is empty; - // the fallback keeps the strategy evaluable. + // Only a job pinned as a matrix lane reads the matrix, so a future + // non-matrix helper is not forced to carry it. + if *matrix { + // `changes` is skipped on `closed`, so its matrix output is empty. + // The placeholder applies only then (`&&` yields the string when + // the result is `skipped`, else `false`, so `||` falls through to + // the output): a missing output after `changes` succeeded still + // makes `fromJSON('')` fail instead of quietly running `current`. assert!( - block.contains( - "fromJSON(needs.changes.outputs.sdk_version_matrix || '[\"current\"]')" - ), - "lane `{id}` must fall back to `[\"current\"]` when the matrix output is \ - empty:\n{block}" + block.contains(MATRIX_EXPR), + "lane `{id}` must read its matrix as `{MATRIX_EXPR}`:\n{block}" + ); + assert_eq!( + block.matches("sdk_version_matrix").count(), + 1, + "lane `{id}` must read `sdk_version_matrix` only through that guarded \ + expression:\n{block}" ); } } - for (name, _) in &expected { + for (name, _, _) in &expected { assert!( ids.iter().any(|id| id == name), "pinned job `{name}` is gone" From 299b2ec6b41533d3a4e5b68b2a07a1ba25c1e0fc Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 17:03:10 +0000 Subject: [PATCH 8/9] test(ci): declare MATRIX_EXPR above the test's statements clippy items_after_statements (-D warnings) rejected the const declared after the test's first let, which stopped the xtask test target compiling in the clippy and Windows debug jobs. Declare it at module level. Signed-off-by: Tomas Saaristola --- xtask/src/workflow_contract.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 698ced2de..fd1646e17 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7448,6 +7448,11 @@ jobs: .to_owned() } + /// How every matrix lane must read its matrix: the placeholder only when + /// `changes` was skipped (a closed PR), the real output otherwise. + const MATRIX_EXPR: &str = "fromJSON(needs.changes.result == 'skipped' && '[\"current\"]' \ + || needs.changes.outputs.sdk_version_matrix)"; + #[test] fn closing_a_pr_supersedes_its_self_hosted_runs() { // A merged or closed PR's queued GPU run is dead work that still takes a @@ -7498,8 +7503,6 @@ jobs: // did, while a prefix check still passed. A new job has no entry here // and fails until it is added, which is the point: someone must decide // how it behaves on `closed`. - const MATRIX_EXPR: &str = "fromJSON(needs.changes.result == 'skipped' && '[\"current\"]' \ - || needs.changes.outputs.sdk_version_matrix)"; let lane_if = |extra: &str, platform: &str| { format!( "always() && needs.changes.result == 'success' && {extra}\ From 147af4ee9082c71a46bb91fd36ba2fb05ce28fac Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 21:54:34 +0000 Subject: [PATCH 9/9] ci: say the report skips too, and name the pinned-if table's fields The changes comment said every lane gates on changes' result, but e2e-report skips because a skipped job's output is empty; say both. Replace the table's unnamed bool with a PinnedIf struct whose matrix_lane field says what it is. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 4 +- xtask/src/workflow_contract.rs | 58 +++++++++++++++++----------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index eaaede47c..eb96c7a9f 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -100,7 +100,9 @@ jobs: # forced true, matching ci.yml. changes: runs-on: ubuntu-24.04 # one LTS: path filter only, nothing here depends on the image - # Skipped on `closed`: every lane gates on `needs.changes.result == 'success'`. + # Skipped on `closed`, and with it every lane and `e2e-report`: the lanes gate + # on `needs.changes.result == 'success'`, the report on + # `needs.changes.outputs.serve`, which is empty for a skipped job. if: github.event_name != 'pull_request' || github.event.action != 'closed' permissions: contents: read diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index fd1646e17..90b32bf58 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7448,6 +7448,14 @@ jobs: .to_owned() } + /// The `if` a job must carry so it skips on a closed PR, and whether the job + /// is a matrix lane (and so must also read its matrix through `MATRIX_EXPR`). + struct PinnedIf { + job: &'static str, + condition: String, + matrix_lane: bool, + } + /// How every matrix lane must read its matrix: the placeholder only when /// `changes` was skipped (a closed PR), the real output otherwise. const MATRIX_EXPR: &str = "fromJSON(needs.changes.result == 'skipped' && '[\"current\"]' \ @@ -7512,33 +7520,38 @@ jobs: && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" ) }; - let expected: Vec<(&str, String, bool)> = vec![ - ( + let lane = |job, extra: &str, platform: &str| PinnedIf { + job, + condition: lane_if(extra, platform), + matrix_lane: true, + }; + let expected = vec![ + lane( "e2e-gpu", - lane_if("github.event_name != 'merge_group' && ", "app-dev-gpu"), - true, + "github.event_name != 'merge_group' && ", + "app-dev-gpu", ), - ("e2e-gpu-strix-ubuntu", lane_if("", "strix-ubuntu"), true), - ("e2e-gpu-strix-windows", lane_if("", "strix-windows"), true), - ("e2e-wsl", lane_if("", "strix-wsl"), true), - ("e2e-gpu-w7900", lane_if("", "w7900"), true), + lane("e2e-gpu-strix-ubuntu", "", "strix-ubuntu"), + lane("e2e-gpu-strix-windows", "", "strix-windows"), + lane("e2e-wsl", "", "strix-wsl"), + lane("e2e-gpu-w7900", "", "w7900"), // Skips because the `serve` output of a skipped `changes` is empty. - ( - "e2e-report", - "always() && (needs.changes.outputs.serve == 'true' \ - || github.event_name == 'workflow_dispatch')" + PinnedIf { + job: "e2e-report", + condition: "always() && (needs.changes.outputs.serve == 'true' \ + || github.event_name == 'workflow_dispatch')" .to_owned(), - false, - ), + matrix_lane: false, + }, ]; let ids = job_ids(&sh); for id in ids .iter() .filter(|id| !["changes", "pr-closed"].contains(&id.as_str())) { - let (_, want, matrix) = expected + let pinned = expected .iter() - .find(|(name, _, _)| name == id) + .find(|pinned| pinned.job == id) .unwrap_or_else(|| { panic!( "job `{id}` has no pinned `if`: add it, deciding how it behaves on `closed`" @@ -7546,13 +7559,13 @@ jobs: }); let block = uncommented_job_block(&sh, id); assert_eq!( - &job_level_if(&block), - want, + job_level_if(&block), + pinned.condition, "job `{id}` must skip when `changes` is skipped (a closed PR)" ); // Only a job pinned as a matrix lane reads the matrix, so a future // non-matrix helper is not forced to carry it. - if *matrix { + if pinned.matrix_lane { // `changes` is skipped on `closed`, so its matrix output is empty. // The placeholder applies only then (`&&` yields the string when // the result is `skipped`, else `false`, so `||` falls through to @@ -7570,10 +7583,11 @@ jobs: ); } } - for (name, _, _) in &expected { + for pinned in &expected { assert!( - ids.iter().any(|id| id == name), - "pinned job `{name}` is gone" + ids.iter().any(|id| id == pinned.job), + "pinned job `{}` is gone", + pinned.job ); } }