diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 398f9ded6..eb96c7a9f 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -31,6 +31,13 @@ on: # Mirror ci.yml: run on PRs against ANY base branch so a stacked PR still gets # the GPU lanes. Affected-crate/heavy selection diffs against github.base_ref. pull_request: + # `closed` (merge OR close) joins the PR's concurrency group below, so it + # supersedes that PR's earlier run. It runs only the no-op `pr-closed` job; + # every GPU lane skips because `changes` skips. This frees runners only where + # the cancel completes: a job queued for a runner that is not picking it up + # (W7900 while its one runner is busy) may not finish cancelling, so this + # does NOT clear such queued runs. + types: [opened, synchronize, reopened, closed] merge_group: branches: [main] types: [checks_requested] @@ -93,6 +100,10 @@ jobs: # forced true, matching ci.yml. changes: runs-on: ubuntu-24.04 # one LTS: path filter only, nothing here depends on the image + # Skipped on `closed`, and with it every lane and `e2e-report`: the lanes gate + # on `needs.changes.result == 'success'`, the report on + # `needs.changes.outputs.serve`, which is empty for a skipped job. + if: github.event_name != 'pull_request' || github.event.action != 'closed' permissions: contents: read pull-requests: read @@ -183,6 +194,19 @@ jobs: echo 'matrix=["current"]' >> "$GITHUB_OUTPUT" fi + # Exists only to start a run in the PR's concurrency group when the PR is + # merged or closed: that run cancels the PR's older run. A GPU job of that run + # which is queued for a runner that is not picking it up may not finish + # cancelling (observed on W7900-queued runs). + pr-closed: + name: Supersede runs of a closed PR + if: github.event_name == 'pull_request' && github.event.action == 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - run: echo "PR closed; its earlier E2E runs are superseded by this run." + e2e-gpu: name: E2E tests (MI300X, ${{ matrix.sdk_version }}) timeout-minutes: 90 @@ -199,7 +223,7 @@ jobs: # single-element `["current"]` matrix, so the ordinary PR/merge-queue path # runs exactly one job, unchanged from before this matrix existed. matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # No build-and-test gate (cross-workflow needs is unavailable): the job builds # the rocm binary itself, and this check isn't in the required list (see the # file header), so a broken build here still can't block a merge; ci.yml's @@ -463,7 +487,7 @@ jobs: # See `e2e-gpu` (ROCMAI-430): layered alongside the pre-existing # `channel` dimension above, not replacing it — this lane is the only # per-PR job with two independent matrix axes. - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # The nightly channel only carries the latest (pre-)release builds, so # it cannot resolve the n-1/n-2 released versions the matrix pins # (`rocm install sdk` fails with "no mutually compatible TheRock ... @@ -618,7 +642,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-windows. if: >- always() @@ -966,7 +990,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-wsl. Gated on # `serve` like the sibling lanes rather than the broader `heavy`: the # consolidated report gates on `serve` too, so a heavy-but-not-serve change @@ -1574,7 +1598,7 @@ jobs: strategy: fail-fast: false matrix: - sdk_version: ${{ fromJSON(needs.changes.outputs.sdk_version_matrix) }} + sdk_version: ${{ fromJSON(needs.changes.result == 'skipped' && '["current"]' || needs.changes.outputs.sdk_version_matrix) }} if: >- always() && needs.changes.result == 'success' diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index e5185829d..5d34089d1 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -364,7 +364,12 @@ lands is the per-PR canary (`serve-05`), which runs when the `serve` filter matches. `push` fires on `main` and on any `release/**` branch, so pushing to a release branch runs the full self-hosted matrix ahead of cutting the `v*` tag that `release.yml` builds/publishes from — this is the release-candidate regression gate; `pull_request` fires against any base -branch. +branch. Merging or closing a pull request also starts a run (`closed` is a +listed `pull_request` activity type) that joins that PR's group and runs only a +no-op job, so it supersedes the PR's earlier run and frees runners where the +cancel completes. A run whose job is still queued for a runner may not finish +cancelling, so it can keep its place in the queue until the runner picks the +job up. On a release-branch push, every self-hosted GPU job additionally expands into a `strategy.matrix.sdk_version` of `[current, n-1, n-2]` (ROCMAI-430) — three diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 12a0726bd..90b32bf58 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -7416,6 +7416,182 @@ jobs: ); } + /// `text` with every comment removed, so a match cannot come from prose. + fn yaml_without_comments(text: &str) -> String { + text.lines() + .map(strip_comment) + .collect::>() + .join("\n") + } + + /// A job's block with comments removed. + fn uncommented_job_block(text: &str, job: &str) -> String { + yaml_without_comments(job_block(text, job)) + } + + /// The job-level `if:` of an already uncommented job block, with a folded + /// (`>-`) condition joined onto one line. Empty when the job has none. + fn job_level_if(block: &str) -> String { + let mut lines = block.lines(); + let Some(first) = lines.find(|l| l.starts_with(" if:")) else { + return String::new(); + }; + let mut parts = vec![first.trim_start_matches(" if:").trim().to_owned()]; + // Continuation lines are indented deeper than the 4-space job keys. + parts.extend( + lines + .take_while(|l| l.starts_with(" ")) + .map(|l| l.trim().to_owned()), + ); + normalized_whitespace(&parts.join(" ")) + .trim_start_matches(">- ") + .to_owned() + } + + /// The `if` a job must carry so it skips on a closed PR, and whether the job + /// is a matrix lane (and so must also read its matrix through `MATRIX_EXPR`). + struct PinnedIf { + job: &'static str, + condition: String, + matrix_lane: bool, + } + + /// How every matrix lane must read its matrix: the placeholder only when + /// `changes` was skipped (a closed PR), the real output otherwise. + const MATRIX_EXPR: &str = "fromJSON(needs.changes.result == 'skipped' && '[\"current\"]' \ + || needs.changes.outputs.sdk_version_matrix)"; + + #[test] + fn closing_a_pr_supersedes_its_self_hosted_runs() { + // A merged or closed PR's queued GPU run is dead work that still takes a + // runner. `closed` is not a default `pull_request` activity type, so it + // must be listed; the run it starts joins the PR's group (same key as the + // opened/synchronize runs) and so cancels the earlier one. It must run + // nothing but the no-op job: `changes` skips, and with it every lane. + // Every assertion reads one job's comment-free block, so prose or another + // job cannot satisfy it. + let sh = read_workflow("e2e-selfhosted.yml"); + let on = normalized_whitespace(&yaml_without_comments(&top_level_block(&sh, "on"))); + assert!( + on.contains("types: [opened, synchronize, reopened, closed]"), + "pull_request must list `closed` next to the default types, got `{on}`" + ); + + let changes = uncommented_job_block(&sh, "changes"); + assert_eq!( + job_level_if(&changes), + "github.event_name != 'pull_request' || github.event.action != 'closed'", + "the `changes` job must skip on `closed`, or every GPU lane would run for a closed PR" + ); + + let closed = uncommented_job_block(&sh, "pr-closed"); + assert_eq!( + job_level_if(&closed), + "github.event_name == 'pull_request' && github.event.action == 'closed'", + "`pr-closed` must run on `closed` and on nothing else" + ); + assert!( + closed.contains(" runs-on: ubuntu-24.04\n"), + "`pr-closed` must run on a GitHub-hosted runner, never queue for a GPU:\n{closed}" + ); + assert!( + closed.contains(" permissions: {}\n"), + "`pr-closed` must hold no token permissions:\n{closed}" + ); + for forbidden in ["uses:", "needs:", "secrets", "checkout"] { + assert!( + !closed.contains(forbidden), + "`pr-closed` is a no-op and must not contain `{forbidden}`:\n{closed}" + ); + } + + // Every other job must skip when `changes` skips. Each `if` is pinned in + // full, not by prefix: an `||` arm appended to a lane's condition (as + // `&&` binds tighter than `||`) would make it true whatever `changes` + // did, while a prefix check still passed. A new job has no entry here + // and fails until it is added, which is the point: someone must decide + // how it behaves on `closed`. + let lane_if = |extra: &str, platform: &str| { + format!( + "always() && needs.changes.result == 'success' && {extra}\ + ( (github.event_name != 'workflow_dispatch' \ + && needs.changes.outputs.serve == 'true') \ + || (github.event_name == 'workflow_dispatch' \ + && (inputs.platform == 'all' || inputs.platform == '{platform}')) )" + ) + }; + let lane = |job, extra: &str, platform: &str| PinnedIf { + job, + condition: lane_if(extra, platform), + matrix_lane: true, + }; + let expected = vec![ + lane( + "e2e-gpu", + "github.event_name != 'merge_group' && ", + "app-dev-gpu", + ), + lane("e2e-gpu-strix-ubuntu", "", "strix-ubuntu"), + lane("e2e-gpu-strix-windows", "", "strix-windows"), + lane("e2e-wsl", "", "strix-wsl"), + lane("e2e-gpu-w7900", "", "w7900"), + // Skips because the `serve` output of a skipped `changes` is empty. + PinnedIf { + job: "e2e-report", + condition: "always() && (needs.changes.outputs.serve == 'true' \ + || github.event_name == 'workflow_dispatch')" + .to_owned(), + matrix_lane: false, + }, + ]; + let ids = job_ids(&sh); + for id in ids + .iter() + .filter(|id| !["changes", "pr-closed"].contains(&id.as_str())) + { + let pinned = expected + .iter() + .find(|pinned| pinned.job == id) + .unwrap_or_else(|| { + panic!( + "job `{id}` has no pinned `if`: add it, deciding how it behaves on `closed`" + ) + }); + let block = uncommented_job_block(&sh, id); + assert_eq!( + job_level_if(&block), + pinned.condition, + "job `{id}` must skip when `changes` is skipped (a closed PR)" + ); + // Only a job pinned as a matrix lane reads the matrix, so a future + // non-matrix helper is not forced to carry it. + if pinned.matrix_lane { + // `changes` is skipped on `closed`, so its matrix output is empty. + // The placeholder applies only then (`&&` yields the string when + // the result is `skipped`, else `false`, so `||` falls through to + // the output): a missing output after `changes` succeeded still + // makes `fromJSON('')` fail instead of quietly running `current`. + assert!( + block.contains(MATRIX_EXPR), + "lane `{id}` must read its matrix as `{MATRIX_EXPR}`:\n{block}" + ); + assert_eq!( + block.matches("sdk_version_matrix").count(), + 1, + "lane `{id}` must read `sdk_version_matrix` only through that guarded \ + expression:\n{block}" + ); + } + } + for pinned in &expected { + assert!( + ids.iter().any(|id| id == pinned.job), + "pinned job `{}` is gone", + pinned.job + ); + } + } + #[test] fn mi300x_lane_is_skipped_in_the_merge_queue() { // The MI300X check is not required for merging, so a merge_group run's