From 627c04e144de7c4f4aa05466475c930f83067661 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 18 Nov 2026 12:00:05 +0200 Subject: [PATCH] ci(nightly): let a failing GPU lane fail the Nightly run (ROCMAI-444) Every nightly E2E lane set job-level `continue-on-error: true`, so a failed lane read `success` both in `needs..result` and in the run's conclusion: a broken lane looked green everywhere but its own log. The masking bought nothing. Each lane is its own job and its matrix sets `fail-fast: false`, so an unmasked failure cancels no other lane or leg, and the consolidated report job runs on `always()`, so a red lane is still reported. Remove `continue-on-error` from the nightly E2E lanes. Add the `e2e_lanes_do_not_mask_failures_with_continue_on_error` contract test: a fail-closed line scan of every self-hosted E2E lane in nightly.yml and e2e-selfhosted.yml, at job and step level, with no allow-list. Its known limits are pinned as unit-test rows, a vacuity guard fails it when no lane is found, and it pins `e2e-report-nightly` to `if: always()`. Correct the workflow comments, xtask comments and docs that described the lanes as masked. The E2E wiki page updates only when e2e-report-nightly succeeded and its other guards hold, and release readiness reads the report job's result and failure total rather than the lane results, so it can read GO while a lane is red. Signed-off-by: Roman Inflianskas --- .github/workflows/e2e-selfhosted.yml | 20 +- .github/workflows/nightly.yml | 40 ++-- docs/ci-hardware-testing.md | 45 ++++- tests/e2e-cucumber/README.md | 5 +- xtask/src/main.rs | 6 +- xtask/src/release_readiness.rs | 35 ++-- xtask/src/workflow_contract.rs | 265 +++++++++++++++++++++++++-- 7 files changed, 342 insertions(+), 74 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 93c2c4c36..5361ebb4b 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -787,7 +787,8 @@ jobs: # CARGO_HOME override (pool hosts have room under the default paths, unlike # the full-partition static host this replaced), though CARGO_TARGET_DIR # still points at RUNNER_WORKSPACE to dodge checkout's `git clean -ffdx` - # mid-job. Non-blocking while this hardware is proven out. + # mid-job. Non-blocking: its check is not a required status check (see the + # header), but a failure still turns it red. e2e-gpu-strix-ubuntu: name: E2E tests (Strix Halo, Ubuntu, ${{ matrix.channel }}, ${{ matrix.sdk_version }}) # 45min: see e2e-gpu — one collapsed job runs all serves + per-scenario @@ -972,7 +973,8 @@ jobs: # First real Windows GPU coverage: the Strix Halo Windows 11 runner. The # existing windows-debug/windows-release jobs use GitHub-hosted windows-latest, - # which has no GPU. Non-blocking so it never gates the PR. + # which has no GPU. Non-blocking: its check is not a required status check + # (see the header), but a failure still turns it red. e2e-gpu-strix-windows: name: E2E tests (Strix Halo, Windows, ${{ matrix.sdk_version }}) # 45min: see e2e-gpu — one collapsed job runs all serves + per-scenario @@ -2007,10 +2009,10 @@ jobs: # A lane that dies before writing report.json (preflight, build, a crash # before results exist) uploads nothing, and `discover()` can only render # the reports that arrived, so that lane would drop out of the grid with - # no row and no FAIL. On a `continue-on-error` lane nothing else is red - # either. So check it here: every lane that ran (any `needs` result but - # `skipped`, and not self-skipped as `stale`) must have uploaded its report - # artifact. This reads the run's + # no row and no FAIL. The lane's own job is red, but nothing in the grid + # would say which report is absent. So check it here: every lane that ran + # (any `needs` result but `skipped`, and not self-skipped as `stale`) must + # have uploaded its report artifact. This reads the run's # artifact list rather than e2e-artifacts/, because download-artifact # flattens a single match into the root and loses its name. The list also # holds earlier attempts' artifacts, so a re-run that uploads nothing can @@ -2022,9 +2024,9 @@ jobs: # stand-in for the whole lane. LANE_REPORTS, and this script's # behaviour, # are pinned by xtask's workflow contract tests. The annotation quotes the - # lane's `needs` result verbatim; on a `continue-on-error` lane it may read - # `success` even though the lane failed, so the absent artifact is the - # signal, not that value. `!cancelled()`, not `always()`: a run + # lane's `needs` result verbatim, but the absent artifact is the signal, + # not that value: a lane whose other leg failed reads `failure` whether or + # not this leg uploaded. `!cancelled()`, not `always()`: a run # superseded mid-flight cuts lanes off before upload, which is not a gap. - name: Check every lane that ran uploaded its report if: ${{ !cancelled() }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b27ff7ab8..32dc4f8c3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -404,14 +404,20 @@ jobs: # to stay fast. # Runs unconditionally on the schedule/dispatch (independent of the nightly # BUILD skip above — a regression run is worth doing even with no new commits). - # Non-blocking, mirrors e2e-selfhosted.yml's e2e-gpu job (minus its PR-only - # stale-run guard) but sets E2E_INCLUDE_NIGHTLY. + # Mirrors e2e-selfhosted.yml's e2e-gpu job (minus its PR-only stale-run + # guard) but sets E2E_INCLUDE_NIGHTLY. + # Like every nightly lane, it sets no `continue-on-error`: a failing leg fails + # this job and the Nightly run, and `needs` downstream reads `failure`. The + # report and release-readiness jobs below still run on such a night. The E2E + # wiki page is updated only when e2e-report-nightly succeeded (it fails, for + # example, when a lane that ran uploaded no report) and the other guards in + # publish-e2e-wiki's `if:` hold; otherwise the previous page stays in place. + # Nightly gates no merge or tag, so a red run blocks nothing; it only reports. e2e-gpu-nightly: name: E2E tests (MI300X, incl. nightly-only) timeout-minutes: 90 # `mi300x`, not `amd-gpu` — see the same pin on e2e-selfhosted.yml's e2e-gpu. runs-on: [self-hosted, linux, mi300x] - continue-on-error: true strategy: fail-fast: false matrix: @@ -530,7 +536,6 @@ jobs: # the 12h run cap and the lane never ran. Pool hosts are cold, # so there are no host-specific HOME or cache-dir overrides. runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] - continue-on-error: true strategy: fail-fast: false matrix: @@ -620,7 +625,6 @@ jobs: name: E2E tests (Strix Halo, Windows, incl. nightly-only) timeout-minutes: 90 runs-on: [self-hosted, windows, devlab-dispatch, strix-halo] - continue-on-error: true strategy: fail-fast: false matrix: @@ -890,7 +894,6 @@ jobs: # conservative starting budget, tune down after a measured run. timeout-minutes: 120 runs-on: [self-hosted, windows, devlab-dispatch, strix-halo] - continue-on-error: true strategy: fail-fast: false matrix: @@ -1341,8 +1344,8 @@ jobs: # `error` makes that a failed step here, in the lane that owns the # copy, where default `warn` would upload nothing without a word. # The consolidated report job's missing-report check flags the absent - # artifact too, and is the only signal on a `continue-on-error` lane: - # a failed step there does not turn the lane's check red. Only the + # artifact too, so the gap is named in the consolidated report job as + # well as here, where the failed step turns this lane red. Only the # WSL lanes need this guard: every other lane writes report.json # directly in-job, with no guest-to-host copy step that can fail on # its own. A harness preflight refusal is not one of those: @@ -1375,9 +1378,9 @@ jobs: - e2e-gpu-nightly-strix-windows - e2e-wsl-nightly # Unconditional `always()`, unlike the per-PR workflow: there is no paths - # filter to consult on a schedule, and the lanes are `continue-on-error`, so - # a failed lane must still be reported. A nightly that broke is precisely - # when the grid is worth reading. + # filter to consult on a schedule, and a failed lane fails its job, which + # would skip a job with the default `success()` condition. A nightly that + # broke is precisely when the grid is worth reading. if: always() steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1534,10 +1537,11 @@ jobs: # step publishes it, so it shows as an EMPTY row (a post-run lane failure # as a FAIL row with lane_failure), not as missing. `discover()` can only # render the reports that arrived, so a lane that uploads nothing would - # drop out of the grid with no row and no FAIL. On a `continue-on-error` - # lane nothing else is red either. So check it here: every lane that ran - # (any `needs` result but `skipped`, and not self-skipped as `stale`) - # must have uploaded its report artifact. This reads the run's + # drop out of the grid with no row and no FAIL. The lane's own job is red, + # but nothing in the grid would say which report is absent. So check it + # here: every lane that ran (any `needs` result but `skipped`, and not + # self-skipped as `stale`) must have uploaded its report artifact. This + # reads the run's # artifact list rather than e2e-artifacts/, because download-artifact # flattens a single match into the root and loses its name. The list also # holds earlier attempts' artifacts, so a re-run that uploads nothing can @@ -1548,9 +1552,9 @@ jobs: # checked explicitly, comma-separated, rather than reading one leg as a # stand-in for the whole lane. LANE_REPORTS, and this script's behaviour, # are pinned by xtask's workflow contract tests. The annotation quotes the - # lane's `needs` result verbatim; on a `continue-on-error` lane it may read - # `success` even though the lane failed, so the absent artifact is the - # signal, not that value. `!cancelled()`, not `always()`: a run + # lane's `needs` result verbatim, but the absent artifact is the signal, + # not that value: a lane whose other leg failed reads `failure` whether or + # not this leg uploaded. `!cancelled()`, not `always()`: a run # superseded mid-flight cuts lanes off before upload, which is not a gap. - name: Check every lane that ran uploaded its report if: ${{ !cancelled() }} diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index d09b6eb6b..837610b3c 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -417,7 +417,10 @@ via `cargo xtask release-readiness`: it gates on the nightly Linux/Windows build results, the consolidated E2E failure total, and the `e2e-report-nightly` job's own result (so a lane that dies before uploading a report still fails the gate), and buckets commits since the last release tag into -features/fixes/breaking changes. Unlike the wiki-publish jobs above, the +features/fixes/breaking changes. It reads `e2e-report-nightly`'s result and +the report's failure total, not the lane jobs' own results, so it can read GO +while a lane is red for a reason the report does not record as a failure. +Unlike the wiki-publish jobs above, the `release-readiness-nightly` and `release-readiness-narrative` jobs have no `main`/repo guard — they run on any branch this workflow is dispatched on. The narrative job drafts prose from those facts via `actions/ai-inference` @@ -706,14 +709,38 @@ The self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, and `e2e-wsl` — all run with check names that are absent from branch protection's required-status-check list (see below), so a hardware failure never gates a PR merge no matter how -it reports. They run with `continue-on-error: false`, so a real -regression shows red instead of always green. Their results also surface in -the self-hosted consolidated report for visibility. A lane that ran but uploaded -no report (it died before writing its results) fails the consolidated report -job, naming the lane, rather than silently dropping out of the grid (a lane that -skipped itself as stale is the exception); on a -`continue-on-error: true` lane that is the only red signal for such a failure. -The nightly report job applies the same check to every nightly lane. +it reports. Each sets `continue-on-error: false`, so a real regression shows +red instead of always green. Their results also surface in the self-hosted +consolidated report for visibility. A lane that ran but uploaded no report (it +died before writing its results) fails the consolidated report job, naming the +lane, rather than silently dropping out of the grid (a lane that skipped itself +as stale is the exception). The nightly report job applies the same check to +every nightly lane. + +No E2E lane in [`nightly.yml`](../.github/workflows/nightly.yml) sets +`continue-on-error`, on the job or on any step, so a leg that fails turns its +lane and the whole Nightly run red. (Outside the lanes it is still used: by +the `release-readiness-narrative` job, by steps of +`publish-release-readiness-wiki`, and by the queue-import steps of +`e2e-report-nightly`, which must never stop the report.) Job-level +`continue-on-error: true` would instead make the failed lane read `success`, +both in `needs..result` and in the run's conclusion, and a step-level one +would let the failing step pass the job. Nightly gates no merge or tag either +way. On a red +night the consolidated report and release-readiness jobs still run, and the E2E +wiki page is updated only when `e2e-report-nightly` succeeded and +`publish-e2e-wiki`'s other guards hold; the report job fails, for example, when +a lane that ran uploaded no report, and then the previous page stays in place. +`e2e_lanes_do_not_mask_failures_with_continue_on_error` in +[`workflow_contract.rs`](../xtask/src/workflow_contract.rs) enforces this with a +line scan of every self-hosted lane in both workflows that is fail-closed for +the spellings it reads: outside comments, any line mentioning +`continue-on-error` that is not exactly `continue-on-error: false` (or +`- continue-on-error: false`) fails the test, even inside a `run:` script, and +the failure names each such line with its lane. There is no exception list: no +self-hosted E2E lane in either workflow may carry such a line. What the scan +cannot see, including spellings it misses, is listed in the +`continue_on_error_lines` doc comment. A lane the harness failed after its run (no scenario executed, or no GPU scenario, `@requires-gpu` or `@requires-multi-gpu`, on an `E2E_EXPECT_GPU=1` diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 967e98ba3..b4aaf4a19 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -246,10 +246,11 @@ exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to skip there. The nightly workflow runs the MI300X and the Strix Halo hardware from the -table above as non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-strix`, +table above, in its own lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-strix`, `e2e-gpu-nightly-strix-windows`, `e2e-wsl-nightly`) with `E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform -grid. R9700, MI350P and W7900 no longer have a self-hosted nightly lane in +grid. They gate no merge, but a failing lane turns the Nightly run red. +R9700, MI350P and W7900 no longer have a self-hosted nightly lane in `nightly.yml`: they run on the queue ([nightly-on-the-queue.md](../../docs/nightly-on-the-queue.md)), and the nightly report adds the queue's lanes to that grid. The shared large-model scenario follows each lane's serve engine: it serves `Qwen/Qwen3.6-27B` through vLLM on the Instinct lanes (MI300X, MI350P), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 0fda6b30e..122d0ce6f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -378,9 +378,9 @@ pub(crate) enum Command { /// That job already fails if any E2E lane that ran didn't upload its /// `report.json` (preflight failure, timeout, runner loss), which would /// otherwise just vanish from the consolidated E2E report's totals, - /// reading as zero failures. Gating on this job's result (rather than - /// each lane job's own `continue-on-error`-masked result, which is - /// always `success`) covers every lane, present and future, for free. + /// reading as zero failures. Gating on this job's result covers every + /// lane, present and future, with no second copy of the lane list. The + /// lane jobs' own results are not read (see docs/ci-hardware-testing.md). /// Caveat: a re-run can pass on an earlier attempt's artifact, so this /// can miss a lane. #[arg(long)] diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index 1c85f2e26..79bcfa63a 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -142,18 +142,17 @@ fn assemble_facts( Some(_) => {} } // The Total row above only counts platforms whose `report.json` actually - // got uploaded — a GPU lane that dies before uploading (preflight - // failure, timeout, runner loss) just drops out of it, silently reading - // as zero failures. Every lane is `continue-on-error`, so the lane job's - // own `needs..result` reads `success` even when it failed; gating on - // that (as an earlier version of this code did) can never fire. Gate on - // `e2e-report-nightly`'s result instead: its "every lane that ran - // uploaded its report" check (`nightly.yml`, `LANE_REPORTS`) already - // fails that job when a lane is silently missing, covering every lane in - // `LANE_REPORTS` — including ones added later — for free (a re-run can - // pass on an earlier attempt's artifact, so it can still miss a lane). Unlike the - // build gates, this job is `if: always()` and has no legitimate - // no-op-night skip, so `skipped` isn't accepted here. + // got uploaded — a GPU lane that dies before uploading (preflight failure, + // timeout, runner loss) just drops out of it, silently reading as zero + // failures. `e2e-report-nightly`'s "every lane that ran uploaded its + // report" check (`nightly.yml`, `LANE_REPORTS`) fails that job when a lane + // is silently missing, covering every lane in `LANE_REPORTS` — including + // ones added later — without a second copy of the lane list here (a re-run + // can pass on an earlier attempt's artifact, so it can still miss a lane), + // so gate on that job's result. The lane jobs' own results are not read; + // docs/ci-hardware-testing.md's readiness paragraph says what that leaves + // out. Unlike the build gates, this job is `if: always()` and has no + // legitimate no-op-night skip, so `skipped` isn't accepted here. if e2e_report_job_result != "success" { failing_gates.push(format!("E2E (lane report check: {e2e_report_job_result})")); } @@ -430,11 +429,13 @@ fn queue_absent_reason(markdown: &str) -> Option { /// /// The second part exists because two kinds of failed row contribute `0` to /// the `Fail` column, so the Total row's own Fail cell has no record of them: -/// an `EMPTY` platform (no `report.json`, or one that's empty/unparseable), -/// and a `FAIL` platform whose only problem is the harness's post-run lane -/// check (`lane_failure` in its `platform.json`, e.g. "0 scenarios executed"), -/// which fails no scenario cell. A `FAIL` row with a nonzero own `Fail` cell -/// is already folded into the Total row and is not counted again. +/// an `EMPTY` platform (a `report.json` with no scenarios, or one that is empty +/// or unparseable, with no `platform.json` beside it; a missing `report.json` +/// gives no row at all), and a `FAIL` platform whose only problem is the +/// harness's post-run lane check (`lane_failure` in its `platform.json`, e.g. +/// "0 scenarios executed"), which fails no scenario cell. A `FAIL` row with a +/// nonzero own `Fail` cell is already folded into the Total row and is not +/// counted again. /// /// A row for a shared-queue lane the queue does not block on (its platform /// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 120d60bc3..4e5bd5b82 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2465,9 +2465,10 @@ printf '%s' \"$pr\" | jq -r \"$q\"\n"; fn every_lane_that_runs_is_checked_for_its_report_artifact() { // A lane that dies before writing report.json uploads nothing, and the // consolidated grid can only render reports that arrived, so the lane - // drops out with no row and no FAIL — and on a `continue-on-error` lane - // nothing else is red either. The report job's missing-report check is - // what catches that, and it only knows a lane through LANE_REPORTS. + // drops out with no row and no FAIL. The lane's own job is red, but + // nothing in the grid says which report is absent. The report job's + // missing-report check is what names it, and it only knows a lane + // through LANE_REPORTS. // // Derived from the lanes, like its siblings: a new lane fails here until // it is added to the map under the exact artifact name its upload step @@ -2741,8 +2742,10 @@ printf '%s' \"$pr\" | jq -r \"$q\"\n"; "the summary must name the lane: {summary}" ); - // A continue-on-error lane may present as `success`; the missing - // artifact alone has to be enough. + // The `needs` result does not say which leg failed to upload, and a + // masked lane (which `e2e_lanes_do_not_mask_failures_with_continue_on_error` + // forbids) would read `success`; the missing artifact alone has to be + // enough. let (code, log, _) = run_report_check(&step, &all("success"), Some(&without_victim)); assert_eq!( code, 1, @@ -3761,9 +3764,8 @@ printf '%s' \"$pr\" | jq -r \"$q\"\n"; /// cucumber measures scenario durations by subtracting `SystemTime` stamps, so /// a guest whose clock steps backwards mid-run kills the suite (#455 makes that /// survivable; this keeps the durations real). The hazard is invisible: the - /// step's absence breaks nothing that any other test here observes, and on the - /// nightly lane job-level `continue-on-error: true` means a regression would - /// not even turn the run red. + /// step's absence breaks nothing that any other test here observes: a clock + /// that never steps backwards on a given night passes the lane either way. /// /// Steps are matched as WHOLE LINES, not found as substrings of the job: the /// surrounding comments discuss the clock at length, and a comment line that @@ -5630,13 +5632,15 @@ fi /// `release-readiness-nightly` must gate E2E lane coverage through /// `e2e-report-nightly`'s own result, not through each lane job's own - /// `continue-on-error: true`-masked result (which is always `success` - /// regardless of what actually happened). Ties the gate to the one place - /// that already fails correctly when a lane silently drops out — see + /// result. Readiness reads failing scenarios from the consolidated report's + /// failure total and a lane that uploaded no report from the report job's + /// own result (docs/ci-hardware-testing.md says what that leaves out). Ties + /// the gate to the one place that fails correctly when a lane silently + /// drops out — see /// `every_lane_that_runs_is_checked_for_its_report_artifact`, which pins /// that `e2e-report-nightly` itself is driven by `LANE_REPORTS` and so - /// covers every lane, including ones added later, without this job - /// needing its own copy of the lane list. + /// covers every lane, including ones added later, without this job needing + /// its own copy of the lane list. #[test] fn release_readiness_nightly_gates_on_the_e2e_report_job_not_individual_lanes() { let nightly = read_workflow("nightly.yml"); @@ -5669,8 +5673,8 @@ fi assert!( !needs.contains(&lane.to_string()), "release-readiness-nightly must not depend on individual E2E lane job `{lane}` \ - directly: continue-on-error on that job means its own `result` is always \ - `success`, so gating on it can never fire" + directly: it gates on e2e-report-nightly, whose result also covers a lane \ + that uploaded no report, so one copy of the lane list is enough" ); } @@ -5693,6 +5697,235 @@ fi ); } + /// Every line of a self-hosted lane's job block in `text` that may set + /// `continue-on-error` to anything but `false`, as `job: line` (the line + /// trimmed), lanes and lines in file order. + /// + /// A fail-closed line scan rather than a YAML parse. Each line is cut at + /// its first ` #` (a YAML comment; in an indented job block this also + /// empties a whole comment line), and counts when what is left contains + /// `continue-on-error` anywhere, unless it is exactly `continue-on-error: + /// false` or `- continue-on-error: false`. That reports a true value, an + /// expression, a quoted key or value, a flow mapping and extra spaces after + /// the `- `. A `run:` script that merely contains the text is reported too, + /// deliberately: telling script text from YAML keys is the parsing this + /// scan exists to avoid, and a false alarm costs a reword where a missed + /// mask costs a green run over a red lane. + /// + /// What it does not see, by the same choice not to parse YAML: anything + /// after a ` #`, which is usually a comment (a comment cannot mask) but + /// also cuts a quoted value containing ` #` and whatever follows it on the + /// line, such as `- {name: "Run #1", continue-on-error: true}`; the key in + /// another case, such as `CONTINUE-ON-ERROR: true`, since the match is + /// case-sensitive; a value a lane takes from a YAML anchor or merge key + /// (`<<: *defaults`) defined outside its job block; and anything after a + /// comment line indented like a job id (` # note`), where `job_block` + /// ends the lane. In the other direction, a comment at column 0 inside a + /// lane has no space before its `#`, so it is not cut and a mention of the + /// key there is reported. The unit test pins each of these as a known + /// limit. + fn continue_on_error_lines(text: &str) -> Vec { + self_hosted_e2e_jobs(text) + .into_iter() + .flat_map(|(job, _)| { + job_block(text, &job) + .lines() + .map(|line| strip_comment(line).trim()) + .filter(|code| { + code.contains("continue-on-error") + && !matches!( + *code, + "continue-on-error: false" | "- continue-on-error: false" + ) + }) + .map(|code| format!("{job}: {code}")) + .collect::>() + }) + .collect() + } + + /// Why `text` (the workflow named `workflow`) breaks the no-mask rule, as + /// one message listing every line `continue_on_error_lines` reports, as + /// `job: line`; `None` when no self-hosted E2E lane has such a line. + /// + /// A job-level `continue-on-error: true` does more than keep the run going: + /// a failed leg then reads `success` in `needs..result` downstream and + /// in the run's conclusion, so a broken lane is indistinguishable from a + /// passing one everywhere but its own log. A step-level one lets the + /// failing step pass the job. The lanes run in separate jobs with + /// `fail-fast: false`, so an unmasked failure does not cancel a sibling + /// lane. Both consolidated report jobs are pinned to run on `always()`, so + /// a failed lane does not skip them: nightly's `e2e-report-nightly` by + /// `e2e_lanes_do_not_mask_failures_with_continue_on_error`, and + /// e2e-selfhosted's `e2e-report` by + /// `closing_a_pr_supersedes_its_self_hosted_runs`. + fn mask_failure(workflow: &str, text: &str) -> Option { + let lines = continue_on_error_lines(text); + if lines.is_empty() { + return None; + } + Some(format!( + "{workflow}: self-hosted E2E lanes may set `continue-on-error` to something \ + other than `false`, on the job or on a step, in these lines: {lines:?}. On the \ + job that makes a failed lane read `success` in `needs..result` and in \ + the run's conclusion; on a step it lets the failing step pass the job. Either \ + way the run goes green while the lane is red. Remove it, or reword a script \ + that only mentions the key (this scan is deliberately fail-closed)" + )) + } + + #[test] + fn mask_failure_names_every_masked_line_and_passes_an_unmasked_workflow() { + // Two lanes: one masked on the job and on a step, one explicitly unmasked. + let workflow = "on: push\njobs:\n quiet:\n runs-on: [self-hosted, linux, mi300x]\n continue-on-error: false\n loud:\n runs-on: [self-hosted, linux, mi300x]\n continue-on-error: ${{ true }}\n steps:\n - continue-on-error: true\n"; + let failure = mask_failure("w.yml", workflow).expect("a masked lane must fail"); + assert!( + failure.starts_with("w.yml: ") + && failure.contains("\"loud: continue-on-error: ${{ true }}\"") + && failure.contains("\"loud: - continue-on-error: true\"") + && !failure.contains("quiet"), + "{failure}" + ); + + assert_eq!( + mask_failure( + "w.yml", + &one_lane_workflow(" continue-on-error: false\n") + ), + None + ); + assert_eq!( + mask_failure( + "w.yml", + &one_lane_workflow(" steps:\n - run: echo\n") + ), + None + ); + } + + /// A one-lane self-hosted workflow: `job_lines` follow the job's + /// `runs-on`, verbatim. + fn one_lane_workflow(job_lines: &str) -> String { + format!("on: push\njobs:\n lane:\n runs-on: [self-hosted, linux, mi300x]\n{job_lines}") + } + + #[test] + fn continue_on_error_scan_accepts_only_a_plain_false() { + let step = |key_line: &str| { + format!(" steps:\n - name: Run\n {key_line}\n run: echo\n") + }; + for (case, job_lines, flagged) in [ + ("job-level true", " continue-on-error: true\n".to_owned(), true), + ("step-level true", step("continue-on-error: true"), true), + ("expression", step("continue-on-error: ${{ true }}"), true), + ("quoted value", step("continue-on-error: 'true'"), true), + ("capitalised False", step("continue-on-error: False"), true), + ( + "flow mapping", + " steps:\n - {name: x, continue-on-error: true}\n".to_owned(), + true, + ), + ( + "extra spaces after the dash", + " steps:\n - continue-on-error: true\n run: echo\n".to_owned(), + true, + ), + ("quoted key", step("\"continue-on-error\": true"), true), + ( + "text in a run script (fail-closed)", + " steps:\n - name: Run\n run: |\n continue-on-error: true\n" + .to_owned(), + true, + ), + ("plain false", step("continue-on-error: false"), false), + ("false with a trailing comment", step("continue-on-error: false # why"), false), + ( + "dash-line false", + " steps:\n - continue-on-error: false\n run: echo\n".to_owned(), + false, + ), + ( + "a comment line", + " # no continue-on-error: true here\n steps:\n - run: echo\n".to_owned(), + false, + ), + ("absent", step("shell: bash"), false), + // Known limits, pinned so they stay documented behaviour rather + // than silent holes (see the doc comment on the scan). + ( + "limit: a quoted ` #` cuts the rest of the line", + " steps:\n - {name: \"Run #1\", continue-on-error: true}\n".to_owned(), + false, + ), + ("limit: the match is case-sensitive", step("CONTINUE-ON-ERROR: true"), false), + ( + "limit: a merge key from outside the job block", + " <<: *lenient_defaults\n steps:\n - run: echo\n".to_owned(), + false, + ), + ( + "limit: a comment indented like a job id ends the lane (missed)", + " # note\n continue-on-error: true\n".to_owned(), + false, + ), + ( + "limit: a column-0 comment is not cut (flagged)", + "# continue-on-error: true\n steps:\n - run: echo\n".to_owned(), + true, + ), + ] { + assert_eq!( + !continue_on_error_lines(&one_lane_workflow(&job_lines)).is_empty(), + flagged, + "{case}" + ); + } + + assert_eq!( + continue_on_error_lines(&one_lane_workflow(&step("continue-on-error: ${{ true }}"))), + ["lane: continue-on-error: ${{ true }}"] + ); + assert_eq!( + continue_on_error_lines(&one_lane_workflow( + " steps:\n - {name: x, continue-on-error: true}\n" + )), + ["lane: - {name: x, continue-on-error: true}"] + ); + } + + #[test] + fn e2e_lanes_do_not_mask_failures_with_continue_on_error() { + for (workflow, text) in self_hosted_workflows() { + let lanes = self_hosted_e2e_jobs(&text); + // Guards against a vacuous pass: a parsing change that found no + // lanes would otherwise check nothing. + let expected_lane = match workflow { + "e2e-selfhosted.yml" => "e2e-gpu", + "nightly.yml" => "e2e-gpu-nightly", + other => panic!("no known lane for {other}"), + }; + assert!( + lanes.iter().any(|(job, _)| job == expected_lane), + "{workflow}: found no `{expected_lane}` among its self-hosted lanes {lanes:?}, \ + so nothing below would be checked" + ); + if let Some(failure) = mask_failure(workflow, &text) { + panic!("{failure}"); + } + } + + // With the lanes unmasked, a failed lane fails its job, and the default + // `success()` condition would then skip the consolidated report on the + // very night it is worth reading. + let nightly = read_workflow("nightly.yml"); + let condition = job_scalar(job_block(&nightly, "e2e-report-nightly"), "if"); + assert!( + ["always()", "${{ always() }}"].contains(&condition), + "e2e-report-nightly must run on `always()` so a failed lane is still \ + reported; found `if: {condition}`" + ); + } + #[test] fn release_readiness_nightly_runs_even_if_an_upstream_job_failed() { let nightly = read_workflow("nightly.yml"); @@ -6026,7 +6259,7 @@ fi ); let nightly_ids: Vec = nightly_lanes.iter().map(|(job, _)| job.clone()).collect(); assert_eq!( - backticked_list_between(&readme_flat, "as non-blocking lanes (", ") with"), + backticked_list_between(&readme_flat, "above, in its own lanes (", ") with"), nightly_ids, "the E2E README must name every nightly self-hosted lane, in workflow order" );