From a79fbff7ce2f1b6bfdc36c7a33c92582ce1d8c37 Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Tue, 3 Nov 2026 12:23:31 +0200 Subject: [PATCH] ci(e2e): move the AIM serve scenarios off the per-PR Strix Ubuntu lane The self-hosted `E2E tests (Strix Halo, Ubuntu, ...)` jobs have been cancelled at their 45 minute cap on every run that contains the AIM engine (84f951e2). Before it the same lane passed in 25-26 minutes, and runs whose merge ref lacked it still did. Cause: serve-26 and serve-27 are the last two of 183 scenarios in the suite and are tagged `@requires-engine:aim @serve-timeout:1200`, so each may wait up to 20 minutes for readiness. The pool runner is cold on every job, so the AIM image and a 9B checkpoint are pulled each time. About 25 minutes for the other scenarios plus serve-26's wait lands on the 45 minute cap with serve-27 just starting; the cancelled run's command log ends inside serve-27. The W7900 and MI300X lanes never run them (the harness needs Docker and a Radeon/Strix Halo gfx family), so only this lane was affected. A lane failure drops a merge-queue entry. Fix: tag both scenarios `@nightly` (the existing opt-in, so ids and numbering are unchanged), pin that with a unit test, and give the lanes that do opt in room for them: the nightly Strix lane goes from 90 to 120 minutes and the per-PR Strix Ubuntu job gets 105 minutes when an `include_nightly` dispatch is run. Per-PR runs keep the 45 minute cap. The unit test also pins the opt-in side: with the nightly opt-in set, the scenarios never skip for the nightly reason; on a host that lists aim they run exactly when Docker answers, and otherwise skip on the engine. AIM coverage still needs a nightly host with Docker, which is unconfirmed: the nightly Strix job now logs whether Docker is reachable, and the caveat is in docs/testing.md and the feature comment. Signed-off-by: Eugene Volen --- .github/workflows/e2e-selfhosted.yml | 5 +- .github/workflows/nightly.yml | 17 ++++- docs/testing.md | 18 +++++ .../features/model_serving.feature | 12 ++- tests/e2e-cucumber/src/expectation.rs | 75 +++++++++++++++++++ 5 files changed, 123 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 3a0119f900..89d7a97390 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -420,7 +420,10 @@ jobs: # this replaced, and a run took 26m54s against the old 35min cap (run # 34489547654) versus 18-19m on a warm static host — keep the same headroom # ratio the 35 was chosen for. - timeout-minutes: 45 + # An include_nightly dispatch also runs the AIM serve scenarios (serve-26/27, + # @nightly, up to 1200s readiness each on a cold pull) and the large-model + # serve, which cannot fit the per-PR 45min budget. + timeout-minutes: ${{ inputs.include_nightly && 105 || 45 }} runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e9d00ad47a..8175ae0028 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -718,7 +718,12 @@ jobs: # Strix Ubuntu job. e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) - timeout-minutes: 90 + # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits when + # this host has Docker; on a host without it they skip and cost nothing. + # Whether it has Docker is unconfirmed: nothing in this job installs or checks + # it, so the "Report whether Docker is reachable" step logs the answer. If it + # does not, no nightly lane runs AIM serve and a pool-based one is needed. + timeout-minutes: 120 # `native` disambiguates this runner from any host that might carry a bare # `wsl` label alongside `strix-halo` (the static WSL runner this repo used # before the WSL lanes moved to the ephemeral pool) -- the paths below @@ -770,6 +775,16 @@ jobs: fi echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + # serve-26/27 (AIM) need Docker reachable on this host; without it they skip + # silently. Say which it is, so a nightly that never ran them is visible. + - name: Report whether Docker is reachable (AIM serve scenarios need it) + run: | + if docker info >/dev/null 2>&1; then + echo "docker reachable: serve-26/27 (AIM) can run on this host" + else + echo "::warning::docker is not reachable on this host: serve-26/27 (AIM) will skip" + fi + - name: Run E2E tests on Strix Halo (incl. nightly-only) run: | export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" diff --git a/docs/testing.md b/docs/testing.md index 2738bb50d0..4e11f1c002 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -739,6 +739,24 @@ self-hosted CI dispatch (which forwards `--name`): each bypasses every gate above, so on WSL2, Windows or a machine with no GPU it would still run the real install. +`serve-26` and `serve-27` (`features/model_serving.feature`, the AIM serve and +restart canaries) are `@nightly` for cost: each pulls the AIM image and a 9B +checkpoint on a cold runner and may wait up to 1200s for readiness, which on the +per-PR Strix Ubuntu job (about 25 minutes without them, 45 minute cap) used up +the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable +and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` +is set on such a host: the nightly Strix Halo lane, or an `include_nightly` +dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then gets a 105 minute +cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that +they stay reachable when the opt-in is set. + +Caveat: it is unconfirmed that the nightly Strix Halo host has Docker (nothing in +`nightly.yml` installs or checks it). Without it these two scenarios skip on +every nightly run, and only a manual `include_nightly` dispatch on a host with +Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker +is reachable ("Report whether Docker is reachable"); check that line before +relying on this coverage. + ## Runtime Selection And Activation List registered managed or read-only runtimes and their exact side-by-side keys: diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 9a59d533a9..fe8188cbd5 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -350,12 +350,20 @@ Feature: Model serving # verified on (see `HostCapability::engine_available`'s "aim" arm — Radeon # gfx110X/120X and Strix Halo gfx1151, not Instinct); a Strix Halo # self-hosted runner with Docker is where this actually runs. + # `@nightly` keeps both AIM scenarios off the per-PR lanes: each one pulls the + # image and a 9B checkpoint on a cold runner and may wait out the full + # 1200s readiness cap, so the pair on the per-PR Strix Ubuntu job (~25 min + # without them, 45 min cap) ended every run at the cap. They run only when + # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of + # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. + # Unconfirmed: whether the nightly Strix Halo host has Docker; without it both + # skip every night (the nightly job logs whether Docker is reachable). # `@serve-timeout:1200` # widens both the harness's readiness poll and the CLI's own # ROCM_CLI_AIM_READY_TIMEOUT_SECS (see isolate_cmd) — a cold-start HF weight # download for this 9B model legitimately exceeds the engine's default # 5-minute readiness cap. - @id:serve-aim-inference @requires-gpu @requires-engine:aim @serve-timeout:1200 + @id:serve-aim-inference @requires-gpu @requires-engine:aim @serve-timeout:1200 @nightly Scenario: serve-26 - A model served on AIM responds to inference requests Given a managed runtime is active And a model is being served on aim @@ -370,7 +378,7 @@ Feature: Model serving # on (`stop_managed_services`, via the `stop_aim_service` step), and the # same serve+inference fixture `serve-26` uses, rather than adding a new # stop/restart mechanism just for AIM. - @id:serve-aim-lifecycle-parity @requires-gpu @requires-engine:aim @serve-timeout:1200 + @id:serve-aim-lifecycle-parity @requires-gpu @requires-engine:aim @serve-timeout:1200 @nightly Scenario: serve-27 - A stopped AIM service can be restarted and serves inference again Given a managed runtime is active And a model is being served on aim diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 073ac85baa..d0edcc7b38 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1042,6 +1042,81 @@ serve_timeout_secs = 90 assert!(unmatched_also_run_ids(&[], &known).is_empty()); } + /// The tag line directly above `Scenario: ` in `model_serving.feature`. + fn model_serving_tags(prefix: &str) -> Vec { + let feature = include_str!("../features/model_serving.feature"); + let lines: Vec<&str> = feature.lines().collect(); + let at = lines + .iter() + .position(|l| l.trim_start().starts_with(&format!("Scenario: {prefix}"))) + .unwrap_or_else(|| panic!("no `Scenario: {prefix}` in model_serving.feature")); + lines[at - 1] + .split_whitespace() + .map(str::to_owned) + .collect() + } + + #[test] + fn aim_serve_scenarios_stay_off_the_per_pr_lanes() { + // serve-26/27 pull the AIM image and a 9B checkpoint on a cold runner and + // may each wait out a 1200s readiness cap. Run per-PR on the Strix Ubuntu + // pool they pushed the job (~25 min otherwise) into its 45 min cap on every + // run. They must resolve to Skip unless the nightly opt-in is set, and must + // still be reachable (not Skip for the nightly reason) when it is. + let m = Expectations::default(); + for prefix in ["serve-26 -", "serve-27 -"] { + let d = ScenarioDecl::from_tags(&model_serving_tags(prefix)); + assert_eq!(d.requires_engine.as_deref(), Some("aim"), "{prefix}"); + assert!(d.nightly, "{prefix} must be @nightly"); + match resolve(&d, &cap("strix-ubuntu"), &m, Included::default()) { + Expectation::Skip { reason } => { + assert!(reason.contains("E2E_INCLUDE_NIGHTLY"), "{prefix}: {reason}"); + } + other => panic!("{prefix} must skip per-PR, got {other:?}"), + } + // Opted in: the nightly gate must not fire. What remains is the engine + // requirement, which `engine_available("aim")` decides from the host's + // engine list, its GFX family and whether Docker answers. Pin both + // sides of it: a host that lists aim runs the scenario exactly when + // Docker is reachable, and one that does not list aim skips on the + // engine, never on the nightly opt-in. + let opted_in = Included { + nightly: true, + ..Included::default() + }; + let mut with_aim = cap("strix-ubuntu"); + with_aim.available_engines.push("aim".to_owned()); + match resolve(&d, &with_aim, &m, opted_in) { + Expectation::ExpectPass => assert!( + docker_available(), + "{prefix} ran although Docker is unreachable" + ), + Expectation::Skip { reason } => { + assert!( + !docker_available(), + "{prefix} skipped with the opt-in set, aim listed and Docker reachable: {reason}" + ); + assert!( + reason.contains("engine 'aim' cannot start"), + "{prefix}: wrong skip reason with the opt-in set: {reason}" + ); + } + other @ Expectation::ExpectXfail { .. } => { + panic!("{prefix}: unexpected outcome when opted in: {other:?}") + } + } + match resolve(&d, &cap("strix-ubuntu"), &m, opted_in) { + Expectation::Skip { reason } => assert!( + reason.contains("engine 'aim' cannot start"), + "{prefix}: {reason}" + ), + other => { + panic!("{prefix}: host without aim must skip on the engine, got {other:?}") + } + } + } + } + #[test] fn docker_scenario_skips_unless_included() { // A container-backed scenario is opt-in, not merely "docker is here".