From a9d568a801ca5b7a38b4c684ed5d74f189842dfb Mon Sep 17 00:00:00 2001 From: juhovainio Date: Fri, 13 Nov 2026 14:59:00 +0200 Subject: [PATCH 1/3] test(e2e): let an xfail row's serve_timeout_secs beat the scenario's serve-timeout tag serve-06 carries @serve-timeout:2400 for the hosts where it passes. On native Windows with Lemonade it is a known failure (ROCMAI-900) whose serve never becomes ready, and the tag won over the xfail row, so each Windows lane waited the full 2400s (about 41 minutes) before failing. A row means the serve is a known bug on this host, so its wait now takes precedence; the tag still lengthens the wait wherever no row matches. The Windows serve-06 row gets serve_timeout_secs = 600 (the step default). Other hosts keep the 2400s window. Signed-off-by: juhovainio --- tests/e2e-cucumber/expectations.toml | 13 +++-- tests/e2e-cucumber/src/expectation.rs | 48 +++++++++++++------ tests/e2e-cucumber/tests/e2e.rs | 13 ++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 7 ++- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 2274bae5b..02e723538 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -27,6 +27,7 @@ # bug = "EAI-NNNN" ticket reference (shown in the report) # reason = "..." human explanation (shown in the report) # serve_timeout_secs = 90 optional: shorter serve wait for a known bug +# (wins over the scenario's @serve-timeout tag) # flaky = true optional: bug reproduces only intermittently, # so an XPASS is tolerated (reported, not stale) # @@ -265,10 +266,13 @@ flaky = true # suspected cause; it has NOT been reproduced off Windows, so the row names both # facts. Both rows are deterministic (both scenarios failed on each Windows lane # run examined: 2026-10-28, 2026-11-05 and 2026-11-06), so neither is `flaky`: an -# XPASS means the cause is gone and the row should be deleted. Neither row sets -# `serve_timeout_secs`: serve-06's own `@serve-timeout:2400` tag takes precedence -# over a row, so a shorter wait here would have no effect, and the failing -# serve-06 still holds the Windows lane for the full 2400s wait. +# XPASS means the cause is gone and the row should be deleted. Only serve-06's row +# sets `serve_timeout_secs`: a row's wait wins over the scenario's own +# `@serve-timeout:2400` tag, and the failing serve never became ready within that +# window on the Windows lane (2026-10-27 and 2026-11-10 runs), holding the lane for +# about 41 minutes per channel. 600s is the step default, so the scenario still gets +# a normal load; if the cause is fixed, delete the row and the tag's 2400s applies +# again. # # The pin is now Lemonade 2026.44.1, which fetches TheRock from AMD's whl-next # wheel index or its core tarball location, and whose pinned llama.cpp build @@ -287,3 +291,4 @@ reason = "Observed with Lemonade 11.5.1, which fetched TheRock only from repo.am when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" reason = "The Windows large-model serve crashed in the SDK's Tensile library on the first inference request while Lemonade 11.5.1 was pinned: its ROCm 7.13 backend ran against the 7.14.1 SDK because 11.5.1 could not align it (suspected cause, not reproduced off Windows). Not yet re-checked with the 2026.44.1 pin; delete this row if the Windows nightly lane reports it as a stale XPASS." +serve_timeout_secs = 600 diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index aa8408d0d..9c589353e 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -380,14 +380,17 @@ impl Expectations { /// The serve-readiness window a scenario runs with on this host, as the /// harness's `before` hook hands it to the serve steps - /// (`E2eWorld::serve_timeout_override`, read by `serve_and_wait`): a - /// `@serve-timeout:` tag on an expected-pass scenario wins, else the - /// shortest matching xfail `serve_timeout_secs`. `None` for a scenario with - /// no `@id:` tag or with neither, which keeps the step default. + /// (`E2eWorld::serve_timeout_override`, read by `serve_and_wait`): the + /// shortest matching xfail `serve_timeout_secs` wins, else the + /// `@serve-timeout:` tag. A row means the serve is a known bug on this + /// host, so it must fail fast even when the scenario carries a long tag for + /// the hosts where it passes; the tag only lengthens an expected-pass serve. + /// `None` for a scenario with no `@id:` tag or with neither, which keeps the + /// step default. pub fn serve_timeout_override(&self, decl: &ScenarioDecl, cap: &HostCapability) -> Option { let id = decl.id.as_deref()?; - decl.serve_timeout_secs - .or_else(|| self.serve_timeout_for(id, cap, decl.effective_engine(cap))) + self.serve_timeout_for(id, cap, decl.effective_engine(cap)) + .or(decl.serve_timeout_secs) } /// Whether any declared condition for this scenario matches this host — the @@ -2411,9 +2414,9 @@ flaky = true /// was pinned, kept until a nightly shows them stale) apply to native /// Windows with lemonade and to nothing else: not to Linux hosts (including one whose /// default engine is vLLM, where the engine named is what decides), and they are not - /// `flaky` (an XPASS means the cause is gone and the row is stale). Neither sets - /// `serve_timeout_secs`: serve-06's own `@serve-timeout` tag takes precedence over a - /// row, so one would change nothing. + /// `flaky` (an XPASS means the cause is gone and the row is stale). Only serve-06's + /// row sets `serve_timeout_secs`, so the known-bad Windows serve stops waiting out the + /// scenario's 2400s tag. #[test] fn lemonade_alignment_rows_apply_only_to_native_windows_with_lemonade() { let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); @@ -2430,16 +2433,25 @@ flaky = true let entries = m.entries_for(id); assert_eq!(entries.len(), 1, "{id}"); assert!(!entries[0].flaky, "{id} is deterministic"); - assert_eq!(entries[0].serve_timeout_secs, None, "{id}"); } - // The row cannot shorten serve-06's wait: its own tag wins, so the harness - // still hands the 2400s the feature file asks for. + assert_eq!( + m.entries_for("runtime-lemonade-backend-alignment-reported")[0].serve_timeout_secs, + None + ); + // serve-06's row shortens the wait on the failing host and nowhere else: the + // harness hands the row's window on native Windows, and still the 2400s the + // feature file asks for on a host where the scenario is expected to pass. let tags = feature_tags( include_str!("../features/model_serving.feature"), "serve-large-model-inference", ); + let decl = ScenarioDecl::from_tags(&tags); + assert_eq!( + m.serve_timeout_override(&decl, &cap("strix-windows")), + Some(600) + ); assert_eq!( - m.serve_timeout_override(&ScenarioDecl::from_tags(&tags), &cap("strix-windows")), + m.serve_timeout_override(&decl, &cap("strix-ubuntu")), Some(2400) ); } @@ -2489,9 +2501,10 @@ flaky = true } #[test] - fn serve_timeout_override_prefers_the_tag_and_needs_an_id() { + fn serve_timeout_override_prefers_the_row_and_needs_an_id() { let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); - // The row says 90s on this host; a `@serve-timeout` tag lengthens it. + // The row says 90s on this host; a `@serve-timeout` tag must not lengthen a + // known-bug serve back to the full window. let tagged = decl(&[ "id:serve-stop-leaves-no-background-helper", "requires-engine:lemonade", @@ -2499,6 +2512,11 @@ flaky = true ]); assert_eq!( m.serve_timeout_override(&tagged, &cap("strix-ubuntu")), + Some(90) + ); + // Where no row matches the host, the tag is what lengthens the wait. + assert_eq!( + m.serve_timeout_override(&tagged, &cap("strix-windows")), Some(2400) ); // No `@id:` tag, nothing to look up: the step default stays. diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 6174cab2a..3e5537b3f 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -155,8 +155,9 @@ pub struct E2eWorld { pub legacy_rocm_path: Option, /// Per-scenario serve-readiness timeout override (seconds), set by the /// `before` hook from `expectations.toml` when this scenario is a known bug - /// with a `serve_timeout_secs`. Lets an xfail serve that never becomes ready - /// fail fast instead of burning the full cold-start window. `None` → the + /// with a `serve_timeout_secs` (which wins over a `@serve-timeout` tag), else + /// from that tag. Lets an xfail serve that never becomes ready fail fast + /// instead of burning the full cold-start window. `None` → the /// step's default / `E2E_SERVE_TIMEOUT_SECS`. pub serve_timeout_override: Option, /// Whether `expectations.toml` marks this scenario a known bug on this host, @@ -1656,10 +1657,10 @@ async fn run_suite() { let decl = ScenarioDecl::from_tags(&scenario.tags); if let Some(id) = &decl.id { let engine = decl.effective_engine(cap); - // A `@serve-timeout:` tag on an expected-pass scenario (a - // genuinely slow serve, e.g. a large model) takes precedence; - // otherwise fall back to any xfail matrix override (fail-fast for - // a known-bug serve). Absent both → the step default. + // A matching xfail matrix override (fail-fast for a known-bug + // serve) takes precedence; otherwise a `@serve-timeout:` tag + // (a genuinely slow serve, e.g. a large model) applies. Absent + // both → the step default. world.serve_timeout_override = matrix.serve_timeout_override(&decl, cap); world.expect_xfail = matrix.is_xfail(id, cap, engine); } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c649d4de6..daced1820 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -73,10 +73,9 @@ fn serve_timeout_secs() -> u64 { /// Serve-readiness timeout for a scenario: a per-scenario override set by the /// `before` hook takes precedence over the global `E2E_SERVE_TIMEOUT_SECS` / -/// default. The override is a `@serve-timeout:` tag on an expected-pass -/// scenario (lengthen a genuinely slow serve, e.g. a large model), else an -/// `expectations.toml` xfail `serve_timeout_secs` (shorten a known-bug serve so -/// it fails fast). +/// default. The override is an `expectations.toml` xfail `serve_timeout_secs` +/// (shorten a known-bug serve so it fails fast), else a `@serve-timeout:` +/// tag (lengthen a genuinely slow serve, e.g. a large model). pub(crate) fn serve_timeout_for(world: &E2eWorld) -> u64 { world .serve_timeout_override From 772a260f4452c6668c4e9a6ec12db16ea3701d28 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Sat, 14 Nov 2026 15:02:17 +0200 Subject: [PATCH 2/3] test(e2e): describe serve-06's Windows failure the way the lane logs show it The row's comment and reason said the server crashes on the first inference request, while the new serve_timeout_secs paragraph said the serve never becomes ready. The Windows lane logs show the second: the launch exits 0, /v1/models never lists the model within the wait, and the failing step is the serve step. Say that in both places, keeping the Tensile access violation from the service log as the suspected cause. Signed-off-by: juhovainio --- tests/e2e-cucumber/expectations.toml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 02e723538..9f77de06a 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -260,12 +260,14 @@ flaky = true # # runtime-16 asserts the alignment succeeded, which it cannot while the 7.14.x # tarball is unreachable. serve-06 serves a 35B model through the Windows lane's -# 7.13 backend against the SDK's 7.14.1 libraries and the server crashes in the -# SDK's Tensile library on the first inference request (access violation in the -# lane's service log). The mismatch between the backend and the SDK is the -# suspected cause; it has NOT been reproduced off Windows, so the row names both -# facts. Both rows are deterministic (both scenarios failed on each Windows lane -# run examined: 2026-10-28, 2026-11-05 and 2026-11-06), so neither is `flaky`: an +# 7.13 backend against the SDK's 7.14.1 libraries and the engine fails on its own +# after the launch reports success: `rocm serve` exits 0, `/v1/models` never lists +# the model within the wait (the failing step is the serve step, not the chat +# request), and the lane's service log shows an access violation in the SDK's +# Tensile library. The mismatch between the backend and the SDK is the suspected +# cause; it has NOT been reproduced off Windows, so the row names both facts. +# Both rows are deterministic (both scenarios failed on each Windows lane run +# examined: 2026-10-28, 2026-11-05 and 2026-11-06), so neither is `flaky`: an # XPASS means the cause is gone and the row should be deleted. Only serve-06's row # sets `serve_timeout_secs`: a row's wait wins over the scenario's own # `@serve-timeout:2400` tag, and the failing serve never became ready within that @@ -290,5 +292,5 @@ reason = "Observed with Lemonade 11.5.1, which fetched TheRock only from repo.am [["serve-large-model-inference"]] when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" -reason = "The Windows large-model serve crashed in the SDK's Tensile library on the first inference request while Lemonade 11.5.1 was pinned: its ROCm 7.13 backend ran against the 7.14.1 SDK because 11.5.1 could not align it (suspected cause, not reproduced off Windows). Not yet re-checked with the 2026.44.1 pin; delete this row if the Windows nightly lane reports it as a stale XPASS." +reason = "The Windows large-model serve never exposed the model while Lemonade 11.5.1 was pinned (the launch exited 0, then the engine failed on its own; the service log showed a crash in the SDK's Tensile library): its ROCm 7.13 backend ran against the 7.14.1 SDK because 11.5.1 could not align it (suspected cause, not reproduced off Windows). Not yet re-checked with the 2026.44.1 pin; delete this row if the Windows nightly lane reports it as a stale XPASS." serve_timeout_secs = 600 From 8c13922349c3ef919ec506cfd39aafb2bcf43e9c Mon Sep 17 00:00:00 2001 From: juhovainio Date: Wed, 18 Nov 2026 14:07:43 +0200 Subject: [PATCH 3/3] docs(e2e): say an xfail row's serve_timeout_secs overrides @serve-timeout The env-block comment in e2e.rs and the @serve-timeout row in the e2e README described the tag as lengthening only. A matching xfail row now wins over the tag, can be shorter, and then lowers the CLI's own readiness caps too. Signed-off-by: juhovainio --- tests/e2e-cucumber/README.md | 2 +- tests/e2e-cucumber/tests/e2e.rs | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index c68381db7..d81eb6021 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -197,7 +197,7 @@ Scenarios carry stable-id and capability tags: | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch so `rocm serve`'s summary can be verified without a GPU: one that runs out of GPU memory (the OOM guidance), or, in its `ready` mode, one that becomes ready and then fails its smoke test. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | -| `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | +| `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). A matching xfail row's `serve_timeout_secs` overrides it, even with a shorter value, which also lowers the CLI's own readiness caps. | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios, the native-Windows vLLM refusal scenarios serve-33/34/35, and the side-by-side runtime scenarios runtime-lifecycle-19/20/21. | diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 3e5537b3f..8e05b53a0 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -506,9 +506,11 @@ impl E2eWorld { if let Some(path) = &self.legacy_rocm_path { env.push(("ROCM_PATH", path.clone().into_os_string())); } - // When a scenario declares a longer serve-readiness window (via a - // `@serve-timeout:` tag → serve_timeout_override), also raise the - // CLI's OWN per-engine readiness cap(s) to match. `rocm serve --managed` + // When a scenario declares a serve-readiness window (via a + // `@serve-timeout:` tag, or a matching xfail row's + // `serve_timeout_secs`, which wins over the tag and can be shorter → + // serve_timeout_override), also set the CLI's OWN per-engine readiness + // cap(s) to match, lowering them when the override is shorter. `rocm serve --managed` // otherwise kills/fails a server that isn't ready within its default (5 // min, EAI-7393), which a large model's cold load legitimately exceeds — // so extending only the harness's poll (`model_is_ready`) isn't enough;