From 9ba169bb36129ab9f1be8c889e4ecdb349c8da57 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 15:33:12 +0200 Subject: [PATCH] test(e2e): drop the ROCMAI-898 xfail rows (ROCMAI-888) #963 fixed Lemonade serving on Linux by resolving registered model names, but the ROCMAI-898 expected-failure rows stayed in expectations.toml. Every one of them is flaky, so their passes were tolerated as XPASS and never surfaced as stale rows. The lanes have been passing these scenarios since. In the main push and merge-queue E2E runs from #963's merge (2026-11-09) to 2026-11-13, the Strix Halo Ubuntu jobs passed every removed scenario that ran, MI300X passed serve-07 and serve-40 (as it also did on this PR's own MI300X job), and the W7900 jobs in that window passed them too. The dash-lemonade-managed-inference-journey row, added by #631, has passed in all seven Strix Halo Ubuntu jobs since that change merged. Remove twelve rows: the default-engine, readiness-contract, Anthropic inference/tool-use/streaming, explicit Lemonade inference (serve-07) and background-helper (serve-40) serving scenarios, the two chat scenarios, the bench real-serve scenario and the dash managed-inference journey. Keep serve-50 and serve-52 as flaky rows, re-attributed to ROCMAI-1023: their serve becomes ready, but `rocm services list` intermittently shows no `anthropic endpoint:` row within the 30s poll. They set no serve-timeout override, since the serve itself succeeds. Add a guard test that resolves each of the twelve scenarios from its real tag line on the modelled Linux Lemonade hosts (Strix Halo Ubuntu, a WSL2 guest with a usable GPU, gfx1100 bare metal) and, for the ids pinned to Lemonade, on MI300X, failing if any row matches; the pin is held per id and the failure message says how to proceed. Tests that used the deleted rows as fixtures now parse inline matrices. Update docs/testing.md and a model_serving.feature comment that still described the old rows. Signed-off-by: Roman Inflianskas --- docs/testing.md | 54 ++-- tests/e2e-cucumber/expectations.toml | 183 ++--------- .../features/model_serving.feature | 5 +- tests/e2e-cucumber/src/expectation.rs | 285 ++++++++++++------ 4 files changed, 258 insertions(+), 269 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index a21272ebc..c42e31d2d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -720,8 +720,8 @@ place fails it), so a change of route is caught even when the endpoint still ans inference and stops the scenario-owned service. It needs a usable AMD GPU on a Lemonade Linux lane (`@requires-gpu`, `@requires-os:linux`, `@requires-engine:lemonade`). It has no flaky expectations row, so a route -regression fails the lane. Of the other direct-route serves, `serve-07` keeps its -flaky Linux row; `serve-08` has none, so it fails the lane too. +regression fails the lane. The other direct-route serves, `serve-07` and +`serve-08`, have no row either, so a regression there fails the lane too. The daemon's Lemonade contracts in [telemetry_contract.rs](../crates/rocm-dash-daemon/tests/telemetry_contract.rs) @@ -754,18 +754,16 @@ The dashboard scenarios in claims. One remaining model shows its throughput, an empty or unreachable server shows no models, and a reload restores both. - `dash-lemonade-managed-inference-journey`: a GPU-gated real Lemonade serve and - inference followed by dashboard observation. On Linux Lemonade hosts it is - expected to be flaky and its serve is bounded by `serve_timeout_secs` (readiness polling; the CLI process is killed only 300 s after that) in - [expectations.toml](../tests/e2e-cucumber/expectations.toml). The model must + inference followed by dashboard observation. It carries no row in + [expectations.toml](../tests/e2e-cucumber/expectations.toml), so it is + expected to pass on every Linux Lemonade host that runs it; its flaky + ROCMAI-898 row was removed with the others (see + [Lemonade managed-serve failure evidence](#lemonade-managed-serve-failure-evidence)). The model must appear as a full row in Observe, and its instance detail must show it READY on the port it was served on. This is real engine acceptance; the protocol fixtures do not prove GPU inference or engine installation. - The flaky row matches every lane that can run this scenario, so the whole - journey is advisory: its dashboard, instance detail, and post-quit inference - checks cannot block a merge. Only the serve step is intermittent, but - expectations apply per scenario. It also does not gate the - "Lemonade — managed runtime installed" wording. - Unit tests gate that wording instead: + It runs only on GPU lanes, so the + "Lemonade — managed runtime installed" wording is also gated by unit tests: `managed_installed_verdict_follows_detect_result` in [dash.rs](../apps/rocm/src/dash.rs), and the engine pane glyph and verdict tests in [pane.rs](../crates/rocm-dash-tui/src/ui/tabs/pane.rs). No other @@ -1768,9 +1766,20 @@ is the `kill/reap failed` branch, which needs an operating system that refuses t kill a child it just started. Use the evidence to distinguish backend startup, download, and router readiness -failures; ROCMAI-898 tracks the Strix Halo Ubuntu investigation (first tracked -as [issue #897](https://github.com/ROCm/rocm-cli/issues/897), which is closed). A -backend's ready line does not establish that the router serves the requested +failures. The diagnostics were added for the Strix Halo Ubuntu Lemonade serve +failures of ROCMAI-898 (first tracked as +[issue #897](https://github.com/ROCm/rocm-cli/issues/897)), which [#963](https://github.com/ROCm/rocm-cli/pull/963) fixed. +Twelve of the scenarios that carried ROCMAI-898 expected-failure rows in +[expectations.toml](../tests/e2e-cucumber/expectations.toml) no longer carry a +row on Linux lemonade hosts, and +`lemonade_linux_serve_scenarios_have_no_xfail_after_rocmai_898` in +[expectation.rs](../tests/e2e-cucumber/src/expectation.rs) guards them against a +row that matches one of the hosts it models. +The other two, `serve-50` and `serve-52`, keep flaky rows under ROCMAI-1023 for +a different symptom: the serve is ready, but `rocm services list` shows no +`anthropic endpoint:` row. + +A backend's ready line does not establish that the router serves the requested model, and a readiness timeout alone does not establish a shutdown. Diagnostics do not change the CLI's asynchronous managed-launch contract or imply a backend fix. @@ -3223,12 +3232,8 @@ bad config from failing a `serve` after its launch. The user-visible side is `au (mock lane: enable starts a live helper, disabling the last watcher makes it exit) and `serve-40` in [model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature) -(GPU lane: serving with no watcher enabled starts no helper; on a Linux host -that runs it with lemonade it is an expected failure in -[expectations.toml](../tests/e2e-cucumber/expectations.toml) while lemonade -serving on the Strix Halo Ubuntu lanes is broken (ROCMAI-898), so the helper check is -not reached on those lanes. The Strix Halo Windows lane runs it and passes, and -the row also matches other Linux lemonade hosts, where a pass is tolerated). +(GPU lane: serving with no watcher enabled starts no helper; it is pinned to +lemonade and expected to pass on every GPU lane that runs it, Linux and Windows). Anthropic-compatible `/v1/messages` endpoint (ROCMAI-597). The translation and its HTTP front are tested in @@ -3254,7 +3259,14 @@ checks the reply is an event stream that reassembles into text, with usage in `message_delta`) in [model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature). They run on the GPU lanes (`@requires-gpu`), with whichever engine the host -serves with. The proxy's unit tests cover the stream event sequence, a stream +serves with. On a Linux host serving with lemonade, `serve-50` and `serve-52` +carry flaky rows in +[expectations.toml](../tests/e2e-cucumber/expectations.toml) under ROCMAI-1023 +(`services list` intermittently shows no `anthropic endpoint:` row after the +service is ready), and `serve-53` has an expected-failure row under EAI-7223 +on vLLM, which is started without a tool-call parser. + +The proxy's unit tests cover the stream event sequence, a stream that ends early, an engine refusal before the first byte, and the refusal of a streaming request that offers `tools` (no scenario covers that refusal). The tool-use translation (`tools`, `tool_choice`, `tool_use` and `tool_result` diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index feddea530..987780ddf 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -31,6 +31,12 @@ # so an XPASS is tolerated (reported, not stale) # # An empty `when = {}` means "always xfail" (platform-independent bug). +# +# The scenarios in `LEMONADE_LINUX_SERVE_IDS` in src/expectation.rs, whose +# ROCMAI-898 rows were removed when Linux Lemonade serving was fixed, are +# guarded by `lemonade_linux_serve_scenarios_have_no_xfail_after_rocmai_898` +# against a row that matches a host it models. If that test fails, its failure +# message says what to do. # --- EAI-7219: short-name expansion (vLLM does not show an expanded name) --- # What the lane logs show, so the scope below is not read as more than it is: @@ -73,176 +79,53 @@ when = {} bug = "EAI-7219" reason = "A serve of a short name does not show the full model name the engine will serve: vLLM either stops before printing a serve plan or prints the alias unexpanded, and the lemonade half is only reached once that stops happening." -# ROCMAI-898 tracks multiple Strix Halo Ubuntu Lemonade serve signatures (first -# tracked as GitHub issue ROCm/rocm-cli#897, since closed; ROCMAI-898 is the open -# ticket and what the rows below name): -# the child rejects --no-mmap (41984921053), backend readiness on :8001 without -# the model appearing on :11435 (41984503055), and backend download failure -# (41986772728). A backend ready line does not prove a later shutdown. -# EAI-7423 described only the earlier ready-then-unavailable symptom. -# Retain flaky=true because historical runs also passed intermittently; do not -# infer a fix from a single XPASS. The flag is per-row and is NOT derived from -# `bug`, so a new ROCMAI-898 row needs `flaky = true` added explicitly. -# Failed serves now capture configured pins, candidate executable paths and -# fingerprints before stopping the service, and each candidate's --version/--help -# after it. -# Revisit under EAI-7853 before deleting any of these rows: which lemonade build -# a given run used could not be read back from this lane, and until a failed run -# there has carried that evidence, an XPASS streak is not by itself evidence that -# the bug is fixed. +# --- ROCMAI-1023: `services list` does not show the Anthropic endpoint row --- +# On the Strix Halo Ubuntu lanes the lemonade serve itself succeeds and the model +# answers on /v1/models, but `rocm services list` then shows no +# `anthropic endpoint:` row within the 30s poll of `listed_anthropic_endpoint` +# (tests/e2e/serving_steps.rs). Seen intermittently on main after the Lemonade +# serve fix (#963): serve-50 in jobs 129254733795 and 129309536682, serve-52 in +# jobs 129309527901, 129318300169, 129700937355 and 130496332636. # -# os=linux: native Windows passed (29338475133 Strix-Windows). -[["serve-default-engine-working-endpoint"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." -serve_timeout_secs = 90 -flaky = true - -# The same serve precondition blocks inference; native Windows passed. -[["serve-default-engine-inference"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." -serve_timeout_secs = 90 -flaky = true - -# The readiness contract depends on the same real Lemonade serve. -[["serve-readiness-contract"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." -serve_timeout_secs = 90 -flaky = true - -# The Anthropic-endpoint scenarios start from `a model is being served on GPU` -# like the rows above, so they hit the same Linux lemonade serve failure. +# serve-51, serve-53 and serve-54 read the endpoint through the same poll (their +# When steps look it up before sending a request) yet carry no row. In the six +# jobs above, serve-51 passed in all six, and serve-53 and serve-54 (added later) +# ran only in 130496332636 and passed there. Those jobs ran while the three +# still had ROCMAI-898 rows, so each had one serve attempt, not two. +# A row for them would rest on no observed failure; add one if they start +# failing the same way. +# +# `flaky = true` because both scenarios pass in most runs. No +# `serve_timeout_secs`: the serve becomes ready, so a shorter serve wait would +# only cut short a slow but healthy serve. A matching row still tolerates any +# failure of the scenario on these hosts, not only this symptom: a serve that +# never becomes ready is also reported as the expected failure, and the harness +# gives an expected-failure scenario one serve attempt instead of two. [["serve-anthropic-endpoint-advertised"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model, so no endpoint is left to advertise." -serve_timeout_secs = 90 +bug = "ROCMAI-1023" +reason = "After the lemonade service is ready, `rocm services list` intermittently shows no `anthropic endpoint:` row, so the Anthropic endpoint is not advertised." flaky = true -[["serve-anthropic-messages-inference"]] +[["serve-anthropic-endpoint-readiness"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model, so the Anthropic request never gets an answer." -serve_timeout_secs = 90 -flaky = true - -[["serve-anthropic-messages-tool-use"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model, so the tool-use request never gets an answer." -serve_timeout_secs = 90 +bug = "ROCMAI-1023" +reason = "After the lemonade service is ready, `rocm services list` intermittently shows no `anthropic endpoint:` row, so readiness cannot be seen to cover the Anthropic endpoint." flaky = true +# --- EAI-7223: requests with tool definitions rejected (vLLM path) --- [["serve-anthropic-messages-tool-use"]] when = { effective_engine = "vllm" } bug = "EAI-7223" reason = "The served vLLM model is started without a tool-call parser, so it rejects a request that names a tool; the endpoint relays that refusal." serve_timeout_secs = 90 -[["serve-anthropic-endpoint-readiness"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model, so readiness never covers the Anthropic endpoint." -serve_timeout_secs = 90 -flaky = true - -[["serve-anthropic-messages-streaming"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model, so the streaming Anthropic request never gets an answer." -serve_timeout_secs = 90 -flaky = true - -# Explicit Lemonade serving shares the same failure on the Ubuntu lane. -[["serve-lemonade-inference"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." -serve_timeout_secs = 90 -flaky = true - -# serve-40 serves the same small GGUF model through the same lemonade managed serve -# as the rows above, so on a lemonade LINUX host its When step never sees the model -# on /v1/models and panics before the Then assertion runs. Scoped like those rows -# (os=linux). Known from run 41986772728: serve-40 passes on the Strix Halo Windows -# lane, is skipped on the Strix Halo WSL2 lane (no AMD GPU detected), and fails on -# both Strix Halo Ubuntu lanes. The row also matches any other Linux host that runs -# the scenario with lemonade (the MI300X and W7900 lanes had not reported in that -# run); whether lemonade serves there is not verified, and `flaky = true` tolerates -# a pass on such a host. The cause on the Strix Halo Ubuntu lanes is not settled -# and varies between runs (ROCMAI-898), so this row names that ticket rather than -# EAI-7423 -# and carries `flaky = true` like its siblings. While it is in place the -# scenario's own check (serve starts no background helper when no watcher is -# enabled) is not exercised on the hosts it matches, and a genuine failure of that -# check there would also read as a failure that was expected. `flaky` also means a -# fixed ROCMAI-898 is not flagged as a stale row, so delete the row by hand when -# ROCMAI-898 is -# closed. -[["serve-stop-leaves-no-background-helper"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "Lemonade serving fails on the Strix Halo Ubuntu lanes, so the model never appears on /v1/models and the helper check is never reached." -serve_timeout_secs = 90 -flaky = true - -# The dashboard journey does the same real lemonade serve as serve-lemonade-inference, -# so on a Linux lemonade host it is just as intermittent. The bounded serve timeout -# makes a stuck serve fail this one scenario instead of stalling the whole lane. -[["dash-lemonade-managed-inference-journey"]] -when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCMAI-898" -reason = "The managed Lemonade serve on Linux is intermittent: it can reach ready and then shut down, or never become ready, so the journey cannot be relied on to complete." -serve_timeout_secs = 90 -flaky = true - -# --- EAI-7223: chat requests with tool definitions rejected (vLLM path) --- [["chat-tool-definitions-accepted"]] when = { effective_engine = "vllm" } bug = "EAI-7223" reason = "Chat requests carrying tool definitions are not accepted by the served vLLM model." serve_timeout_secs = 90 -# Chat's mock lane must remain expect-pass: only a real GPU serve is affected. -[["chat-tool-definitions-accepted"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." -serve_timeout_secs = 90 -flaky = true - -# EAI-7221 (TUI chat sends hardcoded 'local-model' instead of querying /v1/models) -# was previously mapped here as a vLLM-path xfail, but that bug is TUI-only and this -# scenario exercises the CLI `rocm chat --prompt` path (which correctly discovers the -# model via /v1/models). Verified expect-pass on MI300X vLLM (XPASS on run 29322186691, -# then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat -# path works. Only the real Lemonade Linux serve stays xfail under ROCMAI-898; -# the no-GPU MockServer path and native Windows remain expect-pass. -[["chat-end-to-end-local-model"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." -serve_timeout_secs = 90 -flaky = true - -# Bench's real-serve precondition is blocked by ROCMAI-898 on the same Linux hosts. -# -# This does NOT weaken the coverage the bench fix relies on: the five -# MockServer-backed bench scenarios (bench-01 to bench-05) run unxfail'd on -# every lane, including this one, and they are what pin the request path, the -# failure reporting, the CSV columns, and the rollup-split warning. -[["bench-load-real-serve"]] -when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "ROCMAI-898" -reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." -serve_timeout_secs = 90 -flaky = true - # --- Lemonade 11.5.1 cannot align its llama.cpp backend to a ROCm 7.14.x SDK --- # `rocm engines install lemonade` tries to align Lemonade's `llamacpp:rocm` # backend with the active ROCm SDK. Lemonade 11.5.1, the pin in diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 3623c1e80..852e378c8 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -910,9 +910,8 @@ Feature: Model serving # Linux only: registered names take the router on Windows both with and # without the direct llama-server route, so a Windows run cannot tell the # two behaviours apart. - # No expectations row is added for this scenario. The flaky Linux rows cover - # the direct llama-server route, and this scenario takes the router route, so - # those rows' failure mode does not describe it. It is nightly-only. + # No expectations row: it is expected to pass on every Linux lemonade host + # that runs it. It is nightly-only. @id:serve-lemonade-complex-registered-inference @requires-gpu @requires-os:linux @requires-engine:lemonade @nightly Scenario: serve-57 - A registered vision GGUF retains the Lemonade router and managed lifecycle Given a managed runtime is active diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9ebdca6c6..8a2bbb96f 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -840,51 +840,168 @@ mod tests { } } - #[test] - fn lemonade_serve_failures_track_rocmai_898_without_xfailing_mock_or_windows() { - let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); - for id in [ - "bench-load-real-serve", - "chat-tool-definitions-accepted", - "chat-end-to-end-local-model", - "serve-lemonade-inference", + /// The scenario ids whose `ROCMAI-898` rows were deleted after the Linux + /// Lemonade serve fix (#963), with the feature file that declares each. A + /// slice, so an id can be removed without changing a length. + const LEMONADE_LINUX_SERVE_IDS: &[(&str, &str)] = &[ + ( + "serve-default-engine-working-endpoint", + MODEL_SERVING_FEATURE, + ), + ("serve-default-engine-inference", MODEL_SERVING_FEATURE), + ("serve-readiness-contract", MODEL_SERVING_FEATURE), + ("serve-anthropic-messages-inference", MODEL_SERVING_FEATURE), + ("serve-anthropic-messages-tool-use", MODEL_SERVING_FEATURE), + ("serve-anthropic-messages-streaming", MODEL_SERVING_FEATURE), + ("serve-lemonade-inference", MODEL_SERVING_FEATURE), + ( "serve-stop-leaves-no-background-helper", - ] { - let scenario = decl(&[&format!("id:{id}")]); + MODEL_SERVING_FEATURE, + ), + ("chat-tool-definitions-accepted", CHAT_FEATURE), + ("chat-end-to-end-local-model", CHAT_FEATURE), + ("bench-load-real-serve", BENCH_FEATURE), + ("dash-lemonade-managed-inference-journey", DASH_FEATURE), + ]; + /// The ids in `LEMONADE_LINUX_SERVE_IDS` whose scenario is tagged + /// `@requires-engine:lemonade` (serve-07, serve-40 and dash-52), which is what puts + /// them on the MI300X check. Every entry must also be in that list, so + /// removing a pinned id means removing it from both. + const LEMONADE_PINNED_IDS: &[&str] = &[ + "serve-lemonade-inference", + "serve-stop-leaves-no-background-helper", + "dash-lemonade-managed-inference-journey", + ]; + const MODEL_SERVING_FEATURE: &str = include_str!("../features/model_serving.feature"); + const CHAT_FEATURE: &str = include_str!("../features/chat.feature"); + const BENCH_FEATURE: &str = include_str!("../features/bench.feature"); + const DASH_FEATURE: &str = include_str!("../features/dash.feature"); + + /// The ROCMAI-898 rows (Lemonade managed serve never exposing the model on + /// Linux) are gone. This guards against a silent regression: each scenario + /// resolves expect-pass from its REAL tag line on three modelled Linux + /// lemonade hosts (gfx1151 bare metal, a WSL2 guest with a usable GPU, and + /// gfx1100 bare metal), and the ids in `LEMONADE_PINNED_IDS` also on MI300X. + /// A row that matches one of those hosts fails it; a row that matches none + /// of them (another GPU family, or a combination no modelled host has) does + /// not. + #[test] + fn lemonade_linux_serve_scenarios_have_no_xfail_after_rocmai_898() { + let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + let in_merge_queue = Included { + merge_queue: true, + ..Included::default() + }; + // A W7900-class host: the bare-metal Linux fixture with an RDNA3 + // workstation GPU in place of Strix Halo's. + let gfx1100 = HostCapability { + gfx_target: Some("gfx1100".into()), + platform_slug: "w7900".into(), + ..cap("strix-ubuntu") + }; + let lemonade_hosts = [ + ("Strix Halo Ubuntu", cap("strix-ubuntu")), + ("Strix Halo WSL2", cap("wsl2")), + ("gfx1100 Linux", gfx1100), + ]; + for pinned_id in LEMONADE_PINNED_IDS { assert!( - matches!( - resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), - Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "ROCMAI-898" - ), - "{id}" + LEMONADE_LINUX_SERVE_IDS + .iter() + .any(|(id, _)| id == pinned_id), + "{pinned_id} is in LEMONADE_PINNED_IDS but not in \ + LEMONADE_LINUX_SERVE_IDS; remove it from both", ); + } + let remedy = "the ROCMAI-898 rows were removed after #963 fixed Linux \ + Lemonade serving. A row for a NEW known bug scoped to a host this \ + guard does not model needs no guard change; one that matches this \ + host means removing the id from LEMONADE_LINUX_SERVE_IDS (and from \ + LEMONADE_PINNED_IDS if it is there) in the same change, saying why, \ + which drops the guard for the id on every host"; + for (id, feature) in LEMONADE_LINUX_SERVE_IDS { + let scenario = ScenarioDecl::from_tags(&feature_tags(feature, id)); + for (label, host) in &lemonade_hosts { + assert_eq!( + resolve(&scenario, host, &m, in_merge_queue), + Expectation::ExpectPass, + "{id} on {label}: {remedy}", + ); + } + // Losing the `@requires-engine:lemonade` tag would quietly drop the + // MI300X check below, so the pin is held per id rather than counted. + let pinned = scenario.requires_engine.as_deref() == Some("lemonade"); assert_eq!( - matrix.serve_timeout_for(id, &cap("strix-ubuntu"), "lemonade"), - Some(90), - "{id}", + pinned, + LEMONADE_PINNED_IDS.contains(id), + "{id}: pinned to lemonade is {pinned}, but LEMONADE_PINNED_IDS says \ + otherwise. If the tag changed on purpose, update LEMONADE_PINNED_IDS \ + in the same change, saying why", ); - for platform in ["mock", "strix-windows"] { + if pinned { assert_eq!( - resolve(&scenario, &cap(platform), &matrix, Included::default()), + resolve(&scenario, &cap("mi300x"), &m, in_merge_queue), Expectation::ExpectPass, - "{id} on {platform}", + "{id} on MI300X: {remedy}", ); } } } - /// The rows that carry `therock_family` and `has_amd_gpu` exist so that the - /// no-GPU lane, where the same scenarios run against MockServer instead of a - /// real Lemonade serve, stays expect-pass. `cap("mock")` cannot prove that: - /// its `os_family` is `other`, so the `os = "linux"` condition excludes it - /// before either GPU guard is consulted. These hosts are Linux, so each guard - /// is the only thing standing between them and an xfail. The two partial - /// hosts isolate the guards from each other: dropping `has_amd_gpu` from a row - /// fails the host that reports only a gfx target, and dropping `therock_family` - /// fails the host that reports only a GPU. + /// serve-50 and serve-52 still fail now and then on Linux lemonade hosts, but + /// with a different symptom from ROCMAI-898: the serve becomes ready and + /// `services list` then shows no `anthropic endpoint:` row. Their rows name + /// that ticket, stay `flaky` (both pass in most runs, and a non-flaky row + /// would turn each pass into a stale-row failure), set no shortened serve wait + /// (the serve itself succeeds), and leave Windows expect-pass. + #[test] + fn anthropic_endpoint_listing_rows_track_rocmai_1023() { + let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + for id in [ + "serve-anthropic-endpoint-advertised", + "serve-anthropic-endpoint-readiness", + ] { + let scenario = ScenarioDecl::from_tags(&feature_tags(MODEL_SERVING_FEATURE, id)); + assert!( + matches!( + resolve(&scenario, &cap("strix-ubuntu"), &m, Included::default()), + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "ROCMAI-1023" + ), + "{id} on Strix Halo Ubuntu", + ); + assert_eq!( + resolve(&scenario, &cap("strix-windows"), &m, Included::default()), + Expectation::ExpectPass, + "{id} on Strix Halo Windows", + ); + let entries = m.entries_for(id); + assert_eq!(entries.len(), 1, "{id}"); + assert_eq!(entries[0].serve_timeout_secs, None, "{id}"); + } + } + + /// A row carrying `therock_family` and `has_amd_gpu` must leave a Linux host + /// without a usable GPU expect-pass (the no-GPU lane runs chat and bench + /// scenarios against MockServer). `cap("mock")` cannot prove that: its + /// `os_family` is `other`, so `os = "linux"` excludes it before either GPU + /// guard is consulted. These hosts are Linux, so each guard is the only thing + /// standing between them and an xfail. The two partial hosts isolate the + /// guards from each other: a matcher that ignored `has_amd_gpu` fails the host + /// that reports only a gfx target, and one that ignored `therock_family` fails + /// the host that reports only a GPU. #[test] - fn real_gpu_lemonade_rows_do_not_xfail_linux_hosts_without_a_usable_gpu() { - let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + fn gpu_guards_in_a_condition_each_exclude_a_linux_host_on_their_own() { + let m = Expectations::parse( + r#" +[["chat-end-to-end-local-model"]] +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } +bug = "EAI-0001" +reason = "Applies only when the scenario uses a real Lemonade GPU serve." +flaky = true +"#, + ) + .unwrap(); + let id = "chat-end-to-end-local-model"; let linux_lemonade = cap("strix-ubuntu"); let hosts = [ ( @@ -912,28 +1029,19 @@ mod tests { }, ), ]; - for id in [ - "bench-load-real-serve", - "chat-tool-definitions-accepted", - "chat-end-to-end-local-model", - ] { - let scenario = decl(&[&format!("id:{id}")]); - for (label, host) in &hosts { - assert_eq!( - resolve(&scenario, host, &matrix, Included::default()), - Expectation::ExpectPass, - "{id} on {label}", - ); - } - // The full real-GPU host is the one that must still be xfailed. - assert!( - matches!( - resolve(&scenario, &linux_lemonade, &matrix, Included::default()), - Expectation::ExpectXfail { bug, .. } if bug == "ROCMAI-898" - ), - "{id} on a real GPU host", + let scenario = decl(&[&format!("id:{id}")]); + for (label, host) in &hosts { + assert_eq!( + resolve(&scenario, host, &m, Included::default()), + Expectation::ExpectPass, + "{label}", ); } + // The full real-GPU host is the one the row still matches. + assert!(matches!( + resolve(&scenario, &linux_lemonade, &m, Included::default()), + Expectation::ExpectXfail { bug, .. } if bug == "EAI-0001" + )); } fn decl(tags: &[&str]) -> ScenarioDecl { @@ -2392,56 +2500,43 @@ flaky = true ); } - /// serve-40 serves through the same lemonade managed serve as the other - /// lemonade serve rows, so its row is scoped the same way: xfail on a - /// lemonade Linux host, with a 90s serve wait, and not an expected failure - /// on native Windows. The row must stay `flaky` (a fixed ROCMAI-898 would - /// otherwise turn the Ubuntu lanes red as a stale XPASS) and must name ROCMAI-898. + /// The serve window the `before` hook hands the serve steps + /// (`E2eWorld::serve_timeout_override`, read by `serve_and_wait`): the + /// matching row's `serve_timeout_secs` when the scenario has no + /// `@serve-timeout` tag — with the engine taken from a `@requires-engine` pin, + /// so it matches on a host whose default engine is another one — a tag over + /// the row, and nothing without an `@id:` to look the row up by. #[test] - fn serve_40_row_is_scoped_like_the_other_lemonade_serve_rows() { - let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); - let id = "serve-stop-leaves-no-background-helper"; - assert!(m.is_xfail(id, &cap("strix-ubuntu"), "lemonade")); + fn serve_timeout_override_prefers_the_tag_and_needs_an_id() { + let m = Expectations::parse( + r#" +[["serve-pinned"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "EAI-0001" +reason = "test row" +serve_timeout_secs = 90 +"#, + ) + .unwrap(); + // No tag: the row's 90s applies, on MI300X too, whose default engine is + // vLLM, because the pin decides the engine. + let pinned = decl(&["id:serve-pinned", "requires-engine:lemonade"]); assert_eq!( - m.serve_timeout_for(id, &cap("strix-ubuntu"), "lemonade"), + m.serve_timeout_override(&pinned, &cap("strix-ubuntu")), Some(90) ); - assert!(!m.is_xfail(id, &cap("strix-windows"), "lemonade")); - let entries = m.entries_for(id); - assert_eq!(entries.len(), 1); - assert!(entries[0].flaky); - assert_eq!(entries[0].bug, "ROCMAI-898"); - } - - /// The row's 90s only matters if the harness hands it to the serve steps: - /// the `before` hook stores `serve_timeout_override(decl, cap)` on the world - /// and `serve_and_wait` waits that long instead of the 300s/600s default. - /// Resolve it from serve-40's REAL tag line, so dropping the `@id:` or the - /// `@requires-engine:lemonade` pin from the feature file fails here rather - /// than silently restoring the full wait. The pin also decides the engine on - /// a host whose default is another engine (MI300X serves vLLM by default). - #[test] - fn serve_40_gets_the_shortened_serve_window_from_its_own_tags() { - let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); - let tags = feature_tags( - include_str!("../features/model_serving.feature"), - "serve-stop-leaves-no-background-helper", - ); - let decl = ScenarioDecl::from_tags(&tags); + assert_eq!(m.serve_timeout_override(&pinned, &cap("mi300x")), Some(90)); + // Without the pin MI300X serves vLLM and the row does not match. + let unpinned = decl(&["id:serve-pinned"]); + assert_eq!(m.serve_timeout_override(&unpinned, &cap("mi300x")), None); + // Native Windows is outside the row: the step default stays. assert_eq!( - m.serve_timeout_override(&decl, &cap("strix-ubuntu")), - Some(90) + m.serve_timeout_override(&pinned, &cap("strix-windows")), + None ); - assert_eq!(m.serve_timeout_override(&decl, &cap("mi300x")), Some(90)); - assert_eq!(m.serve_timeout_override(&decl, &cap("strix-windows")), None); - } - - #[test] - fn serve_timeout_override_prefers_the_tag_and_needs_an_id() { - let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); - // The row says 90s on this host; a `@serve-timeout` tag lengthens it. + // A `@serve-timeout` tag wins over the row. let tagged = decl(&[ - "id:serve-stop-leaves-no-background-helper", + "id:serve-pinned", "requires-engine:lemonade", "serve-timeout:2400", ]);