From c2d3af2667fef193ba0df537b0181a176f24891e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 09:22:18 +0000 Subject: [PATCH 1/3] test(e2e): say why a short-name serve printed no plan serve-01 and serve-02 ran `rocm serve qwen2.5 --engine ...` and kept stdout only, so a run that stopped on a refusal before it printed the serve plan read as "the name was not expanded". On the Strix Halo Ubuntu lane the lemonade run does print `resolved model: Qwen/Qwen2.5-1.5B-Instruct`, while the vLLM run's stdout is empty and its stderr, which would say why, was thrown away. The plan line is printed only after the platform, no-GPU and active-runtime checks and the engine's model resolution, so an empty stdout means one of those stopped the run, not that the alias was left unexpanded. Keep stderr and the exit code per run (`ServeRun`, in a library module so its verdict is unit-tested) and put the engine, exit code and first stderr line in the failure message. Rewrite the two expected-fail rows' wording to say what the logs show. Their all-hosts scope is unchanged: whether the stop is the tracked alias bug or an earlier refusal is not known until a lane run reports the stderr. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/expectations.toml | 14 +- tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/src/serve_plan.rs | 137 ++++++++++++++++++ tests/e2e-cucumber/tests/e2e.rs | 4 + tests/e2e-cucumber/tests/e2e/serving_steps.rs | 64 ++++---- 5 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 tests/e2e-cucumber/src/serve_plan.rs diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 4f26ef6b59..053d76de91 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -32,16 +32,24 @@ # # An empty `when = {}` means "always xfail" (platform-independent bug). -# --- EAI-7219: short-name expansion not surfaced in serve output (any engine) --- +# --- EAI-7219: short-name expansion (vLLM does not resolve the alias) --- +# What the lane logs show, so the scope below is not read as more than it is: +# lemonade DOES print the full name (`resolved model: Qwen/Qwen2.5-1.5B-Instruct`) +# on the Strix Halo Ubuntu and Windows lanes, and vLLM's stdout is empty -- the run +# stopped before it printed a serve plan, so the plan's name line was never +# reached. Whether that stop is the bug above or an earlier refusal (a missing +# active runtime, a no-GPU bail, native Windows refusing vLLM since ROCMAI-823) is +# not known: the steps used to discard stderr. They now report it, so the next lane +# run says which. Until then the rows keep their all-hosts scope. [["serve-short-name-expansion"]] when = {} bug = "EAI-7219" -reason = "Serve output does not surface the expanded full model name for a short name." +reason = "A vLLM serve of a short name prints no serve plan, so no expanded full model name is shown." [["serve-short-name-consistent-across-engines"]] when = {} bug = "EAI-7219" -reason = "Engines do not report a consistent resolved model name for a short name." +reason = "A vLLM serve of a short name prints no serve plan, so its resolved model name cannot match lemonade's." # EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: # which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 110023fe94..9c9bdc62c5 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -21,6 +21,7 @@ pub mod quit_and_wait; pub mod reader_failure; pub mod send_until; pub mod serve_log; +pub mod serve_plan; pub mod shared_runtime; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/serve_plan.rs b/tests/e2e-cucumber/src/serve_plan.rs new file mode 100644 index 0000000000..065a521164 --- /dev/null +++ b/tests/e2e-cucumber/src/serve_plan.rs @@ -0,0 +1,137 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Say why a `rocm serve` run printed no `resolved model:` line. +//! +//! `rocm serve` prints its plan (`serve plan` / `resolved model: ...`) only after a +//! chain of refusals has passed: the platform and engine gate, the no-usable-GPU +//! bail, the "no active ROCm runtime" bail and the engine's model-resolution call. +//! A run that stops at any of them leaves stdout without the line, and the refusal +//! goes to stderr. The short-name scenarios used to keep stdout only, so every +//! such run read as "the name was not expanded" whatever actually stopped it. The +//! lane logs show exactly that: a vLLM serve whose stdout was empty, with nothing +//! to say which refusal it was. + +/// One `rocm serve` run: the engine it asked for and everything it produced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServeRun { + pub engine: String, + pub stdout: String, + pub stderr: String, + pub rc: i32, +} + +impl ServeRun { + /// The model named by the plan's `resolved model:` line, if the plan was + /// printed at all. + #[must_use] + pub fn resolved_model(&self) -> Option<&str> { + resolved_model(&self.stdout) + } + + /// What to say when [`resolved_model`](Self::resolved_model) is `None`. + /// + /// Names the engine, the exit code and the first line of stderr, so a refusal + /// reads as a refusal and a run that exited 0 without a plan reads as that. + #[must_use] + pub fn missing_plan_message(&self) -> String { + let engine = &self.engine; + let rc = self.rc; + let stderr_line = self + .stderr + .lines() + .map(str::trim) + .find(|line| !line.is_empty()); + let verdict = match (rc, stderr_line) { + (0, _) => format!( + "`rocm serve --engine {engine}` exited 0 but printed no serve plan, so the name was never shown" + ), + (_, Some(line)) => format!( + "`rocm serve --engine {engine}` stopped before it printed a serve plan (exit {rc}): {line}" + ), + (_, None) => format!( + "`rocm serve --engine {engine}` stopped before it printed a serve plan (exit {rc}) and wrote nothing to stderr" + ), + }; + format!( + "no 'resolved model' in output: {verdict}\n\nstdout:\n{}\n\nstderr:\n{}", + self.stdout, self.stderr + ) + } +} + +/// The model a serve plan resolved to (`resolved model: `), if it printed one. +/// +/// It can differ from the requested id: `rocm serve ` with no `--engine` +/// picks the recipe's preferred model and engine. `None` means the plan has no +/// such line, for example because the serve stopped before resolving. +#[must_use] +pub fn resolved_model(output: &str) -> Option<&str> { + output + .lines() + .find_map(|l| l.trim().strip_prefix("resolved model:")) + .map(str::trim) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn attempt(engine: &str, stdout: &str, stderr: &str, rc: i32) -> ServeRun { + ServeRun { + engine: engine.to_owned(), + stdout: stdout.to_owned(), + stderr: stderr.to_owned(), + rc, + } + } + + #[test] + fn a_printed_plan_names_its_resolved_model() { + let plan = "serve plan\n requested model: qwen2.5\n resolved model: Qwen/Qwen2.5-1.5B-Instruct\n engine: lemonade\n"; + assert_eq!( + attempt("lemonade", plan, "", 0).resolved_model(), + Some("Qwen/Qwen2.5-1.5B-Instruct") + ); + } + + #[test] + fn a_refusal_is_reported_as_a_refusal_with_its_stderr_line() { + let refused = attempt( + "vllm", + "", + "\nError: no active ROCm runtime is configured; run `rocm runtimes list`\n", + 1, + ); + assert_eq!(refused.resolved_model(), None); + let message = refused.missing_plan_message(); + assert!( + message.contains("`rocm serve --engine vllm` stopped before it printed a serve plan (exit 1): Error: no active ROCm runtime is configured"), + "{message}" + ); + assert!( + !message.contains("exited 0"), + "a non-zero exit must not read as a clean one: {message}" + ); + } + + #[test] + fn an_exit_zero_without_a_plan_is_not_called_a_refusal() { + let message = attempt("lemonade", "", "warning: ignored", 0).missing_plan_message(); + assert!( + message.contains("exited 0 but printed no serve plan"), + "{message}" + ); + assert!(!message.contains("stopped before"), "{message}"); + } + + #[test] + fn a_silent_failure_says_stderr_was_empty() { + let message = attempt("vllm", "", "", 1).missing_plan_message(); + assert!( + message.contains("(exit 1) and wrote nothing to stderr"), + "{message}" + ); + } +} diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 2d56ac2835..28e6f0c4e7 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -76,6 +76,9 @@ pub struct E2eWorld { pub cli_outputs: Option>, pub cli_stderr: Option, pub cli_rc: Option, + /// One entry per `rocm serve` run a scenario made, with the stderr and exit + /// code `cli_outputs` has no room for (see `e2e_cucumber::serve_plan`). + pub serve_runs: Option>, /// Wall-clock time the last measured `rocm` invocation took, for scenarios /// where the *duration* bounds the behaviour under test — /// `service-cleanup-07`, where a prune that returned while the managed-launch @@ -320,6 +323,7 @@ impl Default for E2eWorld { cli_outputs: None, cli_stderr: None, cli_rc: None, + serve_runs: None, cli_elapsed: None, remote_env: Vec::new(), launch_release: None, diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 49ab899ce3..c31baba794 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -15,6 +15,7 @@ use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_ use e2e_cucumber::serve_log::{ ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, }; +use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; /// Model id of the live managed service the reuse scenarios plant. /// @@ -830,18 +831,31 @@ async fn setup_model_detected(world: &mut E2eWorld) { #[when("the user serves a model using its short name")] async fn user_serves_short_name(world: &mut E2eWorld) { - let (stdout, _, _) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", "vllm"]); - world.cli_output = Some(stdout); + let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", "vllm"]); + // stderr and the exit code ride along so a run that stopped on a refusal reads + // as one in the Then step, instead of as "the name was not expanded". + world.serve_runs = Some(vec![ServeRun { + engine: "vllm".to_owned(), + stdout, + stderr, + rc, + }]); } #[when("the user serves the same short name with different engines")] async fn user_serves_multiple_engines(world: &mut E2eWorld) { - let mut outputs = Vec::new(); + let mut runs = Vec::new(); for engine in ["lemonade", "vllm"] { - let (stdout, _, _) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); - outputs.push(stdout); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); + runs.push(ServeRun { + engine: engine.to_owned(), + stdout, + stderr, + rc, + }); } - world.cli_outputs = Some(outputs); + world.serve_runs = Some(runs); } #[when("the user lists running services")] @@ -2133,12 +2147,15 @@ async fn assert_masked_index_message(world: &mut E2eWorld) { #[then("the output shows the full model name")] async fn assert_full_model_name(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no CLI output"); - let resolved = resolved_model(output) - .unwrap_or_else(|| panic!("no 'resolved model' in output:\n{output}")); + let runs = world.serve_runs.as_ref().expect("no serve run"); + let run = runs.first().expect("no serve run"); + let resolved = run + .resolved_model() + .unwrap_or_else(|| panic!("{}", run.missing_plan_message())); assert!( resolved.contains('/'), - "expected a fully qualified model name (org/model), got '{resolved}'\n\nfull output:\n{output}" + "expected a fully qualified model name (org/model), got '{resolved}'\n\nfull output:\n{}", + run.stdout ); } @@ -2158,19 +2175,19 @@ async fn assert_model_list_is_ascii(world: &mut E2eWorld) { #[then("all engines expand to the same full model name")] async fn assert_consistent_expansion(world: &mut E2eWorld) { - let outputs = world.cli_outputs.as_ref().expect("no CLI outputs"); - assert!( - outputs.len() >= 2, - "need at least 2 serve outputs to compare" - ); + let runs = world.serve_runs.as_ref().expect("no serve runs"); + assert!(runs.len() >= 2, "need at least 2 serve outputs to compare"); let mut resolved: Vec = Vec::new(); - for (i, output) in outputs.iter().enumerate() { + for (i, run) in runs.iter().enumerate() { // A missing `resolved model:` line means the engine never got far enough // to expand the name (e.g. it errored before serving). Fail loudly rather // than defaulting to an empty string — otherwise two engines that both // fail would produce equal ("") values and pass this check vacuously. - let Some(model) = resolved_model(output).map(str::to_string) else { - panic!("engine #{i} produced no 'resolved model' line:\n{output}"); + let Some(model) = run.resolved_model().map(str::to_string) else { + panic!( + "engine #{i} produced no 'resolved model' line: {}", + run.missing_plan_message() + ); }; resolved.push(model); } @@ -2257,17 +2274,6 @@ fn selected_engine(output: &str) -> &str { engine } -/// The model a serve plan actually resolved to (`resolved model: `), which -/// can differ from the requested id — `rocm serve ` with no `--engine` -/// picks the recipe's preferred model+engine. `None` if the plan has no such line -/// (e.g. the serve errored before resolving). -fn resolved_model(output: &str) -> Option<&str> { - output - .lines() - .find_map(|l| l.trim().strip_prefix("resolved model:")) - .map(str::trim) -} - /// A distinctive substring of a model id that appears in the served endpoint's /// `/v1/models` response, for [`model_is_ready`]'s containment check. Strips the /// `org/` prefix and the `-GGUF` catalog marker so a resolved catalog id From e97223ba08b43cbf82c2c2350b0df82174c0027f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 07:30:06 +0000 Subject: [PATCH 2/3] test(e2e): address review of the short-name serve diagnostics Word the EAI-7219 xfail reasons and header so they stay true whether the next lane run shows a vLLM serve that stops before its plan or one that prints the alias unexpanded, and say native Windows stops at the designed vLLM refusal. List the refusals in the serve_plan module doc in the order serve_cmd.rs runs them, derive ServeRun.engine from the value the command runs with, render the streams through the shared section helper so an empty stdout is marked, drop the doubled prefix in the consistency step, and cover blank and indented stderr lines. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/expectations.toml | 18 ++++++++----- tests/e2e-cucumber/src/serve_plan.rs | 26 ++++++++++++++++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 10 +++---- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 053d76de91..19ec8e2d4c 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -32,24 +32,28 @@ # # An empty `when = {}` means "always xfail" (platform-independent bug). -# --- EAI-7219: short-name expansion (vLLM does not resolve the alias) --- +# --- EAI-7219: short-name expansion (vLLM does not show an expanded name) --- # What the lane logs show, so the scope below is not read as more than it is: # lemonade DOES print the full name (`resolved model: Qwen/Qwen2.5-1.5B-Instruct`) # on the Strix Halo Ubuntu and Windows lanes, and vLLM's stdout is empty -- the run # stopped before it printed a serve plan, so the plan's name line was never -# reached. Whether that stop is the bug above or an earlier refusal (a missing -# active runtime, a no-GPU bail, native Windows refusing vLLM since ROCMAI-823) is -# not known: the steps used to discard stderr. They now report it, so the next lane -# run says which. Until then the rows keep their all-hosts scope. +# reached. EAI-7219 as reported describes the other shape: a vLLM plan that prints +# `resolved model: qwen2.5`, the alias left unexpanded; the lane logs have not +# shown it. Whether the empty stdout is that bug or an earlier refusal (a missing +# active runtime, a no-GPU bail) is not known: the steps used to discard stderr. +# They now report it, so the next lane run says which. On native Windows the stop +# is the up-front vLLM refusal (ROCMAI-823), a designed refusal and not this bug. +# Until the lane stderr is in, the rows keep their all-hosts scope, and their +# reasons name both shapes so neither becomes a false claim. [["serve-short-name-expansion"]] when = {} bug = "EAI-7219" -reason = "A vLLM serve of a short name prints no serve plan, so no expanded full model name is shown." +reason = "A vLLM serve of a short name shows no expanded full model name: it either stops before printing a serve plan or prints the alias unexpanded." [["serve-short-name-consistent-across-engines"]] when = {} bug = "EAI-7219" -reason = "A vLLM serve of a short name prints no serve plan, so its resolved model name cannot match lemonade's." +reason = "A vLLM serve of a short name does not resolve to lemonade's full model name: it either stops before printing a serve plan or prints the alias unexpanded." # EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: # which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run diff --git a/tests/e2e-cucumber/src/serve_plan.rs b/tests/e2e-cucumber/src/serve_plan.rs index 065a521164..5723d6f540 100644 --- a/tests/e2e-cucumber/src/serve_plan.rs +++ b/tests/e2e-cucumber/src/serve_plan.rs @@ -5,8 +5,10 @@ //! Say why a `rocm serve` run printed no `resolved model:` line. //! //! `rocm serve` prints its plan (`serve plan` / `resolved model: ...`) only after a -//! chain of refusals has passed: the platform and engine gate, the no-usable-GPU -//! bail, the "no active ROCm runtime" bail and the engine's model-resolution call. +//! chain of refusals has passed, among them, in the order `serve_cmd.rs` runs +//! them: the OS gate that rules an engine out (vLLM on native Windows), the +//! no-usable-GPU bail, the Docker-aware engine gate, the "no active ROCm runtime" +//! bail and the engine's model-resolution call. //! A run that stops at any of them leaves stdout without the line, and the refusal //! goes to stderr. The short-name scenarios used to keep stdout only, so every //! such run read as "the name was not expanded" whatever actually stopped it. The @@ -55,8 +57,9 @@ impl ServeRun { ), }; format!( - "no 'resolved model' in output: {verdict}\n\nstdout:\n{}\n\nstderr:\n{}", - self.stdout, self.stderr + "no 'resolved model' in output: {verdict}\n\n{}\n\n{}", + crate::section("stdout", &self.stdout), + crate::section("stderr", &self.stderr) ) } } @@ -133,5 +136,20 @@ mod tests { message.contains("(exit 1) and wrote nothing to stderr"), "{message}" ); + assert!( + message.contains("--- stdout: (empty) ---"), + "an empty stream is marked, not left as a bare heading: {message}" + ); + } + + #[test] + fn blank_stderr_lines_are_skipped_and_the_reported_line_is_trimmed() { + let indented = attempt("vllm", "", " \n\t\n Error: refused\n", 1).missing_plan_message(); + assert!(indented.contains("(exit 1): Error: refused"), "{indented}"); + let blank_only = attempt("vllm", "", " \n", 1).missing_plan_message(); + assert!( + blank_only.contains("(exit 1) and wrote nothing to stderr"), + "whitespace is not a stderr line: {blank_only}" + ); } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c31baba794..bce78d7ebe 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -831,11 +831,12 @@ async fn setup_model_detected(world: &mut E2eWorld) { #[when("the user serves a model using its short name")] async fn user_serves_short_name(world: &mut E2eWorld) { - let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", "vllm"]); + let engine = "vllm"; + let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); // stderr and the exit code ride along so a run that stopped on a refusal reads // as one in the Then step, instead of as "the name was not expanded". world.serve_runs = Some(vec![ServeRun { - engine: "vllm".to_owned(), + engine: engine.to_owned(), stdout, stderr, rc, @@ -2184,10 +2185,7 @@ async fn assert_consistent_expansion(world: &mut E2eWorld) { // than defaulting to an empty string — otherwise two engines that both // fail would produce equal ("") values and pass this check vacuously. let Some(model) = run.resolved_model().map(str::to_string) else { - panic!( - "engine #{i} produced no 'resolved model' line: {}", - run.missing_plan_message() - ); + panic!("engine #{i}: {}", run.missing_plan_message()); }; resolved.push(model); } From 0a3fcd66eb8393a891b39b899b6bd29fb38d0338 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 08:24:26 +0000 Subject: [PATCH 3/3] test(e2e): record why the short-name serve scenarios stop on the Linux lanes The lane run that first reported stderr shows vLLM stopping at the missing active runtime before any alias handling, so say that in the EAI-7219 comment instead of calling the cause unknown, and what the rows wait on. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/expectations.toml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 19ec8e2d4c..d678d701ef 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -39,12 +39,15 @@ # stopped before it printed a serve plan, so the plan's name line was never # reached. EAI-7219 as reported describes the other shape: a vLLM plan that prints # `resolved model: qwen2.5`, the alias left unexpanded; the lane logs have not -# shown it. Whether the empty stdout is that bug or an earlier refusal (a missing -# active runtime, a no-GPU bail) is not known: the steps used to discard stderr. -# They now report it, so the next lane run says which. On native Windows the stop -# is the up-front vLLM refusal (ROCMAI-823), a designed refusal and not this bug. -# Until the lane stderr is in, the rows keep their all-hosts scope, and their -# reasons name both shapes so neither becomes a false claim. +# shown it. The steps used to discard stderr, so the cause of the empty stdout was +# not known. The first lane run that reported it (run 41811714290: Strix Halo +# Ubuntu release and nightly, and MI300X) shows vLLM stopping at "no active ROCm +# runtime is configured", before any alias handling, so on those lanes the rows are +# red for a missing runtime and not for this bug. On native Windows the stop is the +# up-front vLLM refusal (ROCMAI-823), a designed refusal and not this bug either. +# The rows keep their all-hosts scope until the scenarios get a managed runtime or +# are gated on the engine and the rows are narrowed to the real bug; their reasons +# name both shapes so neither becomes a false claim. [["serve-short-name-expansion"]] when = {} bug = "EAI-7219"