diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 4f26ef6b59..d678d701ef 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -32,16 +32,31 @@ # # An empty `when = {}` means "always xfail" (platform-independent bug). -# --- EAI-7219: short-name expansion not surfaced in serve output (any engine) --- +# --- EAI-7219: short-name expansion (vLLM does not show an expanded name) --- +# What the lane logs show, so the scope below is not read as more than it is: +# lemonade DOES print the full name (`resolved model: Qwen/Qwen2.5-1.5B-Instruct`) +# on the Strix Halo Ubuntu and Windows lanes, and vLLM's stdout is empty -- the run +# stopped before it printed a serve plan, so the plan's name line was never +# reached. EAI-7219 as reported describes the other shape: a vLLM plan that prints +# `resolved model: qwen2.5`, the alias left unexpanded; the lane logs have not +# shown it. The steps used to discard stderr, so the cause of the empty stdout was +# not known. The first lane run that reported it (run 41811714290: Strix Halo +# Ubuntu release and nightly, and MI300X) shows vLLM stopping at "no active ROCm +# runtime is configured", before any alias handling, so on those lanes the rows are +# red for a missing runtime and not for this bug. On native Windows the stop is the +# up-front vLLM refusal (ROCMAI-823), a designed refusal and not this bug either. +# The rows keep their all-hosts scope until the scenarios get a managed runtime or +# are gated on the engine and the rows are narrowed to the real bug; their reasons +# name both shapes so neither becomes a false claim. [["serve-short-name-expansion"]] when = {} bug = "EAI-7219" -reason = "Serve output does not surface the expanded full model name for a short name." +reason = "A vLLM serve of a short name shows no expanded full model name: it either stops before printing a serve plan or prints the alias unexpanded." [["serve-short-name-consistent-across-engines"]] when = {} bug = "EAI-7219" -reason = "Engines do not report a consistent resolved model name for a short name." +reason = "A vLLM serve of a short name does not resolve to lemonade's full model name: it either stops before printing a serve plan or prints the alias unexpanded." # EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: # which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 110023fe94..9c9bdc62c5 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -21,6 +21,7 @@ pub mod quit_and_wait; pub mod reader_failure; pub mod send_until; pub mod serve_log; +pub mod serve_plan; pub mod shared_runtime; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/serve_plan.rs b/tests/e2e-cucumber/src/serve_plan.rs new file mode 100644 index 0000000000..5723d6f540 --- /dev/null +++ b/tests/e2e-cucumber/src/serve_plan.rs @@ -0,0 +1,155 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Say why a `rocm serve` run printed no `resolved model:` line. +//! +//! `rocm serve` prints its plan (`serve plan` / `resolved model: ...`) only after a +//! chain of refusals has passed, among them, in the order `serve_cmd.rs` runs +//! them: the OS gate that rules an engine out (vLLM on native Windows), the +//! no-usable-GPU bail, the Docker-aware engine gate, the "no active ROCm runtime" +//! bail and the engine's model-resolution call. +//! A run that stops at any of them leaves stdout without the line, and the refusal +//! goes to stderr. The short-name scenarios used to keep stdout only, so every +//! such run read as "the name was not expanded" whatever actually stopped it. The +//! lane logs show exactly that: a vLLM serve whose stdout was empty, with nothing +//! to say which refusal it was. + +/// One `rocm serve` run: the engine it asked for and everything it produced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServeRun { + pub engine: String, + pub stdout: String, + pub stderr: String, + pub rc: i32, +} + +impl ServeRun { + /// The model named by the plan's `resolved model:` line, if the plan was + /// printed at all. + #[must_use] + pub fn resolved_model(&self) -> Option<&str> { + resolved_model(&self.stdout) + } + + /// What to say when [`resolved_model`](Self::resolved_model) is `None`. + /// + /// Names the engine, the exit code and the first line of stderr, so a refusal + /// reads as a refusal and a run that exited 0 without a plan reads as that. + #[must_use] + pub fn missing_plan_message(&self) -> String { + let engine = &self.engine; + let rc = self.rc; + let stderr_line = self + .stderr + .lines() + .map(str::trim) + .find(|line| !line.is_empty()); + let verdict = match (rc, stderr_line) { + (0, _) => format!( + "`rocm serve --engine {engine}` exited 0 but printed no serve plan, so the name was never shown" + ), + (_, Some(line)) => format!( + "`rocm serve --engine {engine}` stopped before it printed a serve plan (exit {rc}): {line}" + ), + (_, None) => format!( + "`rocm serve --engine {engine}` stopped before it printed a serve plan (exit {rc}) and wrote nothing to stderr" + ), + }; + format!( + "no 'resolved model' in output: {verdict}\n\n{}\n\n{}", + crate::section("stdout", &self.stdout), + crate::section("stderr", &self.stderr) + ) + } +} + +/// The model a serve plan resolved to (`resolved model: `), if it printed one. +/// +/// It can differ from the requested id: `rocm serve ` with no `--engine` +/// picks the recipe's preferred model and engine. `None` means the plan has no +/// such line, for example because the serve stopped before resolving. +#[must_use] +pub fn resolved_model(output: &str) -> Option<&str> { + output + .lines() + .find_map(|l| l.trim().strip_prefix("resolved model:")) + .map(str::trim) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn attempt(engine: &str, stdout: &str, stderr: &str, rc: i32) -> ServeRun { + ServeRun { + engine: engine.to_owned(), + stdout: stdout.to_owned(), + stderr: stderr.to_owned(), + rc, + } + } + + #[test] + fn a_printed_plan_names_its_resolved_model() { + let plan = "serve plan\n requested model: qwen2.5\n resolved model: Qwen/Qwen2.5-1.5B-Instruct\n engine: lemonade\n"; + assert_eq!( + attempt("lemonade", plan, "", 0).resolved_model(), + Some("Qwen/Qwen2.5-1.5B-Instruct") + ); + } + + #[test] + fn a_refusal_is_reported_as_a_refusal_with_its_stderr_line() { + let refused = attempt( + "vllm", + "", + "\nError: no active ROCm runtime is configured; run `rocm runtimes list`\n", + 1, + ); + assert_eq!(refused.resolved_model(), None); + let message = refused.missing_plan_message(); + assert!( + message.contains("`rocm serve --engine vllm` stopped before it printed a serve plan (exit 1): Error: no active ROCm runtime is configured"), + "{message}" + ); + assert!( + !message.contains("exited 0"), + "a non-zero exit must not read as a clean one: {message}" + ); + } + + #[test] + fn an_exit_zero_without_a_plan_is_not_called_a_refusal() { + let message = attempt("lemonade", "", "warning: ignored", 0).missing_plan_message(); + assert!( + message.contains("exited 0 but printed no serve plan"), + "{message}" + ); + assert!(!message.contains("stopped before"), "{message}"); + } + + #[test] + fn a_silent_failure_says_stderr_was_empty() { + let message = attempt("vllm", "", "", 1).missing_plan_message(); + assert!( + message.contains("(exit 1) and wrote nothing to stderr"), + "{message}" + ); + assert!( + message.contains("--- stdout: (empty) ---"), + "an empty stream is marked, not left as a bare heading: {message}" + ); + } + + #[test] + fn blank_stderr_lines_are_skipped_and_the_reported_line_is_trimmed() { + let indented = attempt("vllm", "", " \n\t\n Error: refused\n", 1).missing_plan_message(); + assert!(indented.contains("(exit 1): Error: refused"), "{indented}"); + let blank_only = attempt("vllm", "", " \n", 1).missing_plan_message(); + assert!( + blank_only.contains("(exit 1) and wrote nothing to stderr"), + "whitespace is not a stderr line: {blank_only}" + ); + } +} diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 2d56ac2835..28e6f0c4e7 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -76,6 +76,9 @@ pub struct E2eWorld { pub cli_outputs: Option>, pub cli_stderr: Option, pub cli_rc: Option, + /// One entry per `rocm serve` run a scenario made, with the stderr and exit + /// code `cli_outputs` has no room for (see `e2e_cucumber::serve_plan`). + pub serve_runs: Option>, /// Wall-clock time the last measured `rocm` invocation took, for scenarios /// where the *duration* bounds the behaviour under test — /// `service-cleanup-07`, where a prune that returned while the managed-launch @@ -320,6 +323,7 @@ impl Default for E2eWorld { cli_outputs: None, cli_stderr: None, cli_rc: None, + serve_runs: None, cli_elapsed: None, remote_env: Vec::new(), launch_release: None, diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 49ab899ce3..bce78d7ebe 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -15,6 +15,7 @@ use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_ use e2e_cucumber::serve_log::{ ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, }; +use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; /// Model id of the live managed service the reuse scenarios plant. /// @@ -830,18 +831,32 @@ async fn setup_model_detected(world: &mut E2eWorld) { #[when("the user serves a model using its short name")] async fn user_serves_short_name(world: &mut E2eWorld) { - let (stdout, _, _) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", "vllm"]); - world.cli_output = Some(stdout); + let engine = "vllm"; + let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); + // stderr and the exit code ride along so a run that stopped on a refusal reads + // as one in the Then step, instead of as "the name was not expanded". + world.serve_runs = Some(vec![ServeRun { + engine: engine.to_owned(), + stdout, + stderr, + rc, + }]); } #[when("the user serves the same short name with different engines")] async fn user_serves_multiple_engines(world: &mut E2eWorld) { - let mut outputs = Vec::new(); + let mut runs = Vec::new(); for engine in ["lemonade", "vllm"] { - let (stdout, _, _) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); - outputs.push(stdout); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]); + runs.push(ServeRun { + engine: engine.to_owned(), + stdout, + stderr, + rc, + }); } - world.cli_outputs = Some(outputs); + world.serve_runs = Some(runs); } #[when("the user lists running services")] @@ -2133,12 +2148,15 @@ async fn assert_masked_index_message(world: &mut E2eWorld) { #[then("the output shows the full model name")] async fn assert_full_model_name(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no CLI output"); - let resolved = resolved_model(output) - .unwrap_or_else(|| panic!("no 'resolved model' in output:\n{output}")); + let runs = world.serve_runs.as_ref().expect("no serve run"); + let run = runs.first().expect("no serve run"); + let resolved = run + .resolved_model() + .unwrap_or_else(|| panic!("{}", run.missing_plan_message())); assert!( resolved.contains('/'), - "expected a fully qualified model name (org/model), got '{resolved}'\n\nfull output:\n{output}" + "expected a fully qualified model name (org/model), got '{resolved}'\n\nfull output:\n{}", + run.stdout ); } @@ -2158,19 +2176,16 @@ async fn assert_model_list_is_ascii(world: &mut E2eWorld) { #[then("all engines expand to the same full model name")] async fn assert_consistent_expansion(world: &mut E2eWorld) { - let outputs = world.cli_outputs.as_ref().expect("no CLI outputs"); - assert!( - outputs.len() >= 2, - "need at least 2 serve outputs to compare" - ); + let runs = world.serve_runs.as_ref().expect("no serve runs"); + assert!(runs.len() >= 2, "need at least 2 serve outputs to compare"); let mut resolved: Vec = Vec::new(); - for (i, output) in outputs.iter().enumerate() { + for (i, run) in runs.iter().enumerate() { // A missing `resolved model:` line means the engine never got far enough // to expand the name (e.g. it errored before serving). Fail loudly rather // than defaulting to an empty string — otherwise two engines that both // fail would produce equal ("") values and pass this check vacuously. - let Some(model) = resolved_model(output).map(str::to_string) else { - panic!("engine #{i} produced no 'resolved model' line:\n{output}"); + let Some(model) = run.resolved_model().map(str::to_string) else { + panic!("engine #{i}: {}", run.missing_plan_message()); }; resolved.push(model); } @@ -2257,17 +2272,6 @@ fn selected_engine(output: &str) -> &str { engine } -/// The model a serve plan actually resolved to (`resolved model: `), which -/// can differ from the requested id — `rocm serve ` with no `--engine` -/// picks the recipe's preferred model+engine. `None` if the plan has no such line -/// (e.g. the serve errored before resolving). -fn resolved_model(output: &str) -> Option<&str> { - output - .lines() - .find_map(|l| l.trim().strip_prefix("resolved model:")) - .map(str::trim) -} - /// A distinctive substring of a model id that appears in the served endpoint's /// `/v1/models` response, for [`model_is_ready`]'s containment check. Strips the /// `org/` prefix and the `-GGUF` catalog marker so a resolved catalog id