Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions tests/e2e-cucumber/expectations.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,31 @@
#
# An empty `when = {}` means "always xfail" (platform-independent bug).

# --- EAI-7219: short-name expansion not surfaced in serve output (any engine) ---
# --- EAI-7219: short-name expansion (vLLM does not show an expanded name) ---
# What the lane logs show, so the scope below is not read as more than it is:
# lemonade DOES print the full name (`resolved model: Qwen/Qwen2.5-1.5B-Instruct`)
# on the Strix Halo Ubuntu and Windows lanes, and vLLM's stdout is empty -- the run
# stopped before it printed a serve plan, so the plan's name line was never
# reached. EAI-7219 as reported describes the other shape: a vLLM plan that prints
# `resolved model: qwen2.5`, the alias left unexpanded; the lane logs have not
# shown it. The steps used to discard stderr, so the cause of the empty stdout was
# not known. The first lane run that reported it (run 41811714290: Strix Halo
# Ubuntu release and nightly, and MI300X) shows vLLM stopping at "no active ROCm
# runtime is configured", before any alias handling, so on those lanes the rows are
# red for a missing runtime and not for this bug. On native Windows the stop is the
# up-front vLLM refusal (ROCMAI-823), a designed refusal and not this bug either.
# The rows keep their all-hosts scope until the scenarios get a managed runtime or
# are gated on the engine and the rows are narrowed to the real bug; their reasons
# name both shapes so neither becomes a false claim.
[["serve-short-name-expansion"]]
when = {}
bug = "EAI-7219"
reason = "Serve output does not surface the expanded full model name for a short name."
reason = "A vLLM serve of a short name shows no expanded full model name: it either stops before printing a serve plan or prints the alias unexpanded."

[["serve-short-name-consistent-across-engines"]]
when = {}
bug = "EAI-7219"
reason = "Engines do not report a consistent resolved model name for a short name."
reason = "A vLLM serve of a short name does not resolve to lemonade's full model name: it either stops before printing a serve plan or prints the alias unexpanded."

# EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed:
# which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run
Expand Down
1 change: 1 addition & 0 deletions tests/e2e-cucumber/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub mod quit_and_wait;
pub mod reader_failure;
pub mod send_until;
pub mod serve_log;
pub mod serve_plan;
pub mod shared_runtime;

use std::path::{Path, PathBuf};
Expand Down
155 changes: 155 additions & 0 deletions tests/e2e-cucumber/src/serve_plan.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
// Copyright © Advanced Micro Devices, Inc., or its affiliates.
//
// SPDX-License-Identifier: MIT

//! Say why a `rocm serve` run printed no `resolved model:` line.
//!
//! `rocm serve` prints its plan (`serve plan` / `resolved model: ...`) only after a
//! chain of refusals has passed, among them, in the order `serve_cmd.rs` runs
//! them: the OS gate that rules an engine out (vLLM on native Windows), the
//! no-usable-GPU bail, the Docker-aware engine gate, the "no active ROCm runtime"
//! bail and the engine's model-resolution call.
//! A run that stops at any of them leaves stdout without the line, and the refusal
//! goes to stderr. The short-name scenarios used to keep stdout only, so every
//! such run read as "the name was not expanded" whatever actually stopped it. The
//! lane logs show exactly that: a vLLM serve whose stdout was empty, with nothing
//! to say which refusal it was.

/// One `rocm serve` run: the engine it asked for and everything it produced.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ServeRun {
pub engine: String,
pub stdout: String,
pub stderr: String,
pub rc: i32,
}

impl ServeRun {
/// The model named by the plan's `resolved model:` line, if the plan was
/// printed at all.
#[must_use]
pub fn resolved_model(&self) -> Option<&str> {
resolved_model(&self.stdout)
}

/// What to say when [`resolved_model`](Self::resolved_model) is `None`.
///
/// Names the engine, the exit code and the first line of stderr, so a refusal
/// reads as a refusal and a run that exited 0 without a plan reads as that.
#[must_use]
pub fn missing_plan_message(&self) -> String {
let engine = &self.engine;
let rc = self.rc;
let stderr_line = self
.stderr
.lines()
.map(str::trim)
.find(|line| !line.is_empty());
let verdict = match (rc, stderr_line) {
(0, _) => format!(
"`rocm serve --engine {engine}` exited 0 but printed no serve plan, so the name was never shown"
),
(_, Some(line)) => format!(
"`rocm serve --engine {engine}` stopped before it printed a serve plan (exit {rc}): {line}"
),
(_, None) => format!(
"`rocm serve --engine {engine}` stopped before it printed a serve plan (exit {rc}) and wrote nothing to stderr"
),
};
format!(
"no 'resolved model' in output: {verdict}\n\n{}\n\n{}",
crate::section("stdout", &self.stdout),
crate::section("stderr", &self.stderr)
)
}
}

/// The model a serve plan resolved to (`resolved model: <id>`), if it printed one.
///
/// It can differ from the requested id: `rocm serve <model>` with no `--engine`
/// picks the recipe's preferred model and engine. `None` means the plan has no
/// such line, for example because the serve stopped before resolving.
#[must_use]
pub fn resolved_model(output: &str) -> Option<&str> {
output
.lines()
.find_map(|l| l.trim().strip_prefix("resolved model:"))
.map(str::trim)
}

#[cfg(test)]
mod tests {
use super::*;

fn attempt(engine: &str, stdout: &str, stderr: &str, rc: i32) -> ServeRun {
ServeRun {
engine: engine.to_owned(),
stdout: stdout.to_owned(),
stderr: stderr.to_owned(),
rc,
}
}

#[test]
fn a_printed_plan_names_its_resolved_model() {
let plan = "serve plan\n requested model: qwen2.5\n resolved model: Qwen/Qwen2.5-1.5B-Instruct\n engine: lemonade\n";
assert_eq!(
attempt("lemonade", plan, "", 0).resolved_model(),
Some("Qwen/Qwen2.5-1.5B-Instruct")
);
}

#[test]
fn a_refusal_is_reported_as_a_refusal_with_its_stderr_line() {
let refused = attempt(
"vllm",
"",
"\nError: no active ROCm runtime is configured; run `rocm runtimes list`\n",
1,
);
assert_eq!(refused.resolved_model(), None);
let message = refused.missing_plan_message();
assert!(
message.contains("`rocm serve --engine vllm` stopped before it printed a serve plan (exit 1): Error: no active ROCm runtime is configured"),
"{message}"
);
assert!(
!message.contains("exited 0"),
"a non-zero exit must not read as a clean one: {message}"
);
}

#[test]
fn an_exit_zero_without_a_plan_is_not_called_a_refusal() {
let message = attempt("lemonade", "", "warning: ignored", 0).missing_plan_message();
assert!(
message.contains("exited 0 but printed no serve plan"),
"{message}"
);
assert!(!message.contains("stopped before"), "{message}");
}

#[test]
fn a_silent_failure_says_stderr_was_empty() {
let message = attempt("vllm", "", "", 1).missing_plan_message();
assert!(
message.contains("(exit 1) and wrote nothing to stderr"),
"{message}"
);
assert!(
message.contains("--- stdout: (empty) ---"),
"an empty stream is marked, not left as a bare heading: {message}"
);
}

#[test]
fn blank_stderr_lines_are_skipped_and_the_reported_line_is_trimmed() {
let indented = attempt("vllm", "", " \n\t\n Error: refused\n", 1).missing_plan_message();
assert!(indented.contains("(exit 1): Error: refused"), "{indented}");
let blank_only = attempt("vllm", "", " \n", 1).missing_plan_message();
assert!(
blank_only.contains("(exit 1) and wrote nothing to stderr"),
"whitespace is not a stderr line: {blank_only}"
);
}
}
4 changes: 4 additions & 0 deletions tests/e2e-cucumber/tests/e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ pub struct E2eWorld {
pub cli_outputs: Option<Vec<String>>,
pub cli_stderr: Option<String>,
pub cli_rc: Option<i32>,
/// One entry per `rocm serve` run a scenario made, with the stderr and exit
/// code `cli_outputs` has no room for (see `e2e_cucumber::serve_plan`).
pub serve_runs: Option<Vec<e2e_cucumber::serve_plan::ServeRun>>,
/// Wall-clock time the last measured `rocm` invocation took, for scenarios
/// where the *duration* bounds the behaviour under test —
/// `service-cleanup-07`, where a prune that returned while the managed-launch
Expand Down Expand Up @@ -320,6 +323,7 @@ impl Default for E2eWorld {
cli_outputs: None,
cli_stderr: None,
cli_rc: None,
serve_runs: None,
cli_elapsed: None,
remote_env: Vec::new(),
launch_release: None,
Expand Down
62 changes: 33 additions & 29 deletions tests/e2e-cucumber/tests/e2e/serving_steps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_
use e2e_cucumber::serve_log::{
ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail,
};
use e2e_cucumber::serve_plan::{ServeRun, resolved_model};

/// Model id of the live managed service the reuse scenarios plant.
///
Expand Down Expand Up @@ -830,18 +831,32 @@ async fn setup_model_detected(world: &mut E2eWorld) {

#[when("the user serves a model using its short name")]
async fn user_serves_short_name(world: &mut E2eWorld) {
let (stdout, _, _) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", "vllm"]);
world.cli_output = Some(stdout);
let engine = "vllm";
let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]);
// stderr and the exit code ride along so a run that stopped on a refusal reads
// as one in the Then step, instead of as "the name was not expanded".
world.serve_runs = Some(vec![ServeRun {
engine: engine.to_owned(),
stdout,
stderr,
rc,
}]);
}

#[when("the user serves the same short name with different engines")]
async fn user_serves_multiple_engines(world: &mut E2eWorld) {
let mut outputs = Vec::new();
let mut runs = Vec::new();
for engine in ["lemonade", "vllm"] {
let (stdout, _, _) = crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]);
outputs.push(stdout);
let (stdout, stderr, rc) =
crate::run_rocm(world, &["serve", "qwen2.5", "--engine", engine]);
runs.push(ServeRun {
engine: engine.to_owned(),
stdout,
stderr,
rc,
});
}
world.cli_outputs = Some(outputs);
world.serve_runs = Some(runs);
}

#[when("the user lists running services")]
Expand Down Expand Up @@ -2133,12 +2148,15 @@ async fn assert_masked_index_message(world: &mut E2eWorld) {

#[then("the output shows the full model name")]
async fn assert_full_model_name(world: &mut E2eWorld) {
let output = world.cli_output.as_ref().expect("no CLI output");
let resolved = resolved_model(output)
.unwrap_or_else(|| panic!("no 'resolved model' in output:\n{output}"));
let runs = world.serve_runs.as_ref().expect("no serve run");
let run = runs.first().expect("no serve run");
let resolved = run
.resolved_model()
.unwrap_or_else(|| panic!("{}", run.missing_plan_message()));
assert!(
resolved.contains('/'),
"expected a fully qualified model name (org/model), got '{resolved}'\n\nfull output:\n{output}"
"expected a fully qualified model name (org/model), got '{resolved}'\n\nfull output:\n{}",
run.stdout
);
}

Expand All @@ -2158,19 +2176,16 @@ async fn assert_model_list_is_ascii(world: &mut E2eWorld) {

#[then("all engines expand to the same full model name")]
async fn assert_consistent_expansion(world: &mut E2eWorld) {
let outputs = world.cli_outputs.as_ref().expect("no CLI outputs");
assert!(
outputs.len() >= 2,
"need at least 2 serve outputs to compare"
);
let runs = world.serve_runs.as_ref().expect("no serve runs");
assert!(runs.len() >= 2, "need at least 2 serve outputs to compare");
let mut resolved: Vec<String> = Vec::new();
for (i, output) in outputs.iter().enumerate() {
for (i, run) in runs.iter().enumerate() {
// A missing `resolved model:` line means the engine never got far enough
// to expand the name (e.g. it errored before serving). Fail loudly rather
// than defaulting to an empty string — otherwise two engines that both
// fail would produce equal ("") values and pass this check vacuously.
let Some(model) = resolved_model(output).map(str::to_string) else {
panic!("engine #{i} produced no 'resolved model' line:\n{output}");
let Some(model) = run.resolved_model().map(str::to_string) else {
panic!("engine #{i}: {}", run.missing_plan_message());
};
resolved.push(model);
}
Expand Down Expand Up @@ -2257,17 +2272,6 @@ fn selected_engine(output: &str) -> &str {
engine
}

/// The model a serve plan actually resolved to (`resolved model: <id>`), which
/// can differ from the requested id — `rocm serve <model>` with no `--engine`
/// picks the recipe's preferred model+engine. `None` if the plan has no such line
/// (e.g. the serve errored before resolving).
fn resolved_model(output: &str) -> Option<&str> {
output
.lines()
.find_map(|l| l.trim().strip_prefix("resolved model:"))
.map(str::trim)
}

/// A distinctive substring of a model id that appears in the served endpoint's
/// `/v1/models` response, for [`model_is_ready`]'s containment check. Strips the
/// `org/` prefix and the `-GGUF` catalog marker so a resolved catalog id
Expand Down
Loading