diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index fec5e0154f..17d9e7dbbb 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -164,3 +164,49 @@ bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." serve_timeout_secs = 90 flaky = true + +# -- README-walkthrough defects (EAI-8047, EAI-8049) -- +# Defects found walking the CLI on an MI300X host, each pinned by a +# scenario asserting the CORRECT behaviour. Both are in code paths with no +# hardware, engine or network dependency, and both were re-confirmed in a plain +# Linux container — so each row is scoped `when = {}` and is expected to go stale +# (XPASS) on every lane the day its bug is fixed. Neither is `flaky`: each +# reproduced deterministically. + +# --- EAI-8047: listed automation checks cannot be turned on from the listing --- +# `automations list` shows each check by display name only ("Server recovery"), +# while `automations enable` accepts the internal id ("server-recover") and +# rejects anything else with rc=1. The names do not mechanically reduce to the +# ids, so nothing the listing prints is enough to act on. Remove this row once +# the listing exposes an identifier a user can pass back in. +# +# Either direction closes it, and the scenario accepts both: show the id in the +# listing, or make `enable` accept the label exactly as listed. Note that +# `render_automations_text`'s unit tests currently assert the listing does NOT +# contain `command id:` or the raw ids (apps/rocm/src/automations.rs), so hiding them was +# a deliberate choice; a fix that shows the ids has to update those assertions. +[["automations-listed-checks-can-be-enabled"]] +when = {} +bug = "EAI-8047" +reason = "`automations list` publishes no usable identifier, so the checks it lists cannot be enabled from what it shows." + +# --- EAI-8049: the engine shell's activation hint names a path that is not there --- +# `engines shell` builds the hint by appending the interpreter directory to the +# resolved env_path. Users reach it through the auto-install branch of +# `resolve_engine_env`, which uses the install response's env_path live, and the +# vLLM install reports that as the directory CONTAINING the command +# (`command.parent()`, engines/vllm/src/install.rs). The hint therefore doubles +# the segment and points at a file nobody can source. Container-measured: with a bin-shaped env_path the hint named +# `/bin/bin/activate` (absent) while a root-shaped one named +# `/bin/activate` (present). +# +# The scenario plants a bin-shaped manifest as a stand-in for that response: no +# install writes such a manifest to disk, it is only the cheapest way to hand the +# shell the value the vLLM install reports. A fix in engines/vllm that reports the +# environment root therefore closes EAI-8049 but does NOT flip this row, since the +# planted manifest is unchanged; delete the row and rewrite or delete the +# scenario together with that fix. Only a fix to the hint composition flips it. --- +[["engine-shell-activation-hint-is-usable"]] +when = {} +bug = "EAI-8049" +reason = "`engines shell` prints an activation hint whose path does not exist, because the env_path it is given (as reported by the vLLM install response) already includes the interpreter directory." diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature index 52e668129e..3bcd6dd591 100644 --- a/tests/e2e-cucumber/features/automations.feature +++ b/tests/e2e-cucumber/features/automations.feature @@ -197,3 +197,16 @@ Feature: Automation watchers And a GPU thermal alert for the server reaches the daemon again Then the daemon records that a stop request is already waiting for the server And only one stop request for the server is waiting for review + + # The scenarios above enable a watcher by an id the test already knows. This one pins + # the complementary contract: the listing is the only place a user learns which + # background checks exist, so whatever it shows has to be enough to act on. + # Deriving each check's identifier from the listing the way a reader would IS + # the contract under test — the identifiers are deliberately not written into + # the scenario or the steps, because hard-coding them would pass against a + # listing that exposes nothing. No fixtures, no GPU, no network. + @id:automations-listed-checks-can-be-enabled + Scenario: automations-13 - Every background check that is listed can be turned on + Given a machine with no background checks turned on + When the user lists the background checks + Then every listed check can be turned on from what the listing shows diff --git a/tests/e2e-cucumber/features/engine_shell.feature b/tests/e2e-cucumber/features/engine_shell.feature index c6666c0e19..5696fc456c 100644 --- a/tests/e2e-cucumber/features/engine_shell.feature +++ b/tests/e2e-cucumber/features/engine_shell.feature @@ -22,3 +22,35 @@ Feature: Engine shell And the engine environment's interpreter is the one that runs When the user leaves the engine shell Then the engine shell exits successfully + + # The env_path is planted bin-shaped, the way the vLLM install RESPONSE reports it + # (`command.parent()` in engines/vllm/src/install.rs; users reach it through the + # auto-install branch of resolve_engine_env). No install writes such a manifest + # to disk — planting one is only the cheapest way to hand the shell that value + # without a multi-GiB install. The shell composes its activation hint by + # appending the interpreter directory to whatever env_path it is given, so a + # value that already points inside it yields a path nobody can source. + # + # Consequence for whoever fixes this: a fix in engines/vllm (report the env root) + # closes EAI-8049 but leaves this fixture unchanged, so the scenario must then be + # rewritten or deleted along with its expectation row. Only a fix to the hint + # composition itself makes this row go stale. + # + # The interpreter step is here because the same doubled path also sets a wrong + # VIRTUAL_ENV and puts a nonexistent bin/bin on PATH: a fix to the hint text + # alone would otherwise flip this row while the shell still runs the wrong + # python. The step runs the interpreter and also checks that VIRTUAL_ENV names + # an environment holding bin/activate, so the row flips only once the hint, PATH + # and VIRTUAL_ENV are all right. + # + # Phrased as "the file it names is there", not as "the path is not doubled": the + # first survives whichever way the mismatch is repaired, the second would go + # stale the moment it is. + @id:engine-shell-activation-hint-is-usable @requires-os:linux + Scenario: engine-shell-02 - The activation hint an engine shell prints refers to a real file + Given a machine with an engine environment installed the way the engine records it + When the user opens a shell for that engine + Then the printed activation hint names a file that exists + And the engine environment's interpreter is the one that runs + When the user leaves the engine shell + Then the engine shell exits successfully diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 95972ef56e..ee3262effc 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -19,6 +19,13 @@ //! proposal history (ROCMAI-729). One scenario (automations-12) starts a real //! `rocmd run` and feeds it GPU thermal alerts through its loopback webhook; the //! rest of the watcher/daemon trigger behaviour is covered separately. +//! +//! Two slices live here. The enable/disable/mode steps act on a watcher id the +//! test already knows. The listing steps (scenario 13) assert the complementary +//! discoverability contract: everything the listing shows a user must also be +//! able to act on, so they derive each check's identifier FROM the listing rather +//! than knowing it in advance — hard-coding the real ids would keep passing +//! against a listing that publishes none of them, which is the defect pinned. use std::io::{Read, Write}; use std::process::Stdio; @@ -65,6 +72,112 @@ pub(crate) fn suppress_daemon_spawn(world: &E2eWorld) { .expect("failed to write automation runtime state"); } +/// Slugify a display name: lowercase, words joined by dashes. +/// +/// This is the guess a user is forced into when the listing exposes no real +/// identifier — the current-bug fallback, not the contract. It deliberately does +/// NOT recover the true ids ("Server recovery" → `server-recovery`, not the real +/// `server-recover`), which is exactly why the listing has to publish them. +fn slug_of(display_name: &str) -> String { + display_name + .split_whitespace() + .map(str::to_lowercase) + .collect::>() + .join("-") +} + +/// The identifiers a check block plausibly EXPOSES, most specific first. +/// +/// The contract is "enable-able from what the listing shows", so the moment the +/// product publishes an identifier this must pick it up, or a correct fix would +/// stay xfailed forever (the row would never go stale). These are CANDIDATES, not +/// a verdict: the step tries each and counts the check reachable if any works, so +/// a wrong guess here costs nothing and a missing one is the only risk. +/// - a detail line keyed `id:` / `identifier:` / `slug:`, or any key ending in +/// ` id` (`command id: server-recover`) +/// - bracketed on the header — `Server recovery [server-recover]` +/// - a single parenthesised token beside the label — `Server recovery (server-recover)` +/// +/// Empty only when the block names no identifier at all, which is today's defect. +fn exposed_identifiers(block: &[&str]) -> Vec { + let mut found = Vec::new(); + let mut add = |id: &str| { + let id = id.trim().trim_matches(|c| c == '"' || c == '`'); + if !id.is_empty() && !id.contains(char::is_whitespace) && !found.iter().any(|f| f == id) { + found.push(id.to_owned()); + } + }; + for line in block { + if let Some((key, rest)) = line.trim().split_once(':') { + let key = key.trim().to_ascii_lowercase(); + if matches!(key.as_str(), "id" | "identifier" | "slug") || key.ends_with(" id") { + add(rest); + } + } + } + if let Some(header) = block.first().map(|h| h.trim()) { + for (open, close) in [('[', ']'), ('(', ')')] { + if let (Some(o), Some(c)) = (header.rfind(open), header.rfind(close)) + && o < c + { + add(&header[o + 1..c]); + } + } + } + found +} + +/// The checks the listing advertises, as `(displayed name, identifier to try)`. +/// +/// A check is a two-space-indented header line carrying a parenthesised on/off +/// state — ` Server recovery (off)`. Its own detail lines (`setting:`, `does:`, +/// and potentially an `id:`) are indented further; the report's other sections +/// list events rather than checks. Each header plus the deeper-indented lines +/// under it forms one block, so an identifier the fix exposes on a detail line +/// is seen. The identifier to invoke is the one the block explicitly exposes, +/// falling back to the display-name slug only when it exposes none (today's bug). +fn listed_checks(listing: &str) -> Vec<(String, Vec)> { + let lines: Vec<&str> = listing.lines().collect(); + let is_header = |line: &str| line.starts_with(" ") && !line.starts_with(" "); + let mut checks = Vec::new(); + for (i, line) in lines.iter().enumerate() { + if !is_header(line) { + continue; + } + let Some(name) = line + .trim() + .strip_suffix("(on)") + .or_else(|| line.trim().strip_suffix("(off)")) + .map(str::trim) + .filter(|name| !name.is_empty()) + else { + continue; + }; + // The block is this header plus every following more-indented line, up + // to the next header or a less-indented line. + let mut block = vec![name]; + for detail in &lines[i + 1..] { + if detail.starts_with(" ") { + block.push(detail); + } else { + break; + } + } + // Exposed identifiers first, then the label exactly as listed (a fix may make + // `enable` accept it), then its slug — today's fallback. + let mut candidates = exposed_identifiers(&block); + for fallback in [name.to_owned(), slug_of(name)] { + if !candidates.contains(&fallback) { + candidates.push(fallback); + } + } + checks.push((name.to_owned(), candidates)); + } + checks +} + +// ── Given ────────────────────────────────────────────────────────── + /// Plant an autostart *claim* — the harness's own (live) pid and a fresh spawn /// time — WITHOUT planting the daemon runtime-state. Unlike `suppress_daemon_spawn` /// (which marks the daemon already *running*, so the CLI's pre-check short-circuits @@ -427,6 +540,17 @@ async fn pending_before_review_history(world: &mut E2eWorld) { } } +#[given("a machine with no background checks turned on")] +async fn no_background_checks(world: &mut E2eWorld) { + // The scenario's isolated config starts empty, so every check is already off + // and nothing needs planting for the precondition itself. What DOES need + // planting is the running-daemon marker, for the same reason the sibling + // scenarios plant it: `automations enable` would otherwise launch a detached + // `rocm daemon` that outlives the scenario and accumulates on a persistent + // runner. + suppress_daemon_spawn(world); +} + // ── When ─────────────────────────────────────────────────────────── #[when("the user enables an automation watcher in observe mode")] @@ -502,6 +626,16 @@ async fn show_old_pending(world: &mut E2eWorld) { proposal_action(world, OLD_PENDING_PROPOSAL, "show"); } +#[when("the user lists the background checks")] +async fn user_lists_checks(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["automations", "list"]); + assert_eq!( + rc, 0, + "listing the background checks failed (rc={rc}):\n{stdout}\n{stderr}" + ); + world.cli_output = Some(stdout); +} + // ── Then ─────────────────────────────────────────────────────────── #[then("the CLI still lists the record before the torn tail")] @@ -1170,6 +1304,56 @@ async fn still_one_stop_request_waiting(world: &mut E2eWorld) { ); } +#[then("every listed check can be turned on from what the listing shows")] +async fn assert_listed_checks_enableable(world: &mut E2eWorld) { + let listing = world + .cli_output + .clone() + .expect("the background checks were never listed"); + let checks = listed_checks(&listing); + // Guard the guard: a listing this step failed to parse would otherwise + // "prove" the contract by checking nothing at all. + assert!( + !checks.is_empty(), + "no background checks were found in the listing:\n{listing}" + ); + + let mut unreachable = Vec::new(); + for (name, candidates) in &checks { + let mut last = String::new(); + let mut reached = false; + for candidate in candidates { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["automations", "enable", candidate]); + if rc == 0 { + reached = true; + break; + } + last = format!( + "tried {candidate:?}: rc={rc} {}", + stderr + .trim() + .lines() + .next() + .unwrap_or_else(|| stdout.trim()) + ); + } + if !reached { + unreachable.push(format!( + "{name:?} → none of {} candidate(s) worked; last {last}", + candidates.len() + )); + } + } + assert!( + unreachable.is_empty(), + "the listing names {} background check(s) that cannot be turned on from what it \ + shows:\n {}\nfull listing:\n{listing}", + unreachable.len(), + unreachable.join("\n "), + ); +} + // ── Helpers ──────────────────────────────────────────────────────── fn record(world: &mut E2eWorld, stdout: String, stderr: String, rc: i32) { diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 3db8aaa31c..de8ebd5b15 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -30,12 +30,18 @@ fn planted_env_path(world: &E2eWorld) -> PathBuf { root.path().join("engine-env") } -#[given("a machine with an installed engine environment")] -async fn plant_engine_env(world: &mut E2eWorld) { - // Plant the manifest `rocm engines install` would have written, plus a stub - // interpreter, so the scenario reaches the shell-launch path without a - // multi-GiB engine install or a GPU. Black-box: plain JSON matching the - // CLI's on-disk schema, not a typed import from the product crates. +/// Plant an environment tree with an interpreter and an activation script, plus a +/// manifest naming them, so a scenario reaches the shell-launch path with no +/// multi-GiB engine install and no GPU. Black-box: plain JSON matching the CLI's on-disk schema, not a +/// typed import from the product crates. +/// +/// `recorded_env_path` is the env_path the shell is handed, which is NOT always +/// the environment root: the vLLM install RESPONSE reports the directory holding +/// the engine command (`command.parent()` in `engines/vllm/src/install.rs`). +/// No install writes this manifest to disk; it +/// stands in for that response so the shell is reachable without an install. +/// Callers choose the shape they need. +fn plant_engine_env_recorded_as(world: &E2eWorld, recorded_env_path: &std::path::Path) { let root = world.isolated_root.as_ref().expect("no isolated root"); let env_path = planted_env_path(world); let bin = env_path.join("bin"); @@ -47,6 +53,11 @@ async fn plant_engine_env(world: &mut E2eWorld) { "#!/bin/sh\necho planted-engine-python\n", ) .expect("failed to write the planted interpreter"); + // A real environment ships the activation script next to its interpreter. + // Planting it is what makes "the hint names a file that exists" a statement + // about the CLI rather than about how thorough this fixture was. + std::fs::write(bin.join("activate"), "# planted engine env activation\n") + .expect("failed to write the planted activation script"); let manifests = root .path() @@ -59,7 +70,7 @@ async fn plant_engine_env(world: &mut E2eWorld) { "env_id": ENV_ID, "runtime_id": "therock-release:gfx94X-dcgpu", "python_executable": python.display().to_string(), - "env_path": env_path.display().to_string(), + "env_path": recorded_env_path.display().to_string(), }); std::fs::write( manifests.join(format!("{ENV_ID}.json")), @@ -68,6 +79,20 @@ async fn plant_engine_env(world: &mut E2eWorld) { .expect("failed to write the engine env manifest"); } +#[given("a machine with an installed engine environment")] +async fn plant_engine_env(world: &mut E2eWorld) { + let env_path = planted_env_path(world); + plant_engine_env_recorded_as(world, &env_path); +} + +#[given("a machine with an engine environment installed the way the engine records it")] +async fn plant_engine_env_as_installed(world: &mut E2eWorld) { + // Same tree, but env_path is what the vLLM install response reports: the + // directory the engine command lives in, one level inside the environment. + let bin = planted_env_path(world).join("bin"); + plant_engine_env_recorded_as(world, &bin); +} + #[when("the user opens a shell for that engine")] async fn open_engine_shell(world: &mut E2eWorld) { // `--shell` is pinned rather than inherited: the runner's own $SHELL decides @@ -115,26 +140,92 @@ async fn assert_prompt_marked(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("the marker never reached a prompt line: {e}")); } +/// What the planted interpreter prints when it is run (see +/// `plant_engine_env_recorded_as`). The shell banner never prints it, so seeing it +/// on screen proves the interpreter actually executed. +const PLANTED_INTERPRETER_OUTPUT: &str = "planted-engine-python"; + #[then("the engine environment's interpreter is the one that runs")] async fn assert_engine_interpreter(world: &mut E2eWorld) { - // Guards against the prompt shim breaking the activation it decorates: the - // shim re-runs the user's startup files, which could otherwise reorder PATH. - let expected = planted_env_path(world).join("bin").join("python"); + // Guards against the prompt shim breaking the activation it decorates, and + // against a wrong PATH or VIRTUAL_ENV: the shim re-runs the user's startup + // files, which could otherwise reorder PATH. + // + // This RUNS `python` and waits for the planted interpreter's own output. Do not + // wait for the interpreter's path instead: `engine_shell` prints + // `python: ` in its banner before bash starts, so a path wait is + // satisfied by the banner whatever `command -v` would say. + // + // `` on Unix. + let path = hint.strip_prefix("source ").unwrap_or(hint).trim(); + assert!( + !path.is_empty(), + "the activation hint named no path:\n{screen}" + ); + assert!( + std::path::Path::new(path).exists(), + "the engine shell told the user to activate {path:?}, which does not exist:\n{screen}" + ); +} + #[when("the user leaves the engine shell")] async fn leave_engine_shell(world: &mut E2eWorld) { let session = world.tui.as_mut().expect("no engine shell session"); diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 8e74839df3..5226db9987 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -378,6 +378,39 @@ impl TuiSession { self.screen_snapshot().0 } + /// The visible screen as LOGICAL lines: rows the terminal soft-wrapped are + /// rejoined into the single line the application actually emitted. + /// + /// [`Self::screen_text`] is the right lens for "what does the user see", and + /// most assertions look for a short marker where wrapping is irrelevant. A + /// step that must read a whole VALUE off the screen — a filesystem path, a + /// URL — needs this instead: at 80 columns a long path silently splits + /// across two rows, and a step parsing the raw rows would then assert + /// against a truncated value and fail for a reason that has nothing to do + /// with the product. + pub fn screen_logical_lines(&self) -> Vec { + let parser = self + .parser + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let screen = parser.screen(); + let (rows, cols) = screen.size(); + let mut lines: Vec = Vec::new(); + let mut continuing = false; + for row in 0..rows { + let text = screen.contents_between(row, 0, row, cols); + if continuing { + if let Some(last) = lines.last_mut() { + last.push_str(&text); + } + } else { + lines.push(text); + } + continuing = screen.row_wrapped(row); + } + lines + } + /// Whether the top-left cell carries the dimmed-backdrop wash a popup /// overlay paints behind itself (`grey_overlay`'s fixed RGB(0x1c, 0x1e, /// 0x22)). Popups drawn via `centered_rect` always leave a pad outside