From 81137182deb051134aeb47ccf8b562c3cecf2ab1 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 20 Aug 2026 11:54:47 +0200 Subject: [PATCH 01/11] test(e2e): pin the second README-walkthrough batch (EAI-8046..8049) Add one behavioural scenario per defect, each asserting the CORRECT behaviour and registered as an xfail in expectations.toml citing its EAI id, so the mock lane stays green now and the row goes stale (XPASS) the day the bug is fixed. No product behaviour changed. - EAI-8046: `rocm chat` ignores a piped prompt (reads no stdin, sends no request, exits 0 with a status summary). New run_rocm_with_stdin helper since Command::output gives the child a null stdin; the load-bearing assertion is that the mock received the prompt, not stdout. - EAI-8047: automations list publishes no usable identifier, so the checks it lists cannot be enabled from what it shows. The scenario derives each id from the listing rather than hard-coding the real ids. - EAI-8048: `dash --replay ` opens the interactive view instead of refusing; the step imposes its own deadline so the known hang xfails promptly rather than burning the CI budget. - EAI-8049: the engine shell's activation hint names a path that does not exist, because the recorded env_path already includes the interpreter directory. Phrased as path-exists so the row goes stale on any fix. All four re-confirmed in a plain Linux container against this branch's binary before writing the scenarios. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 54 +++++++ .../e2e-cucumber/features/automations.feature | 13 ++ tests/e2e-cucumber/features/chat.feature | 8 ++ tests/e2e-cucumber/features/dash.feature | 13 ++ .../features/engine_shell.feature | 18 +++ tests/e2e-cucumber/tests/e2e.rs | 40 ++++++ .../tests/e2e/automations_steps.rs | 133 ++++++++++++++++++ tests/e2e-cucumber/tests/e2e/chat_steps.rs | 40 ++++++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 33 +++++ tests/e2e-cucumber/tests/e2e/engines_steps.rs | 71 +++++++++- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 63 +++++++-- 11 files changed, 470 insertions(+), 16 deletions(-) create mode 100644 tests/e2e-cucumber/features/automations.feature create mode 100644 tests/e2e-cucumber/tests/e2e/automations_steps.rs diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 26864cb886..3f7224357c 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -169,3 +169,57 @@ bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." serve_timeout_secs = 90 flaky = true + +# ── Second README-walkthrough batch (EAI-8050) ───────────────────────────── +# Four defects found walking the CLI on an MI300X app-dev pod, each pinned by a +# scenario asserting the CORRECT behaviour. All four are in code paths with no +# hardware, engine or network dependency, and all four were re-confirmed in a +# plain Linux container against this branch's binary — so every row is scoped +# `when = {}` and each is expected to go stale (XPASS) on every lane the day its +# bug is fixed. None is `flaky`: each reproduced deterministically. + +# --- EAI-8046: `rocm chat` ignores a piped prompt --- +# With no `--prompt` and a non-terminal stdin, the command renders an assistant +# status summary and exits 0 without ever reading stdin or contacting the model, +# though `--prompt`'s own help says it reads standard input when omitted and the +# command's EXAMPLES block shows `echo … | rocm chat`. Container-measured: the +# `--prompt` control produced a POST on the served model, the piped run produced +# only the readiness GET. Remove this row once a piped prompt reaches the model. +[["chat-cli-stdin-prompt"]] +when = {} +bug = "EAI-8046" +reason = "`rocm chat` with a piped prompt and no --prompt never sends a request; it prints an assistant status summary and exits 0." + +# --- EAI-8047: listed automation checks cannot be turned on from the listing --- +# `automations list` shows each check by display name only ("Server recovery"), +# while `automations enable` accepts the internal id ("server-recover") and +# rejects anything else with rc=1. The names do not mechanically reduce to the +# ids, so nothing the listing prints is enough to act on. Remove this row once +# the listing exposes an identifier a user can pass back in. +[["automations-listed-checks-can-be-enabled"]] +when = {} +bug = "EAI-8047" +reason = "`automations list` publishes no usable identifier, so the checks it lists cannot be enabled from what it shows." + +# --- EAI-8048: `dash --replay` does not validate its recording --- +# Pointed at a file that does not exist, the dashboard enters its alternate +# screen and waits for input instead of refusing. Container-measured under a +# pseudo-terminal: no output, killed at a 12s timeout (rc=124). The scenario's +# step imposes its own deadline, so this xfails promptly rather than hanging. +# Remove this row once an unreadable recording is rejected before the TUI opens. +[["dash-replay-rejects-an-unreadable-session"]] +when = {} +bug = "EAI-8048" +reason = "`rocm dash --replay` opens the interactive dashboard on a recording that cannot be read instead of refusing it." + +# --- EAI-8049: the engine shell's activation hint names a path that is not there --- +# `engines shell` builds the hint by appending the interpreter directory to the +# manifest's env_path, but the vLLM install records env_path as the directory +# CONTAINING the command, so the hint doubles that segment and points at a file +# nobody can source. Container-measured: with a vLLM-shaped manifest the hint +# named `/bin/bin/activate` (absent) while a root-shaped one named +# `/bin/activate` (present). Remove this row once the two agree. +[["engine-shell-activation-hint-is-usable"]] +when = {} +bug = "EAI-8049" +reason = "`engines shell` prints an activation hint whose path does not exist, because the recorded env_path already includes the interpreter directory." diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature new file mode 100644 index 0000000000..e03bed3512 --- /dev/null +++ b/tests/e2e-cucumber/features/automations.feature @@ -0,0 +1,13 @@ +Feature: Background automation checks + + # A pure contract scenario: no fixtures, no GPU, no network. The listing is the + # only place a user learns which background checks exist, so whatever it shows + # has to be enough to act on. Deriving each check's identifier from the listing + # the way a reader would IS the contract under test — the identifiers are + # deliberately not written into the scenario or the steps, because hard-coding + # them would pass on a listing that exposes nothing. + @id:automations-listed-checks-can-be-enabled + Scenario: 1 - Every background check that is listed can be turned on + Given a machine with no background checks turned on + When the user lists the background checks + Then every listed check can be turned on by name diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index d0cc26099e..47566df4bb 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -75,3 +75,11 @@ Feature: Chat and endpoint detection And the model is registered with the CLI When the user sends a one-shot chat prompt through the CLI Then the CLI prints the assistant's reply + + @id:chat-cli-stdin-prompt + Scenario: 8 - The chat CLI answers a prompt received through standard input + Given a model is being served + And the model is registered with the CLI + When the user pipes a chat prompt to the CLI + Then the CLI sends the piped prompt to the model + And the CLI prints the assistant's reply diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index f1fa35e4f1..3d222814a2 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -110,3 +110,16 @@ Feature: Interactive dashboard Then generation throughput is no longer displayed When the user quits the dashboard Then the dashboard exits successfully + + # Driven under a pseudo-terminal like the rest of this feature: the replay path + # opens the same interactive event loop, and a piped run is refused before it + # gets there (it fails on the terminal, which proves nothing about the file). + # + # The step imposes its own deadline and treats expiry as a FAILURE rather than + # waiting for the scenario timeout. Today the dashboard takes an unreadable + # session as far as the alternate screen and then sits waiting for input, so a + # plain "assert a non-zero exit" step would hang for as long as CI allowed. + @id:dash-replay-rejects-an-unreadable-session @requires-os:linux + Scenario: 10 - Replaying a session that cannot be read is refused promptly + When the user replays a session recording that does not exist + Then the dashboard refuses to start without opening the interactive view diff --git a/tests/e2e-cucumber/features/engine_shell.feature b/tests/e2e-cucumber/features/engine_shell.feature index 3712d41957..ac864152f5 100644 --- a/tests/e2e-cucumber/features/engine_shell.feature +++ b/tests/e2e-cucumber/features/engine_shell.feature @@ -22,3 +22,21 @@ Feature: Engine shell And the engine environment's interpreter is the one that runs When the user leaves the engine shell Then the engine shell exits successfully + + # The engine environment here is planted in the shape the vLLM install actually + # records — its manifest names the directory holding the engine command, not the + # environment root (engines/vllm/src/lib.rs derives it as `command.parent()`). + # That distinction is the whole scenario: the shell composes its activation hint + # by appending the interpreter directory to whatever the manifest recorded, so a + # manifest that already points inside it yields a path nobody can source. + # + # Phrased as "the file it names is there", not as "the path is not doubled": the + # first survives whichever way the mismatch is repaired, the second would go + # stale the moment it is. + @id:engine-shell-activation-hint-is-usable @requires-os:linux + Scenario: 2 - The activation hint an engine shell prints refers to a real file + Given a machine with an engine environment installed the way the engine records it + When the user opens a shell for that engine + Then the printed activation hint names a file that exists + When the user leaves the engine shell + Then the engine shell exits successfully diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index e93d4e5aa0..a0daa07c1a 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -15,6 +15,7 @@ use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_ use tempfile::TempDir; mod e2e { + pub mod automations_steps; pub mod bench_steps; pub mod chat_steps; pub mod dash_steps; @@ -522,6 +523,45 @@ pub fn run_rocm_with_env( ) } +/// Like [`run_rocm`], but writes `stdin` to the child's standard input. +/// +/// [`run_rocm`] uses `Command::output`, which gives the child a NULL stdin, so no +/// scenario could exercise the documented `echo … | rocm chat` path. The pipe is +/// closed once the input is written, exactly as a shell pipeline does, so a +/// command that reads to EOF terminates instead of waiting for more. +pub fn run_rocm_with_stdin(world: &E2eWorld, args: &[&str], stdin: &str) -> (String, String, i32) { + use std::io::Write as _; + + let binary = rocm_binary(); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + cmd.stdin(std::process::Stdio::piped()); + cmd.stdout(std::process::Stdio::piped()); + cmd.stderr(std::process::Stdio::piped()); + let mut child = cmd + .spawn() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + // Take the handle so it is dropped (closing the pipe) as soon as the write + // finishes; otherwise the child would block reading a pipe we still hold. + { + let mut pipe = child.stdin.take().expect("stdin was piped"); + pipe.write_all(stdin.as_bytes()) + .unwrap_or_else(|e| panic!("failed to write to {binary}'s stdin: {e}")); + } + let output = child + .wait_with_output() + .unwrap_or_else(|e| panic!("failed to wait for {binary}: {e}")); + let rc = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + ( + stdout, + String::from_utf8_lossy(&output.stderr).to_string(), + rc, + ) +} + /// Append one `rocm` invocation to `results/commands.jsonl` so the consolidated /// report can build a command × platform coverage table tied to real results. /// Best-effort: a recording failure must never fail a scenario. diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs new file mode 100644 index 0000000000..56dcf80624 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -0,0 +1,133 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `rocm automations`. +//! +//! The scenario asserts a discoverability contract: everything the listing shows +//! a user must also be able to act on. So these steps deliberately derive each +//! check's identifier FROM the listing rather than knowing it in advance — +//! hard-coding the real ids would keep passing against a listing that publishes +//! none of them, which is the defect being pinned. + +use cucumber::{given, then, when}; +use serde_json::json; + +use crate::E2eWorld; + +/// Turn one line of the listing into the identifier a user would try. +/// +/// A reader has only the displayed name to go on, so this applies the obvious +/// reading of it: lowercase, words joined by dashes. That is the *charitable* +/// derivation — if a real identifier were printed the parse would pick it up +/// verbatim, and if a user could reasonably guess it this reproduces the guess. +/// Anything the listing never shows is, by definition, not derivable here. +fn identifier_from(listed_name: &str) -> String { + listed_name + .split_whitespace() + .map(str::to_lowercase) + .collect::>() + .join("-") +} + +/// The checks the listing advertises, as `(displayed name, derived identifier)`. +/// +/// A check is a two-space-indented line carrying a parenthesised on/off state — +/// ` Server recovery (off)`. The surrounding report lines are either less +/// indented (the header block) or more (a check's own `setting:` / `does:` +/// detail), and the report's other sections list events rather than checks, so +/// the state suffix is what distinguishes a check from anything else. +fn listed_checks(listing: &str) -> Vec<(String, String)> { + listing + .lines() + .filter(|line| line.starts_with(" ") && !line.starts_with(" ")) + .filter_map(|line| { + let line = line.trim(); + let name = line + .strip_suffix("(on)") + .or_else(|| line.strip_suffix("(off)"))? + .trim(); + (!name.is_empty()).then(|| (name.to_owned(), identifier_from(name))) + }) + .collect() +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given("a machine with no background checks turned on")] +async fn no_background_checks(world: &mut E2eWorld) { + // The scenario's isolated config starts empty, so every check is already off + // and nothing needs planting for the precondition itself. + // + // What DOES need planting is a background helper: `automations enable` + // launches a detached `rocm daemon` unless the recorded one is alive, and + // that child would outlive the scenario, survive its temp dir, and pile up on + // a persistent runner. Recording this test process as the running helper + // makes the liveness check find one and skip the spawn — the same trick, for + // the same reason, as pointing a planted service record's pids at this + // process (see `E2eWorld::register_mock_service_with`). Black-box: plain JSON + // matching the CLI's on-disk schema, not a typed import from the product. + let root = world.isolated_root.as_ref().expect("no isolated root"); + let automations = root.path().join("data").join("automations"); + std::fs::create_dir_all(&automations).expect("failed to create the automations dir"); + let state = json!({ + "running": true, + "automations_enabled": false, + "daemon_pid": std::process::id(), + "started_at_unix_ms": 1_700_000_000_000_u64, + "last_tick_unix_ms": 1_700_000_000_000_u64, + "active_watchers": [], + }); + std::fs::write( + automations.join("runtime-state.json"), + serde_json::to_vec_pretty(&state).expect("runtime state serialises"), + ) + .expect("failed to write the automation runtime state"); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user lists the background checks")] +async fn user_lists_checks(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["automations", "list"]); + assert_eq!( + rc, 0, + "listing the background checks failed (rc={rc}):\n{stdout}\n{stderr}" + ); + world.cli_output = Some(stdout); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("every listed check can be turned on by name")] +async fn assert_listed_checks_enableable(world: &mut E2eWorld) { + let listing = world + .cli_output + .clone() + .expect("the background checks were never listed"); + let checks = listed_checks(&listing); + // Guard the guard: a listing this step failed to parse would otherwise + // "prove" the contract by checking nothing at all. + assert!( + !checks.is_empty(), + "no background checks were found in the listing:\n{listing}" + ); + + let mut unreachable = Vec::new(); + for (name, identifier) in &checks { + let (stdout, stderr, rc) = crate::run_rocm(world, &["automations", "enable", identifier]); + if rc != 0 { + unreachable.push(format!( + "{name:?} → tried {identifier:?}: rc={rc} {}", + stderr.trim().lines().next().unwrap_or(stdout.trim()) + )); + } + } + assert!( + unreachable.is_empty(), + "the listing names {} background check(s) that cannot be turned on from what it \ + shows:\n {}\nfull listing:\n{listing}", + unreachable.len(), + unreachable.join("\n "), + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index 600a1dd7d3..d31819ed08 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: MIT +use std::time::Duration; + use cucumber::{given, then, when}; use e2e_cucumber::mock_server::MockServer; @@ -80,6 +82,24 @@ async fn user_sends_oneshot_chat(world: &mut E2eWorld) { world.cli_output = Some(stdout); } +#[when("the user pipes a chat prompt to the CLI")] +async fn user_pipes_chat_prompt(world: &mut E2eWorld) { + // Keep --model explicit: this scenario pins the stdin contract, not the + // default-model discovery defect covered separately by EAI-8009. + let model = world.model_name.clone().expect("no model name set"); + let (stdout, stderr, rc) = crate::run_rocm_with_stdin( + world, + &["chat", "--provider", "local", "--model", &model], + "Hello from standard input\n", + ); + assert_eq!( + rc, 0, + "piped rocm chat failed (rc={rc}):\n{stdout}\n{stderr}" + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); +} + // ── Then ─────────────────────────────────────────────────────────── #[then("the served model is listed")] @@ -117,6 +137,26 @@ async fn assert_chat_successful(world: &mut E2eWorld) { ); } +#[then("the CLI sends the piped prompt to the model")] +async fn assert_piped_prompt_sent(world: &mut E2eWorld) { + // This is the load-bearing assertion. Today the command still exits 0 and + // prints an assistant status summary, but the mock receives no completion + // at all. Waiting for the actual request prevents any future incidental + // output from making the scenario pass for the wrong reason. + let request = world + .mock + .as_ref() + .expect("no mock server running") + .wait_for_chat_request(Duration::from_secs(2)) + .await + .unwrap_or_else(|e| panic!("the piped prompt never reached the model: {e}")); + let content = request["messages"][0]["content"].as_str().unwrap_or(""); + assert_eq!( + content, "Hello from standard input", + "the model received the wrong prompt: {request}" + ); +} + #[then("the CLI prints the assistant's reply")] async fn assert_cli_prints_reply(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no chat CLI output"); diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index e12fe3962a..7534eb29e6 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -80,6 +80,21 @@ async fn running_managed_model(world: &mut E2eWorld) { // ── When ─────────────────────────────────────────────────────────── +#[when("the user replays a session recording that does not exist")] +async fn replay_missing_session(world: &mut E2eWorld) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let missing = root.path().join("missing-session.ndjson"); + assert!( + !missing.exists(), + "the replay fixture unexpectedly exists: {}", + missing.display() + ); + let path = missing.to_string_lossy().into_owned(); + let tui = TuiSession::spawn(world, &["dash", "--replay", &path]) + .unwrap_or_else(|e| panic!("failed to launch the replay command: {e}")); + world.tui = Some(tui); +} + #[when("the user opens the dashboard with demo data")] async fn open_dashboard_demo(world: &mut E2eWorld) { // `--demo` replays a deterministic synthetic session, so the dashboard @@ -220,6 +235,24 @@ async fn quit_interactive_chat(world: &mut E2eWorld) { // ── Then ─────────────────────────────────────────────────────────── +#[then("the dashboard refuses to start without opening the interactive view")] +async fn assert_replay_refused_promptly(world: &mut E2eWorld) { + let tui = session(world); + let rc = tui + .wait_for_any_exit(Duration::from_secs(2)) + .await + .unwrap_or_else(|e| { + panic!( + "the dashboard did not reject the unreadable replay promptly; it appears to have \ + opened the interactive view instead: {e}" + ) + }); + assert_ne!( + rc, 0, + "the dashboard accepted a replay recording that does not exist" + ); +} + #[then("the dashboard home view is displayed")] async fn home_view_displayed(world: &mut E2eWorld) { let tui = session(world); diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 504d6afb52..1866eaa5bd 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -30,12 +30,17 @@ fn planted_env_path(world: &E2eWorld) -> PathBuf { root.path().join("engine-env") } -#[given("a machine with an installed engine environment")] -async fn plant_engine_env(world: &mut E2eWorld) { - // Plant the manifest `rocm engines install` would have written, plus a stub - // interpreter, so the scenario reaches the shell-launch path without a - // multi-GiB engine install or a GPU. Black-box: plain JSON matching the - // CLI's on-disk schema, not a typed import from the product crates. +/// Plant the files `rocm engines install` leaves behind — an environment tree +/// with an interpreter and an activation script, plus the manifest naming them — +/// so a scenario reaches the shell-launch path with no multi-GiB engine install +/// and no GPU. Black-box: plain JSON matching the CLI's on-disk schema, not a +/// typed import from the product crates. +/// +/// `recorded_env_path` is what the manifest claims the environment is, which is +/// NOT always the environment root: the vLLM install records the directory +/// holding the engine command instead (`command.parent()` in +/// `engines/vllm/src/lib.rs`). Callers choose the shape they need. +fn plant_engine_env_recorded_as(world: &E2eWorld, recorded_env_path: &std::path::Path) { let root = world.isolated_root.as_ref().expect("no isolated root"); let env_path = planted_env_path(world); let bin = env_path.join("bin"); @@ -50,6 +55,11 @@ async fn plant_engine_env(world: &mut E2eWorld) { std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o755)) .expect("failed to mark the planted interpreter executable"); } + // A real environment ships the activation script next to its interpreter. + // Planting it is what makes "the hint names a file that exists" a statement + // about the CLI rather than about how thorough this fixture was. + std::fs::write(bin.join("activate"), "# planted engine env activation\n") + .expect("failed to write the planted activation script"); let manifests = root .path() @@ -62,7 +72,7 @@ async fn plant_engine_env(world: &mut E2eWorld) { "env_id": ENV_ID, "runtime_id": "therock-release:gfx94X-dcgpu", "python_executable": python.display().to_string(), - "env_path": env_path.display().to_string(), + "env_path": recorded_env_path.display().to_string(), }); std::fs::write( manifests.join(format!("{ENV_ID}.json")), @@ -71,6 +81,20 @@ async fn plant_engine_env(world: &mut E2eWorld) { .expect("failed to write the engine env manifest"); } +#[given("a machine with an installed engine environment")] +async fn plant_engine_env(world: &mut E2eWorld) { + let env_path = planted_env_path(world); + plant_engine_env_recorded_as(world, &env_path); +} + +#[given("a machine with an engine environment installed the way the engine records it")] +async fn plant_engine_env_as_installed(world: &mut E2eWorld) { + // Same tree, but the manifest records what the vLLM install records: the + // directory the engine command lives in, one level inside the environment. + let bin = planted_env_path(world).join("bin"); + plant_engine_env_recorded_as(world, &bin); +} + #[when("the user opens a shell for that engine")] async fn open_engine_shell(world: &mut E2eWorld) { // `--shell` is pinned rather than inherited: the runner's own $SHELL decides @@ -134,6 +158,39 @@ async fn assert_engine_interpreter(world: &mut E2eWorld) { }); } +#[then("the printed activation hint names a file that exists")] +async fn assert_activation_hint_resolves(world: &mut E2eWorld) { + const HINT_LABEL: &str = "activate_hint:"; + + let session = world.tui.as_mut().expect("no engine shell session"); + session + .wait_for_screen(HINT_LABEL, SCREEN_TIMEOUT) + .await + .unwrap_or_else(|e| panic!("the engine shell printed no activation hint: {e}")); + + // Read LOGICAL lines: the hint carries a path long enough to soft-wrap at the + // terminal width, and a truncated path would fail the assertion below for a + // reason that has nothing to do with the product. + let lines = session.screen_logical_lines(); + let screen = session.screen_text(); + let hint = lines + .iter() + .find_map(|line| line.split_once(HINT_LABEL)) + .map(|(_, rest)| rest.trim()) + .unwrap_or_else(|| panic!("no activation hint on screen:\n{screen}")); + // The hint is the command a user would run, which on Unix is `source `. + // Take the path it points at, whichever form it came in. + let path = hint.strip_prefix("source ").unwrap_or(hint).trim(); + assert!( + !path.is_empty(), + "the activation hint named no path:\n{screen}" + ); + assert!( + std::path::Path::new(path).exists(), + "the engine shell told the user to activate {path:?}, which does not exist:\n{screen}" + ); +} + #[when("the user leaves the engine shell")] async fn leave_engine_shell(world: &mut E2eWorld) { let session = world.tui.as_mut().expect("no engine shell session"); diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 9996dba4d4..0fc1bb3bd0 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -215,6 +215,39 @@ impl TuiSession { self.screen_snapshot().0 } + /// The visible screen as LOGICAL lines: rows the terminal soft-wrapped are + /// rejoined into the single line the application actually emitted. + /// + /// [`Self::screen_text`] is the right lens for "what does the user see", and + /// most assertions look for a short marker where wrapping is irrelevant. A + /// step that must read a whole VALUE off the screen — a filesystem path, a + /// URL — needs this instead: at 80 columns a long path silently splits + /// across two rows, and a step parsing the raw rows would then assert + /// against a truncated value and fail for a reason that has nothing to do + /// with the product. + pub fn screen_logical_lines(&self) -> Vec { + let parser = self + .parser + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let screen = parser.screen(); + let (rows, cols) = screen.size(); + let mut lines: Vec = Vec::new(); + let mut continuing = false; + for row in 0..rows { + let text = screen.contents_between(row, 0, row, cols); + if continuing { + if let Some(last) = lines.last_mut() { + last.push_str(&text); + } + } else { + lines.push(text); + } + continuing = screen.row_wrapped(row); + } + lines + } + fn screen_snapshot(&self) -> (String, (u16, u16)) { // Recover a poisoned lock rather than defaulting to a blank screen: the // parser's data is still valid even if some other thread panicked while @@ -364,20 +397,32 @@ impl TuiSession { /// Poll until the child exits, asserting a successful (zero) exit code. pub async fn wait_for_exit(&mut self, timeout: Duration) -> Result<(), String> { + let status = self.wait_for_any_exit(timeout).await?; + if status == 0 { + Ok(()) + } else { + Err(format!( + "TUI exited unsuccessfully (exit code {status}).\n{}", + self.framed_screen() + )) + } + } + + /// Poll until the child exits, returning its exit code without requiring zero. + /// + /// Most journeys require a clean exit and use [`Self::wait_for_exit`]. A + /// command-rejection scenario needs the inverse contract: it must terminate + /// promptly and non-zero. Keeping the deadline/reap/record logic here avoids + /// teaching individual steps how to manipulate the PTY child directly. + pub async fn wait_for_any_exit(&mut self, timeout: Duration) -> Result { let deadline = Instant::now() + timeout; loop { match self.child.try_wait() { Ok(Some(status)) => { self.finished = true; - self.record_once(i32::try_from(status.exit_code()).unwrap_or(-1)); - return if status.success() { - Ok(()) - } else { - Err(format!( - "TUI exited unsuccessfully ({status:?}).\n{}", - self.framed_screen() - )) - }; + let rc = i32::try_from(status.exit_code()).unwrap_or(-1); + self.record_once(rc); + return Ok(rc); } Ok(None) => {} Err(e) => return Err(format!("failed to poll TUI child: {e}")), From 9edd21a8952eef4fe1d84d7e3fb227da98bdfebc Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 20 Aug 2026 13:04:34 +0200 Subject: [PATCH 02/11] test(e2e): assert the durable contract, not each bug's current shape Address pre-PR review: each of the three step assertions pinned the bug's present symptom rather than the behaviour that must hold, so a valid fix could stay xfailed and the row would never go stale. - EAI-8047: parse the identifier the listing explicitly exposes (inline [id]/(id) on the header, or an `id:`/`identifier:` detail line), keeping the display-name slug only as the current-broken-output fallback. A fix that publishes the id now flips the row to XPASS. - EAI-8048: also assert the interactive view never opened, not just a prompt non-zero exit. Adds a latched entered_alternate_screen() to the PTY driver so an "open the TUI, then exit non-zero" impl no longer satisfies the contract. - EAI-8046: assert the user-role message content rather than messages[0], so a fix that prepends a system message still XPASSes. Mock lane unchanged: still 6 xfail / 0 XPASS; each scenario still fails on its assertion step alone. Signed-off-by: fredespi --- .../tests/e2e/automations_steps.rs | 112 ++++++++++++++---- tests/e2e-cucumber/tests/e2e/chat_steps.rs | 18 ++- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 9 ++ tests/e2e-cucumber/tests/e2e/tui_driver.rs | 24 ++++ 4 files changed, 135 insertions(+), 28 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 56dcf80624..adcbe627d2 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -15,41 +15,101 @@ use serde_json::json; use crate::E2eWorld; -/// Turn one line of the listing into the identifier a user would try. +/// Slugify a display name: lowercase, words joined by dashes. /// -/// A reader has only the displayed name to go on, so this applies the obvious -/// reading of it: lowercase, words joined by dashes. That is the *charitable* -/// derivation — if a real identifier were printed the parse would pick it up -/// verbatim, and if a user could reasonably guess it this reproduces the guess. -/// Anything the listing never shows is, by definition, not derivable here. -fn identifier_from(listed_name: &str) -> String { - listed_name +/// This is the guess a user is forced into when the listing exposes no real +/// identifier — the current-bug fallback, not the contract. It deliberately does +/// NOT recover the true ids ("Server recovery" → `server-recovery`, not the real +/// `server-recover`), which is exactly why the listing has to publish them. +fn slug_of(display_name: &str) -> String { + display_name .split_whitespace() .map(str::to_lowercase) .collect::>() .join("-") } -/// The checks the listing advertises, as `(displayed name, derived identifier)`. +/// The identifier a check block explicitly EXPOSES, if any. /// -/// A check is a two-space-indented line carrying a parenthesised on/off state — -/// ` Server recovery (off)`. The surrounding report lines are either less -/// indented (the header block) or more (a check's own `setting:` / `does:` -/// detail), and the report's other sections list events rather than checks, so -/// the state suffix is what distinguishes a check from anything else. +/// The contract is "enable-able from what the listing shows", so the moment the +/// product publishes a real identifier — inline on the header (`Server recovery +/// [server-recover]` or `... (server-recover)`) or on an indented detail line +/// (`id: server-recover`) — this must pick THAT up, or a correct fix would stay +/// xfailed forever (the row would never go stale). Returns `None` only when the +/// block names no identifier at all, which is today's defect. +fn exposed_identifier(block: &[&str]) -> Option { + // A detail line that names the id outright, in the obvious shapes a fix + // might use: "id: server-recover", "identifier = server-recover". + for line in block { + let line = line.trim(); + for key in ["id:", "id =", "identifier:", "identifier ="] { + if let Some(rest) = line.strip_prefix(key) { + let id = rest.trim().trim_matches(|c| c == '"' || c == '`'); + if !id.is_empty() { + return Some(id.to_owned()); + } + } + } + } + // An id printed inline on the header, in brackets or parens after the name: + // "Server recovery [server-recover]". The state suffix "(on)"/"(off)" has + // already been stripped from `header` before this is called. + let header = block.first()?.trim(); + for (open, close) in [('[', ']'), ('(', ')')] { + if let (Some(o), Some(c)) = (header.rfind(open), header.rfind(close)) { + if o < c { + let inner = header[o + 1..c].trim(); + // A single token with no spaces is an identifier; a phrase is + // still part of the display name, not an id. + if !inner.is_empty() && !inner.contains(char::is_whitespace) { + return Some(inner.to_owned()); + } + } + } + } + None +} + +/// The checks the listing advertises, as `(displayed name, identifier to try)`. +/// +/// A check is a two-space-indented header line carrying a parenthesised on/off +/// state — ` Server recovery (off)`. Its own detail lines (`setting:`, `does:`, +/// and potentially an `id:`) are indented further; the report's other sections +/// list events rather than checks. Each header plus the deeper-indented lines +/// under it forms one block, so an identifier the fix exposes on a detail line +/// is seen. The identifier to invoke is the one the block explicitly exposes, +/// falling back to the display-name slug only when it exposes none (today's bug). fn listed_checks(listing: &str) -> Vec<(String, String)> { - listing - .lines() - .filter(|line| line.starts_with(" ") && !line.starts_with(" ")) - .filter_map(|line| { - let line = line.trim(); - let name = line - .strip_suffix("(on)") - .or_else(|| line.strip_suffix("(off)"))? - .trim(); - (!name.is_empty()).then(|| (name.to_owned(), identifier_from(name))) - }) - .collect() + let lines: Vec<&str> = listing.lines().collect(); + let is_header = |line: &str| line.starts_with(" ") && !line.starts_with(" "); + let mut checks = Vec::new(); + for (i, line) in lines.iter().enumerate() { + if !is_header(line) { + continue; + } + let Some(name) = line + .trim() + .strip_suffix("(on)") + .or_else(|| line.trim().strip_suffix("(off)")) + .map(str::trim) + .filter(|name| !name.is_empty()) + else { + continue; + }; + // The block is this header plus every following more-indented line, up + // to the next header or a less-indented line. + let mut block = vec![name]; + for detail in &lines[i + 1..] { + if detail.starts_with(" ") { + block.push(detail); + } else { + break; + } + } + let identifier = exposed_identifier(&block).unwrap_or_else(|| slug_of(name)); + checks.push((name.to_owned(), identifier)); + } + checks } // ── Given ────────────────────────────────────────────────────────── diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index d31819ed08..73c8be1d82 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -150,9 +150,23 @@ async fn assert_piped_prompt_sent(world: &mut E2eWorld) { .wait_for_chat_request(Duration::from_secs(2)) .await .unwrap_or_else(|e| panic!("the piped prompt never reached the model: {e}")); - let content = request["messages"][0]["content"].as_str().unwrap_or(""); + // Find the user-role message rather than assuming it is index 0: a correct + // fix is free to prepend a system prompt, and pinning `messages[0]` would + // keep this xfailed against exactly that valid behaviour. Same shape as the + // managed-chat assertion in dash_steps.rs. + let messages = request + .get("messages") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("chat request had no messages array:\n{request}")); + let user_content = messages + .iter() + .rev() + .find(|m| m.get("role").and_then(serde_json::Value::as_str) == Some("user")) + .and_then(|m| m.get("content")) + .and_then(serde_json::Value::as_str) + .unwrap_or_else(|| panic!("no user message in chat request:\n{request}")); assert_eq!( - content, "Hello from standard input", + user_content, "Hello from standard input", "the model received the wrong prompt: {request}" ); } diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 78ab796244..3102d3d8e8 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -255,6 +255,15 @@ async fn assert_replay_refused_promptly(world: &mut E2eWorld) { opened the interactive view instead: {e}" ) }); + // Both halves of the contract. A prompt non-zero exit alone would be + // satisfied by an implementation that opens the full-screen TUI, then + // notices the unreadable file and exits — which still violates "instead of + // entering the TUI". Assert it never switched to the alternate screen too; + // that latched flag survives the switch-back an exiting TUI would perform. + assert!( + !tui.entered_alternate_screen(), + "the dashboard opened the interactive view before rejecting the unreadable replay" + ); assert_ne!( rc, 0, "the dashboard accepted a replay recording that does not exist" diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 88f60d9f51..866c29d76b 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -94,6 +94,12 @@ pub struct TuiSession { /// terminal failure state after the one-time diagnostic is consumed, so a /// retry cannot lose the cause while the reader thread is still finishing. reader_failure: Arc, + /// Latches `true` the first time the child switches the terminal to its + /// alternate screen (the full-screen TUI buffer). Set by the reader thread + /// after each parse and never cleared, so it survives the exit sequence that + /// switches back — a plain `screen().alternate_screen()` snapshot taken after + /// the process exits would read `false` and miss that the TUI ever opened. + entered_alternate_screen: Arc, /// Kept alive for the lifetime of the session: the reader/writer are cloned /// from it, and dropping it early would close the PTY. master: Box, @@ -204,11 +210,13 @@ impl TuiSession { let parser = Arc::new(Mutex::new(vt100::Parser::new(ROWS, COLS, 0))); let reader_stop = Arc::new(AtomicBool::new(false)); let reader_failure = Arc::new(ReaderFailure::default()); + let entered_alternate_screen = Arc::new(AtomicBool::new(false)); let reader = spawn_reader( reader, Arc::clone(&parser), Arc::clone(&reader_stop), Arc::clone(&reader_failure), + Arc::clone(&entered_alternate_screen), ); Ok(Self { @@ -218,6 +226,7 @@ impl TuiSession { reader_stop, reader: Some(reader), reader_failure, + entered_alternate_screen, master: pair.master, finished: false, recorded: false, @@ -232,6 +241,14 @@ impl TuiSession { self.finished } + /// Whether the child ever switched the terminal to its alternate screen — + /// i.e. opened the full-screen interactive view. Latched, so it stays true + /// after the child switches back on exit. Lets a "must be refused before the + /// TUI opens" contract be asserted, not just "exited non-zero". + pub fn entered_alternate_screen(&self) -> bool { + self.entered_alternate_screen.load(Ordering::Relaxed) + } + /// The current visible screen as plain text (one row per line). `vt100` has /// already resolved escape sequences and styling into cells, so this is /// exactly what a user sees — color/attribute independent. @@ -624,6 +641,7 @@ fn spawn_reader( parser: Arc>, stop: Arc, reader_failure: Arc, + entered_alternate_screen: Arc, ) -> JoinHandle<()> { std::thread::spawn(move || { let mut buf = [0u8; 8192]; @@ -640,6 +658,12 @@ fn spawn_reader( let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { p.process(&buf[..n]); })); + // Latch alternate-screen use while the parser reflects THIS + // chunk: the switch back on exit would otherwise erase the + // evidence that the full-screen TUI ever opened. + if result.is_ok() && p.screen().alternate_screen() { + entered_alternate_screen.store(true, Ordering::Relaxed); + } drop(p); if let Err(payload) = result { let message = panic_message(&payload); From dd733fea957f5b163f4f453314f3eec25e76da2a Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 20 Aug 2026 14:17:10 +0200 Subject: [PATCH 03/11] test(e2e): make the alternate-screen latch race-free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 review: the latch checked the parser's alternate_screen() state only after processing a whole PTY read, so a chunk carrying both the enter (?1049h) and leave (?1049l) sequences left it false; and wait_for_any_exit returned before the reader drained, so a still-buffered enter could be missed. Either produced a false negative that would let a fixed dashboard open the TUI and still pass the "must not open the interactive view" half. - Detect the ?1049h enter sequence in the RAW byte stream, with a one-short-of-marker carry-over so a boundary-split sequence is still found. Independent of what follows it in the same read. - Drain the reader (bounded by DRAIN_TIMEOUT) after the child exits, before the caller reads the latch — same post-exit drain wait_for_screen uses. Mock lane unchanged: 6 xfail / 0 XPASS. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 61 +++++++++++++++++++--- 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 866c29d76b..24dcb7c4b7 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -533,6 +533,13 @@ impl TuiSession { self.finished = true; let rc = i32::try_from(status.exit_code()).unwrap_or(-1); self.record_once(rc); + // Let the reader consume the child's final bytes before the + // caller inspects post-exit state (e.g. entered_alternate_ + // screen). The child exiting closes the slave, so the reader + // sees EOF and finishes; without this wait, an enter sequence + // still buffered in the pipe would be missed. Bounded so a + // misbehaving PTY can't stall the scenario. + self.drain_reader().await; return Ok(rc); } Ok(None) => {} @@ -557,6 +564,25 @@ impl TuiSession { } } + /// Wait, bounded by [`DRAIN_TIMEOUT`], for the reader thread to finish + /// consuming the child's final output after it exits. The reader sees EOF + /// once the exited child's slave closes, so this normally returns well + /// inside the bound; the cap only guards a misbehaving PTY. Idempotent and + /// safe to call once per exit. + async fn drain_reader(&self) { + let drain_deadline = Instant::now() + DRAIN_TIMEOUT; + while Instant::now() < drain_deadline { + if self + .reader + .as_ref() + .is_none_or(std::thread::JoinHandle::is_finished) + { + return; + } + tokio::time::sleep(POLL_INTERVAL).await; + } + } + /// Record this invocation once for the command-coverage report (so `rocm /// dash` / `rocm chat` count as covered), tied to the scenario for the /// pass/fail join. Best-effort and idempotent. @@ -643,8 +669,20 @@ fn spawn_reader( reader_failure: Arc, entered_alternate_screen: Arc, ) -> JoinHandle<()> { + // The escape sequence a terminal emits to switch to the alternate screen. + // Detected in the raw byte stream rather than via the parser's post-process + // `alternate_screen()` state: a single read can contain both the enter and + // the matching leave (`\x1b[?1049l`), which would leave the parser back on + // the primary screen and the evidence erased. Scanning the bytes catches the + // enter regardless of what follows it in the same chunk. + const ALT_SCREEN_ENTER: &[u8] = b"\x1b[?1049h"; + std::thread::spawn(move || { let mut buf = [0u8; 8192]; + // Carry the last few bytes of each read into the next scan so an enter + // sequence split across a read boundary is still found. One byte short + // of the marker length is all that can overlap. + let mut carry: Vec = Vec::new(); loop { if stop.load(Ordering::Relaxed) { break; @@ -652,18 +690,27 @@ fn spawn_reader( match reader.read(&mut buf) { Ok(0) => break, Ok(n) => { + let chunk = &buf[..n]; + if !entered_alternate_screen.load(Ordering::Relaxed) { + let mut window = std::mem::take(&mut carry); + window.extend_from_slice(chunk); + if window + .windows(ALT_SCREEN_ENTER.len()) + .any(|w| w == ALT_SCREEN_ENTER) + { + entered_alternate_screen.store(true, Ordering::Relaxed); + } + // Keep just enough tail to bridge a boundary-split marker. + let keep = ALT_SCREEN_ENTER.len().saturating_sub(1); + let start = window.len().saturating_sub(keep); + carry = window[start..].to_vec(); + } let mut p = parser .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - p.process(&buf[..n]); + p.process(chunk); })); - // Latch alternate-screen use while the parser reflects THIS - // chunk: the switch back on exit would otherwise erase the - // evidence that the full-screen TUI ever opened. - if result.is_ok() && p.screen().alternate_screen() { - entered_alternate_screen.store(true, Ordering::Relaxed); - } drop(p); if let Err(payload) = result { let message = panic_message(&payload); From 43f1fb37ace0b803a9e996b1afeac00055125628 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 20 Aug 2026 15:38:26 +0200 Subject: [PATCH 04/11] test(e2e): satisfy the e2e-harness clippy lints The CI Clippy (e2e harness + steps) job runs clippy -p e2e-cucumber --test e2e -D warnings, which the local container gate did not (its clippy step excludes e2e-cucumber). Four lints surfaced: - future-not-send: drain_reader took &self, and a shared &TuiSession held across .await is not Send (the struct holds !Sync PTY handles), poisoning every step awaiting it. Take &mut self like the sibling waits. - collapsible-if, or-fun-call (automations_steps) and map-unwrap-or (engines_steps): mechanical rewrites, no behaviour change. Verified with the exact CI commands in a Linux container: both clippy invocations clean, e2e mock lane unchanged (6 xfail / 0 XPASS). Signed-off-by: fredespi --- .../tests/e2e/automations_steps.rs | 22 +++++++++++-------- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 6 +++-- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 17 ++++++++++---- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index adcbe627d2..6d9dd7f0bc 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -56,14 +56,14 @@ fn exposed_identifier(block: &[&str]) -> Option { // already been stripped from `header` before this is called. let header = block.first()?.trim(); for (open, close) in [('[', ']'), ('(', ')')] { - if let (Some(o), Some(c)) = (header.rfind(open), header.rfind(close)) { - if o < c { - let inner = header[o + 1..c].trim(); - // A single token with no spaces is an identifier; a phrase is - // still part of the display name, not an id. - if !inner.is_empty() && !inner.contains(char::is_whitespace) { - return Some(inner.to_owned()); - } + if let (Some(o), Some(c)) = (header.rfind(open), header.rfind(close)) + && o < c + { + let inner = header[o + 1..c].trim(); + // A single token with no spaces is an identifier; a phrase is + // still part of the display name, not an id. + if !inner.is_empty() && !inner.contains(char::is_whitespace) { + return Some(inner.to_owned()); } } } @@ -179,7 +179,11 @@ async fn assert_listed_checks_enableable(world: &mut E2eWorld) { if rc != 0 { unreachable.push(format!( "{name:?} → tried {identifier:?}: rc={rc} {}", - stderr.trim().lines().next().unwrap_or(stdout.trim()) + stderr + .trim() + .lines() + .next() + .unwrap_or_else(|| stdout.trim()) )); } } diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 1866eaa5bd..eef2e12262 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -176,8 +176,10 @@ async fn assert_activation_hint_resolves(world: &mut E2eWorld) { let hint = lines .iter() .find_map(|line| line.split_once(HINT_LABEL)) - .map(|(_, rest)| rest.trim()) - .unwrap_or_else(|| panic!("no activation hint on screen:\n{screen}")); + .map_or_else( + || panic!("no activation hint on screen:\n{screen}"), + |(_, rest)| rest.trim(), + ); // The hint is the command a user would run, which on Unix is `source `. // Take the path it points at, whichever form it came in. let path = hint.strip_prefix("source ").unwrap_or(hint).trim(); diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 24dcb7c4b7..43f66ac18c 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -569,14 +569,23 @@ impl TuiSession { /// once the exited child's slave closes, so this normally returns well /// inside the bound; the cap only guards a misbehaving PTY. Idempotent and /// safe to call once per exit. - async fn drain_reader(&self) { + /// + /// Takes `&mut self` even though it only reads: a shared `&TuiSession` held + /// across an `.await` is not `Send` (the struct holds `!Sync` PTY handles), + /// which `-D clippy::future-not-send` rejects and would propagate to every + /// awaiting step. `&mut TuiSession` is `Send`, matching the sibling waits. + async fn drain_reader(&mut self) { let drain_deadline = Instant::now() + DRAIN_TIMEOUT; while Instant::now() < drain_deadline { - if self + // Compute the check and drop every borrow of `self` BEFORE the await: + // a reference held across `.await` would make this future `!Send`, + // which `-D clippy::future-not-send` rejects (and which propagates to + // every step that awaits it). + let finished = self .reader .as_ref() - .is_none_or(std::thread::JoinHandle::is_finished) - { + .is_none_or(std::thread::JoinHandle::is_finished); + if finished { return; } tokio::time::sleep(POLL_INTERVAL).await; From 1452ee174a87cf05693284b32a8b16fce0cc094a Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 25 Aug 2026 18:39:59 +0200 Subject: [PATCH 05/11] test(e2e): drop the duplicate run_rocm_with_stdin after absorbing main Upstream (EAI-8072) independently added a run_rocm_with_stdin helper that also takes an env slice, and the merge left both definitions in e2e.rs (E0428), with my chat step calling the 3-arg form (E0061). Keep upstream's more general helper and delete mine; the chat stdin step now passes an empty env slice. Behaviour is identical - both pipe stdin and close the handle before waiting, so a command reading to EOF still terminates. Verified with the exact CI commands in a Linux container: both clippy invocations clean, e2e mock lane 6 xfail / 0 XPASS. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e.rs | 39 ---------------------- tests/e2e-cucumber/tests/e2e/chat_steps.rs | 1 + 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 06e7f21a20..ec3be17f12 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -604,45 +604,6 @@ pub fn run_rocm_with_env( ) } -/// Like [`run_rocm`], but writes `stdin` to the child's standard input. -/// -/// [`run_rocm`] uses `Command::output`, which gives the child a NULL stdin, so no -/// scenario could exercise the documented `echo … | rocm chat` path. The pipe is -/// closed once the input is written, exactly as a shell pipeline does, so a -/// command that reads to EOF terminates instead of waiting for more. -pub fn run_rocm_with_stdin(world: &E2eWorld, args: &[&str], stdin: &str) -> (String, String, i32) { - use std::io::Write as _; - - let binary = rocm_binary(); - let mut cmd = std::process::Command::new(&binary); - cmd.args(args); - world.isolate_cmd(&mut cmd); - cmd.stdin(std::process::Stdio::piped()); - cmd.stdout(std::process::Stdio::piped()); - cmd.stderr(std::process::Stdio::piped()); - let mut child = cmd - .spawn() - .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); - // Take the handle so it is dropped (closing the pipe) as soon as the write - // finishes; otherwise the child would block reading a pipe we still hold. - { - let mut pipe = child.stdin.take().expect("stdin was piped"); - pipe.write_all(stdin.as_bytes()) - .unwrap_or_else(|e| panic!("failed to write to {binary}'s stdin: {e}")); - } - let output = child - .wait_with_output() - .unwrap_or_else(|e| panic!("failed to wait for {binary}: {e}")); - let rc = output.status.code().unwrap_or(-1); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - record_command(world.current_scenario.as_deref(), args, rc, &stdout); - ( - stdout, - String::from_utf8_lossy(&output.stderr).to_string(), - rc, - ) -} - /// Append one `rocm` invocation to `results/commands.jsonl` so the consolidated /// report can build a command × platform coverage table tied to real results. /// Best-effort: a recording failure must never fail a scenario. diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index 73c8be1d82..e53e70e9e2 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -91,6 +91,7 @@ async fn user_pipes_chat_prompt(world: &mut E2eWorld) { world, &["chat", "--provider", "local", "--model", &model], "Hello from standard input\n", + &[], ); assert_eq!( rc, 0, From 3490bf6f3551722a3f00f907b986cd074753be2c Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 10 Sep 2026 08:40:22 +0200 Subject: [PATCH 06/11] test(e2e): harden the pinned assertions against valid fixes (PR review) Addresses the non-blocking review points on #292. Each one was a way a CORRECT fix could leave the expectation row xfailed instead of going stale, which is the failure mode these scenarios exist to avoid. - chat: compare the user message trimmed. The step pipes a trailing newline, so forwarding stdin verbatim -- a perfectly good fix -- would otherwise not XPASS. - automations: recognise the identifier in the shapes a fix might publish it -- a detail line (id/identifier/slug/key/name), bracketed, parenthesised, dash-separated, or as a trailing column. A parenthesised or dash/column token must look like an identifier (lowercase with a separator, as every real watcher id is), so a display-name qualifier such as "Server recovery (beta)" is not mistaken for one. The slug-of-display-name fallback is unchanged, so today's listing still fails the contract. - engine shell: accept ". " and a quoted path as well as "source ", so the existence check fails only when the CLI really named a missing file. - dash: replace the bare 2s replay deadline with a documented REPLAY_REJECTION_TIMEOUT. Still bounded (today's behaviour is a hang, and that bound is paid every run until it is fixed), but with enough headroom that a cold start on a loaded runner will not flake once the row is deleted. Signed-off-by: fredespi --- .../tests/e2e/automations_steps.rs | 82 ++++++++++++++++--- tests/e2e-cucumber/tests/e2e/chat_steps.rs | 7 +- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 11 ++- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 13 ++- 4 files changed, 95 insertions(+), 18 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index adc19c2aa0..5bd1f84a77 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -67,20 +67,51 @@ fn slug_of(display_name: &str) -> String { .join("-") } +/// Whether `token` looks like a watcher identifier rather than display text. +/// +/// Every real id is lowercase with a `-`/`_` separator (`server-recover`, +/// `therock-update`, `gpu-metrics`). Requiring that shape is what lets the +/// ambiguous forms below — a parenthesised token, a trailing column, a +/// dash-separated tail — be read as an id without mistaking a display-name +/// qualifier ("Server recovery (beta)") for one. +fn is_identifier_shaped(token: &str) -> bool { + !token.is_empty() + && token.contains(['-', '_']) + && token + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') +} + /// The identifier a check block explicitly EXPOSES, if any. /// /// The contract is "enable-able from what the listing shows", so the moment the -/// product publishes a real identifier — inline on the header (`Server recovery -/// [server-recover]` or `... (server-recover)`) or on an indented detail line -/// (`id: server-recover`) — this must pick THAT up, or a correct fix would stay -/// xfailed forever (the row would never go stale). Returns `None` only when the -/// block names no identifier at all, which is today's defect. +/// product publishes a real identifier this must pick THAT up, or a correct fix +/// would stay xfailed forever (the row would never go stale). Recognises the +/// shapes a fix might plausibly use: +/// - a detail line — `id: server-recover`, `slug = server-recover` +/// - bracketed on the header — `Server recovery [server-recover]` +/// - parenthesised on the header — `Server recovery (server-recover)` +/// - dash-separated on the header — `Server recovery — server-recover` +/// - a trailing column — `Server recovery server-recover` +/// +/// Returns `None` only when the block names no identifier at all, which is +/// today's defect. fn exposed_identifier(block: &[&str]) -> Option { - // A detail line that names the id outright, in the obvious shapes a fix - // might use: "id: server-recover", "identifier = server-recover". + // A detail line that names the id outright. for line in block { let line = line.trim(); - for key in ["id:", "id =", "identifier:", "identifier ="] { + for key in [ + "id:", + "id =", + "identifier:", + "identifier =", + "slug:", + "slug =", + "key:", + "key =", + "name:", + "name =", + ] { if let Some(rest) = line.strip_prefix(key) { let id = rest.trim().trim_matches(|c| c == '"' || c == '`'); if !id.is_empty() { @@ -89,22 +120,47 @@ fn exposed_identifier(block: &[&str]) -> Option { } } } - // An id printed inline on the header, in brackets or parens after the name: - // "Server recovery [server-recover]". The state suffix "(on)"/"(off)" has + // An id printed on the header itself. The state suffix "(on)"/"(off)" has // already been stripped from `header` before this is called. let header = block.first()?.trim(); + + // Bracketed or parenthesised: "Server recovery [server-recover]". for (open, close) in [('[', ']'), ('(', ')')] { if let (Some(o), Some(c)) = (header.rfind(open), header.rfind(close)) && o < c { let inner = header[o + 1..c].trim(); - // A single token with no spaces is an identifier; a phrase is - // still part of the display name, not an id. - if !inner.is_empty() && !inner.contains(char::is_whitespace) { + // Brackets are unambiguous — nothing but an id is written that way. + // Parentheses are not, so those must look like an identifier. + if open == '[' && !inner.is_empty() && !inner.contains(char::is_whitespace) { + return Some(inner.to_owned()); + } + if is_identifier_shaped(inner) { return Some(inner.to_owned()); } } } + + // Dash-separated tail: "Server recovery — server-recover" (em/en dash, or a + // spaced ASCII hyphen). Only the last segment can be the id. + for sep in [" — ", " – ", " - "] { + if let Some((_, tail)) = header.rsplit_once(sep) { + let tail = tail.trim(); + if is_identifier_shaped(tail) { + return Some(tail.to_owned()); + } + } + } + + // A trailing column: "Server recovery server-recover", i.e. the id set + // off by column padding rather than punctuation. + if let Some((_, tail)) = header.rsplit_once(" ") { + let tail = tail.trim(); + if is_identifier_shaped(tail) { + return Some(tail.to_owned()); + } + } + None } diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index e53e70e9e2..ef2376d11a 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -166,8 +166,13 @@ async fn assert_piped_prompt_sent(world: &mut E2eWorld) { .and_then(|m| m.get("content")) .and_then(serde_json::Value::as_str) .unwrap_or_else(|| panic!("no user message in chat request:\n{request}")); + // Compare trimmed: the step pipes a trailing newline (as a shell pipeline + // does), and forwarding stdin verbatim is a perfectly correct fix. Demanding + // the exact untrimmed string would keep this row xfailed against exactly + // that behaviour — the stale-row failure mode this file sets out to avoid. assert_eq!( - user_content, "Hello from standard input", + user_content.trim(), + "Hello from standard input", "the model received the wrong prompt: {request}" ); } diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 9addacfaf7..a31d778232 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -19,6 +19,15 @@ use crate::e2e::tui_driver::{TuiSession, default_timeout}; /// the mock actually received — so the two can never silently drift apart. const MANAGED_MODEL_PROMPT: &str = "hello from the terminal"; +/// How long a rejected `dash --replay` gets to exit before the step calls it a +/// hang. Deliberately its own budget rather than `default_timeout()`: today the +/// dashboard hangs on an unreadable recording, so this bound is paid on every +/// mock-lane run until that is fixed, and 30s per run is real CI time for a +/// known-red scenario. It is still far more headroom than a cold binary start +/// needs on a loaded shared runner, so once the bug is fixed and the xfail row +/// is deleted the scenario does not become a flake. +const REPLAY_REJECTION_TIMEOUT: Duration = Duration::from_secs(10); + /// Borrow the scenario's active TUI session, or fail clearly if none was opened. const fn session(world: &mut E2eWorld) -> &mut TuiSession { world @@ -261,7 +270,7 @@ async fn quit_launcher(world: &mut E2eWorld) { async fn assert_replay_refused_promptly(world: &mut E2eWorld) { let tui = session(world); let rc = tui - .wait_for_any_exit(Duration::from_secs(2)) + .wait_for_any_exit(REPLAY_REJECTION_TIMEOUT) .await .unwrap_or_else(|e| { panic!( diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index eef2e12262..2fdf94453b 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -180,9 +180,16 @@ async fn assert_activation_hint_resolves(world: &mut E2eWorld) { || panic!("no activation hint on screen:\n{screen}"), |(_, rest)| rest.trim(), ); - // The hint is the command a user would run, which on Unix is `source `. - // Take the path it points at, whichever form it came in. - let path = hint.strip_prefix("source ").unwrap_or(hint).trim(); + // The hint is the command a user would run, which on Unix is `source ` + // — but `. ` is the same instruction, and a path with spaces would be + // quoted. Strip whichever wrapper it came in so the existence check below + // fails only when the CLI named a path that is genuinely not there. + let path = hint + .strip_prefix("source ") + .or_else(|| hint.strip_prefix(". ")) + .unwrap_or(hint) + .trim() + .trim_matches(|c| c == '"' || c == '\''); assert!( !path.is_empty(), "the activation hint named no path:\n{screen}" From 2c4b600292cc09533510193ab3079b51e12c324f Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 10 Sep 2026 09:32:11 +0200 Subject: [PATCH 07/11] test(e2e): tighten the pinned assertions against wrong-reason passes (PR review) Second round of pr-review-watcher notes, all about a row staying xfailed or failing for the wrong reason after a valid fix: - chat-09 now waits for the request that carries the piped prompt rather than for whichever chat request lands first, so a future readiness probe on this path cannot satisfy the wait. Its bound is named and the reason for keeping it short is recorded at the constant. - automations-04 sweeps the unambiguous identifier keys (id/identifier/ slug) across the whole block before the ambiguous ones (key/name), which must additionally look like an identifier, so a listing that prints a display name above its id no longer yields the name. - The engine-shell expectation reason now states that the fixture pins the manifest shape an install already recorded, so the row clears on the hint composition rather than on what a fresh install writes. Verified in the Linux container: 6 xfail, 0 XPASS, 0 stale. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 2 +- .../tests/e2e/automations_steps.rs | 48 ++++++++------ tests/e2e-cucumber/tests/e2e/chat_steps.rs | 63 ++++++++++++++----- 3 files changed, 77 insertions(+), 36 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 88b23dd5d6..78ed9b9492 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -261,4 +261,4 @@ reason = "`rocm dash --replay` opens the interactive dashboard on a recording th [["engine-shell-activation-hint-is-usable"]] when = {} bug = "EAI-8049" -reason = "`engines shell` prints an activation hint whose path does not exist, because the recorded env_path already includes the interpreter directory." +reason = "`engines shell` prints an activation hint whose path does not exist, because the recorded env_path already includes the interpreter directory. The scenario plants a manifest in that recorded shape, so it clears only when the hint is composed correctly for an already-recorded env_path — changing what a fresh install writes does not on its own repair the environments already on disk." diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 5bd1f84a77..79a260e4ea 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -74,6 +74,13 @@ fn slug_of(display_name: &str) -> String { /// ambiguous forms below — a parenthesised token, a trailing column, a /// dash-separated tail — be read as an id without mistaking a display-name /// qualifier ("Server recovery (beta)") for one. +/// +/// A separator-less id would not match here, which is deliberate: in those +/// ambiguous positions a bare lowercase word is indistinguishable from display +/// text, and accepting it is the false positive this exists to prevent. Such an +/// id is still recognised through the shapes that need no disambiguation — a +/// bracketed header token, or an `id`/`identifier`/`slug` detail line — so the +/// requirement narrows the guess, never the contract. fn is_identifier_shaped(token: &str) -> bool { !token.is_empty() && token.contains(['-', '_']) @@ -97,26 +104,31 @@ fn is_identifier_shaped(token: &str) -> bool { /// Returns `None` only when the block names no identifier at all, which is /// today's defect. fn exposed_identifier(block: &[&str]) -> Option { - // A detail line that names the id outright. - for line in block { - let line = line.trim(); - for key in [ - "id:", - "id =", - "identifier:", - "identifier =", - "slug:", - "slug =", - "key:", - "key =", - "name:", - "name =", - ] { - if let Some(rest) = line.strip_prefix(key) { + // A detail line that names the id outright. Keys are tried in two tiers, + // each swept across the WHOLE block before the next: `id`/`identifier`/ + // `slug` mean nothing but an identifier, so they win wherever they appear, + // while `key`/`name` are ambiguous and must additionally look like an id. + // Tiering rather than taking the first matching line is what stops a fix + // that prints `name: Server recovery` above its `id:` line from yielding + // the display name. + for (keys, must_look_like_an_id) in [ + (["id", "identifier", "slug"].as_slice(), false), + (["key", "name"].as_slice(), true), + ] { + for line in block { + let line = line.trim(); + for key in keys { + let Some(rest) = line + .strip_prefix(&format!("{key}:")) + .or_else(|| line.strip_prefix(&format!("{key} ="))) + else { + continue; + }; let id = rest.trim().trim_matches(|c| c == '"' || c == '`'); - if !id.is_empty() { - return Some(id.to_owned()); + if id.is_empty() || (must_look_like_an_id && !is_identifier_shaped(id)) { + continue; } + return Some(id.to_owned()); } } } diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index ef2376d11a..e2b1a79b55 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -9,6 +9,22 @@ use e2e_cucumber::mock_server::MockServer; use crate::E2eWorld; +/// The prompt `chat-09` pipes on standard input. Shared by the `When` that +/// writes it and the `Then` that identifies the request carrying it. +const PIPED_PROMPT: &str = "Hello from standard input"; + +/// How long `chat-09` waits for the piped prompt to reach the mock. +/// +/// Deliberately short, and deliberately not `default_timeout()`. The CLI has +/// already run to completion by the time the assertion runs — the mock records +/// a request before the CLI can receive its response — so nothing can arrive +/// later than the process exit that already happened. There is no cold start to +/// absorb here, which is what the 30s waits elsewhere exist for; those poll a +/// live TUI. Until the stdin defect is fixed no request comes at all, so this +/// budget is spent in full on every mock-lane run, and a larger one would buy +/// no additional safety. +const PIPED_PROMPT_TIMEOUT: Duration = Duration::from_secs(2); + // ── Given ────────────────────────────────────────────────────────── #[given("a model is being served")] @@ -90,7 +106,7 @@ async fn user_pipes_chat_prompt(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm_with_stdin( world, &["chat", "--provider", "local", "--model", &model], - "Hello from standard input\n", + &format!("{PIPED_PROMPT}\n"), &[], ); assert_eq!( @@ -138,33 +154,46 @@ async fn assert_chat_successful(world: &mut E2eWorld) { ); } +/// The newest user-role message in a chat request body. +/// +/// Reads the LAST user turn rather than assuming index 0: a correct fix is free +/// to prepend a system prompt, and pinning `messages[0]` would keep this row +/// xfailed against exactly that valid behaviour. Same shape as the managed-chat +/// assertion in `dash_steps.rs`. +fn last_user_message(body: &serde_json::Value) -> Option<&str> { + body.get("messages")? + .as_array()? + .iter() + .rev() + .find(|m| m.get("role").and_then(serde_json::Value::as_str) == Some("user"))? + .get("content")? + .as_str() +} + #[then("the CLI sends the piped prompt to the model")] async fn assert_piped_prompt_sent(world: &mut E2eWorld) { // This is the load-bearing assertion. Today the command still exits 0 and // prints an assistant status summary, but the mock receives no completion // at all. Waiting for the actual request prevents any future incidental // output from making the scenario pass for the wrong reason. + // + // Matched on content rather than taking whichever body landed first: only + // the newest request is retained, and a local endpoint check is itself a + // chat request ("Say ok."). `rocm chat` does not issue one on this path + // today, but if it ever did, waiting for any request at all would let this + // row fail for an unrelated reason and never go stale. Membership, not + // equality, so a prompt that arrives altered still reaches the assertion + // below and reports what was wrong with it. let request = world .mock .as_ref() .expect("no mock server running") - .wait_for_chat_request(Duration::from_secs(2)) + .wait_for_chat_request_where(PIPED_PROMPT_TIMEOUT, |body| { + last_user_message(body).is_some_and(|content| content.contains(PIPED_PROMPT)) + }) .await .unwrap_or_else(|e| panic!("the piped prompt never reached the model: {e}")); - // Find the user-role message rather than assuming it is index 0: a correct - // fix is free to prepend a system prompt, and pinning `messages[0]` would - // keep this xfailed against exactly that valid behaviour. Same shape as the - // managed-chat assertion in dash_steps.rs. - let messages = request - .get("messages") - .and_then(serde_json::Value::as_array) - .unwrap_or_else(|| panic!("chat request had no messages array:\n{request}")); - let user_content = messages - .iter() - .rev() - .find(|m| m.get("role").and_then(serde_json::Value::as_str) == Some("user")) - .and_then(|m| m.get("content")) - .and_then(serde_json::Value::as_str) + let user_content = last_user_message(&request) .unwrap_or_else(|| panic!("no user message in chat request:\n{request}")); // Compare trimmed: the step pipes a trailing newline (as a shell pipeline // does), and forwarding stdin verbatim is a perfectly correct fix. Demanding @@ -172,7 +201,7 @@ async fn assert_piped_prompt_sent(world: &mut E2eWorld) { // that behaviour — the stale-row failure mode this file sets out to avoid. assert_eq!( user_content.trim(), - "Hello from standard input", + PIPED_PROMPT, "the model received the wrong prompt: {request}" ); } From 57c52a840f12cfac94913779a1ce97c3c31a4f79 Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 23 Oct 2026 16:19:25 +0200 Subject: [PATCH 08/11] test(e2e): drop speculative output shapes from the pinned assertions exposed_identifier now recognises only an id:/identifier:/slug: detail line or a bracketed header token; the other shapes were guesses no listing prints and no test exercised. The enable step is reworded to say what it does (try the listed identifier, else the slug) rather than 'by name'. The engine-shell hint strip is reduced to the one form the CLI emits. The expectations header no longer cites a ticket nothing else references. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 2 +- .../e2e-cucumber/features/automations.feature | 2 +- .../tests/e2e/automations_steps.rs | 112 +++--------------- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 12 +- 4 files changed, 21 insertions(+), 107 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 5c24bd7b98..db905e5d71 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -181,7 +181,7 @@ when = { is_wsl = true, therock_family = "gfx*" } bug = "EAI-7998" reason = "`examine --json` reports has_amd_gpu:false on a WSL2 host whose own summary names a gfx target." -# -- Second README-walkthrough batch (EAI-8050) -- +# -- README-walkthrough defects (EAI-8047, EAI-8049) -- # Defects found walking the CLI on an MI300X app-dev pod, each pinned by a # scenario asserting the CORRECT behaviour. Both are in code paths with no # hardware, engine or network dependency, and both were re-confirmed in a plain diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature index 0646b3562d..87e09b5cff 100644 --- a/tests/e2e-cucumber/features/automations.feature +++ b/tests/e2e-cucumber/features/automations.feature @@ -66,4 +66,4 @@ Feature: Automation watchers Scenario: automations-06 - Every background check that is listed can be turned on Given a machine with no background checks turned on When the user lists the background checks - Then every listed check can be turned on by name + Then every listed check can be turned on from what the listing shows diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 81f8c064ef..5b32f00759 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -73,113 +73,35 @@ fn slug_of(display_name: &str) -> String { .join("-") } -/// Whether `token` looks like a watcher identifier rather than display text. -/// -/// Every real id is lowercase with a `-`/`_` separator (`server-recover`, -/// `therock-update`, `gpu-metrics`). Requiring that shape is what lets the -/// ambiguous forms below — a parenthesised token, a trailing column, a -/// dash-separated tail — be read as an id without mistaking a display-name -/// qualifier ("Server recovery (beta)") for one. -/// -/// A separator-less id would not match here, which is deliberate: in those -/// ambiguous positions a bare lowercase word is indistinguishable from display -/// text, and accepting it is the false positive this exists to prevent. Such an -/// id is still recognised through the shapes that need no disambiguation — a -/// bracketed header token, or an `id`/`identifier`/`slug` detail line — so the -/// requirement narrows the guess, never the contract. -fn is_identifier_shaped(token: &str) -> bool { - !token.is_empty() - && token.contains(['-', '_']) - && token - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') -} - /// The identifier a check block explicitly EXPOSES, if any. /// /// The contract is "enable-able from what the listing shows", so the moment the /// product publishes a real identifier this must pick THAT up, or a correct fix -/// would stay xfailed forever (the row would never go stale). Recognises the -/// shapes a fix might plausibly use: -/// - a detail line — `id: server-recover`, `slug = server-recover` +/// would stay xfailed forever (the row would never go stale). Two shapes are +/// recognised, both unambiguous — neither can be mistaken for display text: +/// - a detail line — `id: server-recover` (also `identifier:` / `slug:`) /// - bracketed on the header — `Server recovery [server-recover]` -/// - parenthesised on the header — `Server recovery (server-recover)` -/// - dash-separated on the header — `Server recovery — server-recover` -/// - a trailing column — `Server recovery server-recover` /// -/// Returns `None` only when the block names no identifier at all, which is -/// today's defect. +/// Any other shape a fix might pick leaves the row xfailed until this learns it; +/// that is the cost of not guessing at forms no listing prints today. Returns +/// `None` only when the block names no identifier at all, which is today's defect. fn exposed_identifier(block: &[&str]) -> Option { - // A detail line that names the id outright. Keys are tried in two tiers, - // each swept across the WHOLE block before the next: `id`/`identifier`/ - // `slug` mean nothing but an identifier, so they win wherever they appear, - // while `key`/`name` are ambiguous and must additionally look like an id. - // Tiering rather than taking the first matching line is what stops a fix - // that prints `name: Server recovery` above its `id:` line from yielding - // the display name. - for (keys, must_look_like_an_id) in [ - (["id", "identifier", "slug"].as_slice(), false), - (["key", "name"].as_slice(), true), - ] { - for line in block { - let line = line.trim(); - for key in keys { - let Some(rest) = line - .strip_prefix(&format!("{key}:")) - .or_else(|| line.strip_prefix(&format!("{key} ="))) - else { - continue; - }; + for line in block { + let line = line.trim(); + for key in ["id", "identifier", "slug"] { + if let Some(rest) = line.strip_prefix(&format!("{key}:")) { let id = rest.trim().trim_matches(|c| c == '"' || c == '`'); - if id.is_empty() || (must_look_like_an_id && !is_identifier_shaped(id)) { - continue; + if !id.is_empty() { + return Some(id.to_owned()); } - return Some(id.to_owned()); } } } - // An id printed on the header itself. The state suffix "(on)"/"(off)" has - // already been stripped from `header` before this is called. let header = block.first()?.trim(); - - // Bracketed or parenthesised: "Server recovery [server-recover]". - for (open, close) in [('[', ']'), ('(', ')')] { - if let (Some(o), Some(c)) = (header.rfind(open), header.rfind(close)) - && o < c - { - let inner = header[o + 1..c].trim(); - // Brackets are unambiguous — nothing but an id is written that way. - // Parentheses are not, so those must look like an identifier. - if open == '[' && !inner.is_empty() && !inner.contains(char::is_whitespace) { - return Some(inner.to_owned()); - } - if is_identifier_shaped(inner) { - return Some(inner.to_owned()); - } - } - } - - // Dash-separated tail: "Server recovery — server-recover" (em/en dash, or a - // spaced ASCII hyphen). Only the last segment can be the id. - for sep in [" — ", " – ", " - "] { - if let Some((_, tail)) = header.rsplit_once(sep) { - let tail = tail.trim(); - if is_identifier_shaped(tail) { - return Some(tail.to_owned()); - } - } - } - - // A trailing column: "Server recovery server-recover", i.e. the id set - // off by column padding rather than punctuation. - if let Some((_, tail)) = header.rsplit_once(" ") { - let tail = tail.trim(); - if is_identifier_shaped(tail) { - return Some(tail.to_owned()); - } - } - - None + let (open, close) = (header.rfind('[')?, header.rfind(']')?); + let inner = header.get(open + 1..close)?.trim(); + (open < close && !inner.is_empty() && !inner.contains(char::is_whitespace)) + .then(|| inner.to_owned()) } /// The checks the listing advertises, as `(displayed name, identifier to try)`. @@ -433,7 +355,7 @@ async fn refuse_unknown(world: &mut E2eWorld) { ); } -#[then("every listed check can be turned on by name")] +#[then("every listed check can be turned on from what the listing shows")] async fn assert_listed_checks_enableable(world: &mut E2eWorld) { let listing = world .cli_output diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index dbdb9b77a6..4cd91b2d58 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -182,16 +182,8 @@ async fn assert_activation_hint_resolves(world: &mut E2eWorld) { || panic!("no activation hint on screen:\n{screen}"), |(_, rest)| rest.trim(), ); - // The hint is the command a user would run, which on Unix is `source ` - // — but `. ` is the same instruction, and a path with spaces would be - // quoted. Strip whichever wrapper it came in so the existence check below - // fails only when the CLI named a path that is genuinely not there. - let path = hint - .strip_prefix("source ") - .or_else(|| hint.strip_prefix(". ")) - .unwrap_or(hint) - .trim() - .trim_matches(|c| c == '"' || c == '\''); + // `runtime_python_activation_hint` prints `source ` on Unix. + let path = hint.strip_prefix("source ").unwrap_or(hint).trim(); assert!( !path.is_empty(), "the activation hint named no path:\n{screen}" From 485d697494904ef5729e3d36da473de2112c6780 Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 26 Oct 2026 15:56:30 +0100 Subject: [PATCH 09/11] test(e2e): make engine-shell-02 flip only when the shell runs the engine's interpreter The doubled env_path also sets a wrong VIRTUAL_ENV and a nonexistent bin/bin on PATH, so a fix to the hint text alone would flip this row while the shell still runs the wrong python. Reuse the existing interpreter step so the row clears only once the user's actual problem is fixed. Signed-off-by: fredespi --- tests/e2e-cucumber/features/engine_shell.feature | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e-cucumber/features/engine_shell.feature b/tests/e2e-cucumber/features/engine_shell.feature index 16eed6d23e..50b3da084d 100644 --- a/tests/e2e-cucumber/features/engine_shell.feature +++ b/tests/e2e-cucumber/features/engine_shell.feature @@ -36,6 +36,11 @@ Feature: Engine shell # rewritten or deleted along with its expectation row. Only a fix to the hint # composition itself makes this row go stale. # + # The interpreter step is here because the same doubled path also sets a wrong + # VIRTUAL_ENV and puts a nonexistent bin/bin on PATH: a fix to the hint text + # alone would otherwise flip this row while the shell still runs the wrong + # python. The row flips only once the user's actual problem is fixed. + # # Phrased as "the file it names is there", not as "the path is not doubled": the # first survives whichever way the mismatch is repaired, the second would go # stale the moment it is. @@ -44,5 +49,6 @@ Feature: Engine shell Given a machine with an engine environment installed the way the engine records it When the user opens a shell for that engine Then the printed activation hint names a file that exists + And the engine environment's interpreter is the one that runs When the user leaves the engine shell Then the engine shell exits successfully From 20fcc4a4526239196323bdc78128a029ba246b94 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 27 Oct 2026 11:40:42 +0100 Subject: [PATCH 10/11] test(e2e): make the engine-shell interpreter step able to fail The step waited for the interpreter's path on screen, but engine_shell prints that path in its banner before bash starts, so the wait returned at once and a hint-only fix still flipped engine-shell-02. Run python and wait for the planted interpreter's own output, which the banner never prints. Verified by applying a hint-only fix to the product: the hint step passes and the interpreter step now fails, so the row stays xfailed. Also drop an inaccurate Lemonade contrast and an internal host name from the expectation comments. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 7 ++--- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 30 ++++++++++++------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index db905e5d71..1b52f90b3c 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -182,7 +182,7 @@ bug = "EAI-7998" reason = "`examine --json` reports has_amd_gpu:false on a WSL2 host whose own summary names a gfx target." # -- README-walkthrough defects (EAI-8047, EAI-8049) -- -# Defects found walking the CLI on an MI300X app-dev pod, each pinned by a +# Defects found walking the CLI on an MI300X host, each pinned by a # scenario asserting the CORRECT behaviour. Both are in code paths with no # hardware, engine or network dependency, and both were re-confirmed in a plain # Linux container — so each row is scoped `when = {}` and is expected to go stale @@ -205,9 +205,8 @@ reason = "`automations list` publishes no usable identifier, so the checks it li # resolved env_path. Users reach it through the auto-install branch of # `resolve_engine_env`, which uses the install response's env_path live, and the # vLLM install reports that as the directory CONTAINING the command -# (`command.parent()`, engines/vllm/src/install.rs) while Lemonade reports the -# root. The hint therefore doubles the segment and points at a file nobody can -# source. Container-measured: with a bin-shaped env_path the hint named +# (`command.parent()`, engines/vllm/src/install.rs). The hint therefore doubles +# the segment and points at a file nobody can source. Container-measured: with a bin-shaped env_path the hint named # `/bin/bin/activate` (absent) while a root-shaped one named # `/bin/activate` (present). # diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 4cd91b2d58..403e6a0a5e 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -37,8 +37,8 @@ fn planted_env_path(world: &E2eWorld) -> PathBuf { /// /// `recorded_env_path` is the env_path the shell is handed, which is NOT always /// the environment root: the vLLM install RESPONSE reports the directory holding -/// the engine command (`command.parent()` in `engines/vllm/src/install.rs`), -/// while Lemonade reports the root. No install writes this manifest to disk; it +/// the engine command (`command.parent()` in `engines/vllm/src/install.rs`). +/// No install writes this manifest to disk; it /// stands in for that response so the shell is reachable without an install. /// Callers choose the shape they need. fn plant_engine_env_recorded_as(world: &E2eWorld, recorded_env_path: &std::path::Path) { @@ -140,22 +140,32 @@ async fn assert_prompt_marked(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("the marker never reached a prompt line: {e}")); } +/// What the planted interpreter prints when it is run (see +/// `plant_engine_env_recorded_as`). The shell banner never prints it, so seeing it +/// on screen proves the interpreter actually executed. +const PLANTED_INTERPRETER_OUTPUT: &str = "planted-engine-python"; + #[then("the engine environment's interpreter is the one that runs")] async fn assert_engine_interpreter(world: &mut E2eWorld) { - // Guards against the prompt shim breaking the activation it decorates: the - // shim re-runs the user's startup files, which could otherwise reorder PATH. - let expected = planted_env_path(world).join("bin").join("python"); + // Guards against the prompt shim breaking the activation it decorates, and + // against a wrong PATH/VIRTUAL_ENV: the shim re-runs the user's startup files, + // which could otherwise reorder PATH. + // + // This RUNS `python` and waits for the planted interpreter's own output. Do not + // wait for the interpreter's path instead: `engine_shell` prints + // `python: ` in its banner before bash starts, so a path wait is + // satisfied by the banner whatever `command -v` would say. let session = world.tui.as_mut().expect("no engine shell session"); session - .send("command -v python\n") - .unwrap_or_else(|e| panic!("failed to query the interpreter: {e}")); + .send("python\n") + .unwrap_or_else(|e| panic!("failed to run the interpreter: {e}")); session - .wait_for_screen(&expected.display().to_string(), SCREEN_TIMEOUT) + .wait_for_screen(PLANTED_INTERPRETER_OUTPUT, SCREEN_TIMEOUT) .await .unwrap_or_else(|e| { panic!( - "the engine env interpreter did not win on PATH (wanted {}): {e}", - expected.display() + "running `python` in the engine shell did not run the engine env's interpreter \ + (wanted it to print {PLANTED_INTERPRETER_OUTPUT:?}): {e}" ) }); } From 9f4bc826117552220ff43ce0db348ec32832970b Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 27 Oct 2026 14:06:40 +0100 Subject: [PATCH 11/11] test(e2e): stop the interpreter step hanging the run; check VIRTUAL_ENV The interpreter step typed a bare python into the shell. When PATH is wrong that can start a real REPL as the terminal's foreground job; it outlives the kill of rocm, keeps the PTY open and hangs teardown instead of failing the scenario. Run python with stdin at EOF so a wrong interpreter exits on its own. The same step now also checks that VIRTUAL_ENV names an environment holding bin/activate, so engine-shell-02 flips only once the hint, PATH and VIRTUAL_ENV are all right. Verified with two product mutations: a hint-only fix and a hint+PATH fix each leave the row xfailed, and the run completes. automations-06 now tries every plausible identifier per check (exposed ids, including a 'command id:' key or a parenthesised token, then the label as listed, then its slug) and counts the check reachable if any works. The row comment says which fix directions are accepted. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 6 ++ .../features/engine_shell.feature | 4 +- .../tests/e2e/automations_steps.rs | 88 +++++++++++++------ tests/e2e-cucumber/tests/e2e/engines_steps.rs | 28 +++++- 4 files changed, 93 insertions(+), 33 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 1b52f90b3c..fb1382345c 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -195,6 +195,12 @@ reason = "`examine --json` reports has_amd_gpu:false on a WSL2 host whose own su # rejects anything else with rc=1. The names do not mechanically reduce to the # ids, so nothing the listing prints is enough to act on. Remove this row once # the listing exposes an identifier a user can pass back in. +# +# Either direction closes it, and the scenario accepts both: show the id in the +# listing, or make `enable` accept the label exactly as listed. Note that +# `render_automations_text`'s unit tests currently assert the listing does NOT +# contain `command id:` or the raw ids (apps/rocm/src/main.rs), so hiding them was +# a deliberate choice; a fix that shows the ids has to update those assertions. [["automations-listed-checks-can-be-enabled"]] when = {} bug = "EAI-8047" diff --git a/tests/e2e-cucumber/features/engine_shell.feature b/tests/e2e-cucumber/features/engine_shell.feature index 50b3da084d..5696fc456c 100644 --- a/tests/e2e-cucumber/features/engine_shell.feature +++ b/tests/e2e-cucumber/features/engine_shell.feature @@ -39,7 +39,9 @@ Feature: Engine shell # The interpreter step is here because the same doubled path also sets a wrong # VIRTUAL_ENV and puts a nonexistent bin/bin on PATH: a fix to the hint text # alone would otherwise flip this row while the shell still runs the wrong - # python. The row flips only once the user's actual problem is fixed. + # python. The step runs the interpreter and also checks that VIRTUAL_ENV names + # an environment holding bin/activate, so the row flips only once the hint, PATH + # and VIRTUAL_ENV are all right. # # Phrased as "the file it names is there", not as "the path is not doubled": the # first survives whichever way the mismatch is repaired, the second would go diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 5b32f00759..f0c01fcfb1 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -73,35 +73,45 @@ fn slug_of(display_name: &str) -> String { .join("-") } -/// The identifier a check block explicitly EXPOSES, if any. +/// The identifiers a check block plausibly EXPOSES, most specific first. /// /// The contract is "enable-able from what the listing shows", so the moment the -/// product publishes a real identifier this must pick THAT up, or a correct fix -/// would stay xfailed forever (the row would never go stale). Two shapes are -/// recognised, both unambiguous — neither can be mistaken for display text: -/// - a detail line — `id: server-recover` (also `identifier:` / `slug:`) +/// product publishes an identifier this must pick it up, or a correct fix would +/// stay xfailed forever (the row would never go stale). These are CANDIDATES, not +/// a verdict: the step tries each and counts the check reachable if any works, so +/// a wrong guess here costs nothing and a missing one is the only risk. +/// - a detail line keyed `id:` / `identifier:` / `slug:`, or any key ending in +/// ` id` (`command id: server-recover`) /// - bracketed on the header — `Server recovery [server-recover]` +/// - a single parenthesised token beside the label — `Server recovery (server-recover)` /// -/// Any other shape a fix might pick leaves the row xfailed until this learns it; -/// that is the cost of not guessing at forms no listing prints today. Returns -/// `None` only when the block names no identifier at all, which is today's defect. -fn exposed_identifier(block: &[&str]) -> Option { +/// Empty only when the block names no identifier at all, which is today's defect. +fn exposed_identifiers(block: &[&str]) -> Vec { + let mut found = Vec::new(); + let mut add = |id: &str| { + let id = id.trim().trim_matches(|c| c == '"' || c == '`'); + if !id.is_empty() && !id.contains(char::is_whitespace) && !found.iter().any(|f| f == id) { + found.push(id.to_owned()); + } + }; for line in block { - let line = line.trim(); - for key in ["id", "identifier", "slug"] { - if let Some(rest) = line.strip_prefix(&format!("{key}:")) { - let id = rest.trim().trim_matches(|c| c == '"' || c == '`'); - if !id.is_empty() { - return Some(id.to_owned()); - } + if let Some((key, rest)) = line.trim().split_once(':') { + let key = key.trim().to_ascii_lowercase(); + if matches!(key.as_str(), "id" | "identifier" | "slug") || key.ends_with(" id") { + add(rest); + } + } + } + if let Some(header) = block.first().map(|h| h.trim()) { + for (open, close) in [('[', ']'), ('(', ')')] { + if let (Some(o), Some(c)) = (header.rfind(open), header.rfind(close)) + && o < c + { + add(&header[o + 1..c]); } } } - let header = block.first()?.trim(); - let (open, close) = (header.rfind('[')?, header.rfind(']')?); - let inner = header.get(open + 1..close)?.trim(); - (open < close && !inner.is_empty() && !inner.contains(char::is_whitespace)) - .then(|| inner.to_owned()) + found } /// The checks the listing advertises, as `(displayed name, identifier to try)`. @@ -113,7 +123,7 @@ fn exposed_identifier(block: &[&str]) -> Option { /// under it forms one block, so an identifier the fix exposes on a detail line /// is seen. The identifier to invoke is the one the block explicitly exposes, /// falling back to the display-name slug only when it exposes none (today's bug). -fn listed_checks(listing: &str) -> Vec<(String, String)> { +fn listed_checks(listing: &str) -> Vec<(String, Vec)> { let lines: Vec<&str> = listing.lines().collect(); let is_header = |line: &str| line.starts_with(" ") && !line.starts_with(" "); let mut checks = Vec::new(); @@ -140,8 +150,15 @@ fn listed_checks(listing: &str) -> Vec<(String, String)> { break; } } - let identifier = exposed_identifier(&block).unwrap_or_else(|| slug_of(name)); - checks.push((name.to_owned(), identifier)); + // Exposed identifiers first, then the label exactly as listed (a fix may make + // `enable` accept it), then its slug — today's fallback. + let mut candidates = exposed_identifiers(&block); + for fallback in [name.to_owned(), slug_of(name)] { + if !candidates.contains(&fallback) { + candidates.push(fallback); + } + } + checks.push((name.to_owned(), candidates)); } checks } @@ -370,16 +387,29 @@ async fn assert_listed_checks_enableable(world: &mut E2eWorld) { ); let mut unreachable = Vec::new(); - for (name, identifier) in &checks { - let (stdout, stderr, rc) = crate::run_rocm(world, &["automations", "enable", identifier]); - if rc != 0 { - unreachable.push(format!( - "{name:?} → tried {identifier:?}: rc={rc} {}", + for (name, candidates) in &checks { + let mut last = String::new(); + let mut reached = false; + for candidate in candidates { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["automations", "enable", candidate]); + if rc == 0 { + reached = true; + break; + } + last = format!( + "tried {candidate:?}: rc={rc} {}", stderr .trim() .lines() .next() .unwrap_or_else(|| stdout.trim()) + ); + } + if !reached { + unreachable.push(format!( + "{name:?} → none of {} candidate(s) worked; last {last}", + candidates.len() )); } } diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 403e6a0a5e..de8ebd5b15 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -148,16 +148,22 @@ const PLANTED_INTERPRETER_OUTPUT: &str = "planted-engine-python"; #[then("the engine environment's interpreter is the one that runs")] async fn assert_engine_interpreter(world: &mut E2eWorld) { // Guards against the prompt shim breaking the activation it decorates, and - // against a wrong PATH/VIRTUAL_ENV: the shim re-runs the user's startup files, - // which could otherwise reorder PATH. + // against a wrong PATH or VIRTUAL_ENV: the shim re-runs the user's startup + // files, which could otherwise reorder PATH. // // This RUNS `python` and waits for the planted interpreter's own output. Do not // wait for the interpreter's path instead: `engine_shell` prints // `python: ` in its banner before bash starts, so a path wait is // satisfied by the banner whatever `command -v` would say. + // + // `