From 35c0942af065454152d8957e55e7844a8eb07df8 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 12 Aug 2026 19:16:01 +0200 Subject: [PATCH 01/62] test(e2e): pin the contracts a README walkthrough expects (EAI-8024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Walking the public README end to end on an MI300X pod turned up thirteen places where the CLI does not do what the README leads a reader to expect. This adds a scenario for each and registers it as an expected failure, so CI stays green and every row goes stale the day its defect is fixed. No product behaviour changes. Pinned on the no-GPU lane, so they gate every PR: EAI-8011 the help offers `rocm serve qwen2.5-7b-instruct`, which is neither a name the model listing knows nor an owner/repo reference EAI-8011 the help describes running `rocm` with no subcommand as opening the dashboard, which is what it says `rocm dash` does EAI-8010 `rocm update --dry-run` is refused as a usage error, though the README documents --apply and --dry-run as independent EAI-8019 `serve --help` advertises cpu_only and the command refuses it EAI-8020 the diagnosis and the fix preview print different verification steps for the same remedy EAI-8009 chatting locally with no --model ignores a ready local server that is not the built-in assistant model EAI-8014 uninstall reports completion while leaving the servers it manages running, having deleted the records needed to stop them Pinned on the GPU lane, scoped `when = {}` for a first measurement: EAI-8007 a stop that worked reports that it stopped no processes EAI-8015 a remedy names the group-lookup failure UNKNOWN as a group EAI-8020 the diagnosis and the fix propose different group commands EAI-8021 a device the user can already read and write is still offered as a group-permission cause EAI-8006 `--gpu auto` selects none on a machine that has a GPU EAI-8016 a second server is given an address already in use EAI-8018 Lemonade is advertised ready on the GPU but never answers EAI-8017 installing an engine fails when the session provides no scratch directory Harness additions, all small: removing environment variables from the child (not just setting them), a `stat` shim scoped to one device path so a device-group state can be arranged rather than hoped for, and a child process a scenario owns and registers as a managed service so it can ask whether the product stopped it — asserted before teardown, which would otherwise clean up after the product and hide the defect. EAI-8007 was first written against that child-process fixture on the no-GPU lane and XPASSed: a record with no start-time token takes the legacy best-effort path and is counted correctly. It now serves a real model instead, which is the path the pod exercised. Verified in a Linux container: ten new scenarios fail, each for the reason its row states, with no XPASS and no new unexpected failures. The one remaining failure there, engine-shell-marks-the-prompt, reproduces identically on unmodified main — the container runs as root, so the prompt ends in `#` where the assertion looks for `$`. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 165 ++++++++++ tests/e2e-cucumber/features/chat.feature | 11 + tests/e2e-cucumber/features/diagnose.feature | 40 +++ .../features/engine_shell.feature | 24 +- tests/e2e-cucumber/features/examine.feature | 19 ++ .../features/model_serving.feature | 72 +++++ tests/e2e-cucumber/features/update.feature | 16 + tests/e2e-cucumber/src/mock_server.rs | 18 +- tests/e2e-cucumber/tests/e2e.rs | 222 +++++++++++++ tests/e2e-cucumber/tests/e2e/chat_steps.rs | 11 + .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 273 ++++++++++++++++ tests/e2e-cucumber/tests/e2e/engines_steps.rs | 57 ++++ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 113 +++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 300 ++++++++++++++++++ tests/e2e-cucumber/tests/e2e/update_steps.rs | 43 +++ 15 files changed, 1378 insertions(+), 6 deletions(-) create mode 100644 tests/e2e-cucumber/features/update.feature create mode 100644 tests/e2e-cucumber/tests/e2e/update_steps.rs diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index da2de69982..805f7ad802 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -162,3 +162,168 @@ bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." serve_timeout_secs = 90 flaky = true + +# ── Contracts found by walking the public README end to end ───────────────── +# +# The rows below were all filed from one pass through README.md against a +# nightly build on an MI300X pod (EAI-8024 tracks the set). Each scenario asserts +# the behaviour the README leads a reader to expect, and each row here says the +# product does not do that yet — so the day one is fixed its row goes stale and +# deletes itself. + +# --- EAI-8011: the help's own worked examples name a model the CLI cannot +# resolve (`qwen2.5-7b-instruct` — not a catalog alias, and not an `owner/repo` +# reference either). Pure help text, so it holds on every platform. Remove this +# row when the examples name a resolvable model. --- +[["help-serve-example-names-a-resolvable-model"]] +when = {} +bug = "EAI-8011" +reason = "The help offers `rocm serve qwen2.5-7b-instruct`, which is neither a name the model listing knows nor an owner/repo reference." + +# --- EAI-8011: the help says running `rocm` with no subcommand opens the +# dashboard, which is what it also says `rocm dash` does; the README documents +# the plain command as opening the launcher instead, and that is what it really +# does. Pure help text. Remove when the two are described distinctly. --- +[["help-describes-the-default-command"]] +when = {} +bug = "EAI-8011" +reason = "The help describes running the CLI with no subcommand as opening the dashboard, the same thing it says `rocm dash` does." + +# --- EAI-8010: `--dry-run` is declared as requiring `--apply`, so previewing an +# update without asking for one is rejected as a usage error, though the README +# documents the two as independent. Argument handling only, so platform- +# independent. Remove when `rocm update --dry-run` is accepted. --- +[["update-preview-without-applying"]] +when = {} +bug = "EAI-8010" +reason = "`rocm update --dry-run` is refused as a usage error unless --apply is also given." + +# --- EAI-8019: `serve --help` advertises `cpu_only` among its device policies +# and the command then rejects it outright. Scoped to the no-GPU lane, which is +# where the scenario runs (`@requires-no-gpu`) — there every policy meets the +# same "no GPU on this machine" refusal, which makes a refusal aimed at the +# policy itself cleanly visible. Remove when the advertised list and the accepted +# list are the same list. --- +[["serve-rejects-no-advertised-device-policy"]] +when = {} +bug = "EAI-8019" +reason = "`serve` advertises cpu_only as a device policy and then refuses it for being cpu_only." + +# --- EAI-8020: `diagnose` and `fix` each carry their own copy of the +# render-group remedy, and the verification step has drifted between them +# (`diagnose` adds `ls -l /dev/kfd`). Both strings are constants, so this holds +# wherever the catalog runs at all; `@requires-os:linux` on the scenario keeps it +# off Windows, where the recipe does not apply. Remove when the two are one. --- +[["diagnose-and-fix-agree-on-how-to-verify"]] +when = {} +bug = "EAI-8020" +reason = "The diagnosis and the fix preview print different verification steps for the same remedy." + +# --- EAI-8009: with no `--model`, the local chat provider only looks for the +# built-in Lemonade assistant model, so a ready managed service running anything +# else is ignored and the user is told nothing is running. The scenario plants +# its own ready service, so this needs no hardware and holds on every lane. +# Remove when chatting locally uses whichever local server is ready. --- +[["chat-local-uses-the-running-server"]] +when = {} +bug = "EAI-8009" +reason = "The local chat provider only accepts the built-in assistant model, so a ready local server running any other model is not used." + +# --- EAI-8007: MEASURE. `services stop` reports the count of PIDs it signalled, +# which on the pod was 0 for a service it did in fact stop. First tried on the +# no-GPU lane with a plain child process standing in for the engine, and it +# XPASSed there: a record carrying no start-time token takes the legacy +# best-effort path and the child is signalled and counted correctly. So the +# scenario now serves a real model (GPU lane), where the CLI records identity +# tokens for the processes it launches — which is the path the pod exercised. --- +[["services-stop-reports-what-it-stopped"]] +when = {} +bug = "EAI-8007" +reason = "Stopping a running managed service reports that it stopped no processes." + +# --- EAI-8014: `uninstall` removes the managed files and reports completion +# without stopping the servers it manages — and takes the service records with +# it, so the supported way to stop them goes too. It warns that it does not stop +# them, which is the behaviour being disputed, not a defence of it. No hardware +# needed. Remove when uninstall stops what it manages first. --- +[["uninstall-stops-what-it-manages"]] +when = {} +bug = "EAI-8014" +reason = "Uninstall reports completion while leaving the servers it manages running, and removes the records needed to stop them." + +# --- EAI-8015: MEASURE. When the device's owning group has no entry in the local +# group database, `stat` answers `UNKNOWN` and the diagnosis passes that straight +# into a `usermod -a -G UNKNOWN,video` the user cannot run; only an EMPTY group +# name is filtered. The scenario substitutes that `stat` answer, so it needs a +# real device node but not a real broken host. GPU lane only — with no +# `/dev/kfd` the inspection stops before it ever asks `stat`. --- +[["diagnose-commands-name-a-real-group"]] +when = {} +bug = "EAI-8015" +reason = "A diagnosis names the group-lookup failure `UNKNOWN` as a group for the user to join." + +# --- EAI-8020: MEASURE. The other half of the same divergence: `diagnose` names +# the device's actual owning group while the `fix` recipe is hard-coded to +# `render`, so the two propose different changes wherever the device's group is +# not `render`. Needs a real device node, hence the GPU lane. --- +[["diagnose-and-fix-agree-on-the-remedy-command"]] +when = {} +bug = "EAI-8020" +reason = "The diagnosis proposes adding the user to the device's own group while the fix is hard-coded to `render`." + +# --- EAI-8021: MEASURE. The render-group check only ever ADDS score for missing +# group membership and never credits access that demonstrably works, so a device +# the user can already read and write is still diagnosed as a permission problem. +# The scenario substitutes a world-writable device answer and asserts the +# inspection agrees before relying on it. Where the runner's user already belongs +# to the conventional groups the finding scores zero anyway and this XPASSes — +# which is what the first GPU run is for. --- +[["diagnose-credits-a-usable-device"]] +when = {} +bug = "EAI-8021" +reason = "A device the user can already read and write is still offered as a group-permission cause." + +# --- EAI-8006: MEASURE. `--gpu auto` reports `selected none` on a machine that +# has a GPU, because the bundled `amd-smi` cannot execute (its `#!/usr/bin/env +# python3` shebang finds no interpreter on PATH) — even though the CLI has +# already recorded a working one in its own config. Scoped to any GPU host for +# the first run; narrow to the family that reproduces it if the dispatch +# disagrees. --- +[["serve-auto-gpu-selection-names-a-device"]] +when = {} +bug = "EAI-8006" +reason = "Letting the CLI choose a GPU reports that it selected none on a machine that has one." +serve_timeout_secs = 180 + +# --- EAI-8016: MEASURE. The serve path has no address-availability check: the +# default port is used whether or not something already holds it, so a second +# server is handed the address the first is on. The scenario holds the address +# itself rather than starting a second real server. --- +[["serve-second-server-gets-a-free-port"]] +when = {} +bug = "EAI-8016" +reason = "A server started without an address is given the default one even when it is already in use." +serve_timeout_secs = 120 + +# --- EAI-8018: MEASURE. On Instinct the Lemonade install selects a Vulkan +# llama.cpp backend and the engine inventory then says Lemonade is ready on the +# AMD GPU, but the packaged server enumerates only the host CPU under a +# GPU-required policy and answers 503. Expected to be family-scoped +# (`gfx94*`) — the first dispatch decides, and the precedent for narrowing is +# `examine-both-forms-agree-on-gpu`. --- +[["lemonade-serves-on-the-gpu-it-claims"]] +when = {} +bug = "EAI-8018" +reason = "The engine inventory says Lemonade is ready on this GPU, but a model served through it never answers." +serve_timeout_secs = 180 + +# --- EAI-8017: MEASURE. The CLI resolves a writable scratch directory for its +# own directories but passes none to the engine process it launches, so an +# install from a session without one fails before it starts. GPU lane: the +# engine's launcher needs the AMD userspace libraries to get as far as looking +# for a scratch directory, so on a plain runner it dies earlier for an unrelated +# reason and proves nothing. --- +[["engines-install-without-a-session-runtime-dir"]] +when = {} +bug = "EAI-8017" +reason = "Installing an engine fails when the session provides no scratch directory, because the CLI passes none to the engine it launches." diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index d0cc26099e..6e72d47101 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -75,3 +75,14 @@ Feature: Chat and endpoint detection And the model is registered with the CLI When the user sends a one-shot chat prompt through the CLI Then the CLI prints the assistant's reply + + # Expected to FAIL. Scenario 7 names the model and works; this one leaves the + # choice to the CLI and is told nothing is running — on the very machine whose + # services list shows a ready local server. Asking to chat locally should use + # the local server that is there, whichever model the user chose to serve. + @id:chat-local-uses-the-running-server + Scenario: 8 - Chatting locally uses the local server that is already running + Given a model is being served + And the model is registered with the CLI + When the user sends a one-shot chat prompt without naming a model + Then the CLI prints the assistant's reply diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 6497b1f088..7524943d97 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -78,3 +78,43 @@ Feature: Diagnosing failures and listing fixes When the user asks the CLI to apply it without agreeing to the change Then the CLI refuses and explains that it needs agreement And the file the fix would have changed is untouched + + # Expected to FAIL. `diagnose` and `fix` are two views of the same remedy: the + # first tells the user what will put the machine right, while the second is the + # command they are told to run. They must not leave the user with two different + # definitions of what proves the remedy worked. + @id:diagnose-and-fix-agree-on-how-to-verify @requires-os:linux + Scenario: 9 - Diagnosing a problem and previewing its fix agree on how to verify it + Given a user who hit a device-permission failure + When the user compares the diagnosis with the matching fix preview + Then both give the same way to verify that the fix worked + + # Expected to FAIL on a bare-metal GPU host. When the device belongs to a + # group that is not named in the local group database, the diagnosis prints + # that lookup failure as if it were a group the user could join. A proposed + # remedy has to name a group that actually exists on the machine. + @id:diagnose-commands-name-a-real-group @requires-gpu @requires-bare-metal @requires-os:linux + Scenario: 10 - Every group named in a diagnosis is one the machine recognises + Given the GPU device belongs to a group the machine cannot name + When the user asks the CLI to diagnose a device-permission failure + Then every group named in the remedy exists on the machine + + # Expected to FAIL on a bare-metal GPU host whose device group is not the + # hard-coded default. The diagnosis promises one command and its matching fix + # previews another. This is distinct from scenario 9: even after the verify + # text agrees, the actual change must agree too. + @id:diagnose-and-fix-agree-on-the-remedy-command @requires-gpu @requires-bare-metal @requires-os:linux + Scenario: 11 - Diagnosing a problem and previewing its fix agree on the remedy command + Given the GPU device belongs to a recognised non-default group + When the user compares the diagnosis with the matching fix preview + Then both give the same command for applying the remedy + + # Expected to FAIL on a bare-metal GPU host where direct access already works. + # Membership in one conventional group is only a means to access the device, + # not the outcome. A diagnosis should credit the observable access the user + # has instead of recommending a permission repair for a usable device. + @id:diagnose-credits-a-usable-device @requires-gpu @requires-bare-metal @requires-os:linux + Scenario: 12 - A usable GPU device is not diagnosed as a permission failure + Given the user can already read and write the GPU device + When the user asks the CLI to diagnose the machine + Then adding the user to a device group is not the leading remedy diff --git a/tests/e2e-cucumber/features/engine_shell.feature b/tests/e2e-cucumber/features/engine_shell.feature index 3712d41957..edbad57536 100644 --- a/tests/e2e-cucumber/features/engine_shell.feature +++ b/tests/e2e-cucumber/features/engine_shell.feature @@ -1,4 +1,4 @@ -Feature: Engine shell +Feature: Engine shell and engine installation # `rocm engines shell` always did activate the engine environment, but nothing # said so: the prompt marker was passed through the PS1 *environment variable*, @@ -22,3 +22,25 @@ Feature: Engine shell And the engine environment's interpreter is the one that runs When the user leaves the engine shell Then the engine shell exits successfully + + # Expected to FAIL on a GPU host. Installing an engine is a plain command a + # user runs from wherever they happen to be — a container, a bare `ssh` + # session, a service unit — and none of those necessarily give a process its + # own per-session scratch directory. The CLI already copes with that for the + # directories it owns; the engine it launches on the user's behalf must not be + # left to fail for the same reason. + # + # Only the missing-scratch-directory failure is pinned. Anything else the + # install runs into on a given host (no supported GPU backend, a download + # problem) is that host's business and does not make this scenario fail, so it + # goes green the day the CLI stops leaving the engine without one. + # + # @requires-gpu because the engine's own launcher needs the AMD userspace + # libraries to get far enough to look for a scratch directory at all: on a + # plain container it dies earlier, for an unrelated reason, and would prove + # nothing. + @id:engines-install-without-a-session-runtime-dir @requires-gpu @requires-os:linux + Scenario: 2 - Installing an engine works from a session with no scratch directory + Given a session that provides no scratch directory of its own + When the user installs the Lemonade engine + Then the install does not fail for want of a scratch directory diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 0aeef746b0..ffe978da37 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -17,6 +17,25 @@ Feature: GPU detection and system inspection When the user asks for help Then the subcommands are listed in alphabetical order + # Expected to FAIL. The help's own worked examples are the first thing a new + # user copies, so a model named there has to be one this CLI can actually + # serve. The check accepts either form the README documents — a name the model + # listing knows, or an explicit `owner/repo` reference — and so does not + # prescribe which model the examples should use. + @id:help-serve-example-names-a-resolvable-model + Scenario: 11 - Every model the help offers as an example is one the CLI can resolve + When the user reads the serve examples the help offers + Then every model named there is one the CLI can resolve + + # Expected to FAIL. Running the CLI with no subcommand and running its + # dashboard command are two different things, and the help describes them as + # the same one — so a reader has no way to learn what the plain command does + # or that there is anything else to reach. + @id:help-describes-the-default-command + Scenario: 12 - The help tells the two ways of opening a screen apart + When the user asks for help + Then running the CLI with no subcommand is not described as the dashboard command + @id:examine-detects-gpu-and-driver @requires-gpu Scenario: 3 - System inspection detects the GPU and driver Given a machine with an AMD GPU diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a9edaab998..d70060aa51 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -144,6 +144,78 @@ Feature: Model serving Then serving is refused before any engine starts And the user is told no AMD GPU was detected + # Expected to FAIL. `serve` advertises a set of device policies and then + # refuses one of them outright, so the list the user is offered is not the list + # the command accepts. Runs on the no-GPU lane, where a refusal that names the + # policy itself is cleanly distinguishable from the ordinary "this machine has + # no GPU" refusal every policy gets there. + @id:serve-rejects-no-advertised-device-policy @requires-no-gpu + Scenario: 14 - Every device policy the serve command offers is one it accepts + When the user serves a model under each device policy the command offers + Then no policy is refused for being that policy + + # Expected to FAIL on a GPU host. Stopping a running server does stop it, but + # the account it gives of what it did says it stopped nothing — so an operator + # cannot tell a stop that worked from one that found nothing to do. + # + # This needs a real served model, which is why it is on the GPU lane. A plain + # process registered as a managed service was tried first and reported the stop + # correctly (measured on the no-GPU lane), so the fixture would have pinned + # nothing: what differs about a real engine is that the CLI records identity + # tokens for its processes and reasons about them. + @id:services-stop-reports-what-it-stopped @requires-gpu + Scenario: 15 - Stopping a running server reports that it stopped it + Given a managed runtime is active + And a model is being served on GPU + When the user stops the server that is running + Then the CLI reports that it stopped a process + + # Expected to FAIL. Removing the CLI's managed files reports completion while + # the server it was managing is left running — and with the records gone, the + # supported way to stop it has been removed along with them. Nothing here + # touches the installed program: the removal is scoped to this scenario's own + # directories and keeps the binaries. + @id:uninstall-stops-what-it-manages @requires-os:linux + Scenario: 16 - Removing the CLI's managed files stops the servers it manages + Given a local server this machine manages is running + When the user removes the CLI's managed files + Then the removal is reported as complete + And the server is no longer running + + # Expected to FAIL on a GPU host. Asking the CLI to choose a GPU on a machine + # that has one must end with a device chosen: reporting that it selected none + # and carrying on leaves the user unable to tell which GPU their model will + # run on, or whether it will run on one at all. + @id:serve-auto-gpu-selection-names-a-device @requires-gpu + Scenario: 17 - Letting the CLI choose the GPU names the device it chose + Given a machine with an AMD GPU + When the user serves a model letting the CLI choose the GPU + Then the plan names the device it chose + + # Expected to FAIL on a GPU host. A second server started while the usual + # address is already taken is handed that same address anyway, so it collides + # with the server already there. Either outcome is fine — pick a free address, + # or say the usual one is busy — but silently reusing it is not. + @id:serve-second-server-gets-a-free-port @requires-gpu + Scenario: 18 - A second server does not take an address already in use + Given a managed runtime is active + And the address a new server would use is already taken + When the user serves a model without choosing an address + Then the new server does not try to use the taken address + + # Expected to FAIL on Instinct. The engine inventory tells the user Lemonade is + # ready to serve on their AMD GPU; that claim is only worth making if a model + # served through Lemonade then answers. Where the claim is not made this + # scenario does not run, so it never reads as a failure on a host that was + # honest about it. + @id:lemonade-serves-on-the-gpu-it-claims @requires-gpu @requires-engine:lemonade + Scenario: 19 - Lemonade serves on the GPU the engine inventory says it is ready on + Given a managed runtime is active + And the engine inventory says Lemonade is ready on this GPU + And a GGUF model is being served on lemonade + When the user sends a chat completion request + Then the response contains a model reply + # Honest device selection: a `--gpu` index that does not exist on the host is # rejected outright, never silently remapped to another device (no device-0 # fallback). Runs on GPU hardware: on a no-GPU host the GPU-required pre-flight diff --git a/tests/e2e-cucumber/features/update.feature b/tests/e2e-cucumber/features/update.feature new file mode 100644 index 0000000000..3248f50a61 --- /dev/null +++ b/tests/e2e-cucumber/features/update.feature @@ -0,0 +1,16 @@ +Feature: Checking for and previewing a ROCm update + + # `rocm update` is a query by default and only changes the machine when asked + # to. Nothing here installs anything: the scenarios cover what the command + # accepts, which is pure argument handling and so needs no GPU, no runtime, + # and no network. + + # Expected to FAIL. Asking to see what an update would do, without asking for + # it to be done, is refused as a misuse — even though checking is what this + # command does when left alone. The two choices are documented as independent + # of each other, so a user who wants a preview before committing to anything + # is turned away from the one command that would give them it. + @id:update-preview-without-applying + Scenario: 1 - Previewing an update without asking to install it is accepted + When the user asks to see what updating would do without asking for it to be done + Then the request is accepted rather than refused as a misuse diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 3f69b9d20f..db9a292fc8 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -644,6 +644,12 @@ impl MockServer { /// record shape. #[derive(Debug, Clone, Copy)] pub struct ServiceRecordOptions { + /// Id of the recorded service, used both as the record's own `service_id` + /// and as its file name. Defaults to `e2e-mock`, which the cucumber + /// teardown recognises as the planted record with no process behind it and + /// leaves alone; a scenario that registers a REAL process it owns passes its + /// own id so that teardown also tries to stop it. + pub service_id: &'static str, pub status: &'static str, pub startup_phase: Option<&'static str>, pub supervisor_pid: u32, @@ -653,6 +659,7 @@ pub struct ServiceRecordOptions { impl Default for ServiceRecordOptions { fn default() -> Self { Self { + service_id: "e2e-mock", status: "ready", startup_phase: None, supervisor_pid: 0, @@ -685,11 +692,12 @@ pub fn write_service_record_with( options: ServiceRecordOptions, ) { std::fs::create_dir_all(services_dir).expect("failed to create services dir"); + let manifest = services_dir.join(format!("{}.json", options.service_id)); // The CLI only extracts host:port from `endpoint_url` and appends // `/v1/models` for its readiness probe, which the mock serves. let record = json!({ - "service_id": "e2e-mock", + "service_id": options.service_id, "engine": "vllm", "model_ref": model, "canonical_model_id": model, @@ -711,13 +719,13 @@ pub fn write_service_record_with( "engine_recipe_json": null, "restart_count": 0, "last_restart_unix_ms": null, - "manifest_path": services_dir.join("e2e-mock.json"), - "log_path": services_dir.join("e2e-mock.log"), - "engine_state_path": services_dir.join("e2e-mock.state.json"), + "manifest_path": manifest, + "log_path": services_dir.join(format!("{}.log", options.service_id)), + "engine_state_path": services_dir.join(format!("{}.state.json", options.service_id)), "created_at_unix_ms": 1_700_000_000_000_u64, }); std::fs::write( - services_dir.join("e2e-mock.json"), + &manifest, serde_json::to_vec_pretty(&record).expect("failed to serialize record"), ) .expect("failed to write service record"); diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index e93d4e5aa0..b144057126 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -25,6 +25,7 @@ mod e2e { pub mod runtime_steps; pub mod serving_steps; pub mod tui_driver; + pub mod update_steps; } // ── World ────────────────────────────────────────────────────────── @@ -76,6 +77,66 @@ pub struct E2eWorld { /// dir, captured logs). `Some` only for `@lifecycle` scenarios; all its paths /// are rooted in `isolated_root` so teardown removes them with the temp dir. pub lifecycle: Option, + /// An address a scenario is deliberately holding, so a serve started without + /// one meets an address that is already in use. Released with the World. + pub occupied_address: Option, + /// A `PATH` a scenario prepared for the `rocm` invocations that follow (see + /// [`stat_shim_path`]). Carried on the World because the situation it sets up + /// belongs to the Given, while the invocations that must see it are in later + /// steps. + pub path_override: Option, + /// Plain child processes a scenario spawned itself and registered with the + /// CLI as managed services, so a step can assert the PRODUCT stopped them. + /// + /// Killed in `Drop` as a panic-path backstop only — never as the thing under + /// test. The scenarios that use these assert the child is gone *before* + /// teardown runs, because a `Drop` that cleans up after the product failed + /// would turn the very defect being pinned into a pass. + pub owned_processes: Vec, +} + +/// A child process this scenario owns, kept alive until the product is asked to +/// stop it (or teardown reaps it). +#[derive(Debug)] +pub struct OwnedProcess { + pub child: std::process::Child, +} + +impl OwnedProcess { + /// The child's PID. + pub fn pid(&self) -> u32 { + self.child.id() + } + + /// Whether the process is still running. + /// + /// This handle is the child's parent, so reaping it is what turns "signalled + /// and gone" into an observable exit — a terminated child stays a zombie + /// until then, and asking the OS whether the PID exists would still say yes. + /// Polls briefly: the product signals the process and this only has to + /// outlast the moment between the signal landing and the kernel finishing + /// with it, not any grace period the product itself waits out. + pub fn is_running(&mut self) -> bool { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + match self.child.try_wait() { + Ok(Some(_)) => return false, + Ok(None) if std::time::Instant::now() < deadline => { + std::thread::sleep(std::time::Duration::from_millis(100)); + } + Ok(None) => return true, + // Already reaped by an earlier call. + Err(_) => return false, + } + } + } +} + +impl Drop for OwnedProcess { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } } /// Resolve a CI-provided shared-directory env var to a validated, existing path. @@ -180,6 +241,9 @@ impl Default for E2eWorld { tui: None, chat_use_mock: false, lifecycle: None, + occupied_address: None, + path_override: None, + owned_processes: Vec::new(), } } } @@ -332,6 +396,52 @@ impl E2eWorld { self.register_mock_service_with(ServiceRecordOptions::default()); } + /// Spawn a plain, long-lived child process and register it with the CLI as a + /// ready managed service, exactly as a real `rocm serve --managed` would + /// record its own processes. Returns the child's PID. + /// + /// The child is a sleeper rather than a server: every scenario that uses + /// this asks whether the CLI *stopped the process it recorded*, and a + /// listening socket would add a second thing to get wrong without making + /// that question any easier to answer. Both PID fields point at the child so + /// the record looks like one the CLI itself wrote. + /// + /// Unix only — Windows has no equivalent one-liner sleeper on PATH, and the + /// scenarios that call this are `@requires-os:linux`. + pub fn start_managed_service_process(&mut self, service_id: &'static str, model: &str) -> u32 { + use std::process::Stdio; + + let root = self.isolated_root.as_ref().expect("no isolated root"); + let services = root.path().join("data").join("services"); + // Long enough to outlive any scenario, short enough that a leaked child + // on a persistent runner reaps itself rather than lingering forever. + let child = std::process::Command::new("sleep") + .arg("600") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to spawn the scenario's managed-service process"); + let pid = child.id(); + self.owned_processes.push(OwnedProcess { child }); + + // The port is never connected to; it only has to be a plausible value in + // a well-formed record. + write_service_record_with( + &services, + model, + 18_765, + ServiceRecordOptions { + service_id, + status: "ready", + startup_phase: None, + supervisor_pid: pid, + engine_pid: Some(pid), + }, + ); + pid + } + pub fn register_mock_service_with(&self, options: ServiceRecordOptions) { let root = self.isolated_root.as_ref().expect("no isolated root"); let mock = self.mock.as_ref().expect("no mock server running"); @@ -522,6 +632,118 @@ pub fn run_rocm_with_env( ) } +/// Install a `stat` shim on the front of `PATH` that answers for ONE device path +/// and delegates every other invocation to the real binary, and return the `PATH` +/// value to hand the child. +/// +/// The CLI learns a device's mode and owning group by shelling out to `stat`, and +/// several contracts are about what it then does with unusual answers — an owning +/// group the machine cannot name, or one that is not the conventional default. +/// Those states are properties of a machine's device nodes, not something a test +/// can arrange on a shared runner, so the answer is substituted instead. +/// +/// What this proves and what it does not: it exercises how the CLI HANDLES such +/// an answer, not that any particular host produces one. `stat` really does print +/// `UNKNOWN` for an id with no entry in the group database, which is where the +/// substituted answer comes from. +/// +/// Scoped as tightly as possible: only the named path is answered for, so every +/// other `stat` the CLI makes still reads the real filesystem. +/// +/// Unix only — panics elsewhere. Every scenario that reaches it is +/// `@requires-os:linux`, so the panic is unreachable rather than a limitation. +#[cfg(unix)] +pub fn stat_shim_path( + world: &E2eWorld, + device: &str, + mode: &str, + owner: &str, + group: &str, +) -> String { + use std::os::unix::fs::PermissionsExt as _; + + let root = world.isolated_root.as_ref().expect("no isolated root"); + let bin = root.path().join("stat-shim"); + std::fs::create_dir_all(&bin).expect("failed to create the stat shim directory"); + + let real = ["/usr/bin/stat", "/bin/stat", "/usr/local/bin/stat"] + .into_iter() + .find(|candidate| std::path::Path::new(candidate).is_file()) + .expect("no real `stat` binary to delegate to"); + // `-c %A|%U|%G` is the only form the CLI asks for, and the shim answers only + // when the target path is the one under test; anything else is the real + // binary's business. + let script = format!( + "#!/bin/sh\nfor arg in \"$@\"; do\n if [ \"$arg\" = \"{device}\" ]; then\n \ + printf '%s|%s|%s\\n' '{mode}' '{owner}' '{group}'\n exit 0\n fi\ndone\nexec {real} \"$@\"\n" + ); + let shim = bin.join("stat"); + std::fs::write(&shim, script).expect("failed to write the stat shim"); + std::fs::set_permissions(&shim, std::fs::Permissions::from_mode(0o755)) + .expect("failed to mark the stat shim executable"); + + let existing = std::env::var("PATH").unwrap_or_default(); + format!("{}:{existing}", bin.display()) +} + +#[cfg(not(unix))] +pub fn stat_shim_path( + _world: &E2eWorld, + _device: &str, + _mode: &str, + _owner: &str, + _group: &str, +) -> String { + unreachable!("the stat shim is only reached by @requires-os:linux scenarios") +} + +/// Every group name the machine's own group database knows. +/// +/// Read from `/etc/group` rather than asked of a tool, so the check does not +/// depend on `getent` being installed. Unix only, for the same reason as +/// [`stat_shim_path`]. +pub fn machine_group_names() -> Vec { + std::fs::read_to_string("/etc/group") + .unwrap_or_default() + .lines() + .filter_map(|line| line.split(':').next()) + .filter(|name| !name.trim().is_empty()) + .map(str::to_owned) + .collect() +} + +/// Like [`run_rocm`], but with environment variables REMOVED from the child. +/// +/// [`run_rocm_with_env`] can only set variables, and some contracts are about +/// what the CLI does when something the ambient environment usually provides is +/// absent. Removing them here — rather than depending on the runner not to have +/// them — makes the precondition the scenario's own, so the result means the +/// same thing on a bare container and on a full desktop session. +pub fn run_rocm_without_env( + world: &E2eWorld, + args: &[&str], + removed: &[&str], +) -> (String, String, i32) { + let binary = rocm_binary(); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + for key in removed { + cmd.env_remove(key); + } + let output = cmd + .output() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + let rc = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + ( + stdout, + String::from_utf8_lossy(&output.stderr).to_string(), + rc, + ) +} + /// Append one `rocm` invocation to `results/commands.jsonl` so the consolidated /// report can build a command × platform coverage table tied to real results. /// Best-effort: a recording failure must never fail a scenario. diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index 600a1dd7d3..b6abb9a2cb 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -80,6 +80,17 @@ async fn user_sends_oneshot_chat(world: &mut E2eWorld) { world.cli_output = Some(stdout); } +#[when("the user sends a one-shot chat prompt without naming a model")] +async fn user_sends_oneshot_chat_without_model(world: &mut E2eWorld) { + // The same command as the scenario above, minus `--model`. Leaving the model + // out is the ordinary way to say "use whatever is running here", and the + // services list on this machine shows exactly one ready local server. + let (stdout, stderr, rc) = + crate::run_rocm(world, &["chat", "--provider", "local", "--prompt", "Hello"]); + world.cli_output = Some(format!("{stdout}{stderr}")); + world.cli_rc = Some(rc); +} + // ── Then ─────────────────────────────────────────────────────────── #[then("the served model is listed")] diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index fac3fc4ba4..dead977254 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -380,6 +380,279 @@ async fn assert_refuses_without_agreement(world: &mut E2eWorld) { ); } +// ── Diagnosis and fix as two views of one remedy ─────────────────── + +/// The remedy both commands describe for a device-permission problem: the CLI +/// can apply it, and both `diagnose` and `fix` have their own copy of what it +/// does — which is exactly why the two can drift apart. +const DEVICE_PERMISSION_FIX_ID: &str = "fix-4-render-group"; + +/// A symptom that makes the device-permission cause score on any host, so the +/// comparison never depends on what happens to be wrong with the runner. +const DEVICE_PERMISSION_SYMPTOM: &str = "unable to open /dev/kfd"; + +/// The device whose owning group the remedy is about. +const GPU_DEVICE: &str = "/dev/kfd"; + +/// The group the fix recipe is hard-coded to add the user to. +const DEFAULT_DEVICE_GROUP: &str = "render"; + +/// The line a diagnosis prints for how to check the remedy worked. +const DIAGNOSE_VERIFY_PREFIX: &str = "verify after fix:"; + +/// The line a fix preview prints for the same thing. +const FIX_VERIFY_PREFIX: &str = "Verify:"; + +/// The value after `label` on the first line that carries it. +fn labelled(text: &str, label: &str) -> Option { + text.lines() + .filter_map(|line| line.trim().strip_prefix(label)) + .map(|value| value.trim().to_owned()) + .next() +} + +/// The `usermod` invocation a report proposes, wherever it prints it. Both +/// commands render it differently — one as a shell line under the plan, the +/// other as the command it is about to run — so match on the invocation itself. +fn proposed_group_command(text: &str) -> Option { + text.lines() + .map(str::trim) + .find(|line| line.contains("usermod") && line.contains("-G")) + .map(|line| { + // Strip whichever prefix the renderer used ("$ ", "Will run: ") so the + // two are compared as the same kind of thing. + line.trim_start_matches("$ ") + .trim_start_matches("Will run: ") + .to_owned() + }) +} + +/// The group names in a `-G a,b` argument of a proposed command. +fn groups_in_command(command: &str) -> Vec { + command + .split_whitespace() + .skip_while(|token| *token != "-G") + .nth(1) + .unwrap_or_default() + .trim_matches('"') + .split(',') + .filter(|name| !name.trim().is_empty()) + .map(|name| name.trim().to_owned()) + .collect() +} + +/// Run the diagnosis and the matching fix preview, and stash both. +fn compare_diagnosis_with_fix(world: &mut E2eWorld, path_override: Option<&str>) { + let args = ["diagnose", "--symptom", DEVICE_PERMISSION_SYMPTOM]; + let (diagnosis, _, _) = match path_override { + Some(path) => crate::run_rocm_with_env(world, &args, &[("PATH", path)]), + None => crate::run_rocm(world, &args), + }; + // No `--yes`, so nothing is applied; the preview is the user-facing account + // of what the fix would do. + let (preview, preview_err, _) = + crate::run_rocm(world, &["fix", DEVICE_PERMISSION_FIX_ID, "--dry-run"]); + world.cli_output = Some(diagnosis); + world.cli_stderr = Some(format!("{preview}{preview_err}")); +} + +#[given("a user who hit a device-permission failure")] +async fn user_hit_device_permission_failure(world: &mut E2eWorld) { + world.model_name = Some(DEVICE_PERMISSION_SYMPTOM.to_string()); +} + +#[when("the user compares the diagnosis with the matching fix preview")] +async fn user_compares_diagnosis_with_fix(world: &mut E2eWorld) { + let path_override = world.path_override.clone(); + compare_diagnosis_with_fix(world, path_override.as_deref()); +} + +#[then("both give the same way to verify that the fix worked")] +async fn assert_verification_agrees(world: &mut E2eWorld) { + let diagnosis = world.cli_output.as_ref().expect("no diagnosis output"); + let preview = world.cli_stderr.as_ref().expect("no fix preview output"); + assert!( + diagnosis.contains(DEVICE_PERMISSION_FIX_ID), + "the symptom did not produce the device-permission cause, so there is nothing to \ + compare:\n{diagnosis}" + ); + let from_diagnosis = labelled(diagnosis, DIAGNOSE_VERIFY_PREFIX) + .unwrap_or_else(|| panic!("the diagnosis gave no way to verify the fix:\n{diagnosis}")); + let from_preview = labelled(preview, FIX_VERIFY_PREFIX) + .unwrap_or_else(|| panic!("the fix preview gave no way to verify itself:\n{preview}")); + // A user told two different things to run cannot know which one settles it. + assert_eq!( + from_diagnosis, from_preview, + "the diagnosis and the fix preview give different ways to verify the same remedy" + ); +} + +#[then("both give the same command for applying the remedy")] +async fn assert_remedy_command_agrees(world: &mut E2eWorld) { + let diagnosis = world.cli_output.as_ref().expect("no diagnosis output"); + let preview = world.cli_stderr.as_ref().expect("no fix preview output"); + let from_diagnosis = proposed_group_command(diagnosis).unwrap_or_else(|| { + panic!("the diagnosis proposed no group command:\n{diagnosis}"); + }); + let from_preview = proposed_group_command(preview) + .unwrap_or_else(|| panic!("the fix preview proposed no group command:\n{preview}")); + // Compare the groups rather than the whole line: the two renderings name the + // user differently (one leaves `$USER` for the shell, the other resolves it), + // and that difference is presentation, not disagreement about the remedy. + assert_eq!( + groups_in_command(&from_diagnosis), + groups_in_command(&from_preview), + "the diagnosis and the fix preview would add the user to different groups:\n \ + diagnosis: {from_diagnosis}\n preview: {from_preview}" + ); +} + +// ── Device group states induced for the machine under test ───────── + +#[given("the GPU device belongs to a group the machine cannot name")] +async fn given_unnameable_device_group(world: &mut E2eWorld) { + // What `stat` itself prints for an owning id with no entry in the group + // database — the state a container inherits when the device's group exists + // on the host but not inside. + world.path_override = Some(crate::stat_shim_path( + world, + GPU_DEVICE, + "crw-rw----", + "root", + "UNKNOWN", + )); +} + +#[given("the GPU device belongs to a recognised non-default group")] +async fn given_non_default_device_group(world: &mut E2eWorld) { + // Any real group on this machine other than the one the fix recipe assumes. + // `video` is excluded too: the remedy adds it alongside, so using it would + // make the two sides agree by coincidence. + let group = crate::machine_group_names() + .into_iter() + .find(|name| name != DEFAULT_DEVICE_GROUP && name != "video") + .expect("this machine names no group other than the default"); + world.path_override = Some(crate::stat_shim_path( + world, + GPU_DEVICE, + "crw-rw----", + "root", + &group, + )); +} + +#[given("the user can already read and write the GPU device")] +async fn given_usable_device(world: &mut E2eWorld) { + let path = crate::stat_shim_path(world, GPU_DEVICE, "crw-rw-rw-", "root", "root"); + // Prove the premise through the CLI's own eyes before relying on it: if the + // inspection does not agree the device is usable, the scenario would be + // measuring something else. + let (report, _, _) = + crate::run_rocm_with_env(world, &["examine", "--json"], &[("PATH", path.as_str())]); + let json: serde_json::Value = serde_json::from_str(&report) + .unwrap_or_else(|e| panic!("`examine --json` did not emit valid JSON ({e}):\n{report}")); + for field in ["user_can_read", "user_can_write"] { + let stated = find_bool(&json, field); + assert_eq!( + stated, + Some(true), + "the inspection does not report the device as {field}, so the premise this scenario \ + needs is absent:\n{report}" + ); + } + world.path_override = Some(path); +} + +/// The first value of `field` anywhere in the document. The inspection nests the +/// device it describes, and where exactly is its own business. +fn find_bool(value: &serde_json::Value, field: &str) -> Option { + match value { + serde_json::Value::Object(map) => { + if let Some(found) = map.get(field).and_then(serde_json::Value::as_bool) { + return Some(found); + } + map.values().find_map(|child| find_bool(child, field)) + } + serde_json::Value::Array(items) => items.iter().find_map(|item| find_bool(item, field)), + _ => None, + } +} + +#[when("the user asks the CLI to diagnose a device-permission failure")] +async fn user_diagnoses_device_permission(world: &mut E2eWorld) { + let path = world + .path_override + .clone() + .expect("no shimmed PATH prepared"); + let (stdout, _, rc) = crate::run_rocm_with_env( + world, + &["diagnose", "--symptom", DEVICE_PERMISSION_SYMPTOM], + &[("PATH", path.as_str())], + ); + world.cli_output = Some(stdout); + world.cli_rc = Some(rc); +} + +#[when("the user asks the CLI to diagnose the machine")] +async fn user_diagnoses_machine(world: &mut E2eWorld) { + let path = world + .path_override + .clone() + .expect("no shimmed PATH prepared"); + // No symptom: the question is what this machine looks like, not what a + // particular error message suggests. + let (stdout, _, rc) = + crate::run_rocm_with_env(world, &["diagnose", "--json"], &[("PATH", path.as_str())]); + world.cli_output = Some(stdout); + world.cli_rc = Some(rc); +} + +#[then("every group named in the remedy exists on the machine")] +async fn assert_named_groups_exist(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no diagnose output"); + let command = proposed_group_command(output) + .unwrap_or_else(|| panic!("the diagnosis proposed no group command:\n{output}")); + let known = crate::machine_group_names(); + let unknown: Vec = groups_in_command(&command) + .into_iter() + .filter(|group| !known.iter().any(|name| name == group)) + .collect(); + // Handing the user a command naming a group that does not exist gives them + // something that cannot work and no way to see why. + assert!( + unknown.is_empty(), + "the remedy tells the user to join {unknown:?}, which this machine does not \ + recognise:\n {command}" + ); +} + +#[then("adding the user to a device group is not the leading remedy")] +async fn assert_no_device_group_remedy(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no diagnose output"); + let report: serde_json::Value = + serde_json::from_str(output).expect("diagnose --json did not emit valid JSON"); + let matched = report + .get("matched") + .and_then(|m| m.as_array()) + .expect("diagnose JSON has no 'matched' array"); + let offered = matched.iter().find(|entry| { + entry.get("id").and_then(serde_json::Value::as_str) == Some(DEVICE_PERMISSION_FIX_ID) + }); + let score = offered + .and_then(|entry| entry.get("score")) + .and_then(serde_json::Value::as_i64) + .unwrap_or(0); + // Belonging to a conventional group is one way to reach the device, not the + // point of doing so. With access already demonstrated, repairing permissions + // is not a cause of anything, and offering it sends the user to change their + // group membership for no reason. + assert_eq!( + score, 0, + "the device is readable and writable, yet the diagnosis still offers \ + '{DEVICE_PERMISSION_FIX_ID}' as a cause (score {score})" + ); +} + #[then("the file the fix would have changed is untouched")] async fn assert_rc_file_untouched(world: &mut E2eWorld) { let rc_file = fix_rc_file(world); diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 504d6afb52..827bd83fa4 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -155,3 +155,60 @@ async fn assert_engine_shell_exited(world: &mut E2eWorld) { "no engine shell session was opened for this scenario" ); } + +// ── Installing an engine ─────────────────────────────────────────── + +/// The variables a session uses to tell a process where its own scratch +/// directory is. A container, a bare remote shell, and a service started without +/// one all present the same way: neither is set. +const SESSION_SCRATCH_VARS: [&str; 2] = ["XDG_RUNTIME_DIR", "RUNTIME_DIRECTORY"]; + +/// How the failure under test reads when it happens. Matched loosely — on the +/// two words that identify it — so a reworded message still counts as the same +/// failure and does not quietly turn this scenario green. +const SCRATCH_DIR_FAILURE: [&str; 2] = ["runtime directory", "RUNTIME_DIRECTORY"]; + +#[given("a session that provides no scratch directory of its own")] +async fn given_no_session_scratch_dir(world: &mut E2eWorld) { + // Nothing to arrange beyond recording the intent: the removal happens on the + // child in the When step, because it is the CLI's own process that must cope. + world.model_name = Some("lemonade".to_owned()); +} + +#[when("the user installs the Lemonade engine")] +async fn user_installs_lemonade(world: &mut E2eWorld) { + let engine = world.model_name.clone().expect("no engine chosen"); + // Removed rather than assumed absent: on a runner that happens to have a + // session scratch directory the precondition would silently not hold, and + // the scenario would report on something else entirely. + let (stdout, stderr, rc) = crate::run_rocm_without_env( + world, + &["engines", "install", &engine], + &SESSION_SCRATCH_VARS, + ); + world.cli_output = Some(format!("{stdout}{stderr}")); + world.cli_rc = Some(rc); +} + +#[then("the install does not fail for want of a scratch directory")] +async fn assert_install_not_blocked_by_scratch_dir(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no install output"); + // Deliberately not "the install succeeded". What an engine install runs into + // on a given machine — no supported GPU backend, a download that did not + // come through — is that machine's business and no business of this + // scenario's. The one outcome pinned here is the CLI leaving the engine it + // launched without somewhere to work, which it already knows how to solve + // for the directories it owns. + let blocked = SESSION_SCRATCH_VARS + .iter() + .chain(SCRATCH_DIR_FAILURE.iter()) + .filter(|needle| output.contains(**needle)) + .count() + > 0; + assert!( + !blocked, + "installing the engine failed because the session gave it no scratch directory — the \ + CLI resolves one for its own directories and must do the same for the engine it \ + launches:\n{output}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index dc872cd8bb..106ac4ff4e 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -47,6 +47,119 @@ async fn user_asks_help(world: &mut E2eWorld) { world.cli_output = Some(stdout); } +// ── What the help itself promises ────────────────────────────────── + +/// Model references named after `rocm serve ` in an examples block, from both +/// the top-level help and `serve`'s own. A worked example is copied verbatim, so +/// the name in it is the one that has to work. +fn serve_example_models(help: &str) -> Vec { + help.lines() + .filter_map(|line| line.trim().strip_prefix("rocm serve ")) + .filter_map(|rest| rest.split_whitespace().next()) + // The generic placeholders the help uses to describe the FORM of a model + // reference are not names anyone is expected to run. + .filter(|model| !model.starts_with('<')) + .map(str::to_owned) + .collect() +} + +/// Every name the CLI's own model listing answers to: each recipe's canonical id +/// plus its aliases, as `rocm model --verbose` prints them. +fn known_model_names(listing: &str) -> Vec { + let mut names = Vec::new(); + for line in listing.lines() { + let Some((head, rest)) = line.trim().split_once(" aliases=[") else { + continue; + }; + names.push(head.trim().to_owned()); + if let Some((aliases, _)) = rest.split_once(']') { + names.extend(aliases.split(',').map(|alias| alias.trim().to_owned())); + } + } + names +} + +#[when("the user reads the serve examples the help offers")] +async fn user_reads_serve_examples(world: &mut E2eWorld) { + // Both example blocks that name a model: the top-level one a new user meets + // first, and `serve`'s own. + let (general, _, _) = crate::run_rocm(world, &["help"]); + let (serve, _, _) = crate::run_rocm(world, &["serve", "--help"]); + let (listing, _, _) = crate::run_rocm(world, &["model", "--verbose"]); + world.cli_output = Some(format!("{general}\n{serve}")); + world.cli_stderr = Some(listing); +} + +#[then("every model named there is one the CLI can resolve")] +async fn assert_example_models_resolve(world: &mut E2eWorld) { + let help = world.cli_output.as_ref().expect("no help output"); + let listing = world.cli_stderr.as_ref().expect("no model listing"); + let examples = serve_example_models(help); + assert!( + !examples.is_empty(), + "no `rocm serve ` example found in the help:\n{help}" + ); + let known = known_model_names(listing); + assert!( + !known.is_empty(), + "could not read any model name out of the listing:\n{listing}" + ); + // Two forms are legitimate, and the README documents both: a name this CLI's + // own catalog answers to, or an explicit `owner/repo` reference to fetch. The + // check accepts either, so it says the example must WORK without dictating + // which model it should be. + let unresolvable: Vec<&String> = examples + .iter() + .filter(|model| { + !model.contains('/') && !known.iter().any(|name| name.eq_ignore_ascii_case(model)) + }) + .collect(); + assert!( + unresolvable.is_empty(), + "the help offers {unresolvable:?} as models to serve, but they are neither a name the \ + model listing knows nor an `owner/repo` reference — copying the example verbatim \ + cannot work.\nmodel listing knows: {known:?}" + ); +} + +#[then("running the CLI with no subcommand is not described as the dashboard command")] +async fn assert_default_command_described_distinctly(world: &mut E2eWorld) { + let help = world.cli_output.as_ref().expect("no help output"); + // What the help says the dedicated dashboard command does. Taken from the + // help itself rather than hardcoded, so this stays true if the wording + // changes. + let dash_row = help + .lines() + .map(str::trim) + .find(|line| line.starts_with("dash ")) + .unwrap_or_default() + .to_ascii_lowercase(); + assert!( + dash_row.contains("dashboard"), + "expected the help to list a `dash` command that opens the dashboard:\n{help}" + ); + // The sentence describing what running the CLI with no subcommand does. + let default_sentence = help + .lines() + .map(str::trim) + .find(|line| line.contains("with no subcommand")) + .unwrap_or_default() + .to_ascii_lowercase(); + assert!( + !default_sentence.is_empty(), + "the help does not say what running the CLI with no subcommand does:\n{help}" + ); + // Two commands, two different screens. Describing the plain command as the + // dashboard leaves the reader unable to tell them apart, and unaware that + // the plain command opens something else entirely. + assert!( + !default_sentence.contains("dashboard"), + "the help describes running the CLI with no subcommand as opening the dashboard, which \ + is what it also says `dash` does — the two are different screens:\n \ + default: {default_sentence}\n dash: {dash_row}" + ); +} + #[then("a version string is returned")] async fn assert_version_returned(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 5c917bb848..d4193a3801 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -740,6 +740,306 @@ async fn user_serves_absent_gpu_index(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +// ── Device policies the command offers ───────────────────────────── + +/// The device policies `serve --help` advertises, read out of the help rather +/// than listed here so the check follows whatever the command currently offers. +fn advertised_device_policies(help: &str) -> Vec { + help.lines() + .find_map(|line| { + let (_, rest) = line.split_once("[possible values:")?; + let (values, _) = rest.split_once(']')?; + Some( + values + .split(',') + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + .collect::>(), + ) + }) + .unwrap_or_default() +} + +#[when("the user serves a model under each device policy the command offers")] +async fn user_serves_under_each_device_policy(world: &mut E2eWorld) { + let (help, _, _) = crate::run_rocm(world, &["serve", "--help"]); + let policies = advertised_device_policies(&help); + assert!( + !policies.is_empty(), + "`serve --help` advertises no device policies to check:\n{help}" + ); + let (model, engine, _) = host_serve_target(); + // One line per policy, so a failure names which one was refused. This runs + // only where there is no GPU, so no serve can actually start and each of + // these returns promptly. + let mut transcript = Vec::new(); + for policy in &policies { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["serve", model, "--engine", engine, "--device", policy], + ); + transcript.push(format!("{policy}\t{rc}\t{stdout}{stderr}")); + } + world.cli_outputs = Some(transcript); +} + +#[then("no policy is refused for being that policy")] +async fn assert_no_policy_refused(world: &mut E2eWorld) { + let transcript = world + .cli_outputs + .as_ref() + .expect("no serve attempts recorded"); + // On this lane every policy is refused for the same host-level reason — the + // machine has no GPU — and that refusal is not what this scenario is about. + // What it must not find is a refusal aimed at the POLICY: a command that + // offers a choice and then rejects the user for making it. + let rejected: Vec<&String> = transcript + .iter() + .filter(|line| { + let lower = line.to_lowercase(); + lower.contains("cpu mode is not a fallback path") + || lower.contains("unsupported device policy") + }) + .collect(); + assert!( + rejected.is_empty(), + "`serve --help` offers these device policies, but the command refuses one for being \ + what it is:\n{}", + rejected + .iter() + .map(|line| format!(" {line}")) + .collect::>() + .join("\n") + ); +} + +// ── A local server this scenario owns ────────────────────────────── + +/// The recorded id of the server a scenario starts for itself. Distinct from the +/// planted mock's id so the suite's teardown treats it as a real service and +/// tries to stop it too, as a backstop behind the scenario's own assertion. +const OWNED_SERVICE_ID: &str = "e2e-owned-server"; + +#[given("a local server this machine manages is running")] +async fn given_owned_managed_server(world: &mut E2eWorld) { + let pid = world.start_managed_service_process(OWNED_SERVICE_ID, "TestModel/E2E-1B"); + // The CLI must agree the server is there before anything is asked of it, + // otherwise a later "it is gone" would prove nothing. + let (listing, _, _) = crate::run_rocm(world, &["services", "list"]); + assert!( + listing.contains(OWNED_SERVICE_ID), + "the CLI does not list the running server (pid {pid}), so there is nothing to \ + stop:\n{listing}" + ); +} + +#[when("the user stops the server that is running")] +async fn user_stops_running_server(world: &mut E2eWorld) { + // Whichever service this scenario's serve registered. Read from the record + // the CLI wrote in its own (isolated) services directory — the same on-disk + // schema the suite's teardown reads — rather than assuming an id. + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") + .join("services"); + let service_id = std::fs::read_dir(&root) + .into_iter() + .flatten() + .flatten() + .filter(|entry| entry.path().extension().is_some_and(|e| e == "json")) + .find_map(|entry| { + let bytes = std::fs::read(entry.path()).ok()?; + let record: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + Some(record.get("service_id")?.as_str()?.to_owned()) + }) + .unwrap_or_else(|| panic!("no managed service recorded under {}", root.display())); + let (stdout, stderr, rc) = crate::run_rocm(world, &["services", "stop", &service_id, "--yes"]); + world.cli_output = Some(format!("{stdout}{stderr}")); + world.cli_rc = Some(rc); +} + +#[when("the user removes the CLI's managed files")] +async fn user_removes_managed_files(world: &mut E2eWorld) { + // `--keep-binaries` is what keeps this safe to run on a shared runner: the + // config, data and cache directories are this scenario's own temporary ones, + // so the removal is real but reaches nothing outside the scenario. It does + // not weaken the scenario — whether the CLI stops what it manages has + // nothing to do with whether it also deletes the program. + let (stdout, stderr, rc) = crate::run_rocm(world, &["uninstall", "--yes", "--keep-binaries"]); + world.cli_output = Some(format!("{stdout}{stderr}")); + world.cli_rc = Some(rc); +} + +#[then("the removal is reported as complete")] +async fn assert_removal_complete(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no uninstall output"); + assert_eq!(world.cli_rc, Some(0), "removal did not succeed:\n{output}"); + assert!( + output.contains("uninstall complete"), + "removal did not report completion:\n{output}" + ); +} + +#[then("the server is no longer running")] +async fn assert_owned_server_stopped(world: &mut E2eWorld) { + let output = world.cli_output.clone().unwrap_or_default(); + // Asked of the OS, and asked HERE — before this scenario's teardown, which + // also kills what it owns. Deferring the question until after teardown would + // report the harness's own cleanup as the product's work, turning the very + // defect under test into a pass. + let mut still_running = Vec::new(); + for process in &mut world.owned_processes { + if process.is_running() { + still_running.push(process.pid()); + } + } + assert!( + still_running.is_empty(), + "the CLI reported success but the server it manages is still running (pid \ + {still_running:?}):\n{output}" + ); +} + +#[then("the CLI reports that it stopped a process")] +async fn assert_stop_count_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no stop output"); + assert_eq!(world.cli_rc, Some(0), "stopping did not succeed:\n{output}"); + let reported = output + .lines() + .find_map(|line| line.trim().strip_prefix("stopped processes:")) + .map_or_else( + || panic!("the stop gave no account of what it stopped:\n{output}"), + str::trim, + ); + let count: u32 = reported + .parse() + .unwrap_or_else(|_| panic!("could not read the reported count {reported:?}:\n{output}")); + // An operator reading "0" has been told the stop found nothing to do, which + // is the opposite of what happened. + assert!( + count >= 1, + "the server was running and is now stopped, but the CLI reports it stopped \ + {count}:\n{output}" + ); +} + +// ── Choosing a device and an address ─────────────────────────────── + +#[when("the user serves a model letting the CLI choose the GPU")] +async fn user_serves_with_auto_gpu(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + // `--gpu auto` is the default, named here so the scenario is about the + // choice rather than about what happens to be configured. The plan the CLI + // prints on its way to launching is where it says which device it picked, + // and that line is all this scenario reads — whether the serve then goes on + // to succeed is a different scenario's business. + ensure_serve_port_free().await; + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + model, + "--engine", + engine, + "--gpu", + "auto", + "--managed", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the plan names the device it chose")] +async fn assert_auto_gpu_named(world: &mut E2eWorld) { + let output = serve_output(world); + let chosen = output + .lines() + .find_map(|line| line.trim().strip_prefix("gpu: auto (selected ")) + .and_then(|rest| rest.split(')').next()) + .map_or_else( + || panic!("the plan does not say which GPU was chosen:\n{output}"), + str::trim, + ); + // "none" on a machine that has a GPU leaves the user unable to tell whether + // their model is about to run on the device at all. + assert_ne!( + chosen, "none", + "this machine has an AMD GPU, but letting the CLI choose one selected \ + nothing:\n{output}" + ); +} + +#[given("the address a new server would use is already taken")] +async fn given_default_serve_address_taken(world: &mut E2eWorld) { + // Hold the address a server started without `--port` would take, so the next + // serve meets exactly the situation a user's second server meets. + ensure_serve_port_free().await; + let listener = std::net::TcpListener::bind(("127.0.0.1", SERVE_PORT)) + .unwrap_or_else(|e| panic!("could not take the default serve address: {e}")); + // Held on the World so it stays taken for the whole scenario and is released + // with it. + world.occupied_address = Some(listener); +} + +#[when("the user serves a model without choosing an address")] +async fn user_serves_without_choosing_address(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + // No `--port`, so the CLI picks the address itself — which is the whole + // question. The serve is expected not to come up (something else holds the + // address), so nothing here waits for readiness; the planned address is + // printed before the engine is launched. + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the new server does not try to use the taken address")] +async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { + let output = serve_output(world); + let planned = output + .lines() + .find_map(|line| line.trim().strip_prefix("port:")) + .map(str::trim); + // Two honest outcomes: pick somewhere else, or say the usual address is + // busy. Planning to use it anyway is the one that collides with the server + // already there. + match planned { + None => assert!( + world.cli_rc != Some(0), + "the CLI neither planned an address nor refused:\n{output}" + ), + Some(port) => assert_ne!( + port, + SERVE_PORT.to_string(), + "the address is already taken, but the new server plans to use it \ + anyway:\n{output}" + ), + } +} + +#[given("the engine inventory says Lemonade is ready on this GPU")] +async fn given_lemonade_claims_gpu_readiness(world: &mut E2eWorld) { + let (listing, _, _) = crate::run_rocm(world, &["engines", "list"]); + // Where the claim is not made, there is nothing to hold the CLI to, so the + // scenario stops here rather than reading as a failure on a host that was + // honest about what it can do. + let claims = listing + .lines() + .any(|line| line.contains("Lemonade is ready") && line.contains("GPU")); + assert!( + claims, + "the engine inventory does not claim Lemonade is ready on this GPU, so there is no \ + claim to hold it to:\n{listing}" + ); +} + #[when("the CLI reports the service as ready")] async fn when_cli_reports_ready(world: &mut E2eWorld) { // Read readiness from the CLI's own view (`services list`), not a direct diff --git a/tests/e2e-cucumber/tests/e2e/update_steps.rs b/tests/e2e-cucumber/tests/e2e/update_steps.rs new file mode 100644 index 0000000000..51e6d5abd8 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/update_steps.rs @@ -0,0 +1,43 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `rocm update`. +//! +//! Argument handling only: nothing here contacts a package index or changes the +//! machine, so these run on every lane. + +use cucumber::{then, when}; + +use crate::E2eWorld; + +/// The exit code a CLI uses to reject the way it was CALLED, as opposed to +/// failing at the work it was asked to do. Anything the command decides about +/// the machine — no runtime registered, nothing to update — is a different +/// outcome and not what this scenario is about. +const USAGE_ERROR: i32 = 2; + +#[when("the user asks to see what updating would do without asking for it to be done")] +async fn user_previews_update(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["update", "--dry-run"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the request is accepted rather than refused as a misuse")] +async fn assert_preview_accepted(world: &mut E2eWorld) { + let combined = format!( + "{}{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ); + // Deliberately NOT "exits 0": a host with no ROCm install registered has + // nothing to check and says so, which is a legitimate answer to a legitimate + // question. The contract is only that asking was allowed. + assert_ne!( + world.cli_rc, + Some(USAGE_ERROR), + "asking to preview an update was rejected as a misuse of the command:\n{combined}" + ); +} From f06f4175c710abcc264805560dc9a3aeb2b529e9 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 12 Aug 2026 19:52:12 +0200 Subject: [PATCH 02/62] test(e2e): correct the GPU rows against what run 188 measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first app-dev-gpu dispatch answered the MEASURE questions the rows were carrying, and three of them came back differently from the plan. EAI-8007 loses its row. Neither fixture reproduces it: a plain process registered as a managed service is counted correctly (no-GPU lane), and so is a real vLLM serve (MI300X). A row would be a stale XPASS on both lanes and would fail them, so the scenario ships as a guard for the contract the pod violated instead. What was different about that host is still unknown. EAI-8021 needed a second substituted answer. The scenario made the device world-writable and expected the group-permission cause to stop being offered, but the GPU runner's user is already in the conventional groups, so the cause scored zero there for a reason that has nothing to do with device access and the scenario passed. It now substitutes the group answer too, so the state it describes — access works, membership absent — holds on any host. EAI-8006 was failing for the wrong reason: the serve was refused for having no active ROCm runtime, before it ever chose a device. It now takes the same managed-runtime precondition its sibling serve scenarios do. Two scenarios could not be measured at all, because `rocm engines install lemonade` fails on the GPU runner within two seconds: - EAI-8017 was passing vacuously — the install never got far enough to leave the engine without a scratch directory, and the assertion tolerated that. It now requires the install to have got under way before it will speak. - EAI-8018 asserted a claim the inventory never makes there, because Lemonade is not installed in the scenario's isolated directories. It now installs it first and reports the install output when the claim is absent. Both will say why on the next dispatch. Also scopes the diagnosis comparisons to the cause under test rather than the first one reported (which cause ranks highest is host-dependent), tightens the scratch-directory match so ordinary path output cannot satisfy it, and stops the uninstall assertion passing vacuously when no server was registered. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 18 +++--- .../features/model_serving.feature | 18 +++--- tests/e2e-cucumber/tests/e2e.rs | 41 +++++++++++++ .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 57 +++++++++++++++---- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 31 +++++++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 15 +++++ 6 files changed, 138 insertions(+), 42 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 805f7ad802..c57873b49e 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -229,17 +229,13 @@ when = {} bug = "EAI-8009" reason = "The local chat provider only accepts the built-in assistant model, so a ready local server running any other model is not used." -# --- EAI-8007: MEASURE. `services stop` reports the count of PIDs it signalled, -# which on the pod was 0 for a service it did in fact stop. First tried on the -# no-GPU lane with a plain child process standing in for the engine, and it -# XPASSed there: a record carrying no start-time token takes the legacy -# best-effort path and the child is signalled and counted correctly. So the -# scenario now serves a real model (GPU lane), where the CLI records identity -# tokens for the processes it launches — which is the path the pod exercised. --- -[["services-stop-reports-what-it-stopped"]] -when = {} -bug = "EAI-8007" -reason = "Stopping a running managed service reports that it stopped no processes." +# EAI-8007 has NO row on purpose. `services stop` reported stopping 0 processes +# for a service it did in fact stop on the pod, but neither fixture reproduces it +# here: a plain child process registered as a managed service is counted +# correctly (no-GPU lane), and so is a real vLLM serve (MI300X, run 188). Its +# scenario therefore ships as a guard rather than an expected failure — a row +# would be a stale XPASS on both lanes and would fail them. The ticket keeps the +# pod observation; what distinguishes that host is still unknown. # --- EAI-8014: `uninstall` removes the managed files and reports completion # without stopping the servers it manages — and takes the service records with diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index d70060aa51..61288dffcc 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -154,15 +154,12 @@ Feature: Model serving When the user serves a model under each device policy the command offers Then no policy is refused for being that policy - # Expected to FAIL on a GPU host. Stopping a running server does stop it, but - # the account it gives of what it did says it stopped nothing — so an operator - # cannot tell a stop that worked from one that found nothing to do. - # - # This needs a real served model, which is why it is on the GPU lane. A plain - # process registered as a managed service was tried first and reported the stop - # correctly (measured on the no-GPU lane), so the fixture would have pinned - # nothing: what differs about a real engine is that the CLI records identity - # tokens for its processes and reasons about them. + # A guard, not a finding. Stopping a running server was reported as having + # stopped nothing on the pod this set came from, but neither fixture tried here + # reproduces that: a plain process registered as a managed service is counted + # correctly (measured on the no-GPU lane), and so is a real vLLM serve + # (measured on MI300X, run 188). So this carries no expected-failure row — it + # holds the contract the pod violated, and goes red if CI ever meets it. @id:services-stop-reports-what-it-stopped @requires-gpu Scenario: 15 - Stopping a running server reports that it stopped it Given a managed runtime is active @@ -188,7 +185,8 @@ Feature: Model serving # run on, or whether it will run on one at all. @id:serve-auto-gpu-selection-names-a-device @requires-gpu Scenario: 17 - Letting the CLI choose the GPU names the device it chose - Given a machine with an AMD GPU + Given a managed runtime is active + And a machine with an AMD GPU When the user serves a model letting the CLI choose the GPU Then the plan names the device it chose diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index b144057126..e4ab57f06c 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -697,6 +697,47 @@ pub fn stat_shim_path( unreachable!("the stat shim is only reached by @requires-os:linux scenarios") } +/// Add an `id` shim to the same directory [`stat_shim_path`] uses, reporting the +/// current user as belonging to `groups` instead of their real ones, and return +/// the `PATH` carrying both shims. +/// +/// The CLI reads group membership by shelling out to `id -Gn`. Whether the user +/// running the suite belongs to a device group is a property of the runner, and +/// on the GPU runner they already do — so a contract about what the CLI concludes +/// for a user who does NOT can only be measured by substituting the answer. Call +/// after [`stat_shim_path`], whose directory this reuses. +#[cfg(unix)] +pub fn id_shim_path(world: &E2eWorld, groups: &[&str]) -> String { + use std::os::unix::fs::PermissionsExt as _; + + let root = world.isolated_root.as_ref().expect("no isolated root"); + let bin = root.path().join("stat-shim"); + std::fs::create_dir_all(&bin).expect("failed to create the shim directory"); + + let real = ["/usr/bin/id", "/bin/id"] + .into_iter() + .find(|candidate| std::path::Path::new(candidate).is_file()) + .expect("no real `id` binary to delegate to"); + // `-Gn` is the only form the CLI asks for; every other invocation is the real + // binary's business. + let script = format!( + "#!/bin/sh\nif [ \"$1\" = \"-Gn\" ]; then\n echo '{}'\n exit 0\nfi\nexec {real} \"$@\"\n", + groups.join(" ") + ); + let shim = bin.join("id"); + std::fs::write(&shim, script).expect("failed to write the id shim"); + std::fs::set_permissions(&shim, std::fs::Permissions::from_mode(0o755)) + .expect("failed to mark the id shim executable"); + + let existing = std::env::var("PATH").unwrap_or_default(); + format!("{}:{existing}", bin.display()) +} + +#[cfg(not(unix))] +pub fn id_shim_path(_world: &E2eWorld, _groups: &[&str]) -> String { + unreachable!("the id shim is only reached by @requires-os:linux scenarios") +} + /// Every group name the machine's own group database knows. /// /// Read from `/etc/group` rather than asked of a tool, so the check does not diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index dead977254..70fc2bf443 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -411,6 +411,23 @@ fn labelled(text: &str, label: &str) -> Option { .next() } +/// The part of a diagnosis that belongs to one cause. +/// +/// A diagnosis reports several causes, each with its own plan and its own way to +/// verify it, and which one comes first depends on the machine. Narrowing to the +/// named cause first is what keeps the comparison about the remedy under test +/// rather than about whichever cause happened to rank highest here. +fn section_for(report: &str, fix_id: &str) -> Option { + let start = report.lines().position(|line| line.contains(fix_id))?; + // Causes are separated by a blank line, so the next one ends this section. + let body: Vec<&str> = report + .lines() + .skip(start) + .take_while(|line| !line.trim().is_empty()) + .collect(); + Some(body.join("\n")) +} + /// The `usermod` invocation a report proposes, wherever it prints it. Both /// commands render it differently — one as a shell line under the plan, the /// other as the command it is about to run — so match on the invocation itself. @@ -471,13 +488,14 @@ async fn user_compares_diagnosis_with_fix(world: &mut E2eWorld) { async fn assert_verification_agrees(world: &mut E2eWorld) { let diagnosis = world.cli_output.as_ref().expect("no diagnosis output"); let preview = world.cli_stderr.as_ref().expect("no fix preview output"); - assert!( - diagnosis.contains(DEVICE_PERMISSION_FIX_ID), - "the symptom did not produce the device-permission cause, so there is nothing to \ - compare:\n{diagnosis}" - ); - let from_diagnosis = labelled(diagnosis, DIAGNOSE_VERIFY_PREFIX) - .unwrap_or_else(|| panic!("the diagnosis gave no way to verify the fix:\n{diagnosis}")); + let cause = section_for(diagnosis, DEVICE_PERMISSION_FIX_ID).unwrap_or_else(|| { + panic!( + "the symptom did not produce the device-permission cause, so there is nothing to \ + compare:\n{diagnosis}" + ) + }); + let from_diagnosis = labelled(&cause, DIAGNOSE_VERIFY_PREFIX) + .unwrap_or_else(|| panic!("the diagnosis gave no way to verify the fix:\n{cause}")); let from_preview = labelled(preview, FIX_VERIFY_PREFIX) .unwrap_or_else(|| panic!("the fix preview gave no way to verify itself:\n{preview}")); // A user told two different things to run cannot know which one settles it. @@ -491,9 +509,14 @@ async fn assert_verification_agrees(world: &mut E2eWorld) { async fn assert_remedy_command_agrees(world: &mut E2eWorld) { let diagnosis = world.cli_output.as_ref().expect("no diagnosis output"); let preview = world.cli_stderr.as_ref().expect("no fix preview output"); - let from_diagnosis = proposed_group_command(diagnosis).unwrap_or_else(|| { - panic!("the diagnosis proposed no group command:\n{diagnosis}"); + let cause = section_for(diagnosis, DEVICE_PERMISSION_FIX_ID).unwrap_or_else(|| { + panic!( + "the symptom did not produce the device-permission cause, so there is nothing to \ + compare:\n{diagnosis}" + ) }); + let from_diagnosis = proposed_group_command(&cause) + .unwrap_or_else(|| panic!("the diagnosis proposed no group command:\n{cause}")); let from_preview = proposed_group_command(preview) .unwrap_or_else(|| panic!("the fix preview proposed no group command:\n{preview}")); // Compare the groups rather than the whole line: the two renderings name the @@ -543,7 +566,14 @@ async fn given_non_default_device_group(world: &mut E2eWorld) { #[given("the user can already read and write the GPU device")] async fn given_usable_device(world: &mut E2eWorld) { - let path = crate::stat_shim_path(world, GPU_DEVICE, "crw-rw-rw-", "root", "root"); + // Two halves of one situation: the device is open to everyone, AND the user + // is in none of the groups conventionally used to reach it. Both are + // substituted rather than hoped for — the GPU runner's user IS in those + // groups, which makes the finding score zero there for a reason that has + // nothing to do with the access the scenario is about (measured: it passed + // on MI300X until the group answer was substituted too). + crate::stat_shim_path(world, GPU_DEVICE, "crw-rw-rw-", "root", "root"); + let path = crate::id_shim_path(world, &["users"]); // Prove the premise through the CLI's own eyes before relying on it: if the // inspection does not agree the device is usable, the scenario would be // measuring something else. @@ -610,8 +640,11 @@ async fn user_diagnoses_machine(world: &mut E2eWorld) { #[then("every group named in the remedy exists on the machine")] async fn assert_named_groups_exist(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no diagnose output"); - let command = proposed_group_command(output) - .unwrap_or_else(|| panic!("the diagnosis proposed no group command:\n{output}")); + let cause = section_for(output, DEVICE_PERMISSION_FIX_ID).unwrap_or_else(|| { + panic!("the symptom did not produce the device-permission cause:\n{output}") + }); + let command = proposed_group_command(&cause) + .unwrap_or_else(|| panic!("the diagnosis proposed no group command:\n{cause}")); let known = crate::machine_group_names(); let unknown: Vec = groups_in_command(&command) .into_iter() diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 827bd83fa4..dd8eed2361 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -163,10 +163,13 @@ async fn assert_engine_shell_exited(world: &mut E2eWorld) { /// one all present the same way: neither is set. const SESSION_SCRATCH_VARS: [&str; 2] = ["XDG_RUNTIME_DIR", "RUNTIME_DIRECTORY"]; -/// How the failure under test reads when it happens. Matched loosely — on the -/// two words that identify it — so a reworded message still counts as the same -/// failure and does not quietly turn this scenario green. -const SCRATCH_DIR_FAILURE: [&str; 2] = ["runtime directory", "RUNTIME_DIRECTORY"]; +/// How the failure under test reads when it happens: the engine's launcher says +/// it could not resolve a writable runtime directory and names the variables it +/// looked in. Matched on the distinctive phrase and on the variable names rather +/// than the whole sentence, so a reworded message still counts as the same +/// failure and does not quietly turn this scenario green — but not on the bare +/// words "runtime directory", which appear in ordinary path output. +const SCRATCH_DIR_FAILURE: [&str; 2] = ["writable runtime directory", "RUNTIME_DIRECTORY"]; #[given("a session that provides no scratch directory of its own")] async fn given_no_session_scratch_dir(world: &mut E2eWorld) { @@ -190,21 +193,31 @@ async fn user_installs_lemonade(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// What the install prints once it is genuinely under way. Reaching this is what +/// makes the rest of the assertion mean anything. +const INSTALL_STARTED: &str = "Preparing Lemonade"; + #[then("the install does not fail for want of a scratch directory")] async fn assert_install_not_blocked_by_scratch_dir(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no install output"); + // An install that stops before it gets going cannot say anything about what + // the engine needs once it is running, and letting that count as a pass + // would make this scenario green while proving nothing. + assert!( + output.contains(INSTALL_STARTED), + "the install never got under way, so this scenario cannot say whether the engine was \ + left without a scratch directory:\n{output}" + ); // Deliberately not "the install succeeded". What an engine install runs into // on a given machine — no supported GPU backend, a download that did not // come through — is that machine's business and no business of this // scenario's. The one outcome pinned here is the CLI leaving the engine it // launched without somewhere to work, which it already knows how to solve // for the directories it owns. - let blocked = SESSION_SCRATCH_VARS + let blocked = SCRATCH_DIR_FAILURE .iter() - .chain(SCRATCH_DIR_FAILURE.iter()) - .filter(|needle| output.contains(**needle)) - .count() - > 0; + .chain(SESSION_SCRATCH_VARS.iter()) + .any(|needle| output.contains(needle)); assert!( !blocked, "installing the engine failed because the session gave it no scratch directory — the \ diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index d4193a3801..394d4f4eb1 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -886,6 +886,13 @@ async fn assert_removal_complete(world: &mut E2eWorld) { #[then("the server is no longer running")] async fn assert_owned_server_stopped(world: &mut E2eWorld) { let output = world.cli_output.clone().unwrap_or_default(); + // Guards the assertion below against passing vacuously: with no process + // registered there is nothing that could still be running, and the scenario + // would go green having proved nothing. + assert!( + !world.owned_processes.is_empty(), + "this scenario registered no server, so there is nothing to have stopped" + ); // Asked of the OS, and asked HERE — before this scenario's teardown, which // also kills what it owns. Deferring the question until after teardown would // report the harness's own cleanup as the product's work, turning the very @@ -1026,7 +1033,15 @@ async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { #[given("the engine inventory says Lemonade is ready on this GPU")] async fn given_lemonade_claims_gpu_readiness(world: &mut E2eWorld) { + // The claim is only made once the engine is installed, so install it first — + // into this scenario's own isolated directories, not the runner's. + let (install, install_err, _) = crate::run_rocm(world, &["engines", "install", "lemonade"]); let (listing, _, _) = crate::run_rocm(world, &["engines", "list"]); + assert!( + !listing.contains("Lemonade is not installed"), + "Lemonade could not be installed here, so the claim this scenario holds the CLI to is \ + never made:\n--- install ---\n{install}{install_err}\n--- inventory ---\n{listing}" + ); // Where the claim is not made, there is nothing to hold the CLI to, so the // scenario stops here rather than reading as a failure on a host that was // honest about what it can do. From e9b1547075a5644c56b206eea79e1d74077eddbf Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 12 Aug 2026 20:11:17 +0200 Subject: [PATCH 03/62] test(e2e): drop what the runners cannot exercise, keep what they can MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second dispatch settled the remaining MEASURE questions. EAI-8021 now fails for its stated reason once the group answer is substituted alongside the device one (score 45 with the device readable and writable), so its row stands. EAI-8006 loses its row and ships as a guard. `--gpu auto` reported selecting none on the pod, where the bundled amd-smi could not execute; the MI300X runner's amd-smi runs, so the CLI names a device and the scenario passes there. EAI-8017 and EAI-8018 lose their scenarios entirely. Both need a working Lemonade install, and Lemonade's launcher cannot start on either lane: on the MI300X runner it wants GLIBC_2.38 and GLIBCXX_3.4.32, which that OS does not have, and in a plain container it wants libdrm_amdgpu.so.1. Each dies before reaching the behaviour under test, so the scenarios were passing vacuously — which reads as coverage while proving nothing. Better no scenario than that one; both stay on their tickets, reproduced by hand on the pod, until a lane can run Lemonade. Removing the Lemonade install from the engine-inventory scenario also stops the suite writing into the runner's shared pre-warm tree, which it had begun doing. That leaves nine of the thirteen pinned as expected failures, two shipping as guards for a contract our lanes uphold and the pod did not, and two not covered. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 48 ++++--------- .../features/engine_shell.feature | 24 +------ .../features/model_serving.feature | 13 ---- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 70 ------------------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 24 ------- 5 files changed, 16 insertions(+), 163 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index c57873b49e..7e3849f754 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -279,17 +279,12 @@ when = {} bug = "EAI-8021" reason = "A device the user can already read and write is still offered as a group-permission cause." -# --- EAI-8006: MEASURE. `--gpu auto` reports `selected none` on a machine that -# has a GPU, because the bundled `amd-smi` cannot execute (its `#!/usr/bin/env -# python3` shebang finds no interpreter on PATH) — even though the CLI has -# already recorded a working one in its own config. Scoped to any GPU host for -# the first run; narrow to the family that reproduces it if the dispatch -# disagrees. --- -[["serve-auto-gpu-selection-names-a-device"]] -when = {} -bug = "EAI-8006" -reason = "Letting the CLI choose a GPU reports that it selected none on a machine that has one." -serve_timeout_secs = 180 +# EAI-8006 has NO row on purpose. `--gpu auto` reported `selected none` on the +# pod, where the bundled `amd-smi` could not execute (its `#!/usr/bin/env +# python3` shebang found no interpreter). The MI300X runner's `amd-smi` runs, so +# the CLI names a device there and the scenario passes (measured, runs 190/191). +# It ships as a guard instead: a row would be a stale XPASS and would fail the +# lane. # --- EAI-8016: MEASURE. The serve path has no address-availability check: the # default port is used whether or not something already holds it, so a second @@ -301,25 +296,12 @@ bug = "EAI-8016" reason = "A server started without an address is given the default one even when it is already in use." serve_timeout_secs = 120 -# --- EAI-8018: MEASURE. On Instinct the Lemonade install selects a Vulkan -# llama.cpp backend and the engine inventory then says Lemonade is ready on the -# AMD GPU, but the packaged server enumerates only the host CPU under a -# GPU-required policy and answers 503. Expected to be family-scoped -# (`gfx94*`) — the first dispatch decides, and the precedent for narrowing is -# `examine-both-forms-agree-on-gpu`. --- -[["lemonade-serves-on-the-gpu-it-claims"]] -when = {} -bug = "EAI-8018" -reason = "The engine inventory says Lemonade is ready on this GPU, but a model served through it never answers." -serve_timeout_secs = 180 - -# --- EAI-8017: MEASURE. The CLI resolves a writable scratch directory for its -# own directories but passes none to the engine process it launches, so an -# install from a session without one fails before it starts. GPU lane: the -# engine's launcher needs the AMD userspace libraries to get as far as looking -# for a scratch directory, so on a plain runner it dies earlier for an unrelated -# reason and proves nothing. --- -[["engines-install-without-a-session-runtime-dir"]] -when = {} -bug = "EAI-8017" -reason = "Installing an engine fails when the session provides no scratch directory, because the CLI passes none to the engine it launches." +# EAI-8017 and EAI-8018 have NO rows and NO scenarios, because neither can be +# exercised on any lane available here. Both need a working Lemonade install, +# and Lemonade's own launcher cannot start on either lane: on the MI300X runner +# `lemond` wants GLIBC_2.38 / GLIBCXX_3.4.32, which that OS does not have +# (measured, runs 190/191), and in a plain container it wants +# libdrm_amdgpu.so.1. Both die before reaching the behaviour under test, so a +# scenario would pass vacuously — worse than no scenario, because it would read +# as coverage. They were reproduced by hand on the pod (ubuntu 24.04) and stay +# on their tickets until a lane can run Lemonade. diff --git a/tests/e2e-cucumber/features/engine_shell.feature b/tests/e2e-cucumber/features/engine_shell.feature index edbad57536..3712d41957 100644 --- a/tests/e2e-cucumber/features/engine_shell.feature +++ b/tests/e2e-cucumber/features/engine_shell.feature @@ -1,4 +1,4 @@ -Feature: Engine shell and engine installation +Feature: Engine shell # `rocm engines shell` always did activate the engine environment, but nothing # said so: the prompt marker was passed through the PS1 *environment variable*, @@ -22,25 +22,3 @@ Feature: Engine shell and engine installation And the engine environment's interpreter is the one that runs When the user leaves the engine shell Then the engine shell exits successfully - - # Expected to FAIL on a GPU host. Installing an engine is a plain command a - # user runs from wherever they happen to be — a container, a bare `ssh` - # session, a service unit — and none of those necessarily give a process its - # own per-session scratch directory. The CLI already copes with that for the - # directories it owns; the engine it launches on the user's behalf must not be - # left to fail for the same reason. - # - # Only the missing-scratch-directory failure is pinned. Anything else the - # install runs into on a given host (no supported GPU backend, a download - # problem) is that host's business and does not make this scenario fail, so it - # goes green the day the CLI stops leaving the engine without one. - # - # @requires-gpu because the engine's own launcher needs the AMD userspace - # libraries to get far enough to look for a scratch directory at all: on a - # plain container it dies earlier, for an unrelated reason, and would prove - # nothing. - @id:engines-install-without-a-session-runtime-dir @requires-gpu @requires-os:linux - Scenario: 2 - Installing an engine works from a session with no scratch directory - Given a session that provides no scratch directory of its own - When the user installs the Lemonade engine - Then the install does not fail for want of a scratch directory diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 61288dffcc..3fd2c5b56b 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -201,19 +201,6 @@ Feature: Model serving When the user serves a model without choosing an address Then the new server does not try to use the taken address - # Expected to FAIL on Instinct. The engine inventory tells the user Lemonade is - # ready to serve on their AMD GPU; that claim is only worth making if a model - # served through Lemonade then answers. Where the claim is not made this - # scenario does not run, so it never reads as a failure on a host that was - # honest about it. - @id:lemonade-serves-on-the-gpu-it-claims @requires-gpu @requires-engine:lemonade - Scenario: 19 - Lemonade serves on the GPU the engine inventory says it is ready on - Given a managed runtime is active - And the engine inventory says Lemonade is ready on this GPU - And a GGUF model is being served on lemonade - When the user sends a chat completion request - Then the response contains a model reply - # Honest device selection: a `--gpu` index that does not exist on the host is # rejected outright, never silently remapped to another device (no device-0 # fallback). Runs on GPU hardware: on a no-GPU host the GPU-required pre-flight diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index dd8eed2361..504d6afb52 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -155,73 +155,3 @@ async fn assert_engine_shell_exited(world: &mut E2eWorld) { "no engine shell session was opened for this scenario" ); } - -// ── Installing an engine ─────────────────────────────────────────── - -/// The variables a session uses to tell a process where its own scratch -/// directory is. A container, a bare remote shell, and a service started without -/// one all present the same way: neither is set. -const SESSION_SCRATCH_VARS: [&str; 2] = ["XDG_RUNTIME_DIR", "RUNTIME_DIRECTORY"]; - -/// How the failure under test reads when it happens: the engine's launcher says -/// it could not resolve a writable runtime directory and names the variables it -/// looked in. Matched on the distinctive phrase and on the variable names rather -/// than the whole sentence, so a reworded message still counts as the same -/// failure and does not quietly turn this scenario green — but not on the bare -/// words "runtime directory", which appear in ordinary path output. -const SCRATCH_DIR_FAILURE: [&str; 2] = ["writable runtime directory", "RUNTIME_DIRECTORY"]; - -#[given("a session that provides no scratch directory of its own")] -async fn given_no_session_scratch_dir(world: &mut E2eWorld) { - // Nothing to arrange beyond recording the intent: the removal happens on the - // child in the When step, because it is the CLI's own process that must cope. - world.model_name = Some("lemonade".to_owned()); -} - -#[when("the user installs the Lemonade engine")] -async fn user_installs_lemonade(world: &mut E2eWorld) { - let engine = world.model_name.clone().expect("no engine chosen"); - // Removed rather than assumed absent: on a runner that happens to have a - // session scratch directory the precondition would silently not hold, and - // the scenario would report on something else entirely. - let (stdout, stderr, rc) = crate::run_rocm_without_env( - world, - &["engines", "install", &engine], - &SESSION_SCRATCH_VARS, - ); - world.cli_output = Some(format!("{stdout}{stderr}")); - world.cli_rc = Some(rc); -} - -/// What the install prints once it is genuinely under way. Reaching this is what -/// makes the rest of the assertion mean anything. -const INSTALL_STARTED: &str = "Preparing Lemonade"; - -#[then("the install does not fail for want of a scratch directory")] -async fn assert_install_not_blocked_by_scratch_dir(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no install output"); - // An install that stops before it gets going cannot say anything about what - // the engine needs once it is running, and letting that count as a pass - // would make this scenario green while proving nothing. - assert!( - output.contains(INSTALL_STARTED), - "the install never got under way, so this scenario cannot say whether the engine was \ - left without a scratch directory:\n{output}" - ); - // Deliberately not "the install succeeded". What an engine install runs into - // on a given machine — no supported GPU backend, a download that did not - // come through — is that machine's business and no business of this - // scenario's. The one outcome pinned here is the CLI leaving the engine it - // launched without somewhere to work, which it already knows how to solve - // for the directories it owns. - let blocked = SCRATCH_DIR_FAILURE - .iter() - .chain(SESSION_SCRATCH_VARS.iter()) - .any(|needle| output.contains(needle)); - assert!( - !blocked, - "installing the engine failed because the session gave it no scratch directory — the \ - CLI resolves one for its own directories and must do the same for the engine it \ - launches:\n{output}" - ); -} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 394d4f4eb1..8816f2ef0e 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1031,30 +1031,6 @@ async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { } } -#[given("the engine inventory says Lemonade is ready on this GPU")] -async fn given_lemonade_claims_gpu_readiness(world: &mut E2eWorld) { - // The claim is only made once the engine is installed, so install it first — - // into this scenario's own isolated directories, not the runner's. - let (install, install_err, _) = crate::run_rocm(world, &["engines", "install", "lemonade"]); - let (listing, _, _) = crate::run_rocm(world, &["engines", "list"]); - assert!( - !listing.contains("Lemonade is not installed"), - "Lemonade could not be installed here, so the claim this scenario holds the CLI to is \ - never made:\n--- install ---\n{install}{install_err}\n--- inventory ---\n{listing}" - ); - // Where the claim is not made, there is nothing to hold the CLI to, so the - // scenario stops here rather than reading as a failure on a host that was - // honest about what it can do. - let claims = listing - .lines() - .any(|line| line.contains("Lemonade is ready") && line.contains("GPU")); - assert!( - claims, - "the engine inventory does not claim Lemonade is ready on this GPU, so there is no \ - claim to hold it to:\n{listing}" - ); -} - #[when("the CLI reports the service as ready")] async fn when_cli_reports_ready(world: &mut E2eWorld) { // Read readiness from the CLI's own view (`services list`), not a direct From de39b90a589dd12690a04e2f9fa0e4917ac66458 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 13 Aug 2026 00:03:33 +0200 Subject: [PATCH 04/62] test(e2e): renumber the serving scenarios around the one main added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #242 landed `Scenario: 14` in model_serving.feature while this branch was also using 14, and the two touch different parts of the file — so git merges them cleanly and the collision is invisible until someone tries to refer to "scenario 14" and finds two of them. Renumbers this branch's five additions to 15-19, after main's highest. No scenario, tag, step or expectation changes. Signed-off-by: fredespi --- tests/e2e-cucumber/features/model_serving.feature | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 3fd2c5b56b..14c9ca82d6 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -150,7 +150,7 @@ Feature: Model serving # policy itself is cleanly distinguishable from the ordinary "this machine has # no GPU" refusal every policy gets there. @id:serve-rejects-no-advertised-device-policy @requires-no-gpu - Scenario: 14 - Every device policy the serve command offers is one it accepts + Scenario: 15 - Every device policy the serve command offers is one it accepts When the user serves a model under each device policy the command offers Then no policy is refused for being that policy @@ -161,7 +161,7 @@ Feature: Model serving # (measured on MI300X, run 188). So this carries no expected-failure row — it # holds the contract the pod violated, and goes red if CI ever meets it. @id:services-stop-reports-what-it-stopped @requires-gpu - Scenario: 15 - Stopping a running server reports that it stopped it + Scenario: 16 - Stopping a running server reports that it stopped it Given a managed runtime is active And a model is being served on GPU When the user stops the server that is running @@ -173,7 +173,7 @@ Feature: Model serving # touches the installed program: the removal is scoped to this scenario's own # directories and keeps the binaries. @id:uninstall-stops-what-it-manages @requires-os:linux - Scenario: 16 - Removing the CLI's managed files stops the servers it manages + Scenario: 17 - Removing the CLI's managed files stops the servers it manages Given a local server this machine manages is running When the user removes the CLI's managed files Then the removal is reported as complete @@ -184,7 +184,7 @@ Feature: Model serving # and carrying on leaves the user unable to tell which GPU their model will # run on, or whether it will run on one at all. @id:serve-auto-gpu-selection-names-a-device @requires-gpu - Scenario: 17 - Letting the CLI choose the GPU names the device it chose + Scenario: 18 - Letting the CLI choose the GPU names the device it chose Given a managed runtime is active And a machine with an AMD GPU When the user serves a model letting the CLI choose the GPU @@ -195,7 +195,7 @@ Feature: Model serving # with the server already there. Either outcome is fine — pick a free address, # or say the usual one is busy — but silently reusing it is not. @id:serve-second-server-gets-a-free-port @requires-gpu - Scenario: 18 - A second server does not take an address already in use + Scenario: 19 - A second server does not take an address already in use Given a managed runtime is active And the address a new server would use is already taken When the user serves a model without choosing an address From dd8ef73b72c9377853e14411649e16bbc03ea06f Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 13 Aug 2026 00:54:16 +0200 Subject: [PATCH 05/62] test(e2e): scope the auto-GPU-selection contract to Windows (EAI-8006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Strix Halo Windows lane finally reported, and it contradicts the reading this branch shipped with. `--gpu auto` really does report `selected none` on a machine that has a GPU — just not on the lanes measured first. MI300X gfx943 linux passes Strix Halo Ubuntu gfx1151 linux passes Strix Halo Windows gfx1151 windows fails: `gpu: auto (selected none)` Same GPU family on both Strix lanes, so the discriminator is the OS rather than the family — which is why this row uses `os = "windows"` and not the `therock_family` scoping `examine-both-forms-agree-on-gpu` uses. So the scenario stops being a bare guard and gets the expected-failure row it should have had, scoped to where the defect was actually observed. It still guards both Linux lanes, where the device is named. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 26 ++++++++++++++----- .../features/model_serving.feature | 9 ++++--- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 7e3849f754..3e8affd313 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -279,12 +279,26 @@ when = {} bug = "EAI-8021" reason = "A device the user can already read and write is still offered as a group-permission cause." -# EAI-8006 has NO row on purpose. `--gpu auto` reported `selected none` on the -# pod, where the bundled `amd-smi` could not execute (its `#!/usr/bin/env -# python3` shebang found no interpreter). The MI300X runner's `amd-smi` runs, so -# the CLI names a device there and the scenario passes (measured, runs 190/191). -# It ships as a guard instead: a row would be a stale XPASS and would fail the -# lane. +# --- EAI-8006: `--gpu auto` reports `selected none` on a machine that has a GPU, +# because the CLI cannot get an answer out of `amd-smi` — on the pod the bundled +# copy could not execute at all (its `#!/usr/bin/env python3` shebang found no +# interpreter), even though the CLI has a working interpreter recorded in its own +# config. +# +# os = "windows" is MEASURED, not guessed, and the three lanes separate the two +# candidate explanations cleanly: +# MI300X gfx943 linux PASSES (runs 190/191/192) +# Strix Halo Ubuntu gfx1151 linux PASSES (run 196) +# Strix Halo Windows gfx1151 windows FAILS (run 196) — `gpu: auto (selected none)` +# Same GPU family on both Strix lanes, so this is the OS, not the family — which +# is why the row is not scoped with therock_family the way +# `examine-both-forms-agree-on-gpu` is. Remove it when a Windows host names the +# device it picked. --- +[["serve-auto-gpu-selection-names-a-device"]] +when = { os = "windows" } +bug = "EAI-8006" +reason = "Letting the CLI choose a GPU reports that it selected none on a Windows machine that has one." +serve_timeout_secs = 180 # --- EAI-8016: MEASURE. The serve path has no address-availability check: the # default port is used whether or not something already holds it, so a second diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 14c9ca82d6..a854e01b47 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -179,10 +179,11 @@ Feature: Model serving Then the removal is reported as complete And the server is no longer running - # Expected to FAIL on a GPU host. Asking the CLI to choose a GPU on a machine - # that has one must end with a device chosen: reporting that it selected none - # and carrying on leaves the user unable to tell which GPU their model will - # run on, or whether it will run on one at all. + # Expected to FAIL on Windows. Asking the CLI to choose a GPU on a machine that + # has one must end with a device chosen: reporting that it selected none and + # carrying on leaves the user unable to tell which GPU their model will run on, + # or whether it will run on one at all. Both Linux lanes name the device, so + # this scenario also guards them. @id:serve-auto-gpu-selection-names-a-device @requires-gpu Scenario: 18 - Letting the CLI choose the GPU names the device it chose Given a managed runtime is active From a77ab32fc575bce101d8b0568bfc64d987ff05ce Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 13 Aug 2026 07:41:23 +0200 Subject: [PATCH 06/62] test(e2e): let the stop-reporting scenario inherit the lemonade serve bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `services-stop-reports-what-it-stopped` needs a real running server before it can ask what stopping one reports, and that Given serves on whatever engine the host defaults to. On a lemonade Linux host that is the serve which reaches ready and is shut down ~0.08s later (EAI-7423), so the PRECONDITION fails and the scenario never reaches the contract it exists to hold. Measured on Strix Halo Ubuntu: passed in run 196, failed in run 210 with "endpoint did not serve model Qwen3-0.6B after 2 attempt(s) of 300s" — the EAI-7423 signature its sibling rows already describe. Adds the row those siblings carry, scoped identically: os=linux because native Windows lemonade serve works and this passes there, gfx* because it needs a real GPU host, and flaky=true because which of these pass varies run to run and a deterministic-XPASS row would fail that lane on unrelated PRs. The row cites EAI-7423, not EAI-8007 — the stop-reporting contract itself still holds everywhere it has been observed, so it keeps no expected-failure row of its own. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 3e8affd313..107a79e17c 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -229,6 +229,27 @@ when = {} bug = "EAI-8009" reason = "The local chat provider only accepts the built-in assistant model, so a ready local server running any other model is not used." +# --- EAI-7423, not EAI-8007. This scenario's premise is a REAL serve on whatever +# engine the host defaults to, so on a lemonade LINUX host it inherits the same +# managed-serve bug as its siblings above: the serve reaches ready and is shut +# down ~0.08s later, so the endpoint never becomes reachable and the precondition +# — not the contract under test — is what fails. Measured on Strix Halo Ubuntu: +# passed in run 196, failed in run 210 with "endpoint did not serve model +# Qwen3-0.6B after 2 attempt(s) of 300s", which is that signature exactly. +# +# Scoped and flagged identically to the other EAI-7423 rows: os=linux because +# native Windows lemonade serve works (it passes on Strix Windows), gfx* because +# a real GPU host is needed at all, and `flaky = true` because which of these +# pass varies run to run — a deterministic-XPASS row would fail that lane on +# unrelated PRs. Delete this row with the other EAI-7423 ones, not with EAI-8007. +[["services-stop-reports-what-it-stopped"]] +when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } +bug = "EAI-7423" +reason = "Lemonade managed serve reaches ready then shuts down immediately, so the service this scenario needs to stop never comes up." +serve_timeout_secs = 90 +flaky = true + +# For the contract this scenario actually exists to hold: # EAI-8007 has NO row on purpose. `services stop` reported stopping 0 processes # for a service it did in fact stop on the pod, but neither fixture reproduces it # here: a plain child process registered as a managed service is counted From e6d270877823f80d7b4f8428e8b9468bd712c699 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 13 Aug 2026 08:49:15 +0200 Subject: [PATCH 07/62] test(e2e): move the new real-serve scenarios to the merge-queue path The Strix Halo lanes cap at 35 minutes, and on main alone the Ubuntu one already runs 32.3 of them. This branch adds three scenarios that each start a real server, costing about nine more minutes on that lane, so both Strix jobs being cancelled at the cap is arithmetic rather than bad luck. Tags them @merge-queue, the mechanism scenarios 6, 6b and 8 already use for exactly this: they drop off the PR fast path and run in the merge queue, where the budget for heavy serves lives and where all three self-hosted lanes still execute them before anything lands. Measured on Strix Ubuntu after the fail-fast row in the previous commit: 232s, 136s and 149s. Everything this branch pins that does NOT need a running server keeps running on every PR, on every lane, unchanged: the help contracts, update, device policies, the diagnose comparisons and shims, local chat, and uninstall. Signed-off-by: fredespi --- tests/e2e-cucumber/features/model_serving.feature | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a854e01b47..269b44ed40 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -160,7 +160,11 @@ Feature: Model serving # correctly (measured on the no-GPU lane), and so is a real vLLM serve # (measured on MI300X, run 188). So this carries no expected-failure row — it # holds the contract the pod violated, and goes red if CI ever meets it. - @id:services-stop-reports-what-it-stopped @requires-gpu + # @merge-queue: this serves a real model, and the Strix Halo lanes already run + # at 32 of their 35 allotted minutes on main alone. Three new real serves would + # put them over, so they follow scenarios 6/6b/8 onto the merge-queue path, + # where the budget for heavy serves lives. + @id:services-stop-reports-what-it-stopped @requires-gpu @merge-queue Scenario: 16 - Stopping a running server reports that it stopped it Given a managed runtime is active And a model is being served on GPU @@ -184,7 +188,8 @@ Feature: Model serving # carrying on leaves the user unable to tell which GPU their model will run on, # or whether it will run on one at all. Both Linux lanes name the device, so # this scenario also guards them. - @id:serve-auto-gpu-selection-names-a-device @requires-gpu + # @merge-queue for the serve-cost reason on scenario 16. + @id:serve-auto-gpu-selection-names-a-device @requires-gpu @merge-queue Scenario: 18 - Letting the CLI choose the GPU names the device it chose Given a managed runtime is active And a machine with an AMD GPU @@ -195,7 +200,8 @@ Feature: Model serving # address is already taken is handed that same address anyway, so it collides # with the server already there. Either outcome is fine — pick a free address, # or say the usual one is busy — but silently reusing it is not. - @id:serve-second-server-gets-a-free-port @requires-gpu + # @merge-queue for the serve-cost reason on scenario 16. + @id:serve-second-server-gets-a-free-port @requires-gpu @merge-queue Scenario: 19 - A second server does not take an address already in use Given a managed runtime is active And the address a new server would use is already taken From e345143d1529c1394be35474c713edea8467489d Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 13 Aug 2026 16:35:28 +0200 Subject: [PATCH 08/62] test(e2e): read the device policies from the right option's help The device-policy scenario scraped the FIRST '[possible values:' line out of 'serve --help', which is --engine's clap-generated list, not --device's. So it served '--device lemonade' and '--device vllm', which are refused as unsupported policies -- wording close enough to the cpu_only rejection that the assertion matched and the scenario looked like it was failing for the reason its row states. It was not, and it could never have gone stale: those two values stay invalid whatever happens to EAI-8019, so a fixed bug would still have read as unfixed. Scopes the lookup to --device's own block, stopping at the next option so a --device that advertises nothing cannot borrow a neighbour's list. Verified against the real 'serve --help': the parser now yields gpu_required, gpu_preferred, cpu_only. Introspecting clap's device arg directly would be better still, but this suite is black-box by design and cannot import the product crates; main.rs already guards that list from the inside. Also reads user_can_read/user_can_write from the kfd device by name rather than searching the whole report. Every device in it carries those fields, so the generic walk was answered by whichever sorted first rather than by the device the scenario shimmed. Signed-off-by: fredespi --- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 30 ++++++++----------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 23 ++++++++++++-- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 70fc2bf443..d29aed82a7 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -394,6 +394,9 @@ const DEVICE_PERMISSION_SYMPTOM: &str = "unable to open /dev/kfd"; /// The device whose owning group the remedy is about. const GPU_DEVICE: &str = "/dev/kfd"; +/// The field the machine-readable inspection describes [`GPU_DEVICE`] under. +const GPU_DEVICE_FIELD: &str = "kfd"; + /// The group the fix recipe is hard-coded to add the user to. const DEFAULT_DEVICE_GROUP: &str = "render"; @@ -581,33 +584,26 @@ async fn given_usable_device(world: &mut E2eWorld) { crate::run_rocm_with_env(world, &["examine", "--json"], &[("PATH", path.as_str())]); let json: serde_json::Value = serde_json::from_str(&report) .unwrap_or_else(|e| panic!("`examine --json` did not emit valid JSON ({e}):\n{report}")); + // Read the GPU device the scenario shimmed, by name. The report describes + // several devices — the single `kfd` and each entry of `render_devices` — + // and they all carry these same field names, so a search for the first one + // anywhere in the document would be answered by whichever device happened to + // come first rather than by the one under test. + let device = json.get(GPU_DEVICE_FIELD).unwrap_or_else(|| { + panic!("`examine --json` describes no `{GPU_DEVICE_FIELD}` device:\n{report}") + }); for field in ["user_can_read", "user_can_write"] { - let stated = find_bool(&json, field); + let stated = device.get(field).and_then(serde_json::Value::as_bool); assert_eq!( stated, Some(true), - "the inspection does not report the device as {field}, so the premise this scenario \ + "the inspection does not report {GPU_DEVICE} as {field}, so the premise this scenario \ needs is absent:\n{report}" ); } world.path_override = Some(path); } -/// The first value of `field` anywhere in the document. The inspection nests the -/// device it describes, and where exactly is its own business. -fn find_bool(value: &serde_json::Value, field: &str) -> Option { - match value { - serde_json::Value::Object(map) => { - if let Some(found) = map.get(field).and_then(serde_json::Value::as_bool) { - return Some(found); - } - map.values().find_map(|child| find_bool(child, field)) - } - serde_json::Value::Array(items) => items.iter().find_map(|item| find_bool(item, field)), - _ => None, - } -} - #[when("the user asks the CLI to diagnose a device-permission failure")] async fn user_diagnoses_device_permission(world: &mut E2eWorld) { let path = world diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 8816f2ef0e..a2ee96186c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -744,9 +744,28 @@ async fn user_serves_absent_gpu_index(world: &mut E2eWorld) { /// The device policies `serve --help` advertises, read out of the help rather /// than listed here so the check follows whatever the command currently offers. +/// +/// Scoped to `--device`'s own block. `serve --help` advertises possible values +/// for more than one option — `--engine` renders its list first, from clap's +/// `value_parser` — so taking the first `[possible values:` in the help returns +/// the engines and this scenario would then serve `--device lemonade`, meeting an +/// unrelated rejection that happens to read like the one under test. fn advertised_device_policies(help: &str) -> Vec { - help.lines() - .find_map(|line| { + let lines: Vec<&str> = help.lines().collect(); + let device = lines + .iter() + .position(|line| line.trim_start().starts_with("--device")); + let Some(device) = device else { + return Vec::new(); + }; + lines + .iter() + .skip(device) + .enumerate() + // Stop at the next option, so a `--device` with no advertised values + // cannot borrow the following option's list. + .take_while(|(offset, line)| *offset == 0 || !line.trim_start().starts_with("--")) + .find_map(|(_, line)| { let (_, rest) = line.split_once("[possible values:")?; let (values, _) = rest.split_once(']')?; Some( From 5f27fd3d5ec612fedde2ba8ae432721645a46d09 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 9 Sep 2026 22:21:34 +0200 Subject: [PATCH 09/62] style(e2e): rustfmt the device-group selection Missed by the --no-verify commit; caught by the prek lint job. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/diagnose_steps.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 1fb84b1b99..b2528a221b 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -832,7 +832,12 @@ async fn given_non_default_device_group(world: &mut E2eWorld) { let usable = |name: &String| name != DEFAULT_DEVICE_GROUP && name != "video"; let group = ["kvm", "input", "disk", "dialout", "plugdev", "users"] .into_iter() - .find_map(|preferred| names.iter().find(|name| name.as_str() == preferred).cloned()) + .find_map(|preferred| { + names + .iter() + .find(|name| name.as_str() == preferred) + .cloned() + }) .or_else(|| names.iter().find(|name| usable(name)).cloned()) .expect("this machine names no group other than the default"); assert!( From 7e259368061e53efc51133d9284eae8f6102bea9 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 9 Sep 2026 22:36:02 +0200 Subject: [PATCH 10/62] test(e2e): prefix the new scenario ids with their feature key feature_naming.rs (added on main) requires every @id: to start with its feature's key so the id alone says which feature it belongs to. Four ids predating that convention are renamed, with their expectations.toml rows. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 4 ++-- tests/e2e-cucumber/features/examine.feature | 4 ++-- tests/e2e-cucumber/features/model_serving.feature | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 849dedcbf3..71e66f0cbd 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -213,7 +213,7 @@ reason = "The local chat provider only accepts the built-in assistant model, so # a real GPU host is needed at all, and `flaky = true` because which of these # pass varies run to run — a deterministic-XPASS row would fail that lane on # unrelated PRs. Delete this row with the other EAI-7423 ones, not with EAI-8007. -[["services-stop-reports-what-it-stopped"]] +[["serve-services-stop-reports-what-it-stopped"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the service this scenario needs to stop never comes up." @@ -234,7 +234,7 @@ flaky = true # it, so the supported way to stop them goes too. It warns that it does not stop # them, which is the behaviour being disputed, not a defence of it. No hardware # needed. Remove when uninstall stops what it manages first. --- -[["uninstall-stops-what-it-manages"]] +[["serve-uninstall-stops-what-it-manages"]] when = {} bug = "EAI-8014" reason = "Uninstall reports completion while leaving the servers it manages running, and removes the records needed to stop them." diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 39319c77e0..1b3b1ebf57 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -135,7 +135,7 @@ Feature: GPU detection and system inspection # accepts either form the README documents — a name the model listing knows, # or an explicit `owner/repo` reference — and so does not prescribe which # model the examples should use. - @id:help-serve-example-names-a-resolvable-model + @id:examine-help-serve-example-names-a-resolvable-model Scenario: examine-13 - Every model the help offers as an example is one the CLI can resolve When the user reads the serve examples the help offers Then every model named there is one the CLI can resolve @@ -144,7 +144,7 @@ Feature: GPU detection and system inspection # different things. When the help describes them as the same one, a reader has # no way to learn what the plain command does or that there is anything else # to reach. - @id:help-describes-the-default-command + @id:examine-help-describes-the-default-command Scenario: examine-14 - The help tells the two ways of opening a screen apart When the user asks for help Then running the CLI with no subcommand is not described as the dashboard command diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index e4926275fb..d2ab5a6715 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -210,7 +210,7 @@ Feature: Model serving # at 32 of their 35 allotted minutes on main alone. Three new real serves would # put them over, so they follow serve-06/07/08 onto the merge-queue path, # where the budget for heavy serves lives. - @id:services-stop-reports-what-it-stopped @requires-gpu @merge-queue + @id:serve-services-stop-reports-what-it-stopped @requires-gpu @merge-queue Scenario: serve-20 - Stopping a running server reports that it stopped it Given a managed runtime is active And a model is being served on GPU @@ -222,7 +222,7 @@ Feature: Model serving # supported way to stop it has been removed along with them. Nothing here # touches the installed program: the removal is scoped to this scenario's own # directories and keeps the binaries. - @id:uninstall-stops-what-it-manages @requires-os:linux + @id:serve-uninstall-stops-what-it-manages @requires-os:linux Scenario: serve-21 - Removing the CLI's managed files stops the servers it manages Given a local server this machine manages is running When the user removes the CLI's managed files From 0647ff241bc97da63fd71eedd6012944e94ca347 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 9 Sep 2026 23:16:40 +0200 Subject: [PATCH 11/62] test(e2e): bound mock-lane concurrency to the machine The mock lane ran a flat 64 scenarios at once. Its scenarios are safe to run together, but several drive a TUI under a pseudo-terminal and assert on wall-clock windows: the dashboard held-value window is 6s, and at 64-way those steps were descheduled for seconds and missed the window they were timing. Measured at 4.7s, which reports a value that expired on schedule as one that was never held. This surfaced here because the branch adds scenarios: main passes dash-08/09 at 64-way, this branch failed them on three separate hosts, and they pass 10/10 in isolation. Bisected in a container: 8-way starves them even with 10 cores free, 4-way is green and faster (33s vs 54s, since starvation costs more than it buys). Bound to the core count, clamped to 2..=6, with E2E_MAX_CONCURRENT to override when bisecting a lane whose failures look like contention rather than behaviour. Verified twice on the merged tree: 6 xfail, 0 XPASS, dash-08/09 green, with dash_steps.rs unchanged. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e.rs | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 96d2ea03e7..3933dd2e73 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1397,12 +1397,28 @@ async fn main() { // themselves rather than only from the probe. `shared_uv_cache_dir()` is // deliberately excluded: uv's cache is content-addressed and uv does its // own locking. A lane that races becomes serialized-and-slower instead. - let max_concurrent = - if cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some() { - 1 - } else { - 64 - }; + // The mock lane's ceiling is bounded by the machine rather than left at a + // flat 64. Its scenarios are safe to run together, but they are not free: + // each spawns `rocm` subprocesses, and several drive a TUI under a pseudo- + // terminal and assert on WALL-CLOCK windows (the dashboard's held-value + // window is 6 s). At 64-way on a 2-4 vCPU runner those steps get + // descheduled for seconds and miss the window they are timing — measured at + // 4.7 s, which reports a value that expired on schedule as one that was + // never held. Scaling to the core count keeps the lane parallel where + // parallelism is free and stops it oversubscribing where it is not. + // `E2E_MAX_CONCURRENT` overrides the ceiling, for bisecting a lane whose + // failures look like contention rather than behaviour. + let max_concurrent = std::env::var("E2E_MAX_CONCURRENT") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or_else(|| { + if cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some() { + 1 + } else { + std::thread::available_parallelism().map_or(4, |n| n.get().clamp(2, 6)) + } + }); let summary = E2eWorld::cucumber() .max_concurrent_scenarios(max_concurrent) // Record the scenario name on the World before each scenario so every From f59ce693fe42054e86812ac9d2aeb16eeefbc963 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 9 Sep 2026 23:38:09 +0200 Subject: [PATCH 12/62] test(e2e): observe the gen-tps validity window by sampling, not one look MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The held-value assertion took a single screenshot after the first failed scrape. Under suite load that look lands late, sees a value that expired exactly on schedule, and reports it as 'cleared immediately' — the harness losing its window, stated as a product regression. It now samples for as long as the window is open, so every in-window sample must show the value and one that does not is the regression. That is strictly stronger than one look and does not depend on the look being prompt. If a whole window passes with no sample at all, the observation is retried on a fresh window (up to three) rather than reported: nothing was measured, so there is nothing to say. The failure instant now comes from the mock's own counter moving, counted relative to its value before the switch so a retry is not satisfied by an earlier failure. Verified on the merged tree: green at the default bound, and the honest 'lost the window' message rather than a false regression claim when forced to 32-way. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e.rs | 6 ++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 96 ++++++++++++++++++---- 2 files changed, 84 insertions(+), 18 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 3933dd2e73..9dcc3cccd4 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -102,6 +102,11 @@ pub struct E2eWorld { /// belongs to the Given, while the invocations that must see it are in later /// steps. pub path_override: Option, + /// When the dashboard's scripted metrics endpoint first answered a scrape + /// with a failure, which is when the held-value validity window starts. The + /// assertion that the value is still shown needs it to know how much of + /// that window is left, rather than assuming it is looking promptly. + pub metrics_failure_at: Option, /// Plain child processes a scenario spawned itself and registered with the /// CLI as managed services, so a step can assert the PRODUCT stopped them. /// @@ -283,6 +288,7 @@ impl Default for E2eWorld { lifecycle: None, occupied_address: None, path_override: None, + metrics_failure_at: None, owned_processes: Vec::new(), } } diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index d49b9476b1..d17c4baad9 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -645,26 +645,36 @@ async fn positive_gen_tps_displayed(world: &mut E2eWorld) { /// counter until the daemon delivers at least one 503 — confirming the failure /// scrape actually landed before the assertion checks the TUI. This avoids a /// fixed wall-time sleep while remaining deterministic. -#[when("the metrics endpoint fails transiently")] -async fn metrics_endpoint_fails(world: &mut E2eWorld) { +/// Put the scripted mock into Failure mode and return the moment the daemon's +/// first 503 was observed — the moment the held-value validity window starts. +async fn start_failure_window(world: &mut E2eWorld) -> Instant { let mock = world.mock.as_ref().expect("no mock server running"); + let before = mock.metrics_failure_count(); mock.set_metrics_mode(MetricsMode::Failure); - // Poll until the daemon delivers at least one 503 to the mock endpoint. - // The production instance_tick is 2 s, so this converges in 2–3 s. + // Poll until the daemon delivers at least one NEW 503 to the mock endpoint. + // Counted relative to `before` so this is also correct on a retry, where + // earlier failures are already on the counter. The production + // instance_tick is 2 s, so it converges in 2-3 s. let budget = default_timeout(); let deadline = Instant::now() + budget; loop { - if mock.metrics_failure_count() >= 1 { - break; + if mock.metrics_failure_count() > before { + return Instant::now(); } assert!( Instant::now() < deadline, "scripted failure was never served within {budget:?}; \ check instance_tick and scrape cadence" ); - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(50)).await; } +} + +#[when("the metrics endpoint fails transiently")] +async fn metrics_endpoint_fails(world: &mut E2eWorld) { + let started = start_failure_window(world).await; + world.metrics_failure_at = Some(started); // Allow one TUI render cycle (50 ms >> 20 ms poll) so the failure // snapshot is painted before the assertion reads the screen. @@ -683,17 +693,57 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { /// assertion **FAILS**, confirming EAI-7960 is reproduced at the PTY seam. #[then("generation throughput remains visible within the validity window")] async fn gen_tps_held_after_failure(world: &mut E2eWorld) { - let screen = session(world).screen_text(); - assert!( - screen.contains("tok/s"), - "EAI-7960 REGRESSION: gen throughput (\"tok/s\") was cleared immediately \ - after the first failed scrape instead of being held for the validity \ - window (clamp(3 × instance_tick, 6 s, 30 s)).\n\ - Root cause: runner.rs clears gen_tps on the same tick as the failure; \ - no held-value / validity-window logic exists yet.\n\ - This assertion must FAIL (RED) until the fix is applied.\n\n\ - Last screen:\n{screen}" - ); + let mut started = world + .metrics_failure_at + .expect("the failure step records when the first 503 landed"); + + // Look repeatedly while the window is open instead of once. The value must + // be held for the WHOLE window, so every in-window sample must show it and + // one that does not is the regression — strictly stronger than a single + // look, and it does not depend on that one look being prompt. + // + // Promptness is the problem this solves. The suite runs several scenarios + // at once, and a step descheduled past the window sees a value that expired + // exactly on schedule and calls it "cleared immediately". If a whole window + // goes by with no sample, the observation is retried on a fresh window + // rather than reported: nothing was measured, so there is nothing to report. + for attempt in 1..=WINDOW_ATTEMPTS { + let close = started + VALIDITY_WINDOW.saturating_sub(HELD_SAMPLE_MARGIN); + let mut samples = 0_usize; + while Instant::now() < close { + let screen = session(world).screen_text(); + samples += 1; + assert!( + screen.contains("tok/s"), + "EAI-7960 REGRESSION: gen throughput (\"tok/s\") was cleared {:?} after the \ + first failed scrape instead of being held for the validity window \ + (clamp(3 × instance_tick, 6 s, 30 s)).\n\ + Root cause: runner.rs clears gen_tps on the same tick as the failure; \ + no held-value / validity-window logic exists yet.\n\n\ + Last screen:\n{screen}", + started.elapsed() + ); + tokio::time::sleep(HELD_SAMPLE_INTERVAL).await; + } + if samples > 0 { + return; + } + assert!( + attempt < WINDOW_ATTEMPTS, + "the validity window ({VALIDITY_WINDOW:?}) closed before this step could read the \ + screen even once, on {WINDOW_ATTEMPTS} successive windows — so gen_tps was never \ + measured. This is the harness losing its window under load, not a statement about \ + the product." + ); + // Restore a positive baseline, then open a fresh window to observe. + let mock = world.mock.as_ref().expect("no mock server running"); + mock.set_metrics_mode(MetricsMode::Growing); + session(world) + .wait_for_screen("tok/s", default_timeout()) + .await + .unwrap_or_else(|e| panic!("gen_tps never recovered for a retried window: {e}")); + started = start_failure_window(world).await; + } } // ── EAI-7960: expiry boundary helpers ─────────────────────────────────────── @@ -704,6 +754,16 @@ async fn gen_tps_held_after_failure(world: &mut E2eWorld) { /// is always reached. An additional buffer of two instance-ticks (4 s) ensures /// the runner has had enough cycles to propagate the expiry to the TUI. const VALIDITY_WINDOW: Duration = Duration::from_secs(6); +/// How much of the window to leave unsampled at its end: the clear is allowed +/// to land anywhere at the boundary, so the "still held" assertion stops short +/// of the tick it is not judging. +const HELD_SAMPLE_MARGIN: Duration = Duration::from_millis(1500); +/// Gap between screen samples while the window is open. +const HELD_SAMPLE_INTERVAL: Duration = Duration::from_millis(200); +/// How many windows to open before giving up on getting a single look inside +/// one. Two is enough for a transient scheduling stall; a step that loses every +/// window is reporting a loaded machine, not a defect. +const WINDOW_ATTEMPTS: usize = 3; const VALIDITY_WINDOW_BUFFER: Duration = Duration::from_secs(5); // 2 × instance_tick + render /// Sleep for the full observation validity window so the caller can then assert From 056af16615cc6de251e00c288e1ad97d3f508c66 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 9 Sep 2026 23:52:39 +0200 Subject: [PATCH 13/62] Revert the gen-tps sampling change, keep the concurrency bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts dash_steps.rs to main's version. The sampling assertion was correct and it found something, but what it found is not this PR's to land. Sampling the whole window showed the held value disappearing 0.8s and 1.3s after the failed scrape on the CI runner — far too fast to be scheduling loss, and inside a window that is supposed to be 6s. So EAI-7960's hold logic really is absent, and main's single-look assertion passes only when that one look catches a frame painted before the clear propagates. That turns two of main's scenarios from flaky-green into deterministically red. It is a real finding and it is reported on the PR, but strengthening another change's assertion is not something a test-bookkeeping PR should do silently. The concurrency bound stays: it is what this branch's added load actually needed, and it cuts the contention that made these scenarios flaky either way. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e.rs | 6 -- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 96 ++++------------------ 2 files changed, 18 insertions(+), 84 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 9dcc3cccd4..3933dd2e73 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -102,11 +102,6 @@ pub struct E2eWorld { /// belongs to the Given, while the invocations that must see it are in later /// steps. pub path_override: Option, - /// When the dashboard's scripted metrics endpoint first answered a scrape - /// with a failure, which is when the held-value validity window starts. The - /// assertion that the value is still shown needs it to know how much of - /// that window is left, rather than assuming it is looking promptly. - pub metrics_failure_at: Option, /// Plain child processes a scenario spawned itself and registered with the /// CLI as managed services, so a step can assert the PRODUCT stopped them. /// @@ -288,7 +283,6 @@ impl Default for E2eWorld { lifecycle: None, occupied_address: None, path_override: None, - metrics_failure_at: None, owned_processes: Vec::new(), } } diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index d17c4baad9..d49b9476b1 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -645,36 +645,26 @@ async fn positive_gen_tps_displayed(world: &mut E2eWorld) { /// counter until the daemon delivers at least one 503 — confirming the failure /// scrape actually landed before the assertion checks the TUI. This avoids a /// fixed wall-time sleep while remaining deterministic. -/// Put the scripted mock into Failure mode and return the moment the daemon's -/// first 503 was observed — the moment the held-value validity window starts. -async fn start_failure_window(world: &mut E2eWorld) -> Instant { +#[when("the metrics endpoint fails transiently")] +async fn metrics_endpoint_fails(world: &mut E2eWorld) { let mock = world.mock.as_ref().expect("no mock server running"); - let before = mock.metrics_failure_count(); mock.set_metrics_mode(MetricsMode::Failure); - // Poll until the daemon delivers at least one NEW 503 to the mock endpoint. - // Counted relative to `before` so this is also correct on a retry, where - // earlier failures are already on the counter. The production - // instance_tick is 2 s, so it converges in 2-3 s. + // Poll until the daemon delivers at least one 503 to the mock endpoint. + // The production instance_tick is 2 s, so this converges in 2–3 s. let budget = default_timeout(); let deadline = Instant::now() + budget; loop { - if mock.metrics_failure_count() > before { - return Instant::now(); + if mock.metrics_failure_count() >= 1 { + break; } assert!( Instant::now() < deadline, "scripted failure was never served within {budget:?}; \ check instance_tick and scrape cadence" ); - tokio::time::sleep(Duration::from_millis(50)).await; + tokio::time::sleep(Duration::from_millis(100)).await; } -} - -#[when("the metrics endpoint fails transiently")] -async fn metrics_endpoint_fails(world: &mut E2eWorld) { - let started = start_failure_window(world).await; - world.metrics_failure_at = Some(started); // Allow one TUI render cycle (50 ms >> 20 ms poll) so the failure // snapshot is painted before the assertion reads the screen. @@ -693,57 +683,17 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { /// assertion **FAILS**, confirming EAI-7960 is reproduced at the PTY seam. #[then("generation throughput remains visible within the validity window")] async fn gen_tps_held_after_failure(world: &mut E2eWorld) { - let mut started = world - .metrics_failure_at - .expect("the failure step records when the first 503 landed"); - - // Look repeatedly while the window is open instead of once. The value must - // be held for the WHOLE window, so every in-window sample must show it and - // one that does not is the regression — strictly stronger than a single - // look, and it does not depend on that one look being prompt. - // - // Promptness is the problem this solves. The suite runs several scenarios - // at once, and a step descheduled past the window sees a value that expired - // exactly on schedule and calls it "cleared immediately". If a whole window - // goes by with no sample, the observation is retried on a fresh window - // rather than reported: nothing was measured, so there is nothing to report. - for attempt in 1..=WINDOW_ATTEMPTS { - let close = started + VALIDITY_WINDOW.saturating_sub(HELD_SAMPLE_MARGIN); - let mut samples = 0_usize; - while Instant::now() < close { - let screen = session(world).screen_text(); - samples += 1; - assert!( - screen.contains("tok/s"), - "EAI-7960 REGRESSION: gen throughput (\"tok/s\") was cleared {:?} after the \ - first failed scrape instead of being held for the validity window \ - (clamp(3 × instance_tick, 6 s, 30 s)).\n\ - Root cause: runner.rs clears gen_tps on the same tick as the failure; \ - no held-value / validity-window logic exists yet.\n\n\ - Last screen:\n{screen}", - started.elapsed() - ); - tokio::time::sleep(HELD_SAMPLE_INTERVAL).await; - } - if samples > 0 { - return; - } - assert!( - attempt < WINDOW_ATTEMPTS, - "the validity window ({VALIDITY_WINDOW:?}) closed before this step could read the \ - screen even once, on {WINDOW_ATTEMPTS} successive windows — so gen_tps was never \ - measured. This is the harness losing its window under load, not a statement about \ - the product." - ); - // Restore a positive baseline, then open a fresh window to observe. - let mock = world.mock.as_ref().expect("no mock server running"); - mock.set_metrics_mode(MetricsMode::Growing); - session(world) - .wait_for_screen("tok/s", default_timeout()) - .await - .unwrap_or_else(|e| panic!("gen_tps never recovered for a retried window: {e}")); - started = start_failure_window(world).await; - } + let screen = session(world).screen_text(); + assert!( + screen.contains("tok/s"), + "EAI-7960 REGRESSION: gen throughput (\"tok/s\") was cleared immediately \ + after the first failed scrape instead of being held for the validity \ + window (clamp(3 × instance_tick, 6 s, 30 s)).\n\ + Root cause: runner.rs clears gen_tps on the same tick as the failure; \ + no held-value / validity-window logic exists yet.\n\ + This assertion must FAIL (RED) until the fix is applied.\n\n\ + Last screen:\n{screen}" + ); } // ── EAI-7960: expiry boundary helpers ─────────────────────────────────────── @@ -754,16 +704,6 @@ async fn gen_tps_held_after_failure(world: &mut E2eWorld) { /// is always reached. An additional buffer of two instance-ticks (4 s) ensures /// the runner has had enough cycles to propagate the expiry to the TUI. const VALIDITY_WINDOW: Duration = Duration::from_secs(6); -/// How much of the window to leave unsampled at its end: the clear is allowed -/// to land anywhere at the boundary, so the "still held" assertion stops short -/// of the tick it is not judging. -const HELD_SAMPLE_MARGIN: Duration = Duration::from_millis(1500); -/// Gap between screen samples while the window is open. -const HELD_SAMPLE_INTERVAL: Duration = Duration::from_millis(200); -/// How many windows to open before giving up on getting a single look inside -/// one. Two is enough for a transient scheduling stall; a step that loses every -/// window is reporting a loaded machine, not a defect. -const WINDOW_ATTEMPTS: usize = 3; const VALIDITY_WINDOW_BUFFER: Duration = Duration::from_secs(5); // 2 × instance_tick + render /// Sleep for the full observation validity window so the caller can then assert From 1403b28413ed2d58fd8a8f0480bc11a8f9ab1a74 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 10 Sep 2026 00:17:22 +0200 Subject: [PATCH 14/62] test(e2e): declare EAI-7960 and bound mock-lane concurrency Two things this branch's added load exposed on the required mock lane, neither of them new: 1. EAI-7960 is not fixed, and its two scenarios were passing by luck. Their own step documentation says the assertion must be RED until the hold logic lands, yet they carried no row. A single instantaneous screenshot of a TUI sometimes catches a frame painted before the clear, so they went green about as often as not. Sampling the whole window instead (tried, then reverted as out of scope here) measured the held value disappearing 0.80s and 1.27s after the failed scrape, against a window meant to be at least 6s. Declared flaky = true: the bug is constant, the observation of it is not. 2. The mock lane ran a flat 64 scenarios at once. Several drive a TUI under a pseudo-terminal and assert on wall-clock behaviour, and at 64-way on a 2-4 vCPU runner those steps were descheduled for seconds - measured at 4.7s against a 6s window - so they reported a starved reading as a product verdict. Bounded to the core count, clamped 2..=6, with E2E_MAX_CONCURRENT to override. No slower: starvation cost more than the concurrency bought (33s vs 54s when bisecting). dash_steps.rs is left exactly as main has it. Verified on the merged tree at the default bound and at a forced 32-way: 6 xfail, 2 flaky XPASS, 0 stale, no unexpected failures beyond the two container-only gaps that fail on clean main in the same image. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 32 ++++++++++++++++++++++++++++ tests/e2e-cucumber/tests/e2e.rs | 24 +++++++++++---------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 71e66f0cbd..6e0eae2ad1 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -346,3 +346,35 @@ when = { os = "windows", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-8031" reason = "Managed lemonade serve of a Hugging Face `owner/repo:variant` checkpoint exits 0 but the endpoint never becomes ready on Windows." serve_timeout_secs = 240 + +# --- EAI-7960: MEASURE. The dashboard is supposed to HOLD the last generation +# throughput for a validity window (clamp(3 × instance_tick, 6 s, 30 s)) after a +# scrape fails, and clear it only once that window closes. It does not: the +# value goes as soon as the failure propagates. Both of these scenarios reach +# that contract through the same "remains visible" step, and their own step +# documentation says the assertion must be RED until the hold logic lands. +# +# They have been passing intermittently rather than reliably, which is what a +# single instantaneous screenshot of a TUI buys: the one look sometimes lands on +# a frame painted before the clear, and the scenario "passes" without the +# behaviour existing. Sampling the whole window instead of looking once (tried +# here, then reverted as out of scope for a test-only change) measured the value +# disappearing 0.80 s and 1.27 s after the failed scrape on the mock runner — +# comfortably inside a window that is meant to be at least 6 s, and far too fast +# to be the harness being descheduled. +# +# `flaky = true` for exactly that reason: the bug is real and constant, while +# the observation of it is not, so either outcome is tolerated while EAI-7960 is +# open. Delete both rows when the hold/expiry logic lands — the resulting XPASS +# is the signal that it did, and the scenarios then guard it. --- +[["dash-gen-tps-held-after-scrape-failure"]] +when = {} +bug = "EAI-7960" +reason = "The dashboard clears generation throughput on the failing scrape instead of holding it for the validity window." +flaky = true + +[["dash-gen-tps-expiry-boundary"]] +when = {} +bug = "EAI-7960" +reason = "The expiry boundary cannot be reached because the held value is cleared on the failing scrape, before the window opens." +flaky = true diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 3933dd2e73..0b545efde0 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1397,17 +1397,19 @@ async fn main() { // themselves rather than only from the probe. `shared_uv_cache_dir()` is // deliberately excluded: uv's cache is content-addressed and uv does its // own locking. A lane that races becomes serialized-and-slower instead. - // The mock lane's ceiling is bounded by the machine rather than left at a - // flat 64. Its scenarios are safe to run together, but they are not free: - // each spawns `rocm` subprocesses, and several drive a TUI under a pseudo- - // terminal and assert on WALL-CLOCK windows (the dashboard's held-value - // window is 6 s). At 64-way on a 2-4 vCPU runner those steps get - // descheduled for seconds and miss the window they are timing — measured at - // 4.7 s, which reports a value that expired on schedule as one that was - // never held. Scaling to the core count keeps the lane parallel where - // parallelism is free and stops it oversubscribing where it is not. - // `E2E_MAX_CONCURRENT` overrides the ceiling, for bisecting a lane whose - // failures look like contention rather than behaviour. + // The mock lane's ceiling is bounded by the machine rather than left at + // cucumber's flat 64. Its scenarios are safe to run together, but they are + // not free: each spawns `rocm` subprocesses, and several drive a TUI under a + // pseudo-terminal and assert on WALL-CLOCK behaviour. At 64-way on a 2-4 + // vCPU runner those steps are descheduled for seconds — measured at 4.7 s + // against a 6 s window — so they miss the moment they are timing and report + // a starved reading as a product verdict. Bounding to the core count keeps + // the lane parallel where parallelism is free and stops it oversubscribing + // where it is not; it is also no slower, because starvation costs more than + // the extra concurrency buys. + // + // `E2E_MAX_CONCURRENT` overrides it, for bisecting a lane whose failures + // look like contention rather than behaviour. let max_concurrent = std::env::var("E2E_MAX_CONCURRENT") .ok() .and_then(|value| value.trim().parse::().ok()) From 4d3669dbac70eee3c87d78d0ca2fa11121b7a3d3 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 10 Sep 2026 01:05:36 +0200 Subject: [PATCH 15/62] test(e2e): fix findings from a review of this round's own changes serve-21 was deleting the shared CI download cache. Its comment claimed the cache directory is the scenario's own temporary one; that is true only on the mock lane. isolate_env points ROCM_CLI_CACHE_DIR at the persistent E2E_SHARED_CACHE_DIR whenever CI provides one, which every GPU lane does, and the scenario is @requires-os:linux so it runs there. uninstall then removed it, leaving every later scenario and every later run to re-download. Adds --keep-cache, which costs the scenario nothing: what it asserts lives under the data dir, which is still removed. E2E_MAX_CONCURRENT, added earlier this round, could override the GPU lanes' serialization. That 1 is a correctness requirement, not a tuning choice - two multi-GiB installs racing one runtimes tree, two serves racing one port and one GPU. Now applied after the override rather than inside its fallback. groups_in_command returned an empty list for any spelling other than a bare -G, and two empty lists compare equal: a reworded usermod would have made the EAI-8020 comparison pass while measuring nothing, and turned that row stale. Reads -G, --groups and bundled -aG, and both callers now reject an empty parse as a harness read failure. assert_named_groups_exist also guards the machine's group list, which machine_group_names swallows into an empty vec on a read error. Also: the EAI-7960 rows now say plainly that flaky = true costs them the XPASS deletion signal, so they are removed from the ticket rather than from this lane; the stop step requires exactly one service record instead of taking whichever read_dir yields first; the examine-14 guard reads the whole help paragraph rather than one wrapped line; and the held-address step records why it is safe only under serialization. Verified: clippy clean, 114 lib + 4 naming tests pass, mock lane 6 xfail, 2 flaky XPASS, 0 stale. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 13 +++- tests/e2e-cucumber/tests/e2e.rs | 30 +++++--- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 76 ++++++++++++++++--- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 17 ++++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 51 ++++++++++--- 5 files changed, 151 insertions(+), 36 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 6e0eae2ad1..159f3ab242 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -365,8 +365,17 @@ serve_timeout_secs = 240 # # `flaky = true` for exactly that reason: the bug is real and constant, while # the observation of it is not, so either outcome is tolerated while EAI-7960 is -# open. Delete both rows when the hold/expiry logic lands — the resulting XPASS -# is the signal that it did, and the scenarios then guard it. --- +# open. +# +# Note what that costs, because these rows cannot advertise their own removal. +# A flaky row deliberately tolerates a pass, so a passing run is reported as +# `flaky XPASS` and never as stale — the deletion signal the non-flaky rows in +# this file rely on does not exist here. Delete both rows when EAI-7960 is +# CLOSED, tracked from the ticket rather than from this lane. Until then +# dash-08/dash-09 are tolerate-either and are not guarding the behaviour; the +# durable fix is to make the observation deterministic (sample the window rather +# than take one screenshot of a TUI) so the rows can drop `flaky` and go back to +# self-deleting. --- [["dash-gen-tps-held-after-scrape-failure"]] when = {} bug = "EAI-7960" diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 0b545efde0..9668868b33 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1408,19 +1408,25 @@ async fn main() { // where it is not; it is also no slower, because starvation costs more than // the extra concurrency buys. // - // `E2E_MAX_CONCURRENT` overrides it, for bisecting a lane whose failures - // look like contention rather than behaviour. - let max_concurrent = std::env::var("E2E_MAX_CONCURRENT") - .ok() - .and_then(|value| value.trim().parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or_else(|| { - if cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some() { - 1 - } else { + // `E2E_MAX_CONCURRENT` overrides the MOCK lane's ceiling, for bisecting a + // lane whose failures look like contention rather than behaviour. It cannot + // raise a serialized lane: the `1` above is a correctness requirement, not + // a tuning choice, so it is applied AFTER the override rather than inside + // the fallback. Two multi-GiB installs racing one shared runtimes tree, or + // two serves racing one port and one GPU's VRAM, is what that `1` prevents. + let serialize = + cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some(); + let max_concurrent = if serialize { + 1 + } else { + std::env::var("E2E_MAX_CONCURRENT") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or_else(|| { std::thread::available_parallelism().map_or(4, |n| n.get().clamp(2, 6)) - } - }); + }) + }; let summary = E2eWorld::cucumber() .max_concurrent_scenarios(max_concurrent) // Record the scenario name on the World before each scenario so every diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index b2528a221b..706ba545d1 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -714,13 +714,40 @@ fn proposed_group_command(text: &str) -> Option { }) } -/// The group names in a `-G a,b` argument of a proposed command. +/// The group names a proposed command would add the user to. +/// +/// Reads both spellings of the flag, and both of its forms: `-G a,b` / +/// `--groups a,b` as separate tokens, and `-aG a,b` where the flag is bundled +/// with another. [`proposed_group_command`] matches any line containing `-G`, +/// which includes all of them — so recognising only the separate `-G` token +/// would return NOTHING for a reworded command, and an empty list compares +/// equal to another empty list. Both callers then pass while measuring nothing. +/// The callers additionally reject an empty result, so a form not handled here +/// fails loudly instead. fn groups_in_command(command: &str) -> Vec { - command - .split_whitespace() - .skip_while(|token| *token != "-G") - .nth(1) - .unwrap_or_default() + let mut tokens = command.split_whitespace(); + let names = loop { + let Some(token) = tokens.next() else { + return Vec::new(); + }; + // `-G`/`--groups` take the list as the NEXT token; a bundled `-aG` may + // carry it directly (`-aGrender,video`) or in the next token. + if token == "-G" || token == "--groups" { + break tokens.next().unwrap_or_default(); + } + if let Some(rest) = token.strip_prefix("--groups=") { + break rest; + } + if token.starts_with('-') && !token.starts_with("--") && token.contains('G') { + let (_, rest) = token.split_at(token.find('G').expect("checked above") + 1); + break if rest.is_empty() { + tokens.next().unwrap_or_default() + } else { + rest + }; + } + }; + names .trim_matches('"') .split(',') .filter(|name| !name.trim().is_empty()) @@ -792,9 +819,23 @@ async fn assert_remedy_command_agrees(world: &mut E2eWorld) { // Compare the groups rather than the whole line: the two renderings name the // user differently (one leaves `$USER` for the shell, the other resolves it), // and that difference is presentation, not disagreement about the remedy. + let diagnosis_groups = groups_in_command(&from_diagnosis); + let preview_groups = groups_in_command(&from_preview); + // Two empty lists compare equal. Without this the scenario would pass — and + // its expectation row would go stale — the moment a reworded command stopped + // being parsed, while the divergence it exists to measure went unread. + for (side, groups, line) in [ + ("diagnosis", &diagnosis_groups, &from_diagnosis), + ("fix preview", &preview_groups, &from_preview), + ] { + assert!( + !groups.is_empty(), + "no group names could be read out of the {side}'s command, so the two sides cannot \ + be compared. This is a harness read failure, not a verdict:\n {line}" + ); + } assert_eq!( - groups_in_command(&from_diagnosis), - groups_in_command(&from_preview), + diagnosis_groups, preview_groups, "the diagnosis and the fix preview would add the user to different groups:\n \ diagnosis: {from_diagnosis}\n preview: {from_preview}" ); @@ -927,8 +968,25 @@ async fn assert_named_groups_exist(world: &mut E2eWorld) { }); let command = proposed_group_command(&cause) .unwrap_or_else(|| panic!("the diagnosis proposed no group command:\n{cause}")); + let proposed = groups_in_command(&command); + // Both sides are guarded because an empty either side makes the check below + // vacuous: no proposed groups means nothing is tested, and no known groups + // means everything is reported unknown. `machine_group_names` swallows a + // read error into an empty list, so the second is a real possibility on a + // host where `/etc/group` cannot be read — a harness failure that must not + // be reported as the product naming groups that do not exist. + assert!( + !proposed.is_empty(), + "no group names could be read out of the proposed command, so nothing was checked. \ + This is a harness read failure, not a verdict:\n {command}" + ); let known = crate::machine_group_names(); - let unknown: Vec = groups_in_command(&command) + assert!( + !known.is_empty(), + "this machine's group database could not be read, so no proposed group can be judged \ + against it. This is a harness read failure, not a verdict:\n {command}" + ); + let unknown: Vec = proposed .into_iter() .filter(|group| !known.iter().any(|name| name == group)) .collect(); diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index bd8bd5057e..a0e3c1a59f 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -159,12 +159,21 @@ async fn assert_default_command_described_distinctly(world: &mut E2eWorld) { "expected the help to list a `dash` command that opens the dashboard:\n{help}" ); // The sentence describing what running the CLI with no subcommand does. - let default_sentence = help + // + // Read from the whole PARAGRAPH the phrase sits in, not from its one line: + // clap wraps `long_about` at the terminal width, and this sentence sits near + // a wrap boundary. Matching a single line would let a future wording that + // does call the plain command the dashboard slip through whenever the wrap + // happened to fall between "subcommand" and "dashboard" — passing while the + // defect this exists to catch is present. + let paragraph: Vec<&str> = help + .split("\n\n") + .find(|block| block.contains("with no subcommand")) + .unwrap_or_default() .lines() .map(str::trim) - .find(|line| line.contains("with no subcommand")) - .unwrap_or_default() - .to_ascii_lowercase(); + .collect(); + let default_sentence = paragraph.join(" ").to_ascii_lowercase(); assert!( !default_sentence.is_empty(), "the help does not say what running the CLI with no subcommand does:\n{help}" diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index e07205d03d..91ba864c6c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1055,17 +1055,30 @@ async fn user_stops_running_server(world: &mut E2eWorld) { .path() .join("data") .join("services"); - let service_id = std::fs::read_dir(&root) + let recorded: Vec = std::fs::read_dir(&root) .into_iter() .flatten() .flatten() .filter(|entry| entry.path().extension().is_some_and(|e| e == "json")) - .find_map(|entry| { + .filter_map(|entry| { let bytes = std::fs::read(entry.path()).ok()?; let record: serde_json::Value = serde_json::from_slice(&bytes).ok()?; Some(record.get("service_id")?.as_str()?.to_owned()) }) - .unwrap_or_else(|| panic!("no managed service recorded under {}", root.display())); + .collect(); + // Exactly one, not the first `read_dir` happens to yield. Directory order is + // unspecified, so with a second record present — a retried serve's leftover, + // or a planted mock alongside a real one — taking the first would stop an + // arbitrary service and the assertion that follows would judge that + // unrelated stop. + assert_eq!( + recorded.len(), + 1, + "expected exactly one managed service under {} so there is no doubt which one is being \ + stopped, found {recorded:?}", + root.display() + ); + let service_id = recorded.into_iter().next().expect("length checked above"); let (stdout, stderr, rc) = crate::run_rocm(world, &["services", "stop", &service_id, "--yes"]); world.cli_output = Some(format!("{stdout}{stderr}")); world.cli_rc = Some(rc); @@ -1073,12 +1086,23 @@ async fn user_stops_running_server(world: &mut E2eWorld) { #[when("the user removes the CLI's managed files")] async fn user_removes_managed_files(world: &mut E2eWorld) { - // `--keep-binaries` is what keeps this safe to run on a shared runner: the - // config, data and cache directories are this scenario's own temporary ones, - // so the removal is real but reaches nothing outside the scenario. It does - // not weaken the scenario — whether the CLI stops what it manages has - // nothing to do with whether it also deletes the program. - let (stdout, stderr, rc) = crate::run_rocm(world, &["uninstall", "--yes", "--keep-binaries"]); + // `--keep-binaries` and `--keep-cache` are what keep this safe to run on a + // shared runner. Config and data ARE this scenario's own temporary ones, so + // removing them is real and reaches nothing outside the scenario. The cache + // is NOT: `isolate_env` points `ROCM_CLI_CACHE_DIR` at the persistent + // `E2E_SHARED_CACHE_DIR` whenever CI provides one, which every GPU lane + // does — and this scenario is `@requires-os:linux`, so it runs there. + // Without `--keep-cache` the uninstall would delete the shared download + // cache out from under every later scenario AND every later run. + // + // Neither flag weakens the scenario: what it asserts is whether the CLI + // stops the servers it manages, which has nothing to do with whether it + // also deletes the program or the download cache. The records it must not + // orphan live under the data dir, which is still removed. + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["uninstall", "--yes", "--keep-binaries", "--keep-cache"], + ); world.cli_output = Some(format!("{stdout}{stderr}")); world.cli_rc = Some(rc); } @@ -1200,6 +1224,15 @@ async fn given_default_serve_address_taken(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("could not take the default serve address: {e}")); // Held on the World so it stays taken for the whole scenario and is released // with it. + // + // Safe only because this scenario is `@requires-gpu`, and every GPU lane is + // serialized to one scenario at a time. `ensure_serve_port_free` clears the + // port with `fuser -k`/`kill -9`, and from the moment this listener exists + // the process holding that port is the cucumber binary itself — so a + // concurrent scenario calling it would kill the whole suite mid-run. The + // serialization is enforced in `e2e.rs` and cannot be lifted by + // `E2E_MAX_CONCURRENT`; if that ever changes, this needs an ephemeral port + // rather than the shared default. world.occupied_address = Some(listener); } From 2e8772b2de796a8540fe5524cf8b52ebd7a5a444 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 10 Sep 2026 01:50:54 +0200 Subject: [PATCH 16/62] test(e2e): wait for the engine-shell prompt line, not the banner assert_prompt_marked waited for the marker to appear anywhere on screen, then read the screen once and required it on a prompt line. The banner the CLI prints before handing over to bash already contains the marker, so the wait returned before bash had painted its prompt and the single read that followed landed in the gap, failing a shell that was simply still starting. Poll the real condition instead. The assertion is unchanged - a shell whose prompt is never marked still fails, at the timeout rather than instantly. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index 504d6afb52..d191bf5dff 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -9,7 +9,7 @@ //! from — exists only in what the terminal renders, so a piped run cannot see it. use std::path::PathBuf; -use std::time::Duration; +use std::time::{Duration, Instant}; use cucumber::{given, then, when}; @@ -98,20 +98,33 @@ async fn assert_prompt_marked(world: &mut E2eWorld) { let marker = format!("(rocm:{ENGINE})"); // The marker must be on the rendered prompt line, which is what the user // actually sees — not merely somewhere in the informational block above it. - session - .wait_for_screen(&marker, SCREEN_TIMEOUT) - .await - .unwrap_or_else(|e| panic!("engine shell prompt was not marked: {e}")); - - let screen = session.screen_text(); - let on_a_prompt_line = screen - .lines() - .filter(|line| line.contains(&marker)) - .any(|line| line.contains('$')); - assert!( - on_a_prompt_line, - "the marker never reached a prompt line:\n{screen}" - ); + // + // Waited for as ONE condition rather than two. Waiting for the marker alone + // and then reading the screen once is satisfied by the banner above the + // prompt ("your prompt is now prefixed (rocm:vllm)"), which the CLI prints + // before bash has painted anything — so the single read that follows lands + // in the gap between the two whenever the machine is busy, and reports a + // shell that simply had not reached its prompt yet as one whose prompt is + // unmarked. Polling the real condition removes the gap without weakening + // it: the assertion below is unchanged, and a shell that never marks its + // prompt still fails, just at the timeout instead of instantly. + let deadline = Instant::now() + SCREEN_TIMEOUT; + let mut screen = session.screen_text(); + loop { + let on_a_prompt_line = screen + .lines() + .filter(|line| line.contains(&marker)) + .any(|line| line.contains('$')); + if on_a_prompt_line { + return; + } + assert!( + Instant::now() < deadline, + "the marker never reached a prompt line within {SCREEN_TIMEOUT:?}:\n{screen}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + screen = session.screen_text(); + } } #[then("the engine environment's interpreter is the one that runs")] From 03f144d0d00f492cc2b2cdbb47c2f9be5dcf731f Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 10 Sep 2026 10:16:01 +0200 Subject: [PATCH 17/62] test(e2e): make two guards discriminating rather than silently satisfiable is_running() answered false on any try_wait() error. Its only caller asserts the product stopped the process, so that answered 'the product did its job' for a wait the harness could not perform - failing open on the defect the scenario exists to catch. Child caches the status once reaped, so a repeated call returns Ok(Some(_)) and no benign case is left to absorb; panic instead. update-02 only checked the preview was not refused as a usage error. Correct for the bug it pins, but once that row goes it becomes a permanent guard that a binary which accepted the preview and then performed the update would still pass. Assert the preview installed nothing, read back through the CLI's own report against this scenario's empty isolated data dir. Signed-off-by: fredespi --- tests/e2e-cucumber/features/update.feature | 1 + tests/e2e-cucumber/tests/e2e.rs | 13 ++++++++-- tests/e2e-cucumber/tests/e2e/update_steps.rs | 27 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/features/update.feature b/tests/e2e-cucumber/features/update.feature index 76a0b85405..9c826f7e1f 100644 --- a/tests/e2e-cucumber/features/update.feature +++ b/tests/e2e-cucumber/features/update.feature @@ -27,3 +27,4 @@ Feature: Update report Scenario: update-02 - Previewing an update without asking to install it is accepted When the user asks to see what updating would do without asking for it to be done Then the request is accepted rather than refused as a misuse + And the preview has not installed anything diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 9668868b33..2bbefc25d3 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -142,8 +142,17 @@ impl OwnedProcess { std::thread::sleep(std::time::Duration::from_millis(100)); } Ok(None) => return true, - // Already reaped by an earlier call. - Err(_) => return false, + // Not treated as "stopped". The only caller asserts that the + // product stopped this process, so answering `false` here is + // answering "the product did its job" — which would turn a wait + // this harness could not perform into a pass for the very defect + // the scenario exists to catch. `Child` caches the status once + // it has reaped, so a repeated call returns `Ok(Some(_))` rather + // than an error and there is no benign case left to absorb. + Err(error) => panic!( + "could not determine whether pid {} is still running: {error}", + self.pid() + ), } } } diff --git a/tests/e2e-cucumber/tests/e2e/update_steps.rs b/tests/e2e-cucumber/tests/e2e/update_steps.rs index abf4cffafb..e261222b98 100644 --- a/tests/e2e-cucumber/tests/e2e/update_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/update_steps.rs @@ -98,6 +98,33 @@ async fn assert_preview_accepted(world: &mut E2eWorld) { ); } +#[then("the preview has not installed anything")] +async fn assert_preview_installed_nothing(world: &mut E2eWorld) { + // The half of the contract that outlives the defect. Once `--dry-run` is + // accepted, this scenario stops pinning a bug and becomes a guard — and a + // guard that only checked the command was ALLOWED would be satisfied by a + // binary that accepted the preview and then performed the update, which is + // the one outcome a preview must never produce. + // + // Asked of the CLI's own report rather than of the filesystem, keyed on the + // line `update-01` already pins, so it reads the machine the way a user + // would. Host-invariant: this scenario runs against an isolated and empty + // `ROCM_CLI_DATA_DIR`, so nothing is managed before the preview and any + // runtime named after it was installed by the preview. + // + // Today this never runs — the step above fails first on the usage error — + // so it neither weakens nor satisfies the row that pins EAI-8010. + let (stdout, stderr, rc) = crate::run_rocm(world, &["update"]); + assert_eq!( + rc, 0, + "could not read back what the machine manages after the preview:\n{stdout}{stderr}" + ); + assert!( + stdout.contains("managed runtimes: none"), + "previewing an update installed a runtime; the machine managed none before it:\n{stdout}" + ); +} + // ── Helpers ──────────────────────────────────────────────────────── fn ok_output(world: &E2eWorld) -> String { From 19cc22092a528b0ca138ec2fd91f2447ad3fa07f Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 10 Sep 2026 12:06:01 +0200 Subject: [PATCH 18/62] test(e2e): scope update-02's readback to what its fixture can prove The step added last round claimed to guard against a preview that performed the update. It cannot: on an empty registry `--apply` bails in select_runtime_update_source (main.rs:15939) because update upgrades a managed runtime and never installs a first one, so 'manages none' is guaranteed by the fixture rather than by the product. A comment asserting otherwise would mislead the next reader. Renamed to say what it does hold - the readback path stays clean - and record why the real non-mutation contract is not proven here: it needs a managed runtime, and with one present the report resolves the latest version per runtime (therock.rs:810) and reaches the TheRock index, which the mock lane has no network for. Tracked on EAI-8010. Also note why panicking in is_running is safe (no Drop path, cucumber catches per step), and cite the reverted sampling commit in the EAI-7960 note. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 3 +- tests/e2e-cucumber/features/update.feature | 2 +- tests/e2e-cucumber/tests/e2e.rs | 6 +++ tests/e2e-cucumber/tests/e2e/update_steps.rs | 42 ++++++++++++-------- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 159f3ab242..e47b4b1569 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -358,7 +358,8 @@ serve_timeout_secs = 240 # single instantaneous screenshot of a TUI buys: the one look sometimes lands on # a frame painted before the clear, and the scenario "passes" without the # behaviour existing. Sampling the whole window instead of looking once (tried -# here, then reverted as out of scope for a test-only change) measured the value +# here, then reverted in `88f46e3` as out of scope for a test-only change; the +# reverted diff is where to start if this is picked up) measured the value # disappearing 0.80 s and 1.27 s after the failed scrape on the mock runner — # comfortably inside a window that is meant to be at least 6 s, and far too fast # to be the harness being descheduled. diff --git a/tests/e2e-cucumber/features/update.feature b/tests/e2e-cucumber/features/update.feature index 9c826f7e1f..face4fd50d 100644 --- a/tests/e2e-cucumber/features/update.feature +++ b/tests/e2e-cucumber/features/update.feature @@ -27,4 +27,4 @@ Feature: Update report Scenario: update-02 - Previewing an update without asking to install it is accepted When the user asks to see what updating would do without asking for it to be done Then the request is accepted rather than refused as a misuse - And the preview has not installed anything + And the machine still manages no runtimes diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 2bbefc25d3..ddd135d50b 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -149,6 +149,12 @@ impl OwnedProcess { // the scenario exists to catch. `Child` caches the status once // it has reaped, so a repeated call returns `Ok(Some(_))` rather // than an error and there is no benign case left to absorb. + // + // Panicking is safe here specifically: nothing calls this from + // `Drop` (the only caller is the serve-21 step), so there is no + // unwind-during-drop abort to worry about, and cucumber catches + // a step panic per scenario — the failure is reported and this + // handle's `Drop` still runs its kill/wait cleanup. Err(error) => panic!( "could not determine whether pid {} is still running: {error}", self.pid() diff --git a/tests/e2e-cucumber/tests/e2e/update_steps.rs b/tests/e2e-cucumber/tests/e2e/update_steps.rs index e261222b98..11d6941d6f 100644 --- a/tests/e2e-cucumber/tests/e2e/update_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/update_steps.rs @@ -98,30 +98,40 @@ async fn assert_preview_accepted(world: &mut E2eWorld) { ); } -#[then("the preview has not installed anything")] -async fn assert_preview_installed_nothing(world: &mut E2eWorld) { - // The half of the contract that outlives the defect. Once `--dry-run` is - // accepted, this scenario stops pinning a bug and becomes a guard — and a - // guard that only checked the command was ALLOWED would be satisfied by a - // binary that accepted the preview and then performed the update, which is - // the one outcome a preview must never produce. +#[then("the machine still manages no runtimes")] +async fn assert_still_manages_no_runtimes(world: &mut E2eWorld) { + // Scoped deliberately narrowly, because the obvious stronger claim is one + // this fixture CANNOT make. It would be natural to read this as "the + // preview did not perform the update", but on an empty registry that + // outcome is unreachable whether the product is honest or not: `--apply` + // resolves a runtime to upgrade through `select_runtime_update_source`, + // which bails with "no managed runtimes are registered" when none is + // (`apps/rocm/src/main.rs:15939`). `update` upgrades a managed runtime; it + // never installs a first one. So "manages none" here is guaranteed by the + // fixture, and an assertion resting on it would be satisfied forever. // - // Asked of the CLI's own report rather than of the filesystem, keyed on the - // line `update-01` already pins, so it reads the machine the way a user - // would. Host-invariant: this scenario runs against an isolated and empty - // `ROCM_CLI_DATA_DIR`, so nothing is managed before the preview and any - // runtime named after it was installed by the preview. + // What it does hold is the readback path: after a preview, `update` still + // answers, exits 0, and reports the same machine `update-01` pins. That is + // worth asserting and it can fail — but it is not the non-mutation contract. // - // Today this never runs — the step above fails first on the usage error — - // so it neither weakens nor satisfies the row that pins EAI-8010. + // Proving THAT needs a scenario with a managed runtime registered, where a + // performed update is observable. It is not done here on purpose: with a + // manifest present, `render_update_report` resolves the latest version per + // runtime (`apps/rocm/src/therock.rs:810` → `resolve_latest_for_manifest`), + // which reaches the TheRock index — and this scenario runs on the mock lane, + // which has no network. Tracked on EAI-8010 rather than forced in here. + // + // Today this never runs: the step above fails first on the usage error, so + // it neither weakens nor satisfies the row that pins EAI-8010. let (stdout, stderr, rc) = crate::run_rocm(world, &["update"]); assert_eq!( rc, 0, - "could not read back what the machine manages after the preview:\n{stdout}{stderr}" + "`update` stopped answering after the preview:\n{stdout}{stderr}" ); assert!( stdout.contains("managed runtimes: none"), - "previewing an update installed a runtime; the machine managed none before it:\n{stdout}" + "reading the machine back after the preview did not report the empty registry \ + this scenario runs against:\n{stdout}" ); } From d6825bb8a9ffef38a581d49841a0a1c6d33b372e Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 14 Sep 2026 16:38:52 +0200 Subject: [PATCH 19/62] test(e2e): delete the EAI-7960 rows, which main has made stale main's #380 landed the durable fix these rows' own note prescribed: an injected logical clock that advances with daemon cycles, so neither boundary depends on wall-clock scheduling. It also replaced dash-08/dash-09 with the deterministic versions and carries no expectation rows for them, which means main's CI already gates both unconditionally. So the rows no longer describe scenarios that exist. Keeping them would only suppress a real signal on two scenarios main now guards - and because they are flaky they would never report as stale, so nothing would have said so. Verified on the merged tree: 6 xfail, 0 XPASS (0 flaky, 0 stale), and both dash scenarios pass as guards. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 41 ---------------------------- 1 file changed, 41 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index b99541108b..56cc96ac82 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -348,44 +348,3 @@ bug = "EAI-8031" reason = "Managed lemonade serve of a Hugging Face `owner/repo:variant` checkpoint exits 0 but the endpoint never becomes ready on Windows." serve_timeout_secs = 240 -# --- EAI-7960: MEASURE. The dashboard is supposed to HOLD the last generation -# throughput for a validity window (clamp(3 × instance_tick, 6 s, 30 s)) after a -# scrape fails, and clear it only once that window closes. It does not: the -# value goes as soon as the failure propagates. Both of these scenarios reach -# that contract through the same "remains visible" step, and their own step -# documentation says the assertion must be RED until the hold logic lands. -# -# They have been passing intermittently rather than reliably, which is what a -# single instantaneous screenshot of a TUI buys: the one look sometimes lands on -# a frame painted before the clear, and the scenario "passes" without the -# behaviour existing. Sampling the whole window instead of looking once (tried -# here, then reverted in `88f46e3` as out of scope for a test-only change; the -# reverted diff is where to start if this is picked up) measured the value -# disappearing 0.80 s and 1.27 s after the failed scrape on the mock runner — -# comfortably inside a window that is meant to be at least 6 s, and far too fast -# to be the harness being descheduled. -# -# `flaky = true` for exactly that reason: the bug is real and constant, while -# the observation of it is not, so either outcome is tolerated while EAI-7960 is -# open. -# -# Note what that costs, because these rows cannot advertise their own removal. -# A flaky row deliberately tolerates a pass, so a passing run is reported as -# `flaky XPASS` and never as stale — the deletion signal the non-flaky rows in -# this file rely on does not exist here. Delete both rows when EAI-7960 is -# CLOSED, tracked from the ticket rather than from this lane. Until then -# dash-08/dash-09 are tolerate-either and are not guarding the behaviour; the -# durable fix is to make the observation deterministic (sample the window rather -# than take one screenshot of a TUI) so the rows can drop `flaky` and go back to -# self-deleting. --- -[["dash-gen-tps-held-after-scrape-failure"]] -when = {} -bug = "EAI-7960" -reason = "The dashboard clears generation throughput on the failing scrape instead of holding it for the validity window." -flaky = true - -[["dash-gen-tps-expiry-boundary"]] -when = {} -bug = "EAI-7960" -reason = "The expiry boundary cannot be reached because the held value is cleared on the failing scrape, before the window opens." -flaky = true From 542ebe04460cb7f07e2eb25a0f3a3ece54e09aa0 Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 14 Sep 2026 16:46:28 +0200 Subject: [PATCH 20/62] test(e2e): drop the trailing blank line left by deleting the last rows The EAI-7960 block was at the end of expectations.toml, so removing it left the file ending in a blank line and prek's end-of-file hook rewrote it. Verified with the hook set CI runs (prek run --all-files --no-group local-tools): all nine pass. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 56cc96ac82..1ac57e735b 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -347,4 +347,3 @@ when = { os = "windows", therock_family = "gfx*", has_amd_gpu = true } bug = "EAI-8031" reason = "Managed lemonade serve of a Hugging Face `owner/repo:variant` checkpoint exits 0 but the endpoint never becomes ready on Windows." serve_timeout_secs = 240 - From ea4fd910831145e86a753b8ab37682a5840188da Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 14 Sep 2026 17:15:14 +0200 Subject: [PATCH 21/62] test(e2e): stop bounding mock-lane concurrency, which broke two of main's scenarios The mock lane's ceiling was bounded to the core count to stop TUI steps being descheduled past the wall-clock windows they measure. main's #380 changed what that knob means: dash-08/dash-09 now measure a validity window whose elapsed time advances with DAEMON CYCLES, not wall time, so a LESS loaded lane burns the window faster and clears the held value before the assertion looks. Both scenarios pass on main at 64-way and failed on this branch at 2..=6, with their dash files byte-identical to main - so the bound, not the scenarios, was the difference. A contract-pinning change should not carry a suite-wide scheduling knob that flips unrelated verdicts, so the lane goes back to main's 64 and the E2E_MAX_CONCURRENT override goes with it. The starvation this was meant to fix is better handled per scenario, the way engine-shell-01 now polls for its prompt line instead of sampling once. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e.rs | 43 ++++++------------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 6 +-- 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 44f3a707f5..58e2ad5dbd 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1413,36 +1413,21 @@ async fn main() { // themselves rather than only from the probe. `shared_uv_cache_dir()` is // deliberately excluded: uv's cache is content-addressed and uv does its // own locking. A lane that races becomes serialized-and-slower instead. - // The mock lane's ceiling is bounded by the machine rather than left at - // cucumber's flat 64. Its scenarios are safe to run together, but they are - // not free: each spawns `rocm` subprocesses, and several drive a TUI under a - // pseudo-terminal and assert on WALL-CLOCK behaviour. At 64-way on a 2-4 - // vCPU runner those steps are descheduled for seconds — measured at 4.7 s - // against a 6 s window — so they miss the moment they are timing and report - // a starved reading as a product verdict. Bounding to the core count keeps - // the lane parallel where parallelism is free and stops it oversubscribing - // where it is not; it is also no slower, because starvation costs more than - // the extra concurrency buys. // - // `E2E_MAX_CONCURRENT` overrides the MOCK lane's ceiling, for bisecting a - // lane whose failures look like contention rather than behaviour. It cannot - // raise a serialized lane: the `1` above is a correctness requirement, not - // a tuning choice, so it is applied AFTER the override rather than inside - // the fallback. Two multi-GiB installs racing one shared runtimes tree, or - // two serves racing one port and one GPU's VRAM, is what that `1` prevents. - let serialize = - cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some(); - let max_concurrent = if serialize { - 1 - } else { - std::env::var("E2E_MAX_CONCURRENT") - .ok() - .and_then(|value| value.trim().parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or_else(|| { - std::thread::available_parallelism().map_or(4, |n| n.get().clamp(2, 6)) - }) - }; + // The mock lane's 64 is deliberately left as main has it. Bounding it to the + // core count was tried on this branch and reverted: dash-08/dash-09 measure a + // validity window whose elapsed time advances with DAEMON CYCLES (the logical + // clock added in #380), so a less-loaded lane burns that window faster and + // clears the held value before the assertion looks. Both scenarios pass on + // main at 64-way and failed here at 2..=6 — a suite-wide scheduling knob + // changing two unrelated scenarios' verdicts is reason enough for a + // contract-pinning change not to touch it. + let max_concurrent = + if cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some() { + 1 + } else { + 64 + }; let summary = E2eWorld::cucumber() .max_concurrent_scenarios(max_concurrent) // Record the scenario name on the World before each scenario so every diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 91ba864c6c..b6bd6a5736 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1230,9 +1230,9 @@ async fn given_default_serve_address_taken(world: &mut E2eWorld) { // port with `fuser -k`/`kill -9`, and from the moment this listener exists // the process holding that port is the cucumber binary itself — so a // concurrent scenario calling it would kill the whole suite mid-run. The - // serialization is enforced in `e2e.rs` and cannot be lifted by - // `E2E_MAX_CONCURRENT`; if that ever changes, this needs an ephemeral port - // rather than the shared default. + // serialization is enforced in `e2e.rs`, which pins a GPU lane to one + // scenario regardless of the mock lane's ceiling; if that ever changes, this + // needs an ephemeral port rather than the shared default. world.occupied_address = Some(listener); } From a80429f0a4f3a1076142177df4a83deae7c80969 Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 14 Sep 2026 19:02:39 +0200 Subject: [PATCH 22/62] test(e2e): close three gaps where a guard read more loosely than it claimed The orphan-row test scanned whole feature files with split_whitespace, so an @id: written inside a # comment counted as a live scenario and vouched for a row that measures nothing - the exact failure the test exists to catch. Read tags off tag lines only, the way scenarios_of in feature_naming.rs does. Mutation-checked: a commented-out id plus a bogus row now fails the test. The stat shim keyed on the device path alone while its comment claimed it answered only 'stat -c %A|%U|%G '. A later stat of the same device in any other form would have been handed this mode|owner|group line and parsed the wrong shape. Match the format argument too. stop_managed_services rebuilt three directory variables by hand while its doc said it used 'the same isolated env the scenario used'. That dropped the rest of isolate_env, including the shared-cache redirection CI lanes export. Pass the scenario's real environment instead. Signed-off-by: fredespi --- tests/e2e-cucumber/src/expectation.rs | 18 ++++++++-- tests/e2e-cucumber/tests/e2e.rs | 34 ++++++++++++++----- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 94a7f61adc..4f7af11cc9 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1209,9 +1209,21 @@ flaky = true } let text = std::fs::read_to_string(&path) .unwrap_or_else(|e| panic!("could not read {}: {e}", path.display())); - for tag in text.split_whitespace() { - if let Some(id) = tag.strip_prefix("@id:") { - scenario_ids.insert(id.to_owned()); + // Read tags off TAG LINES only, the way `scenarios_of` in + // tests/feature_naming.rs does. Scanning the whole file would also + // pick up an `@id:` written inside a `#` comment — and since this + // set is what decides whether a row is an orphan, a commented-out + // id would vouch for a row that measures nothing, which is the one + // thing this test exists to catch. + for line in text.lines() { + let line = line.trim(); + if !line.starts_with('@') { + continue; + } + for tag in line.split_whitespace() { + if let Some(id) = tag.strip_prefix("@id:") { + scenario_ids.insert(id.to_owned()); + } } } } diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 58e2ad5dbd..6ea79392a9 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -657,8 +657,9 @@ impl Drop for E2eWorld { // failure is ignored rather than panicking (which would abort the run) — // hence the returned status is discarded here. The serve retry, where a // failed stop changes the next attempt's meaning, does read it. + let env = self.isolate_env(); if let Some(root) = &self.isolated_root { - stop_managed_services(root.path()); + stop_managed_services(root.path(), &env); } // `isolated_root` is a `TempDir`; its own Drop removes the directory. } @@ -675,7 +676,10 @@ impl Drop for E2eWorld { /// record existed yet, or the stop failed, the next attempt runs against a device /// the previous one still owns, and the run must say so rather than let it look /// like a genuinely broken serve. -fn stop_managed_services(root: &std::path::Path) -> String { +fn stop_managed_services( + root: &std::path::Path, + env: &[(&'static str, std::ffi::OsString)], +) -> String { let services_dir = root.join("data").join("services"); let entries = match std::fs::read_dir(&services_dir) { Ok(entries) => entries, @@ -707,9 +711,14 @@ fn stop_managed_services(root: &std::path::Path) -> String { } let mut cmd = std::process::Command::new(rocm_binary()); cmd.args(["services", "stop", service_id, "--yes"]); - cmd.env("ROCM_CLI_CONFIG_DIR", root.join("config")); - cmd.env("ROCM_CLI_DATA_DIR", root.join("data")); - cmd.env("ROCM_CLI_CACHE_DIR", root.join("cache")); + // The scenario's own environment, not a hand-rolled subset of it. The + // three directory variables were previously rebuilt here, which quietly + // dropped everything else `isolate_env` sets — including the shared + // cache redirection a CI lane exports — so this teardown was talking to + // a slightly different machine than the scenario it is cleaning up after. + for (key, value) in env { + cmd.env(key, value); + } outcomes.push(match cmd.output() { Ok(out) if out.status.success() => format!("{service_id}: stopped"), Ok(out) => format!( @@ -917,12 +926,19 @@ pub fn stat_shim_path( .into_iter() .find(|candidate| std::path::Path::new(candidate).is_file()) .expect("no real `stat` binary to delegate to"); - // `-c %A|%U|%G` is the only form the CLI asks for, and the shim answers only - // when the target path is the one under test; anything else is the real + // Answers only the one call this scenario is substituting: the CLI's + // `stat -c %A|%U|%G ` (crates/rocm-core/src/examine.rs:1303). + // Both the format AND the path must match — keying on the path alone would + // hand this `mode|owner|group` line to any other `stat` of the same device + // added later, in whatever format it asked for, and the caller would parse + // the wrong shape without anything saying so. Anything else is the real // binary's business. let script = format!( - "#!/bin/sh\nfor arg in \"$@\"; do\n if [ \"$arg\" = \"{device}\" ]; then\n \ - printf '%s|%s|%s\\n' '{mode}' '{owner}' '{group}'\n exit 0\n fi\ndone\nexec {real} \"$@\"\n" + "#!/bin/sh\nfmt=\nwant=\nfor arg in \"$@\"; do\n \ + if [ \"$arg\" = '%A|%U|%G' ]; then fmt=1; fi\n \ + if [ \"$arg\" = \"{device}\" ]; then want=1; fi\ndone\n\ + if [ -n \"$fmt\" ] && [ -n \"$want\" ]; then\n \ + printf '%s|%s|%s\\n' '{mode}' '{owner}' '{group}'\n exit 0\nfi\nexec {real} \"$@\"\n" ); let shim = bin.join("stat"); std::fs::write(&shim, script).expect("failed to write the stat shim"); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index b6bd6a5736..b87370ebfc 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -449,7 +449,7 @@ fn default_engine_serve_target() -> &'static str { fn stop_scenario_services(world: &E2eWorld) -> String { world.isolated_root.as_ref().map_or_else( || "not attempted: scenario has no isolated root".to_owned(), - |root| crate::stop_managed_services(root.path()), + |root| crate::stop_managed_services(root.path(), &world.isolate_env()), ) } From 201f7905d8b9108ec480a45bc6f8166132cd41d9 Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 14 Sep 2026 19:54:23 +0200 Subject: [PATCH 23/62] test(e2e): cite the two refusal strings, correct the checker's platform list The policy guard matched two product wordings with nothing pointing at where they live, so a reword would have narrowed it to clap's branch alone while still passing. Cite both with file and line, and say what a miss means. The diagnose symptom comment attributed check_1_arch_not_in_wheel to a LINUX_AND_WINDOWS constant. That constant is real but belongs to fix recipes in fix.rs; the checker is registered with a literal linux/windows/wsl list at diagnose.rs:1964, so the comment named the wrong subsystem and dropped WSL. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/diagnose_steps.rs | 17 +++++++++-------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 10 ++++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index e6ff021443..861c444742 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -7,14 +7,15 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; -/// A symptom string that scores a catalog match on both Linux and Windows. It -/// keys off `check_1_arch_not_in_wheel` (a `LINUX_AND_WINDOWS` checker), which -/// scores 50 on the `HSA_STATUS_ERROR_INVALID_ISA` keyword regardless of host -/// state — the covered-arch penalty only applies when a framework arch list is -/// present, so with none installed the match always renders. (The earlier -/// "/dev/kfd" symptom keyed only off the Linux-only render-group checker and so -/// produced no match on Windows.) The specific fix-id is environment-dependent, -/// so scenarios assert the shape of a match, not the id. +/// A symptom string that scores a catalog match on every platform these +/// scenarios run on. It keys off `check_1_arch_not_in_wheel`, registered for +/// `["linux", "windows", "wsl"]` (`crates/rocm-core/src/diagnose.rs:1964`), +/// which scores 50 on the `HSA_STATUS_ERROR_INVALID_ISA` keyword regardless of +/// host state — the covered-arch penalty only applies when a framework arch +/// list is present, so with none installed the match always renders. (The +/// earlier "/dev/kfd" symptom keyed only off the Linux-only render-group +/// checker and so produced no match on Windows.) The specific fix-id is +/// environment-dependent, so scenarios assert the shape of a match, not the id. const KNOWN_SYMPTOM: &str = "HSA_STATUS_ERROR_INVALID_ISA"; /// The error text a vLLM engine-startup import failure leaves behind, as a user diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index b87370ebfc..f608175f1f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1002,6 +1002,16 @@ async fn assert_no_policy_refused(world: &mut E2eWorld) { // Three wordings are recognised: the two the product uses to turn a policy // down for being that policy, and clap's own, which is what a value that is // advertised but not accepted as a value looks like. + // + // The two product wordings are cited so a reword is a grep away rather than + // a silent narrowing. If either moves, this guard quietly falls back to the + // clap branch alone — still a real check, but no longer the one described + // above — so treat a miss here as needing the strings re-sourced, not as + // evidence the command stopped refusing: + // "CPU mode is not a fallback path" apps/rocm/src/main.rs:18688 + // "unsupported device policy" apps/rocm/src/main.rs:18690 + // Matched lowercased, which is why the literals below differ in case from + // the source. let rejected: Vec<&String> = transcript .iter() .filter(|line| { From 91747cbae13252bff84d3f05c4a78eb6a2d3bbd8 Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 14 Sep 2026 21:05:27 +0200 Subject: [PATCH 24/62] test(e2e): wait through the driver, and name what the docs and a grep get wrong The engine-shell prompt wait hand-rolled its poll loop, so it lost what wait_for_screen does around the waiting: a dead shell or a panicked pty reader was reported as the marker never arriving, after burning the full 30s, instead of as the thing that actually went wrong. Generalise that helper into wait_for_screen_where and wait through it; wait_for_screen now delegates too, so both paths share the crash and drain handling. The README claimed there is no tag-filter tiering, which stopped being true when @merge-queue landed - and this PR adds three more such scenarios. Say which three tags tier the suite, and what that costs: a @merge-queue scenario runs on no per-PR check, on lanes ci.yml marks non-blocking. Add @merge-queue and @requires-no-gpu to the tag table, which never listed either. Name the command that prints the serve-23 port: line. port_status writes an identical line for another command, and two reviewers have now grepped their way to the wrong one while checking this step. Signed-off-by: fredespi --- tests/e2e-cucumber/README.md | 17 ++++++-- tests/e2e-cucumber/tests/e2e/engines_steps.rs | 42 +++++++++---------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 10 +++++ tests/e2e-cucumber/tests/e2e/tui_driver.rs | 35 ++++++++++++---- 4 files changed, 71 insertions(+), 33 deletions(-) diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 5b9b6b7b13..d43b461976 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -101,10 +101,17 @@ E2E_INCLUDE_LIFECYCLE=1 E2E_ONLY_LIFECYCLE=1 cargo xtask e2e ## Tags and per-scenario expectations -There is no tag-filter tiering. Each CI job runs the **whole** suite -(`cargo xtask e2e`, no `-t` filter); the harness resolves every scenario to -**pass / xfail / skip** at runtime from its capability tags plus the known-bug -matrix, then reconciles the actual result against that expectation. +Each CI job runs the suite as a whole (`cargo xtask e2e`, no `-t` filter); the +harness resolves every scenario to **pass / xfail / skip** at runtime from its +capability tags plus the known-bug matrix, then reconciles the actual result +against that expectation. + +Three tags tier the suite at runtime rather than by a `-t` filter, so a scenario +carrying one runs only where its env var is set: `@nightly` +(`E2E_INCLUDE_NIGHTLY`), `@lifecycle` (`E2E_INCLUDE_LIFECYCLE`) and +`@merge-queue` (`E2E_MERGE_QUEUE`, set on the merge-queue lanes). Note what that +means for gating: a `@merge-queue` scenario does not run on ordinary per-PR CI +at all, and the lanes that do run it are marked non-blocking in `ci.yml`. ### Naming @@ -140,6 +147,8 @@ Scenarios carry stable-id and capability tags: | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | +| `@requires-no-gpu` | Premise is a host with no usable AMD GPU (e.g. a refusal that only happens without one). The inverse of `@requires-gpu`; resolves to **skip** on a GPU host. | +| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the merge-queue lanes do. Those lanes are non-blocking, so such a scenario is telemetry rather than a gate. | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. | diff --git a/tests/e2e-cucumber/tests/e2e/engines_steps.rs b/tests/e2e-cucumber/tests/e2e/engines_steps.rs index d191bf5dff..14d95e38c2 100644 --- a/tests/e2e-cucumber/tests/e2e/engines_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/engines_steps.rs @@ -9,7 +9,7 @@ //! from — exists only in what the terminal renders, so a piped run cannot see it. use std::path::PathBuf; -use std::time::{Duration, Instant}; +use std::time::Duration; use cucumber::{given, then, when}; @@ -105,26 +105,26 @@ async fn assert_prompt_marked(world: &mut E2eWorld) { // before bash has painted anything — so the single read that follows lands // in the gap between the two whenever the machine is busy, and reports a // shell that simply had not reached its prompt yet as one whose prompt is - // unmarked. Polling the real condition removes the gap without weakening - // it: the assertion below is unchanged, and a shell that never marks its - // prompt still fails, just at the timeout instead of instantly. - let deadline = Instant::now() + SCREEN_TIMEOUT; - let mut screen = session.screen_text(); - loop { - let on_a_prompt_line = screen - .lines() - .filter(|line| line.contains(&marker)) - .any(|line| line.contains('$')); - if on_a_prompt_line { - return; - } - assert!( - Instant::now() < deadline, - "the marker never reached a prompt line within {SCREEN_TIMEOUT:?}:\n{screen}" - ); - tokio::time::sleep(Duration::from_millis(100)).await; - screen = session.screen_text(); - } + // unmarked. Waiting on the real condition removes the gap without weakening + // it: a shell that never marks its prompt still fails, at the timeout. + // + // Through `wait_for_screen_where` rather than a local poll loop, so a shell + // that dies or a pty reader that panics is reported as THAT, instead of + // running out the full timeout and blaming the prompt marker for a failure + // that happened earlier and elsewhere. + session + .wait_for_screen_where( + &format!("{marker} on a prompt line"), + SCREEN_TIMEOUT, + |screen| { + screen + .lines() + .filter(|line| line.contains(&marker)) + .any(|line| line.contains('$')) + }, + ) + .await + .unwrap_or_else(|e| panic!("the engine shell's prompt was never marked: {e}")); } #[then("the engine environment's interpreter is the one that runs")] diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index f608175f1f..7e8ac09ed5 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1263,6 +1263,16 @@ async fn user_serves_without_choosing_address(world: &mut E2eWorld) { #[then("the new server does not try to use the taken address")] async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { let output = serve_output(world); + // The `port:` line read here is the one `serve` prints in its "serve plan" + // block (`apps/rocm/src/main.rs:5389`), which is what this scenario ran. + // + // Named because grepping the product for `port: {port}` finds a decoy first: + // `port_status` writes an identical line at `main.rs:13571` for a different + // command entirely. Two reviewers have now landed on that one while checking + // this step. `serve`'s is indented and `port_status`'s is not, which is why + // the `trim()` above is load-bearing rather than defensive — and also why + // matching on indentation instead would be the wrong way to tell them apart, + // since only one of the two can appear in this scenario's output at all. let planned = output .lines() .find_map(|line| line.trim().strip_prefix("port:")) diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 6d644d375f..779a2dfb9e 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -408,14 +408,33 @@ impl TuiSession { /// deadline that includes the last screen for diagnosis. Also fails fast if /// the child exits before the marker appears. pub async fn wait_for_screen(&mut self, marker: &str, timeout: Duration) -> Result<(), String> { + let wanted = format!("{marker:?}"); + self.wait_for_screen_where(&wanted, timeout, |screen| screen.contains(marker)) + .await + } + + /// The same wait as [`Self::wait_for_screen`], for a condition a substring + /// cannot express. + /// + /// Callers that hand-roll this loop lose what the waiting is really for: a + /// shell that dies, or a pty reader that panics, is reported as the real + /// cause here instead of running out the full timeout and blaming whatever + /// the caller happened to be looking for. `wanted` is quoted into those + /// messages, so it should read as the thing being waited for. + pub async fn wait_for_screen_where( + &mut self, + wanted: &str, + timeout: Duration, + matches: impl Fn(&str) -> bool, + ) -> Result<(), String> { let deadline = Instant::now() + timeout; loop { - if self.screen_text().contains(marker) { + if matches(&self.screen_text()) { return Ok(()); } if let Some(panic_message) = self.take_reader_panic() { return Err(format!( - "pty reader thread panicked while waiting for {marker:?}: {panic_message}\n{}", + "pty reader thread panicked while waiting for {wanted}: {panic_message}\n{}", self.framed_screen() )); } @@ -427,12 +446,12 @@ impl TuiSession { self.record_once(i32::try_from(status.exit_code()).unwrap_or(-1)); let drain_deadline = Instant::now() + DRAIN_TIMEOUT; while Instant::now() < drain_deadline { - if self.screen_text().contains(marker) { + if matches(&self.screen_text()) { return Ok(()); } if let Some(panic_message) = self.take_reader_panic() { return Err(format!( - "pty reader thread panicked while draining the final frame for {marker:?}: {panic_message}\n{}", + "pty reader thread panicked while draining the final frame for {wanted}: {panic_message}\n{}", self.framed_screen() )); } @@ -448,7 +467,7 @@ impl TuiSession { // Final check after the drain window closes: the reader may have // committed the last frame between the loop's screen check and the // `is_finished`/deadline exit, so re-read before declaring failure. - if self.screen_text().contains(marker) { + if matches(&self.screen_text()) { return Ok(()); } // A reader panic landing exactly on the drain deadline would @@ -457,18 +476,18 @@ impl TuiSession { // unwind). Surface it here so the real cause wins. if let Some(panic_message) = self.take_reader_panic() { return Err(format!( - "pty reader thread panicked while draining the final frame for {marker:?}: {panic_message}\n{}", + "pty reader thread panicked while draining the final frame for {wanted}: {panic_message}\n{}", self.framed_screen() )); } return Err(format!( - "process exited ({status:?}) before {marker:?} appeared.\n{}", + "process exited ({status:?}) before {wanted} appeared.\n{}", self.framed_screen() )); } if Instant::now() >= deadline { return Err(format!( - "timed out after {timeout:?} waiting for {marker:?}.\n{}", + "timed out after {timeout:?} waiting for {wanted}.\n{}", self.framed_screen() )); } From 2b36fec7a8701f053fcb174f2b30f33b5955e779 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 09:15:41 +0200 Subject: [PATCH 25/62] test(e2e): point the merge-queue docs at the right file, and guard the prose The README line added last round said those lanes are non-blocking in ci.yml. Both halves were wrong: the lanes live in e2e-selfhosted.yml, and they are non-blocking because their check names are not in branch protection, not because of continue-on-error, which is false on five of the six. ci.yml's own comment made the same claim and sent a reviewer to the wrong conclusion while checking mine, so correct it there too. Add every_expected_to_fail_comment_still_has_a_row. Feature files restate in prose what expectations.toml owns, and prose does not move when a row does - four rows were deleted during this PR alone. Mutation-checked: deleting a row whose scenario still claims Expected to FAIL now fails the test. Drop a dead world.model_name write that made a Given look like it fed its When, and stop the serve-20 comment restating measurements expectations.toml already records, including a CI run number the two copies could drift on. Signed-off-by: fredespi --- .github/workflows/ci.yml | 7 +- tests/e2e-cucumber/README.md | 7 +- .../features/model_serving.feature | 22 +++---- tests/e2e-cucumber/src/expectation.rs | 64 +++++++++++++++++++ .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 12 +++- 5 files changed, 95 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9b677be8f..df888ab2cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -986,8 +986,11 @@ jobs: # e2e-gpu-strix-windows). They live there — not here — so a job queued on an # offline self-hosted runner (which GitHub cannot cancel) can never hold this # workflow's concurrency group and stall its merge-required checks (EAI-7548). - # They are non-blocking (continue-on-error) and fork-safe (self-hosted runners - # do not execute untrusted fork PRs). See docs/ci-hardware-testing.md. + # They are non-blocking because none of their check names are in branch + # protection's required-status-check list — NOT because of `continue-on-error`, + # which is `false` on all but one of them so a real hardware regression shows + # red (see that file's own header). They are also fork-safe (self-hosted + # runners do not execute untrusted fork PRs). See docs/ci-hardware-testing.md. # ── E2E tests (cucumber-rs) ─────────────────────────────────────────────── # diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d43b461976..b6e5510a23 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -111,7 +111,10 @@ carrying one runs only where its env var is set: `@nightly` (`E2E_INCLUDE_NIGHTLY`), `@lifecycle` (`E2E_INCLUDE_LIFECYCLE`) and `@merge-queue` (`E2E_MERGE_QUEUE`, set on the merge-queue lanes). Note what that means for gating: a `@merge-queue` scenario does not run on ordinary per-PR CI -at all, and the lanes that do run it are marked non-blocking in `ci.yml`. +at all, and the lanes that do run it — the self-hosted ones in +`.github/workflows/e2e-selfhosted.yml`, not `ci.yml` — are non-blocking because +none of their check names are in branch protection's required-status-check list. +Not because of `continue-on-error`, which is `false` on all but one of them. ### Naming @@ -148,7 +151,7 @@ Scenarios carry stable-id and capability tags: | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@requires-no-gpu` | Premise is a host with no usable AMD GPU (e.g. a refusal that only happens without one). The inverse of `@requires-gpu`; resolves to **skip** on a GPU host. | -| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the merge-queue lanes do. Those lanes are non-blocking, so such a scenario is telemetry rather than a gate. | +| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the self-hosted lanes in `e2e-selfhosted.yml` do on a `merge_group` event. Those check names are not in branch protection's required list, so such a scenario is telemetry rather than a gate. | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. | diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index d2ab5a6715..e62ccb2e55 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -199,17 +199,17 @@ Feature: Model serving # A guard, not a finding. Stopping a running server was reported as having # stopped nothing on the pod this set came from, but neither fixture tried here - # reproduces that: a plain process registered as a managed service is counted - # correctly (measured on the no-GPU lane), and so is a real vLLM serve - # (measured on MI300X, run 188). It therefore holds the contract the pod - # violated and goes red if CI ever meets it — unconditionally, EXCEPT on the - # lemonade-default Linux GPU lane, where a separate serve bug (EAI-7423) stops - # the service coming up at all and a `flaky` row covers the precondition - # failing; see the note at that row in `expectations.toml`. - # @merge-queue: this serves a real model, and the Strix Halo lanes already run - # at 32 of their 35 allotted minutes on main alone. Three new real serves would - # put them over, so they follow serve-06/07/08 onto the merge-queue path, - # where the budget for heavy serves lives. + # reproduces that. Why it ships as a guard rather than an expected failure, and + # what was measured on which lane, is recorded once at the EAI-8007 note in + # `expectations.toml` — not restated here, so the two cannot drift. + # + # It holds that contract unconditionally EXCEPT on the lemonade-default Linux + # GPU lane, where a separate serve bug (EAI-7423) stops the service coming up + # at all and a `flaky` row covers the precondition failing; see that row. + # + # @merge-queue because it serves a real model and the Strix Halo lanes have + # little headroom left on main alone, so three more real serves follow + # serve-06/07/08 onto the merge-queue path where the budget for them lives. @id:serve-services-stop-reports-what-it-stopped @requires-gpu @merge-queue Scenario: serve-20 - Stopping a running server reports that it stopped it Given a managed runtime is active diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 35e6522d31..955b937693 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1219,6 +1219,70 @@ flaky = true ); } + /// The inverse orphan: a scenario whose comment still says "Expected to + /// FAIL" after its row has been deleted. + /// + /// `expectations.toml` owns that state, but the feature files restate it in + /// prose for the reader, and prose does not move when a row does. When a bug + /// is fixed the harness flips correctly and silently — leaving a comment that + /// tells the next reader the opposite of what the suite now enforces. Four + /// rows were deleted while this file was being written, so the drift is not + /// hypothetical. + #[test] + fn every_expected_to_fail_comment_still_has_a_row() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut claimed = Vec::new(); + let features = std::fs::read_dir(root.join("features")).expect("no features directory"); + for entry in features { + let path = entry.expect("unreadable features directory entry").path(); + if path.extension().is_none_or(|ext| ext != "feature") { + continue; + } + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("could not read {}: {e}", path.display())); + // A scenario is written comment block → tag line(s) → `Scenario:`, so + // carry the claim forward from the comments and bind it to the ids on + // the tag lines, clearing at the `Scenario:` that ends the block. + let (mut expects_failure, mut pending_ids) = (false, Vec::new()); + for line in text.lines() { + let line = line.trim(); + if line.starts_with('#') { + expects_failure |= line.contains("Expected to FAIL"); + } else if line.starts_with('@') { + for tag in line.split_whitespace() { + if let Some(id) = tag.strip_prefix("@id:") { + pending_ids.push(id.to_owned()); + } + } + } else if line.starts_with("Scenario:") || line.starts_with("Scenario Outline:") { + if expects_failure { + claimed.append(&mut pending_ids.clone()); + } + expects_failure = false; + pending_ids.clear(); + } + } + } + assert!( + !claimed.is_empty(), + "found no scenario claiming 'Expected to FAIL', so this check would pass vacuously" + ); + + let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + let declared: std::collections::BTreeSet<&str> = m.declared_ids().collect(); + let lying: Vec<&String> = claimed + .iter() + .filter(|id| !declared.contains(id.as_str())) + .collect(); + assert!( + lying.is_empty(), + "these scenarios still say 'Expected to FAIL' but have no expectations row, so the \ + comment asserts the opposite of what the suite enforces: {lying:?}\n\ + Delete the comment along with the row — the bug it described is fixed, and the \ + scenario now guards the fix." + ); + } + #[test] fn glob_matches_family() { assert!(glob_match("gfx94*", "gfx942")); diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 861c444742..00dd793af6 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -1141,8 +1141,16 @@ fn compare_diagnosis_with_fix(world: &mut E2eWorld, path_override: Option<&str>) } #[given("a user who hit a device-permission failure")] -async fn user_hit_device_permission_failure(world: &mut E2eWorld) { - world.model_name = Some(DEVICE_PERMISSION_SYMPTOM.to_string()); +async fn user_hit_device_permission_failure(_world: &mut E2eWorld) { + // States the premise; carries no fixture. The symptom is `DEVICE_PERMISSION_SYMPTOM`, + // which `compare_diagnosis_with_fix` passes directly because it is shared with + // the other `When` that calls it — one that this Given does not precede. + // + // This used to write `world.model_name`, which nothing read: the step LOOKED + // like it parameterised the comparison, so changing the constant here would + // have had no effect and said nothing about it. Left empty rather than + // wired up, because routing a constant through a mutable field for one of + // two callers is the fragility, not the fix. } #[when("the user compares the diagnosis with the matching fix preview")] From 22ee61d1027b1a6e49ea89ad79e67df0582b1c95 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 10:55:46 +0200 Subject: [PATCH 26/62] test(e2e): stop the new prose guard blaming an innocent scenario expects_failure was only cleared at Scenario:, so a comment merely MENTIONING Expected to FAIL - a file header, a note in an earlier body - carried forward and condemned whichever scenario came next. Reproduced it, then fixed it by resetting on a blank line: a comment block is contiguous with the tags it introduces, so anything a blank line away belongs to something else. Checked both directions after: the header case now passes, and deleting a row whose comment stands still fails. Correct two more things in the ci.yml block rewritten last commit. It named three self-hosted jobs when six lanes exist - drop the enumeration rather than extend it, since a partial list is what misled a reader here already - and it still called those lanes non-blocking while proven out, which contradicts the structural reason established eleven lines above. Fix the Given comment's reason too: compare_diagnosis_with_fix has one caller, not two. What is shared with a second When is the constant. Signed-off-by: fredespi --- .github/workflows/ci.yml | 9 +++++---- tests/e2e-cucumber/src/expectation.rs | 12 +++++++++++- tests/e2e-cucumber/tests/e2e/diagnose_steps.rs | 12 +++++++----- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df888ab2cb..c95afa2bf2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -982,8 +982,8 @@ jobs: run: cargo xtask verify-commits --base "$BASE" --require-verified # Hardware (real AMD GPU) smoke tests run on dedicated self-hosted runners in a - # SEPARATE workflow, e2e-selfhosted.yml (jobs e2e-gpu, e2e-gpu-strix-ubuntu, - # e2e-gpu-strix-windows). They live there — not here — so a job queued on an + # SEPARATE workflow, e2e-selfhosted.yml — see that file for the current lanes, + # deliberately not restated here. They live there — not here — so a job queued on an # offline self-hosted runner (which GitHub cannot cancel) can never hold this # workflow's concurrency group and stall its merge-required checks (EAI-7548). # They are non-blocking because none of their check names are in branch @@ -998,8 +998,9 @@ jobs: # job per platform: the harness resolves every scenario to pass / xfail / skip # per host from its @id + @requires-* tags, a capability probe, and # expectations.toml (no tier flag, no tag filter). The mock job (GitHub-hosted, - # no GPU) gates the PR; the self-hosted GPU jobs are non-blocking while proven - # out. Each job writes a platform.json sidecar the consolidated report joins by + # no GPU) gates the PR; the self-hosted GPU jobs do not, because their check + # names are not in branch protection's required list (see the note above). + # Each job writes a platform.json sidecar the consolidated report joins by # scenario id to render the (scenario × platform) expectation grid. # Blocking mock job (GitHub-hosted, no GPU): must stay green. @requires-gpu diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 955b937693..5764bb23e1 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1243,10 +1243,20 @@ flaky = true // A scenario is written comment block → tag line(s) → `Scenario:`, so // carry the claim forward from the comments and bind it to the ids on // the tag lines, clearing at the `Scenario:` that ends the block. + // + // A blank line resets it. A comment block is contiguous with the tags + // it introduces, so anything separated by a blank line — a file + // header, a note in a previous scenario's body — belongs to something + // else. Without this, a header merely MENTIONING the phrase condemns + // whichever scenario happens to come first, and the failure names an + // innocent id: loud, but pointing at the wrong place. let (mut expects_failure, mut pending_ids) = (false, Vec::new()); for line in text.lines() { let line = line.trim(); - if line.starts_with('#') { + if line.is_empty() { + expects_failure = false; + pending_ids.clear(); + } else if line.starts_with('#') { expects_failure |= line.contains("Expected to FAIL"); } else if line.starts_with('@') { for tag in line.split_whitespace() { diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 00dd793af6..8e23ba84d8 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -1142,15 +1142,17 @@ fn compare_diagnosis_with_fix(world: &mut E2eWorld, path_override: Option<&str>) #[given("a user who hit a device-permission failure")] async fn user_hit_device_permission_failure(_world: &mut E2eWorld) { - // States the premise; carries no fixture. The symptom is `DEVICE_PERMISSION_SYMPTOM`, - // which `compare_diagnosis_with_fix` passes directly because it is shared with - // the other `When` that calls it — one that this Given does not precede. + // States the premise; carries no fixture. The symptom is + // `DEVICE_PERMISSION_SYMPTOM`, which the `When` this precedes passes + // directly. That constant is also used by a second `When` further down, + // which runs its own `diagnose` invocation under a shimmed PATH — so it + // belongs to the pair of them, not to this step. // // This used to write `world.model_name`, which nothing read: the step LOOKED // like it parameterised the comparison, so changing the constant here would // have had no effect and said nothing about it. Left empty rather than - // wired up, because routing a constant through a mutable field for one of - // two callers is the fragility, not the fix. + // wired up, because a Given that fed only one of the two would make the + // constant look local to this scenario when it is not. } #[when("the user compares the diagnosis with the matching fix preview")] From b6d1fdac537adb28854ce5c27abcbd3d5ffa13e0 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 12:33:49 +0200 Subject: [PATCH 27/62] test(e2e): make the prose guard report what it cannot bind, and re-source four comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard added last round reset on a blank line so a file header could not condemn an innocent scenario. That traded a loud wrong answer for a quiet one: a blank line between a comment block and its tag line silently removed that scenario from the guard, and nothing in the suite covered the new branch at all. The scan is now a pure `claimed_failure_ids` with three unit tests (header mention, contiguous block, blank-line separation), and a claim that binds to no scenario id is REPORTED with its file and line rather than dropped. Both mutations that beat the old version now fail: deleting the reset arm, and inserting one blank line before `@id:update-preview-without-applying` while deleting its row. `features/` is flat and this scan does not recurse, so it now refuses to run if a subdirectory appears instead of quietly covering less. Also, four comments that a reader could disprove: - `expectations.toml` said the EAI-8021 scenario XPASSes where the runner is already in the conventional groups. It shims both `stat` and `id`, so host membership cannot reach the result — the note invited a spurious `flaky`. - `diagnose_steps.rs` said `fix-4-render-group` exits early once the user is in the groups. `run_render_group` has no membership check; it exits 3 when `usermod`, or `sudo` when not root, is off `$PATH`. - `serving_steps.rs` cited two refusal strings by line number; they had drifted ~170 lines. Cited by symbol (`parse_device_policy`) instead. - The branch-protection fact was asserted independently in four places. It has no in-repo source of truth, so `e2e-selfhosted.yml`'s header now says that once and `ci.yml` and the README point there. Two test-quality fixes: `labelled` returned `Some("")` for a bare label, so a diagnosis and a fix preview that both said nothing compared equal; and a second command's stdout is now carried in its own `cli_other_output` rather than borrowed from `cli_stderr`, which holds real stderr everywhere else. Signed-off-by: Frederic Espinosa --- .github/workflows/ci.yml | 18 +- .github/workflows/e2e-selfhosted.yml | 8 + tests/e2e-cucumber/README.md | 17 +- tests/e2e-cucumber/expectations.toml | 9 +- tests/e2e-cucumber/src/expectation.rs | 210 ++++++++++++++---- tests/e2e-cucumber/tests/e2e.rs | 8 + .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 31 ++- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 4 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 9 +- 9 files changed, 238 insertions(+), 76 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c95afa2bf2..d121fceac0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -982,15 +982,12 @@ jobs: run: cargo xtask verify-commits --base "$BASE" --require-verified # Hardware (real AMD GPU) smoke tests run on dedicated self-hosted runners in a - # SEPARATE workflow, e2e-selfhosted.yml — see that file for the current lanes, - # deliberately not restated here. They live there — not here — so a job queued on an - # offline self-hosted runner (which GitHub cannot cancel) can never hold this - # workflow's concurrency group and stall its merge-required checks (EAI-7548). - # They are non-blocking because none of their check names are in branch - # protection's required-status-check list — NOT because of `continue-on-error`, - # which is `false` on all but one of them so a real hardware regression shows - # red (see that file's own header). They are also fork-safe (self-hosted - # runners do not execute untrusted fork PRs). See docs/ci-hardware-testing.md. + # SEPARATE workflow, e2e-selfhosted.yml — see that file for the current lanes + # and for why they are non-blocking, neither restated here. They live there — + # not here — so a job queued on an offline self-hosted runner (which GitHub + # cannot cancel) can never hold this workflow's concurrency group and stall its + # merge-required checks (EAI-7548). They are also fork-safe (self-hosted runners + # do not execute untrusted fork PRs). See docs/ci-hardware-testing.md. # ── E2E tests (cucumber-rs) ─────────────────────────────────────────────── # @@ -998,8 +995,7 @@ jobs: # job per platform: the harness resolves every scenario to pass / xfail / skip # per host from its @id + @requires-* tags, a capability probe, and # expectations.toml (no tier flag, no tag filter). The mock job (GitHub-hosted, - # no GPU) gates the PR; the self-hosted GPU jobs do not, because their check - # names are not in branch protection's required list (see the note above). + # no GPU) gates the PR; the self-hosted GPU jobs do not (see the note above). # Each job writes a platform.json sidecar the consolidated report joins by # scenario id to render the (scenario × platform) expectation grid. diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 3650a0e372..57c6652a10 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -15,6 +15,14 @@ name: E2E self-hosted # landed while `E2E tests (Strix Halo, Ubuntu)` reported `cancelled` on the # merge_group ref, which a required check under ALLGREEN would have blocked; # renaming a lane is therefore safe. +# +# THIS HEADER IS THE ONLY PLACE THE REPO RECORDS THAT FACT. Branch protection is +# a GitHub repository setting with no in-tree source of truth, so nothing in a +# checkout can verify the claim — it is a dated observation, and re-confirming it +# means reading the setting (`gh api repos/:owner/:repo/branches/main/protection`) +# or watching what a red lane actually blocks. ci.yml and +# tests/e2e-cucumber/README.md point here rather than asserting it again, so +# there is one line to re-date rather than four to notice. on: push: diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 8045fb9fc9..27ddd1a59a 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -112,9 +112,10 @@ carrying one runs only where its env var is set: `@nightly` `@merge-queue` (`E2E_MERGE_QUEUE`, set on the merge-queue lanes). Note what that means for gating: a `@merge-queue` scenario does not run on ordinary per-PR CI at all, and the lanes that do run it — the self-hosted ones in -`.github/workflows/e2e-selfhosted.yml`, not `ci.yml` — are non-blocking because -none of their check names are in branch protection's required-status-check list. -Not because of `continue-on-error`, which is `false` on all but one of them. +`.github/workflows/e2e-selfhosted.yml`, not `ci.yml` — are non-blocking. That +file's header says why, and is the only place the repo records it; branch +protection is a GitHub setting no checkout can verify, so it is not restated +here. ### Naming @@ -152,7 +153,7 @@ Scenarios carry stable-id and capability tags: | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@requires-no-gpu` | Premise is a host with no usable AMD GPU (e.g. a refusal that only happens without one). The inverse of `@requires-gpu`; resolves to **skip** on a GPU host. | -| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the self-hosted lanes in `e2e-selfhosted.yml` do on a `merge_group` event. Those check names are not in branch protection's required list, so such a scenario is telemetry rather than a gate. | +| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the self-hosted lanes in `e2e-selfhosted.yml` do on a `merge_group` event. Those lanes are non-blocking (that file's header says why), so such a scenario is telemetry rather than a gate. | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. | @@ -165,6 +166,14 @@ matches a condition is expected to fail (xfail); if it then passes, that is an bug. See `src/expectation.rs` for the resolver and `expectations.toml`'s header for the condition grammar. +When a scenario's comment block says **"Expected to FAIL"**, write it directly +above that scenario's `@id:` tag line with **no blank line between them**. A test +binds each such claim to the tags immediately beneath it and fails if the row has +since been deleted — so the prose cannot quietly come to say the opposite of what +the suite enforces. A blank line breaks that binding, and the same test then +fails naming the comment's own file and line rather than letting the scenario +drop out of the check unnoticed. + On a GPU host, a `serve` precondition that never publishes its model is relaunched once — but only for a scenario expected to pass; a known bug keeps its shortened `serve_timeout_secs` and fails on the first attempt. The stalled diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 83c46ece24..99f6d7d92b 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -254,9 +254,12 @@ reason = "The diagnosis proposes adding the user to the device's own group while # group membership and never credits access that demonstrably works, so a device # the user can already read and write is still diagnosed as a permission problem. # The scenario substitutes a world-writable device answer and asserts the -# inspection agrees before relying on it. Where the runner's user already belongs -# to the conventional groups the finding scores zero anyway and this XPASSes — -# which is what the first GPU run is for. --- +# inspection agrees before relying on it. It shims BOTH `stat` (device +# `crw-rw-rw- root root`) and `id` (groups `users` — none of the conventional +# ones), so the runner's own group membership cannot reach the result: the +# outcome is the same on every bare-metal Linux GPU host. If this XPASSes, the +# product changed; it is not a host difference, and NOT a reason to add +# `flaky = true`. --- [["diagnose-credits-a-usable-device"]] when = {} bug = "EAI-8021" diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 2d45bde09e..1049970e25 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1345,14 +1345,8 @@ flaky = true /// caught here instead. #[test] fn every_expectation_row_names_a_scenario_that_exists() { - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); let mut scenario_ids = std::collections::BTreeSet::new(); - let features = std::fs::read_dir(root.join("features")).expect("no features directory"); - for entry in features { - let path = entry.expect("unreadable features directory entry").path(); - if path.extension().is_none_or(|ext| ext != "feature") { - continue; - } + for path in feature_files() { let text = std::fs::read_to_string(&path) .unwrap_or_else(|e| panic!("could not read {}: {e}", path.display())); // Read tags off TAG LINES only, the way `scenarios_of` in @@ -1390,6 +1384,146 @@ flaky = true ); } + /// Every `.feature` file the suite runs. + /// + /// `features/` is flat, as `scenarios_of` in tests/feature_naming.rs also + /// assumes, so this does not recurse. It refuses to run rather than quietly + /// covering less if that ever stops being true: a subdirectory would simply + /// vanish from the checks below, and a check that silently stops looking at + /// something is worse than no check at all. + fn feature_files() -> Vec { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("features"); + let mut paths = Vec::new(); + for entry in std::fs::read_dir(&dir).expect("no features directory") { + let path = entry.expect("unreadable features directory entry").path(); + assert!( + !path.is_dir(), + "features/ has grown a subdirectory ({}), which this scan does not descend \ + into — make it recursive, here and in tests/feature_naming.rs, before moving \ + any .feature file into one", + path.display() + ); + if path.extension().is_some_and(|ext| ext == "feature") { + paths.push(path); + } + } + assert!( + !paths.is_empty(), + "found no .feature files in {}", + dir.display() + ); + paths + } + + /// What one feature file claims about expected failures. + struct FailureClaims { + /// Scenario ids introduced by a comment block claiming "Expected to + /// FAIL". + ids: Vec, + /// 1-indexed lines whose claim reached no scenario id. + unbound: Vec, + } + + /// Scan one feature file for "Expected to FAIL" claims. + /// + /// A scenario is written comment block → tag line(s) → `Scenario:`, and the + /// block is contiguous with the tags it introduces, so a blank line ends a + /// claim's reach: a file header or a note left in a previous scenario's + /// body describes something other than whatever comes next. + /// + /// A claim that reaches nothing is REPORTED, not dropped. Dropping it is + /// how a scenario leaves this check's coverage without anyone noticing — + /// one blank line between a comment and its tag line would be enough, and + /// a blank line is the same character the file already uses to separate + /// scenarios. Reporting it also keeps the failure honest about where the + /// problem is: the comment, not the scenario that happens to follow it. + fn claimed_failure_ids(text: &str) -> FailureClaims { + let (mut ids, mut unbound) = (Vec::new(), Vec::new()); + let (mut claim_lines, mut pending_ids): (Vec, Vec) = + (Vec::new(), Vec::new()); + for (index, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + // Clearing `pending_ids` here means a blank line between a tag + // line and its `Scenario:` would drop the ids too. Deliberate, + // and no longer silent: the claim above them then reaches + // nothing and is reported as unbound. + unbound.append(&mut claim_lines); + pending_ids.clear(); + } else if line.starts_with('#') { + if line.contains("Expected to FAIL") { + claim_lines.push(index + 1); + } + } else if line.starts_with('@') { + for tag in line.split_whitespace() { + if let Some(id) = tag.strip_prefix("@id:") { + pending_ids.push(id.to_owned()); + } + } + } else if line.starts_with("Scenario:") || line.starts_with("Scenario Outline:") { + if !claim_lines.is_empty() { + if pending_ids.is_empty() { + unbound.append(&mut claim_lines); + } else { + ids.extend(pending_ids.iter().cloned()); + } + } + claim_lines.clear(); + pending_ids.clear(); + } + } + unbound.append(&mut claim_lines); + FailureClaims { ids, unbound } + } + + #[test] + fn a_contiguous_comment_block_binds_to_its_scenario() { + let claims = claimed_failure_ids( + "Feature: f\n\ + \n\ + \x20 # Expected to FAIL. The thing is broken.\n\ + \x20 @id:thing-is-broken\n\ + \x20 Scenario: f-01 - The thing works\n\ + \x20 Given a thing\n", + ); + assert_eq!(claims.ids, ["thing-is-broken"]); + assert!(claims.unbound.is_empty(), "{:?}", claims.unbound); + } + + #[test] + fn a_file_header_mentioning_the_phrase_condemns_no_scenario() { + let claims = claimed_failure_ids( + "# Historical note: f-01 was once Expected to FAIL.\n\ + \n\ + Feature: f\n\ + \n\ + \x20 @id:thing-works\n\ + \x20 Scenario: f-01 - The thing works\n\ + \x20 Given a thing\n", + ); + // Not attributed to `thing-works` — that was the round-9 false positive. + assert!(claims.ids.is_empty(), "{:?}", claims.ids); + // But not swallowed either: it is reported against its own line. + assert_eq!(claims.unbound, [1]); + } + + #[test] + fn a_comment_a_blank_line_from_its_tag_is_reported_not_dropped() { + let claims = claimed_failure_ids( + "Feature: f\n\ + \n\ + \x20 # Expected to FAIL. The thing is broken.\n\ + \n\ + \x20 @id:thing-is-broken\n\ + \x20 Scenario: f-01 - The thing works\n\ + \x20 Given a thing\n", + ); + // The scenario has silently left the guard's coverage in every earlier + // version of this scan. It must not do so silently. + assert!(claims.ids.is_empty(), "{:?}", claims.ids); + assert_eq!(claims.unbound, [3]); + } + /// The inverse orphan: a scenario whose comment still says "Expected to /// FAIL" after its row has been deleted. /// @@ -1401,49 +1535,33 @@ flaky = true /// hypothetical. #[test] fn every_expected_to_fail_comment_still_has_a_row() { - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); let mut claimed = Vec::new(); - let features = std::fs::read_dir(root.join("features")).expect("no features directory"); - for entry in features { - let path = entry.expect("unreadable features directory entry").path(); - if path.extension().is_none_or(|ext| ext != "feature") { - continue; - } + let mut unbound = Vec::new(); + for path in feature_files() { let text = std::fs::read_to_string(&path) .unwrap_or_else(|e| panic!("could not read {}: {e}", path.display())); - // A scenario is written comment block → tag line(s) → `Scenario:`, so - // carry the claim forward from the comments and bind it to the ids on - // the tag lines, clearing at the `Scenario:` that ends the block. - // - // A blank line resets it. A comment block is contiguous with the tags - // it introduces, so anything separated by a blank line — a file - // header, a note in a previous scenario's body — belongs to something - // else. Without this, a header merely MENTIONING the phrase condemns - // whichever scenario happens to come first, and the failure names an - // innocent id: loud, but pointing at the wrong place. - let (mut expects_failure, mut pending_ids) = (false, Vec::new()); - for line in text.lines() { - let line = line.trim(); - if line.is_empty() { - expects_failure = false; - pending_ids.clear(); - } else if line.starts_with('#') { - expects_failure |= line.contains("Expected to FAIL"); - } else if line.starts_with('@') { - for tag in line.split_whitespace() { - if let Some(id) = tag.strip_prefix("@id:") { - pending_ids.push(id.to_owned()); - } - } - } else if line.starts_with("Scenario:") || line.starts_with("Scenario Outline:") { - if expects_failure { - claimed.append(&mut pending_ids.clone()); - } - expects_failure = false; - pending_ids.clear(); - } - } + let scan = claimed_failure_ids(&text); + claimed.extend(scan.ids); + unbound.extend( + scan.unbound + .into_iter() + .map(|number| format!("{}:{number}", path.display())), + ); } + // Before asking whether the claims still have rows, insist that every + // claim reached a scenario at all. A claim that reaches none is not + // harmless: it means that scenario is no longer covered by this check, + // and the whole point of the check is that nothing drops out of it + // quietly. Naming the file and line sends the reader to the comment + // itself rather than to whichever scenario happened to follow it. + assert!( + unbound.is_empty(), + "these lines say 'Expected to FAIL' but bind to no scenario id, so the scenario they \ + describe is not covered by this check: {unbound:?}\n\ + A claim reaches only the tag line directly beneath it — no blank line in between. \ + Move the comment against its tag line, or reword it if it is not a claim about a \ + specific scenario." + ); assert!( !claimed.is_empty(), "found no scenario claiming 'Expected to FAIL', so this check would pass vacuously" diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index dab93f5a03..022b31b732 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -54,6 +54,13 @@ pub struct E2eWorld { pub cli_output: Option, pub cli_outputs: Option>, pub cli_stderr: Option, + /// STDOUT of a SECOND command, for a scenario whose whole point is that two + /// commands agree — a diagnosis against a fix preview, help examples against + /// the model listing. Its own slot rather than a spare one: `cli_stderr` + /// holds real stderr at three dozen other call sites, and borrowing it to + /// mean "the other stdout" reads, at every one of them, as a claim about + /// stderr that is not being made. + pub cli_other_output: Option, pub cli_rc: Option, /// Name of the scenario currently executing, set by the `before` hook. Used /// to tie each recorded `rocm` invocation to its scenario so the coverage @@ -292,6 +299,7 @@ impl Default for E2eWorld { cli_output: None, cli_outputs: None, cli_stderr: None, + cli_other_output: None, cli_rc: None, current_scenario: None, isolated_root: Some(root), diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 8e23ba84d8..1c1f8b7d8e 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -37,9 +37,14 @@ const PREVIEW_FIX_ID: &str = "fix-1-arch"; /// A recipe that would really change the machine, used to prove the CLI asks /// first. Of the four AUTO recipes this is the only one that reaches the /// confirmation gate on a host with nothing installed: `fix-2-unset-override` -/// never calls it on Linux, `fix-4-render-group` exits early once the user is -/// already in the groups, and `fix-6-path` exits early with "no ROCm install -/// found". This one needs only `--device-index`, which the scenario supplies. +/// never calls it on Linux (`run_unset_override_linux` only prints), `fix-6-path` +/// exits 3 with "No ROCm install found; nothing to add to PATH", and +/// `fix-4-render-group` exits 3 unless `usermod` — and `sudo`, when not root — +/// are both on `$PATH`, which is why the command-failure scenario below has to +/// shim them in. (`run_render_group` has no group-membership check at all; an +/// earlier version of this comment said it exited early "once the user is +/// already in the groups", which was never true.) This one needs only +/// `--device-index`, which the scenario supplies. const MUTATING_FIX_ID: &str = "fix-9-igpu-dgpu"; /// The recipe used to prove a failed helper command is explained on stderr with @@ -1044,11 +1049,17 @@ const DIAGNOSE_VERIFY_PREFIX: &str = "verify after fix:"; const FIX_VERIFY_PREFIX: &str = "Verify:"; /// The value after `label` on the first line that carries it. +/// +/// A bare label with nothing after it is `None`, not `Some("")`. The callers +/// compare two of these for equality, and two empty strings compare equal — so +/// returning `Some("")` would let a diagnosis and a fix preview that BOTH said +/// nothing pass as agreeing. `None` sends the caller to its own "gave no way to +/// verify this" panic, which is what an empty label actually means. fn labelled(text: &str, label: &str) -> Option { text.lines() .filter_map(|line| line.trim().strip_prefix(label)) .map(|value| value.trim().to_owned()) - .next() + .find(|value| !value.is_empty()) } /// The part of a diagnosis that belongs to one cause. @@ -1137,7 +1148,7 @@ fn compare_diagnosis_with_fix(world: &mut E2eWorld, path_override: Option<&str>) let (preview, preview_err, _) = crate::run_rocm(world, &["fix", DEVICE_PERMISSION_FIX_ID, "--dry-run"]); world.cli_output = Some(diagnosis); - world.cli_stderr = Some(format!("{preview}{preview_err}")); + world.cli_other_output = Some(format!("{preview}{preview_err}")); } #[given("a user who hit a device-permission failure")] @@ -1164,7 +1175,10 @@ async fn user_compares_diagnosis_with_fix(world: &mut E2eWorld) { #[then("both give the same way to verify that the fix worked")] async fn assert_verification_agrees(world: &mut E2eWorld) { let diagnosis = world.cli_output.as_ref().expect("no diagnosis output"); - let preview = world.cli_stderr.as_ref().expect("no fix preview output"); + let preview = world + .cli_other_output + .as_ref() + .expect("no fix preview output"); let cause = section_for(diagnosis, DEVICE_PERMISSION_FIX_ID).unwrap_or_else(|| { panic!( "the symptom did not produce the device-permission cause, so there is nothing to \ @@ -1185,7 +1199,10 @@ async fn assert_verification_agrees(world: &mut E2eWorld) { #[then("both give the same command for applying the remedy")] async fn assert_remedy_command_agrees(world: &mut E2eWorld) { let diagnosis = world.cli_output.as_ref().expect("no diagnosis output"); - let preview = world.cli_stderr.as_ref().expect("no fix preview output"); + let preview = world + .cli_other_output + .as_ref() + .expect("no fix preview output"); let cause = section_for(diagnosis, DEVICE_PERMISSION_FIX_ID).unwrap_or_else(|| { panic!( "the symptom did not produce the device-permission cause, so there is nothing to \ diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index b0f621dc25..3ae1a75c41 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -107,13 +107,13 @@ async fn user_reads_serve_examples(world: &mut E2eWorld) { let (serve, _, _) = crate::run_rocm(world, &["serve", "--help"]); let (listing, _, _) = crate::run_rocm(world, &["model", "--verbose"]); world.cli_output = Some(format!("{general}\n{serve}")); - world.cli_stderr = Some(listing); + world.cli_other_output = Some(listing); } #[then("every model named there is one the CLI can resolve")] async fn assert_example_models_resolve(world: &mut E2eWorld) { let help = world.cli_output.as_ref().expect("no help output"); - let listing = world.cli_stderr.as_ref().expect("no model listing"); + let listing = world.cli_other_output.as_ref().expect("no model listing"); let examples = serve_example_models(help); assert!( !examples.is_empty(), diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 22a989534f..2ffc341194 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1007,9 +1007,12 @@ async fn assert_no_policy_refused(world: &mut E2eWorld) { // a silent narrowing. If either moves, this guard quietly falls back to the // clap branch alone — still a real check, but no longer the one described // above — so treat a miss here as needing the strings re-sourced, not as - // evidence the command stopped refusing: - // "CPU mode is not a fallback path" apps/rocm/src/main.rs:18688 - // "unsupported device policy" apps/rocm/src/main.rs:18690 + // evidence the command stopped refusing. Both are emitted by + // `parse_device_policy` in `apps/rocm/src/main.rs`: + // "CPU mode is not a fallback path" + // "unsupported device policy" + // Cited by symbol rather than line: an earlier draft gave line numbers, and + // they had drifted by ~170 lines before this PR was through review. // Matched lowercased, which is why the literals below differ in case from // the source. let rejected: Vec<&String> = transcript From 82506e8a8eeba9d2d68e2ed8eef1a16bf5c747bb Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 15:24:14 +0200 Subject: [PATCH 28/62] test(e2e): stop claiming uniqueness for a fact recorded in two more places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last round consolidated the branch-protection restatements and then asserted that e2e-selfhosted.yml's header was "the only place the repo records" the fact. docs/ci-hardware-testing.md records it twice — in "Blocking vs. non-blocking" and again in "Required-check history", carrying the same confirmed 2026-09-11 date — and ci.yml's own comment block points the reader at that doc in the same breath as saying the fact is not restated there. So the commit whose purpose was removing comments a reader can disprove introduced one. The uniqueness claim is gone rather than made true: it is an assertion about the whole repo that nobody adding a sixth mention would think to falsify. Both files now name the two places that carry the date and say they must be re-dated together. Also for the scan added last round: its doc comment now states that it is line-shaped rather than a Gherkin parse (the phrase is assumed to appear only in real `#` comments, and `Background:` is not a block terminator), and the flush after the loop — a claim dangling at EOF, the one arm the three new tests did not reach — has a test. Verified by removing the flush, which fails it. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 19 ++++++++++++------- tests/e2e-cucumber/README.md | 8 ++++---- tests/e2e-cucumber/src/expectation.rs | 23 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 57c6652a10..c786245ca0 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -16,13 +16,18 @@ name: E2E self-hosted # merge_group ref, which a required check under ALLGREEN would have blocked; # renaming a lane is therefore safe. # -# THIS HEADER IS THE ONLY PLACE THE REPO RECORDS THAT FACT. Branch protection is -# a GitHub repository setting with no in-tree source of truth, so nothing in a -# checkout can verify the claim — it is a dated observation, and re-confirming it -# means reading the setting (`gh api repos/:owner/:repo/branches/main/protection`) -# or watching what a red lane actually blocks. ci.yml and -# tests/e2e-cucumber/README.md point here rather than asserting it again, so -# there is one line to re-date rather than four to notice. +# Branch protection is a GitHub repository setting with no in-tree source of +# truth, so nothing in a checkout can verify that claim — it is a dated +# observation. Re-confirming it means reading the setting +# (`gh api repos/:owner/:repo/branches/main/protection`) or watching what a red +# lane actually blocks. +# +# TWO places carry it, and BOTH need re-dating together: this header, and +# docs/ci-hardware-testing.md (its "Blocking vs. non-blocking" section and its +# "Required-check history" paragraph, which carries the same date and the fuller +# story of why these names used to be required). ci.yml and +# tests/e2e-cucumber/README.md only point at these two rather than asserting it +# themselves. on: push: diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 27ddd1a59a..0a09f5504b 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -112,10 +112,10 @@ carrying one runs only where its env var is set: `@nightly` `@merge-queue` (`E2E_MERGE_QUEUE`, set on the merge-queue lanes). Note what that means for gating: a `@merge-queue` scenario does not run on ordinary per-PR CI at all, and the lanes that do run it — the self-hosted ones in -`.github/workflows/e2e-selfhosted.yml`, not `ci.yml` — are non-blocking. That -file's header says why, and is the only place the repo records it; branch -protection is a GitHub setting no checkout can verify, so it is not restated -here. +`.github/workflows/e2e-selfhosted.yml`, not `ci.yml` — are non-blocking. Branch +protection is a GitHub setting no checkout can verify, so the reason is not +restated here: that file's header gives it, and `docs/ci-hardware-testing.md` +carries the fuller history. ### Naming diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 1049970e25..42ea86a0d7 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1437,6 +1437,14 @@ flaky = true /// a blank line is the same character the file already uses to separate /// scenarios. Reporting it also keeps the failure honest about where the /// problem is: the comment, not the scenario that happens to follow it. + /// + /// This is line-shaped, not a Gherkin parse. It assumes the phrase appears + /// only in real `#` comments (not inside a docstring or table cell) and + /// that `Scenario:`/`Scenario Outline:` are the only block terminators — + /// `Background:` is not treated as one. Neither construct exists anywhere in + /// `features/` today, and both would fail loudly (an unbound claim) rather + /// than silently if one appeared, but a reader adding either should revisit + /// this scan first. fn claimed_failure_ids(text: &str) -> FailureClaims { let (mut ids, mut unbound) = (Vec::new(), Vec::new()); let (mut claim_lines, mut pending_ids): (Vec, Vec) = @@ -1524,6 +1532,21 @@ flaky = true assert_eq!(claims.unbound, [3]); } + #[test] + fn a_claim_dangling_at_end_of_file_is_reported_not_dropped() { + // The `Scenario:` that would have bound this claim was deleted, leaving + // the comment as the last thing in the file. Covers the flush after the + // loop, which the blank-line and `Scenario:` cases never reach. + let claims = claimed_failure_ids( + "Feature: f\n\ + \n\ + \x20 # Expected to FAIL. The thing is broken.\n\ + \x20 @id:thing-is-broken\n", + ); + assert!(claims.ids.is_empty(), "{:?}", claims.ids); + assert_eq!(claims.unbound, [3]); + } + /// The inverse orphan: a scenario whose comment still says "Expected to /// FAIL" after its row has been deleted. /// From f71356064b369ed467699210bb1a752fb4a7523b Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 16:33:07 +0200 Subject: [PATCH 29/62] test(e2e): retire the cli_stderr exception instead of documenting around it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `cli_other_output` doc added last round said `cli_stderr` holds real stderr at every other call site. It did not: `user_inspects_both_ways` put a second command's stdout there, and its own adjacent comment said so — 33 of 34 assignments stored stderr, and that one did not. Same shape as the uniqueness claim retracted in `94a8ea2c`: a sweeping statement about the whole tree that nobody adding the next exception comes back to falsify. So the exception is gone rather than footnoted. That step and its three reader steps move to `cli_other_output`, which is the field they wanted and did not have when they were written; all 33 remaining assignments now store a stderr capture. The doc comment no longer counts call sites. It states the rule the slot keeps and the one-line grep that checks it, which is a claim a reader can settle rather than one they have to trust. Signed-off-by: Frederic Espinosa --- tests/e2e-cucumber/tests/e2e.rs | 12 ++++++++---- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 14 ++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 022b31b732..1faee49e5e 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -56,10 +56,14 @@ pub struct E2eWorld { pub cli_stderr: Option, /// STDOUT of a SECOND command, for a scenario whose whole point is that two /// commands agree — a diagnosis against a fix preview, help examples against - /// the model listing. Its own slot rather than a spare one: `cli_stderr` - /// holds real stderr at three dozen other call sites, and borrowing it to - /// mean "the other stdout" reads, at every one of them, as a claim about - /// stderr that is not being made. + /// the model listing, the human `examine` report against its `--json` form. + /// + /// Put a second stdout HERE rather than in `cli_stderr`. The rule that slot + /// keeps is "this is what the command wrote to stderr", and a reader checks + /// it with `grep 'cli_stderr = '` — every hit should be storing a stderr + /// capture. Borrowing it for a second stdout breaks that at a distance: the + /// step that reads it back looks like it is making a claim about stderr. + /// This field exists so nothing has to. pub cli_other_output: Option, pub cli_rc: Option, /// Name of the scenario currently executing, set by the `before` hook. Used diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 3ae1a75c41..eafd6e8492 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -451,9 +451,11 @@ fn parsed_json(world: &E2eWorld) -> serde_json::Value { async fn user_inspects_both_ways(world: &mut E2eWorld) { let (human, _, _) = crate::run_rocm(world, &["examine"]); let (json, _, rc) = crate::run_rocm(world, &["examine", "--json"]); - // Both are needed by the comparison step; the human form goes in the stderr - // slot rather than adding a World field for one scenario. - world.cli_stderr = Some(human); + // Both are needed by the comparison step. The human form is a second + // command's STDOUT, so it goes in `cli_other_output` — the field that did + // not exist when this step was written, which is why it used to borrow the + // stderr slot. + world.cli_other_output = Some(human); world.cli_output = Some(json); world.cli_rc = Some(rc); } @@ -527,7 +529,7 @@ fn human_states(human: &str, label: &str) -> Option { #[then("the machine-readable form states everything the readable one does")] async fn assert_json_states_what_human_does(world: &mut E2eWorld) { let human = world - .cli_stderr + .cli_other_output .as_ref() .expect("the human report was not captured"); let value = parsed_json(world); @@ -553,7 +555,7 @@ async fn assert_json_states_what_human_does(world: &mut E2eWorld) { #[then("both reports agree on whether this machine has an AMD GPU")] async fn assert_forms_agree_on_gpu(world: &mut E2eWorld) { let human = world - .cli_stderr + .cli_other_output .as_ref() .expect("the human report was not captured"); let json = parsed_json(world); @@ -576,7 +578,7 @@ async fn assert_forms_agree_on_gpu(world: &mut E2eWorld) { #[then("both reports agree on whether this platform is in scope")] async fn assert_forms_agree_on_platform(world: &mut E2eWorld) { let human = world - .cli_stderr + .cli_other_output .as_ref() .expect("the human report was not captured"); let json = parsed_json(world); From 275f4c05018319e878506ca2ce8c8ad5c250bd7f Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 18:05:08 +0200 Subject: [PATCH 30/62] test(e2e): cite product code by symbol, and stop writing counts that rot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four comments added by this PR gave line numbers that were already wrong at this head: the serve-plan `port:` line (5389 vs 5442), the `port_status` decoy (13571 vs 13742, in `run_chat_port_status_tool`), `select_runtime_update_source` (15939 vs 16427) and `resolve_latest_for_manifest` (therock.rs 810 vs 1333). The `serving_steps.rs` one is the worst of them: that comment exists to stop a reader landing on the wrong one of two identical `port: {port}` lines, says two reviewers already had, and then gave numbers pointing at neither. And the same round's commit message records fixing this exact defect for the two refusal strings while these four shipped beside it. All six line citations in this suite are now symbols — including the two that happened to still be correct, since leaving those while adding a convention against them would be the same self-contradiction. The convention is now in tests/e2e-cucumber/README.md: cite a symbol, never a line number. main.rs is over twenty thousand lines, nothing checks these numbers, and four of the six written during this review had drifted before the change adding them merged. Three smaller ones in the same spirit: - The `cli_other_output` doc offered `grep 'cli_stderr = '` as the check that settles its claim, and that pattern matched the comment itself. Anchored to the assignment so it returns the 33 real sites and not the sentence describing them. The doc also said "STDOUT", while the fix-preview site stores both streams concatenated; widened to say so. - `pending_ids.clear()` on the blank-line arm was dead to the suite. Without it a later claim silently inherits a stale id and the row check then vouches for the wrong scenario. Pinned, and verified by deleting the statement. - The workflow header said "TWO places carry it". Naming them is the useful part; the tally is the part that rots. The count is gone. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 13 ++++++------ tests/e2e-cucumber/README.md | 8 ++++++++ tests/e2e-cucumber/src/expectation.rs | 19 ++++++++++++++++++ tests/e2e-cucumber/tests/e2e.rs | 20 +++++++++++-------- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 2 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 9 ++++++--- tests/e2e-cucumber/tests/e2e/update_steps.rs | 10 +++++----- 7 files changed, 58 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index c786245ca0..4c3f1b1c0b 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -22,12 +22,13 @@ name: E2E self-hosted # (`gh api repos/:owner/:repo/branches/main/protection`) or watching what a red # lane actually blocks. # -# TWO places carry it, and BOTH need re-dating together: this header, and -# docs/ci-hardware-testing.md (its "Blocking vs. non-blocking" section and its -# "Required-check history" paragraph, which carries the same date and the fuller -# story of why these names used to be required). ci.yml and -# tests/e2e-cucumber/README.md only point at these two rather than asserting it -# themselves. +# Re-date it here AND in docs/ci-hardware-testing.md — its "Blocking vs. +# non-blocking" section and its "Required-check history" paragraph, which +# carries the same date and the fuller story of why these names used to be +# required. Those are the places known to carry it; ci.yml and +# tests/e2e-cucumber/README.md point at them rather than asserting it +# themselves. Deliberately no count of how many places there are: naming them is +# what helps, and a tally is what goes stale the moment someone adds another. on: push: diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 0a09f5504b..64dba9edbe 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -166,6 +166,14 @@ matches a condition is expected to fail (xfail); if it then passes, that is an bug. See `src/expectation.rs` for the resolver and `expectations.toml`'s header for the condition grammar. +When a comment here points at product code, **cite a symbol, never a line +number** — `select_runtime_update_source` in `apps/rocm/src/main.rs`, not +`main.rs:16427`. `apps/rocm/src/main.rs` alone is over twenty thousand lines and +moves every week; nothing checks these numbers, so a stale one sends the reader +somewhere confidently wrong, which is worse than no pointer at all. Six such +citations were written during this suite's review and four of them had drifted — +one by five hundred lines — before the change that added them had merged. + When a scenario's comment block says **"Expected to FAIL"**, write it directly above that scenario's `@id:` tag line with **no blank line between them**. A test binds each such claim to the tags immediately beneath it and fails if the row has diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 42ea86a0d7..cab13ed274 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1532,6 +1532,25 @@ flaky = true assert_eq!(claims.unbound, [3]); } + #[test] + fn a_claim_does_not_inherit_a_stale_id_from_across_a_blank_line() { + // Pins the `pending_ids.clear()` half of the blank-line arm, which the + // other scan tests leave dead: without it `stale-id` would still be + // pending when the claim below is bound, and the claim would be + // attributed to a scenario it says nothing about — silently, and with + // the row check then vouching for the wrong id. + let claims = claimed_failure_ids( + "Feature: f\n\ + \n\ + \x20 @id:stale-id\n\ + \n\ + \x20 # Expected to FAIL. The thing is broken.\n\ + \x20 Scenario: f-01 - The thing works\n", + ); + assert!(claims.ids.is_empty(), "{:?}", claims.ids); + assert_eq!(claims.unbound, [5]); + } + #[test] fn a_claim_dangling_at_end_of_file_is_reported_not_dropped() { // The `Scenario:` that would have bound this claim was deleted, leaving diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 1faee49e5e..60f5e25da4 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -54,16 +54,20 @@ pub struct E2eWorld { pub cli_output: Option, pub cli_outputs: Option>, pub cli_stderr: Option, - /// STDOUT of a SECOND command, for a scenario whose whole point is that two + /// Output of a SECOND command, for a scenario whose whole point is that two /// commands agree — a diagnosis against a fix preview, help examples against /// the model listing, the human `examine` report against its `--json` form. + /// Usually that command's stdout; where a scenario does not care which + /// stream carried a line, it may hold both concatenated (the fix preview + /// does). /// - /// Put a second stdout HERE rather than in `cli_stderr`. The rule that slot - /// keeps is "this is what the command wrote to stderr", and a reader checks - /// it with `grep 'cli_stderr = '` — every hit should be storing a stderr - /// capture. Borrowing it for a second stdout breaks that at a distance: the - /// step that reads it back looks like it is making a claim about stderr. - /// This field exists so nothing has to. + /// Put it HERE rather than in `cli_stderr`. The rule that slot keeps is + /// "this is what the command wrote to stderr", and a reader settles it with + /// `grep -rnE '^\s*world\.cli_stderr = ' tests/e2e-cucumber/` — anchored to + /// the assignment so this comment is not itself a hit. Every result should + /// be storing a stderr capture. Borrowing that slot for a second command's + /// output breaks the rule at a distance: the step reading it back looks like + /// it is making a claim about stderr. This field exists so nothing has to. pub cli_other_output: Option, pub cli_rc: Option, /// Name of the scenario currently executing, set by the `before` hook. Used @@ -945,7 +949,7 @@ pub fn stat_shim_path( .find(|candidate| std::path::Path::new(candidate).is_file()) .expect("no real `stat` binary to delegate to"); // Answers only the one call this scenario is substituting: the CLI's - // `stat -c %A|%U|%G ` (crates/rocm-core/src/examine.rs:1303). + // `stat -c %A|%U|%G ` in `stat_device` (crates/rocm-core/src/examine.rs). // Both the format AND the path must match — keying on the path alone would // hand this `mode|owner|group` line to any other `stat` of the same device // added later, in whatever format it asked for, and the caller would parse diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 1c1f8b7d8e..4fefd35b22 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -9,7 +9,7 @@ use crate::e2e::tui_driver::{TuiSession, default_timeout}; /// A symptom string that scores a catalog match on every platform these /// scenarios run on. It keys off `check_1_arch_not_in_wheel`, registered for -/// `["linux", "windows", "wsl"]` (`crates/rocm-core/src/diagnose.rs:1964`), +/// `["linux", "windows", "wsl"]` in `CHECKERS` (`crates/rocm-core/src/diagnose.rs`), /// which scores 50 on the `HSA_STATUS_ERROR_INVALID_ISA` keyword regardless of /// host state — the covered-arch penalty only applies when a framework arch /// list is present, so with none installed the match always renders. (The diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 2ffc341194..adbd9ec5d8 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1267,12 +1267,15 @@ async fn user_serves_without_choosing_address(world: &mut E2eWorld) { async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { let output = serve_output(world); // The `port:` line read here is the one `serve` prints in its "serve plan" - // block (`apps/rocm/src/main.rs:5389`), which is what this scenario ran. + // block — the `println!` under `serve plan` in `serve` itself + // (`apps/rocm/src/main.rs`) — which is what this scenario ran. // // Named because grepping the product for `port: {port}` finds a decoy first: - // `port_status` writes an identical line at `main.rs:13571` for a different + // `run_chat_port_status_tool` writes an identical line for a different // command entirely. Two reviewers have now landed on that one while checking - // this step. `serve`'s is indented and `port_status`'s is not, which is why + // this step, and a third landed on neither because an earlier draft of this + // comment gave line numbers that had already drifted — hence symbols only. + // `serve`'s is indented and the chat tool's is not, which is why // the `trim()` above is load-bearing rather than defensive — and also why // matching on indentation instead would be the wrong way to tell them apart, // since only one of the two can appear in this scenario's output at all. diff --git a/tests/e2e-cucumber/tests/e2e/update_steps.rs b/tests/e2e-cucumber/tests/e2e/update_steps.rs index 13818a5f98..d72678f39d 100644 --- a/tests/e2e-cucumber/tests/e2e/update_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/update_steps.rs @@ -120,10 +120,10 @@ async fn assert_still_manages_no_runtimes(world: &mut E2eWorld) { // this fixture CANNOT make. It would be natural to read this as "the // preview did not perform the update", but on an empty registry that // outcome is unreachable whether the product is honest or not: `--apply` - // resolves a runtime to upgrade through `select_runtime_update_source`, - // which bails with "no managed runtimes are registered" when none is - // (`apps/rocm/src/main.rs:15939`). `update` upgrades a managed runtime; it - // never installs a first one. So "manages none" here is guaranteed by the + // resolves a runtime to upgrade through `select_runtime_update_source` + // (`apps/rocm/src/main.rs`), which bails with "no managed runtimes are + // registered" when none is. `update` upgrades a managed runtime; it never + // installs a first one. So "manages none" here is guaranteed by the // fixture, and an assertion resting on it would be satisfied forever. // // What it does hold is the readback path: after a preview, `update` still @@ -133,7 +133,7 @@ async fn assert_still_manages_no_runtimes(world: &mut E2eWorld) { // Proving THAT needs a scenario with a managed runtime registered, where a // performed update is observable. It is not done here on purpose: with a // manifest present, `render_update_report` resolves the latest version per - // runtime (`apps/rocm/src/therock.rs:810` → `resolve_latest_for_manifest`), + // runtime (`resolve_latest_for_manifest` in `apps/rocm/src/therock.rs`), // which reaches the TheRock index — and this scenario runs on the mock lane, // which has no network. Tracked on EAI-8010 rather than forced in here. // From 5c5ae676928cf136c1315b07927047483767d8b4 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 19:06:20 +0200 Subject: [PATCH 31/62] test(e2e): correct the non-blocking reason in the report, and pin the scan's refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The consolidated report told readers the self-hosted lanes are non-blocking "while proven out" — the same provisional framing removed from ci.yml earlier in this PR, wrong for the same reason. They are non-blocking because their check names are not in branch protection's required list. Both spots in crates/e2e-report now say that and point at e2e-selfhosted.yml's header, which lists them among the places to re-date. Three smaller ones: - The sentence arguing that counts rot contained a count, and it was wrong (eight citations, not six). Replaced with the observation it was there to support. - `feature_files()`'s subdirectory refusal could only ever fire against the real features/ directory, so nothing exercised it. Split the scan into `feature_files_in` and pinned both branches against a temp tree. - Said plainly in the `cli_other_output` doc that the move is readability, not a behaviour fix: every assertion that moved would pass unchanged if the same strings went back into `cli_stderr`. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 5 +++-- crates/e2e-report/src/lib.rs | 9 ++++---- tests/e2e-cucumber/README.md | 6 +++--- tests/e2e-cucumber/src/expectation.rs | 31 +++++++++++++++++++++++++-- tests/e2e-cucumber/tests/e2e.rs | 5 +++++ 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 4c3f1b1c0b..928a40e4b8 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -25,8 +25,9 @@ name: E2E self-hosted # Re-date it here AND in docs/ci-hardware-testing.md — its "Blocking vs. # non-blocking" section and its "Required-check history" paragraph, which # carries the same date and the fuller story of why these names used to be -# required. Those are the places known to carry it; ci.yml and -# tests/e2e-cucumber/README.md point at them rather than asserting it +# required. Those are the places known to carry it; ci.yml, +# tests/e2e-cucumber/README.md and crates/e2e-report (which prints the reason +# into the consolidated report) point at them rather than asserting it # themselves. Deliberately no count of how many places there are: naming them is # what helps, and a tally is what goes stale the moment someone adds another. diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 5d0d3e157e..4cc424bcd7 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -1303,9 +1303,9 @@ pub fn consolidated_summary_markdown(inputs: &[(String, PathBuf)]) -> String { engines — the rows below are the list, so naming them here would only go \ stale as lanes are added. They are **non-blocking**: they still run and \ are reported here, but \ - a failure does NOT block the PR from merging (the hardware/runners are still \ - being proven out, so their results are informational rather than a merge \ - gate).\n\n\ + a failure does NOT block the PR from merging — their check names are not in \ + branch protection's required list. See `.github/workflows/e2e-selfhosted.yml`'s \ + header for the dated confirmation and where else it is recorded.\n\n\ Column meanings: **Pass** = scenarios that passed as expected; \ **Xfail** = known bugs that failed as expected (healthy — the bug still \ reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ @@ -1939,7 +1939,8 @@ fn legend() -> Markup { } li { b { "Every other platform" } - " — real self-hosted GPU hardware; non-blocking while proven out." + " — real self-hosted GPU hardware; non-blocking because these \ + check names are not in branch protection's required list." } li { b { "Status" } diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 64dba9edbe..c3cd0f0555 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -170,9 +170,9 @@ When a comment here points at product code, **cite a symbol, never a line number** — `select_runtime_update_source` in `apps/rocm/src/main.rs`, not `main.rs:16427`. `apps/rocm/src/main.rs` alone is over twenty thousand lines and moves every week; nothing checks these numbers, so a stale one sends the reader -somewhere confidently wrong, which is worse than no pointer at all. Six such -citations were written during this suite's review and four of them had drifted — -one by five hundred lines — before the change that added them had merged. +somewhere confidently wrong, which is worse than no pointer at all. Most of the +citations written during this suite's review had already drifted — one by five +hundred lines — before the change that added them had merged. When a scenario's comment block says **"Expected to FAIL"**, write it directly above that scenario's `@id:` tag line with **no blank line between them**. A test diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index cab13ed274..eb3f7f4ba1 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1392,9 +1392,15 @@ flaky = true /// vanish from the checks below, and a check that silently stops looking at /// something is worse than no check at all. fn feature_files() -> Vec { - let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("features"); + feature_files_in(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("features")) + } + + /// The directory scan itself, split out so the subdirectory refusal can be + /// exercised against a temporary tree — against the real `features/` it + /// could only ever fire by someone breaking the repository. + fn feature_files_in(dir: &std::path::Path) -> Vec { let mut paths = Vec::new(); - for entry in std::fs::read_dir(&dir).expect("no features directory") { + for entry in std::fs::read_dir(dir).expect("no features directory") { let path = entry.expect("unreadable features directory entry").path(); assert!( !path.is_dir(), @@ -1415,6 +1421,27 @@ flaky = true paths } + #[test] + fn the_feature_scan_takes_the_flat_files_it_finds() { + let dir = tempfile::tempdir().expect("no temp dir"); + std::fs::write(dir.path().join("a.feature"), "Feature: a\n").unwrap(); + std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); + let found = feature_files_in(dir.path()); + assert_eq!(found.len(), 1, "{found:?}"); + assert!(found[0].ends_with("a.feature"), "{found:?}"); + } + + #[test] + #[should_panic(expected = "has grown a subdirectory")] + fn the_feature_scan_refuses_a_subdirectory_rather_than_skipping_it() { + // The branch that stops this scan quietly covering less than it claims. + // Unreachable against the real `features/`, so it is pinned here. + let dir = tempfile::tempdir().expect("no temp dir"); + std::fs::write(dir.path().join("a.feature"), "Feature: a\n").unwrap(); + std::fs::create_dir(dir.path().join("nested")).unwrap(); + let _ = feature_files_in(dir.path()); + } + /// What one feature file claims about expected failures. struct FailureClaims { /// Scenario ids introduced by a comment block claiming "Expected to diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 60f5e25da4..62fbda2ef8 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -68,6 +68,11 @@ pub struct E2eWorld { /// be storing a stderr capture. Borrowing that slot for a second command's /// output breaks the rule at a distance: the step reading it back looks like /// it is making a claim about stderr. This field exists so nothing has to. + /// + /// Readability only — NOT a bug fix. Every assertion that moved here would + /// pass unchanged if the same strings went back into `cli_stderr`; what + /// changes is whether a reader of those steps is misled about which stream + /// they are asserting on. pub cli_other_output: Option, pub cli_rc: Option, /// Name of the scenario currently executing, set by the `before` hook. Used From bc87b89326dd775976f396f925d56b121c689478 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 20:36:11 +0200 Subject: [PATCH 32/62] test(e2e): fix the proven-out reason where it is most visible, and guard the sibling scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings that both amount to fixing a class in one place and leaving it in the next one over. The report crate now sends readers to e2e-selfhosted.yml's header for why the self-hosted lanes are non-blocking, but three per-lane comments IN THAT FILE still gave the provisional "proven out" reason — the one this PR has now replaced three times elsewhere. Corrected at :387, :704 and :890; the file no longer contradicts its own header. `feature_files()` in src/expectation.rs refuses to run against a features/ subdirectory rather than silently covering less, and its message tells the maintainer to fix tests/feature_naming.rs too. That scan had no such guard, so the same subdirectory would have failed one loudly and been skipped by the other. It now carries the same refusal, pointing back. Verified by creating features/nested-probe: the naming target fails naming the path, and passes again once removed. Also, update-04 asserted a known bug without saying which side of the disagreement is wrong. A reviewer reasonably read `--dry-run` requiring `--apply` as possibly deliberate, since bare `update` is already the preview. The row and the scenario now cite README.md's synopsis — which brackets `[--apply]` and `[--dry-run]` independently — as the evidence, and say the row asserts only that documentation and behaviour disagree: correcting the synopsis would retire it just as well as accepting the flag. Which one is a maintainer call, and the row no longer pre-empts it. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 11 +++++---- tests/e2e-cucumber/expectations.toml | 14 ++++++++--- tests/e2e-cucumber/features/update.feature | 11 ++++++--- tests/e2e-cucumber/tests/feature_naming.rs | 27 ++++++++++++++++++---- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 928a40e4b8..d4b82a30cd 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -384,7 +384,8 @@ jobs: # CARGO_HOME override (pool hosts have room under the default paths, unlike # the full-partition static host this replaced), though CARGO_TARGET_DIR # still points at RUNNER_WORKSPACE to dodge checkout's `git clean -ffdx` - # mid-job. Non-blocking while this hardware is proven out. + # mid-job. Non-blocking for the reason in this file's header, not because + # this hardware is still being proven out. e2e-gpu-strix-ubuntu: name: E2E tests (Strix Halo, Ubuntu) # 45min: see e2e-gpu — one collapsed job runs all serves + per-scenario @@ -700,8 +701,9 @@ jobs: # boundary AND whatever GPU access WSL exposes. Same suite as every other # platform — scenarios the host cannot satisfy resolve to skip from the # capability probe, so nothing is filtered out here. `wsl` in `runs-on` - # disambiguates it from the `native` Strix Linux runner. Non-blocking while - # GPU-on-WSL is proven out. + # disambiguates it from the `native` Strix Linux runner. Non-blocking for the + # reason in this file's header, not because GPU-on-WSL is still being proven + # out. e2e-wsl: name: E2E tests (Strix Halo, WSL2) # A manual include_nightly dispatch runs the 2400s large-model readiness @@ -887,7 +889,8 @@ jobs: # Third AMD GPU target: the rad3 cluster's Radeon AI PRO R9700S (gfx1201, 32GB), # targeted by the `r9700` label. This runner is a KEDA-scaled pod (0<->1 off the # job queue), so the card is only reserved while a job exists (dev-tooling EAI-8065). - # Non-blocking while this hardware is proven out. + # Non-blocking for the reason in this file's header, not because this hardware + # is still being proven out. e2e-gpu-rad3: name: E2E tests (rad3 R9700) # 90min: matches e2e-gpu — same collapsed suite, and the card is smaller, so diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 99f6d7d92b..1acd8ad364 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -174,9 +174,17 @@ flaky = true # deletes itself. # --- EAI-8010: `--dry-run` is declared as requiring `--apply`, so previewing an -# update without asking for one is rejected as a usage error, though the README -# documents the two as independent. Argument handling only, so platform- -# independent. Remove when `rocm update --dry-run` is accepted. --- +# update without asking for one is rejected as a usage error, though README.md's +# synopsis line for `rocm update` brackets `[--apply]` and `[--dry-run]` +# independently. Argument handling only, so platform-independent. +# +# WHICH SIDE IS WRONG IS A MAINTAINER CALL, and this row does not presume it. +# The defensible alternative reading is that bare `rocm update` is already the +# preview, making `--dry-run` meaningful only alongside `--apply` — in which case +# the synopsis is what misleads. This row asserts only that the documented +# contract and the shipped behaviour disagree, which is true either way. +# Remove it when they agree: either `rocm update --dry-run` is accepted, or the +# README stops offering it as a standalone flag. --- [["update-preview-without-applying"]] when = {} bug = "EAI-8010" diff --git a/tests/e2e-cucumber/features/update.feature b/tests/e2e-cucumber/features/update.feature index 5050b09c16..aac85a16ca 100644 --- a/tests/e2e-cucumber/features/update.feature +++ b/tests/e2e-cucumber/features/update.feature @@ -32,9 +32,14 @@ Feature: Update report # # Expected to FAIL. Asking to see what an update would do, without asking for # it to be done, is refused as a misuse — even though checking is what this - # command does when left alone. The two choices are documented as independent - # of each other, so a user who wants a preview before committing to anything - # is turned away from the one command that would give them it. + # command does when left alone. README.md's synopsis for `rocm update` brackets + # `[--apply]` and `[--dry-run]` as independent options, so a user who wants a + # preview before committing to anything is turned away from the one command + # the documentation told them would give it. + # + # The row in expectations.toml records why this does not presume which side is + # wrong: closing the gap by correcting the synopsis would satisfy it just as + # well as accepting the flag. @id:update-preview-without-applying Scenario: update-04 - Previewing an update without asking to install it is accepted When the user asks to see what updating would do without asking for it to be done diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index e86b1bd583..467a30ea5d 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -49,12 +49,29 @@ fn features_dir() -> PathBuf { } /// Every `.feature` file actually present, by file name. +/// +/// Flat, not recursive — and it refuses to run rather than quietly covering +/// less if that stops being true. `feature_files` in `src/expectation.rs` makes +/// the same assumption and its refusal message names THIS scan as the other +/// place to fix, so the two guards belong together: a subdirectory that made +/// one of them fail loudly while the other silently skipped it would be the +/// worst of both. fn feature_files() -> Vec { - let mut names: Vec = std::fs::read_dir(features_dir()) - .expect("features dir") - .map(|e| e.expect("dir entry").file_name().to_string_lossy().into()) - .filter(|n: &String| n.ends_with(".feature")) - .collect(); + let mut names: Vec = Vec::new(); + for entry in std::fs::read_dir(features_dir()).expect("features dir") { + let path = entry.expect("dir entry").path(); + assert!( + !path.is_dir(), + "features/ has grown a subdirectory ({}), which this scan does not descend \ + into — make it recursive, here and in src/expectation.rs, before moving any \ + .feature file into one", + path.display() + ); + let name = path.file_name().expect("dir entry name").to_string_lossy(); + if name.ends_with(".feature") { + names.push(name.into_owned()); + } + } names.sort(); names } From f911952bb15aa0ad4fd1decd55796234dfb1edd3 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 21:04:19 +0200 Subject: [PATCH 33/62] test(e2e): pin the sibling refusal, and widen the update row to its real scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings, all the same class the last round was meant to close. The subdirectory refusal added to feature_naming.rs last commit was itself unpinned — added in the commit whose point was that the two scans should not diverge, while its twin in src/expectation.rs already had a should_panic test. Split `feature_files_in(dir)` the same way and pinned both branches. Verified by deleting the assert!, which now fails the test rather than nothing. The EAI-8010 row named only `--dry-run`, but `--runtime` and `--activate` carry the same `requires = "apply"` (apps/rocm/src/main.rs, the `Update` variant), and the README synopsis brackets all four as independent — so three of its four options are documented as usable alone and none is. The row now says so, and says to fix all three together: a synopsis correction covering `--dry-run` alone would retire this row while leaving its two siblings broken. And a fourth lane in e2e-selfhosted.yml still justified itself as "Non-blocking so it never gates the PR" — restating the conclusion instead of the header's reason. Last round fixed the three that said "proven out"; this one did not match that grep. Now deferring to the header like the others, and a sweep of the file says these four were all of them. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 2 +- tests/e2e-cucumber/expectations.toml | 18 ++++++++---- tests/e2e-cucumber/tests/feature_naming.rs | 32 +++++++++++++++++++++- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index d4b82a30cd..cb63147434 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -527,7 +527,7 @@ jobs: # First real Windows GPU coverage: the Strix Halo Windows 11 runner. The # existing windows-build-and-test uses GitHub-hosted windows-latest, which has - # no GPU. Non-blocking so it never gates the PR. + # no GPU. Non-blocking for the reason in this file's header. e2e-gpu-strix-windows: name: E2E tests (Strix Halo, Windows) # 45min: see e2e-gpu — one collapsed job runs all serves + per-scenario diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 1acd8ad364..d021293e72 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -178,13 +178,21 @@ flaky = true # synopsis line for `rocm update` brackets `[--apply]` and `[--dry-run]` # independently. Argument handling only, so platform-independent. # +# NOT ONLY `--dry-run`. `--runtime` and `--activate` carry the same +# `requires = "apply"`, and that one synopsis line brackets all four as if +# independent — so three of its four options are documented as usable on their +# own and none of them is. This scenario exercises `--dry-run` because it is the +# one a README reader reaches for first, not because it is the only case. +# # WHICH SIDE IS WRONG IS A MAINTAINER CALL, and this row does not presume it. # The defensible alternative reading is that bare `rocm update` is already the -# preview, making `--dry-run` meaningful only alongside `--apply` — in which case -# the synopsis is what misleads. This row asserts only that the documented -# contract and the shipped behaviour disagree, which is true either way. -# Remove it when they agree: either `rocm update --dry-run` is accepted, or the -# README stops offering it as a standalone flag. --- +# preview, making these meaningful only alongside `--apply` — in which case the +# synopsis is what misleads. This row asserts only that the documented contract +# and the shipped behaviour disagree, which is true either way. +# +# Remove it when they agree — and fix all three together, because a synopsis +# correction that covers `--dry-run` alone leaves the identical defect standing +# for its two siblings while this row goes quiet. --- [["update-preview-without-applying"]] when = {} bug = "EAI-8010" diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index 467a30ea5d..7901c29286 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -57,8 +57,17 @@ fn features_dir() -> PathBuf { /// one of them fail loudly while the other silently skipped it would be the /// worst of both. fn feature_files() -> Vec { + feature_files_in(&features_dir()) +} + +/// The scan itself, split out so the subdirectory refusal can be exercised +/// against a temporary tree — against the real `features/` it could only fire +/// by someone breaking the repository. Mirrors `feature_files_in` in +/// `src/expectation.rs`, which is split for the same reason: the two guards +/// should match in coverage, not only in wording. +fn feature_files_in(dir: &Path) -> Vec { let mut names: Vec = Vec::new(); - for entry in std::fs::read_dir(features_dir()).expect("features dir") { + for entry in std::fs::read_dir(dir).expect("features dir") { let path = entry.expect("dir entry").path(); assert!( !path.is_dir(), @@ -76,6 +85,27 @@ fn feature_files() -> Vec { names } +#[test] +fn the_feature_scan_takes_the_flat_files_it_finds() { + let dir = tempfile::tempdir().expect("no temp dir"); + std::fs::write(dir.path().join("b.feature"), "Feature: b\n").unwrap(); + std::fs::write(dir.path().join("a.feature"), "Feature: a\n").unwrap(); + std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); + assert_eq!(feature_files_in(dir.path()), ["a.feature", "b.feature"]); +} + +#[test] +#[should_panic(expected = "has grown a subdirectory")] +fn the_feature_scan_refuses_a_subdirectory_rather_than_skipping_it() { + // The branch that stops this scan quietly covering less than it claims. + // Unreachable against the real `features/`, so it is pinned here — the same + // way `src/expectation.rs` pins its twin. + let dir = tempfile::tempdir().expect("no temp dir"); + std::fs::write(dir.path().join("a.feature"), "Feature: a\n").unwrap(); + std::fs::create_dir(dir.path().join("nested")).unwrap(); + let _ = feature_files_in(dir.path()); +} + /// The `@id:` tags and scenario names in one feature file, paired in declaration /// order. Tags precede their scenario, so the most recent id seen belongs to the /// next scenario line. From 462d9def59b9349ab1002b66b737ace8c0288d30 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 21:56:58 +0200 Subject: [PATCH 34/62] test(e2e): make the two feature scans actually agree, and fix the header they rely on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim that feature_naming.rs's scan "mirrors" the one in src/expectation.rs was wrong in a way that mattered: the sibling filters on Path::extension() and asserts the result is non-empty, while this one matched a `.feature` string suffix and asserted nothing. A file named exactly `.feature` has no extension, so one scan would have taken it and the other skipped it — the precise divergence the pair exists to prevent, in the commit that claimed to close it. Both now use extension() and both assert non-empty, and both tests plant a `.feature` file so the agreement is pinned rather than asserted. Verified by reverting this scan to the suffix match, which now fails its test. Also: - The flat-file test looked like it pinned `names.sort()`, and did not: read_dir order is unspecified but comes back alphabetical, so deleting the sort leaves every test green (reproduced). It pins filtering and membership; the comment now says so rather than leaving the next reader to repeat the assumption the fixture layout encodes. - This file's header said `continue-on-error` is `false` "below", and four lane comments now defer to that header for their reason. e2e-gpu-mi350p sets it `true`. Corrected to name the exception, since the sentence is load-bearing for the four comments pointing at it. - The EAI-8010 row said the synopsis "brackets all four as if independent". It spans two lines and six options. Scoped to the three that carry requires = "apply", and says why --json and --timeout-secs are excluded: the README prose under the synopsis states their relationship and the code agrees. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 7 ++++-- tests/e2e-cucumber/expectations.toml | 13 ++++++---- tests/e2e-cucumber/src/expectation.rs | 5 ++++ tests/e2e-cucumber/tests/feature_naming.rs | 28 ++++++++++++++++++---- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index cb63147434..e35accd198 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -10,8 +10,11 @@ name: E2E self-hosted # # These lanes are non-blocking because none of their check names are in # branch protection's required-status-check list (confirmed 2026-09-11) — not -# because of `continue-on-error`, which is `false` below so a real hardware -# regression shows red instead of always green. PRs #334, #335 and #343 all +# because of `continue-on-error`, which is `false` on every lane below except +# `e2e-gpu-mi350p`, so a real hardware regression shows red instead of always +# green. (mi350p alone sets it `true`; that lane reports green regardless, which +# is a property of that lane and not the reason any of them is non-blocking.) +# PRs #334, #335 and #343 all # landed while `E2E tests (Strix Halo, Ubuntu)` reported `cancelled` on the # merge_group ref, which a required check under ALLGREEN would have blocked; # renaming a lane is therefore safe. diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index d021293e72..a05aa68f27 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -179,10 +179,15 @@ flaky = true # independently. Argument handling only, so platform-independent. # # NOT ONLY `--dry-run`. `--runtime` and `--activate` carry the same -# `requires = "apply"`, and that one synopsis line brackets all four as if -# independent — so three of its four options are documented as usable on their -# own and none of them is. This scenario exercises `--dry-run` because it is the -# one a README reader reaches for first, not because it is the only case. +# `requires = "apply"`, and the synopsis brackets all three alongside `[--apply]` +# as if independent — so three options are documented as usable on their own and +# none of them is. This scenario exercises `--dry-run` because it is the one a +# README reader reaches for first, not because it is the only case. +# +# Scoped to those three on purpose. The synopsis spans two lines and six +# options; the other two, `--json` and `--timeout-secs`, are NOT part of this: +# the README prose under it spells their relationship out ("`--timeout-secs` +# requires `--json`; both conflict with `--apply`"), and the code agrees. # # WHICH SIDE IS WRONG IS A MAINTAINER CALL, and this row does not presume it. # The defensible alternative reading is that bare `rocm update` is already the diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index eb3f7f4ba1..46d0a6ce9a 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1426,6 +1426,11 @@ flaky = true let dir = tempfile::tempdir().expect("no temp dir"); std::fs::write(dir.path().join("a.feature"), "Feature: a\n").unwrap(); std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); + // A file named exactly `.feature` has no extension, so this scan skips + // it. Pinned on both sides — `tests/feature_naming.rs` once matched on a + // `.feature` string suffix instead, which would have taken this one and + // left the two scans covering different sets. + std::fs::write(dir.path().join(".feature"), "not a feature file\n").unwrap(); let found = feature_files_in(dir.path()); assert_eq!(found.len(), 1, "{found:?}"); assert!(found[0].ends_with("a.feature"), "{found:?}"); diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index 7901c29286..d740161786 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -62,9 +62,14 @@ fn feature_files() -> Vec { /// The scan itself, split out so the subdirectory refusal can be exercised /// against a temporary tree — against the real `features/` it could only fire -/// by someone breaking the repository. Mirrors `feature_files_in` in -/// `src/expectation.rs`, which is split for the same reason: the two guards -/// should match in coverage, not only in wording. +/// by someone breaking the repository. +/// +/// Deliberately the same shape as `feature_files_in` in `src/expectation.rs`: +/// same refusal, the same `Path::extension()` test rather than a `.feature` +/// string suffix, and the same non-empty assertion. The extension test is not +/// interchangeable with the suffix one — a file named exactly `.feature` has no +/// extension and would have been taken by one scan and skipped by the other, +/// which is the divergence this pair exists to prevent. fn feature_files_in(dir: &Path) -> Vec { let mut names: Vec = Vec::new(); for entry in std::fs::read_dir(dir).expect("features dir") { @@ -76,21 +81,34 @@ fn feature_files_in(dir: &Path) -> Vec { .feature file into one", path.display() ); - let name = path.file_name().expect("dir entry name").to_string_lossy(); - if name.ends_with(".feature") { + if path.extension().is_some_and(|ext| ext == "feature") { + let name = path.file_name().expect("dir entry name").to_string_lossy(); names.push(name.into_owned()); } } + assert!( + !names.is_empty(), + "found no .feature files in {}", + dir.display() + ); names.sort(); names } #[test] fn the_feature_scan_takes_the_flat_files_it_finds() { + // Pins FILTERING and MEMBERSHIP, not sort order. `read_dir` order is + // unspecified and happens to come back alphabetical here, so removing + // `names.sort()` leaves this green — verified. Nothing can force a real + // directory to yield entries out of order, so the ordering is asserted for + // a stable comparison rather than because this test proves it. let dir = tempfile::tempdir().expect("no temp dir"); std::fs::write(dir.path().join("b.feature"), "Feature: b\n").unwrap(); std::fs::write(dir.path().join("a.feature"), "Feature: a\n").unwrap(); std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); + // A file named exactly `.feature` has no extension, so neither this scan nor + // its sibling takes it. Present here so the two stay agreed on that. + std::fs::write(dir.path().join(".feature"), "not a feature file\n").unwrap(); assert_eq!(feature_files_in(dir.path()), ["a.feature", "b.feature"]); } From e8482bfa817dbb57c371fd5df85fc8ff92405431 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 22:27:27 +0200 Subject: [PATCH 35/62] test(e2e): pin the non-empty assertion the last commit added but did not test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both feature scans assert they found at least one .feature file — the guard that stops every check downstream of them looping over nothing and agreeing with everything. Neither side had a test that reached it: no fixture used an empty directory, so deleting the assertion left all tests green on both sides (reproduced). The one guarantee that commit added was the one nothing pinned. A #[should_panic(expected = "found no .feature files")] test over a temp dir holding only a non-.feature file now covers it in each file, in the same shape as the existing subdirectory refusal. Mutation-verified on both sides: deleting either assertion now fails its own test and nothing else. Also reflows the e2e-selfhosted.yml header paragraph, where the previous reword left `PRs #334, #335 and #343 all` as a 29-character orphan line in the middle of a sentence. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 7 +++---- tests/e2e-cucumber/src/expectation.rs | 12 ++++++++++++ tests/e2e-cucumber/tests/feature_naming.rs | 13 +++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index e35accd198..65519962dd 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -14,10 +14,9 @@ name: E2E self-hosted # `e2e-gpu-mi350p`, so a real hardware regression shows red instead of always # green. (mi350p alone sets it `true`; that lane reports green regardless, which # is a property of that lane and not the reason any of them is non-blocking.) -# PRs #334, #335 and #343 all -# landed while `E2E tests (Strix Halo, Ubuntu)` reported `cancelled` on the -# merge_group ref, which a required check under ALLGREEN would have blocked; -# renaming a lane is therefore safe. +# PRs #334, #335 and #343 all landed while `E2E tests (Strix Halo, Ubuntu)` +# reported `cancelled` on the merge_group ref, which a required check under +# ALLGREEN would have blocked; renaming a lane is therefore safe. # # Branch protection is a GitHub repository setting with no in-tree source of # truth, so nothing in a checkout can verify that claim — it is a dated diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 46d0a6ce9a..cfd7bf90f9 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1436,6 +1436,18 @@ flaky = true assert!(found[0].ends_with("a.feature"), "{found:?}"); } + #[test] + #[should_panic(expected = "found no .feature files")] + fn the_feature_scan_refuses_an_empty_directory_rather_than_passing_vacuously() { + // Without this, the callers that walk the scan's result would walk + // nothing and agree with everything. Unreachable against the real + // `features/`, so pinned here — and pinned the same way in + // `tests/feature_naming.rs`, whose scan carries the same assertion. + let dir = tempfile::tempdir().expect("no temp dir"); + std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); + let _ = feature_files_in(dir.path()); + } + #[test] #[should_panic(expected = "has grown a subdirectory")] fn the_feature_scan_refuses_a_subdirectory_rather_than_skipping_it() { diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index d740161786..e5400ff9ef 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -112,6 +112,19 @@ fn the_feature_scan_takes_the_flat_files_it_finds() { assert_eq!(feature_files_in(dir.path()), ["a.feature", "b.feature"]); } +#[test] +#[should_panic(expected = "found no .feature files")] +fn the_feature_scan_refuses_an_empty_directory_rather_than_passing_vacuously() { + // Without this, every check in this file is a loop over nothing: a + // `features/` that stopped yielding files would pass the whole guard rather + // than fail it. Unreachable against the real directory, so pinned here — + // and pinned identically in `src/expectation.rs`, whose scan makes the same + // assertion for the same reason. + let dir = tempfile::tempdir().expect("no temp dir"); + std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); + let _ = feature_files_in(dir.path()); +} + #[test] #[should_panic(expected = "has grown a subdirectory")] fn the_feature_scan_refuses_a_subdirectory_rather_than_skipping_it() { From f5629a94b485d63fc52ca237e8602c7fb57f0839 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 22:54:09 +0200 Subject: [PATCH 36/62] test(e2e): say what the non-empty assertion actually buys, not what it does not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two comments added with the last commit's tests claimed that without the non-empty assertion the downstream checks would pass vacuously. They would not. Pointing features_dir() at an empty directory with the assertion deleted fails four of the five checks in feature_naming.rs — one on the reciprocal "FEATURE_KEYS lists X, which does not exist", three on a read error inside scenarios_of — and both feature_files() callers in expectation.rs carry their own explicit vacuity assertions, so they fail too. The assertion is still worth having: it reports an emptied features/ ONCE, at the scan, naming the directory, instead of surfacing as a confusing key mismatch or a file-not-found deep in another check. Both comments now say that, which is the honest reason and leaves the next maintainer with the right conclusion about the other checks rather than the opposite one. Also renames both tests to ..._refuses_a_directory_with_no_feature_files: the fixture writes a notes.md on purpose, because what the scan refuses is an empty RESULT and not an empty directory, and the old name invited someone to "tidy" that file away and quietly narrow the test. Signed-off-by: Frederic Espinosa --- tests/e2e-cucumber/src/expectation.rs | 19 ++++++++++++++----- tests/e2e-cucumber/tests/feature_naming.rs | 22 ++++++++++++++++------ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index cfd7bf90f9..3a3546e487 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1438,11 +1438,20 @@ flaky = true #[test] #[should_panic(expected = "found no .feature files")] - fn the_feature_scan_refuses_an_empty_directory_rather_than_passing_vacuously() { - // Without this, the callers that walk the scan's result would walk - // nothing and agree with everything. Unreachable against the real - // `features/`, so pinned here — and pinned the same way in - // `tests/feature_naming.rs`, whose scan carries the same assertion. + fn the_feature_scan_refuses_a_directory_with_no_feature_files() { + // The assertion this pins reports an emptied `features/` once, at the + // scan, naming the directory. It is not what stops the two callers + // agreeing with everything — both already guard themselves ("read no + // `@id:` tags at all, so this check would pass vacuously" and its + // counterpart over claimed ids), so they fail either way; this just + // fails first and says what actually went wrong. + // + // Unreachable against the real `features/`, so pinned here — and + // pinned the same way in `tests/feature_naming.rs`, whose scan carries + // the same assertion. + // + // The fixture writes a non-`.feature` file on purpose: what the scan + // refuses is an empty RESULT, not an empty directory. let dir = tempfile::tempdir().expect("no temp dir"); std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); let _ = feature_files_in(dir.path()); diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index e5400ff9ef..0b0791438d 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -114,12 +114,22 @@ fn the_feature_scan_takes_the_flat_files_it_finds() { #[test] #[should_panic(expected = "found no .feature files")] -fn the_feature_scan_refuses_an_empty_directory_rather_than_passing_vacuously() { - // Without this, every check in this file is a loop over nothing: a - // `features/` that stopped yielding files would pass the whole guard rather - // than fail it. Unreachable against the real directory, so pinned here — - // and pinned identically in `src/expectation.rs`, whose scan makes the same - // assertion for the same reason. +fn the_feature_scan_refuses_a_directory_with_no_feature_files() { + // The assertion this pins turns a `features/` that stopped yielding files + // into ONE clear failure naming the directory. It is not what stops the + // other checks passing vacuously — they already fail on their own, just + // confusingly: `feature_files_and_declared_keys_agree` reports the first + // FEATURE_KEYS entry as one that "does not exist", and the other three + // panic inside `scenarios_of` with a read error. Verified by pointing + // `features_dir()` at an empty directory with the assertion deleted: four + // of the five checks fail, none of them saying the directory is empty. + // + // Unreachable against the real directory, so pinned here — and pinned the + // same way in `src/expectation.rs`, whose scan carries the same assertion. + // + // The fixture writes a non-`.feature` file on purpose: what the scan + // refuses is an empty RESULT, not an empty directory, and this covers the + // stronger case. let dir = tempfile::tempdir().expect("no temp dir"); std::fs::write(dir.path().join("notes.md"), "ignored\n").unwrap(); let _ = feature_files_in(dir.path()); From 4e2a22b72a6b7e057913d2b0c6be9c4d4c18daea Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 15 Sep 2026 23:43:22 +0200 Subject: [PATCH 37/62] docs(ci): re-date the required-check observation, and say how it was read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-blocking claim these lanes rest on is a dated observation of a GitHub repository setting, with no in-tree source of truth — the workflow header says so and tells the next person to re-date it in both places when they re-confirm it. It was last read on 2026-09-11 and was questioned again today, so this is that re-confirmation rather than a restatement. Read from required_status_checks.contexts: sixteen names, none of which is a self-hosted lane. `ci.yml`'s mock `E2E tests` and `E2E consolidated report` are required and are distinct strings from every self-hosted name, including `E2E consolidated report (self-hosted)` — the one pair close enough to collide if GitHub matched loosely, which it does not. The docs paragraph now records the field that was read, so a later reader can repeat the check rather than take the date on trust. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 2 +- docs/ci-hardware-testing.md | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 65519962dd..e0dd12ef67 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -9,7 +9,7 @@ name: E2E self-hosted # THIS workflow's supersession, never the required checks in ci.yml. # # These lanes are non-blocking because none of their check names are in -# branch protection's required-status-check list (confirmed 2026-09-11) — not +# branch protection's required-status-check list (confirmed 2026-09-15) — not # because of `continue-on-error`, which is `false` on every lane below except # `e2e-gpu-mi350p`, so a real hardware regression shows red instead of always # green. (mi350p alone sets it `true`; that lane reports green regardless, which diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 845068250f..70c90f0989 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -256,13 +256,14 @@ fails, just later. (self-hosted)` — used to be in `main`'s required-status-check list, where a required check that *never reports* (because its self-hosted runner is offline) is treated as missing and still blocks the merge, `continue-on-error` -notwithstanding. That branch-protection change has since landed (confirmed -2026-09-11): none of the self-hosted lane names, nor the self-hosted -consolidated report, are in the required list anymore (`ci.yml`'s own mock -`E2E tests` / `E2E consolidated report` are the required checks — similarly -prefixed but distinct from the self-hosted names above, so no name string -actually collides). An offline or unclaimed self-hosted runner can no longer -block a merge. +notwithstanding. That branch-protection change has since landed (re-confirmed +2026-09-15 by reading `required_status_checks.contexts`, which lists sixteen +names and none of these): none of the self-hosted lane names, nor the +self-hosted consolidated report, are in the required list anymore (`ci.yml`'s +own mock `E2E tests` / `E2E consolidated report` are the required checks — +similarly prefixed but distinct from the self-hosted names above, so no name +string actually collides). An offline or unclaimed self-hosted runner can no +longer block a merge. ## Fork safety From 14e0c553e62595091a6467aafa2d32b5d9ca77b0 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 16 Sep 2026 00:02:27 +0200 Subject: [PATCH 38/62] test(xtask): make the two copies of the required-check date fail when they drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim these lanes rest on is a dated reading of a GitHub setting with no in-tree source of truth, so it is written down twice — in e2e-selfhosted.yml's header and in docs/ci-hardware-testing.md — and the header tells the next person to re-date both. That instruction was a comment, and a comment does not fail: re-dating one carrier and leaving the other leaves two dates disagreeing about one reading, with nothing to say which is current. The workflow-contract check already cross-reads exactly these two files to keep the documented lane rows honest against the declared jobs, so the guard goes there: both must carry exactly one `confirmed `, and the two must be equal. It deliberately does not check the claim, which nothing in a checkout can. The extractor flattens line structure first. Both carriers wrap the sentence, so the word and its date routinely land on different lines — and in the YAML each of those lines starts with a `#`. A per-line match would find nothing on the docs side today and the guard would have passed by never looking. Mutation-verified three ways: setting the workflow's date back to 2026-09-11 fails on the mismatch, rewording the docs sentence so it no longer says "confirmed" fails on the count, and the unmutated tree passes. Signed-off-by: Frederic Espinosa --- xtask/src/workflow_contract.rs | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 54cb6cec8c..2844be3d8c 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -415,6 +415,75 @@ mod tests { .collect() } + /// Every `confirmed ` date in `text`, in order. + /// + /// Line structure is thrown away first: both carriers wrap the sentence, so + /// the word and its date routinely land on different lines (and in the YAML + /// each of those lines starts with a `#`). Matching per line would find + /// nothing and the guard would pass by never looking. `re-confirmed` is + /// matched too — it ends with the same word. + fn confirmed_dates(text: &str) -> Vec { + let flat = text + .lines() + .map(|line| line.trim().trim_start_matches('#').trim()) + .collect::>() + .join(" "); + flat.match_indices("confirmed ") + .filter_map(|(at, word)| { + let date: String = flat[at + word.len()..].chars().take(10).collect(); + let shaped = date.len() == 10 + && date.chars().enumerate().all(|(i, c)| { + if i == 4 || i == 7 { + c == '-' + } else { + c.is_ascii_digit() + } + }); + shaped.then_some(date) + }) + .collect() + } + + /// The two carriers of the required-check observation must name the same day. + /// + /// That observation is of a GitHub repository setting with no in-tree source + /// of truth, so it is a dated reading rather than something a checkout can + /// verify — which is exactly why it is written down twice, in the workflow + /// header and in the docs, and why the header tells the next person to + /// re-date BOTH when they re-read the setting. That instruction was a + /// comment, and a comment does not fail: re-dating one carrier and leaving + /// the other would have left two dates disagreeing about one reading, with + /// nothing to say which was current. + /// + /// This does not check the claim itself, which nothing in a checkout can. + #[test] + fn the_required_check_observation_is_dated_the_same_in_both_places() { + let workflow = read_workflow("e2e-selfhosted.yml"); + let docs = std::fs::read_to_string(repo_root().join("docs/ci-hardware-testing.md")) + .expect("read hardware testing docs"); + + let in_workflow = confirmed_dates(&workflow); + let in_docs = confirmed_dates(&docs); + assert_eq!( + in_workflow.len(), + 1, + "e2e-selfhosted.yml's header must carry exactly one `confirmed ` for the \ + required-check observation, so there is one thing to re-date: {in_workflow:?}" + ); + assert_eq!( + in_docs.len(), + 1, + "docs/ci-hardware-testing.md must carry exactly one `confirmed ` for the \ + same observation: {in_docs:?}" + ); + assert_eq!( + in_workflow[0], in_docs[0], + "the required-check observation is one reading of one setting, so its date must \ + match in e2e-selfhosted.yml's header and in docs/ci-hardware-testing.md — \ + re-date both, or neither is trustworthy" + ); + } + /// Every backtick-delimited span in `text`, in order. fn backticked_items(text: &str) -> Vec { text.split('`') From 2b84f9c17948dcc2c24a210d83adcadaf2a10e6f Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 16 Sep 2026 00:21:15 +0200 Subject: [PATCH 39/62] test(xtask): correct the reason given for flattening the date extractor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doc comment added with the guard claimed both carriers wrap the sentence, and that a per-line rule would leave the guard passing by never looking. Neither holds. The workflow keeps `(confirmed )` on one line; only the docs sentence wraps, between `re-confirmed` and its date. And a per-line rule makes the docs side yield zero dates, which the caller's count assertion catches immediately — reproduced by joining on newlines instead of spaces: the test fails with 0 against 1. The true reason is the better argument anyway: the carriers disagree about line structure, so a per-line rule reads one and stops seeing the other, and flattening is what keeps them comparable. The comment now says that, and records that the count assertions are what turn a blind extractor into a failure rather than a pass — the opposite of what it claimed before, and wrong in the direction that made the guard look weaker than it is. Signed-off-by: Frederic Espinosa --- xtask/src/workflow_contract.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 2844be3d8c..c1e56893bd 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -417,11 +417,17 @@ mod tests { /// Every `confirmed ` date in `text`, in order. /// - /// Line structure is thrown away first: both carriers wrap the sentence, so - /// the word and its date routinely land on different lines (and in the YAML - /// each of those lines starts with a `#`). Matching per line would find - /// nothing and the guard would pass by never looking. `re-confirmed` is - /// matched too — it ends with the same word. + /// Line structure is thrown away first, because the two carriers do not + /// agree on it: the workflow keeps the whole parenthetical on one line, + /// while the docs sentence wraps between `re-confirmed` and its date. A + /// per-line rule would therefore read the workflow and stop seeing the + /// docs — flattening is what keeps the two comparable. (The comment markers + /// go with it, since the YAML carrier is a comment block.) + /// + /// That mistake would not go quiet: the caller asserts each side holds + /// exactly one date, so a rule that stops seeing one of them fails on the + /// count rather than passing. `re-confirmed` is matched too — it ends with + /// the same word. fn confirmed_dates(text: &str) -> Vec { let flat = text .lines() From ad44496797e5ac20f85ae59f1a8cec3276d86e30 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 16 Sep 2026 13:51:56 +0200 Subject: [PATCH 40/62] style(e2e): apply rustfmt to the merged drain helper The `None` arm of `drain_final_frame` exceeded the line width once it became a delegating call, so rustfmt wants it braced across three lines. The required `prek (lint / hygiene)` check caught it; my local run had reported it too and I misread the output. Signed-off-by: Frederic Espinosa --- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 094237dc6c..7a1601fba4 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -783,7 +783,10 @@ impl TuiSession { // No marker: drain for the window and report nothing found, which is // what the exit path wants. `|_| false` never short-circuits, so the // loop runs to its deadline exactly as before. - None => self.drain_final_frame_where("the final frame", &|_: &str| false).await, + None => { + self.drain_final_frame_where("the final frame", &|_: &str| false) + .await + } } } From 13d7a189adcf63a36ec4a5af277009de09e4b42c Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 16 Sep 2026 15:52:27 +0200 Subject: [PATCH 41/62] test(e2e): retire the dead drain arm the merge left, and tighten four claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last merge resolution left `drain_final_frame` as a two-armed wrapper whose `Some(marker)` arm nothing calls: `wait_for_screen_where` reaches `drain_final_frame_where` directly, and the only remaining caller passes `None`. So `stop_on` was always `None`, the `Some` arm was dead, and the returned bool could only be `false` — while the doc comment described a caller that does not exist. That comment is the claim my own merge message repeated ("remains the substring-shaped wrapper main's second caller uses"), and it is what let the dead arm survive the resolution. Collapsed to a no-argument wrapper returning `()`, with the doc describing only the exit-drain path it actually serves. The same resolution had also degraded a diagnostic: passing a literal label into a format string that already says "draining the final frame for {wanted}" produced "...for the final frame". `drain_panic_message` takes the empty label as "no context", restoring the wording the base tip produced for this path. Four claims elsewhere that were wider than what the code enforced: - The busy-address scenario promised the CLI would "say the usual address is busy", but the no-plan arm only required a nonzero exit — any unrelated failure would have been accepted as an honest refusal. It now has to mention the address. - The diagnose/fix comparison shimmed PATH for `diagnose` only. Harmless while the fix recipe never reads the device, but the natural fix for the divergence this pins is to make `fix` read what `diagnose` reads — at which point a one-sided shim silently becomes a shimmed-against-real comparison. - The workflow header told the next person to re-date the docs' "Blocking vs. non-blocking" section, which carries no date and defers to the paragraph that does. Following it literally adds a second date and fails the contract test added earlier in this branch (verified: the test reports two dates against one). It now names only the paragraph that carries it. - "which lists sixteen names" is a tally of an out-of-tree setting that nothing can re-check, in a branch that already has a commit retiring counts that rot. The field name alone lets a reader repeat the reading. And `every_expectation_row_names_a_scenario_that_exists` guarded non-vacuity on the feature-file side only: an `expectations.toml` that declared nothing would leave `orphans` empty and pass, reporting "no stale rows" about a file it never read. Both sides are asserted now; mutation-verified by forcing `declared_ids()` to yield nothing, which fails exactly that test. Signed-off-by: Frederic Espinosa --- .github/workflows/e2e-selfhosted.yml | 18 +++--- docs/ci-hardware-testing.md | 14 ++--- tests/e2e-cucumber/src/expectation.rs | 15 ++++- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 15 ++++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 24 ++++++-- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 55 ++++++++++--------- 6 files changed, 91 insertions(+), 50 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index e0dd12ef67..08772eb03f 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -24,14 +24,16 @@ name: E2E self-hosted # (`gh api repos/:owner/:repo/branches/main/protection`) or watching what a red # lane actually blocks. # -# Re-date it here AND in docs/ci-hardware-testing.md — its "Blocking vs. -# non-blocking" section and its "Required-check history" paragraph, which -# carries the same date and the fuller story of why these names used to be -# required. Those are the places known to carry it; ci.yml, -# tests/e2e-cucumber/README.md and crates/e2e-report (which prints the reason -# into the consolidated report) point at them rather than asserting it -# themselves. Deliberately no count of how many places there are: naming them is -# what helps, and a tally is what goes stale the moment someone adds another. +# Re-date it here AND in docs/ci-hardware-testing.md's "Required-check history" +# paragraph, which carries the same date and the fuller story of why these names +# used to be required. Those two are the only places that carry the date, and a +# contract test asserts they agree — that doc's "Blocking vs. non-blocking" +# section states the fact but defers the date to that paragraph, so do not add +# one there. ci.yml, tests/e2e-cucumber/README.md and crates/e2e-report (which +# prints the reason into the consolidated report) point at these rather than +# asserting it themselves. Deliberately no count of how many places there are: +# naming them is what helps, and a tally is what goes stale the moment someone +# adds another. on: push: diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 70c90f0989..df6edfd133 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -257,13 +257,13 @@ fails, just later. required check that *never reports* (because its self-hosted runner is offline) is treated as missing and still blocks the merge, `continue-on-error` notwithstanding. That branch-protection change has since landed (re-confirmed -2026-09-15 by reading `required_status_checks.contexts`, which lists sixteen -names and none of these): none of the self-hosted lane names, nor the -self-hosted consolidated report, are in the required list anymore (`ci.yml`'s -own mock `E2E tests` / `E2E consolidated report` are the required checks — -similarly prefixed but distinct from the self-hosted names above, so no name -string actually collides). An offline or unclaimed self-hosted runner can no -longer block a merge. +2026-09-15 by reading `required_status_checks.contexts`, which names none of +these): none of the self-hosted lane names, nor the self-hosted consolidated +report, are in the required list anymore (`ci.yml`'s own mock `E2E tests` / +`E2E consolidated report` are the required checks — similarly prefixed but +distinct from the self-hosted names above, so no name string actually +collides). An offline or unclaimed self-hosted runner can no longer block a +merge. ## Fork safety diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 3a3546e487..ece1fd429d 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1373,8 +1373,19 @@ flaky = true ); let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); - let orphans: Vec<&str> = m - .declared_ids() + // Both sides, not just the scenarios. An `expectations.toml` that + // declared nothing would make `orphans` empty and this check green — + // it would be reporting "no stale rows" about a file it never read. + // A sibling test would catch the empty parse, but this one should not + // depend on that to mean what it says. + let declared: Vec<&str> = m.declared_ids().collect(); + assert!( + !declared.is_empty(), + "expectations.toml declared no rows at all, so this check would pass vacuously" + ); + let orphans: Vec<&str> = declared + .iter() + .copied() .filter(|id| !scenario_ids.contains(*id)) .collect(); assert!( diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 4fefd35b22..2392f7d753 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -1145,8 +1145,19 @@ fn compare_diagnosis_with_fix(world: &mut E2eWorld, path_override: Option<&str>) }; // No `--yes`, so nothing is applied; the preview is the user-facing account // of what the fix would do. - let (preview, preview_err, _) = - crate::run_rocm(world, &["fix", DEVICE_PERMISSION_FIX_ID, "--dry-run"]); + // + // The shim goes to BOTH sides. Today the fix recipe never consults the + // device, so passing it only to `diagnose` would happen to work — but the + // whole point of this pair is that the two disagree, and the natural fix is + // to make `fix` read what `diagnose` read. On the day that lands, a shim on + // one side only would silently turn this into a shimmed-against-real + // comparison and the scenario would report a divergence that is the + // fixture's, not the product's. + let fix_args = ["fix", DEVICE_PERMISSION_FIX_ID, "--dry-run"]; + let (preview, preview_err, _) = match path_override { + Some(path) => crate::run_rocm_with_env(world, &fix_args, &[("PATH", path)]), + None => crate::run_rocm(world, &fix_args), + }; world.cli_output = Some(diagnosis); world.cli_other_output = Some(format!("{preview}{preview_err}")); } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index adbd9ec5d8..ba85ed561a 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1287,10 +1287,26 @@ async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { // busy. Planning to use it anyway is the one that collides with the server // already there. match planned { - None => assert!( - world.cli_rc != Some(0), - "the CLI neither planned an address nor refused:\n{output}" - ), + None => { + assert!( + world.cli_rc != Some(0), + "the CLI neither planned an address nor refused:\n{output}" + ); + // A nonzero exit alone is not the second outcome. Any unrelated + // failure — a missing model, a broken runtime — would also plan no + // address and exit nonzero, and would be accepted here as though + // the CLI had honestly reported the conflict. Require it to say so. + let said = output.to_lowercase(); + let port = SERVE_PORT.to_string(); + assert!( + said.contains(&port) + || said.contains("in use") + || said.contains("already") + || said.contains("address"), + "the CLI refused, but not about the address that is taken — so this \ + says nothing about whether it noticed the conflict:\n{output}" + ); + } Some(port) => assert_ne!( port, SERVE_PORT.to_string(), diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 7a1601fba4..a2151a975c 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -738,7 +738,7 @@ impl TuiSession { self.finished = true; let code = i32::try_from(status.exit_code()).unwrap_or(-1); self.record_once(code); - self.drain_final_frame(None).await?; + self.drain_final_frame().await?; return Ok(code); } Ok(None) => {} @@ -766,33 +766,26 @@ impl TuiSession { /// single poll is not enough when a large frame is still buffered behind the /// process exit notification. /// - /// The one drain loop for both exit paths, so they cannot drift: pass - /// `stop_on: Some(marker)` to also return as soon as `marker` appears (that - /// caller is racing the drain against a screen assertion), or `None` to just - /// wait out the window. Returns whether `stop_on` was found; `Err` if the - /// reader thread panicked, which must win over the caller's generic timeout - /// or "process exited" message (and would otherwise be swallowed entirely - /// when `Drop` runs during another unwind). - async fn drain_final_frame(&mut self, stop_on: Option<&str>) -> Result { - match stop_on { - Some(marker) => { - let wanted = format!("{marker:?}"); - self.drain_final_frame_where(&wanted, &|screen: &str| screen.contains(marker)) - .await - } - // No marker: drain for the window and report nothing found, which is - // what the exit path wants. `|_| false` never short-circuits, so the - // loop runs to its deadline exactly as before. - None => { - self.drain_final_frame_where("the final frame", &|_: &str| false) - .await - } - } + /// This path has nothing to watch for: it waits the window out and returns. + /// `wait_for_screen_where` races its own drain against a screen predicate + /// and so calls [`Self::drain_final_frame_where`] directly; the two share + /// that one loop rather than keeping a copy each. + /// + /// `Err` if the reader thread panicked, which must win over the caller's + /// generic timeout or "process exited" message (and would otherwise be + /// swallowed entirely when `Drop` runs during another unwind). + async fn drain_final_frame(&mut self) -> Result<(), String> { + // `|_| false` never short-circuits, so the loop runs to its deadline. + // Nothing is being looked for, so the `bool` is uninformative here and + // the empty `wanted` keeps it out of the reader-panic message. + self.drain_final_frame_where("", &|_: &str| false) + .await + .map(|_| ()) } - /// [`drain_final_frame`] against an arbitrary predicate rather than a - /// substring, for callers that match a screen some other way. `wanted` - /// names what was being waited for in the diagnostics. + /// [`Self::drain_final_frame`] against a predicate, for the caller that is + /// racing the drain against a screen assertion. `wanted` names what is being + /// waited for, and is read back in the diagnostics. async fn drain_final_frame_where( &mut self, wanted: &str, @@ -832,8 +825,16 @@ impl TuiSession { /// Reader-panic diagnostic for [`drain_final_frame_where`], naming what the /// drain was racing. fn drain_panic_message(&self, wanted: &str, panic_message: &str) -> String { + // The exit drain waits for nothing and passes "", so it reads as it did + // before this loop was shared: naming a thing it is not waiting for + // would be worse than naming nothing. + let context = if wanted.is_empty() { + String::new() + } else { + format!(" for {wanted}") + }; format!( - "pty reader thread panicked while draining the final frame for {wanted}: {panic_message}\n{}", + "pty reader thread panicked while draining the final frame{context}: {panic_message}\n{}", self.framed_screen() ) } From 8ad6f76d6a4fb25439307ee68c7436b361dd8f02 Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 16 Sep 2026 16:24:49 +0200 Subject: [PATCH 42/62] test(e2e): narrow the busy-address match, and keep the drain sentinel private MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three claims still wider than what the code enforced. The busy-address arm accepted a bare `already` or `address` anywhere in the output, while the commit message said it "has to mention the address". `already` is a word the serve path uses for unrelated conditions — an equivalent managed service already running, a model already being served — so the assertion admitted refusals that say nothing about this conflict. Narrowed to the port number or "in use", which is what naming THIS conflict looks like, and what the message claimed all along. The empty-label sentinel that keeps the exit drain's panic message reading as it did was private by intent only: `wait_for_screen_where` is `pub` and passes `wanted` straight through, so a future caller passing "" would silently lose the context rather than fail. It is now a documented precondition on that parameter with a `debug_assert!` behind it. And `drain_final_frame_where`'s doc named one caller when it has two — the screen-assertion racer and the exit-drain wrapper directly above it. This PR has already produced two findings where a comment contradicted the code it sat on; naming both is worth the line. Signed-off-by: Frederic Espinosa --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 14 ++++++++----- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 21 +++++++++++++++---- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index ba85ed561a..997a0d03c7 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1296,14 +1296,18 @@ async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { // failure — a missing model, a broken runtime — would also plan no // address and exit nonzero, and would be accepted here as though // the CLI had honestly reported the conflict. Require it to say so. + // + // Only these two. A bare "already" would admit the serve path's own + // unrelated uses of the word ("already running" for an equivalent + // managed service, "already being served" for a model), and a bare + // "address" would admit any sentence that happens to mention one. + // The port number or "in use" is what naming THIS conflict looks + // like. let said = output.to_lowercase(); let port = SERVE_PORT.to_string(); assert!( - said.contains(&port) - || said.contains("in use") - || said.contains("already") - || said.contains("address"), - "the CLI refused, but not about the address that is taken — so this \ + said.contains(&port) || said.contains("in use"), + "the CLI refused, but without naming the address that is taken — so this \ says nothing about whether it noticed the conflict:\n{output}" ); } diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index a2151a975c..89617f012c 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -451,13 +451,21 @@ impl TuiSession { /// shell that dies, or a pty reader that panics, is reported as the real /// cause here instead of running out the full timeout and blaming whatever /// the caller happened to be looking for. `wanted` is quoted into those - /// messages, so it should read as the thing being waited for. + /// messages, so it should read as the thing being waited for — and must be + /// non-empty, because the empty string is the exit drain's private "nothing + /// in particular" sentinel and would silently drop the context from a + /// reader-panic message rather than fail. pub async fn wait_for_screen_where( &mut self, wanted: &str, timeout: Duration, matches: impl Fn(&str) -> bool + Sync, ) -> Result<(), String> { + debug_assert!( + !wanted.is_empty(), + "wait_for_screen_where needs a label naming what is being waited for; \ + the empty string is reserved for the exit drain" + ); let deadline = Instant::now() + timeout; loop { if matches(&self.screen_text()) { @@ -783,9 +791,14 @@ impl TuiSession { .map(|_| ()) } - /// [`Self::drain_final_frame`] against a predicate, for the caller that is - /// racing the drain against a screen assertion. `wanted` names what is being - /// waited for, and is read back in the diagnostics. + /// The drain loop itself, against a predicate. Two callers: + /// `wait_for_screen_where`, which races the drain against a screen + /// assertion, and [`Self::drain_final_frame`] directly above, which passes + /// a predicate that never matches and waits the window out. + /// + /// `wanted` names what is being waited for and is read back in the + /// diagnostics; the empty string means "nothing in particular", which is + /// what the exit drain passes. async fn drain_final_frame_where( &mut self, wanted: &str, From c2e929ee8ac1e38a224c8a6a6fa26459f00bbc11 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 14:10:36 +0200 Subject: [PATCH 43/62] test(e2e): say which guards guard a fix, and make the absent label unrepresentable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three scenarios ship as guards because the defects they were written for landed on main while this branch was open, but only `diagnose-21` said so. A reader meeting `examine-16`, `examine-17` or `serve-22` without that line has no way to tell a guard from a test certifying a defect green. Each now names the fix it guards — EAI-8011 for the two help contracts, EAI-8019 for the device-policy list — in the same shape `diagnose-21` already used. The exit drain's "no label" case was an empty string: a sentinel private by intent, on a parameter reachable through a `pub` function, held only by a `debug_assert!` that compiles out under release. Nothing builds this harness in release today, so it was latent rather than live — but documented-and-asserted is the weaker form of a state that can simply be made unrepresentable. The drain loop and its diagnostic take `Option<&str>` now; the racing caller passes `Some(wanted)`, the exit drain passes `None`, and the assertion is gone because there is nothing left to assert. Also corrects one assertion message that still explained a lane's non-blocking status by `continue-on-error` — the framing this branch replaced everywhere it appears in prose. It now gives the structural reason and points at the workflow header that carries the dated observation. Signed-off-by: fredespi --- tests/e2e-cucumber/features/examine.feature | 7 ++++ .../features/model_serving.feature | 4 ++ tests/e2e-cucumber/tests/e2e/tui_driver.rs | 37 +++++++------------ xtask/src/workflow_contract.rs | 5 ++- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 4f00f78d13..74bbbdd298 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -180,6 +180,10 @@ Feature: GPU detection and system inspection # accepts either form the README documents — a name the model listing knows, # or an explicit `owner/repo` reference — and so does not prescribe which # model the examples should use. + # + # Since fixed on main (EAI-8011 swapped the unresolvable example name for a + # real catalog alias), so this ships as a guard rather than an expected + # failure. @id:examine-help-serve-example-names-a-resolvable-model Scenario: examine-16 - Every model the help offers as an example is one the CLI can resolve When the user reads the serve examples the help offers @@ -189,6 +193,9 @@ Feature: GPU detection and system inspection # different things. When the help describes them as the same one, a reader has # no way to learn what the plain command does or that there is anything else # to reach. + # + # Since fixed on main (the same EAI-8011 change now describes bare `rocm` as + # the launcher), so this ships as a guard rather than an expected failure. @id:examine-help-describes-the-default-command Scenario: examine-17 - The help tells the two ways of opening a screen apart When the user asks for help diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 37ed982b04..369589bd1f 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -260,6 +260,10 @@ Feature: Model serving # has to be the list the command accepts. Runs on the no-GPU lane, where a # refusal that names the policy itself is cleanly distinguishable from the # ordinary "this machine has no GPU" refusal every policy gets there. + # + # Since fixed on main (EAI-8019 made `--device` a clap `ValueEnum`, so the + # advertised list and the accepted list are one list), so this ships as a + # guard rather than an expected failure. @id:serve-rejects-no-advertised-device-policy @requires-no-gpu Scenario: serve-22 - Every device policy the serve command offers is one it accepts When the user serves a model under each device policy the command offers diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 89617f012c..7dc4bfe7ed 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -451,21 +451,13 @@ impl TuiSession { /// shell that dies, or a pty reader that panics, is reported as the real /// cause here instead of running out the full timeout and blaming whatever /// the caller happened to be looking for. `wanted` is quoted into those - /// messages, so it should read as the thing being waited for — and must be - /// non-empty, because the empty string is the exit drain's private "nothing - /// in particular" sentinel and would silently drop the context from a - /// reader-panic message rather than fail. + /// messages, so it should read as the thing being waited for. pub async fn wait_for_screen_where( &mut self, wanted: &str, timeout: Duration, matches: impl Fn(&str) -> bool + Sync, ) -> Result<(), String> { - debug_assert!( - !wanted.is_empty(), - "wait_for_screen_where needs a label naming what is being waited for; \ - the empty string is reserved for the exit drain" - ); let deadline = Instant::now() + timeout; loop { if matches(&self.screen_text()) { @@ -483,7 +475,7 @@ impl TuiSession { if let Ok(Some(status)) = self.child.try_wait() { self.finished = true; self.record_once(i32::try_from(status.exit_code()).unwrap_or(-1)); - if self.drain_final_frame_where(wanted, &matches).await? { + if self.drain_final_frame_where(Some(wanted), &matches).await? { return Ok(()); } return Err(format!( @@ -785,8 +777,8 @@ impl TuiSession { async fn drain_final_frame(&mut self) -> Result<(), String> { // `|_| false` never short-circuits, so the loop runs to its deadline. // Nothing is being looked for, so the `bool` is uninformative here and - // the empty `wanted` keeps it out of the reader-panic message. - self.drain_final_frame_where("", &|_: &str| false) + // `None` keeps the absent label out of the reader-panic message. + self.drain_final_frame_where(None, &|_: &str| false) .await .map(|_| ()) } @@ -797,11 +789,12 @@ impl TuiSession { /// a predicate that never matches and waits the window out. /// /// `wanted` names what is being waited for and is read back in the - /// diagnostics; the empty string means "nothing in particular", which is - /// what the exit drain passes. + /// diagnostics; `None` means "nothing in particular", which is what the + /// exit drain passes — unrepresentable as a label rather than spelled with + /// one, so no caller can supply it by accident. async fn drain_final_frame_where( &mut self, - wanted: &str, + wanted: Option<&str>, matches: &(impl Fn(&str) -> bool + Sync + ?Sized), ) -> Result { let found = |session: &Self| matches(&session.screen_text()); @@ -837,15 +830,11 @@ impl TuiSession { /// Reader-panic diagnostic for [`drain_final_frame_where`], naming what the /// drain was racing. - fn drain_panic_message(&self, wanted: &str, panic_message: &str) -> String { - // The exit drain waits for nothing and passes "", so it reads as it did - // before this loop was shared: naming a thing it is not waiting for - // would be worse than naming nothing. - let context = if wanted.is_empty() { - String::new() - } else { - format!(" for {wanted}") - }; + fn drain_panic_message(&self, wanted: Option<&str>, panic_message: &str) -> String { + // The exit drain waits for nothing, so it reads as it did before this + // loop was shared: naming a thing it is not waiting for would be worse + // than naming nothing. + let context = wanted.map_or_else(String::new, |wanted| format!(" for {wanted}")); format!( "pty reader thread panicked while draining the final frame{context}: {panic_message}\n{}", self.framed_screen() diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 3b89036506..32fad5f1ab 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1138,8 +1138,9 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" for row in &self_hosted_rows { assert_eq!( row[3], "no", - "self-hosted lane `{}` is continue-on-error, so the README must not \ - document it as blocking", + "self-hosted lane `{}` is non-blocking — its check name is not in branch \ + protection's required list (see e2e-selfhosted.yml's header) — so the \ + README must not document it as blocking", row[0] ); } From 05717c12ccee4d593078e5a8246dca05e7bc1057 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 15:27:35 +0200 Subject: [PATCH 44/62] test(e2e): drop an invented bound, and fix two wait diagnostics that misread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things in the merged wait helper, none of which the compiler objects to. `+ Send` on the predicate is mine and appears in neither parent: main's takes a bare `FnMut`, this branch's took `Fn + Sync`. I added it reflexively after an earlier `future_not_send` error on a DIFFERENT signature — one that held an `&impl Fn` across an await in a Send context. This one does not need it: the step futures are `LocalBoxFuture`, nothing spawns them, and nothing moves the closure across a thread. Dropped, and clippy's `future_not_send` is clean without it in the Linux container, which is the lint that would object. The doc said `describe` "is quoted in every diagnostic". All three templates interpolate it with Display, never Debug — nothing is quoted. It only looked that way because `wait_for_screen` pre-quotes its own marker, so the one caller that appeared to prove the claim was the one supplying the quotes itself. A caller trusting that sentence would omit quotes the helper never adds. And routing `wait_for_screen` through the general helper left its messages reading `waiting until "Ready"` and `before "Ready".`, because the label is now read as a clause rather than a noun. It passes a phrase, so both caller shapes produce sentences again: `waiting until the screen shows "Ready"`. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index f4fb177890..8dd6d39825 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -439,7 +439,10 @@ impl TuiSession { /// deadline that includes the last screen for diagnosis. Also fails fast if /// the child exits before the marker appears. pub async fn wait_for_screen(&mut self, marker: &str, timeout: Duration) -> Result<(), String> { - let wanted = format!("{marker:?}"); + // A phrase, not the bare marker: the general helper reads its label as + // "waiting until {describe}", so a lone `"Ready"` would land as + // `waiting until "Ready"`. + let wanted = format!("the screen shows {marker:?}"); self.wait_for_screen_where(&wanted, |screen| screen.contains(marker), timeout) .await } @@ -452,7 +455,9 @@ impl TuiSession { /// The general form of `wait_for_screen`, for evidence a frame is current /// that is not "it contains this string" — a cleared table cell, or a /// marker the frame stopped showing. `describe` names the condition being - /// waited on and is quoted in every diagnostic. + /// waited on and is interpolated into every diagnostic as-is, so a caller + /// wanting quotes around it quotes its own text — `wait_for_screen` does, + /// which is why its messages read as quoted and these do not. /// /// A child that has exited does not end the wait on its own: the reader is /// given a bounded window to commit whatever was still buffered behind the @@ -461,7 +466,7 @@ impl TuiSession { pub async fn wait_for_screen_where( &mut self, describe: &str, - mut is_ready: impl FnMut(&str) -> bool + Send, + mut is_ready: impl FnMut(&str) -> bool, timeout: Duration, ) -> Result<(), String> { let deadline = Instant::now() + timeout; @@ -801,7 +806,7 @@ impl TuiSession { async fn drain_final_frame_where( &mut self, wanted: Option<&str>, - is_ready: &mut (impl FnMut(&str) -> bool + Send + ?Sized), + is_ready: &mut (impl FnMut(&str) -> bool + ?Sized), ) -> Result { let drain_deadline = Instant::now() + DRAIN_TIMEOUT; loop { From 40674206844247e35975a963c0b2636dd2fd0180 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 17:20:00 +0200 Subject: [PATCH 45/62] test(e2e): make the fourth wait diagnostic fit the clause, and state the rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last commit moved `wait_for_screen`'s label from a bare marker to a clause so three diagnostics would read as sentences, and missed the fourth: the drain panic message interpolates the same string into `for {wanted}`, a slot that wants a noun. So the one path that commit did not check is the one it made worse — `…draining the final frame for the screen shows "Ready"`. That template is now clause-shaped like the other three, which fixes it for every caller rather than for the one that prompted it. The doc comment that was supposed to prevent exactly this said `describe` is "interpolated into every diagnostic" without saying the templates have different grammatical frames — so it described the substitution and not the constraint. It also explained the quoting convention by contrasting callers, and the contrast was false: `dash_steps.rs` builds its label with `{…:?}`, so one of the three callers it grouped under "these do not" does carry quotes. Both replaced by the rule itself: `describe` is interpolated verbatim into four named templates, must read as a clause after each, and carries whatever quoting the caller puts in it. Stating the contract cannot go stale against a caller the way a survey of callers can. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 8dd6d39825..87563f33fd 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -454,10 +454,15 @@ impl TuiSession { /// /// The general form of `wait_for_screen`, for evidence a frame is current /// that is not "it contains this string" — a cleared table cell, or a - /// marker the frame stopped showing. `describe` names the condition being - /// waited on and is interpolated into every diagnostic as-is, so a caller - /// wanting quotes around it quotes its own text — `wait_for_screen` does, - /// which is why its messages read as quoted and these do not. + /// marker the frame stopped showing. + /// + /// `describe` is interpolated verbatim — no quoting, no rewording — into + /// four diagnostics, which read `waiting until {describe}`, `timed out … + /// waiting until {describe}`, `before {describe}`, and `… draining the + /// final frame, waiting until {describe}`. So it must be a clause that + /// fits all four ("the screen shows X", "the table clears X"), not a bare + /// noun; and it carries whatever quoting the caller puts in it, since none + /// is added here. /// /// A child that has exited does not end the wait on its own: the reader is /// given a bounded window to commit whatever was still buffered behind the @@ -841,10 +846,11 @@ impl TuiSession { /// Reader-panic diagnostic for [`drain_final_frame_where`], naming what the /// drain was racing. fn drain_panic_message(&self, wanted: Option<&str>, panic_message: &str) -> String { - // The exit drain waits for nothing, so it reads as it did before this - // loop was shared: naming a thing it is not waiting for would be worse - // than naming nothing. - let context = wanted.map_or_else(String::new, |wanted| format!(" for {wanted}")); + // Clause-shaped, like the three templates in `wait_for_screen_where`: + // `describe` is a clause, and " for " does not parse. The exit + // drain waits for nothing and adds no context at all, so it reads as it + // did before this loop was shared. + let context = wanted.map_or_else(String::new, |wanted| format!(", waiting until {wanted}")); format!( "pty reader thread panicked while draining the final frame{context}: {panic_message}\n{}", self.framed_screen() From 3d09ade0c0482b8f73b81b862f6c8a04c164c3cd Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 17:39:42 +0200 Subject: [PATCH 46/62] docs(e2e): mark the neighbouring wait convention deliberate, and cite a real caller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tui_driver.rs` carries two diagnostic conventions for the same wait, and `send_until` uses both: through `wait_for_screen` it emits `waiting until the screen shows "Ready"`, while `terminal_state_after_wait` emits `waiting for "Ready"` for that same marker. Each is coherent — the second takes a bare marker and quotes it itself, because its one caller holds the marker rather than a description of a condition — but only the first says so, so the second reads as drift from the contract stated next door. It now documents its own shape and that the difference is intended. The contract's second example, "the table clears X", matched no caller. The cleared-cell caller it was meant to evoke reads "the scripted instance's TTFT cell leaves \"50ms\"", and a neighbouring one reads "generation throughput leaves the screen" — so the example is now that real one. An invented illustration is the same defect as a survey of callers that has gone stale, just earlier: it shows a shape nothing has to keep true. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 87563f33fd..6026f062ef 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -364,6 +364,12 @@ impl TuiSession { /// Retrieve a terminal failure that landed after a wait's final poll but /// before the retry loop decides whether another key is safe to send. + /// + /// Takes a bare marker and quotes it here, so its messages read + /// `waiting for "Ready"` — deliberately NOT the clause convention + /// `wait_for_screen_where` documents. Its callers hold the marker itself + /// rather than a description of a condition, and there is nothing for them + /// to phrase. fn terminal_state_after_wait(&mut self, marker: &str) -> TerminalState { let reader_finished = self .reader @@ -460,8 +466,8 @@ impl TuiSession { /// four diagnostics, which read `waiting until {describe}`, `timed out … /// waiting until {describe}`, `before {describe}`, and `… draining the /// final frame, waiting until {describe}`. So it must be a clause that - /// fits all four ("the screen shows X", "the table clears X"), not a bare - /// noun; and it carries whatever quoting the caller puts in it, since none + /// fits all four ("the screen shows X", "generation throughput leaves the + /// screen"), not a bare noun; and it carries whatever quoting the caller puts in it, since none /// is added here. /// /// A child that has exited does not end the wait on its own: the reader is From 3cf2d939b5acda19588e51ff926c8c04d4f32440 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 18:20:48 +0200 Subject: [PATCH 47/62] test(e2e): pin the wait diagnostics their doc comments describe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review rounds running found this file's doc comments describing templates they no longer matched — each caught only because a person read both. Nothing in the suite connected the two, so every fix restored the description without making the next drift fail. Three tests now read the driver source and assert the templates the docs name are the templates that exist: the four clause-shaped ones `wait_for_screen_where` enumerates, the two bare-marker ones `terminal_state_after_wait` documents as deliberately different, and the absence of a `{describe:?}` anywhere, which would double-quote a label the contract promises is interpolated verbatim. A reworded message now fails with the name of the doc comment it invalidated. Source text rather than rendered output on purpose: rendering one needs a live pty, a child process and a reader thread, which is what the scenarios are for. What rots here is the wording, and the wording is in the file. Mutation-verified, one at a time: rewording a clause template, reverting the noun template to the clause form (the regression two rounds ago), and Debug-formatting the clause label each fail exactly the test that names them. Also closes the two doc items from this round. `terminal_state_after_wait` said "its messages read `waiting for \"Ready\"`" while having two marker-bearing templates, not one — and `"Ready"` was invented, where its sole caller passes `"● Observe"`. Both templates are now enumerated with the real marker, the way the sibling block enumerates its four. The 100-character line left by the previous edit is reflowed to the block's width. Signed-off-by: fredespi --- tests/e2e-cucumber/src/send_until.rs | 77 ++++++++++++++++++++++ tests/e2e-cucumber/tests/e2e/tui_driver.rs | 21 +++--- 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/tests/e2e-cucumber/src/send_until.rs b/tests/e2e-cucumber/src/send_until.rs index 767cb5eee4..78049885ad 100644 --- a/tests/e2e-cucumber/src/send_until.rs +++ b/tests/e2e-cucumber/src/send_until.rs @@ -72,6 +72,83 @@ where } } +#[cfg(test)] +mod diagnostic_wording { + //! The TUI driver's wait diagnostics are described by doc comments in + //! `tests/e2e/tui_driver.rs` — which templates exist, and what shape of + //! label each expects. Nothing ran those descriptions, so three review + //! rounds in a row found them describing code they no longer matched, each + //! time only because a person read both. + //! + //! These read the driver source and assert the templates the docs name are + //! the templates that exist. Reword a message and this fails, naming the + //! doc that has gone stale; the doc is then wrong for as long as it takes + //! to run the tests, rather than until someone notices. + //! + //! Source text rather than rendered output on purpose: rendering one needs + //! a live pty, a child process and a reader thread, which is what the + //! scenarios themselves are for. What rots here is the wording, and the + //! wording is in the file. + + fn driver_source() -> String { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/e2e/tui_driver.rs"); + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) + } + + /// The four templates `wait_for_screen_where`'s doc enumerates, which its + /// contract requires a caller's `describe` clause to read correctly after. + #[test] + fn the_clause_templates_are_the_ones_the_contract_names() { + let source = driver_source(); + for template in [ + "panicked while waiting until {describe}", + "timed out after {timeout:?} waiting until {describe}", + "before {describe}.", + "draining the final frame{context}", + ", waiting until {wanted}", + ] { + assert!( + source.contains(template), + "tui_driver.rs no longer contains {template:?}, so the clause contract \ + documented on `wait_for_screen_where` describes a template that is gone \ + — update that doc comment with this change" + ); + } + } + + /// `terminal_state_after_wait` quotes its own bare marker, which its doc + /// states is deliberate and distinct from the clause convention above. + #[test] + fn the_bare_marker_templates_still_quote_the_marker_themselves() { + let source = driver_source(); + for template in [ + "panicked while waiting for {marker:?}", + "before {marker:?} appeared.", + ] { + assert!( + source.contains(template), + "tui_driver.rs no longer contains {template:?}, so the noun convention \ + documented on `terminal_state_after_wait` describes a template that is \ + gone — update that doc comment with this change" + ); + } + } + + /// The contract says the label is interpolated verbatim. A `{describe:?}` + /// anywhere would quote it a second time on top of whatever the caller + /// already put in, which is what the doc promises does not happen. + #[test] + fn the_clause_label_is_never_debug_formatted() { + let source = driver_source(); + assert!( + !source.contains("{describe:?}"), + "a diagnostic Debug-formats `describe`, but `wait_for_screen_where` \ + documents it as interpolated verbatim — callers that quote their own \ + text would now be double-quoted" + ); + } +} + #[cfg(test)] mod tests { use super::{RetryTiming, TerminalState, send_until}; diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 6026f062ef..8845d12bba 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -365,11 +365,16 @@ impl TuiSession { /// Retrieve a terminal failure that landed after a wait's final poll but /// before the retry loop decides whether another key is safe to send. /// - /// Takes a bare marker and quotes it here, so its messages read - /// `waiting for "Ready"` — deliberately NOT the clause convention - /// `wait_for_screen_where` documents. Its callers hold the marker itself - /// rather than a description of a condition, and there is nothing for them - /// to phrase. + /// Takes a bare marker and quotes it here, so both of its marker-bearing + /// messages read as nouns — deliberately NOT the clause convention + /// `wait_for_screen_where` documents. For `send_until`'s marker they are + /// `pty reader thread panicked while waiting for "● Observe"` and + /// `process exited (…) before "● Observe" appeared.`; a third path + /// reports a poll failure and names no marker at all. + /// + /// Reached only through [`send_until`](Self::send_until), whose own + /// signature takes the marker rather than a description of a condition, so + /// there is nothing for a caller to phrase. fn terminal_state_after_wait(&mut self, marker: &str) -> TerminalState { let reader_finished = self .reader @@ -466,9 +471,9 @@ impl TuiSession { /// four diagnostics, which read `waiting until {describe}`, `timed out … /// waiting until {describe}`, `before {describe}`, and `… draining the /// final frame, waiting until {describe}`. So it must be a clause that - /// fits all four ("the screen shows X", "generation throughput leaves the - /// screen"), not a bare noun; and it carries whatever quoting the caller puts in it, since none - /// is added here. + /// fits all four ("the screen shows X", "generation throughput leaves + /// the screen"), not a bare noun; and it carries whatever quoting the + /// caller puts in it, since none is added here. /// /// A child that has exited does not end the wait on its own: the reader is /// given a bounded window to commit whatever was still buffered behind the From 807e35b72624d18e78b67abe51e6934daa3c6378 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 18:34:15 +0200 Subject: [PATCH 48/62] fix(e2e): build the pinned templates instead of writing them as literals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clippy's `literal_string_with_formatting_args` fires on the template strings the new wording guards compare against: a literal containing `{describe}` reads as a formatting argument nobody interpolated. It is a fair complaint — these ARE format placeholders, just quoted from a different format string in another file — and it fails the required `clippy` check, which runs `--workspace --all-targets` and so covers the lib's test target. Each placeholder is now spelled through a small `arg()` helper, so no literal in the file carries an uninterpolated brace and the strings compared are identical to before. `format!` with escaped braces was the obvious fix and is wrong for the same family of reasons: no arguments means `useless_format`. Found by CI rather than by my own gate, because I ran `clippy -p e2e-cucumber --lib` and `--test e2e` — neither of which covers the lib's test target where these tests live. The gate now runs CI's exact command. Signed-off-by: fredespi --- tests/e2e-cucumber/src/send_until.rs | 37 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/src/send_until.rs b/tests/e2e-cucumber/src/send_until.rs index 78049885ad..6a0b315d79 100644 --- a/tests/e2e-cucumber/src/send_until.rs +++ b/tests/e2e-cucumber/src/send_until.rs @@ -95,20 +95,36 @@ mod diagnostic_wording { std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) } + /// A format placeholder as it appears in source: `arg("describe")` is + /// `{describe}`. Built rather than written so no literal in this file + /// contains an uninterpolated `{…}`. + fn arg(name: &str) -> String { + format!("{{{name}}}") + } + /// The four templates `wait_for_screen_where`'s doc enumerates, which its /// contract requires a caller's `describe` clause to read correctly after. #[test] fn the_clause_templates_are_the_ones_the_contract_names() { let source = driver_source(); + // Each brace-delimited placeholder is spelled through `arg()` rather + // than written into the literal: these are fragments of OTHER format + // strings, and a literal containing `{describe}` reads to clippy as a + // formatting argument nobody interpolated + // (`literal_string_with_formatting_args`). for template in [ - "panicked while waiting until {describe}", - "timed out after {timeout:?} waiting until {describe}", - "before {describe}.", - "draining the final frame{context}", - ", waiting until {wanted}", + format!("panicked while waiting until {}", arg("describe")), + format!( + "timed out after {} waiting until {}", + arg("timeout:?"), + arg("describe") + ), + format!("before {}.", arg("describe")), + format!("draining the final frame{}", arg("context")), + format!(", waiting until {}", arg("wanted")), ] { assert!( - source.contains(template), + source.contains(&template), "tui_driver.rs no longer contains {template:?}, so the clause contract \ documented on `wait_for_screen_where` describes a template that is gone \ — update that doc comment with this change" @@ -121,12 +137,13 @@ mod diagnostic_wording { #[test] fn the_bare_marker_templates_still_quote_the_marker_themselves() { let source = driver_source(); + // Spelled through `arg()`, for the reason given in the sibling test. for template in [ - "panicked while waiting for {marker:?}", - "before {marker:?} appeared.", + format!("panicked while waiting for {}", arg("marker:?")), + format!("before {} appeared.", arg("marker:?")), ] { assert!( - source.contains(template), + source.contains(&template), "tui_driver.rs no longer contains {template:?}, so the noun convention \ documented on `terminal_state_after_wait` describes a template that is \ gone — update that doc comment with this change" @@ -141,7 +158,7 @@ mod diagnostic_wording { fn the_clause_label_is_never_debug_formatted() { let source = driver_source(); assert!( - !source.contains("{describe:?}"), + !source.contains(&arg("describe:?")), "a diagnostic Debug-formats `describe`, but `wait_for_screen_where` \ documents it as interpolated verbatim — callers that quote their own \ text would now be double-quoted" From b44bf2689e893d74b8ac2ea07917788abe7fc9f3 Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 18:49:50 +0200 Subject: [PATCH 49/62] test(e2e): check the doc side too, which the guard's own name claimed it did MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wording guard added last commit asserted one side of a two-sided relation. Its module doc said the tests "assert the templates the docs name are the templates that exist" and the test was called `the_clause_templates_are_the_ones_the_contract_names`, but nothing read a doc comment: the assertions looked for hardcoded substrings in the driver source. Reverting the whole doc change, deleting the `wait_for_screen_where` doc block outright, or adding a fifth clause diagnostic while the doc kept saying four all left every test green. Reproduced, all three. That gap points away from the failures that actually recurred. The last two rounds found docs that were wrong WHEN WRITTEN — an invented example marker, and "its messages" for a function with two templates — not docs that drifted after the code moved. A guard that only looks for templates in the code cannot see either. Both directions are checked now. Each documented diagnostic must appear as a template in the code AND as a rendering in the doc block that claims to enumerate it; the number of diagnostics carrying a caller's clause must equal the number the doc names, so a fifth cannot be added silently; and the example marker must still be the one its caller passes. Doc text is compared with `///` markers stripped and wrapping collapsed, since where a comment wraps is not part of what it says. Mutation-verified, including the three that previously passed: doc-block deletion, a fifth clause diagnostic (on its own line and sharing a line with the fourth — counting occurrences rather than lines is what catches the second), and changing the marker in `dash_steps.rs`. Each fails exactly the test that names it. Moved out of `send_until.rs`, which is a generic retry helper with no relationship to any of this, into its own `src/diagnostic_wording.rs`. Both guarded doc blocks now name that file as the place a reword has to be made too, the way `tests/feature_naming.rs` and `src/expectation.rs` already name each other. Signed-off-by: fredespi --- tests/e2e-cucumber/src/diagnostic_wording.rs | 205 +++++++++++++++++++ tests/e2e-cucumber/src/lib.rs | 2 + tests/e2e-cucumber/src/send_until.rs | 94 --------- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 9 + 4 files changed, 216 insertions(+), 94 deletions(-) create mode 100644 tests/e2e-cucumber/src/diagnostic_wording.rs diff --git a/tests/e2e-cucumber/src/diagnostic_wording.rs b/tests/e2e-cucumber/src/diagnostic_wording.rs new file mode 100644 index 0000000000..cb7c229422 --- /dev/null +++ b/tests/e2e-cucumber/src/diagnostic_wording.rs @@ -0,0 +1,205 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Keeps the TUI driver's wait diagnostics and the doc comments describing +//! them from disagreeing. +//! +//! `tests/e2e/tui_driver.rs` documents two deliberately different conventions +//! for its wait messages: `wait_for_screen_where` interpolates a caller's +//! clause verbatim into four templates, and `terminal_state_after_wait` quotes +//! its own bare marker in two. Both are prose, and prose is the part no test +//! runs — this file's own history is the argument for these checks, with three +//! consecutive review rounds finding those comments describing code they did +//! not match, each caught only because a person read both. +//! +//! So this asserts BOTH directions, because the two failures that actually +//! recurred point opposite ways: +//! +//! - a template changes and the doc still describes the old one (drift), and +//! - the doc describes something the code never had (wrong when written) — +//! which is what the last two rounds found, and what a one-sided check that +//! only looks for templates in the code would miss entirely. +//! +//! Each documented rendering must appear in its doc block AND its template +//! must appear in the code, and the number of `{describe}` diagnostics must be +//! the number the doc claims — so a fifth one cannot be added while the doc +//! goes on saying four. +//! +//! Source text rather than rendered output on purpose: rendering a message +//! needs a live pty, a child process and a reader thread, which is what the +//! scenarios themselves are for. What rots here is wording, and the wording is +//! in the file. The weakness that remains is that this reads source rather +//! than behaviour: a template could be spelled correctly and never reached. + +/// The driver whose diagnostics and doc comments this file holds together. +const DRIVER: &str = "tests/e2e/tui_driver.rs"; + +/// The step file whose marker the driver's doc quotes as its example. +const MARKER_CALLER: &str = "tests/e2e/dash_steps.rs"; + +/// The marker `send_until`'s only caller passes, quoted in the driver's doc as +/// the example rendering. Pinned here so the doc cannot go stale against it. +const EXAMPLE_MARKER: &str = "● Observe"; + +fn read(relative: &str) -> String { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(relative); + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +/// The file's doc comments as flowing text: `///` markers stripped and line +/// breaks collapsed, so a documented rendering can be searched for without +/// caring where the author happened to wrap it. +fn doc_text(source: &str) -> String { + source + .lines() + .map(str::trim_start) + .filter_map(|line| { + line.strip_prefix("///") + .or_else(|| line.strip_prefix("//!")) + }) + .map(str::trim) + .collect::>() + .join(" ") +} + +/// A format placeholder as it appears in source: `arg("describe")` is +/// `{describe}`. Built rather than written so no literal in this file carries +/// an uninterpolated `{…}`, which clippy reads as a missed interpolation. +fn arg(name: &str) -> String { + format!("{{{name}}}") +} + +/// One documented diagnostic: the `format!` template in the code, and the +/// rendering the doc comment shows for it. Both must be present. +struct Documented { + template: String, + documented_as: String, +} + +fn clause_diagnostics() -> Vec { + let describe = arg("describe"); + let timeout = arg("timeout:?"); + vec![ + Documented { + template: format!("panicked while waiting until {describe}"), + documented_as: format!("waiting until {describe}"), + }, + Documented { + template: format!("timed out after {timeout} waiting until {describe}"), + documented_as: format!("timed out … waiting until {describe}"), + }, + Documented { + template: format!("before {describe}."), + documented_as: format!("before {describe}"), + }, + Documented { + // The fourth is assembled from two fragments: the drain message + // interpolates a context that the label is built into. + template: format!("draining the final frame{}", arg("context")), + documented_as: format!("draining the final frame, waiting until {describe}"), + }, + ] +} + +fn marker_diagnostics() -> Vec { + let marker = arg("marker:?"); + vec![ + Documented { + template: format!("panicked while waiting for {marker}"), + documented_as: format!("panicked while waiting for {EXAMPLE_MARKER:?}"), + }, + Documented { + template: format!("before {marker} appeared."), + documented_as: format!("before {EXAMPLE_MARKER:?} appeared."), + }, + ] +} + +fn assert_both_sides(source: &str, documented: &[Documented], owner: &str) { + let docs = doc_text(source); + for Documented { + template, + documented_as, + } in documented + { + assert!( + source.contains(template.as_str()), + "{DRIVER} no longer contains the template {template:?}, but `{owner}`'s doc \ + comment still describes it — update the doc with this change" + ); + assert!( + docs.contains(documented_as.as_str()), + "`{owner}`'s doc comment no longer shows {documented_as:?}, but {DRIVER} still \ + emits that diagnostic — the doc must keep naming every message it claims to \ + enumerate, or this guard is checking nothing" + ); + } +} + +/// The four clause-shaped diagnostics, in both directions. +#[test] +fn every_clause_diagnostic_is_both_emitted_and_documented() { + assert_both_sides( + &read(DRIVER), + &clause_diagnostics(), + "wait_for_screen_where", + ); +} + +/// The two bare-marker diagnostics, in both directions. +#[test] +fn every_marker_diagnostic_is_both_emitted_and_documented() { + assert_both_sides( + &read(DRIVER), + &marker_diagnostics(), + "terminal_state_after_wait", + ); +} + +/// The doc says "four diagnostics". A fifth one added without touching the doc +/// would satisfy every check above — each documented message would still be +/// present — while the count silently became a lie. This is the direction a +/// list of substrings cannot see. +#[test] +fn the_documented_count_of_clause_diagnostics_is_the_real_one() { + let source = read(DRIVER); + let describe = arg("describe"); + // The drain message reaches the label through `{context}` rather than + // naming it, so it is counted by its own placeholder. Both are code lines: + // the doc's own occurrences sit behind `///` and are filtered out. + let context = arg("context"); + // Occurrences, not lines: two placeholders on one line are two diagnostics + // as far as a caller's clause is concerned, and counting lines would miss + // the second. + let emitted: usize = source + .lines() + .map(str::trim_start) + .filter(|line| !line.starts_with("//")) + .map(|line| { + line.matches(describe.as_str()).count() + line.matches(context.as_str()).count() + }) + .sum(); + let documented = clause_diagnostics().len(); + assert_eq!( + emitted, documented, + "{DRIVER} emits {emitted} diagnostics carrying the caller's clause, but \ + `wait_for_screen_where`'s doc enumerates {documented}. A clause has to read \ + correctly after every one of them, so a new diagnostic belongs in that list — \ + and in this file." + ); +} + +/// The driver's doc quotes a real caller's marker as its example. Nothing else +/// connects the two files, so changing the marker would leave the doc quietly +/// illustrating a call that no longer happens. +#[test] +fn the_documented_example_marker_is_the_one_its_caller_passes() { + let caller = read(MARKER_CALLER); + assert!( + caller.contains(&format!("{EXAMPLE_MARKER:?}")), + "{MARKER_CALLER} no longer passes {EXAMPLE_MARKER:?}, which \ + `terminal_state_after_wait`'s doc quotes as its worked example — update the \ + doc, and this constant, to whatever it passes now" + ); +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 0d01ba4de2..87c8bb6006 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -3,6 +3,8 @@ // SPDX-License-Identifier: MIT pub mod capability; +#[cfg(test)] +mod diagnostic_wording; pub mod expectation; pub mod http_server; pub mod installer_fixture; diff --git a/tests/e2e-cucumber/src/send_until.rs b/tests/e2e-cucumber/src/send_until.rs index 6a0b315d79..767cb5eee4 100644 --- a/tests/e2e-cucumber/src/send_until.rs +++ b/tests/e2e-cucumber/src/send_until.rs @@ -72,100 +72,6 @@ where } } -#[cfg(test)] -mod diagnostic_wording { - //! The TUI driver's wait diagnostics are described by doc comments in - //! `tests/e2e/tui_driver.rs` — which templates exist, and what shape of - //! label each expects. Nothing ran those descriptions, so three review - //! rounds in a row found them describing code they no longer matched, each - //! time only because a person read both. - //! - //! These read the driver source and assert the templates the docs name are - //! the templates that exist. Reword a message and this fails, naming the - //! doc that has gone stale; the doc is then wrong for as long as it takes - //! to run the tests, rather than until someone notices. - //! - //! Source text rather than rendered output on purpose: rendering one needs - //! a live pty, a child process and a reader thread, which is what the - //! scenarios themselves are for. What rots here is the wording, and the - //! wording is in the file. - - fn driver_source() -> String { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/e2e/tui_driver.rs"); - std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) - } - - /// A format placeholder as it appears in source: `arg("describe")` is - /// `{describe}`. Built rather than written so no literal in this file - /// contains an uninterpolated `{…}`. - fn arg(name: &str) -> String { - format!("{{{name}}}") - } - - /// The four templates `wait_for_screen_where`'s doc enumerates, which its - /// contract requires a caller's `describe` clause to read correctly after. - #[test] - fn the_clause_templates_are_the_ones_the_contract_names() { - let source = driver_source(); - // Each brace-delimited placeholder is spelled through `arg()` rather - // than written into the literal: these are fragments of OTHER format - // strings, and a literal containing `{describe}` reads to clippy as a - // formatting argument nobody interpolated - // (`literal_string_with_formatting_args`). - for template in [ - format!("panicked while waiting until {}", arg("describe")), - format!( - "timed out after {} waiting until {}", - arg("timeout:?"), - arg("describe") - ), - format!("before {}.", arg("describe")), - format!("draining the final frame{}", arg("context")), - format!(", waiting until {}", arg("wanted")), - ] { - assert!( - source.contains(&template), - "tui_driver.rs no longer contains {template:?}, so the clause contract \ - documented on `wait_for_screen_where` describes a template that is gone \ - — update that doc comment with this change" - ); - } - } - - /// `terminal_state_after_wait` quotes its own bare marker, which its doc - /// states is deliberate and distinct from the clause convention above. - #[test] - fn the_bare_marker_templates_still_quote_the_marker_themselves() { - let source = driver_source(); - // Spelled through `arg()`, for the reason given in the sibling test. - for template in [ - format!("panicked while waiting for {}", arg("marker:?")), - format!("before {} appeared.", arg("marker:?")), - ] { - assert!( - source.contains(&template), - "tui_driver.rs no longer contains {template:?}, so the noun convention \ - documented on `terminal_state_after_wait` describes a template that is \ - gone — update that doc comment with this change" - ); - } - } - - /// The contract says the label is interpolated verbatim. A `{describe:?}` - /// anywhere would quote it a second time on top of whatever the caller - /// already put in, which is what the doc promises does not happen. - #[test] - fn the_clause_label_is_never_debug_formatted() { - let source = driver_source(); - assert!( - !source.contains(&arg("describe:?")), - "a diagnostic Debug-formats `describe`, but `wait_for_screen_where` \ - documents it as interpolated verbatim — callers that quote their own \ - text would now be double-quoted" - ); - } -} - #[cfg(test)] mod tests { use super::{RetryTiming, TerminalState, send_until}; diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 8845d12bba..8982d70b7c 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -375,6 +375,10 @@ impl TuiSession { /// Reached only through [`send_until`](Self::send_until), whose own /// signature takes the marker rather than a description of a condition, so /// there is nothing for a caller to phrase. + /// + /// `src/diagnostic_wording.rs` holds this comment and these templates + /// together in both directions, so reword one there too — including the + /// example marker, which it reads from the caller. fn terminal_state_after_wait(&mut self, marker: &str) -> TerminalState { let reader_finished = self .reader @@ -475,6 +479,11 @@ impl TuiSession { /// the screen"), not a bare noun; and it carries whatever quoting the /// caller puts in it, since none is added here. /// + /// `src/diagnostic_wording.rs` pins this list against the templates below + /// — both that each one still exists and that this comment still names it, + /// and that there are no more of them than it claims. Change a message and + /// that file is where the failure points. + /// /// A child that has exited does not end the wait on its own: the reader is /// given a bounded window to commit whatever was still buffered behind the /// exit notification, because the frame that satisfies `is_ready` is often From a5cb1fc96ed204f85d928dbf66d13393fa260bfe Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 17 Sep 2026 19:26:05 +0200 Subject: [PATCH 50/62] test(e2e): pin every fragment, and read the count the doc actually states The wording guard had two gaps a reviewer found by mutation, and both were mine: answering an earlier note about "the doc says four, the loop iterates five" by deleting the fifth check rather than fixing the count. The fourth clause diagnostic is built in two places -- drain_panic_message phrases the label into a {context} that the drain template interpolates. Only the outer template was pinned, so rewording the label to " for {wanted}" -- the exact defect 40674206 fixed by hand -- passed green. Documented now carries a list of fragments, because a message assembled in more than one place is only as pinned as its least-pinned fragment. The count test compared the code against its own Vec::len, never reading the word "four" in the doc; and doc_text flattened the whole file, so the owner argument was decorative and the enumeration could be moved onto an unrelated method. It now asserts the spelled-out number beside its noun, in the doc block immediately above the named function. Also from that review: the first entry's rendering was a substring of the timeout one, so deleting it from the doc left the tests green; the marker side had no count guard; the dropped {describe:?} guard is back; and the example marker is matched at the send_until call rather than anywhere in the file. Placeholder counting is scoped to lines that frame a screen, so signatures and forwarding calls are not miscounted as diagnostics. Nine mutations checked to redden, including all three the reviewer cited. Signed-off-by: fredespi --- tests/e2e-cucumber/src/diagnostic_wording.rs | 258 ++++++++++++++----- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 15 +- 2 files changed, 196 insertions(+), 77 deletions(-) diff --git a/tests/e2e-cucumber/src/diagnostic_wording.rs b/tests/e2e-cucumber/src/diagnostic_wording.rs index cb7c229422..34b4387726 100644 --- a/tests/e2e-cucumber/src/diagnostic_wording.rs +++ b/tests/e2e-cucumber/src/diagnostic_wording.rs @@ -9,28 +9,35 @@ //! for its wait messages: `wait_for_screen_where` interpolates a caller's //! clause verbatim into four templates, and `terminal_state_after_wait` quotes //! its own bare marker in two. Both are prose, and prose is the part no test -//! runs — this file's own history is the argument for these checks, with three -//! consecutive review rounds finding those comments describing code they did -//! not match, each caught only because a person read both. +//! runs — this file's own history is the argument for these checks, with +//! several consecutive review rounds finding those comments describing code +//! they did not match, each caught only because a person read both. //! -//! So this asserts BOTH directions, because the two failures that actually -//! recurred point opposite ways: +//! What is asserted, for each convention: //! -//! - a template changes and the doc still describes the old one (drift), and -//! - the doc describes something the code never had (wrong when written) — -//! which is what the last two rounds found, and what a one-sided check that -//! only looks for templates in the code would miss entirely. +//! - every documented rendering appears in the doc block of the function that +//! claims to enumerate it — not merely somewhere in the file, so the +//! enumeration cannot be moved off the function it describes; +//! - every `format!` template it names is still in the code, INCLUDING the +//! fragments of any message assembled in more than one place; +//! - the count the doc states in words matches the number of diagnostics that +//! actually carry the caller's text, so a message cannot be added or removed +//! while the prose goes on claiming a different number; +//! - the caller's clause is never Debug-formatted, which would quote it a +//! second time on top of whatever quoting the caller supplied; +//! - the marker the doc quotes as its worked example is still the one its +//! caller passes. //! -//! Each documented rendering must appear in its doc block AND its template -//! must appear in the code, and the number of `{describe}` diagnostics must be -//! the number the doc claims — so a fifth one cannot be added while the doc -//! goes on saying four. +//! Both directions are checked because both have failed here: a template has +//! been reworded while the doc kept describing the old one, and a doc has been +//! written describing something the code never had. A guard that only looks +//! for templates in the code sees neither. //! //! Source text rather than rendered output on purpose: rendering a message //! needs a live pty, a child process and a reader thread, which is what the //! scenarios themselves are for. What rots here is wording, and the wording is -//! in the file. The weakness that remains is that this reads source rather -//! than behaviour: a template could be spelled correctly and never reached. +//! in the file. The weakness that remains is exactly that: a template can be +//! spelled correctly here and never reached at runtime. /// The driver whose diagnostics and doc comments this file holds together. const DRIVER: &str = "tests/e2e/tui_driver.rs"; @@ -47,17 +54,38 @@ fn read(relative: &str) -> String { std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) } -/// The file's doc comments as flowing text: `///` markers stripped and line -/// breaks collapsed, so a documented rendering can be searched for without -/// caring where the author happened to wrap it. -fn doc_text(source: &str) -> String { - source - .lines() - .map(str::trim_start) - .filter_map(|line| { - line.strip_prefix("///") - .or_else(|| line.strip_prefix("//!")) - }) +/// The doc comment attached to `fn `, as flowing text: the contiguous +/// `///` block immediately above it, markers stripped and wrapping collapsed. +/// +/// Scoped to the function on purpose. Flattening the whole file would let an +/// enumeration be moved onto an unrelated method and still satisfy every check +/// — the doc would be somewhere, just not on the thing it claims to describe. +fn doc_block_of(source: &str, name: &str) -> String { + let lines: Vec<&str> = source.lines().collect(); + let signature = lines + .iter() + .position(|line| line.contains(&format!("fn {name}("))) + .unwrap_or_else(|| panic!("{DRIVER} no longer declares `fn {name}`")); + + let mut first = signature; + while first > 0 { + let candidate = lines[first - 1].trim_start(); + // Attributes and ordinary comments sit between a doc block and its + // item, so step over them; anything else ends the block. + if candidate.starts_with("///") + || candidate.starts_with("#[") + || candidate.starts_with("//") + { + first -= 1; + } else { + break; + } + } + + lines[first..signature] + .iter() + .map(|line| line.trim_start()) + .filter_map(|line| line.strip_prefix("///")) .map(str::trim) .collect::>() .join(" ") @@ -70,33 +98,48 @@ fn arg(name: &str) -> String { format!("{{{name}}}") } -/// One documented diagnostic: the `format!` template in the code, and the -/// rendering the doc comment shows for it. Both must be present. +/// One documented diagnostic: the `format!` fragments that build it in the +/// code, and the rendering the doc comment shows for it. +/// +/// `templates` is a list because a message assembled in more than one place is +/// only as pinned as its least-pinned fragment. The drain message is the +/// standing example: its label arrives through a `{context}` built by a +/// separate function, and pinning only the outer template left that function +/// free to reword the label into a shape the doc denies. struct Documented { - template: String, + templates: Vec, documented_as: String, } fn clause_diagnostics() -> Vec { let describe = arg("describe"); let timeout = arg("timeout:?"); + let wanted = arg("wanted"); vec![ Documented { - template: format!("panicked while waiting until {describe}"), - documented_as: format!("waiting until {describe}"), + templates: vec![format!("panicked while waiting until {describe}: ")], + // Distinguishing prefix on purpose: a bare `waiting until + // {describe}` is a substring of the timeout rendering below, so + // deleting this line from the doc would leave the check green. + documented_as: format!("panicked while waiting until {describe}"), }, Documented { - template: format!("timed out after {timeout} waiting until {describe}"), + templates: vec![format!( + "timed out after {timeout} waiting until {describe}" + )], documented_as: format!("timed out … waiting until {describe}"), }, Documented { - template: format!("before {describe}."), - documented_as: format!("before {describe}"), + templates: vec![format!("before {describe}.")], + documented_as: format!("before {describe}."), }, Documented { - // The fourth is assembled from two fragments: the drain message - // interpolates a context that the label is built into. - template: format!("draining the final frame{}", arg("context")), + // Two fragments in two functions: the drain message interpolates a + // context, and the context is where the label is actually phrased. + templates: vec![ + format!("draining the final frame{}", arg("context")), + format!(", waiting until {wanted}"), + ], documented_as: format!("draining the final frame, waiting until {describe}"), }, ] @@ -106,38 +149,76 @@ fn marker_diagnostics() -> Vec { let marker = arg("marker:?"); vec![ Documented { - template: format!("panicked while waiting for {marker}"), + templates: vec![format!("panicked while waiting for {marker}: ")], documented_as: format!("panicked while waiting for {EXAMPLE_MARKER:?}"), }, Documented { - template: format!("before {marker} appeared."), + templates: vec![format!("before {marker} appeared.")], documented_as: format!("before {EXAMPLE_MARKER:?} appeared."), }, ] } fn assert_both_sides(source: &str, documented: &[Documented], owner: &str) { - let docs = doc_text(source); + let docs = doc_block_of(source, owner); for Documented { - template, + templates, documented_as, } in documented { - assert!( - source.contains(template.as_str()), - "{DRIVER} no longer contains the template {template:?}, but `{owner}`'s doc \ - comment still describes it — update the doc with this change" - ); + for template in templates { + assert!( + source.contains(template.as_str()), + "{DRIVER} no longer contains the fragment {template:?}, but `{owner}`'s doc \ + comment still describes the message it builds as {documented_as:?} — update \ + the doc with this change" + ); + } assert!( docs.contains(documented_as.as_str()), - "`{owner}`'s doc comment no longer shows {documented_as:?}, but {DRIVER} still \ + "`{owner}`'s own doc comment no longer shows {documented_as:?}, but {DRIVER} still \ emits that diagnostic — the doc must keep naming every message it claims to \ enumerate, or this guard is checking nothing" ); } } -/// The four clause-shaped diagnostics, in both directions. +/// The doc states its count in words. Assert the word, not this file's list +/// length: comparing the code against our own `Vec` would pass whatever the +/// prose said, which is the defect this test is named for. +fn assert_documented_count(source: &str, owner: &str, count: usize, noun: &str) { + let spelled = match count { + 2 => "two", + 3 => "three", + 4 => "four", + 5 => "five", + 6 => "six", + other => panic!("no spelling for {other}; add one when the list grows"), + }; + let docs = doc_block_of(source, owner); + assert!( + docs.contains(&format!("{spelled} {noun}")), + "`{owner}`'s doc comment does not say {spelled:?} {noun}, but that is how many there \ + are — the stated count and the real one must agree, or the prose is telling the next \ + reader something the code denies" + ); +} + +/// The lines that build a wait diagnostic: every one of them ends its message +/// with the framed screen, which is what makes it a report about a stuck +/// terminal rather than an ordinary error string. Signatures, forwarding calls +/// and the clause `wait_for_screen` assembles for its own caller all mention +/// the same placeholders without being diagnostics, so counting placeholders +/// file-wide would count those too. +fn diagnostic_lines(source: &str) -> impl Iterator { + source + .lines() + .map(str::trim_start) + .filter(|line| !line.starts_with("//")) + .filter(|line| line.contains("\\n{}")) +} + +/// The four clause-shaped diagnostics, in both directions, fragments included. #[test] fn every_clause_diagnostic_is_both_emitted_and_documented() { assert_both_sides( @@ -157,49 +238,84 @@ fn every_marker_diagnostic_is_both_emitted_and_documented() { ); } -/// The doc says "four diagnostics". A fifth one added without touching the doc -/// would satisfy every check above — each documented message would still be -/// present — while the count silently became a lie. This is the direction a -/// list of substrings cannot see. +/// A diagnostic added or removed without touching the doc satisfies every +/// check above — each documented message is still present — while the count +/// silently becomes a lie. Both the code's count and the doc's stated one are +/// read here, so neither side can drift alone. #[test] fn the_documented_count_of_clause_diagnostics_is_the_real_one() { let source = read(DRIVER); - let describe = arg("describe"); + let expected = clause_diagnostics().len(); + assert_documented_count(&source, "wait_for_screen_where", expected, "diagnostics"); + // The drain message reaches the label through `{context}` rather than - // naming it, so it is counted by its own placeholder. Both are code lines: - // the doc's own occurrences sit behind `///` and are filtered out. + // naming it, so it is counted by its own placeholder. Occurrences, not + // lines: two placeholders on one line are two diagnostics as far as a + // caller's clause is concerned. Doc lines are excluded — they quote these + // same placeholders. + let describe = arg("describe"); let context = arg("context"); - // Occurrences, not lines: two placeholders on one line are two diagnostics - // as far as a caller's clause is concerned, and counting lines would miss - // the second. - let emitted: usize = source - .lines() - .map(str::trim_start) - .filter(|line| !line.starts_with("//")) + let emitted: usize = diagnostic_lines(&source) .map(|line| { line.matches(describe.as_str()).count() + line.matches(context.as_str()).count() }) .sum(); - let documented = clause_diagnostics().len(); assert_eq!( - emitted, documented, + emitted, expected, "{DRIVER} emits {emitted} diagnostics carrying the caller's clause, but \ - `wait_for_screen_where`'s doc enumerates {documented}. A clause has to read \ - correctly after every one of them, so a new diagnostic belongs in that list — \ - and in this file." + `wait_for_screen_where` documents {expected}. A clause has to read correctly after \ + every one of them, so a new diagnostic belongs in that list — and in this file." + ); +} + +/// `terminal_state_after_wait`'s doc says a third path names no marker. A +/// third marker-bearing message would contradict it silently. +#[test] +fn the_documented_count_of_marker_diagnostics_is_the_real_one() { + let source = read(DRIVER); + let expected = marker_diagnostics().len(); + assert_documented_count(&source, "terminal_state_after_wait", expected, "messages"); + + let marker = arg("marker:?"); + let emitted: usize = diagnostic_lines(&source) + .map(|line| line.matches(marker.as_str()).count()) + .sum(); + assert_eq!( + emitted, expected, + "{DRIVER} emits {emitted} messages quoting a bare marker, but \ + `terminal_state_after_wait` documents {expected}" + ); +} + +/// The contract says the clause is interpolated verbatim. A `{describe:?}` +/// anywhere would quote it a second time on top of whatever the caller already +/// put in — and would also slip past the count above, which looks for the +/// undecorated placeholder. +#[test] +fn the_clause_label_is_never_debug_formatted() { + let source = read(DRIVER); + assert!( + !source.contains(&arg("describe:?")), + "a diagnostic Debug-formats the caller's clause, but `wait_for_screen_where` \ + documents it as interpolated verbatim — a caller that quotes its own text would \ + now be double-quoted" ); } /// The driver's doc quotes a real caller's marker as its example. Nothing else /// connects the two files, so changing the marker would leave the doc quietly /// illustrating a call that no longer happens. +/// +/// Matched at the call itself rather than anywhere in the file: a stale comment +/// still mentioning the old marker would otherwise satisfy this. #[test] fn the_documented_example_marker_is_the_one_its_caller_passes() { let caller = read(MARKER_CALLER); + let call = format!("send_until(\"4\", {EXAMPLE_MARKER:?}"); assert!( - caller.contains(&format!("{EXAMPLE_MARKER:?}")), - "{MARKER_CALLER} no longer passes {EXAMPLE_MARKER:?}, which \ - `terminal_state_after_wait`'s doc quotes as its worked example — update the \ - doc, and this constant, to whatever it passes now" + caller.contains(&call), + "{MARKER_CALLER} no longer calls `send_until` with {EXAMPLE_MARKER:?}, which \ + `terminal_state_after_wait`'s doc quotes as its worked example — update the doc, \ + and this constant, to whatever it passes now" ); } diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 8982d70b7c..8f416389ef 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -367,9 +367,10 @@ impl TuiSession { /// /// Takes a bare marker and quotes it here, so both of its marker-bearing /// messages read as nouns — deliberately NOT the clause convention - /// `wait_for_screen_where` documents. For `send_until`'s marker they are - /// `pty reader thread panicked while waiting for "● Observe"` and - /// `process exited (…) before "● Observe" appeared.`; a third path + /// `wait_for_screen_where` documents. There are two messages, and for + /// `send_until`'s marker they read + /// `panicked while waiting for "● Observe"` and + /// `before "● Observe" appeared.`; a third path /// reports a poll failure and names no marker at all. /// /// Reached only through [`send_until`](Self::send_until), whose own @@ -472,9 +473,11 @@ impl TuiSession { /// marker the frame stopped showing. /// /// `describe` is interpolated verbatim — no quoting, no rewording — into - /// four diagnostics, which read `waiting until {describe}`, `timed out … - /// waiting until {describe}`, `before {describe}`, and `… draining the - /// final frame, waiting until {describe}`. So it must be a clause that + /// four diagnostics, which read `panicked while waiting until {describe}`, + /// `timed out … waiting until {describe}`, `before {describe}.`, and `… + /// draining the final frame, waiting until {describe}`. The last of those + /// is assembled from two fragments in two functions, and both carry the + /// clause. So it must be a clause that /// fits all four ("the screen shows X", "generation throughput leaves /// the screen"), not a bare noun; and it carries whatever quoting the /// caller puts in it, since none is added here. From 97615d63e1b67ff53096256b37752dffcf80db23 Mon Sep 17 00:00:00 2001 From: fredespi Date: Sat, 19 Sep 2026 14:34:03 +0200 Subject: [PATCH 51/62] fix(e2e): read the human report from the slot its own When writes examine-15 asserted against world.cli_stderr, but the When it depends on -- "the user inspects the system both for reading and for scripting" -- runs `examine` and `examine --json` as two commands and puts the human form in cli_other_output. Its own comment says so: the field did not exist when the step was written, which is why it used to borrow the stderr slot. The sibling Then was moved across; this one was not, so it read None and panicked with "the human report was not captured" before asserting anything. Four GPU lanes on e9d1f8a4 -- MI300X, MI350P, Strix Halo Ubuntu and Windows -- each reported exactly one unexpected failure, and it was this, at examine_steps.rs:529 in all four. The scenario is @requires-gpu because its Given installs a managed runtime, so no hosted lane runs it and none of them went red. Checked against what those runs captured rather than against intent: the human report prints active_runtime_root, and framework_notes names an interpreter inside that root with framework_source "managed-runtime", so the assertions the step never reached do hold on that data. The fifth red lane, rad3 R9700, is unrelated -- cancelled at the 24h ceiling with no steps started, so no runner ever picked it up. Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 24b7468e49..e81d74f184 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -528,8 +528,11 @@ fn human_states(human: &str, label: &str) -> Option { #[then("the framework report names the runtime's interpreter")] async fn assert_framework_names_the_runtimes_interpreter(world: &mut E2eWorld) { + // `cli_other_output`, not `cli_stderr`: the human form is a second command's + // STDOUT, and the `When` above moved it out of the stderr slot it used to + // borrow. Reading the old slot found `None` on every GPU lane. let human = world - .cli_stderr + .cli_other_output .as_ref() .expect("the human report was not captured"); // Read the runtime from the human form: `examine --json` carries no runtime From 6b3256d1db459584d7f6fb38eab0039a1572750b Mon Sep 17 00:00:00 2001 From: fredespi Date: Sat, 19 Sep 2026 14:37:17 +0200 Subject: [PATCH 52/62] test(e2e): read the code side from code, not from the doc vouching for itself The previous commit added a period to the third documented rendering so it matched its entry. That made the doc line contain `before {describe}.` verbatim -- and the code-side assertion was a `contains` over the whole driver file, doc comments included. So the prose satisfied the assertion that exists to check the code against it. Reworded the emitted template to "prior to {describe}." and all six tests passed, with the only remaining occurrence of the pinned string being the comment. A one-line fix would have been to drop the period again. Both sides of this check live in one file, so the trap is general and would be walked into by the next reword: `assert_both_sides` now matches templates against a `code_text` with comment lines removed, the way `diagnostic_lines` already did for counting, and the `Documented` doc says why the code side must never be read from the raw file. Also pins the drain's call site. `drain_final_frame_where` takes an Option, so passing None would leave all six templates intact while the message quietly stopped carrying the caller's clause -- with the doc still counting it among the four that do. Mutation-verified: each of the six pinned templates reworded one at a time reddens on its own, as does the blocking case above, dropping the clause at the drain call, and the six doc-side mutations from the previous round. Signed-off-by: fredespi --- tests/e2e-cucumber/src/diagnostic_wording.rs | 37 +++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/src/diagnostic_wording.rs b/tests/e2e-cucumber/src/diagnostic_wording.rs index 34b4387726..0c7f6f7b6d 100644 --- a/tests/e2e-cucumber/src/diagnostic_wording.rs +++ b/tests/e2e-cucumber/src/diagnostic_wording.rs @@ -106,6 +106,13 @@ fn arg(name: &str) -> String { /// standing example: its label arrives through a `{context}` built by a /// separate function, and pinning only the outer template left that function /// free to reword the label into a shape the doc denies. +/// +/// `templates` must be looked for in CODE ONLY, never the raw file. Both sides +/// of this check live in the same source, and a doc rendering is often the +/// template spelled out — so a file-wide `contains` lets the prose vouch for +/// itself and the code side stops testing anything. That is not hypothetical: +/// adding a period to the third rendering, to match its `documented_as`, did +/// exactly this and went unnoticed for a commit. struct Documented { templates: Vec, documented_as: String, @@ -159,8 +166,19 @@ fn marker_diagnostics() -> Vec { ] } +/// The driver with every comment line removed — the only text a code-side +/// assertion may be made against. See [`Documented`] for why. +fn code_text(source: &str) -> String { + source + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .collect::>() + .join("\n") +} + fn assert_both_sides(source: &str, documented: &[Documented], owner: &str) { let docs = doc_block_of(source, owner); + let code = code_text(source); for Documented { templates, documented_as, @@ -168,7 +186,7 @@ fn assert_both_sides(source: &str, documented: &[Documented], owner: &str) { { for template in templates { assert!( - source.contains(template.as_str()), + code.contains(template.as_str()), "{DRIVER} no longer contains the fragment {template:?}, but `{owner}`'s doc \ comment still describes the message it builds as {documented_as:?} — update \ the doc with this change" @@ -287,6 +305,23 @@ fn the_documented_count_of_marker_diagnostics_is_the_real_one() { ); } +/// The drain message is the one diagnostic whose clause is optional at the +/// call: `drain_final_frame_where` takes an `Option`, and passing `None` would +/// leave every template above intact while the message silently stopped +/// carrying the caller's text. The doc counts it among the four, so the call +/// that makes that true is pinned too. +#[test] +fn the_drain_is_still_reached_with_the_callers_clause() { + let code = code_text(&read(DRIVER)); + let describe = arg("describe"); + assert!( + code.contains("drain_final_frame_where(Some(describe),"), + "`wait_for_screen_where` no longer hands its clause to the drain, but its doc still \ + counts `draining the final frame, waiting until {describe}` among the four \ + diagnostics that carry it" + ); +} + /// The contract says the clause is interpolated verbatim. A `{describe:?}` /// anywhere would quote it a second time on top of whatever the caller already /// put in — and would also slip past the count above, which looks for the From e8626d385511fde05bcf02b3017ea95be9da8b92 Mon Sep 17 00:00:00 2001 From: fredespi Date: Sat, 19 Sep 2026 14:41:15 +0200 Subject: [PATCH 53/62] test(e2e): cover the four expectation branches nothing was asserting Each is production-consumed and could be removed or inverted with all 42 tests still green: - the `is_wsl` arm of `Condition::matches`. `os_family` is `linux` on WSL2 and on bare metal, so this key is the only thing that tells them apart; the test asserts both polarities on both hosts, since a check that always returns true satisfies a one-sided assertion. - `deny_unknown_fields` on `Condition` and on `XfailEntry`. This is the failure the `Condition` doc describes at length: a misspelled key parses to an all-`None` condition, which matches every host, so one row becomes a permanent suite-wide xfail. Both structs are covered -- `engine` for `effective_engine`, and `flakey` for `flaky`. - the `requires_gpu &&` conjunct on the engine gate. Asserted from both sides on one host with one unstartable engine (vLLM on Windows, the real case), so dropping the conjunct flips exactly one of them. - `ResolvedScenario::new`, which builds every row of the platform report. A row that loses its bug id or its flaky flag loses it in the artifact a reader uses to tell an expected failure from a real one; all three expectation shapes are checked, including the two `None`s that distinguish them. Mutation-verified: seven mutations, one per branch plus three inside `ResolvedScenario::new`, each reddening its own test and no other. Signed-off-by: fredespi --- tests/e2e-cucumber/src/expectation.rs | 134 ++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index ece1fd429d..39e9152d44 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1695,4 +1695,138 @@ flaky = true assert!(!glob_match("gfx94*", "gfx1151")); assert!(glob_match("gfx1151", "gfx1151")); } + + /// `is_wsl` is what tells a WSL2 host from bare metal — `os_family` is + /// `linux` on both. A row conditioned on it must hold on one and not the + /// other, in both polarities, or the key is decorative. + #[test] + fn a_row_conditioned_on_wsl_holds_only_there() { + let d = decl(&["id:x"]); + for (value, xfails_on) in [(true, "wsl2"), (false, "strix-ubuntu")] { + let m = Expectations::parse(&format!( + "[[\"x\"]]\nwhen = {{ is_wsl = {value} }}\nbug = \"EAI-1\"\n\ + reason = \"r\"\n" + )) + .unwrap(); + let other = if value { "strix-ubuntu" } else { "wsl2" }; + assert!( + matches!( + resolve(&d, &cap(xfails_on), &m, false, false, false), + Expectation::ExpectXfail { .. } + ), + "is_wsl = {value} should hold on {xfails_on}" + ); + assert_eq!( + resolve(&d, &cap(other), &m, false, false, false), + Expectation::ExpectPass, + "is_wsl = {value} should not hold on {other}" + ); + } + } + + /// The typo this guards against is the one the `Condition` doc describes: a + /// misspelled key parses to an all-`None` condition, which matches every + /// host, turning one row into a permanent suite-wide xfail. Asserted on both + /// structs, since `XfailEntry` carries the row's own keys. + #[test] + fn a_misspelled_key_is_rejected_rather_than_matching_everything() { + let typo_in_condition = Expectations::parse( + "[[\"x\"]]\nwhen = { engine = \"vllm\" }\nbug = \"EAI-1\"\nreason = \"r\"\n", + ); + assert!( + typo_in_condition.is_err(), + "`engine` is not `effective_engine`; accepting it would xfail every platform" + ); + let typo_in_entry = Expectations::parse( + "[[\"x\"]]\nwhen = { os = \"linux\" }\nbug = \"EAI-1\"\nreason = \"r\"\n\ + flakey = true\n", + ); + assert!( + typo_in_entry.is_err(), + "`flakey` is not `flaky`; accepting it would silently drop the tolerance it asks for" + ); + } + + /// The engine gate is deliberately conjoined with `requires_gpu`: a scenario + /// that never serves must not be skipped because some engine cannot start. + /// Both sides are asserted on the same host and the same unstartable engine + /// — vLLM on Windows, which is the real case — so dropping the conjunct + /// changes exactly one of them. + #[test] + fn only_gpu_scenarios_are_gated_on_the_engine_starting() { + let m = Expectations::parse("").unwrap(); + let host = cap("strix-windows"); + assert!( + !host.engine_available("vllm"), + "this test needs an engine that cannot start here" + ); + let gpu = decl(&["id:x", "requires-gpu", "requires-engine:vllm"]); + assert!( + matches!( + resolve(&gpu, &host, &m, false, false, false), + Expectation::Skip { .. } + ), + "a GPU scenario cannot run where its engine will not start" + ); + let no_gpu = decl(&["id:x", "requires-engine:vllm"]); + assert_eq!( + resolve(&no_gpu, &host, &m, false, false, false), + Expectation::ExpectPass, + "a scenario that never serves has no stake in whether the engine starts" + ); + } + + /// `ResolvedScenario` is what the platform report is built from, so a row + /// that loses its bug id or its flaky flag here loses it in the artifact a + /// reader uses to tell an expected failure from a real one. + #[test] + fn the_report_row_carries_each_expectations_own_metadata() { + let xfail = ResolvedScenario::new( + "x", + "f", + "s", + "vllm", + &Expectation::ExpectXfail { + bug: "EAI-1".into(), + reason: "r".into(), + flaky: true, + }, + ); + assert_eq!(xfail.bug.as_deref(), Some("EAI-1")); + assert_eq!(xfail.reason.as_deref(), Some("r")); + assert!(xfail.flaky); + assert_eq!( + xfail.expected, + Expectation::ExpectXfail { + bug: "EAI-1".into(), + reason: "r".into(), + flaky: true, + } + .label() + ); + assert_eq!(xfail.id, "x"); + assert_eq!(xfail.feature, "f"); + assert_eq!(xfail.scenario, "s"); + assert_eq!(xfail.effective_engine, "vllm"); + + // A skip keeps its reason but has no bug, and a pass has neither — the + // two `None`s a reader distinguishes an expected failure by. + let skipped = ResolvedScenario::new( + "x", + "f", + "s", + "vllm", + &Expectation::Skip { + reason: "no gpu".into(), + }, + ); + assert_eq!(skipped.bug, None); + assert_eq!(skipped.reason.as_deref(), Some("no gpu")); + assert!(!skipped.flaky); + + let passing = ResolvedScenario::new("x", "f", "s", "vllm", &Expectation::ExpectPass); + assert_eq!(passing.bug, None); + assert_eq!(passing.reason, None); + assert!(!passing.flaky); + } } From f87ab1d99745ceb0f55c72933d2067ecd2ce0b7d Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 25 Sep 2026 17:37:33 +0200 Subject: [PATCH 54/62] test(e2e): close the block-comment hole, and make an absent cause mean cleared Three non-blocking notes from the last review. `code_text` dropped only `//` lines, so an own-line `/* ... */` comment could still vouch for a template the way the doc comment did before. Reproduced: reworded the third diagnostic and parked the old wording in a block comment, all seven tests green. Block-comment lines (`/*` and a leading `*`) are now prose too; the same mutation reddens exactly the code-side test. The diagnose-23 step read a missing device-permission entry as score 0. That is the outcome it asks for, but a run that failed, or routed out of scope before any device check ran, also produces no entry. The step now requires exit 0 and no out-of-scope verdict before an absent entry counts, and an entry present without a readable score panics instead of reading as zero. Not `!matched.is_empty()`, as suggested: the scenario runs diagnose with no symptom on a machine whose device works, where matching nothing at all is a correct answer and asserting otherwise would fail the fix it waits for. The wait_for_screen_where doc paragraph is reflowed; the orphaned short line came from inserting the two-fragment sentence. Signed-off-by: fredespi --- tests/e2e-cucumber/src/diagnostic_wording.rs | 5 +++- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 30 ++++++++++++++++--- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 8 ++--- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/tests/e2e-cucumber/src/diagnostic_wording.rs b/tests/e2e-cucumber/src/diagnostic_wording.rs index 0c7f6f7b6d..7c81961119 100644 --- a/tests/e2e-cucumber/src/diagnostic_wording.rs +++ b/tests/e2e-cucumber/src/diagnostic_wording.rs @@ -171,7 +171,10 @@ fn marker_diagnostics() -> Vec { fn code_text(source: &str) -> String { source .lines() - .filter(|line| !line.trim_start().starts_with("//")) + .map(str::trim_start) + // `/*` and a leading `*` too: an own-line block comment is prose just + // as a `//` line is, and could vouch for a template the same way. + .filter(|line| !line.starts_with("//") && !line.starts_with("/*") && !line.starts_with('*')) .collect::>() .join("\n") } diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 2392f7d753..b3d799ef80 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -1412,6 +1412,22 @@ async fn assert_no_device_group_remedy(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no diagnose output"); let report: serde_json::Value = serde_json::from_str(output).expect("diagnose --json did not emit valid JSON"); + // An absent entry is the outcome asked for, so it must be absent because the + // checks ran and cleared it — not because the run failed or routed out + // before evaluating anything. Not `!matched.is_empty()`: with no symptom + // given, a healthy machine correctly matches nothing at all. + assert_eq!( + world.cli_rc, + Some(0), + "diagnose should exit 0 (it is a query):\n{output}" + ); + assert!( + report + .get("out_of_scope") + .is_none_or(serde_json::Value::is_null), + "this bare-metal Linux host was routed out of scope, so the device checks never \ + ran and an absent cause proves nothing:\n{output}" + ); let matched = report .get("matched") .and_then(|m| m.as_array()) @@ -1419,10 +1435,16 @@ async fn assert_no_device_group_remedy(world: &mut E2eWorld) { let offered = matched.iter().find(|entry| { entry.get("id").and_then(serde_json::Value::as_str) == Some(DEVICE_PERMISSION_FIX_ID) }); - let score = offered - .and_then(|entry| entry.get("score")) - .and_then(serde_json::Value::as_i64) - .unwrap_or(0); + // Absent means not offered; present without a readable score is a report + // this step cannot judge, and is not allowed to read as a zero. + let score = offered.map_or(0, |entry| { + entry + .get("score") + .and_then(serde_json::Value::as_i64) + .unwrap_or_else(|| { + panic!("the '{DEVICE_PERMISSION_FIX_ID}' entry has no score:\n{output}") + }) + }); // Belonging to a conventional group is one way to reach the device, not the // point of doing so. With access already demonstrated, repairing permissions // is not a cause of anything, and offering it sends the user to change their diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 8f416389ef..995668439d 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -477,10 +477,10 @@ impl TuiSession { /// `timed out … waiting until {describe}`, `before {describe}.`, and `… /// draining the final frame, waiting until {describe}`. The last of those /// is assembled from two fragments in two functions, and both carry the - /// clause. So it must be a clause that - /// fits all four ("the screen shows X", "generation throughput leaves - /// the screen"), not a bare noun; and it carries whatever quoting the - /// caller puts in it, since none is added here. + /// clause. So it must be a clause that fits all four ("the screen shows + /// X", "generation throughput leaves the screen"), not a bare noun; and it + /// carries whatever quoting the caller puts in it, since none is added + /// here. /// /// `src/diagnostic_wording.rs` pins this list against the templates below /// — both that each one still exists and that this comment still names it, From 94f1c00f5c9a4274fad8a43ee7aed369553d9f49 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 27 Oct 2026 13:00:51 +0100 Subject: [PATCH 55/62] test(e2e): address review round on the README-contract pins Move examine-29 readers onto cli_other_output, stop absence waits being satisfied by a dead process's last frame, gate diagnose-42 to bare metal, and fix citations, stray doc lines and comment drift. Signed-off-by: fredespi --- .github/workflows/e2e-selfhosted.yml | 2 +- tests/e2e-cucumber/README.md | 10 ++++---- tests/e2e-cucumber/expectations.toml | 4 ++-- tests/e2e-cucumber/features/chat.feature | 2 +- tests/e2e-cucumber/features/diagnose.feature | 2 +- tests/e2e-cucumber/src/diagnostic_wording.rs | 4 ++-- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 14 +++++++---- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 23 +++++++++++++++---- tests/e2e-cucumber/tests/e2e/update_steps.rs | 1 - xtask/src/ascii_output_contract.rs | 7 +++--- 10 files changed, 45 insertions(+), 24 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 788ad63615..36013187fc 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -1532,7 +1532,7 @@ jobs: || (github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' || inputs.platform == 'w7900')) ) - # Non-blocking while this hardware is proven out, as the MI350P lane is. + # continue-on-error while this hardware is proven out. # Flip to `false` once it has a track record — that is what the older lanes # run with, so a real regression on this silicon shows red instead of # always green. diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 941c0e4be2..e81244b4d6 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -114,9 +114,9 @@ carrying one runs only where its env var is set: `@nightly` `@merge-queue` (`E2E_MERGE_QUEUE`, set on the merge-queue lanes). Note what that means for gating: a `@merge-queue` scenario does not run on ordinary per-PR CI at all, and the lanes that do run it — the self-hosted ones in -`.github/workflows/e2e-selfhosted.yml`, not `ci.yml` — are non-blocking. Branch +[`.github/workflows/e2e-selfhosted.yml`](../../.github/workflows/e2e-selfhosted.yml), not `ci.yml` — are non-blocking. Branch protection is a GitHub setting no checkout can verify, so the reason is not -restated here: that file's header gives it, and `docs/ci-hardware-testing.md` +restated here: that file's header gives it, and [`docs/ci-hardware-testing.md`](../../docs/ci-hardware-testing.md) carries the fuller history. ### Naming @@ -156,7 +156,7 @@ Scenarios carry stable-id and capability tags: | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@requires-no-gpu` | Premise is a host with no usable AMD GPU (e.g. a refusal that only happens without one). The inverse of `@requires-gpu`; resolves to **skip** on a GPU host. | -| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the self-hosted lanes in `e2e-selfhosted.yml` do on a `merge_group` event. Those lanes are non-blocking (that file's header says why), so such a scenario is telemetry rather than a gate. | +| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the self-hosted lanes in [`e2e-selfhosted.yml`](../../.github/workflows/e2e-selfhosted.yml) do on a `merge_group` event. Those lanes are non-blocking (that file's header says why), so such a scenario is telemetry rather than a gate. | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. | @@ -170,8 +170,8 @@ bug. See `src/expectation.rs` for the resolver and `expectations.toml`'s header for the condition grammar. When a comment here points at product code, **cite a symbol, never a line -number** — `select_runtime_update_source` in `apps/rocm/src/main.rs`, not -`main.rs:16427`. `apps/rocm/src/main.rs` alone is over twenty thousand lines and +number** — `select_runtime_update_source` in [`apps/rocm/src/main.rs`](../../apps/rocm/src/main.rs), not +`main.rs:16427`. `main.rs` alone is over twenty thousand lines and moves every week; nothing checks these numbers, so a stale one sends the reader somewhere confidently wrong, which is worse than no pointer at all. Most of the citations written during this suite's review had already drifted — one by five diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 8e824b09f9..66487cbfc4 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -176,8 +176,8 @@ flaky = true # --- EAI-8020: `diagnose` and `fix` each carry their own copy of the # render-group remedy, and the verification step has drifted between them # (`diagnose` adds `ls -l /dev/kfd`). Both strings are constants, so this holds -# wherever the catalog runs at all; `@requires-os:linux` on the scenario keeps it -# off Windows, where the recipe does not apply. Remove when the two are one. --- +# wherever the catalog runs on bare metal; `@requires-os:linux` and +# `@requires-bare-metal` keep it off Windows and WSL2, where the recipe does not apply. Remove when the two are one. --- [["diagnose-and-fix-agree-on-how-to-verify"]] when = {} bug = "EAI-8020" diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index 4d9cbd1b6f..1eaf31a850 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -108,7 +108,7 @@ Feature: Chat and endpoint detection Then the CLI prints the assistant's reply And the model receives the piped prompt with its whitespace intact - # Expected to FAIL. Scenario 7 names the model and works; this one leaves the + # Expected to FAIL. Scenario 8 names the model and works; this one leaves the # choice to the CLI and is told nothing is running — on the very machine whose # services list shows a ready local server. Asking to chat locally should use # the local server that is there, whichever model the user chose to serve. diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 8bb7facdc7..0de90b6d02 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -638,7 +638,7 @@ Feature: Diagnosing failures and listing fixes # first tells the user what will put the machine right, while the second is the # command they are told to run. They must not leave the user with two different # definitions of what proves the remedy worked. - @id:diagnose-and-fix-agree-on-how-to-verify @requires-os:linux + @id:diagnose-and-fix-agree-on-how-to-verify @requires-os:linux @requires-bare-metal Scenario: diagnose-42 - Diagnosing a problem and previewing its fix agree on how to verify it Given a user who hit a device-permission failure When the user compares the diagnosis with the matching fix preview diff --git a/tests/e2e-cucumber/src/diagnostic_wording.rs b/tests/e2e-cucumber/src/diagnostic_wording.rs index 6a50e47a4f..fc437321b9 100644 --- a/tests/e2e-cucumber/src/diagnostic_wording.rs +++ b/tests/e2e-cucumber/src/diagnostic_wording.rs @@ -282,7 +282,7 @@ fn the_documented_count_of_clause_diagnostics_is_the_real_one() { // with its own doc, not a fifth message of this list. let scoped = format!( "{}\n{}", - body_of(&source, "wait_for_screen_where"), + body_of(&source, "wait_loop"), body_of(&source, "drain_panic_message") ); let emitted: usize = diagnostic_lines(&scoped) @@ -301,7 +301,7 @@ fn the_documented_count_of_clause_diagnostics_is_the_real_one() { /// The body of `fn `: from its signature to the brace that closes it. /// /// The marker convention is not unique to one function — `wait_until_gone` -/// quotes its own bare marker in messages of its own — so a count of `{marker:?}` +/// formats its own bare marker into its `describe` label — so a count of `{marker:?}` /// across the file measures every function that happens to name its parameter /// `marker`, not the one whose doc states the number. fn body_of<'a>(source: &'a str, name: &str) -> String { diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 80138c7e0c..f55f83a669 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -639,7 +639,7 @@ async fn user_inspects_for_scripting_first(world: &mut E2eWorld) { // second, so the comparison step can hold the two to each other. let (json, _, rc) = crate::run_rocm(world, &["examine", "--json"]); let (human, _, _) = crate::run_rocm(world, &["examine"]); - world.cli_stderr = Some(human); + world.cli_other_output = Some(human); world.cli_output = Some(json); world.cli_rc = Some(rc); } @@ -842,7 +842,7 @@ async fn assert_json_states_what_human_does(world: &mut E2eWorld) { async fn assert_json_names_setup_runtime_folder(world: &mut E2eWorld) { let planted = planted_setup_runtime_root(world).display().to_string(); let human = world - .cli_stderr + .cli_other_output .as_ref() .expect("the human report was not captured"); let value = parsed_json(world); @@ -2096,7 +2096,10 @@ fn age_expected_from_date(record: &serde_json::Value) -> &'static str { )] async fn assert_examine_reports_driver_version_and_date(world: &mut E2eWorld) { let json = parsed_json(world); - let text = world.cli_stderr.as_ref().expect("no readable inspection"); + let text = world + .cli_other_output + .as_ref() + .expect("no readable inspection"); let record = windows_driver_record(&json).unwrap_or_else(|| { panic!("the machine-readable form carries no Windows driver record:\n{json:#}") }); @@ -2124,7 +2127,10 @@ async fn assert_examine_reports_driver_version_and_date(world: &mut E2eWorld) { #[then("the inspection calls the driver old only when its date is before 27 July 2023")] async fn assert_examine_driver_age_follows_the_date(world: &mut E2eWorld) { let json = parsed_json(world); - let text = world.cli_stderr.as_ref().expect("no readable inspection"); + let text = world + .cli_other_output + .as_ref() + .expect("no readable inspection"); let record = windows_driver_record(&json) .unwrap_or_else(|| panic!("no Windows driver record in the machine-readable form")); let expected = age_expected_from_date(record); diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index d42ea71cf4..565574322d 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -740,7 +740,7 @@ impl TuiSession { // "waiting until {describe}", so a lone `"Ready"` would land as // `waiting until "Ready"`. let wanted = format!("the screen shows {marker:?}"); - self.wait_for_screen_where(&wanted, |screen| screen.contains(marker), timeout) + self.wait_loop(&wanted, |screen| screen.contains(marker), timeout, true) .await } @@ -807,10 +807,24 @@ impl TuiSession { /// recovery is given up on purpose: a diagnosed exit is worth more than a /// condition that only ever held in a dying process's last frame. pub async fn wait_for_screen_where( + &mut self, + describe: &str, + is_ready: impl FnMut(&str) -> bool, + timeout: Duration, + ) -> Result<(), String> { + self.wait_loop(describe, is_ready, timeout, false).await + } + + /// The shared loop. `drain_on_exit` is set only by the positive form + /// ([`wait_for_screen`](Self::wait_for_screen)), where a marker that + /// appeared as the child exited did genuinely appear; an absence or + /// cleared-cell wait must never be satisfied by a dead process's last frame. + async fn wait_loop( &mut self, describe: &str, mut is_ready: impl FnMut(&str) -> bool, timeout: Duration, + drain_on_exit: bool, ) -> Result<(), String> { let deadline = Instant::now() + timeout; loop { @@ -823,9 +837,10 @@ impl TuiSession { if let Ok(Some(status)) = self.child.try_wait() { self.finished = true; self.record_once(i32::try_from(status.exit_code()).unwrap_or(-1)); - if self - .drain_final_frame_where(Some(describe), &mut is_ready) - .await? + if drain_on_exit + && self + .drain_final_frame_where(Some(describe), &mut is_ready) + .await? { return Ok(()); } diff --git a/tests/e2e-cucumber/tests/e2e/update_steps.rs b/tests/e2e-cucumber/tests/e2e/update_steps.rs index 09f9e4578b..f1e4fb51e7 100644 --- a/tests/e2e-cucumber/tests/e2e/update_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/update_steps.rs @@ -14,7 +14,6 @@ //! update-07 and update-08 do plant managed runtimes, and serve the TheRock //! catalog they resolve against from a loopback server, so they stay off the //! network too. -//! binary (EAI-8072). Mock lane. //! //! The flag-handling steps below are argument handling only: nothing there //! contacts a package index or changes the machine either, so they run on every diff --git a/xtask/src/ascii_output_contract.rs b/xtask/src/ascii_output_contract.rs index 31bff5641e..3d66eb73ea 100644 --- a/xtask/src/ascii_output_contract.rs +++ b/xtask/src/ascii_output_contract.rs @@ -243,8 +243,9 @@ mod tests { /// /// A whole-tree walk rather than a list of source directories, so a crate /// added under a new top-level directory is covered without anyone - /// remembering to extend this. `target*/` (`target-container/` is the - /// container gate's build dir) and dot-directories are skipped: + /// remembering to extend this. `target/`, `target-*/` (alternate cargo + /// target dirs, e.g. a `CARGO_TARGET_DIR=target-foo` build) and dot-directories + /// are skipped: /// the latter include the `.claude/worktrees` checkouts of other /// branches, and scanning those would make the verdict depend on state /// outside the commit under test. @@ -270,7 +271,7 @@ mod tests { let skip = path .file_name() .and_then(|name| name.to_str()) - .is_none_or(|name| name.starts_with("target") || name.starts_with('.')); + .is_none_or(|name| name == "target" || name.starts_with("target-") || name.starts_with('.')); if skip { continue; } From 7ce66d5bec912f4adfe3992a2fc68457e18302e9 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 27 Oct 2026 13:12:59 +0100 Subject: [PATCH 56/62] style(xtask): rustfmt the ascii-contract directory skip Signed-off-by: fredespi --- xtask/src/ascii_output_contract.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xtask/src/ascii_output_contract.rs b/xtask/src/ascii_output_contract.rs index 3d66eb73ea..5cef099a6c 100644 --- a/xtask/src/ascii_output_contract.rs +++ b/xtask/src/ascii_output_contract.rs @@ -271,7 +271,9 @@ mod tests { let skip = path .file_name() .and_then(|name| name.to_str()) - .is_none_or(|name| name == "target" || name.starts_with("target-") || name.starts_with('.')); + .is_none_or(|name| { + name == "target" || name.starts_with("target-") || name.starts_with('.') + }); if skip { continue; } From ceb3fb12869ca3d2a7e3db0ab73cef7c6cf50bc7 Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 30 Oct 2026 10:25:08 +0100 Subject: [PATCH 57/62] test(e2e): tighten wording guard and label test, fix doc drift from review Scope the diagnostic wording guard to wait_loop and drain_panic_message so the sibling popup wait cannot vouch for a reworded template, extract and unit-test the @id tag scan, assert the report label literals, and correct the driver doc, README and citation text. Signed-off-by: fredespi --- tests/e2e-cucumber/README.md | 11 ++-- tests/e2e-cucumber/src/diagnostic_wording.rs | 20 ++++++- tests/e2e-cucumber/src/expectation.rs | 58 ++++++++++--------- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 3 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- tests/e2e-cucumber/tests/e2e/tui_driver.rs | 19 +++--- 6 files changed, 69 insertions(+), 44 deletions(-) diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index e5c00b0d04..86a39dd291 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -108,10 +108,11 @@ harness resolves every scenario to **pass / xfail / skip** at runtime from its capability tags plus the known-bug matrix, then reconciles the actual result against that expectation. -Three tags tier the suite at runtime rather than by a `-t` filter, so a scenario +Four tags tier the suite at runtime rather than by a `-t` filter, so a scenario carrying one runs only where its env var is set: `@nightly` -(`E2E_INCLUDE_NIGHTLY`), `@lifecycle` (`E2E_INCLUDE_LIFECYCLE`) and -`@merge-queue` (`E2E_MERGE_QUEUE`, set on the merge-queue lanes). Note what that +(`E2E_INCLUDE_NIGHTLY`), `@lifecycle` (`E2E_INCLUDE_LIFECYCLE`), +`@requires-docker` (`E2E_INCLUDE_DOCKER`) and `@merge-queue` (`E2E_MERGE_QUEUE`, +set on the self-hosted merge-queue lanes). Note what that means for gating: a `@merge-queue` scenario does not run on ordinary per-PR CI at all, and the lanes that do run it — the self-hosted ones in [`.github/workflows/e2e-selfhosted.yml`](../../.github/workflows/e2e-selfhosted.yml), not `ci.yml` — are non-blocking. Branch @@ -156,7 +157,7 @@ Scenarios carry stable-id and capability tags: | `@requires-os:` | Premise is OS-specific; skip on other OSes. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@requires-no-gpu` | Premise is a host with no usable AMD GPU (e.g. a refusal that only happens without one). The inverse of `@requires-gpu`; resolves to **skip** on a GPU host. | -| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which the self-hosted lanes in [`e2e-selfhosted.yml`](../../.github/workflows/e2e-selfhosted.yml) do on a `merge_group` event. Those lanes are non-blocking (that file's header says why), so such a scenario is telemetry rather than a gate. | +| `@merge-queue` | Too expensive for per-PR CI (a real serve). Skipped unless `E2E_MERGE_QUEUE` is set, which every self-hosted lane in [`e2e-selfhosted.yml`](../../.github/workflows/e2e-selfhosted.yml) except the MI300X one does on a `merge_group` event. Those lanes are non-blocking (that file's header says why), so such a scenario is telemetry rather than a gate. | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios. | @@ -171,7 +172,7 @@ for the condition grammar. When a comment here points at product code, **cite a symbol, never a line number** — `select_runtime_update_source` in [`apps/rocm/src/main.rs`](../../apps/rocm/src/main.rs), not -`main.rs:16427`. `main.rs` alone is over twenty thousand lines and +`main.rs:16427`. `main.rs` alone is many thousands of lines and moves every week; nothing checks these numbers, so a stale one sends the reader somewhere confidently wrong, which is worse than no pointer at all. Most of the citations written during this suite's review had already drifted — one by five diff --git a/tests/e2e-cucumber/src/diagnostic_wording.rs b/tests/e2e-cucumber/src/diagnostic_wording.rs index fc437321b9..26e2363d7e 100644 --- a/tests/e2e-cucumber/src/diagnostic_wording.rs +++ b/tests/e2e-cucumber/src/diagnostic_wording.rs @@ -179,9 +179,23 @@ fn code_text(source: &str) -> String { .join("\n") } -fn assert_both_sides(source: &str, documented: &[Documented], owner: &str) { +/// `code_owners` names the functions that build the messages. The templates are +/// searched in their bodies only: `wait_for_popup_body_where` repeats three of +/// them verbatim, so a file-wide search stays green when the real one is reworded. +fn assert_both_sides( + source: &str, + documented: &[Documented], + owner: &str, + code_owners: &[&str], +) { let docs = doc_block_of(source, owner); - let code = code_text(source); + let code = code_text( + &code_owners + .iter() + .map(|name| body_of(source, name)) + .collect::>() + .join("\n"), + ); for Documented { templates, documented_as, @@ -246,6 +260,7 @@ fn every_clause_diagnostic_is_both_emitted_and_documented() { &read(DRIVER), &clause_diagnostics(), "wait_for_screen_where", + &["wait_loop", "drain_panic_message"], ); } @@ -256,6 +271,7 @@ fn every_marker_diagnostic_is_both_emitted_and_documented() { &read(DRIVER), &marker_diagnostics(), "terminal_state_after_wait", + &["terminal_state_after_wait"], ); } diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 5a7d350ad5..5d4d704445 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1989,29 +1989,41 @@ flaky = true /// that nothing measures. Nothing at runtime notices — resolution is keyed /// FROM the scenario TO this file, never the other way — so the orphan is /// caught here instead. + /// The `@id:` tags declared on tag lines. Tag lines only, the way + /// `scenarios_of` in tests/feature_naming.rs reads them: this set decides + /// whether a row is an orphan, so an `@id:` written inside a `#` comment + /// must not vouch for a row that measures nothing. + fn tag_ids(text: &str) -> std::collections::BTreeSet { + let mut ids = std::collections::BTreeSet::new(); + for line in text.lines() { + let line = line.trim(); + if !line.starts_with('@') { + continue; + } + for tag in line.split_whitespace() { + if let Some(id) = tag.strip_prefix("@id:") { + ids.insert(id.to_owned()); + } + } + } + ids + } + + #[test] + fn a_commented_out_id_does_not_declare_a_scenario() { + let text = " # @id:ghost\n @requires-os:linux @id:real\n Scenario: x\n"; + let ids = tag_ids(text); + assert!(ids.contains("real"), "{ids:?}"); + assert!(!ids.contains("ghost"), "{ids:?}"); + } + #[test] fn every_expectation_row_names_a_scenario_that_exists() { let mut scenario_ids = std::collections::BTreeSet::new(); for path in feature_files() { let text = std::fs::read_to_string(&path) .unwrap_or_else(|e| panic!("could not read {}: {e}", path.display())); - // Read tags off TAG LINES only, the way `scenarios_of` in - // tests/feature_naming.rs does. Scanning the whole file would also - // pick up an `@id:` written inside a `#` comment — and since this - // set is what decides whether a row is an orphan, a commented-out - // id would vouch for a row that measures nothing, which is the one - // thing this test exists to catch. - for line in text.lines() { - let line = line.trim(); - if !line.starts_with('@') { - continue; - } - for tag in line.split_whitespace() { - if let Some(id) = tag.strip_prefix("@id:") { - scenario_ids.insert(id.to_owned()); - } - } - } + scenario_ids.extend(tag_ids(&text)); } assert!( !scenario_ids.is_empty(), @@ -2441,15 +2453,7 @@ flaky = true assert_eq!(xfail.bug.as_deref(), Some("EAI-1")); assert_eq!(xfail.reason.as_deref(), Some("r")); assert!(xfail.flaky); - assert_eq!( - xfail.expected, - Expectation::ExpectXfail { - bug: "EAI-1".into(), - reason: "r".into(), - flaky: true, - } - .label() - ); + assert_eq!(xfail.expected, "xfail"); assert_eq!(xfail.id, "x"); assert_eq!(xfail.feature, "f"); assert_eq!(xfail.scenario, "s"); @@ -2466,11 +2470,13 @@ flaky = true reason: "no gpu".into(), }, ); + assert_eq!(skipped.expected, "skip"); assert_eq!(skipped.bug, None); assert_eq!(skipped.reason.as_deref(), Some("no gpu")); assert!(!skipped.flaky); let passing = ResolvedScenario::new("x", "f", "s", "vllm", &Expectation::ExpectPass); + assert_eq!(passing.expected, "pass"); assert_eq!(passing.bug, None); assert_eq!(passing.reason, None); assert!(!passing.flaky); diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index ca57025116..fe26d6e849 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -1654,7 +1654,8 @@ async fn assert_interactive_decline_reported(world: &mut E2eWorld) { /// does — which is exactly why the two can drift apart. const DEVICE_PERMISSION_FIX_ID: &str = "fix-4-render-group"; -/// A symptom that makes the device-permission cause score on any host, so the +/// A symptom that makes the device-permission cause score on any bare-metal +/// Linux host (the check is not registered on WSL or Windows), so the /// comparison never depends on what happens to be wrong with the runner. const DEVICE_PERMISSION_SYMPTOM: &str = "unable to open /dev/kfd"; diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index b9b352648a..dc7a5a0b29 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1484,7 +1484,7 @@ async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { let output = serve_output(world); // The `port:` line read here is the one `serve` prints in its "serve plan" // block — the `println!` under `serve plan` in `serve` itself - // (`apps/rocm/src/main.rs`) — which is what this scenario ran. + // (`apps/rocm/src/serve_cmd.rs`) — which is what this scenario ran. // // Named because grepping the product for `port: {port}` finds a decoy first: // `run_chat_port_status_tool` writes an identical line for a different diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 565574322d..438d75b065 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -732,7 +732,8 @@ impl TuiSession { /// This only gates on the action you're waiting *for* if `marker` is /// genuinely absent before it — the marker is tested before the first /// sleep, so a call whose marker is already on screen returns - /// immediately and proves nothing about what happens afterward. A step + /// immediately (while the child is alive and the reader has not panicked) + /// and proves nothing about what happens afterward. A step /// that re-waits on a marker a prior step already waited for is a silent /// no-op, not a barrier. pub async fn wait_for_screen(&mut self, marker: &str, timeout: Duration) -> Result<(), String> { @@ -785,8 +786,8 @@ impl TuiSession { /// /// `src/diagnostic_wording.rs` pins this list against the templates below. /// - /// Unlike `wait_for_screen`, the liveness checks run *before* the predicate, - /// and deliberately so. The order is only observable on a poll where one of + /// The liveness checks run *before* the predicate, in this form and in + /// `wait_for_screen` alike, and deliberately so. The order is only observable on a poll where one of /// them fires — the child has exited, or the reader thread has panicked — /// and there that check returns before the predicate is ever consulted, so /// the only outcome the ordering can change is a would-be success into a @@ -800,12 +801,12 @@ impl TuiSession { /// The cost falls on predicates that require something to be *present* /// (a cell that must still be rendered, only with a different value): if /// such a condition first holds in the frame the child left behind, this - /// reports the exit instead. The positive form can afford the opposite - /// order — and drains the final frame after `try_wait`, via - /// [`drain_final_frame`](Self::drain_final_frame) — because a marker - /// that appeared as the child exited did genuinely appear. Here that - /// recovery is given up on purpose: a diagnosed exit is worth more than a - /// condition that only ever held in a dying process's last frame. + /// reports the exit instead. Only the positive form + /// ([`wait_for_screen`](Self::wait_for_screen)) drains the final frame after + /// `try_wait` — a marker that appeared as the child exited did genuinely + /// appear. Here that recovery is given up on purpose: a diagnosed exit is + /// worth more than a condition that only ever held in a dying process's + /// last frame. pub async fn wait_for_screen_where( &mut self, describe: &str, From 090539f83c55777f492c20a6577db2aa0d01159a Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 30 Oct 2026 12:31:27 +0100 Subject: [PATCH 58/62] test(e2e): check the stop really stopped, frame uninstall as a proposed contract serve-29 now also asserts the stopped server stops answering. serve-30's row and the expectations header say it is a proposal the README does not make. Also rustfmt the wording guard and trim history narration from two comments. Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 26 ++++++++++++------- .../features/model_serving.feature | 12 ++++++--- tests/e2e-cucumber/src/diagnostic_wording.rs | 7 +---- tests/e2e-cucumber/tests/e2e.rs | 13 ++++------ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 20 ++++++++++++++ tests/e2e-cucumber/tests/e2e/tui_driver.rs | 2 ++ 6 files changed, 52 insertions(+), 28 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 66487cbfc4..5d70509aa6 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -167,11 +167,14 @@ flaky = true # ── Contracts found by walking the public README end to end ───────────────── # -# The rows below were all filed from one pass through README.md against a -# nightly build on an MI300X pod (EAI-8024 tracks the set). Each scenario asserts -# the behaviour the README leads a reader to expect, and each row here says the -# product does not do that yet — so the day one is fixed its row goes stale and -# deletes itself. +# The rows below came out of one walkthrough of the README against a nightly build +# on an MI300X pod (EAI-8024 tracks the set). Most scenarios assert behaviour the +# walkthrough leads a reader to expect; a few are contracts the walkthrough implies +# rather than states (diagnose and fix agreeing, a usable device not being +# diagnosed as a permission failure, a second server avoiding a taken address), and +# one is a proposal the README does not make (uninstall stopping what it manages, +# below). Each row says the product does not do that yet — so the day one is fixed +# its row goes stale and deletes itself. # --- EAI-8020: `diagnose` and `fix` each carry their own copy of the # render-group remedy, and the verification step has drifted between them @@ -222,11 +225,14 @@ flaky = true # would be a stale XPASS on both lanes and would fail them. The ticket keeps the # pod observation; what distinguishes that host is still unknown. -# --- EAI-8014: `uninstall` removes the managed files and reports completion -# without stopping the servers it manages — and takes the service records with -# it, so the supported way to stop them goes too. It warns that it does not stop -# them, which is the behaviour being disputed, not a defence of it. No hardware -# needed. Remove when uninstall stops what it manages first. --- +# --- EAI-8014: a PROPOSED contract, not a README one. The README documents that +# `uninstall` warns it does not stop managed background processes; this row pins +# the stronger behaviour the ticket asks for — `uninstall` removes the managed +# files and reports completion without stopping the servers it manages, and takes +# the service records with it, so the supported way to stop them goes too. The +# warning is documented behaviour, so this stays a row only until the product and +# README change together. No hardware needed. Remove when uninstall stops what it +# manages first, or when the contract is dropped. --- [["serve-uninstall-stops-what-it-manages"]] when = {} bug = "EAI-8014" diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 09593d670b..68af1d8303 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -412,10 +412,14 @@ Feature: Model serving And a model is being served on GPU When the user stops the server that is running Then the CLI reports that it stopped a process - - # Expected to FAIL. Removing the CLI's managed files reports completion while - # the server it was managing is left running — and with the records gone, the - # supported way to stop it has been removed along with them. Nothing here + And the stopped server no longer answers + + # Expected to FAIL. A PROPOSED contract, not one the README promises: the + # README says `uninstall` warns that it does not stop managed background + # processes. This scenario asserts the stronger behaviour — that removing the + # CLI's managed files reports completion while the server it was managing is + # left running, and with the records gone, the supported way to stop it has + # been removed along with them. Nothing here # touches the installed program: the removal is scoped to this scenario's own # directories and keeps the binaries. @id:serve-uninstall-stops-what-it-manages @requires-os:linux diff --git a/tests/e2e-cucumber/src/diagnostic_wording.rs b/tests/e2e-cucumber/src/diagnostic_wording.rs index 26e2363d7e..22e90c2526 100644 --- a/tests/e2e-cucumber/src/diagnostic_wording.rs +++ b/tests/e2e-cucumber/src/diagnostic_wording.rs @@ -182,12 +182,7 @@ fn code_text(source: &str) -> String { /// `code_owners` names the functions that build the messages. The templates are /// searched in their bodies only: `wait_for_popup_body_where` repeats three of /// them verbatim, so a file-wide search stays green when the real one is reworded. -fn assert_both_sides( - source: &str, - documented: &[Documented], - owner: &str, - code_owners: &[&str], -) { +fn assert_both_sides(source: &str, documented: &[Documented], owner: &str, code_owners: &[&str]) { let docs = doc_block_of(source, owner); let code = code_text( &code_owners diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 03834bb337..f8ad8305bc 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1562,14 +1562,11 @@ async fn main() { // deliberately excluded: uv's cache is content-addressed and uv does its // own locking. A lane that races becomes serialized-and-slower instead. // - // The mock lane's 64 is deliberately left as main has it. Bounding it to the - // core count was tried on this branch and reverted: dash-08/dash-09 measure a - // validity window whose elapsed time advances with DAEMON CYCLES (the logical - // clock added in #380), so a less-loaded lane burns that window faster and - // clears the held value before the assertion looks. Both scenarios pass on - // main at 64-way and failed here at 2..=6 — a suite-wide scheduling knob - // changing two unrelated scenarios' verdicts is reason enough for a - // contract-pinning change not to touch it. + // The mock lane stays at 64: dash-08/dash-09 measure a validity window whose + // elapsed time advances with DAEMON CYCLES (the logical clock added in #380), + // so a less-loaded lane burns that window faster and clears the held value + // before the assertion looks. Both pass at 64-way and fail at 2..=6, so a + // tighter cap here would change two unrelated scenarios' verdicts. let max_concurrent = if cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some() { 1 diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index dc7a5a0b29..9d5eba2514 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1373,6 +1373,26 @@ async fn assert_owned_server_stopped(world: &mut E2eWorld) { ); } +#[then("the stopped server no longer answers")] +async fn assert_stopped_server_stops_answering(world: &mut E2eWorld) { + let endpoint = world.endpoint.as_ref().expect("no endpoint configured"); + let url = format!("{endpoint}/models"); + // A signalled server can take a moment to release its port, so the answer is + // given a bounded window to stop. A server that is still listening at the end + // of it was not stopped, whatever the CLI reported. + let deadline = Instant::now() + Duration::from_secs(60); + loop { + if reqwest::get(&url).await.is_err() { + return; + } + assert!( + Instant::now() < deadline, + "the CLI reported the stop, but {url} still answers a minute later" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + #[then("the CLI reports that it stopped a process")] async fn assert_stop_count_reported(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no stop output"); diff --git a/tests/e2e-cucumber/tests/e2e/tui_driver.rs b/tests/e2e-cucumber/tests/e2e/tui_driver.rs index 438d75b065..226b88df89 100644 --- a/tests/e2e-cucumber/tests/e2e/tui_driver.rs +++ b/tests/e2e-cucumber/tests/e2e/tui_driver.rs @@ -780,6 +780,8 @@ impl TuiSession { /// `timed out … waiting until {describe}`, `before {describe}.`, and `… /// draining the final frame, waiting until {describe}`. The last is /// assembled from two fragments in two functions, and both carry the clause. + /// It is reached only through [`wait_for_screen`](Self::wait_for_screen), + /// the one form that drains on exit; this form reaches the other three. /// So it must fit all four ("the screen shows X", "generation throughput /// leaves the screen"), not be a bare noun; it carries whatever quoting the /// caller puts in it, since none is added here. From 4002c3e46372624171d9d135055c0cfd437f50eb Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 30 Oct 2026 13:47:56 +0100 Subject: [PATCH 59/62] test(e2e): use from_mins for the stop-check deadline (clippy) Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 9d5eba2514..607d2a8ba3 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1380,7 +1380,7 @@ async fn assert_stopped_server_stops_answering(world: &mut E2eWorld) { // A signalled server can take a moment to release its port, so the answer is // given a bounded window to stop. A server that is still listening at the end // of it was not stopped, whatever the CLI reported. - let deadline = Instant::now() + Duration::from_secs(60); + let deadline = Instant::now() + Duration::from_mins(1); loop { if reqwest::get(&url).await.is_err() { return; From 3012275c252f6edca69bdb4aab7c12b51b6d86f3 Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 2 Nov 2026 17:33:45 +0100 Subject: [PATCH 60/62] e2e: tighten the README-contract steps per review Stop the owned server only when the scenario has an isolated root, require the stopped endpoint to have answered first, require conflict wording for a port clash, match --device exactly, select service records by manifest name, and drop an unrelated comment. Co-Authored-By: Claude Signed-off-by: fredespi --- tests/e2e-cucumber/tests/e2e.rs | 13 ++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 58 ++++++++++++++----- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index d0da573c3a..09b35bca2d 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -243,13 +243,13 @@ impl OwnedProcess { /// Polls briefly: the product signals the process and this only has to /// outlast the moment between the signal landing and the kernel finishing /// with it, not any grace period the product itself waits out. - pub fn is_running(&mut self) -> bool { + pub async fn is_running(&mut self) -> bool { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); loop { match self.child.try_wait() { Ok(Some(_)) => return false, Ok(None) if std::time::Instant::now() < deadline => { - std::thread::sleep(std::time::Duration::from_millis(100)); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } Ok(None) => return true, // Not treated as "stopped". The only caller asserts that the @@ -834,8 +834,8 @@ impl Drop for E2eWorld { // failure is ignored rather than panicking (which would abort the run) — // hence the returned status is discarded here. The serve retry, where a // failed stop changes the next attempt's meaning, does read it. - let env = self.isolate_env(); if let Some(root) = &self.isolated_root { + let env = self.isolate_env(); stop_managed_services(root.path(), &env); } // A scenario that made the audit directory read-only must give it back @@ -1581,12 +1581,7 @@ async fn main() { // themselves rather than only from the probe. `shared_uv_cache_dir()` is // deliberately excluded: uv's cache is content-addressed and uv does its // own locking. A lane that races becomes serialized-and-slower instead. - // - // The mock lane stays at 64: dash-08/dash-09 measure a validity window whose - // elapsed time advances with DAEMON CYCLES (the logical clock added in #380), - // so a less-loaded lane burns that window faster and clears the held value - // before the assertion looks. Both pass at 64-way and fail at 2..=6, so a - // tighter cap here would change two unrelated scenarios' verdicts. + let max_concurrent = if cap.has_amd_gpu || shared_cache_dir().is_some() || shared_runtimes_dir().is_some() { 1 diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 607d2a8ba3..376f69401c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1113,11 +1113,20 @@ async fn user_serves_absent_gpu_index(world: &mut E2eWorld) { /// nothing" — it silently removes the very list the scenario exists to check. /// The caller's guard turns that into a loud failure, but recognising both /// shapes is what keeps the scenario measuring the contract at all. +/// A help line that starts a new option: `--long` or `-x, --long`. +fn is_option_line(line: &str) -> bool { + let line = line.trim_start(); + line.starts_with("--") + || (line.len() > 1 && line.starts_with('-') && line.as_bytes()[1].is_ascii_alphanumeric()) +} + fn advertised_device_policies(help: &str) -> Vec { let lines: Vec<&str> = help.lines().collect(); - let device = lines - .iter() - .position(|line| line.trim_start().starts_with("--device")); + let device = lines.iter().position(|line| { + line.trim_start() + .strip_prefix("--device") + .is_some_and(|rest| rest.starts_with([' ', '=', '<']) || rest.is_empty()) + }); let Some(device) = device else { return Vec::new(); }; @@ -1127,7 +1136,7 @@ fn advertised_device_policies(help: &str) -> Vec { .iter() .skip(device) .enumerate() - .take_while(|(offset, line)| *offset == 0 || !line.trim_start().starts_with("--")) + .take_while(|(offset, line)| *offset == 0 || !is_option_line(line)) .map(|(_, line)| *line) .collect(); @@ -1288,13 +1297,27 @@ async fn user_stops_running_server(world: &mut E2eWorld) { .into_iter() .flatten() .flatten() - .filter(|entry| entry.path().extension().is_some_and(|e| e == "json")) .filter_map(|entry| { + // The manifest is `.json`; `.state.json` and other sidecars + // share the extension, so match the manifest name itself. + let name = entry.file_name().into_string().ok()?; + let id = name.strip_suffix(".json")?; + if id.contains('.') { + return None; + } let bytes = std::fs::read(entry.path()).ok()?; let record: serde_json::Value = serde_json::from_slice(&bytes).ok()?; - Some(record.get("service_id")?.as_str()?.to_owned()) + let recorded = record.get("service_id")?.as_str()?; + (recorded == id).then(|| id.to_owned()) }) .collect(); + // The server must answer before the stop, or the "no longer answers" check + // that follows could pass on a wrong URL. + let endpoint = world.endpoint.as_ref().expect("no endpoint configured"); + assert!( + model_is_ready(&format!("{endpoint}/models"), None, 30).await, + "{endpoint} does not answer before the stop, so a later silence would prove nothing" + ); // Exactly one, not the first `read_dir` happens to yield. Directory order is // unspecified, so with a second record present — a retried serve's leftover, // or a planted mock alongside a real one — taking the first would stop an @@ -1362,7 +1385,7 @@ async fn assert_owned_server_stopped(world: &mut E2eWorld) { // defect under test into a pass. let mut still_running = Vec::new(); for process in &mut world.owned_processes { - if process.is_running() { + if process.is_running().await { still_running.push(process.pid()); } } @@ -1533,16 +1556,21 @@ async fn assert_new_server_avoids_taken_address(world: &mut E2eWorld) { // address and exit nonzero, and would be accepted here as though // the CLI had honestly reported the conflict. Require it to say so. // - // Only these two. A bare "already" would admit the serve path's own - // unrelated uses of the word ("already running" for an equivalent - // managed service, "already being served" for a model), and a bare - // "address" would admit any sentence that happens to mention one. - // The port number or "in use" is what naming THIS conflict looks - // like. + // Conflict wording, not just the number: an unrelated failure can + // print an endpoint URL that contains the port. A bare "already" + // would admit the serve path's own unrelated uses of the word + // ("already running", "already being served"). let said = output.to_lowercase(); - let port = SERVE_PORT.to_string(); assert!( - said.contains(&port) || said.contains("in use"), + [ + "in use", + "address already", + "port already", + "already bound", + "unavailable" + ] + .iter() + .any(|phrase| said.contains(phrase)), "the CLI refused, but without naming the address that is taken — so this \ says nothing about whether it noticed the conflict:\n{output}" ); From d180cba902bf6afa045bf4d915315a0aa6df4733 Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 3 Nov 2026 17:27:58 +0100 Subject: [PATCH 61/62] e2e: cite the runs behind the measured rows, and cite stat_device by symbol Replace the MEASURE markers on the EAI-8016, EAI-8020 and EAI-8021 rows with the lanes and runs that measured them, say that no blocking lane holds EAI-8007, and drop a source path the examine split may have moved. Co-Authored-By: Claude Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 29 ++++++++++++++++++---------- tests/e2e-cucumber/tests/e2e.rs | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 5d70509aa6..31e26917a8 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -219,10 +219,12 @@ flaky = true # For the contract this scenario actually exists to hold: # EAI-8007 has NO row on purpose. `services stop` reported stopping 0 processes # for a service it did in fact stop on the pod, but neither fixture reproduces it -# here: a plain child process registered as a managed service is counted -# correctly (no-GPU lane), and so is a real vLLM serve (MI300X, run 188). Its -# scenario therefore ships as a guard rather than an expected failure — a row -# would be a stale XPASS on both lanes and would fail them. The ticket keeps the +# here: a plain child process registered as a managed service was counted +# correctly in a one-off no-GPU check during development, and so was a real vLLM +# serve (MI300X, run 188). The shipped scenario is the real-serve one, which is +# `@requires-gpu @merge-queue`, so no blocking lane holds this contract. It ships +# as a guard rather than an expected failure, because a row would be a stale +# XPASS and would fail the lane. The ticket keeps the # pod observation; what distinguishes that host is still unknown. # --- EAI-8014: a PROPOSED contract, not a README one. The README documents that @@ -238,16 +240,19 @@ when = {} bug = "EAI-8014" reason = "Uninstall reports completion while leaving the servers it manages running, and removes the records needed to stop them." -# --- EAI-8020: MEASURE. The other half of the same divergence: `diagnose` names +# --- EAI-8020. The other half of the same divergence: `diagnose` names # the device's actual owning group while the `fix` recipe is hard-coded to # `render`, so the two propose different changes wherever the device's group is -# not `render`. Needs a real device node, hence the GPU lane. --- +# not `render`. Needs a real device node, hence the GPU lane. Measured on the +# MI300X lane (E2E self-hosted ROCm/rocm-cli run 192) and the Strix Halo Ubuntu lane +# (run 196), failing for this reason on both, with 0 XPASS. The group is substituted +# by a `stat` shim, so the outcome does not depend on the host. --- [["diagnose-and-fix-agree-on-the-remedy-command"]] when = {} bug = "EAI-8020" reason = "The diagnosis proposes adding the user to the device's own group while the fix is hard-coded to `render`." -# --- EAI-8021: MEASURE. The render-group check only ever ADDS score for missing +# --- EAI-8021. The render-group check only ever ADDS score for missing # group membership and never credits access that demonstrably works, so a device # the user can already read and write is still diagnosed as a permission problem. # The scenario substitutes a world-writable device answer and asserts the @@ -256,7 +261,8 @@ reason = "The diagnosis proposes adding the user to the device's own group while # ones), so the runner's own group membership cannot reach the result: the # outcome is the same on every bare-metal Linux GPU host. If this XPASSes, the # product changed; it is not a host difference, and NOT a reason to add -# `flaky = true`. --- +# `flaky = true`. Measured on MI300X (run 192) and Strix Halo Ubuntu (run 196), +# failing for this reason on both, with 0 XPASS. --- [["diagnose-credits-a-usable-device"]] when = {} bug = "EAI-8021" @@ -283,10 +289,13 @@ bug = "EAI-8006" reason = "Letting the CLI choose a GPU reports that it selected none on a Windows machine that has one." serve_timeout_secs = 180 -# --- EAI-8016: MEASURE. The serve path has no address-availability check: the +# --- EAI-8016. The serve path has no address-availability check: the # default port is used whether or not something already holds it, so a second # server is handed the address the first is on. The scenario holds the address -# itself rather than starting a second real server. --- +# itself rather than starting a second real server. Measured on all three GPU +# lanes before the scenario became `@merge-queue`: MI300X (run 192), Strix Halo +# Ubuntu (run 196) and Strix Halo Windows (runs 196 and 210). It failed for this +# reason on each, Windows included, with 0 XPASS. --- [["serve-second-server-gets-a-free-port"]] when = {} bug = "EAI-8016" diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 58661d7ac3..e08a662652 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1191,7 +1191,7 @@ pub fn stat_shim_path( .find(|candidate| std::path::Path::new(candidate).is_file()) .expect("no real `stat` binary to delegate to"); // Answers only the one call this scenario is substituting: the CLI's - // `stat -c %A|%U|%G ` in `stat_device` (crates/rocm-core/src/examine.rs). + // `stat -c %A|%U|%G ` in `rocm_core`'s `stat_device`. // Both the format AND the path must match — keying on the path alone would // hand this `mode|owner|group` line to any other `stat` of the same device // added later, in whatever format it asked for, and the caller would parse From 0703a1a1409b2ae49393e517e4ab588c5ea2ecbe Mon Sep 17 00:00:00 2001 From: fredespi Date: Thu, 5 Nov 2026 09:58:47 +0100 Subject: [PATCH 62/62] style: end expectations.toml with a single newline Co-Authored-By: Claude Signed-off-by: fredespi --- tests/e2e-cucumber/expectations.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 7478bbf652..47d3445ca3 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -311,4 +311,3 @@ serve_timeout_secs = 120 # scenario would pass vacuously — worse than no scenario, because it would read # as coverage. They were reproduced by hand on the pod (ubuntu 24.04) and stay # on their tickets until a lane can run Lemonade. -