From 8f4b529a30942c3afe099417da77247e32f3699d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 07:31:38 +0000 Subject: [PATCH 01/13] test(e2e): judge the serve drain-wait by the pool the engine allocates from On an APU the VRAM total is only the BIOS carveout (512 MiB on the Strix Halo lane hosts) while the engine allocates from GTT (about 100 GiB), so the 90% floor taken from the carveout could never be met and every GPU serve waited out the full two-minute drain deadline. Choose GTT when the VRAM total is under the 8 GiB carveout limit and a non-empty GTT reading exists, as gpu-preflight.sh already does, and keep VRAM everywhere else. The choice is a pure function with unit tests built from the preflight log's real readings. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 274 ++++++++++++++++++ tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/tests/e2e/serving_steps.rs | 136 +++++---- 3 files changed, 350 insertions(+), 61 deletions(-) create mode 100644 tests/e2e-cucumber/src/device_memory.rs diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs new file mode 100644 index 0000000000..46b97935e1 --- /dev/null +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -0,0 +1,274 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Which device memory pool the serve drain-wait judges, and the floor it +//! demands of it. +//! +//! The wait between serve scenarios (`wait_for_free_vram` in the serving +//! steps) exists to see a just-killed engine hand its device memory back. It +//! has to look at the pool the engine allocates from. On a discrete card that +//! is VRAM. On an APU such as Strix Halo the VRAM total is only the fixed BIOS +//! carveout (512 MiB on the CI hosts) while the engine draws on GTT-backed +//! system memory (about 100 GiB there), so a floor taken from the carveout can +//! never be met and every GPU serve spent its whole drain deadline waiting. +//! `.github/scripts/gpu-preflight.sh` makes the same switch for the same reason. + +/// Upper bound on the free-memory floor (MiB). +/// +/// Sized so the largest single +/// scenario model can allocate its engine's memory share without tripping its +/// startup check: Qwen3.6-27B plus vLLM's default fraction-of-total KV +/// reservation needs the MI300X mostly clear. Only a data-center GPU has this +/// much, so on smaller pools the floor is capped to 90% of the pool total (see +/// [`required_free_mib`]) — otherwise the check could never pass. Note the cap +/// no longer sits above what vLLM asks for: rocm-cli used to pin utilization +/// below it, and now defers to vLLM's own (higher) default, so the wait has +/// little headroom left and can time out on a device that is merely holding +/// display memory. The wait is best-effort and the serve proceeds regardless, +/// so this costs time rather than correctness; sizing a deliberate margin is +/// follow-up work. +pub const MAX_FREE_FLOOR_MIB: u64 = 150_000; + +/// A VRAM total below this is an APU's BIOS carveout, not private device memory. +/// +/// The engine's allocations come out of GTT instead. It is the +/// 8 GiB floor the Strix Halo preflight lanes pass as +/// `GPU_PREFLIGHT_MIN_FREE_GIB`, which `gpu-preflight.sh` compares the VRAM +/// total against for the same purpose. Every discrete card the lanes run on is +/// far above it (48 GiB and up). +pub const CARVEOUT_MAX_MIB: u64 = 8 * 1024; + +/// One memory pool's size and free space, in MiB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Pool { + pub total_mib: u64, + pub free_mib: u64, +} + +/// What the device tooling reported for GPU 0. `gtt` is `None` when the tool +/// that answered has no GTT figure (or none was readable). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeviceMemory { + pub vram: Pool, + pub gtt: Option, +} + +/// Which pool a [`DrainTarget`] was read from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolKind { + Vram, + Gtt, +} + +/// The pool to wait on, its current state, and the floor it must reach. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DrainTarget { + pub kind: PoolKind, + pub total_mib: u64, + pub free_mib: u64, + pub floor_mib: u64, +} + +impl DrainTarget { + /// Whether the pool has handed back enough memory. + #[must_use] + pub const fn drained(&self) -> bool { + self.free_mib >= self.floor_mib + } +} + +/// The free-memory floor to wait for on a pool of `total_mib`: the +/// model-sized ceiling, but never more than 90% of the pool. +#[must_use] +pub fn required_free_mib(total_mib: u64) -> u64 { + MAX_FREE_FLOOR_MIB.min(total_mib / 100 * 90) +} + +/// Choose the pool the engine allocates from and the floor to hold it to. +/// +/// A VRAM total under [`CARVEOUT_MAX_MIB`] is a carveout, so the GTT pool is +/// used when one was read and is non-empty. A zero GTT total is a failed +/// reading, not a pool, and falls back to VRAM, as it does in +/// `gpu-preflight.sh`. Without GTT data the answer is VRAM, which is what +/// every host got before this choice existed. +#[must_use] +pub fn select_drain_target(memory: DeviceMemory) -> DrainTarget { + let (kind, pool) = match memory.gtt { + Some(gtt) if memory.vram.total_mib < CARVEOUT_MAX_MIB && gtt.total_mib > 0 => { + (PoolKind::Gtt, gtt) + } + _ => (PoolKind::Vram, memory.vram), + }; + DrainTarget { + kind, + total_mib: pool.total_mib, + free_mib: pool.free_mib, + floor_mib: required_free_mib(pool.total_mib), + } +} + +/// Read `(total, free)` in MiB from `rocm-smi --showmeminfo --csv` output. +/// +/// `total_column` and `used_column` are substrings of the two header +/// names (`Total Memory` and `Total Used Memory` for VRAM; the `GTT ` forms for +/// GTT); the first data row is the first GPU. Values are bytes in the output. +#[must_use] +pub fn parse_rocm_smi_csv_pool( + text: &str, + total_column: &str, + used_column: &str, +) -> Option<(u64, u64)> { + let mut lines = text.lines(); + let header = lines.next()?; + let cols: Vec<&str> = header.split(',').collect(); + let total_idx = cols.iter().position(|c| c.contains(total_column))?; + let used_idx = cols.iter().position(|c| c.contains(used_column))?; + let row = lines.next()?; + let vals: Vec<&str> = row.split(',').collect(); + let total: u64 = vals.get(total_idx)?.trim().parse().ok()?; + let used: u64 = vals.get(used_idx)?.trim().parse().ok()?; + let mib = 1024 * 1024; + Some((total / mib, total.saturating_sub(used) / mib)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The pools `gpu-preflight.sh` printed on a Strix Halo lane host + /// (`VRAM 203321344/536870912, GTT 41787392/107374182400`, used/total in + /// bytes), and the same reading in MiB. + fn strix_halo() -> DeviceMemory { + DeviceMemory { + vram: Pool { + total_mib: 512, + free_mib: 318, + }, + gtt: Some(Pool { + total_mib: 102_400, + free_mib: 102_360, + }), + } + } + + fn mi300x() -> DeviceMemory { + DeviceMemory { + vram: Pool { + total_mib: 196_592, + free_mib: 196_309, + }, + gtt: Some(Pool { + total_mib: 1_000_000, + free_mib: 1_000_000, + }), + } + } + + #[test] + fn a_carveout_host_is_judged_by_gtt_and_a_healthy_strix_halo_counts_as_drained() { + let target = select_drain_target(strix_halo()); + assert_eq!(target.kind, PoolKind::Gtt); + assert_eq!(target.total_mib, 102_400); + assert_eq!(target.floor_mib, 92_160); + assert!( + target.drained(), + "an idle Strix Halo host must not wait out the drain deadline: {target:?}" + ); + } + + #[test] + fn a_carveout_host_whose_gtt_is_mostly_held_is_not_drained() { + let mut memory = strix_halo(); + memory.gtt = Some(Pool { + total_mib: 102_400, + free_mib: 60_000, + }); + let target = select_drain_target(memory); + assert_eq!(target.kind, PoolKind::Gtt); + assert!(!target.drained(), "{target:?}"); + } + + #[test] + fn a_discrete_card_is_still_judged_by_vram_with_the_model_sized_cap() { + let target = select_drain_target(mi300x()); + assert_eq!(target.kind, PoolKind::Vram); + assert_eq!(target.total_mib, 196_592); + assert_eq!(target.floor_mib, MAX_FREE_FLOOR_MIB); + assert!(target.drained()); + } + + #[test] + fn a_discrete_card_holding_most_of_its_vram_is_not_drained() { + let mut memory = mi300x(); + memory.vram.free_mib = 40_000; + assert!(!select_drain_target(memory).drained()); + } + + #[test] + fn a_host_without_gtt_data_keeps_the_vram_behaviour() { + let mut memory = strix_halo(); + memory.gtt = None; + let target = select_drain_target(memory); + assert_eq!(target.kind, PoolKind::Vram); + assert_eq!( + target.floor_mib, 450, + "the floor the CI log showed for the carveout" + ); + assert!(!target.drained()); + } + + #[test] + fn a_zero_gtt_total_is_a_failed_reading_not_a_pool() { + let mut memory = strix_halo(); + memory.gtt = Some(Pool { + total_mib: 0, + free_mib: 0, + }); + assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + } + + #[test] + fn a_vram_total_at_the_carveout_limit_is_private_memory() { + let mut memory = strix_halo(); + memory.vram = Pool { + total_mib: CARVEOUT_MAX_MIB, + free_mib: CARVEOUT_MAX_MIB, + }; + assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + } + + #[test] + fn rocm_smi_csv_gtt_is_read_in_mib() { + let text = "device,GTT Total Memory (B),GTT Total Used Memory (B)\n\ + card0,107374182400,41787392\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + Some((102_400, 102_360)) + ); + } + + #[test] + fn rocm_smi_csv_vram_keeps_its_original_column_match() { + let text = "device,VRAM Total Memory (B),VRAM Total Used Memory (B)\n\ + card0,536870912,203321344\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "Total Memory", "Total Used Memory"), + Some((512, 318)) + ); + } + + #[test] + fn rocm_smi_csv_without_the_pool_is_none() { + let text = "device,VRAM Total Memory (B),VRAM Total Used Memory (B)\n\ + card0,536870912,203321344\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + None + ); + assert_eq!( + parse_rocm_smi_csv_pool("", "Total Memory", "Total Used Memory"), + None + ); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 110023fe94..f2ef8e83eb 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -4,6 +4,7 @@ pub mod capability; pub mod chat_request; +pub mod device_memory; pub mod expectation; pub mod fixture_binary; pub mod harness_diagnostics; diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 49ab899ce3..b89cefc76a 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -11,6 +11,10 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; +use e2e_cucumber::device_memory::{ + CARVEOUT_MAX_MIB, DeviceMemory, DrainTarget, Pool, PoolKind, parse_rocm_smi_csv_pool, + select_drain_target, +}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, @@ -256,37 +260,15 @@ async fn ensure_serve_port_free() -> String { wait_for_free_vram().await } -/// Upper bound on the free-VRAM floor (MiB). Sized so the largest single -/// scenario model can allocate its engine's memory share without tripping its -/// startup check: Qwen3.6-27B plus vLLM's default fraction-of-total KV -/// reservation needs the MI300X mostly clear. Only a data-center GPU has this -/// much, so on smaller cards (e.g. Strix Halo's smaller unified-memory pool) the -/// floor is capped to 90% of the device total (see [`required_free_vram_mib`]) — -/// otherwise the check could never pass. Note the cap no longer sits above what -/// vLLM asks for: rocm-cli used to pin utilization below it, and now defers to -/// vLLM's own (higher) default, so the wait has little headroom left and can -/// time out on a device that is merely holding display memory. The wait is -/// best-effort and the serve proceeds regardless, so this costs time rather -/// than correctness; sizing a deliberate margin is follow-up work. -const MAX_FREE_VRAM_FLOOR_MIB: u64 = 150_000; - -/// The free-VRAM floor to wait for on this host: the model-sized ceiling, but -/// never more than 90% of the device's total VRAM. A hardcoded 150 GiB floor is -/// unreachable on a small card (e.g. Strix Halo's ~48 GiB unified VRAM), so -/// `wait_for_free_vram` would burn its full deadline on every serve scenario; -/// scaling to the device keeps the drain-check meaningful everywhere. -fn required_free_vram_mib(total_mib: u64) -> u64 { - MAX_FREE_VRAM_FLOOR_MIB.min(total_mib / 100 * 90) -} - /// How long to wait for a stopped engine to hand its device memory back. const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); -/// Best-effort: wait until the GPU reports enough free VRAM (see -/// [`required_free_vram_mib`]), so a just-killed serve's memory is actually -/// reclaimed before the next serve starts. Queries `amd-smi` then `rocm-smi`; -/// if neither is present (mock/local, no ROCm), returns immediately so non-GPU -/// runs are unaffected. +/// Best-effort: wait until the pool the engine allocates from reports enough +/// free memory (see [`select_drain_target`]: VRAM on a discrete card, GTT on an +/// APU whose VRAM is only a BIOS carveout), so a just-killed serve's memory is +/// actually reclaimed before the next serve starts. Queries `amd-smi` then +/// `rocm-smi`; if neither is present (mock/local, no ROCm), returns immediately +/// so non-GPU runs are unaffected. /// /// The wait is bounded and best-effort: on timeout the serve still starts, /// because a stale reading must not turn a slow drain into a hard failure. The @@ -295,25 +277,25 @@ const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); /// without it the failure looks identical to a genuinely broken serve. async fn wait_for_free_vram() -> String { // No GPU tooling → nothing to wait on (mock/local). Probe once up front. - let Some(total) = total_vram_mib() else { + if device_memory_mib().is_none() { return "device state: no GPU tooling (mock/local run)".to_owned(); - }; - let floor = required_free_vram_mib(total); + } let deadline = Instant::now() + VRAM_DRAIN_DEADLINE; loop { - let free = free_vram_mib(); - if let Some(free) = free - && free >= floor + let target = device_memory_mib().map(select_drain_target); + if let Some(target) = target + && target.drained() { - return format!( - "device state: drained ({free} MiB free of {total} MiB, floor {floor} MiB)" - ); + return format!("device state: drained ({})", describe_target(target)); } if Instant::now() >= deadline { - let free = free.map_or_else(|| "unreadable".to_owned(), |mib| format!("{mib} MiB")); + let held = target.map_or_else( + || "the memory reading was unreadable".to_owned(), + describe_target, + ); return format!( - "device state: NOT drained after {}s ({free} free of {total} MiB, floor \ - {floor} MiB) — a previous engine is still holding the GPU", + "device state: NOT drained after {}s ({held}) — a previous engine may still \ + be holding the GPU", VRAM_DRAIN_DEADLINE.as_secs() ); } @@ -321,16 +303,56 @@ async fn wait_for_free_vram() -> String { } } -/// Free device VRAM in MiB for GPU 0, via `amd-smi` (then `rocm-smi`). `None` -/// when no such tool exists or its output can't be parsed. -fn free_vram_mib() -> Option { - vram_mib().map(|(_total, free)| free) +/// `" MiB free of MiB[ GTT], floor MiB"`. VRAM is the +/// unlabelled default, so the line keeps the shape it always had on discrete +/// cards; only the GTT pool is named. +fn describe_target(target: DrainTarget) -> String { + let pool = match target.kind { + PoolKind::Vram => "", + PoolKind::Gtt => " GTT", + }; + format!( + "{} MiB free of {} MiB{pool}, floor {} MiB", + target.free_mib, target.total_mib, target.floor_mib + ) } -/// Total device VRAM in MiB for GPU 0. `None` when no GPU tool is available — -/// used as the "is there a GPU to wait on at all?" probe in mock/local runs. -fn total_vram_mib() -> Option { - vram_mib().map(|(total, _free)| total) +/// Device memory for GPU 0: VRAM via `amd-smi` (then `rocm-smi`), plus the GTT +/// pool from `rocm-smi`. `None` when no tool reports VRAM or its output can't +/// be parsed. GTT is read only when VRAM is small enough to be an APU's +/// carveout (see [`CARVEOUT_MAX_MIB`]), so a discrete-card host never makes the +/// extra call. `amd-smi` is not asked for GTT: its output has no field this +/// harness has been checked against, whereas the preflight step already reads +/// `rocm-smi`'s GTT rows on these hosts. +fn device_memory_mib() -> Option { + use std::process::Command; + let (vram_total, vram_free) = vram_mib()?; + let gtt = if vram_total < CARVEOUT_MAX_MIB { + Command::new("rocm-smi") + .args(["--showmeminfo", "gtt", "--csv"]) + .output() + .ok() + .filter(|out| out.status.success()) + .and_then(|out| { + parse_rocm_smi_csv_pool( + &String::from_utf8_lossy(&out.stdout), + "GTT Total Memory", + "GTT Total Used Memory", + ) + }) + } else { + None + }; + Some(DeviceMemory { + vram: Pool { + total_mib: vram_total, + free_mib: vram_free, + }, + gtt: gtt.map(|(total_mib, free_mib)| Pool { + total_mib, + free_mib, + }), + }) } /// `(total, free)` device VRAM in MiB for GPU 0, via `amd-smi` (then `rocm-smi`). @@ -358,20 +380,12 @@ fn vram_mib() -> Option<(u64, u64)> { .output() && out.status.success() { - let text = String::from_utf8_lossy(&out.stdout); // CSV columns include "VRAM Total Memory (B)" and "VRAM Total Used Memory (B)". - // Parse the first data row's total and used to derive free. - let mut lines = text.lines(); - let header = lines.next()?; - let cols: Vec<&str> = header.split(',').collect(); - let total_idx = cols.iter().position(|c| c.contains("Total Memory"))?; - let used_idx = cols.iter().position(|c| c.contains("Total Used Memory"))?; - let row = lines.next()?; - let vals: Vec<&str> = row.split(',').collect(); - let total: u64 = vals.get(total_idx)?.trim().parse().ok()?; - let used: u64 = vals.get(used_idx)?.trim().parse().ok()?; - let mib = 1024 * 1024; - return Some((total / mib, total.saturating_sub(used) / mib)); + return parse_rocm_smi_csv_pool( + &String::from_utf8_lossy(&out.stdout), + "Total Memory", + "Total Used Memory", + ); } None } From 32daa972ba2ee9b841579ea6df1beb86b84d7973 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 08:17:10 +0000 Subject: [PATCH 02/13] test(e2e): bound the smi probes in the serve drain wait The drain wait ran amd-smi and rocm-smi with a blocking output() call, so a tool that stalled on a wedged driver blocked the test worker instead of letting the best-effort wait expire, and the first probe ran before the wait's deadline existed. Every probe in the wait now goes through bounded_stdout, which gives the call ten seconds, then kills and reaps the child and reports an unreadable sample (the wait falls back to the VRAM reading, as before). The deadline is created before the first probe, and the probes run under spawn_blocking so they do not occupy an async worker. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 134 ++++++++++++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 58 ++++---- 2 files changed, 161 insertions(+), 31 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 46b97935e1..b460f4a170 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -14,6 +14,12 @@ //! never be met and every GPU serve spent its whole drain deadline waiting. //! `.github/scripts/gpu-preflight.sh` makes the same switch for the same reason. +use std::io::Read; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + /// Upper bound on the free-memory floor (MiB). /// /// Sized so the largest single @@ -132,6 +138,64 @@ pub fn parse_rocm_smi_csv_pool( Some((total / mib, total.saturating_sub(used) / mib)) } +/// How long one `amd-smi` or `rocm-smi` call may run before it is an unreadable sample. +/// +/// These tools can stall on a wedged driver, and the drain wait is best-effort: +/// a stalled probe must cost one sample, not the worker. +pub const SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +const SMI_POLL_INTERVAL: Duration = Duration::from_millis(25); + +/// How long to wait for the reader after the child has exited. A grandchild +/// that inherited the pipe can keep it open, so the reader is abandoned +/// rather than joined. +const SMI_DRAIN_GRACE: Duration = Duration::from_millis(200); + +/// Run `command` and return its stdout if it exits successfully within `timeout`. +/// +/// A spawn failure, a non-zero exit, or a timeout is `None`; on a timeout the +/// child is killed and reaped before this returns. +/// +/// Written against `std` alone because `rocm_core::process::run_with_timeout` +/// is reachable here only as a dev-dependency, and this module is library code. +pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Option { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + let Some(mut pipe) = child.stdout.take() else { + let _ = child.kill(); + let _ = child.wait(); + return None; + }; + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let mut bytes = Vec::new(); + let _ = pipe.read_to_end(&mut bytes); + let _ = sender.send(bytes); + }); + + let deadline = Instant::now() + timeout; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() < deadline => thread::sleep(SMI_POLL_INTERVAL), + _ => { + let _ = child.kill(); + let _ = child.wait(); + return None; + } + } + }; + if !status.success() { + return None; + } + let bytes = receiver.recv_timeout(SMI_DRAIN_GRACE).ok()?; + Some(String::from_utf8_lossy(&bytes).into_owned()) +} + #[cfg(test)] mod tests { use super::*; @@ -271,4 +335,74 @@ mod tests { None ); } + + #[cfg(unix)] + fn shell(script: &str, args: &[&str]) -> Command { + let mut command = Command::new("sh"); + command.arg("-c").arg(script).arg("sh").args(args); + command + } + + #[cfg(unix)] + #[test] + fn a_probe_that_succeeds_returns_its_stdout() { + let out = bounded_stdout(shell("echo hello", &[]), Duration::from_secs(30)); + assert_eq!(out.as_deref(), Some("hello\n")); + } + + #[cfg(unix)] + #[test] + fn a_probe_that_exits_non_zero_is_unreadable() { + let out = bounded_stdout(shell("echo hello; exit 3", &[]), Duration::from_secs(30)); + assert_eq!(out, None); + } + + /// The Copilot finding on the GTT probe: a tool that never returns must cost + /// one sample, not the worker, and the child must not be left running or + /// unreaped. A zombie still answers `kill -0`, so a failing `kill -0` shows + /// the child was reaped, not merely signalled. + #[cfg(unix)] + #[test] + fn a_probe_that_outlives_its_deadline_is_killed_reaped_and_unreadable() { + let dir = tempfile::tempdir().expect("tempdir"); + let pid_file = dir.path().join("pid"); + let started = Instant::now(); + let out = bounded_stdout( + shell( + "echo $$ > \"$1\"; exec sleep 60", + &[pid_file.to_str().expect("utf-8 path")], + ), + Duration::from_millis(1500), + ); + let elapsed = started.elapsed(); + assert_eq!(out, None, "a timed-out probe is an unreadable sample"); + assert!( + elapsed < Duration::from_secs(20), + "returned after {elapsed:?}, far past the 1.5s deadline" + ); + let pid = std::fs::read_to_string(&pid_file) + .expect("the child wrote its pid before the deadline") + .trim() + .to_owned(); + let still_there = Command::new("kill") + .args(["-0", &pid]) + .stderr(Stdio::null()) + .status() + .expect("kill runs") + .success(); + assert!(!still_there, "child {pid} was left running or unreaped"); + } + + /// Windows runs only the deadline half; the reaped-child check above needs + /// `kill -0`. + #[cfg(windows)] + #[test] + fn a_probe_that_outlives_its_deadline_returns_unreadable_in_time() { + let mut command = Command::new("ping.exe"); + command.args(["-n", "60", "127.0.0.1"]); + let started = Instant::now(); + let out = bounded_stdout(command, Duration::from_millis(1500)); + assert_eq!(out, None); + assert!(started.elapsed() < Duration::from_secs(20)); + } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index b89cefc76a..189251e4d5 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -12,8 +12,8 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::device_memory::{ - CARVEOUT_MAX_MIB, DeviceMemory, DrainTarget, Pool, PoolKind, parse_rocm_smi_csv_pool, - select_drain_target, + CARVEOUT_MAX_MIB, DeviceMemory, DrainTarget, Pool, PoolKind, SMI_PROBE_TIMEOUT, bounded_stdout, + parse_rocm_smi_csv_pool, select_drain_target, }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ @@ -276,13 +276,14 @@ const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); /// single most likely reason the serve that follows never becomes ready, and /// without it the failure looks identical to a genuinely broken serve. async fn wait_for_free_vram() -> String { + // Started before the first probe so that probe counts against the wait. + let deadline = Instant::now() + VRAM_DRAIN_DEADLINE; // No GPU tooling → nothing to wait on (mock/local). Probe once up front. - if device_memory_mib().is_none() { + if probe_device_memory().await.is_none() { return "device state: no GPU tooling (mock/local run)".to_owned(); } - let deadline = Instant::now() + VRAM_DRAIN_DEADLINE; loop { - let target = device_memory_mib().map(select_drain_target); + let target = probe_device_memory().await.map(select_drain_target); if let Some(target) = target && target.drained() { @@ -317,6 +318,15 @@ fn describe_target(target: DrainTarget) -> String { ) } +/// [`device_memory_mib`] off the async worker: it runs subprocesses, each bounded +/// by [`SMI_PROBE_TIMEOUT`], so the blocking thread it occupies is released. +async fn probe_device_memory() -> Option { + tokio::task::spawn_blocking(device_memory_mib) + .await + .ok() + .flatten() +} + /// Device memory for GPU 0: VRAM via `amd-smi` (then `rocm-smi`), plus the GTT /// pool from `rocm-smi`. `None` when no tool reports VRAM or its output can't /// be parsed. GTT is read only when VRAM is small enough to be an APU's @@ -328,18 +338,11 @@ fn device_memory_mib() -> Option { use std::process::Command; let (vram_total, vram_free) = vram_mib()?; let gtt = if vram_total < CARVEOUT_MAX_MIB { - Command::new("rocm-smi") - .args(["--showmeminfo", "gtt", "--csv"]) - .output() - .ok() - .filter(|out| out.status.success()) - .and_then(|out| { - parse_rocm_smi_csv_pool( - &String::from_utf8_lossy(&out.stdout), - "GTT Total Memory", - "GTT Total Used Memory", - ) - }) + let mut command = Command::new("rocm-smi"); + command.args(["--showmeminfo", "gtt", "--csv"]); + bounded_stdout(command, SMI_PROBE_TIMEOUT).and_then(|text| { + parse_rocm_smi_csv_pool(&text, "GTT Total Memory", "GTT Total Used Memory") + }) } else { None }; @@ -360,10 +363,9 @@ fn device_memory_mib() -> Option { fn vram_mib() -> Option<(u64, u64)> { use std::process::Command; // amd-smi: lines like " TOTAL_VRAM: 196592 MB" / "FREE_VRAM: 196309 MB". - if let Ok(out) = Command::new("amd-smi").args(["metric", "-m"]).output() - && out.status.success() - { - let text = String::from_utf8_lossy(&out.stdout); + let mut amd_smi = Command::new("amd-smi"); + amd_smi.args(["metric", "-m"]); + if let Some(text) = bounded_stdout(amd_smi, SMI_PROBE_TIMEOUT) { let field = |name: &str| { text.lines() .find_map(|l| l.trim().strip_prefix(name)) @@ -375,17 +377,11 @@ fn vram_mib() -> Option<(u64, u64)> { } } // rocm-smi fallback: `--showmeminfo vram --csv` → vram total/used per card. - if let Ok(out) = Command::new("rocm-smi") - .args(["--showmeminfo", "vram", "--csv"]) - .output() - && out.status.success() - { + let mut rocm_smi = Command::new("rocm-smi"); + rocm_smi.args(["--showmeminfo", "vram", "--csv"]); + if let Some(text) = bounded_stdout(rocm_smi, SMI_PROBE_TIMEOUT) { // CSV columns include "VRAM Total Memory (B)" and "VRAM Total Used Memory (B)". - return parse_rocm_smi_csv_pool( - &String::from_utf8_lossy(&out.stdout), - "Total Memory", - "Total Used Memory", - ); + return parse_rocm_smi_csv_pool(&text, "Total Memory", "Total Used Memory"); } None } From 393ca08279aa303b4832bf1825917b976f9d2952 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 11:54:54 +0000 Subject: [PATCH 03/13] test(e2e): tell an unreadable device probe from an absent one in the drain wait A GTT probe that stalled or failed on a carveout host used to look the same as "this host has no GTT pool", so the wait judged the small VRAM carveout and could report "drained" while the engine still held its GTT-backed memory. A probe now reports absent (tool not installed), unreadable (ran, no usable reading) or a reading. An unreadable GTT read makes the whole sample unreadable, and the "no GPU tooling" shortcut fires only when the tools are absent. The decision to read GTT at all moves next to the pool-selection rule, sharing one carveout predicate, so a test can reach it. The wait reuses its first sample instead of probing twice, the grandchild-holds-the-pipe limit of the bounded probe is documented, and the timeout test waits for the child's pid file instead of reading it once. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 321 +++++++++++++++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 106 +++--- 2 files changed, 328 insertions(+), 99 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index b460f4a170..8f971e0e5b 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -45,6 +45,61 @@ pub const MAX_FREE_FLOOR_MIB: u64 = 150_000; /// far above it (48 GiB and up). pub const CARVEOUT_MAX_MIB: u64 = 8 * 1024; +/// Whether a VRAM total is an APU's carveout rather than private device memory. +/// +/// Both the decision to read GTT at all ([`sample_device_memory`]) and the +/// choice of pool ([`select_drain_target`]) go through this, so they cannot +/// disagree. +#[must_use] +pub const fn is_carveout(vram_total_mib: u64) -> bool { + vram_total_mib < CARVEOUT_MAX_MIB +} + +/// One attempt to read from a device tool. +/// +/// `Absent` and `Unreadable` are kept apart on purpose. With no tool installed +/// there is nothing to wait on (a mock or local run). A tool that is present +/// but stalled, failed, or printed something unparseable is a device that +/// could not be read right now, which must never be mistaken for either "no +/// tooling" or "drained". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Probe { + /// The tool is not installed. + Absent, + /// The tool is there but gave no usable reading. + Unreadable, + Read(T), +} + +impl Probe { + /// Turn a raw reading into a parsed one; a reading `f` rejects is `Unreadable`. + #[must_use] + pub fn parse(self, f: impl FnOnce(T) -> Option) -> Probe { + match self { + Self::Absent => Probe::Absent, + Self::Unreadable => Probe::Unreadable, + Self::Read(value) => f(value).map_or(Probe::Unreadable, Probe::Read), + } + } + + /// Fall back to `next` when this probe produced no reading. + /// + /// A reading from either wins. Without one, the result is `Absent` only if + /// every tool tried was absent; if any tool was present and gave nothing, + /// it is `Unreadable`. + #[must_use] + pub fn or_else(self, next: impl FnOnce() -> Self) -> Self { + if matches!(self, Self::Read(_)) { + return self; + } + match (self, next()) { + (_, read @ Self::Read(_)) => read, + (Self::Absent, Self::Absent) => Self::Absent, + _ => Self::Unreadable, + } + } +} + /// One memory pool's size and free space, in MiB. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Pool { @@ -93,17 +148,14 @@ pub fn required_free_mib(total_mib: u64) -> u64 { /// Choose the pool the engine allocates from and the floor to hold it to. /// -/// A VRAM total under [`CARVEOUT_MAX_MIB`] is a carveout, so the GTT pool is -/// used when one was read and is non-empty. A zero GTT total is a failed -/// reading, not a pool, and falls back to VRAM, as it does in -/// `gpu-preflight.sh`. Without GTT data the answer is VRAM, which is what -/// every host got before this choice existed. +/// A VRAM total that [`is_carveout`] means the GTT pool is used, when one was +/// read. Without GTT data the answer is VRAM, which is what every host got +/// before this choice existed; [`sample_device_memory`] only leaves GTT out +/// when the host has no tool that reports it, never when a read failed. #[must_use] pub fn select_drain_target(memory: DeviceMemory) -> DrainTarget { let (kind, pool) = match memory.gtt { - Some(gtt) if memory.vram.total_mib < CARVEOUT_MAX_MIB && gtt.total_mib > 0 => { - (PoolKind::Gtt, gtt) - } + Some(gtt) if is_carveout(memory.vram.total_mib) => (PoolKind::Gtt, gtt), _ => (PoolKind::Vram, memory.vram), }; DrainTarget { @@ -138,6 +190,50 @@ pub fn parse_rocm_smi_csv_pool( Some((total / mib, total.saturating_sub(used) / mib)) } +/// Assemble one device-memory sample from a VRAM probe and a GTT probe. +/// +/// `gtt` runs only when the VRAM total [`is_carveout`], so a discrete-card host +/// never makes the extra call. On a carveout host the GTT reading is what the +/// engine's memory comes out of, so: +/// +/// - a GTT reading makes the sample; +/// - a GTT probe that ran but gave nothing usable (a stall, a failure, output +/// not understood, or a zero total, which is a failed read rather than an +/// empty pool) makes the whole sample `Unreadable`. Judging the tiny +/// carveout instead could report "drained" while the engine still holds its +/// GTT-backed memory. `gpu-preflight.sh` treats a zero GTT total the same way, +/// as an unreadable sample to retry; +/// - no tool that reports GTT at all (`Absent`) leaves `gtt` empty, and the +/// wait judges VRAM as it did before GTT was considered. +pub fn sample_device_memory( + vram: impl FnOnce() -> Probe<(u64, u64)>, + gtt: impl FnOnce() -> Probe<(u64, u64)>, +) -> Probe { + let (vram_total, vram_free) = match vram() { + Probe::Read(reading) => reading, + Probe::Absent => return Probe::Absent, + Probe::Unreadable => return Probe::Unreadable, + }; + let vram = Pool { + total_mib: vram_total, + free_mib: vram_free, + }; + if !is_carveout(vram_total) { + return Probe::Read(DeviceMemory { vram, gtt: None }); + } + match gtt() { + Probe::Read((total_mib, free_mib)) if total_mib > 0 => Probe::Read(DeviceMemory { + vram, + gtt: Some(Pool { + total_mib, + free_mib, + }), + }), + Probe::Read(_) | Probe::Unreadable => Probe::Unreadable, + Probe::Absent => Probe::Read(DeviceMemory { vram, gtt: None }), + } +} + /// How long one `amd-smi` or `rocm-smi` call may run before it is an unreadable sample. /// /// These tools can stall on a wedged driver, and the drain wait is best-effort: @@ -153,22 +249,33 @@ const SMI_DRAIN_GRACE: Duration = Duration::from_millis(200); /// Run `command` and return its stdout if it exits successfully within `timeout`. /// -/// A spawn failure, a non-zero exit, or a timeout is `None`; on a timeout the -/// child is killed and reaped before this returns. +/// A program that is not installed is [`Probe::Absent`]. Any other spawn +/// failure, a non-zero exit, or a timeout is [`Probe::Unreadable`]; on a +/// timeout the child is killed and reaped before this returns. +/// +/// Only the direct child is killed. A grandchild that inherited the stdout +/// pipe keeps the reader thread blocked in `read_to_end` until it exits, so a +/// timed-out probe can leak one thread. The probes here (`amd-smi`, +/// `rocm-smi`) do not fork helpers that outlive them, and the drain wait makes +/// a handful of calls, so this is accepted rather than guarded. /// /// Written against `std` alone because `rocm_core::process::run_with_timeout` /// is reachable here only as a dev-dependency, and this module is library code. -pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Option { - let mut child = command +pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe { + let mut child = match command .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() - .ok()?; + { + Ok(child) => child, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Probe::Absent, + Err(_) => return Probe::Unreadable, + }; let Some(mut pipe) = child.stdout.take() else { let _ = child.kill(); let _ = child.wait(); - return None; + return Probe::Unreadable; }; let (sender, receiver) = mpsc::channel(); thread::spawn(move || { @@ -185,15 +292,18 @@ pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Option _ => { let _ = child.kill(); let _ = child.wait(); - return None; + return Probe::Unreadable; } } }; if !status.success() { - return None; + return Probe::Unreadable; } - let bytes = receiver.recv_timeout(SMI_DRAIN_GRACE).ok()?; - Some(String::from_utf8_lossy(&bytes).into_owned()) + receiver + .recv_timeout(SMI_DRAIN_GRACE) + .map_or(Probe::Unreadable, |bytes| { + Probe::Read(String::from_utf8_lossy(&bytes).into_owned()) + }) } #[cfg(test)] @@ -282,16 +392,6 @@ mod tests { assert!(!target.drained()); } - #[test] - fn a_zero_gtt_total_is_a_failed_reading_not_a_pool() { - let mut memory = strix_halo(); - memory.gtt = Some(Pool { - total_mib: 0, - free_mib: 0, - }); - assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); - } - #[test] fn a_vram_total_at_the_carveout_limit_is_private_memory() { let mut memory = strix_halo(); @@ -300,6 +400,115 @@ mod tests { free_mib: CARVEOUT_MAX_MIB, }; assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + assert!(is_carveout(CARVEOUT_MAX_MIB - 1)); + assert!(!is_carveout(CARVEOUT_MAX_MIB)); + } + + /// Run `sample_device_memory` with canned probes, reporting whether the GTT + /// probe was consulted. + fn sample(vram: Probe<(u64, u64)>, gtt: Probe<(u64, u64)>) -> (Probe, bool) { + let mut gtt_asked = false; + let result = sample_device_memory( + || vram, + || { + gtt_asked = true; + gtt + }, + ); + (result, gtt_asked) + } + + #[test] + fn a_carveout_host_reads_gtt_and_the_sample_carries_it() { + let (result, gtt_asked) = sample(Probe::Read((512, 318)), Probe::Read((102_400, 102_360))); + assert!(gtt_asked, "a carveout host must have its GTT read"); + assert_eq!(result, Probe::Read(strix_halo())); + } + + #[test] + fn a_discrete_host_never_makes_the_gtt_call() { + let (result, gtt_asked) = sample( + Probe::Read((196_592, 196_309)), + Probe::Read((1_000_000, 1_000_000)), + ); + assert!( + !gtt_asked, + "GTT must not be read when VRAM is private memory" + ); + let Probe::Read(memory) = result else { + panic!("a readable VRAM probe makes a sample: {result:?}"); + }; + assert_eq!(memory.gtt, None); + assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + } + + #[test] + fn a_carveout_host_whose_gtt_probe_failed_is_unreadable_not_judged_by_vram() { + // VRAM alone would read as "drained" here (free is above its floor), so + // falling back to it would hide a GTT-backed allocation still held. + let (result, gtt_asked) = sample(Probe::Read((512, 500)), Probe::Unreadable); + assert!(gtt_asked); + assert_eq!(result, Probe::Unreadable); + } + + #[test] + fn a_zero_gtt_total_is_a_failed_reading_not_a_pool() { + let (result, _) = sample(Probe::Read((512, 500)), Probe::Read((0, 0))); + assert_eq!(result, Probe::Unreadable); + } + + #[test] + fn a_carveout_host_with_no_gtt_tool_keeps_the_vram_behaviour() { + let (result, gtt_asked) = sample(Probe::Read((512, 318)), Probe::Absent); + assert!(gtt_asked); + let Probe::Read(memory) = result else { + panic!("a host without a GTT tool still has a VRAM sample: {result:?}"); + }; + assert_eq!(memory.gtt, None); + assert_eq!(select_drain_target(memory).kind, PoolKind::Vram); + } + + #[test] + fn a_vram_probe_that_is_absent_or_unreadable_is_passed_through_without_asking_for_gtt() { + assert_eq!( + sample(Probe::Absent, Probe::Read((102_400, 102_360))), + (Probe::Absent, false) + ); + assert_eq!( + sample(Probe::Unreadable, Probe::Read((102_400, 102_360))), + (Probe::Unreadable, false) + ); + } + + #[test] + fn a_fallback_tool_supplies_the_reading_the_first_could_not() { + assert_eq!(Probe::Unreadable.or_else(|| Probe::Read(7)), Probe::Read(7)); + assert_eq!(Probe::Absent.or_else(|| Probe::Read(7)), Probe::Read(7)); + assert_eq!( + Probe::Read(1).or_else(|| panic!("the fallback must not run after a reading")), + Probe::Read(1) + ); + } + + #[test] + fn a_fallback_is_absent_only_when_every_tool_was_absent() { + assert_eq!(Probe::::Absent.or_else(|| Probe::Absent), Probe::Absent); + assert_eq!( + Probe::::Absent.or_else(|| Probe::Unreadable), + Probe::Unreadable + ); + assert_eq!( + Probe::::Unreadable.or_else(|| Probe::Absent), + Probe::Unreadable + ); + } + + #[test] + fn parse_keeps_absent_and_unreadable_and_rejects_what_it_cannot_read() { + assert_eq!(Probe::::Absent.parse(Some), Probe::Absent); + assert_eq!(Probe::::Unreadable.parse(Some), Probe::Unreadable); + assert_eq!(Probe::Read(3_u8).parse(|n| Some(n + 1)), Probe::Read(4)); + assert_eq!(Probe::Read(3_u8).parse(|_| None::), Probe::Unreadable); } #[test] @@ -347,20 +556,29 @@ mod tests { #[test] fn a_probe_that_succeeds_returns_its_stdout() { let out = bounded_stdout(shell("echo hello", &[]), Duration::from_secs(30)); - assert_eq!(out.as_deref(), Some("hello\n")); + assert_eq!(out, Probe::Read("hello\n".to_owned())); } #[cfg(unix)] #[test] fn a_probe_that_exits_non_zero_is_unreadable() { let out = bounded_stdout(shell("echo hello; exit 3", &[]), Duration::from_secs(30)); - assert_eq!(out, None); + assert_eq!(out, Probe::Unreadable); + } + + #[test] + fn a_probe_whose_program_is_not_installed_is_absent() { + let out = bounded_stdout( + Command::new("rocm-cli-test-no-such-smi-tool"), + Duration::from_secs(30), + ); + assert_eq!(out, Probe::Absent); } - /// The Copilot finding on the GTT probe: a tool that never returns must cost - /// one sample, not the worker, and the child must not be left running or - /// unreaped. A zombie still answers `kill -0`, so a failing `kill -0` shows - /// the child was reaped, not merely signalled. + /// A tool that never returns must cost one sample, not the worker, and the + /// child must not be left running or unreaped. A zombie still answers + /// `kill -0`, so a failing `kill -0` shows the child was reaped, not merely + /// signalled. #[cfg(unix)] #[test] fn a_probe_that_outlives_its_deadline_is_killed_reaped_and_unreadable() { @@ -372,18 +590,35 @@ mod tests { "echo $$ > \"$1\"; exec sleep 60", &[pid_file.to_str().expect("utf-8 path")], ), - Duration::from_millis(1500), + Duration::from_secs(3), ); let elapsed = started.elapsed(); - assert_eq!(out, None, "a timed-out probe is an unreadable sample"); + assert_eq!( + out, + Probe::Unreadable, + "a timed-out probe is an unreadable sample" + ); + assert!( + elapsed < Duration::from_secs(30), + "returned after {elapsed:?}, far past the 3s deadline" + ); + // The shell creates the file and then writes the pid; give a loaded + // runner a moment to finish the write rather than read it half-done. + let mut pid = String::new(); + for _ in 0..100 { + pid = std::fs::read_to_string(&pid_file) + .unwrap_or_default() + .trim() + .to_owned(); + if !pid.is_empty() { + break; + } + thread::sleep(Duration::from_millis(50)); + } assert!( - elapsed < Duration::from_secs(20), - "returned after {elapsed:?}, far past the 1.5s deadline" + !pid.is_empty(), + "the child wrote its pid before the deadline" ); - let pid = std::fs::read_to_string(&pid_file) - .expect("the child wrote its pid before the deadline") - .trim() - .to_owned(); let still_there = Command::new("kill") .args(["-0", &pid]) .stderr(Stdio::null()) @@ -402,7 +637,7 @@ mod tests { command.args(["-n", "60", "127.0.0.1"]); let started = Instant::now(); let out = bounded_stdout(command, Duration::from_millis(1500)); - assert_eq!(out, None); + assert_eq!(out, Probe::Unreadable); assert!(started.elapsed() < Duration::from_secs(20)); } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 189251e4d5..fe8c296150 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -12,8 +12,8 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::device_memory::{ - CARVEOUT_MAX_MIB, DeviceMemory, DrainTarget, Pool, PoolKind, SMI_PROBE_TIMEOUT, bounded_stdout, - parse_rocm_smi_csv_pool, select_drain_target, + DeviceMemory, DrainTarget, PoolKind, Probe, SMI_PROBE_TIMEOUT, bounded_stdout, + parse_rocm_smi_csv_pool, sample_device_memory, select_drain_target, }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ @@ -278,12 +278,18 @@ const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); async fn wait_for_free_vram() -> String { // Started before the first probe so that probe counts against the wait. let deadline = Instant::now() + VRAM_DRAIN_DEADLINE; - // No GPU tooling → nothing to wait on (mock/local). Probe once up front. - if probe_device_memory().await.is_none() { + let mut sample = probe_device_memory().await; + // No GPU tooling installed → nothing to wait on (mock/local). A tool that is + // present but gave no reading is a different case: it keeps waiting, and + // says so if it never reads. + if sample == Probe::Absent { return "device state: no GPU tooling (mock/local run)".to_owned(); } loop { - let target = probe_device_memory().await.map(select_drain_target); + let target = match sample { + Probe::Read(memory) => Some(select_drain_target(memory)), + Probe::Absent | Probe::Unreadable => None, + }; if let Some(target) = target && target.drained() { @@ -301,6 +307,7 @@ async fn wait_for_free_vram() -> String { ); } tokio::time::sleep(Duration::from_secs(3)).await; + sample = probe_device_memory().await; } } @@ -320,70 +327,57 @@ fn describe_target(target: DrainTarget) -> String { /// [`device_memory_mib`] off the async worker: it runs subprocesses, each bounded /// by [`SMI_PROBE_TIMEOUT`], so the blocking thread it occupies is released. -async fn probe_device_memory() -> Option { +async fn probe_device_memory() -> Probe { tokio::task::spawn_blocking(device_memory_mib) .await - .ok() - .flatten() + .unwrap_or(Probe::Unreadable) } /// Device memory for GPU 0: VRAM via `amd-smi` (then `rocm-smi`), plus the GTT -/// pool from `rocm-smi`. `None` when no tool reports VRAM or its output can't -/// be parsed. GTT is read only when VRAM is small enough to be an APU's -/// carveout (see [`CARVEOUT_MAX_MIB`]), so a discrete-card host never makes the -/// extra call. `amd-smi` is not asked for GTT: its output has no field this -/// harness has been checked against, whereas the preflight step already reads -/// `rocm-smi`'s GTT rows on these hosts. -fn device_memory_mib() -> Option { +/// pool from `rocm-smi` where the host needs it (see [`sample_device_memory`] +/// for when GTT is read and what a failed read means). `amd-smi` is not asked +/// for GTT: its output has no field this harness has been checked against, +/// whereas the preflight step already reads `rocm-smi`'s GTT rows on these +/// hosts. +fn device_memory_mib() -> Probe { + sample_device_memory(vram_mib, gtt_mib) +} + +/// `(total, free)` GTT in MiB for GPU 0, via `rocm-smi`. +fn gtt_mib() -> Probe<(u64, u64)> { use std::process::Command; - let (vram_total, vram_free) = vram_mib()?; - let gtt = if vram_total < CARVEOUT_MAX_MIB { - let mut command = Command::new("rocm-smi"); - command.args(["--showmeminfo", "gtt", "--csv"]); - bounded_stdout(command, SMI_PROBE_TIMEOUT).and_then(|text| { - parse_rocm_smi_csv_pool(&text, "GTT Total Memory", "GTT Total Used Memory") - }) - } else { - None - }; - Some(DeviceMemory { - vram: Pool { - total_mib: vram_total, - free_mib: vram_free, - }, - gtt: gtt.map(|(total_mib, free_mib)| Pool { - total_mib, - free_mib, - }), - }) + let mut command = Command::new("rocm-smi"); + command.args(["--showmeminfo", "gtt", "--csv"]); + bounded_stdout(command, SMI_PROBE_TIMEOUT) + .parse(|text| parse_rocm_smi_csv_pool(&text, "GTT Total Memory", "GTT Total Used Memory")) } /// `(total, free)` device VRAM in MiB for GPU 0, via `amd-smi` (then `rocm-smi`). -/// `None` when no such tool exists or its output can't be parsed. -fn vram_mib() -> Option<(u64, u64)> { +fn vram_mib() -> Probe<(u64, u64)> { use std::process::Command; // amd-smi: lines like " TOTAL_VRAM: 196592 MB" / "FREE_VRAM: 196309 MB". - let mut amd_smi = Command::new("amd-smi"); - amd_smi.args(["metric", "-m"]); - if let Some(text) = bounded_stdout(amd_smi, SMI_PROBE_TIMEOUT) { - let field = |name: &str| { - text.lines() - .find_map(|l| l.trim().strip_prefix(name)) - .and_then(|v| v.split_whitespace().next()) - .and_then(|n| n.parse::().ok()) - }; - if let (Some(total), Some(free)) = (field("TOTAL_VRAM:"), field("FREE_VRAM:")) { - return Some((total, free)); - } - } + let amd_smi = || { + let mut command = Command::new("amd-smi"); + command.args(["metric", "-m"]); + bounded_stdout(command, SMI_PROBE_TIMEOUT).parse(|text| { + let field = |name: &str| { + text.lines() + .find_map(|l| l.trim().strip_prefix(name)) + .and_then(|v| v.split_whitespace().next()) + .and_then(|n| n.parse::().ok()) + }; + Some((field("TOTAL_VRAM:")?, field("FREE_VRAM:")?)) + }) + }; // rocm-smi fallback: `--showmeminfo vram --csv` → vram total/used per card. - let mut rocm_smi = Command::new("rocm-smi"); - rocm_smi.args(["--showmeminfo", "vram", "--csv"]); - if let Some(text) = bounded_stdout(rocm_smi, SMI_PROBE_TIMEOUT) { + let rocm_smi = || { + let mut command = Command::new("rocm-smi"); + command.args(["--showmeminfo", "vram", "--csv"]); // CSV columns include "VRAM Total Memory (B)" and "VRAM Total Used Memory (B)". - return parse_rocm_smi_csv_pool(&text, "Total Memory", "Total Used Memory"); - } - None + bounded_stdout(command, SMI_PROBE_TIMEOUT) + .parse(|text| parse_rocm_smi_csv_pool(&text, "Total Memory", "Total Used Memory")) + }; + amd_smi().or_else(rocm_smi) } /// Kill whatever process is listening on `port`. Best-effort and From a3b2e16ca5d6a70e2f7c032bb9ae0927f855bad4 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 12:00:05 +0000 Subject: [PATCH 04/13] test(e2e): make the drain wait's per-sample decision a tested function The choice between 'no GPU tooling', 'present but unreadable' and 'a reading to judge' lived inline in the async wait loop, where no unit test could reach it, so reverting an unreadable first probe to 'no tooling' would have stayed green. It is now judge() in device_memory.rs with tests, and the DeviceMemory docs state that an unreadable GTT read never reaches a sample. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 44 ++++++++++++++++++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 16 +++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 8f971e0e5b..d572b1e731 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -107,8 +107,12 @@ pub struct Pool { pub free_mib: u64, } -/// What the device tooling reported for GPU 0. `gtt` is `None` when the tool -/// that answered has no GTT figure (or none was readable). +/// What the device tooling reported for GPU 0. +/// +/// `gtt` is `None` when GTT was not read: a discrete card, or a host with no +/// tool that reports it. A GTT read +/// that failed never gets here (the sample is `Unreadable`, see +/// [`sample_device_memory`]), so a `Some` pool always has a non-zero total. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DeviceMemory { pub vram: Pool, @@ -146,6 +150,28 @@ pub fn required_free_mib(total_mib: u64) -> u64 { MAX_FREE_FLOOR_MIB.min(total_mib / 100 * 90) } +/// What one sample means for the drain wait. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Judgement { + /// No device tool is installed: nothing to wait on (mock/local run). + NoTooling, + /// A tool is present but gave no usable reading. This is not "no tooling" + /// and not "drained": the wait keeps trying. + Unreadable, + /// A reading, and the pool and floor it is held to. + Target(DrainTarget), +} + +/// Turn a sample into what the wait does with it. +#[must_use] +pub fn judge(sample: Probe) -> Judgement { + match sample { + Probe::Absent => Judgement::NoTooling, + Probe::Unreadable => Judgement::Unreadable, + Probe::Read(memory) => Judgement::Target(select_drain_target(memory)), + } +} + /// Choose the pool the engine allocates from and the floor to hold it to. /// /// A VRAM total that [`is_carveout`] means the GTT pool is used, when one was @@ -480,6 +506,20 @@ mod tests { ); } + #[test] + fn an_absent_tool_is_no_tooling_but_an_unreadable_one_is_not() { + assert_eq!(judge(Probe::Absent), Judgement::NoTooling); + assert_eq!(judge(Probe::Unreadable), Judgement::Unreadable); + } + + #[test] + fn a_reading_is_judged_against_the_pool_the_engine_allocates_from() { + assert_eq!( + judge(Probe::Read(strix_halo())), + Judgement::Target(select_drain_target(strix_halo())) + ); + } + #[test] fn a_fallback_tool_supplies_the_reading_the_first_could_not() { assert_eq!(Probe::Unreadable.or_else(|| Probe::Read(7)), Probe::Read(7)); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index fe8c296150..c218732abb 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -12,8 +12,8 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::device_memory::{ - DeviceMemory, DrainTarget, PoolKind, Probe, SMI_PROBE_TIMEOUT, bounded_stdout, - parse_rocm_smi_csv_pool, sample_device_memory, select_drain_target, + DeviceMemory, DrainTarget, Judgement, PoolKind, Probe, SMI_PROBE_TIMEOUT, bounded_stdout, + judge, parse_rocm_smi_csv_pool, sample_device_memory, }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ @@ -278,17 +278,17 @@ const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); async fn wait_for_free_vram() -> String { // Started before the first probe so that probe counts against the wait. let deadline = Instant::now() + VRAM_DRAIN_DEADLINE; - let mut sample = probe_device_memory().await; + let mut judgement = judge(probe_device_memory().await); // No GPU tooling installed → nothing to wait on (mock/local). A tool that is // present but gave no reading is a different case: it keeps waiting, and // says so if it never reads. - if sample == Probe::Absent { + if judgement == Judgement::NoTooling { return "device state: no GPU tooling (mock/local run)".to_owned(); } loop { - let target = match sample { - Probe::Read(memory) => Some(select_drain_target(memory)), - Probe::Absent | Probe::Unreadable => None, + let target = match judgement { + Judgement::Target(target) => Some(target), + Judgement::NoTooling | Judgement::Unreadable => None, }; if let Some(target) = target && target.drained() @@ -307,7 +307,7 @@ async fn wait_for_free_vram() -> String { ); } tokio::time::sleep(Duration::from_secs(3)).await; - sample = probe_device_memory().await; + judgement = judge(probe_device_memory().await); } } From 9e628ebd462d8de19329df67e822480afadd7695 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 07:38:23 +0000 Subject: [PATCH 05/13] test(e2e): treat a zero VRAM total as an unreadable drain-wait sample A zero GTT total was already a failed read, but a parseable zero VRAM total was accepted as a pool. With no GTT tool on the host it was judged against a floor of zero, so the wait reported drained on its first sample. Both pools now share the rule, and the DeviceMemory docs say so. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index d572b1e731..34bc0209bf 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -112,7 +112,8 @@ pub struct Pool { /// `gtt` is `None` when GTT was not read: a discrete card, or a host with no /// tool that reports it. A GTT read /// that failed never gets here (the sample is `Unreadable`, see -/// [`sample_device_memory`]), so a `Some` pool always has a non-zero total. +/// [`sample_device_memory`]), so every pool here has a non-zero total, VRAM and +/// GTT alike. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DeviceMemory { pub vram: Pool, @@ -236,9 +237,11 @@ pub fn sample_device_memory( gtt: impl FnOnce() -> Probe<(u64, u64)>, ) -> Probe { let (vram_total, vram_free) = match vram() { - Probe::Read(reading) => reading, + // A zero total is a failed read, not a pool: its floor would be zero and + // every sample would count as drained. + Probe::Read((total, free)) if total > 0 => (total, free), + Probe::Read(_) | Probe::Unreadable => return Probe::Unreadable, Probe::Absent => return Probe::Absent, - Probe::Unreadable => return Probe::Unreadable, }; let vram = Pool { total_mib: vram_total, @@ -483,6 +486,15 @@ mod tests { assert_eq!(result, Probe::Unreadable); } + #[test] + fn a_zero_vram_total_is_a_failed_reading_not_a_drained_host() { + // With no GTT tool the sample would otherwise be judged by a VRAM pool + // whose floor is zero, which is drained by definition. + let (result, gtt_asked) = sample(Probe::Read((0, 0)), Probe::Absent); + assert_eq!(result, Probe::Unreadable); + assert!(!gtt_asked); + } + #[test] fn a_carveout_host_with_no_gtt_tool_keeps_the_vram_behaviour() { let (result, gtt_asked) = sample(Probe::Read((512, 318)), Probe::Absent); From 68e6ae9d9159c09886c2bb5745cd68064ad94eba Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 08:20:14 +0000 Subject: [PATCH 06/13] test(xtask): tie the serve drain carveout limit to the APU preflight floor CARVEOUT_MAX_MIB in the e2e harness is documented as the floor the APU lanes pass as GPU_PREFLIGHT_MIN_FREE_GIB, but nothing connected the two, so changing either left the other silently wrong with every unit test green. A workflow contract test now reads the constant as source text, with no new crate edge, and requires it to equal the smallest preflight floor any lane sets. Signed-off-by: Jussi Elo --- xtask/src/workflow_contract.rs | 76 ++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index d18d8dd53d..816d9f5d34 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -2041,6 +2041,82 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" body.contains("GPU_PREFLIGHT_MIN_FREE_GIB:-8") || body.contains("else { 8 }") } + /// `CARVEOUT_MAX_MIB` in MiB, read from the harness source as text. + /// + /// Read as text rather than imported: `xtask` has no edge to `e2e-cucumber` + /// and this check is not a reason to add one. Only the two spellings the + /// constant can reasonably take are understood, ` * 1024` and a bare + /// number, and anything else fails loudly instead of guessing. + fn carveout_max_mib(source: &str) -> u64 { + let line = source + .lines() + .find(|l| { + l.trim_start() + .starts_with("pub const CARVEOUT_MAX_MIB: u64 =") + }) + .expect("device_memory.rs declares `pub const CARVEOUT_MAX_MIB: u64 = ...;`"); + let expr = line + .split_once('=') + .and_then(|(_, rhs)| rhs.split_once(';')) + .map(|(expr, _)| expr.trim().replace('_', "")) + .expect("the constant's value ends in a semicolon"); + let number = |s: &str| { + s.trim() + .parse::() + .unwrap_or_else(|_| panic!("cannot read {s:?} in `{expr}` as a number")) + }; + match expr.split_once('*') { + Some((a, b)) => number(a) * number(b), + None => number(&expr), + } + } + + #[test] + fn carveout_max_mib_reads_both_spellings() { + let spelled = |v: &str| format!("/// doc\npub const CARVEOUT_MAX_MIB: u64 = {v};\n"); + assert_eq!(carveout_max_mib(&spelled("8 * 1024")), 8192); + assert_eq!(carveout_max_mib(&spelled("8192")), 8192); + assert_eq!(carveout_max_mib(&spelled("8_192")), 8192); + } + + #[test] + fn the_serve_drain_carveout_limit_is_the_apu_lanes_preflight_floor() { + // The serve drain wait (`tests/e2e-cucumber/src/device_memory.rs`) treats + // a VRAM total under `CARVEOUT_MAX_MIB` as an APU's BIOS carveout and + // judges GTT instead. Its doc says that limit is the floor the APU lanes + // pass as `GPU_PREFLIGHT_MIN_FREE_GIB`, and nothing tied the two together: + // moving either one left the other quietly wrong, with every unit test + // green. A raised limit would send a small discrete card to GTT; a lowered + // floor with the limit left behind would judge a real carveout by VRAM. + // + // The APU floor is the smallest one any lane sets (the discrete lanes set + // a larger one), so this follows the lanes rather than a copy of 8. + let mut floors_gib = Vec::new(); + for (_, text) in self_hosted_workflows() { + for step in gpu_preflight_steps(&text) { + if let Some(floor) = preflight_floor(&step) { + floors_gib.push(floor.parse::().unwrap_or_else(|_| { + panic!("a preflight floor of {floor:?} is not a whole number of GiB") + })); + } + } + } + let apu_floor_gib = *floors_gib + .iter() + .min() + .expect("the self-hosted lanes set a preflight floor"); + let source = + std::fs::read_to_string(repo_root().join("tests/e2e-cucumber/src/device_memory.rs")) + .expect("reading tests/e2e-cucumber/src/device_memory.rs"); + assert_eq!( + carveout_max_mib(&source), + apu_floor_gib * 1024, + "CARVEOUT_MAX_MIB no longer equals the APU lanes' GPU_PREFLIGHT_MIN_FREE_GIB \ + ({apu_floor_gib} GiB); change them together, or the serve drain wait picks \ + the wrong memory pool" + ); + } + #[test] fn every_apu_preflight_block_measures_the_gtt_pool() { // A gfx1151 APU reports its BIOS carveout as VRAM total — 512 MiB on the From 624947c162e25ac42668bc32a44ce40a68c8decb Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 08:25:08 +0000 Subject: [PATCH 07/13] test(e2e): name the test that keeps the carveout limit in step with the lanes The CARVEOUT_MAX_MIB doc says it is the APU lanes' preflight floor; it now also names the xtask contract test that fails when the two stop matching. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 34bc0209bf..a37f56a6b0 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -42,7 +42,9 @@ pub const MAX_FREE_FLOOR_MIB: u64 = 150_000; /// 8 GiB floor the Strix Halo preflight lanes pass as /// `GPU_PREFLIGHT_MIN_FREE_GIB`, which `gpu-preflight.sh` compares the VRAM /// total against for the same purpose. Every discrete card the lanes run on is -/// far above it (48 GiB and up). +/// far above it (48 GiB and up). The xtask test +/// `the_serve_drain_carveout_limit_is_the_apu_lanes_preflight_floor` fails if +/// this stops matching the lanes' floor. pub const CARVEOUT_MAX_MIB: u64 = 8 * 1024; /// Whether a VRAM total is an APU's carveout rather than private device memory. From 5099c1e95fb988f0e4c71f9b336a8023c06d09dd Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 14:54:16 +0000 Subject: [PATCH 08/13] test(e2e): bound the drain wait's probes, give up on a tool that never reads, and say what it saw Review of the serve drain wait raised five points, all addressed here. A host whose GPU tool is installed but never produces a reading waited the whole two-minute deadline on every serve. After three samples in a row with no reading, from a tool that has never read, the wait now ends and reports the device as unreadable; a tool that has read before is still waited on. One sample can make three probes of up to ten seconds each, so the last one could run well past the deadline while the line said 120s. Each probe is now held to the time left, the sleep is capped to it, and the line reports the elapsed time. A carveout-sized VRAM pool judged without GTT data (amd-smi present, rocm-smi absent) now says so on the timeout line, because that wait cannot follow the memory the engine uses. "NOT drained" is only printed for a reading below its floor. A sample the harness could not read is reported as "could not read device memory", with the last reading if there was one. The wait's decisions (`DrainWait`), the lines (`describe_verdict`, `DrainTarget::describe`) and the probe budget now live in `device_memory.rs` with unit tests that drive samples through the wait and assert the resulting line with it. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 334 ++++++++++++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 95 ++--- 2 files changed, 372 insertions(+), 57 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index a37f56a6b0..79f62c28ff 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -144,6 +144,21 @@ impl DrainTarget { pub const fn drained(&self) -> bool { self.free_mib >= self.floor_mib } + + /// `" MiB free of MiB[ GTT], floor MiB"`. VRAM is the + /// unlabelled default, so the line keeps the shape it always had on discrete + /// cards; only the GTT pool is named. + #[must_use] + pub fn describe(&self) -> String { + let pool = match self.kind { + PoolKind::Vram => "", + PoolKind::Gtt => " GTT", + }; + format!( + "{} MiB free of {} MiB{pool}, floor {} MiB", + self.free_mib, self.total_mib, self.floor_mib + ) + } } /// The free-memory floor to wait for on a pool of `total_mib`: the @@ -175,6 +190,136 @@ pub fn judge(sample: Probe) -> Judgement { } } +/// Consecutive samples with no reading, from a tool that is installed and has +/// never once produced one, after which the wait stops. +/// +/// A tool that never reads is broken or wedged, and waiting out the whole +/// deadline on every serve would add that deadline to each of them. A tool that +/// has read before is not given up on: its next sample may well succeed. +pub const UNREAD_SAMPLES_BEFORE_GIVING_UP: u32 = 3; + +/// The least time worth starting a probe with. With less than this left before +/// the deadline the wait ends on what it has rather than spawn a probe that +/// would be killed before it could answer. +pub const MIN_PROBE_BUDGET: Duration = Duration::from_secs(1); + +/// How long one probe may run: [`SMI_PROBE_TIMEOUT`], but never past `deadline`. +/// +/// A sample makes up to three probes in a row, so without this the last sample +/// could run several probe timeouts beyond the wait's deadline. +#[must_use] +pub fn probe_budget(now: Instant, deadline: Instant) -> Duration { + SMI_PROBE_TIMEOUT.min(deadline.saturating_duration_since(now)) +} + +/// How the drain wait ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// No device tool is installed (mock/local run): nothing was waited on. + NoTooling, + /// The pool handed back enough memory. + Drained(DrainTarget), + /// The deadline passed with the pool still below its floor. + NotDrained(DrainTarget), + /// No usable reading at the end: `samples` samples in a row gave none, and + /// `last_reading` is the most recent one the wait did get, if any. This is + /// not "not drained": the harness did not observe that. + Unreadable { + last_reading: Option, + samples: u32, + }, +} + +/// The drain wait's decision state, fed one [`Judgement`] per sample. +#[derive(Debug, Default)] +pub struct DrainWait { + samples: u32, + unread_in_a_row: u32, + last_reading: Option, +} + +impl DrainWait { + /// Take the next sample. `Some` ends the wait; `None` means keep sampling. + pub fn observe(&mut self, judgement: Judgement) -> Option { + let first = self.samples == 0; + self.samples += 1; + match judgement { + Judgement::NoTooling if first => Some(Verdict::NoTooling), + Judgement::Target(target) => { + self.unread_in_a_row = 0; + self.last_reading = Some(target); + target.drained().then_some(Verdict::Drained(target)) + } + // A tool that vanishes mid-wait is a sample nobody could read. + Judgement::NoTooling | Judgement::Unreadable => { + self.unread_in_a_row += 1; + (self.last_reading.is_none() + && self.unread_in_a_row >= UNREAD_SAMPLES_BEFORE_GIVING_UP) + .then(|| self.unreadable()) + } + } + } + + /// The verdict when the deadline passes: not drained if the last sample was + /// a reading, unreadable if it was not. + #[must_use] + pub const fn timed_out(&self) -> Verdict { + match self.last_reading { + Some(target) if self.unread_in_a_row == 0 => Verdict::NotDrained(target), + _ => self.unreadable(), + } + } + + const fn unreadable(&self) -> Verdict { + Verdict::Unreadable { + last_reading: self.last_reading, + samples: self.unread_in_a_row, + } + } +} + +/// The one-line `device state:` report for a verdict, after `elapsed_secs` of waiting. +/// +/// It says only what the harness observed: "NOT drained" is for a reading below +/// its floor, never for a sample that gave none, and a carveout-sized VRAM pool +/// judged without GTT data says why that cannot show whether the engine's +/// memory came back. +#[must_use] +pub fn describe_verdict(verdict: Verdict, elapsed_secs: u64) -> String { + match verdict { + Verdict::NoTooling => "device state: no GPU tooling (mock/local run)".to_owned(), + Verdict::Drained(target) => format!("device state: drained ({})", target.describe()), + Verdict::NotDrained(target) => { + let hint = if target.kind == PoolKind::Vram && is_carveout(target.total_mib) { + "; only the BIOS carveout was read because no GTT reading was available (is \ + rocm-smi installed?), so this wait could not follow the memory the engine uses" + } else { + "" + }; + format!( + "device state: NOT drained after {elapsed_secs}s ({}) — a previous engine may \ + still be holding the GPU{hint}", + target.describe() + ) + } + Verdict::Unreadable { + last_reading: None, + samples, + } => format!( + "device state: could not read device memory ({samples} samples in {elapsed_secs}s; \ + the GPU tool is installed but gave no usable reading)" + ), + Verdict::Unreadable { + last_reading: Some(target), + .. + } => format!( + "device state: could not read device memory at the end of {elapsed_secs}s (last \ + reading: {})", + target.describe() + ), + } +} + /// Choose the pool the engine allocates from and the floor to hold it to. /// /// A VRAM total that [`is_carveout`] means the GTT pool is used, when one was @@ -599,6 +744,195 @@ mod tests { ); } + /// A Strix Halo host whose GTT pool has only `free_mib` MiB free of 102 400. + fn strix_holding(free_mib: u64) -> DeviceMemory { + DeviceMemory { + gtt: Some(Pool { + total_mib: 102_400, + free_mib, + }), + ..strix_halo() + } + } + + /// A sample whose GTT pool is nearly all held: below its 92 160 MiB floor. + fn undrained() -> Judgement { + Judgement::Target(select_drain_target(strix_holding(1_000))) + } + + /// The healthy Strix Halo reading: above its floor. + fn drained() -> Judgement { + Judgement::Target(select_drain_target(strix_halo())) + } + + #[test] + fn a_target_is_described_with_its_pool_and_only_gtt_is_named() { + assert_eq!( + select_drain_target(strix_halo()).describe(), + "102360 MiB free of 102400 MiB GTT, floor 92160 MiB" + ); + assert_eq!( + select_drain_target(mi300x()).describe(), + "196309 MiB free of 196592 MiB, floor 150000 MiB" + ); + } + + #[test] + fn a_drained_sample_ends_the_wait_and_the_line_names_the_pool_it_read() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + let verdict = wait.observe(drained()).expect("a drained sample ends it"); + assert_eq!(verdict, Verdict::Drained(select_drain_target(strix_halo()))); + assert_eq!( + describe_verdict(verdict, 6), + "device state: drained (102360 MiB free of 102400 MiB GTT, floor 92160 MiB)" + ); + } + + #[test] + fn no_tooling_ends_the_wait_on_the_first_sample_only() { + assert_eq!( + DrainWait::default().observe(Judgement::NoTooling), + Some(Verdict::NoTooling) + ); + assert_eq!( + describe_verdict(Verdict::NoTooling, 0), + "device state: no GPU tooling (mock/local run)" + ); + // A tool that disappears mid-wait is a sample nobody could read, not a + // reason to report a mock run. + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + for _ in 0..UNREAD_SAMPLES_BEFORE_GIVING_UP * 2 { + assert_eq!(wait.observe(Judgement::NoTooling), None); + } + } + + #[test] + fn a_tool_that_never_reads_is_given_up_on_and_called_unreadable_not_undrained() { + let mut wait = DrainWait::default(); + for _ in 1..UNREAD_SAMPLES_BEFORE_GIVING_UP { + assert_eq!(wait.observe(Judgement::Unreadable), None); + } + let verdict = wait + .observe(Judgement::Unreadable) + .expect("the last allowed unread sample ends the wait"); + assert_eq!( + verdict, + Verdict::Unreadable { + last_reading: None, + samples: UNREAD_SAMPLES_BEFORE_GIVING_UP, + } + ); + let line = describe_verdict(verdict, 24); + assert_eq!( + line, + "device state: could not read device memory (3 samples in 24s; the GPU tool is \ + installed but gave no usable reading)" + ); + assert!(!line.contains("NOT drained"), "{line}"); + } + + #[test] + fn a_tool_that_has_read_is_waited_on_through_unreadable_samples() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(Judgement::Unreadable), None); + assert_eq!(wait.observe(Judgement::Unreadable), None); + assert_eq!(wait.observe(undrained()), None); + for _ in 0..UNREAD_SAMPLES_BEFORE_GIVING_UP * 3 { + assert_eq!(wait.observe(Judgement::Unreadable), None); + } + let verdict = wait.timed_out(); + assert_eq!( + verdict, + Verdict::Unreadable { + last_reading: Some(select_drain_target(strix_holding(1_000))), + samples: UNREAD_SAMPLES_BEFORE_GIVING_UP * 3, + } + ); + assert_eq!( + describe_verdict(verdict, 120), + "device state: could not read device memory at the end of 120s (last reading: \ + 1000 MiB free of 102400 MiB GTT, floor 92160 MiB)" + ); + } + + #[test] + fn the_deadline_after_a_reading_is_not_drained_and_after_a_miss_it_is_not() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + let target = select_drain_target(strix_holding(1_000)); + assert_eq!(wait.timed_out(), Verdict::NotDrained(target)); + assert_eq!( + describe_verdict(wait.timed_out(), 120), + "device state: NOT drained after 120s (1000 MiB free of 102400 MiB GTT, floor \ + 92160 MiB) — a previous engine may still be holding the GPU" + ); + // The most recent sample was one nobody could read: that is not a claim + // about the device. + assert_eq!(wait.observe(Judgement::Unreadable), None); + assert_eq!( + wait.timed_out(), + Verdict::Unreadable { + last_reading: Some(target), + samples: 1, + } + ); + } + + #[test] + fn an_undrained_carveout_judged_without_gtt_says_it_could_not_follow_the_engines_memory() { + // The real assembly path: amd-smi read the carveout, no tool reports GTT. + let (memory, _) = sample(Probe::Read((512, 318)), Probe::Absent); + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(judge(memory)), None); + let line = describe_verdict(wait.timed_out(), 120); + assert!( + line.starts_with( + "device state: NOT drained after 120s (318 MiB free of 512 MiB, floor 450 MiB)" + ), + "{line}" + ); + assert!( + line.contains( + "only the BIOS carveout was read because no GTT reading was available (is \ + rocm-smi installed?)" + ), + "{line}" + ); + // No such note when the pool judged is the one the engine uses. + for held in [ + Verdict::NotDrained(select_drain_target(strix_holding(1_000))), + Verdict::NotDrained(select_drain_target(DeviceMemory { + vram: Pool { + total_mib: 196_592, + free_mib: 1_000, + }, + gtt: None, + })), + ] { + let line = describe_verdict(held, 120); + assert!(!line.contains("carveout"), "{line}"); + } + } + + #[test] + fn a_probe_gets_the_probe_timeout_or_the_time_left_whichever_is_less() { + let now = Instant::now(); + assert_eq!( + probe_budget(now, now + Duration::from_secs(100)), + SMI_PROBE_TIMEOUT + ); + assert_eq!( + probe_budget(now, now + Duration::from_secs(4)), + Duration::from_secs(4) + ); + assert_eq!( + probe_budget(now + Duration::from_secs(5), now), + Duration::ZERO + ); + } + #[cfg(unix)] fn shell(script: &str, args: &[&str]) -> Command { let mut command = Command::new("sh"); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c218732abb..2a5f9a867a 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -12,8 +12,8 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::device_memory::{ - DeviceMemory, DrainTarget, Judgement, PoolKind, Probe, SMI_PROBE_TIMEOUT, bounded_stdout, - judge, parse_rocm_smi_csv_pool, sample_device_memory, + DeviceMemory, DrainWait, MIN_PROBE_BUDGET, Probe, bounded_stdout, describe_verdict, judge, + parse_rocm_smi_csv_pool, probe_budget, sample_device_memory, }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ @@ -263,6 +263,9 @@ async fn ensure_serve_port_free() -> String { /// How long to wait for a stopped engine to hand its device memory back. const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); +/// Pause between two samples of the device. +const DRAIN_POLL_INTERVAL: Duration = Duration::from_secs(3); + /// Best-effort: wait until the pool the engine allocates from reports enough /// free memory (see [`select_drain_target`]: VRAM on a discrete card, GTT on an /// APU whose VRAM is only a BIOS carveout), so a just-killed serve's memory is @@ -272,63 +275,41 @@ const VRAM_DRAIN_DEADLINE: Duration = Duration::from_mins(2); /// /// The wait is bounded and best-effort: on timeout the serve still starts, /// because a stale reading must not turn a slow drain into a hard failure. The -/// returned line records which of the two happened — an undrained device is the -/// single most likely reason the serve that follows never becomes ready, and -/// without it the failure looks identical to a genuinely broken serve. +/// returned line records what happened — an undrained device is the single most +/// likely reason the serve that follows never becomes ready, and without it the +/// failure looks identical to a genuinely broken serve. A tool that is installed +/// but never produces a reading is given up on after a few samples +/// (`UNREAD_SAMPLES_BEFORE_GIVING_UP`) and reported as unreadable, not as +/// undrained; the decisions and the wording live in `DrainWait` and +/// `describe_verdict`, where they are tested. async fn wait_for_free_vram() -> String { // Started before the first probe so that probe counts against the wait. - let deadline = Instant::now() + VRAM_DRAIN_DEADLINE; - let mut judgement = judge(probe_device_memory().await); - // No GPU tooling installed → nothing to wait on (mock/local). A tool that is - // present but gave no reading is a different case: it keeps waiting, and - // says so if it never reads. - if judgement == Judgement::NoTooling { - return "device state: no GPU tooling (mock/local run)".to_owned(); - } + let started = Instant::now(); + let deadline = started + VRAM_DRAIN_DEADLINE; + let mut wait = DrainWait::default(); loop { - let target = match judgement { - Judgement::Target(target) => Some(target), - Judgement::NoTooling | Judgement::Unreadable => None, - }; - if let Some(target) = target - && target.drained() - { - return format!("device state: drained ({})", describe_target(target)); + // Every probe is held to the time left, so the wait ends at its deadline + // rather than a few probe timeouts after it. + let sample = judge(probe_device_memory(deadline).await); + if let Some(verdict) = wait.observe(sample) { + return describe_verdict(verdict, started.elapsed().as_secs()); } - if Instant::now() >= deadline { - let held = target.map_or_else( - || "the memory reading was unreadable".to_owned(), - describe_target, - ); - return format!( - "device state: NOT drained after {}s ({held}) — a previous engine may still \ - be holding the GPU", - VRAM_DRAIN_DEADLINE.as_secs() - ); + let left = deadline.saturating_duration_since(Instant::now()); + if left < MIN_PROBE_BUDGET { + return describe_verdict(wait.timed_out(), started.elapsed().as_secs()); + } + tokio::time::sleep(left.min(DRAIN_POLL_INTERVAL)).await; + if deadline.saturating_duration_since(Instant::now()) < MIN_PROBE_BUDGET { + return describe_verdict(wait.timed_out(), started.elapsed().as_secs()); } - tokio::time::sleep(Duration::from_secs(3)).await; - judgement = judge(probe_device_memory().await); } } -/// `" MiB free of MiB[ GTT], floor MiB"`. VRAM is the -/// unlabelled default, so the line keeps the shape it always had on discrete -/// cards; only the GTT pool is named. -fn describe_target(target: DrainTarget) -> String { - let pool = match target.kind { - PoolKind::Vram => "", - PoolKind::Gtt => " GTT", - }; - format!( - "{} MiB free of {} MiB{pool}, floor {} MiB", - target.free_mib, target.total_mib, target.floor_mib - ) -} - /// [`device_memory_mib`] off the async worker: it runs subprocesses, each bounded -/// by [`SMI_PROBE_TIMEOUT`], so the blocking thread it occupies is released. -async fn probe_device_memory() -> Probe { - tokio::task::spawn_blocking(device_memory_mib) +/// by `probe_budget` (at most the probe timeout, never past `deadline`), so the +/// blocking thread it occupies is released. +async fn probe_device_memory(deadline: Instant) -> Probe { + tokio::task::spawn_blocking(move || device_memory_mib(deadline)) .await .unwrap_or(Probe::Unreadable) } @@ -339,27 +320,27 @@ async fn probe_device_memory() -> Probe { /// for GTT: its output has no field this harness has been checked against, /// whereas the preflight step already reads `rocm-smi`'s GTT rows on these /// hosts. -fn device_memory_mib() -> Probe { - sample_device_memory(vram_mib, gtt_mib) +fn device_memory_mib(deadline: Instant) -> Probe { + sample_device_memory(|| vram_mib(deadline), || gtt_mib(deadline)) } /// `(total, free)` GTT in MiB for GPU 0, via `rocm-smi`. -fn gtt_mib() -> Probe<(u64, u64)> { +fn gtt_mib(deadline: Instant) -> Probe<(u64, u64)> { use std::process::Command; let mut command = Command::new("rocm-smi"); command.args(["--showmeminfo", "gtt", "--csv"]); - bounded_stdout(command, SMI_PROBE_TIMEOUT) + bounded_stdout(command, probe_budget(Instant::now(), deadline)) .parse(|text| parse_rocm_smi_csv_pool(&text, "GTT Total Memory", "GTT Total Used Memory")) } /// `(total, free)` device VRAM in MiB for GPU 0, via `amd-smi` (then `rocm-smi`). -fn vram_mib() -> Probe<(u64, u64)> { +fn vram_mib(deadline: Instant) -> Probe<(u64, u64)> { use std::process::Command; // amd-smi: lines like " TOTAL_VRAM: 196592 MB" / "FREE_VRAM: 196309 MB". let amd_smi = || { let mut command = Command::new("amd-smi"); command.args(["metric", "-m"]); - bounded_stdout(command, SMI_PROBE_TIMEOUT).parse(|text| { + bounded_stdout(command, probe_budget(Instant::now(), deadline)).parse(|text| { let field = |name: &str| { text.lines() .find_map(|l| l.trim().strip_prefix(name)) @@ -374,7 +355,7 @@ fn vram_mib() -> Probe<(u64, u64)> { let mut command = Command::new("rocm-smi"); command.args(["--showmeminfo", "vram", "--csv"]); // CSV columns include "VRAM Total Memory (B)" and "VRAM Total Used Memory (B)". - bounded_stdout(command, SMI_PROBE_TIMEOUT) + bounded_stdout(command, probe_budget(Instant::now(), deadline)) .parse(|text| parse_rocm_smi_csv_pool(&text, "Total Memory", "Total Used Memory")) }; amd_smi().or_else(rocm_smi) From 61e80b46ea03dbbc60733813816ce07d455bb5b2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 15:47:30 +0000 Subject: [PATCH 09/13] test(e2e): show what a GPU tool printed when its output is not understood, and test the amd-smi parser A second review of the drain wait asked for two things. The `rocm-smi --showmeminfo gtt --csv` column names the Strix Halo fix relies on are only confirmed against the text output the preflight script parses. If they are wrong every sample on a carveout host is unreadable and the wait gives up after three samples with only a generic message, so the fix would silently do nothing. A probe whose output the parser rejects is now `Unparsed`, carrying ` printed ""` (the header row, printable ASCII only, cut to 120 characters). It reaches the final `device state:` line through `Judgement::Unparsed`, the wait's decision state and `Verdict::Unreadable`, is kept across later plain failures, and is dropped when a reading arrives. The amd-smi parser had been moved into a closure with no test. It is now `parse_amd_smi_vram` beside `parse_rocm_smi_csv_pool`, parses exactly as before, and has tests for both fields, a missing field, a zero total, non-numeric values, the first of two GPUs and the visible-VRAM lines it must not read. The probes now return a named `Pool` instead of a bare `(u64, u64)`, and `device_memory_mib`, a one-line delegate, is inlined into `probe_device_memory`. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 433 +++++++++++++++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 62 ++- 2 files changed, 410 insertions(+), 85 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 79f62c28ff..6d73805e54 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -64,12 +64,16 @@ pub const fn is_carveout(vram_total_mib: u64) -> bool { /// but stalled, failed, or printed something unparseable is a device that /// could not be read right now, which must never be mistaken for either "no /// tooling" or "drained". -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Probe { /// The tool is not installed. Absent, /// The tool is there but gave no usable reading. Unreadable, + /// The tool ran and exited successfully, but its output was not understood. + /// Like `Unreadable` for every decision, it also carries what the tool + /// printed (see [`output_hint`]) so a log can show what the parser met. + Unparsed(String), Read(T), } @@ -80,6 +84,7 @@ impl Probe { match self { Self::Absent => Probe::Absent, Self::Unreadable => Probe::Unreadable, + Self::Unparsed(hint) => Probe::Unparsed(hint), Self::Read(value) => f(value).map_or(Probe::Unreadable, Probe::Read), } } @@ -87,8 +92,10 @@ impl Probe { /// Fall back to `next` when this probe produced no reading. /// /// A reading from either wins. Without one, the result is `Absent` only if - /// every tool tried was absent; if any tool was present and gave nothing, - /// it is `Unreadable`. + /// every tool tried was absent; if either tool ran and printed something not + /// understood the result keeps what it printed (`Unparsed`, the first tool's + /// if both); otherwise any tool that was present and gave nothing makes it + /// `Unreadable`. #[must_use] pub fn or_else(self, next: impl FnOnce() -> Self) -> Self { if matches!(self, Self::Read(_)) { @@ -97,11 +104,64 @@ impl Probe { match (self, next()) { (_, read @ Self::Read(_)) => read, (Self::Absent, Self::Absent) => Self::Absent, + (Self::Unparsed(hint), _) | (_, Self::Unparsed(hint)) => Self::Unparsed(hint), _ => Self::Unreadable, } } } +impl Probe { + /// Parse a tool's stdout with `f`. Output `f` rejects is not `Unreadable` + /// but `Unparsed`, carrying the first line of it (see [`output_hint`]) so the + /// log says what the tool printed; `source` names the command. + #[must_use] + pub fn parse_text(self, source: &str, f: impl FnOnce(&str) -> Option) -> Probe { + match self { + Self::Absent => Probe::Absent, + Self::Unreadable => Probe::Unreadable, + Self::Unparsed(hint) => Probe::Unparsed(hint), + Self::Read(text) => { + f(&text).map_or_else(|| Probe::Unparsed(output_hint(source, &text)), Probe::Read) + } + } + } +} + +/// The longest first line of tool output [`output_hint`] repeats. +pub const HINT_MAX_CHARS: usize = 120; + +/// What a tool printed, as one bounded, printable line for a log. +/// +/// It is `source`, then the first non-blank line of `text` with anything outside +/// printable ASCII replaced by `?` and cut to [`HINT_MAX_CHARS`] characters. +/// Meant for the header row of a CSV a parser could not match, so the next run +/// shows the real column names. +#[must_use] +pub fn output_hint(source: &str, text: &str) -> String { + let Some(line) = text.lines().map(str::trim).find(|line| !line.is_empty()) else { + return format!("{source} printed no output"); + }; + let clean: String = line + .chars() + .map(|c| { + if c.is_ascii_graphic() || c == ' ' { + c + } else { + '?' + } + }) + .collect(); + let shown = if clean.chars().count() > HINT_MAX_CHARS { + format!( + "{}...", + clean.chars().take(HINT_MAX_CHARS).collect::() + ) + } else { + clean + }; + format!("{source} printed {shown:?}") +} + /// One memory pool's size and free space, in MiB. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Pool { @@ -109,6 +169,16 @@ pub struct Pool { pub free_mib: u64, } +impl Pool { + #[must_use] + pub const fn new(total_mib: u64, free_mib: u64) -> Self { + Self { + total_mib, + free_mib, + } + } +} + /// What the device tooling reported for GPU 0. /// /// `gtt` is `None` when GTT was not read: a discrete card, or a host with no @@ -169,13 +239,16 @@ pub fn required_free_mib(total_mib: u64) -> u64 { } /// What one sample means for the drain wait. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Judgement { /// No device tool is installed: nothing to wait on (mock/local run). NoTooling, /// A tool is present but gave no usable reading. This is not "no tooling" /// and not "drained": the wait keeps trying. Unreadable, + /// As `Unreadable`, and the tool's output was not understood: what it + /// printed, for the report (see [`output_hint`]). + Unparsed(String), /// A reading, and the pool and floor it is held to. Target(DrainTarget), } @@ -186,6 +259,7 @@ pub fn judge(sample: Probe) -> Judgement { match sample { Probe::Absent => Judgement::NoTooling, Probe::Unreadable => Judgement::Unreadable, + Probe::Unparsed(hint) => Judgement::Unparsed(hint), Probe::Read(memory) => Judgement::Target(select_drain_target(memory)), } } @@ -213,7 +287,7 @@ pub fn probe_budget(now: Instant, deadline: Instant) -> Duration { } /// How the drain wait ended. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Verdict { /// No device tool is installed (mock/local run): nothing was waited on. NoTooling, @@ -222,11 +296,14 @@ pub enum Verdict { /// The deadline passed with the pool still below its floor. NotDrained(DrainTarget), /// No usable reading at the end: `samples` samples in a row gave none, and - /// `last_reading` is the most recent one the wait did get, if any. This is - /// not "not drained": the harness did not observe that. + /// `last_reading` is the most recent one the wait did get, if any. `hint` is + /// what a tool printed that was not understood, when one did (the latest + /// such sample, kept until a reading arrives). This is not "not drained": + /// the harness did not observe that. Unreadable { last_reading: Option, samples: u32, + hint: Option, }, } @@ -236,6 +313,7 @@ pub struct DrainWait { samples: u32, unread_in_a_row: u32, last_reading: Option, + hint: Option, } impl DrainWait { @@ -248,32 +326,40 @@ impl DrainWait { Judgement::Target(target) => { self.unread_in_a_row = 0; self.last_reading = Some(target); + // A reading shows the tool's output is understood after all. + self.hint = None; target.drained().then_some(Verdict::Drained(target)) } - // A tool that vanishes mid-wait is a sample nobody could read. - Judgement::NoTooling | Judgement::Unreadable => { - self.unread_in_a_row += 1; - (self.last_reading.is_none() - && self.unread_in_a_row >= UNREAD_SAMPLES_BEFORE_GIVING_UP) - .then(|| self.unreadable()) + Judgement::Unparsed(hint) => { + self.hint = Some(hint); + self.unread() } + // A tool that vanishes mid-wait is a sample nobody could read. + Judgement::NoTooling | Judgement::Unreadable => self.unread(), } } + fn unread(&mut self) -> Option { + self.unread_in_a_row += 1; + (self.last_reading.is_none() && self.unread_in_a_row >= UNREAD_SAMPLES_BEFORE_GIVING_UP) + .then(|| self.unreadable()) + } + /// The verdict when the deadline passes: not drained if the last sample was /// a reading, unreadable if it was not. #[must_use] - pub const fn timed_out(&self) -> Verdict { + pub fn timed_out(&self) -> Verdict { match self.last_reading { Some(target) if self.unread_in_a_row == 0 => Verdict::NotDrained(target), _ => self.unreadable(), } } - const fn unreadable(&self) -> Verdict { + fn unreadable(&self) -> Verdict { Verdict::Unreadable { last_reading: self.last_reading, samples: self.unread_in_a_row, + hint: self.hint.clone(), } } } @@ -305,21 +391,30 @@ pub fn describe_verdict(verdict: Verdict, elapsed_secs: u64) -> String { Verdict::Unreadable { last_reading: None, samples, + hint, } => format!( "device state: could not read device memory ({samples} samples in {elapsed_secs}s; \ - the GPU tool is installed but gave no usable reading)" + the GPU tool is installed but gave no usable reading{})", + hint_suffix(hint.as_deref()) ), Verdict::Unreadable { last_reading: Some(target), + hint, .. } => format!( "device state: could not read device memory at the end of {elapsed_secs}s (last \ - reading: {})", - target.describe() + reading: {}{})", + target.describe(), + hint_suffix(hint.as_deref()) ), } } +/// `"; "` for a report line, or nothing without one. +fn hint_suffix(hint: Option<&str>) -> String { + hint.map_or_else(String::new, |hint| format!("; {hint}")) +} + /// Choose the pool the engine allocates from and the floor to hold it to. /// /// A VRAM total that [`is_carveout`] means the GTT pool is used, when one was @@ -346,11 +441,7 @@ pub fn select_drain_target(memory: DeviceMemory) -> DrainTarget { /// names (`Total Memory` and `Total Used Memory` for VRAM; the `GTT ` forms for /// GTT); the first data row is the first GPU. Values are bytes in the output. #[must_use] -pub fn parse_rocm_smi_csv_pool( - text: &str, - total_column: &str, - used_column: &str, -) -> Option<(u64, u64)> { +pub fn parse_rocm_smi_csv_pool(text: &str, total_column: &str, used_column: &str) -> Option { let mut lines = text.lines(); let header = lines.next()?; let cols: Vec<&str> = header.split(',').collect(); @@ -361,7 +452,24 @@ pub fn parse_rocm_smi_csv_pool( let total: u64 = vals.get(total_idx)?.trim().parse().ok()?; let used: u64 = vals.get(used_idx)?.trim().parse().ok()?; let mib = 1024 * 1024; - Some((total / mib, total.saturating_sub(used) / mib)) + Some(Pool::new(total / mib, total.saturating_sub(used) / mib)) +} + +/// Read VRAM from `amd-smi metric -m` output. +/// +/// The `TOTAL_VRAM:` and `FREE_VRAM:` lines look like `TOTAL_VRAM: 196592 MB`; +/// the number in front of the unit is taken as MiB. The first GPU's lines win. A +/// zero total parses here; it is [`sample_device_memory`] that treats it as a +/// failed read. +#[must_use] +pub fn parse_amd_smi_vram(text: &str) -> Option { + let field = |name: &str| { + text.lines() + .find_map(|line| line.trim().strip_prefix(name)) + .and_then(|value| value.split_whitespace().next()) + .and_then(|number| number.parse::().ok()) + }; + Some(Pool::new(field("TOTAL_VRAM:")?, field("FREE_VRAM:")?)) } /// Assemble one device-memory sample from a VRAM probe and a GTT probe. @@ -373,39 +481,35 @@ pub fn parse_rocm_smi_csv_pool( /// - a GTT reading makes the sample; /// - a GTT probe that ran but gave nothing usable (a stall, a failure, output /// not understood, or a zero total, which is a failed read rather than an -/// empty pool) makes the whole sample `Unreadable`. Judging the tiny +/// empty pool) makes the whole sample `Unreadable` (`Unparsed`, with what the +/// tool printed, when it ran but its output was not understood). Judging the tiny /// carveout instead could report "drained" while the engine still holds its /// GTT-backed memory. `gpu-preflight.sh` treats a zero GTT total the same way, /// as an unreadable sample to retry; /// - no tool that reports GTT at all (`Absent`) leaves `gtt` empty, and the /// wait judges VRAM as it did before GTT was considered. pub fn sample_device_memory( - vram: impl FnOnce() -> Probe<(u64, u64)>, - gtt: impl FnOnce() -> Probe<(u64, u64)>, + vram: impl FnOnce() -> Probe, + gtt: impl FnOnce() -> Probe, ) -> Probe { - let (vram_total, vram_free) = match vram() { + let vram = match vram() { // A zero total is a failed read, not a pool: its floor would be zero and // every sample would count as drained. - Probe::Read((total, free)) if total > 0 => (total, free), + Probe::Read(pool) if pool.total_mib > 0 => pool, Probe::Read(_) | Probe::Unreadable => return Probe::Unreadable, + Probe::Unparsed(hint) => return Probe::Unparsed(hint), Probe::Absent => return Probe::Absent, }; - let vram = Pool { - total_mib: vram_total, - free_mib: vram_free, - }; - if !is_carveout(vram_total) { + if !is_carveout(vram.total_mib) { return Probe::Read(DeviceMemory { vram, gtt: None }); } match gtt() { - Probe::Read((total_mib, free_mib)) if total_mib > 0 => Probe::Read(DeviceMemory { + Probe::Read(pool) if pool.total_mib > 0 => Probe::Read(DeviceMemory { vram, - gtt: Some(Pool { - total_mib, - free_mib, - }), + gtt: Some(pool), }), Probe::Read(_) | Probe::Unreadable => Probe::Unreadable, + Probe::Unparsed(hint) => Probe::Unparsed(hint), Probe::Absent => Probe::Read(DeviceMemory { vram, gtt: None }), } } @@ -582,7 +686,7 @@ mod tests { /// Run `sample_device_memory` with canned probes, reporting whether the GTT /// probe was consulted. - fn sample(vram: Probe<(u64, u64)>, gtt: Probe<(u64, u64)>) -> (Probe, bool) { + fn sample(vram: Probe, gtt: Probe) -> (Probe, bool) { let mut gtt_asked = false; let result = sample_device_memory( || vram, @@ -596,7 +700,10 @@ mod tests { #[test] fn a_carveout_host_reads_gtt_and_the_sample_carries_it() { - let (result, gtt_asked) = sample(Probe::Read((512, 318)), Probe::Read((102_400, 102_360))); + let (result, gtt_asked) = sample( + Probe::Read(Pool::new(512, 318)), + Probe::Read(Pool::new(102_400, 102_360)), + ); assert!(gtt_asked, "a carveout host must have its GTT read"); assert_eq!(result, Probe::Read(strix_halo())); } @@ -604,8 +711,8 @@ mod tests { #[test] fn a_discrete_host_never_makes_the_gtt_call() { let (result, gtt_asked) = sample( - Probe::Read((196_592, 196_309)), - Probe::Read((1_000_000, 1_000_000)), + Probe::Read(Pool::new(196_592, 196_309)), + Probe::Read(Pool::new(1_000_000, 1_000_000)), ); assert!( !gtt_asked, @@ -622,14 +729,17 @@ mod tests { fn a_carveout_host_whose_gtt_probe_failed_is_unreadable_not_judged_by_vram() { // VRAM alone would read as "drained" here (free is above its floor), so // falling back to it would hide a GTT-backed allocation still held. - let (result, gtt_asked) = sample(Probe::Read((512, 500)), Probe::Unreadable); + let (result, gtt_asked) = sample(Probe::Read(Pool::new(512, 500)), Probe::Unreadable); assert!(gtt_asked); assert_eq!(result, Probe::Unreadable); } #[test] fn a_zero_gtt_total_is_a_failed_reading_not_a_pool() { - let (result, _) = sample(Probe::Read((512, 500)), Probe::Read((0, 0))); + let (result, _) = sample( + Probe::Read(Pool::new(512, 500)), + Probe::Read(Pool::new(0, 0)), + ); assert_eq!(result, Probe::Unreadable); } @@ -637,14 +747,14 @@ mod tests { fn a_zero_vram_total_is_a_failed_reading_not_a_drained_host() { // With no GTT tool the sample would otherwise be judged by a VRAM pool // whose floor is zero, which is drained by definition. - let (result, gtt_asked) = sample(Probe::Read((0, 0)), Probe::Absent); + let (result, gtt_asked) = sample(Probe::Read(Pool::new(0, 0)), Probe::Absent); assert_eq!(result, Probe::Unreadable); assert!(!gtt_asked); } #[test] fn a_carveout_host_with_no_gtt_tool_keeps_the_vram_behaviour() { - let (result, gtt_asked) = sample(Probe::Read((512, 318)), Probe::Absent); + let (result, gtt_asked) = sample(Probe::Read(Pool::new(512, 318)), Probe::Absent); assert!(gtt_asked); let Probe::Read(memory) = result else { panic!("a host without a GTT tool still has a VRAM sample: {result:?}"); @@ -656,11 +766,11 @@ mod tests { #[test] fn a_vram_probe_that_is_absent_or_unreadable_is_passed_through_without_asking_for_gtt() { assert_eq!( - sample(Probe::Absent, Probe::Read((102_400, 102_360))), + sample(Probe::Absent, Probe::Read(Pool::new(102_400, 102_360))), (Probe::Absent, false) ); assert_eq!( - sample(Probe::Unreadable, Probe::Read((102_400, 102_360))), + sample(Probe::Unreadable, Probe::Read(Pool::new(102_400, 102_360))), (Probe::Unreadable, false) ); } @@ -716,7 +826,7 @@ mod tests { card0,107374182400,41787392\n"; assert_eq!( parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), - Some((102_400, 102_360)) + Some(Pool::new(102_400, 102_360)) ); } @@ -726,7 +836,7 @@ mod tests { card0,536870912,203321344\n"; assert_eq!( parse_rocm_smi_csv_pool(text, "Total Memory", "Total Used Memory"), - Some((512, 318)) + Some(Pool::new(512, 318)) ); } @@ -822,6 +932,7 @@ mod tests { Verdict::Unreadable { last_reading: None, samples: UNREAD_SAMPLES_BEFORE_GIVING_UP, + hint: None, } ); let line = describe_verdict(verdict, 24); @@ -848,6 +959,7 @@ mod tests { Verdict::Unreadable { last_reading: Some(select_drain_target(strix_holding(1_000))), samples: UNREAD_SAMPLES_BEFORE_GIVING_UP * 3, + hint: None, } ); assert_eq!( @@ -876,6 +988,7 @@ mod tests { Verdict::Unreadable { last_reading: Some(target), samples: 1, + hint: None, } ); } @@ -883,7 +996,7 @@ mod tests { #[test] fn an_undrained_carveout_judged_without_gtt_says_it_could_not_follow_the_engines_memory() { // The real assembly path: amd-smi read the carveout, no tool reports GTT. - let (memory, _) = sample(Probe::Read((512, 318)), Probe::Absent); + let (memory, _) = sample(Probe::Read(Pool::new(512, 318)), Probe::Absent); let mut wait = DrainWait::default(); assert_eq!(wait.observe(judge(memory)), None); let line = describe_verdict(wait.timed_out(), 120); @@ -916,6 +1029,222 @@ mod tests { } } + /// What `amd-smi metric -m` prints, per GPU, with the lines the parser reads + /// among others it must not confuse with them. + const AMD_SMI_GPU0: &str = "GPU: 0\n MEM_USAGE:\n TOTAL_VRAM: 196592 MB\n \ + USED_VRAM: 283 MB\n FREE_VRAM: 196309 MB\n \ + TOTAL_VISIBLE_VRAM: 196592 MB\n FREE_VISIBLE_VRAM: 196309 MB\n"; + + #[test] + fn amd_smi_vram_reads_the_total_and_free_lines_in_mib() { + assert_eq!( + parse_amd_smi_vram(AMD_SMI_GPU0), + Some(Pool::new(196_592, 196_309)) + ); + } + + #[test] + fn amd_smi_vram_takes_the_first_gpu_and_ignores_indentation_and_units() { + let two = format!("{AMD_SMI_GPU0}GPU: 1\n TOTAL_VRAM: 1 MB\n FREE_VRAM: 2 MB\n"); + assert_eq!(parse_amd_smi_vram(&two), Some(Pool::new(196_592, 196_309))); + // Tabs, wide gaps, any unit word and a missing space after the colon all + // read the same way: the number after the name is the value. + assert_eq!( + parse_amd_smi_vram("\tTOTAL_VRAM: 512 MiB\n FREE_VRAM:318\n"), + Some(Pool::new(512, 318)) + ); + assert_eq!( + parse_amd_smi_vram("TOTAL_VRAM: 512\r\nFREE_VRAM: 318 MB\r\n"), + Some(Pool::new(512, 318)) + ); + } + + #[test] + fn amd_smi_vram_needs_both_fields_as_numbers() { + assert_eq!(parse_amd_smi_vram("TOTAL_VRAM: 512 MB\n"), None); + assert_eq!(parse_amd_smi_vram("FREE_VRAM: 318 MB\n"), None); + assert_eq!( + parse_amd_smi_vram("TOTAL_VRAM: N/A\nFREE_VRAM: 318 MB\n"), + None + ); + assert_eq!(parse_amd_smi_vram("no such fields here"), None); + assert_eq!(parse_amd_smi_vram(""), None); + // The visible-VRAM lines are different fields, not the ones asked for. + assert_eq!( + parse_amd_smi_vram("TOTAL_VISIBLE_VRAM: 512 MB\nFREE_VISIBLE_VRAM: 318 MB\n"), + None + ); + } + + #[test] + fn a_zero_amd_smi_total_parses_and_is_then_rejected_as_a_failed_read() { + let zero = "TOTAL_VRAM: 0 MB\nFREE_VRAM: 0 MB\n"; + let parsed = parse_amd_smi_vram(zero).expect("parses"); + assert_eq!(parsed, Pool::new(0, 0)); + let (result, gtt_asked) = sample(Probe::Read(parsed), Probe::Absent); + assert_eq!(result, Probe::Unreadable); + assert!(!gtt_asked); + } + + #[test] + fn output_hint_is_the_first_nonblank_line_bounded_and_printable() { + assert_eq!( + output_hint( + "rocm-smi gtt", + "\n \ndevice,VRAM Total Memory (B)\ncard0,1\n" + ), + "rocm-smi gtt printed \"device,VRAM Total Memory (B)\"" + ); + assert_eq!(output_hint("tool", ""), "tool printed no output"); + assert_eq!(output_hint("tool", " \n\t\n"), "tool printed no output"); + // Non-ASCII and control characters are replaced, not repeated. + assert_eq!( + output_hint("tool", "GTT \u{1b}[31mT\u{f6}tal\u{0}"), + "tool printed \"GTT ?[31mT?tal?\"" + ); + // Quotes and backslashes are escaped by the quoting. + assert_eq!( + output_hint("tool", "a\"b\\c"), + "tool printed \"a\\\"b\\\\c\"" + ); + let long = "x".repeat(HINT_MAX_CHARS + 50); + let hint = output_hint("tool", &long); + assert_eq!( + hint, + format!("tool printed \"{}...\"", "x".repeat(HINT_MAX_CHARS)) + ); + let exact = "y".repeat(HINT_MAX_CHARS); + assert_eq!( + output_hint("tool", &exact), + format!("tool printed \"{exact}\"") + ); + } + + #[test] + fn output_a_parser_rejects_is_unparsed_with_what_it_printed_not_plain_unreadable() { + let header = "device,GTT Total (B),GTT Used (B)\ncard0,107374182400,41787392\n"; + let probe = Probe::Read(header.to_owned()).parse_text("rocm-smi gtt", |text| { + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory") + }); + assert_eq!( + probe, + Probe::Unparsed( + "rocm-smi gtt printed \"device,GTT Total (B),GTT Used (B)\"".to_owned() + ) + ); + // Right headers still read, and a tool that is absent or failed is not + // given a hint it has not earned. + let good = "device,GTT Total Memory (B),GTT Total Used Memory (B)\ncard0,1048576,0\n"; + assert_eq!( + Probe::Read(good.to_owned()).parse_text("x", |t| { + parse_rocm_smi_csv_pool(t, "GTT Total Memory", "GTT Total Used Memory") + }), + Probe::Read(Pool::new(1, 1)) + ); + assert_eq!( + Probe::::Absent.parse_text("x", |_| Some(0_u8)), + Probe::Absent + ); + assert_eq!( + Probe::::Unreadable.parse_text("x", |_| Some(0_u8)), + Probe::Unreadable + ); + } + + #[test] + fn a_fallback_tool_that_printed_something_unknown_keeps_what_it_printed() { + let unparsed = || Probe::::Unparsed("amd-smi printed \"GPU: 0\"".to_owned()); + assert_eq!( + unparsed().or_else(|| Probe::Absent), + unparsed(), + "an absent fallback leaves the first tool's output visible" + ); + assert_eq!(Probe::Absent.or_else(unparsed), unparsed()); + assert_eq!(Probe::Unreadable.or_else(unparsed), unparsed()); + assert_eq!(unparsed().or_else(|| Probe::Unreadable), unparsed()); + assert_eq!(unparsed().or_else(|| Probe::Read(7)), Probe::Read(7)); + // The first tool's output wins when both printed something. + assert_eq!( + unparsed().or_else(|| Probe::Unparsed("rocm-smi printed \"x\"".to_owned())), + unparsed() + ); + } + + #[test] + fn a_carveout_host_whose_gtt_output_was_not_understood_carries_what_it_printed() { + let hint = "rocm-smi gtt printed \"device,GTT Total (B)\""; + let (result, gtt_asked) = sample( + Probe::Read(Pool::new(512, 318)), + Probe::Unparsed(hint.to_owned()), + ); + assert!(gtt_asked); + assert_eq!(result, Probe::Unparsed(hint.to_owned())); + // And a VRAM probe's unparsed output passes through without asking for GTT. + let (result, gtt_asked) = sample(Probe::Unparsed(hint.to_owned()), Probe::Absent); + assert_eq!(result, Probe::Unparsed(hint.to_owned())); + assert!(!gtt_asked); + assert_eq!(judge(result), Judgement::Unparsed(hint.to_owned())); + } + + #[test] + fn what_a_tool_printed_reaches_the_final_line_when_it_never_reads() { + let hint = "rocm-smi --showmeminfo gtt --csv printed \"device,GTT Total (B)\""; + let mut wait = DrainWait::default(); + for _ in 1..UNREAD_SAMPLES_BEFORE_GIVING_UP { + assert_eq!(wait.observe(Judgement::Unparsed(hint.to_owned())), None); + } + let verdict = wait + .observe(Judgement::Unparsed(hint.to_owned())) + .expect("the third unread sample ends the wait"); + assert_eq!( + describe_verdict(verdict, 9), + "device state: could not read device memory (3 samples in 9s; the GPU tool is \ + installed but gave no usable reading; rocm-smi --showmeminfo gtt --csv printed \ + \"device,GTT Total (B)\")" + ); + } + + #[test] + fn what_a_tool_printed_is_kept_through_later_plain_failures_and_dropped_by_a_reading() { + let hint = "rocm-smi printed \"hdr\""; + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(Judgement::Unparsed(hint.to_owned())), None); + // A later timeout does not erase the evidence of the earlier sample. + assert_eq!(wait.observe(Judgement::Unreadable), None); + let Verdict::Unreadable { hint: kept, .. } = wait.timed_out() else { + panic!("two unread samples are unreadable at the deadline"); + }; + assert_eq!(kept.as_deref(), Some(hint)); + // A reading shows the output is understood after all. + assert_eq!(wait.observe(undrained()), None); + assert_eq!(wait.observe(Judgement::Unreadable), None); + let Verdict::Unreadable { + hint: after, + last_reading, + .. + } = wait.timed_out() + else { + panic!("the last sample was unreadable"); + }; + assert_eq!(after, None); + assert!(last_reading.is_some()); + } + + #[test] + fn a_hint_follows_the_last_reading_on_the_line_when_the_deadline_finds_it_unreadable() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe(undrained()), None); + assert_eq!( + wait.observe(Judgement::Unparsed("t printed \"h\"".to_owned())), + None + ); + assert_eq!( + describe_verdict(wait.timed_out(), 120), + "device state: could not read device memory at the end of 120s (last reading: \ + 1000 MiB free of 102400 MiB GTT, floor 92160 MiB; t printed \"h\")" + ); + } + #[test] fn a_probe_gets_the_probe_timeout_or_the_time_left_whichever_is_less() { let now = Instant::now(); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 2a5f9a867a..2c6ab5e21c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -12,8 +12,8 @@ use cucumber::{given, then, when}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::device_memory::{ - DeviceMemory, DrainWait, MIN_PROBE_BUDGET, Probe, bounded_stdout, describe_verdict, judge, - parse_rocm_smi_csv_pool, probe_budget, sample_device_memory, + DeviceMemory, DrainWait, MIN_PROBE_BUDGET, Pool, Probe, bounded_stdout, describe_verdict, + judge, parse_amd_smi_vram, parse_rocm_smi_csv_pool, probe_budget, sample_device_memory, }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ @@ -305,50 +305,44 @@ async fn wait_for_free_vram() -> String { } } -/// [`device_memory_mib`] off the async worker: it runs subprocesses, each bounded -/// by `probe_budget` (at most the probe timeout, never past `deadline`), so the -/// blocking thread it occupies is released. +/// Device memory for GPU 0 off the async worker: VRAM via `amd-smi` (then +/// `rocm-smi`), plus the GTT pool from `rocm-smi` where the host needs it (see +/// [`sample_device_memory`] for when GTT is read and what a failed read means). +/// `amd-smi` is not asked for GTT: its output has no field this harness has been +/// checked against, whereas the preflight step already reads `rocm-smi`'s GTT +/// rows on these hosts. It runs subprocesses, each bounded by `probe_budget` (at +/// most the probe timeout, never past `deadline`), so the blocking thread it +/// occupies is released. async fn probe_device_memory(deadline: Instant) -> Probe { - tokio::task::spawn_blocking(move || device_memory_mib(deadline)) - .await - .unwrap_or(Probe::Unreadable) -} - -/// Device memory for GPU 0: VRAM via `amd-smi` (then `rocm-smi`), plus the GTT -/// pool from `rocm-smi` where the host needs it (see [`sample_device_memory`] -/// for when GTT is read and what a failed read means). `amd-smi` is not asked -/// for GTT: its output has no field this harness has been checked against, -/// whereas the preflight step already reads `rocm-smi`'s GTT rows on these -/// hosts. -fn device_memory_mib(deadline: Instant) -> Probe { - sample_device_memory(|| vram_mib(deadline), || gtt_mib(deadline)) + tokio::task::spawn_blocking(move || { + sample_device_memory(|| vram_pool(deadline), || gtt_pool(deadline)) + }) + .await + .unwrap_or(Probe::Unreadable) } -/// `(total, free)` GTT in MiB for GPU 0, via `rocm-smi`. -fn gtt_mib(deadline: Instant) -> Probe<(u64, u64)> { +/// GTT for GPU 0, via `rocm-smi`. The command is named in the report when its +/// output is not understood, so the first run on a host shows the CSV header the +/// column names were meant to match. +fn gtt_pool(deadline: Instant) -> Probe { use std::process::Command; let mut command = Command::new("rocm-smi"); command.args(["--showmeminfo", "gtt", "--csv"]); bounded_stdout(command, probe_budget(Instant::now(), deadline)) - .parse(|text| parse_rocm_smi_csv_pool(&text, "GTT Total Memory", "GTT Total Used Memory")) + .parse_text("rocm-smi --showmeminfo gtt --csv", |text| { + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory") + }) } -/// `(total, free)` device VRAM in MiB for GPU 0, via `amd-smi` (then `rocm-smi`). -fn vram_mib(deadline: Instant) -> Probe<(u64, u64)> { +/// Device VRAM for GPU 0, via `amd-smi` (then `rocm-smi`). +fn vram_pool(deadline: Instant) -> Probe { use std::process::Command; // amd-smi: lines like " TOTAL_VRAM: 196592 MB" / "FREE_VRAM: 196309 MB". let amd_smi = || { let mut command = Command::new("amd-smi"); command.args(["metric", "-m"]); - bounded_stdout(command, probe_budget(Instant::now(), deadline)).parse(|text| { - let field = |name: &str| { - text.lines() - .find_map(|l| l.trim().strip_prefix(name)) - .and_then(|v| v.split_whitespace().next()) - .and_then(|n| n.parse::().ok()) - }; - Some((field("TOTAL_VRAM:")?, field("FREE_VRAM:")?)) - }) + bounded_stdout(command, probe_budget(Instant::now(), deadline)) + .parse_text("amd-smi metric -m", parse_amd_smi_vram) }; // rocm-smi fallback: `--showmeminfo vram --csv` → vram total/used per card. let rocm_smi = || { @@ -356,7 +350,9 @@ fn vram_mib(deadline: Instant) -> Probe<(u64, u64)> { command.args(["--showmeminfo", "vram", "--csv"]); // CSV columns include "VRAM Total Memory (B)" and "VRAM Total Used Memory (B)". bounded_stdout(command, probe_budget(Instant::now(), deadline)) - .parse(|text| parse_rocm_smi_csv_pool(&text, "Total Memory", "Total Used Memory")) + .parse_text("rocm-smi --showmeminfo vram --csv", |text| { + parse_rocm_smi_csv_pool(text, "Total Memory", "Total Used Memory") + }) }; amd_smi().or_else(rocm_smi) } From fc588ecb17e90f0d2748b18735cba9508acb6a16 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 12:42:45 +0000 Subject: [PATCH 10/13] test(e2e): find the rocm-smi CSV header by its column names instead of taking the first line The drain wait read the first line of `rocm-smi --showmeminfo ... --csv` as the header. If the tool prints warning lines before the CSV, the first line names neither column, so every sample was unreadable and the wait could end as "could not read device memory" on a host whose device was fine. The header is now the first line that names both columns, the row after it is GPU 0, and a warning line that mentions only one of the columns is not taken for it. The comment on the GTT probe says which part of the column naming is known (the plain-text labels `gpu-preflight.sh` matches) and which is assumed (that the CSV headers use the same words). Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 39 +++++++++++++++++-- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 7 ++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 6d73805e54..51de93326d 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -440,13 +440,19 @@ pub fn select_drain_target(memory: DeviceMemory) -> DrainTarget { /// `total_column` and `used_column` are substrings of the two header /// names (`Total Memory` and `Total Used Memory` for VRAM; the `GTT ` forms for /// GTT); the first data row is the first GPU. Values are bytes in the output. +/// +/// The header is the first line that names both columns, not the first line: +/// the tool can print warning lines before the CSV, and taking one of those as +/// the header would make every sample unreadable. #[must_use] pub fn parse_rocm_smi_csv_pool(text: &str, total_column: &str, used_column: &str) -> Option { let mut lines = text.lines(); - let header = lines.next()?; - let cols: Vec<&str> = header.split(',').collect(); - let total_idx = cols.iter().position(|c| c.contains(total_column))?; - let used_idx = cols.iter().position(|c| c.contains(used_column))?; + let (total_idx, used_idx) = lines.find_map(|line| { + let cols: Vec<&str> = line.split(',').collect(); + let total_idx = cols.iter().position(|c| c.contains(total_column))?; + let used_idx = cols.iter().position(|c| c.contains(used_column))?; + Some((total_idx, used_idx)) + })?; let row = lines.next()?; let vals: Vec<&str> = row.split(',').collect(); let total: u64 = vals.get(total_idx)?.trim().parse().ok()?; @@ -854,6 +860,31 @@ mod tests { ); } + #[test] + fn rocm_smi_csv_with_warning_lines_before_the_header_is_still_read() { + let text = "WARNING: a device is in a low-power state\n\ + a warning that only says GTT Total Memory\n\ + \n\ + device,GTT Total Memory (B),GTT Total Used Memory (B)\n\ + card0,107374182400,41787392\n\ + card1,1,1\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + Some(Pool::new(102_400, 102_360)), + "the header is the first line naming both columns, and the first row after it is GPU 0" + ); + } + + #[test] + fn rocm_smi_csv_with_a_header_but_no_data_row_is_none() { + let text = "WARNING: nothing to report\n\ + device,GTT Total Memory (B),GTT Total Used Memory (B)\n"; + assert_eq!( + parse_rocm_smi_csv_pool(text, "GTT Total Memory", "GTT Total Used Memory"), + None + ); + } + /// A Strix Halo host whose GTT pool has only `free_mib` MiB free of 102 400. fn strix_holding(free_mib: u64) -> DeviceMemory { DeviceMemory { diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 2c6ab5e21c..64de4ff979 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -324,6 +324,13 @@ async fn probe_device_memory(deadline: Instant) -> Probe { /// GTT for GPU 0, via `rocm-smi`. The command is named in the report when its /// output is not understood, so the first run on a host shows the CSV header the /// column names were meant to match. +/// +/// What is and is not checked: `gpu-preflight.sh` matches the labels +/// `GTT Total Memory` and `GTT Total Used Memory` in `rocm-smi`'s plain-text +/// `--showmeminfo vram gtt` output, so those labels are known to exist there. +/// That `--showmeminfo gtt --csv` uses the same words as its column headers is +/// not checked against real output; it is assumed by analogy with the VRAM CSV. +/// Warning lines before the header are skipped (see [`parse_rocm_smi_csv_pool`]). fn gtt_pool(deadline: Instant) -> Probe { use std::process::Command; let mut command = Command::new("rocm-smi"); From b3229c36d1b2370a198c731c3ab337b0ec2d055d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 13:17:06 +0000 Subject: [PATCH 11/13] test(e2e): do not count a probe the deadline cut short, drop the unused parser, and document the floor and the probe bound A probe killed by the drain wait's own deadline gave no reading because the deadline stopped it, not because the tool stalled, yet it could replace a below-floor reading with "unreadable" in the final line. A sample that starts with less than a full probe timeout left and reads nothing is now ignored, so the wait ends on the last real reading; a truncated sample that does read still counts. `Probe::parse` was only used by its own test and is removed. The doc comments now say how much a 90% floor tolerates on the ~100 GiB GTT pool (about 10 GiB) and that the amd-smi bound is a behaviour change from the old unbounded call, softened by the rocm-smi fallback. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 111 +++++++++++++++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 8 +- 2 files changed, 98 insertions(+), 21 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 51de93326d..683dcc92a3 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -78,17 +78,6 @@ pub enum Probe { } impl Probe { - /// Turn a raw reading into a parsed one; a reading `f` rejects is `Unreadable`. - #[must_use] - pub fn parse(self, f: impl FnOnce(T) -> Option) -> Probe { - match self { - Self::Absent => Probe::Absent, - Self::Unreadable => Probe::Unreadable, - Self::Unparsed(hint) => Probe::Unparsed(hint), - Self::Read(value) => f(value).map_or(Probe::Unreadable, Probe::Read), - } - } - /// Fall back to `next` when this probe produced no reading. /// /// A reading from either wins. Without one, the result is `Absent` only if @@ -233,6 +222,11 @@ impl DrainTarget { /// The free-memory floor to wait for on a pool of `total_mib`: the /// model-sized ceiling, but never more than 90% of the pool. +/// +/// The 90% cap is a tolerance, not a demand for an empty pool: on the CI Strix +/// Halo hosts' ~100 GiB GTT pool (102 400 MiB, floor 92 160 MiB) about 10 GiB +/// can still be held when the wait ends as drained. That is the accepted cost +/// of a floor that a pool with some memory permanently in use can still meet. #[must_use] pub fn required_free_mib(total_mib: u64) -> u64 { MAX_FREE_FLOOR_MIB.min(total_mib / 100 * 90) @@ -286,6 +280,13 @@ pub fn probe_budget(now: Instant, deadline: Instant) -> Duration { SMI_PROBE_TIMEOUT.min(deadline.saturating_duration_since(now)) } +/// Whether a sample started at `now` has its probes held to less than a full +/// [`SMI_PROBE_TIMEOUT`] by `deadline` (see [`DrainWait::observe_truncated`]). +#[must_use] +pub fn sample_is_truncated(now: Instant, deadline: Instant) -> bool { + probe_budget(now, deadline) < SMI_PROBE_TIMEOUT +} + /// How the drain wait ended. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Verdict { @@ -339,6 +340,21 @@ impl DrainWait { } } + /// As [`observe`](Self::observe), for a sample taken when the deadline had + /// already cut its probes short (`truncated`, see [`probe_budget`]). + /// + /// A truncated sample that gave no reading says nothing about the tool: it + /// was stopped by the deadline, not stalled. Counting it would let a probe + /// killed in the last seconds replace a below-floor reading with + /// "unreadable" at the deadline. It is ignored, so the wait ends on the last + /// real reading. A truncated sample that does read counts as usual. + pub fn observe_truncated(&mut self, judgement: Judgement, truncated: bool) -> Option { + if truncated && matches!(judgement, Judgement::Unreadable | Judgement::Unparsed(_)) { + return None; + } + self.observe(judgement) + } + fn unread(&mut self) -> Option { self.unread_in_a_row += 1; (self.last_reading.is_none() && self.unread_in_a_row >= UNREAD_SAMPLES_BEFORE_GIVING_UP) @@ -524,6 +540,15 @@ pub fn sample_device_memory( /// /// These tools can stall on a wedged driver, and the drain wait is best-effort: /// a stalled probe must cost one sample, not the worker. +/// +/// This is a behaviour change on discrete lanes: the previous probe ran +/// `amd-smi metric -m` with no timeout, so a slow call (an 8-GPU `metric` call +/// can be slow) simply took as long as it took, and a wedged one hung the +/// worker. Now `amd-smi` is cut off at this bound, or at the time left before +/// the deadline, and the `rocm-smi` fallback is asked next with its own bound, +/// so a sample costs at most two bounds and still gets a reading whenever +/// `rocm-smi` answers in time. It is not unbounded either way, and a tool that +/// never answers is reported as unreadable after a few samples. pub const SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(10); const SMI_POLL_INTERVAL: Duration = Duration::from_millis(25); @@ -818,14 +843,6 @@ mod tests { ); } - #[test] - fn parse_keeps_absent_and_unreadable_and_rejects_what_it_cannot_read() { - assert_eq!(Probe::::Absent.parse(Some), Probe::Absent); - assert_eq!(Probe::::Unreadable.parse(Some), Probe::Unreadable); - assert_eq!(Probe::Read(3_u8).parse(|n| Some(n + 1)), Probe::Read(4)); - assert_eq!(Probe::Read(3_u8).parse(|_| None::), Probe::Unreadable); - } - #[test] fn rocm_smi_csv_gtt_is_read_in_mib() { let text = "device,GTT Total Memory (B),GTT Total Used Memory (B)\n\ @@ -975,6 +992,44 @@ mod tests { assert!(!line.contains("NOT drained"), "{line}"); } + #[test] + fn a_probe_the_deadline_cut_short_does_not_replace_a_below_floor_reading() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe_truncated(undrained(), false), None); + // The last sample's probe was killed by the deadline and gave nothing. + assert_eq!( + wait.observe_truncated(Judgement::Unreadable, true), + None, + "a truncated sample with no reading is not counted" + ); + assert_eq!( + wait.observe_truncated(Judgement::Unparsed("rocm-smi: killed".to_owned()), true), + None + ); + assert_eq!( + wait.timed_out(), + Verdict::NotDrained(select_drain_target(strix_holding(1_000))), + "the wait ends on the last real reading, not on the probe the deadline stopped" + ); + } + + #[test] + fn an_unread_sample_that_was_not_cut_short_still_counts_and_a_truncated_reading_still_reads() { + let mut wait = DrainWait::default(); + assert_eq!(wait.observe_truncated(undrained(), false), None); + assert_eq!(wait.observe_truncated(Judgement::Unreadable, false), None); + assert!( + matches!(wait.timed_out(), Verdict::Unreadable { samples: 1, .. }), + "a stalled probe with the full budget is a real miss" + ); + let mut wait = DrainWait::default(); + let verdict = wait.observe_truncated(drained(), true); + assert!( + matches!(verdict, Some(Verdict::Drained(_))), + "a truncated sample that read is judged like any other: {verdict:?}" + ); + } + #[test] fn a_tool_that_has_read_is_waited_on_through_unreadable_samples() { let mut wait = DrainWait::default(); @@ -1293,6 +1348,24 @@ mod tests { ); } + #[test] + fn a_sample_is_truncated_only_when_less_than_a_full_probe_timeout_is_left() { + let now = Instant::now(); + assert!(!sample_is_truncated(now, now + Duration::from_secs(100))); + assert!( + !sample_is_truncated(now, now + SMI_PROBE_TIMEOUT), + "exactly a full probe timeout left is not truncated" + ); + assert!(sample_is_truncated( + now, + now + SMI_PROBE_TIMEOUT.saturating_sub(Duration::from_millis(1)) + )); + assert!( + sample_is_truncated(now + Duration::from_secs(5), now), + "past the deadline is truncated" + ); + } + #[cfg(unix)] fn shell(script: &str, args: &[&str]) -> Command { let mut command = Command::new("sh"); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 64de4ff979..f6d1a42b2a 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -14,6 +14,7 @@ use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::device_memory::{ DeviceMemory, DrainWait, MIN_PROBE_BUDGET, Pool, Probe, bounded_stdout, describe_verdict, judge, parse_amd_smi_vram, parse_rocm_smi_csv_pool, probe_budget, sample_device_memory, + sample_is_truncated, }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ @@ -289,9 +290,12 @@ async fn wait_for_free_vram() -> String { let mut wait = DrainWait::default(); loop { // Every probe is held to the time left, so the wait ends at its deadline - // rather than a few probe timeouts after it. + // rather than a few probe timeouts after it. A sample started with less + // than a full probe timeout left is `truncated`: if it gives no reading + // that is the deadline's doing, not the tool's, and is not counted. + let truncated = sample_is_truncated(Instant::now(), deadline); let sample = judge(probe_device_memory(deadline).await); - if let Some(verdict) = wait.observe(sample) { + if let Some(verdict) = wait.observe_truncated(sample, truncated) { return describe_verdict(verdict, started.elapsed().as_secs()); } let left = deadline.saturating_duration_since(Instant::now()); From 23ed368c7465e3d2a0385f8426e089b6f79a4424 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 13:04:13 +0000 Subject: [PATCH 12/13] test(e2e): give a probe that exited cleanly two seconds to hand over its output After a successful exit the probe's output is already in the pipe and the reader thread only has to be scheduled to pass it on. A 200 ms bound could expire on a loaded runner and turn a good sample into an unreadable one, and three of those in a row make the drain wait give up. The wait ends as soon as the bytes arrive, so a longer bound costs nothing in the normal case and matters only when the reader is late or a grandchild holds the pipe. The wait is now a function of the grace it is offered, so a test pins the bound without sleeping. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/device_memory.rs | 59 +++++++++++++++++++++---- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 683dcc92a3..10a07655e5 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -553,10 +553,19 @@ pub const SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(10); const SMI_POLL_INTERVAL: Duration = Duration::from_millis(25); -/// How long to wait for the reader after the child has exited. A grandchild -/// that inherited the pipe can keep it open, so the reader is abandoned -/// rather than joined. -const SMI_DRAIN_GRACE: Duration = Duration::from_millis(200); +/// How long to wait for the reader after the child has exited successfully. +/// +/// The output is already in the pipe by then and the reader only has to be +/// scheduled to hand it over, so the wait ends the moment it does and the bound +/// matters only when it cannot: a loaded runner that has not run the reader +/// thread yet, where a short bound would turn a good sample into an unreadable +/// one (and three of those in a row make the wait give up). A grandchild that +/// inherited the pipe can keep it open for good, so past this bound the reader +/// is abandoned rather than joined. The bound is generous because the cost of +/// that case, one grace per probe, is small next to [`SMI_PROBE_TIMEOUT`] and +/// the probes here do not fork helpers that outlive them. After a kill there is +/// no wait at all: a probe that outlived its deadline is unreadable at once. +const SMI_DRAIN_GRACE: Duration = Duration::from_secs(2); /// Run `command` and return its stdout if it exits successfully within `timeout`. /// @@ -610,11 +619,18 @@ pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe if !status.success() { return Probe::Unreadable; } - receiver - .recv_timeout(SMI_DRAIN_GRACE) - .map_or(Probe::Unreadable, |bytes| { - Probe::Read(String::from_utf8_lossy(&bytes).into_owned()) - }) + output_after_clean_exit(|grace| receiver.recv_timeout(grace).ok()) +} + +/// The reading from a probe that exited successfully, given a way to wait for +/// the reader's bytes for at most the grace it is passed. A function of the wait +/// so a test can pin the bound that is offered without sleeping. +fn output_after_clean_exit( + wait_for_reader: impl FnOnce(Duration) -> Option>, +) -> Probe { + wait_for_reader(SMI_DRAIN_GRACE).map_or(Probe::Unreadable, |bytes| { + Probe::Read(String::from_utf8_lossy(&bytes).into_owned()) + }) } #[cfg(test)] @@ -1366,6 +1382,31 @@ mod tests { ); } + /// A probe that exited successfully has its output in the pipe, and a loaded + /// runner can take longer than the old 200 ms to run the reader thread that + /// hands it over. The wait it is offered must outlast that, the bytes it + /// returns are the reading, and only a reader that never answers (a + /// grandchild holding the pipe) is unreadable. + #[test] + fn a_clean_exit_waits_for_a_slow_reader_and_only_a_silent_one_is_unreadable() { + let mut offered = None; + let out = output_after_clean_exit(|grace| { + offered = Some(grace); + Some(b"MEM 1 2\n".to_vec()) + }); + assert_eq!(out, Probe::Read("MEM 1 2\n".to_owned())); + let offered = offered.expect("the wait was asked for"); + assert!( + offered >= Duration::from_secs(1), + "a reader on a loaded runner was offered only {offered:?}" + ); + assert!( + offered <= SMI_PROBE_TIMEOUT, + "the grace for one probe is {offered:?}, longer than the probe itself" + ); + assert_eq!(output_after_clean_exit(|_| None), Probe::Unreadable); + } + #[cfg(unix)] fn shell(script: &str, args: &[&str]) -> Command { let mut command = Command::new("sh"); From a0053f5ff15acb5412aef4030304001bf5c3814e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 13:05:30 +0000 Subject: [PATCH 13/13] docs(e2e): say why bounded_stdout repeats run_with_timeout instead of calling it rocm-core is only a dev-dependency of the harness crate, and bounded_stdout is library code, so it cannot call rocm_core::process::run_with_timeout. A normal dependency would be a new first-party edge for the crate-edge check to review. Record that where the dependency is declared and where the function is documented, so a change to either side's kill or drain behaviour looks at the other. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/Cargo.toml | 5 +++++ tests/e2e-cucumber/src/device_memory.rs | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 719f32ac35..3d2d02c7e9 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -84,6 +84,11 @@ vt100 = "0.16" # "inference verified" latch is the field the product reads), never the library # itself, and `xtask check-crate-edges` exempts dev edges by design — so keeping # it here needs no allowlist entry. +# +# For the same reason the library's `device_memory::bounded_stdout` cannot call +# `rocm_core::process::run_with_timeout`: it repeats that helper's spawn, poll, +# kill and reap steps on `std` alone, deliberately, and a normal dependency +# would be a new edge to review. A change to either one should look at the other. rocm-core = { path = "../../crates/rocm-core" } # `test-util` pauses tokio's clock in the retry loops' unit tests # (`#[tokio::test(start_paused = true)]`), so their timing assertions read a diff --git a/tests/e2e-cucumber/src/device_memory.rs b/tests/e2e-cucumber/src/device_memory.rs index 10a07655e5..fb3a513e6f 100644 --- a/tests/e2e-cucumber/src/device_memory.rs +++ b/tests/e2e-cucumber/src/device_memory.rs @@ -579,8 +579,13 @@ const SMI_DRAIN_GRACE: Duration = Duration::from_secs(2); /// `rocm-smi`) do not fork helpers that outlive them, and the drain wait makes /// a handful of calls, so this is accepted rather than guarded. /// -/// Written against `std` alone because `rocm_core::process::run_with_timeout` -/// is reachable here only as a dev-dependency, and this module is library code. +/// Written against `std` alone, so it repeats the spawn, poll, kill and reap +/// steps of `rocm_core::process::run_with_timeout` on purpose: `rocm-core` is +/// only a dev-dependency of this crate (see its entry in `Cargo.toml`) and this +/// module is library code, and a normal dependency would be a new first-party +/// edge that `cargo xtask check-crate-edges` makes a reviewed addition. The two +/// can drift, so a change to either one's kill or drain behaviour should look at +/// the other. pub fn bounded_stdout(mut command: Command, timeout: Duration) -> Probe { let mut child = match command .stdin(Stdio::null())