From 462e0e12066b296f1f0e9a9c7dbbffc4916feb48 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 25 Sep 2026 13:45:15 +0300 Subject: [PATCH 01/32] fix(serve): run the KFD pre-flight at every amd-smi launch site (EAI-8457) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `amd-smi` launched against a kernel fusion driver in a bad state blocks in uninterruptible sleep, which no signal escapes — so `rocm serve` hangs and survives both Ctrl-C and SIGKILL. The repo documents that and calls the `/dev/kfd` pre-flight mandatory, but three of the four production launch sites skipped it. Two of those, `gpu_vram_usage` and `detect_gpu_count`, sit on the normal `rocm serve` path for any run that is not CPU-only, and had no timeout either. Give the pre-flight one definition in `rocm-core` and route the three unguarded sites through a single guarded launcher, so a future launch site inherits the guard rather than having to remember it. The gate is Linux-only on purpose. `/dev/kfd` exists nowhere else, and `amd-smi.exe` is a supported Windows binary with a live Strix Halo lane, so a check that merely failed to find the device would have disabled GPU detection on every Windows host. Two consequences worth naming: - On Linux the two `serve` probes now decline to launch when `/dev/kfd` cannot be opened, where before they launched and usually failed anyway. GPU auto-selection degrades to service state alone, which is what it already does whenever amd-smi is unavailable. - `run_command_with_timeout` now drains both pipes on their own threads. Polling `try_wait` without draining deadlocks once the child fills the 64 KiB pipe buffer — which `amd-smi metric --json` clears easily on a multi-GPU host — and that deadlock surfaced as a spurious timeout. Routing the serve path through the helper without this would have traded the hang for silent loss of VRAM telemetry on exactly the machines that matter. Its timeout message no longer quotes partial output, because collecting it means waiting on a child that may be the unkillable one. The dash collector keeps its own pre-flight: it already runs one, it needs an async launcher, and an edge from `rocm-dash-collectors` to `rocm-core` is what the `crate_edges` layering guard exists to stop. Its check is not cfg-gated, so the dash detects no GPU off Linux — a separate pre-existing bug, untouched here. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 76 ++--------- apps/rocmd/src/lib.rs | 76 +---------- crates/rocm-core/src/lib.rs | 262 +++++++++++++++++++++++++++++++++++- 3 files changed, 274 insertions(+), 140 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 3377ee51c4..5491767787 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -15068,7 +15068,7 @@ fn run_rocm_capture_for_paths( .stdout(Stdio::piped()) .stderr(Stdio::piped()); apply_app_path_env(&mut command, paths); - let output = run_command_with_timeout(command, timeout) + let output = rocm_core::run_command_with_timeout(command, timeout) .with_context(|| format!("failed to run {}", rocm_binary.display()))?; Ok(CommandCapture { argv: std::iter::once(rocm_binary.display().to_string()) @@ -15295,45 +15295,6 @@ fn render_install_sdk_dry_run_for_args(paths: &AppPaths, args: &[String]) -> Res .output) } -fn run_command_with_timeout( - mut command: ProcessCommand, - timeout: Duration, -) -> Result { - let mut child = command.spawn().context("failed to spawn child process")?; - let started = std::time::Instant::now(); - loop { - if child - .try_wait() - .context("failed to poll child process")? - .is_some() - { - return child - .wait_with_output() - .context("failed to collect child process output"); - } - if started.elapsed() >= timeout { - let _ = child.kill(); - let output = child - .wait_with_output() - .context("failed to collect timed-out child process output")?; - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - bail!( - "process exceeded {}s timeout: {}", - timeout.as_secs(), - if !stderr.is_empty() { - stderr - } else if !stdout.is_empty() { - stdout - } else { - "no output".to_owned() - } - ); - } - thread::sleep(Duration::from_millis(50)); - } -} - fn internal_mcp_install_sdk_args( arguments: &serde_json::Map, dry_run: bool, @@ -20918,23 +20879,18 @@ fn select_auto_gpu_index( vec![all[0]] } +/// How long the `serve` path waits on `amd-smi` before giving up on it. Both +/// probes below are advisory — GPU auto-selection degrades to service state +/// alone without them — so a stalled `amd-smi` must not hold up a launch. This +/// bounds the ordinary stalls only; the unkillable one is prevented by the +/// pre-flight inside [`rocm_core::amd_smi_json`], not by any timeout. +const AMD_SMI_SERVE_PROBE_TIMEOUT: Duration = Duration::from_secs(10); + /// Best-effort per-GPU VRAM occupancy via `amd-smi metric --json`. Returns /// `None` when amd-smi is unavailable or its output cannot be parsed (callers /// then fall back to service-state-only auto-selection). fn gpu_vram_usage() -> Option> { - let binary = rocm_core::resolve_amd_smi_binary(); - let output = ProcessCommand::new(&binary) - .arg("metric") - .arg("--json") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .output() - .ok()?; - if !output.status.success() { - return None; - } - let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?; + let value = rocm_core::amd_smi_json(&["metric", "--json"], AMD_SMI_SERVE_PROBE_TIMEOUT).ok()?; let rows = parse_gpu_vram_usage(&value); if rows.is_empty() { None } else { Some(rows) } } @@ -21072,19 +21028,7 @@ fn busy_gpu_indices(paths: &AppPaths) -> Vec { /// amd-smi is unavailable or its output cannot be parsed (callers then fall /// back to conservative defaults). fn detect_gpu_count() -> Option { - let binary = rocm_core::resolve_amd_smi_binary(); - let output = ProcessCommand::new(&binary) - .arg("list") - .arg("--json") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .output() - .ok()?; - if !output.status.success() { - return None; - } - let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?; + let value = rocm_core::amd_smi_json(&["list", "--json"], AMD_SMI_SERVE_PROBE_TIMEOUT).ok()?; let count = value.as_array().map(Vec::len)?; if count == 0 { None } else { Some(count) } } diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index dfc287f452..4664daa073 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -18,10 +18,10 @@ use rocm_core::{ AutomationRuntimeState, AutomationTriggerEvent, CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot, DEFAULT_LOCAL_HOST, ExamineSummary, ManagedServiceRecord, ModelRecipeArtifactRecord, RocmCliConfig, WatcherMode, WatcherRuntimeSnapshot, - append_audit_event, append_automation_event, append_automation_proposal, builtin_watcher, - builtin_watchers, daemon_binary_path, default_engine_for_platform, format_host_port, - load_recent_automation_events, model_artifact_cache_status, resolve_amd_smi_binary, - resolve_model_recipe_artifact, unix_time_millis, + amd_smi_json, append_audit_event, append_automation_event, append_automation_proposal, + builtin_watcher, builtin_watchers, daemon_binary_path, default_engine_for_platform, + format_host_port, load_recent_automation_events, model_artifact_cache_status, + resolve_model_recipe_artifact, run_command_with_timeout, unix_time_millis, }; #[cfg(test)] use rocm_engine_protocol::EnginePluginDescriptor; @@ -1468,34 +1468,7 @@ fn gather_gpu_snapshot_for_config(config: &RocmCliConfig) -> CodexBridgeGpuSnaps } fn capture_amd_smi_json(args: &[&str]) -> Result { - let amd_smi_binary = resolve_amd_smi_binary(); - let mut command = ProcessCommand::new(&amd_smi_binary); - command - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let output = run_command_with_timeout(command, AMD_SMI_PROBE_TIMEOUT) - .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - anyhow::bail!( - "amd-smi {} failed: {}", - args.join(" "), - if !stderr.is_empty() { - stderr - } else if !stdout.is_empty() { - stdout - } else { - format!("exit status {}", output.status) - } - ); - } - - serde_json::from_slice(&output.stdout) - .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) + amd_smi_json(args, AMD_SMI_PROBE_TIMEOUT) } fn bridge_engine_inventory() -> Vec { @@ -2402,45 +2375,6 @@ fn run_rocm_capture_for_paths( }) } -fn run_command_with_timeout( - mut command: ProcessCommand, - timeout: Duration, -) -> Result { - let mut child = command.spawn().context("failed to spawn child process")?; - let started = std::time::Instant::now(); - loop { - if child - .try_wait() - .context("failed to poll child process")? - .is_some() - { - return child - .wait_with_output() - .context("failed to collect child process output"); - } - if started.elapsed() >= timeout { - let _ = child.kill(); - let output = child - .wait_with_output() - .context("failed to collect timed-out child process output")?; - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - bail!( - "process exceeded {}s timeout: {}", - timeout.as_secs(), - if !stderr.is_empty() { - stderr - } else if !stdout.is_empty() { - stdout - } else { - "no output".to_owned() - } - ); - } - thread::sleep(Duration::from_millis(50)); - } -} - fn read_tail_lines(path: &std::path::Path, limit: usize) -> Result { let content = fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 77bd6bbd0b..c58c2b16e7 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -6,9 +6,7 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; -#[cfg(windows)] -use std::ffi::OsStr; -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; use std::fs; use std::io::{IsTerminal, Read, Write, stdin, stdout}; use std::net::{IpAddr, TcpStream, ToSocketAddrs}; @@ -7953,6 +7951,160 @@ fn resolve_amd_smi_binary_in_home(home_dir: Option<&Path>) -> OsString { "amd-smi".into() } +/// Device node of the AMD kernel fusion driver. Linux-only; no other platform +/// exposes one. +const KFD_DEVICE: &str = "/dev/kfd"; + +/// Whether it is safe to launch `amd-smi` on this host. +/// +/// On Linux this is a readability check on `/dev/kfd`, and it is **mandatory** +/// before every launch. Against a kernel fusion driver in a bad state — +/// partially installed, version-mismatched, wedged after a GPU fault, or passed +/// into a container from an unhealthy host — `amd-smi` blocks in uninterruptible +/// kernel sleep. No signal escapes that state, so the process survives both +/// Ctrl-C and `SIGKILL`, and a timeout cannot rescue it either: killing a +/// D-state child is a no-op and the `wait` that follows blocks alongside it. +/// Declining to launch is the only mitigation. +/// +/// Off Linux there is no such device and no such kernel state, so this returns +/// `true` without probing anything. That gate is load-bearing rather than +/// pedantic: `amd-smi.exe` is a supported Windows binary (see +/// [`managed_sdk_tool_path`]), so a check that simply failed to find `/dev/kfd` +/// there would silently disable GPU detection on every Windows host. +#[must_use] +pub fn amd_smi_preflight_ok() -> bool { + kfd_readable(Path::new(KFD_DEVICE)) +} + +#[cfg(target_os = "linux")] +fn kfd_readable(device: &Path) -> bool { + fs::OpenOptions::new().read(true).open(device).is_ok() +} + +#[cfg(not(target_os = "linux"))] +fn kfd_readable(_device: &Path) -> bool { + true +} + +/// Run `amd-smi ` and parse its stdout as JSON, declining to launch at all +/// unless [`amd_smi_preflight_ok`] passes. +/// +/// This is the supported way to launch `amd-smi` from a synchronous caller. The +/// pre-flight has to happen at every launch site or it protects nothing, and +/// routing the launch through one function is what keeps that true as sites are +/// added. +/// +/// `timeout` is defence in depth, not the mitigation: see +/// [`run_command_with_timeout`], which kills the child and then waits for it, +/// neither of which dislodges a process in uninterruptible sleep. It bounds the +/// ordinary failures — a slow or wedged-but-signalable `amd-smi` — and nothing +/// more. +pub fn amd_smi_json(args: &[&str], timeout: Duration) -> Result { + amd_smi_json_with(&resolve_amd_smi_binary(), Path::new(KFD_DEVICE), args, timeout) +} + +fn amd_smi_json_with( + binary: &OsStr, + kfd_device: &Path, + args: &[&str], + timeout: Duration, +) -> Result { + if !kfd_readable(kfd_device) { + bail!( + "declined to launch `amd-smi {}`: {} is not readable, and amd-smi can block \ + unkillably in uninterruptible sleep against a kernel driver in that state", + args.join(" "), + kfd_device.display() + ); + } + + let mut command = Command::new(binary); + command + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let output = run_command_with_timeout(command, timeout) + .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + bail!( + "amd-smi {} failed: {}", + args.join(" "), + if stderr.is_empty() { + if stdout.is_empty() { + format!("exit status {}", output.status) + } else { + stdout + } + } else { + stderr + } + ); + } + + serde_json::from_slice(&output.stdout) + .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) +} + +/// Wait for `command` for at most `timeout`, killing it and failing if it +/// overruns. +/// +/// Bounds signalable children only. A child in uninterruptible sleep ignores +/// the `kill`, so callers that can reach one must decline to spawn it in the +/// first place — see [`amd_smi_preflight_ok`]. This function at least returns +/// to its caller in that case rather than joining the child in its wait. +pub fn run_command_with_timeout( + mut command: Command, + timeout: Duration, +) -> Result { + let mut child = command.spawn().context("failed to spawn child process")?; + // Drain both pipes on their own threads for as long as the child runs. + // Polling `try_wait` alone cannot: a child that fills the pipe buffer + // (64 KiB on Linux) blocks mid-write and never exits, so a merely verbose + // command would be reported as having hung. `amd-smi metric --json` on a + // multi-GPU host clears 64 KiB comfortably. + let stdout = child.stdout.take().map(drain_on_thread); + let stderr = child.stderr.take().map(drain_on_thread); + let started = Instant::now(); + loop { + if let Some(status) = child.try_wait().context("failed to poll child process")? { + // The child is gone, so it has dropped its write ends and the + // readers see EOF. A surviving grandchild holding an inherited + // write end can still keep them open, exactly as the + // `wait_with_output` this replaces would have. + return Ok(std::process::Output { + status, + stdout: join_drained(stdout), + stderr: join_drained(stderr), + }); + } + if started.elapsed() >= timeout { + let _ = child.kill(); + // Deliberately neither waits on the child nor joins the readers: an + // unkillable child would block both, which is the hang this timeout + // exists to bound. That costs the partial output the message used to + // quote; the threads end by themselves once the child does. + bail!("process exceeded {}s timeout", timeout.as_secs()); + } + thread::sleep(Duration::from_millis(50)); + } +} + +fn drain_on_thread(mut pipe: R) -> thread::JoinHandle> { + thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = pipe.read_to_end(&mut buffer); + buffer + }) +} + +fn join_drained(reader: Option>>) -> Vec { + reader.map_or_else(Vec::new, |handle| handle.join().unwrap_or_default()) +} + /// A validated managed-service identifier that is safe to use as a single /// filesystem path component. /// @@ -9610,6 +9762,110 @@ mod tests { Ok(()) } + /// The gate has to stop the *spawn*, not merely report a failure after the + /// fact: the launch that already happened is the launch that hangs. The + /// fake `amd-smi` leaves a marker file, so its absence is direct evidence + /// nothing ran — and running the same fixture against a readable device + /// proves the fake would otherwise have executed, which is what stops the + /// blocked case from passing vacuously. + /// + /// Linux-only because `kfd_readable` is deliberately a no-op elsewhere; + /// off Linux there is no gate to exercise. + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_gate_stops_amd_smi_before_it_is_spawned() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-gate-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let marker = temp_root.join("amd-smi-ran"); + let fake_amd_smi = temp_root.join("amd-smi"); + fs::write( + &fake_amd_smi, + format!("#!/bin/sh\ntouch '{}'\necho '{{}}'\n", marker.display()), + )?; + fs::set_permissions(&fake_amd_smi, fs::Permissions::from_mode(0o755))?; + let readable_device = temp_root.join("stand-in-kfd"); + fs::write(&readable_device, b"")?; + + let timeout = Duration::from_secs(10); + let args = ["list", "--json"]; + + let blocked = amd_smi_json_with( + fake_amd_smi.as_os_str(), + &temp_root.join("absent-kfd"), + &args, + timeout, + ); + let ran_while_blocked = marker.exists(); + + let allowed = + amd_smi_json_with(fake_amd_smi.as_os_str(), &readable_device, &args, timeout); + let ran_while_allowed = marker.exists(); + + let _ = fs::remove_dir_all(&temp_root); + + assert!( + blocked.is_err(), + "an unreadable KFD device must stop the launch" + ); + assert!( + !ran_while_blocked, + "amd-smi was spawned even though the pre-flight failed" + ); + assert!( + allowed.is_ok(), + "a readable device must let the launch through: {allowed:?}" + ); + assert!( + ran_while_allowed, + "the fake amd-smi never ran at all, so the blocked case proves nothing" + ); + Ok(()) + } + + /// A merely verbose child must not be mistaken for a hung one. Polling + /// `try_wait` without draining deadlocks the moment the child fills the + /// pipe buffer — 64 KiB on Linux, which `amd-smi metric --json` clears on a + /// multi-GPU host — and that deadlock surfaces as a spurious timeout, which + /// for the `serve` path means GPU auto-selection silently loses its VRAM + /// telemetry. Both pipes are filled because either one alone can stall the + /// child. + #[test] + #[cfg(unix)] + fn a_child_that_outgrows_the_pipe_buffer_is_not_reported_as_hung() -> Result<()> { + const BYTES: usize = 300_000; + + let mut command = Command::new("sh"); + command + .arg("-c") + .arg(format!( + "yes rocm | head -c {BYTES}; yes err | head -c {BYTES} >&2" + )) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let output = run_command_with_timeout(command, Duration::from_secs(10))?; + + assert!(output.status.success(), "the child should have exited 0"); + assert_eq!(output.stdout.len(), BYTES, "stdout was truncated or stalled"); + assert_eq!(output.stderr.len(), BYTES, "stderr was truncated or stalled"); + Ok(()) + } + + /// No host off Linux has a `/dev/kfd`, so the pre-flight must wave the + /// launch through instead of reading the absent node as "unsafe". + /// `amd-smi.exe` is a supported Windows binary, so treating its absence as + /// a failure would disable GPU detection on every Windows host. + #[test] + #[cfg(not(target_os = "linux"))] + fn the_kfd_preflight_waves_through_hosts_that_have_no_kfd() { + assert!(kfd_readable(Path::new("/definitely/not/a/device/node"))); + assert!(amd_smi_preflight_ok()); + } + #[test] fn resolve_amd_smi_binary_prefers_home_rocm_venv_path() -> Result<()> { let temp_root = From 5a2929239e7312454aa93b7bebaddcc01b1432d6 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 25 Sep 2026 14:10:49 +0300 Subject: [PATCH 02/32] style: apply rustfmt to the amd-smi pre-flight change The hand-wrapped import list and the amd_smi_json call did not match what rustfmt produces; the prek hygiene lane caught it. Signed-off-by: Roman Sirokov --- apps/rocmd/src/lib.rs | 10 +++++----- crates/rocm-core/src/lib.rs | 22 +++++++++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 4664daa073..f46d0390ba 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -17,11 +17,11 @@ use rocm_core::{ AppPaths, AuditEventRecord, AutomationEventRecord, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot, DEFAULT_LOCAL_HOST, ExamineSummary, ManagedServiceRecord, - ModelRecipeArtifactRecord, RocmCliConfig, WatcherMode, WatcherRuntimeSnapshot, - amd_smi_json, append_audit_event, append_automation_event, append_automation_proposal, - builtin_watcher, builtin_watchers, daemon_binary_path, default_engine_for_platform, - format_host_port, load_recent_automation_events, model_artifact_cache_status, - resolve_model_recipe_artifact, run_command_with_timeout, unix_time_millis, + ModelRecipeArtifactRecord, RocmCliConfig, WatcherMode, WatcherRuntimeSnapshot, amd_smi_json, + append_audit_event, append_automation_event, append_automation_proposal, builtin_watcher, + builtin_watchers, daemon_binary_path, default_engine_for_platform, format_host_port, + load_recent_automation_events, model_artifact_cache_status, resolve_model_recipe_artifact, + run_command_with_timeout, unix_time_millis, }; #[cfg(test)] use rocm_engine_protocol::EnginePluginDescriptor; diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index c58c2b16e7..425cb1bc3b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -8000,7 +8000,12 @@ fn kfd_readable(_device: &Path) -> bool { /// ordinary failures — a slow or wedged-but-signalable `amd-smi` — and nothing /// more. pub fn amd_smi_json(args: &[&str], timeout: Duration) -> Result { - amd_smi_json_with(&resolve_amd_smi_binary(), Path::new(KFD_DEVICE), args, timeout) + amd_smi_json_with( + &resolve_amd_smi_binary(), + Path::new(KFD_DEVICE), + args, + timeout, + ) } fn amd_smi_json_with( @@ -9800,8 +9805,7 @@ mod tests { ); let ran_while_blocked = marker.exists(); - let allowed = - amd_smi_json_with(fake_amd_smi.as_os_str(), &readable_device, &args, timeout); + let allowed = amd_smi_json_with(fake_amd_smi.as_os_str(), &readable_device, &args, timeout); let ran_while_allowed = marker.exists(); let _ = fs::remove_dir_all(&temp_root); @@ -9850,8 +9854,16 @@ mod tests { let output = run_command_with_timeout(command, Duration::from_secs(10))?; assert!(output.status.success(), "the child should have exited 0"); - assert_eq!(output.stdout.len(), BYTES, "stdout was truncated or stalled"); - assert_eq!(output.stderr.len(), BYTES, "stderr was truncated or stalled"); + assert_eq!( + output.stdout.len(), + BYTES, + "stdout was truncated or stalled" + ); + assert_eq!( + output.stderr.len(), + BYTES, + "stderr was truncated or stalled" + ); Ok(()) } From 2e1e78ae8376a909afdfcdf839139f08376a8416 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 29 Sep 2026 13:28:43 +0300 Subject: [PATCH 03/32] fix(serve): wave the KFD pre-flight through on WSL (EAI-8457) WSL is `target_os = "linux"`, so it compiled the real-open branch of `kfd_readable` -- but it has no `/dev/kfd` at all. It reaches the GPU through `/dev/dxg` and the Windows host driver, which is exactly why the WSL diagnosis catalog skips every bare-metal driver check (docs/wsl.md, examine::WslFacts). So the open always failed there and `amd_smi_json` declined every launch: `gpu_vram_usage` and `detect_gpu_count` on the ordinary serve path, and the daemon bridge snapshot. Both serve probes discard the error with `.ok()?`, so this was a silent loss of GPU detection on a supported platform -- one with its own diagnosis catalog and its own GPU E2E lane. The D-state hazard the gate exists for is an amdgpu/KFD one, which is precisely the stack WSL never loads. Wave WSL through at the top of the Linux `kfd_readable`, with the determination injected via `kfd_readable_with` so the behaviour can be pinned from an ordinary Linux host. Correct the `KFD_DEVICE` comment, which asserted no other platform exposes one without excluding WSL. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 57 +++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 425cb1bc3b..a27054ed51 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -7951,8 +7951,12 @@ fn resolve_amd_smi_binary_in_home(home_dir: Option<&Path>) -> OsString { "amd-smi".into() } -/// Device node of the AMD kernel fusion driver. Linux-only; no other platform -/// exposes one. +/// Device node of the AMD kernel fusion driver. Bare-metal Linux only: no other +/// platform exposes one, and WSL — which *is* `target_os = "linux"` — is not an +/// exception to hand-wave past. It reaches the GPU through `/dev/dxg` and the +/// Windows host driver and has no `/dev/kfd` at all, which is why the WSL +/// diagnosis catalog skips every bare-metal driver check (see `docs/wsl.md` and +/// [`crate::examine::WslFacts`]). const KFD_DEVICE: &str = "/dev/kfd"; /// Whether it is safe to launch `amd-smi` on this host. @@ -7971,6 +7975,15 @@ const KFD_DEVICE: &str = "/dev/kfd"; /// pedantic: `amd-smi.exe` is a supported Windows binary (see /// [`managed_sdk_tool_path`]), so a check that simply failed to find `/dev/kfd` /// there would silently disable GPU detection on every Windows host. +/// +/// WSL needs the same wave-through for the same reason, and cannot get it from +/// `cfg`: it compiles as Linux yet has no `/dev/kfd` (see [`KFD_DEVICE`]), so +/// the real open would fail on every WSL host and decline every launch. WSL is +/// supported here — it has its own diagnosis catalog and its own GPU E2E lane — +/// and both `serve` probes discard the error with `.ok()?`, so that would be a +/// silent loss of GPU detection rather than a visible failure. The D-state hazard +/// this gate exists for is an `amdgpu`/KFD one, which is precisely the stack WSL +/// does not load. #[must_use] pub fn amd_smi_preflight_ok() -> bool { kfd_readable(Path::new(KFD_DEVICE)) @@ -7978,6 +7991,16 @@ pub fn amd_smi_preflight_ok() -> bool { #[cfg(target_os = "linux")] fn kfd_readable(device: &Path) -> bool { + kfd_readable_with(is_wsl_host(), device) +} + +/// [`kfd_readable`] with the WSL determination injected, so the wave-through can +/// be pinned from a test on an ordinary Linux host rather than only on WSL. +#[cfg(target_os = "linux")] +fn kfd_readable_with(is_wsl: bool, device: &Path) -> bool { + if is_wsl { + return true; + } fs::OpenOptions::new().read(true).open(device).is_ok() } @@ -9781,6 +9804,13 @@ mod tests { fn the_kfd_gate_stops_amd_smi_before_it_is_spawned() -> Result<()> { use std::os::unix::fs::PermissionsExt; + // WSL compiles as Linux but is waved through deliberately (see + // `the_kfd_preflight_waves_through_wsl_hosts`), so there is no gate to + // exercise on such a host and the blocked case could never hold. + if is_wsl_host() { + return Ok(()); + } + let temp_root = std::env::temp_dir().join(format!("rocm-cli-kfd-gate-{}", unix_time_millis())); fs::create_dir_all(&temp_root)?; @@ -9878,6 +9908,29 @@ mod tests { assert!(amd_smi_preflight_ok()); } + /// WSL has no `/dev/kfd` either, but unlike Windows it cannot be waved + /// through by `cfg`: it compiles as Linux, so the real open runs, always + /// fails, and declines every routed `amd-smi` launch on a supported + /// platform — silently, since both `serve` probes drop the error with + /// `.ok()?`. Same wave-through as the off-Linux case above, decided at + /// runtime instead. + /// + /// The bare-metal assertion is the control: without it this would still + /// pass with the wave-through deleted. + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_preflight_waves_through_wsl_hosts() { + let absent = Path::new("/definitely/not/a/device/node"); + assert!( + kfd_readable_with(true, absent), + "WSL has no /dev/kfd, so its absence must not decline the launch" + ); + assert!( + !kfd_readable_with(false, absent), + "a bare-metal host with no readable KFD must still be declined" + ); + } + #[test] fn resolve_amd_smi_binary_prefers_home_rocm_venv_path() -> Result<()> { let temp_root = From 877cf0684d2a223de137d0f966be354f559bb455 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 30 Sep 2026 10:16:44 +0300 Subject: [PATCH 04/32] fix(core): reap the child the amd-smi timeout path kills (EAI-8457) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run_command_with_timeout` killed a timed-out child and then dropped the handle. `std::process::Child` has no reaping `Drop`, so the corpse stayed in state `Z` for the lifetime of the parent. Both per-app copies this function replaced called `wait_with_output()` after the `kill`, so the reap used to happen; consolidating them dropped it. That is a real leak rather than a tidiness point because one caller is a daemon. `rocmd` probes `amd-smi` with a 2s timeout from its watcher tick, twice per GPU-telemetry collection, on a 60s cadence. A host whose `/dev/kfd` is readable but whose `amd-smi` simply overruns 2s — ordinary on a busy multi-GPU box — therefore accrues zombies indefinitely. The KFD pre-flight added here does not cover it: that only prevents the spawn when the device is unreadable. Restoring the `wait` inline would reintroduce the hang the timeout exists to bound, since an uninterruptible-sleep child blocks the `wait` exactly as it ignores the `kill`. Hand the child to a detached reaper thread instead, so the calling thread still returns promptly. The reap runs whether or not `kill` succeeded, because the likeliest failure is a child that exited between the poll and the kill — already a zombie, and already in need of reaping. The doc comment is explicit that this is best-effort: an unkillable child parks its reaper instead of leaving a zombie, trading a stranded PID for a stranded thread. That is better than blocking the caller and no worse than the leak it replaces, but it is still a leak, and the pre-flight remains the only actual mitigation. Reapers do not accumulate otherwise — one per timeout, each exiting as soon as its child does. Covered by a Linux-only test that reads the child's recorded pid straight out of `/proc`: without the reap it observes state `Z` for the full ten- second poll budget, with it the entry is gone in under a second. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 142 +++++++++++++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 4 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 51955e9fd7..323e92949b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -8107,6 +8107,14 @@ fn amd_smi_json_with( /// the `kill`, so callers that can reach one must decline to spawn it in the /// first place — see [`amd_smi_preflight_ok`]. This function at least returns /// to its caller in that case rather than joining the child in its wait. +/// +/// On timeout the calling thread returns without waiting on the child; the +/// `wait` is handed to a detached reaper thread instead, because +/// `std::process::Child` has no reaping `Drop` and dropping the handle here +/// would strand a zombie for the lifetime of this process. That reap is +/// best-effort: it completes only if the child actually dies, which is the +/// same condition the `kill` above already depends on. See +/// `reap_in_background`. pub fn run_command_with_timeout( mut command: Command, timeout: Duration, @@ -8133,17 +8141,65 @@ pub fn run_command_with_timeout( }); } if started.elapsed() >= timeout { + // `kill` is allowed to fail — most often because the child exited + // between the `try_wait` above and here, in which case it is already + // a zombie and the reap below is exactly what is needed. Reap + // unconditionally rather than only on a successful kill. let _ = child.kill(); - // Deliberately neither waits on the child nor joins the readers: an - // unkillable child would block both, which is the hang this timeout - // exists to bound. That costs the partial output the message used to - // quote; the threads end by themselves once the child does. + // This thread deliberately joins neither the child nor the readers: + // an unkillable child would block both, which is the hang this + // timeout exists to bound. That costs the partial output the message + // used to quote; the reader threads end by themselves once the child + // does. + reap_in_background(child); bail!("process exceeded {}s timeout", timeout.as_secs()); } thread::sleep(Duration::from_millis(50)); } } +/// Take ownership of a killed child and `wait` for it on a detached thread. +/// +/// `std::process::Child` has no reaping `Drop`: dropping the handle after a +/// `kill` leaves the child a zombie occupying a PID slot for the whole lifetime +/// of this process. That is affordable for a one-shot CLI and not for a daemon +/// — `rocmd`'s watcher tick reaches this path on its GPU-telemetry cadence, so +/// a host where `amd-smi` reliably overruns the probe timeout would accrue +/// zombies until the PID table ran dry. Reaping on a *detached* thread is what +/// lets the `wait` happen without the calling thread paying for it. +/// +/// Guarantees: +/// +/// - The calling thread returns immediately; it never waits on the child. +/// - A child that does die — the normal case after `SIGKILL`, and the case +/// where it had already exited before the `kill` — is reaped, and its PID is +/// released, within roughly the time the kernel takes to tear it down. +/// +/// Does not guarantee: +/// +/// - That the child dies. A process in uninterruptible sleep ignores `SIGKILL`, +/// so its reaper parks until the kernel lets it go, possibly forever. The +/// resource leaked in that case is a blocked thread rather than a zombie PID +/// — no worse than what it replaces, and better than blocking the caller, but +/// still a leak. The only real mitigation is declining to spawn against a +/// suspect device: see [`amd_smi_preflight_ok`]. +/// - That a reaper always exists. If the thread cannot be spawned, the child is +/// dropped and left as a zombie, which is the pre-existing failure mode and +/// strictly better than propagating an error from a path that is already +/// reporting one. +/// +/// Reaper threads do not accumulate under normal operation: one is created per +/// timeout, and each one exits as soon as its child does. Accumulation needs a +/// steady supply of children that never die, which is the uninterruptible-sleep +/// case above. +fn reap_in_background(mut child: std::process::Child) { + let _ = thread::Builder::new() + .name("rocm-reap-timed-out-child".to_owned()) + .spawn(move || { + let _ = child.wait(); + }); +} + fn drain_on_thread(mut pipe: R) -> thread::JoinHandle> { thread::spawn(move || { let mut buffer = Vec::new(); @@ -9920,6 +9976,84 @@ mod tests { Ok(()) } + /// Killing a timed-out child is not the same as reaping it. + /// `std::process::Child` has no reaping `Drop`, so dropping the handle + /// after the `kill` strands the corpse in state `Z` for the lifetime of + /// this process. `rocmd` reaches this path from its watcher tick, so on a + /// host where `amd-smi` reliably overruns the probe timeout that is an + /// unbounded PID leak rather than a one-off. + /// + /// The child records its own PID before sleeping well past the timeout, so + /// this watches the real process table instead of inferring anything from + /// the returned error. Drop `reap_in_background` and the entry sits at `Z` + /// until the poll below gives up. + /// + /// Linux-only: `/proc` is where a zombie is observable. The behaviour is + /// not Linux-specific, only the assertion is. + #[test] + #[cfg(target_os = "linux")] + fn a_timed_out_child_is_reaped_rather_than_left_a_zombie() -> Result<()> { + /// The `stat` state field is the first token after the last `)`, which + /// is where parsing has to start because `comm` may itself contain + /// parentheses. `None` means the entry is gone, i.e. fully reaped. + fn process_state(pid: u32) -> Option { + let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + stat.rsplit_once(')')? + .1 + .split_whitespace() + .next()? + .chars() + .next() + } + + let temp_root = std::env::temp_dir().join(format!("rocm-cli-reap-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let pid_file = temp_root.join("child-pid"); + + // `$$` is this `sh`, which is our direct child whether or not the shell + // execs the `sleep` over itself — either way it is the process we kill + // and therefore the one that must be reaped. The sleep outlasts the + // timeout by enough that the child cannot exit on its own and make the + // assertion pass for the wrong reason. + let mut command = Command::new("sh"); + command + .arg("-c") + .arg(format!("echo $$ > '{}'; sleep 30", pid_file.display())) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let result = run_command_with_timeout(command, Duration::from_millis(300)); + assert!( + result.is_err(), + "the child outlives the timeout, so this must have timed out: {result:?}" + ); + + let pid: u32 = fs::read_to_string(&pid_file) + .context("the child never recorded its pid, so nothing was proven")? + .trim() + .parse() + .context("unparseable child pid")?; + let _ = fs::remove_dir_all(&temp_root); + + // The reaper races the kernel tearing the child down, so poll rather + // than sample once. Without the reap this burns the full budget and + // still reports `Z`. + let deadline = Instant::now() + Duration::from_secs(10); + let mut state = process_state(pid); + while state == Some('Z') && Instant::now() < deadline { + thread::sleep(Duration::from_millis(25)); + state = process_state(pid); + } + + assert_ne!( + state, + Some('Z'), + "pid {pid} is still a zombie: the timeout path killed the child but never reaped it" + ); + Ok(()) + } + /// No host off Linux has a `/dev/kfd`, so the pre-flight must wave the /// launch through instead of reading the absent node as "unsafe". /// `amd-smi.exe` is a supported Windows binary, so treating its absence as From 9d2024ba14a22e5838675572b8441e597f99000f Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 30 Sep 2026 16:17:04 +0300 Subject: [PATCH 05/32] test(core): assert the timed-out child is gone, not merely not-a-zombie (EAI-8457) `assert_ne!(state, Some('Z'))` was satisfied by two different outcomes: a child the timeout path killed and reaped, and a child it never killed at all, which sits in `S` while it sleeps on. Only one of those is the contract, so the assertion could not tell a working timeout path from a broken one. Assert the `/proc` entry is gone instead, and retry on any surviving state rather than only on `Z` so neither the post-`SIGKILL` teardown transient nor the detached reaper reads as a pass. Mutation-checked: unmutated passes; dropping `child.kill()` fails with `Some('S')` where the old assertion passed; dropping `reap_in_background` still fails with `Some('Z')`. Also record why the declined-launch behaviour has no Gherkin scenario, as AGENTS.md section 3 requires when one cannot be written. The premise needs a host that is GPU-present by the KFD/DRM sysfs rule yet cannot open `/dev/kfd`; no lane is that host and the harness cannot manufacture it. Co-Authored-By: Claude Opus 4.7 Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 24 +++++++---- .../features/model_serving.feature | 42 +++++++++++++++++++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 323e92949b..1ffec29b83 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -10036,20 +10036,28 @@ mod tests { .context("unparseable child pid")?; let _ = fs::remove_dir_all(&temp_root); - // The reaper races the kernel tearing the child down, so poll rather - // than sample once. Without the reap this burns the full budget and - // still reports `Z`. + // Poll until the entry is *gone*, which is the only state that proves + // both halves of the contract. `!= Z` would not: a timeout path that + // never killed the child leaves it in `S`, which satisfies `!= Z` while + // the child sleeps on. Two transients make this a poll rather than a + // single sample — the kernel tearing the child down after the `SIGKILL` + // (briefly still `R`/`S`) and the detached reaper waiting on the corpse + // (`Z`) — so anything short of `None` is retried until the deadline. + // + // No PID-reuse hazard: the loop stops at the first `None`, and the slot + // cannot be recycled before then because a zombie still occupies it. let deadline = Instant::now() + Duration::from_secs(10); let mut state = process_state(pid); - while state == Some('Z') && Instant::now() < deadline { + while state.is_some() && Instant::now() < deadline { thread::sleep(Duration::from_millis(25)); state = process_state(pid); } - assert_ne!( - state, - Some('Z'), - "pid {pid} is still a zombie: the timeout path killed the child but never reaped it" + assert_eq!( + state, None, + "pid {pid} was never reaped: the timeout path must kill the child and wait on it \ + (state `Z` means it was killed but not reaped; any live state means it was \ + never killed)" ); Ok(()) } diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 8ecd89f79e..15d1dbff8c 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -255,3 +255,45 @@ Feature: Model serving When the user serves a model with ROCR hiding every GPU a HIP mask names Then serving is refused before any engine starts And the user is told no AMD GPU was detected + + # KNOWN GAP — the `/dev/kfd` pre-flight on the serve probes (EAI-8457) has no + # scenario here, and cannot have one on any lane this suite runs. + # + # The behaviour: on a Linux host where `/dev/kfd` cannot be opened, `--gpu + # auto` no longer launches `amd-smi` at all (`detect_gpu_count`, + # `gpu_vram_usage`), so GPU auto-selection degrades to service state alone + # where it previously attempted the launch. That is user-observable, so §3 of + # AGENTS.md asks for a scenario rather than only the `rocm-core` unit tests + # that pin `kfd_readable` and `amd_smi_json`. + # + # Why no lane can carry it. The premise needs a host that is BOTH GPU-present + # by the KFD/DRM sysfs rule AND unable to open `/dev/kfd` — the real shape of + # a container started without `--device /dev/kfd`, or a user outside the + # render group, where world-readable sysfs still enumerates the device that + # `open(2)` then refuses. No lane is that host: + # + # - the no-GPU mock lane never reaches either probe. `usable_amd_gpu_indices` + # answers an authoritative empty set there and `serve` fails fast on it + # first — that is serve-13 above, which passes on every PR and is the + # evidence for this claim. Code after that bail is unreachable on the lane. + # - `@requires-gpu` lanes have a readable `/dev/kfd` by construction; every + # other GPU scenario depends on it. The declined branch never fires. + # - `@requires-wsl` exercises the wave-through (the gate is skipped on WSL), + # which is the opposite branch. + # + # Nor can the harness manufacture the premise: `KFD_DEVICE` is a compile-time + # constant in `rocm-core` with no path override, the node is root-owned so the + # harness can neither remove it nor make it unreadable, and supplementary + # group membership cannot be dropped without privilege. A scenario planting a + # stub `amd-smi` under `ROCM_CLI_DATA_DIR` would still take the *allowed* + # branch on a GPU lane and never run at all on the mock lane, so it would be + # green without exercising anything — the vacuity the rule exists to prevent. + # + # Covered instead at unit level in `rocm-core`, each confirmed non-vacuous by + # reverting the fix: `the_kfd_gate_stops_amd_smi_before_it_is_spawned` asserts + # the gate stops the *spawn* (a fake `amd-smi` leaves a marker file that must + # not appear), with the same fixture re-run against a readable device to prove + # the fake would otherwise have executed. + # + # Retire this note if a lane ever gains a container with sysfs GPU topology + # and no `/dev/kfd` passed through, which would make the premise reachable. From a3085af04e6aa6811c4340f6f163ee374583db2e Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 6 Oct 2026 16:54:36 +0300 Subject: [PATCH 06/32] fix(serve): restore the low-VRAM test hook my merge dropped Resolving the `gpu_vram_usage` conflict after #251 was squash-merged, I kept this branch's pre-refactor body and then restored main's dispatcher -- but not `simulated_low_vram_usage()` or its `#[cfg(feature = "e2e-test-hooks")]` call, both of which arrived with #251. The symbol count fell from main's 4 to 2, leaving only the two dangling references in the test module. Nothing in the default build reaches the hook, which is why 21 CI checks stayed green over it. It breaks only under the feature: cargo check -p rocm --features rocm/e2e-test-hooks --tests error[E0425]: cannot find function `simulated_low_vram_usage` and the step at serving_steps.rs:865 still set ROCM_E2E_FORCE_LOW_VRAM with nothing left to read it -- a scenario quietly asserting nothing. Restored verbatim from main: both the hook and its gated call are now byte-identical to origin/main, and the symbol count matches again. Verification note: the Linux devbox is unavailable (its pod has been Pending ~90m behind a 502 from the node proxy) and macOS cannot build this workspace -- `cargo check` stops in rocm-engine-lemonade on `u32 & u16` long before reaching this crate -- so the feature-gated compile has NOT been re-run here. It is pending, and the restored text is identical to the known-good tree. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 46cbbdde09..6fce638bac 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -22096,7 +22096,29 @@ const AMD_SMI_SERVE_PROBE_TIMEOUT: Duration = Duration::from_secs(10); /// [`read_drm_vram_usage`] for why the sysfs fallback withholds telemetry on a /// multi-GPU one). Returns `None` only when neither source is readable /// (callers then fall back to service-state-only auto-selection). +#[cfg(feature = "e2e-test-hooks")] +fn simulated_low_vram_usage() -> Option> { + let index: u32 = std::env::var("ROCM_E2E_FORCE_LOW_VRAM") + .ok()? + .trim() + .parse() + .ok()?; + // A single 24 GiB device with ~0.5 GiB free is well below the 0.90 free + // threshold, so `gpu_low_memory_warning` fires; one GPU keeps + // `vram_capacity_is_meaningful` honest on the non-APU vLLM lane. + let total_mb = 24 * 1024; + Some(vec![GpuVramUsage { + index, + used_mb: total_mb - 512, + total_mb, + }]) +} + fn gpu_vram_usage() -> Option> { + #[cfg(feature = "e2e-test-hooks")] + if let Some(forced) = simulated_low_vram_usage() { + return Some(forced); + } gpu_vram_usage_amd_smi().or_else(gpu_vram_usage_sysfs) } From cf427d9a33483ca305a32372382e9e4debbe05ab Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 7 Oct 2026 10:38:27 +0300 Subject: [PATCH 07/32] docs(serve): put the VRAM doc comments back on their own functions My restore of the low-VRAM hook in a3085af0 spliced the function in between `gpu_vram_usage`'s doc comment and `gpu_vram_usage` itself. The result compiles, so CI stayed green, but the amd-smi/sysfs fallback prose ended up describing `simulated_low_vram_usage`, `gpu_vram_usage` was left undocumented, and the hook's own twelve-line comment -- explaining why the seam exists and that production builds never see it -- was dropped entirely. Restores both comments to the functions they describe, matching 70d6aa13 (the last commit before the merge that dropped the hook) byte for byte. No executable code changes: `gpu_vram_usage`'s body is moved verbatim, and moving a top-level item is inert. Verified without a build host: the region is byte-identical to 70d6aa13, the diff touches no statement, `rustfmt --check --edition 2024` parses the file and reports no formatting change, and the three intra-doc link targets (`AUTO_FREE_VRAM_FRACTION`, `vram_capacity_is_meaningful`, `read_drm_vram_usage`) all still resolve. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 6fce638bac..97eb7b5340 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -22096,6 +22096,26 @@ const AMD_SMI_SERVE_PROBE_TIMEOUT: Duration = Duration::from_secs(10); /// [`read_drm_vram_usage`] for why the sysfs fallback withholds telemetry on a /// multi-GPU one). Returns `None` only when neither source is readable /// (callers then fall back to service-state-only auto-selection). +fn gpu_vram_usage() -> Option> { + #[cfg(feature = "e2e-test-hooks")] + if let Some(forced) = simulated_low_vram_usage() { + return Some(forced); + } + gpu_vram_usage_amd_smi().or_else(gpu_vram_usage_sysfs) +} + +/// E2E hook: when `ROCM_E2E_FORCE_LOW_VRAM` names a GPU ordinal, report that GPU +/// as almost entirely occupied, overriding the real telemetry. +/// +/// The serve-plan low-VRAM OOM warning (and, for vLLM, its +/// `--gpu-memory-utilization` note) only fires when a *selected* GPU is below +/// [`AUTO_FREE_VRAM_FRACTION`] free. The GPU lane's real cards are comfortably +/// free, so without this seam the branch is unreachable in E2E and the note has +/// no live coverage. Synthesizing one near-full single-GPU reading makes that +/// path deterministic on the vLLM GPU lane (a non-APU host, so +/// [`vram_capacity_is_meaningful`] still holds) without touching the device the +/// engine actually launches on. Compiled only into the `e2e-test-hooks` binary; +/// production builds never see it. #[cfg(feature = "e2e-test-hooks")] fn simulated_low_vram_usage() -> Option> { let index: u32 = std::env::var("ROCM_E2E_FORCE_LOW_VRAM") @@ -22114,14 +22134,6 @@ fn simulated_low_vram_usage() -> Option> { }]) } -fn gpu_vram_usage() -> Option> { - #[cfg(feature = "e2e-test-hooks")] - if let Some(forced) = simulated_low_vram_usage() { - return Some(forced); - } - gpu_vram_usage_amd_smi().or_else(gpu_vram_usage_sysfs) -} - /// The `amd-smi` half of [`gpu_vram_usage`], routed through the guarded /// launcher so a missing `/dev/kfd` declines the spawn instead of stalling. fn gpu_vram_usage_amd_smi() -> Option> { From 483798ba18daacf326669611f27811eea64d699a Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 10:19:24 +0300 Subject: [PATCH 08/32] fix(core): address review on the amd-smi KFD pre-flight Blocking items from juhovainio's review and the automated one, on top of this round's merge of main. Unbounded output (the one that reinstates a defect #526 already fixed): `amd_smi_json_with` pasted a failing `amd-smi`'s whole stdout/stderr into the error. `rocmd` renders that into its snapshot note and the GPU-metrics watcher copies the note into rewrite-the-whole-file JSONL logs on every poll -- the path that produced a 151K-character log line. Bounded with `MAX_QUOTED_OUTPUT_CHARS = 2000` through `examine::truncate_to_chars`, deliberately the same name, value and helper `fix/rocmai-448-drain-child- pipes` uses one layer down, so converging the two is a deletion rather than a behaviour change. No new abstraction is introduced here. WSL wave-through: `is_wsl_host()` alone repeated the mistake `b4f98bc2` corrected in the dashboard pre-flight -- a WSL host whose ROCDXG plumbing is not ready is one where amd-smi still hangs. Now `is_wsl_host() && has_usable_amd_gpu()`, the same conjunction `dash.rs::gpu_reachable_for_preflight` feeds the dashboard, so the cross-surface claim at `rocm-dash-collectors/src/amd_smi.rs` stays true. `&&` short-circuits, so bare metal never pays for the extra probe. Hardware-probe routing: main's `hardware_root` contract says every `/dev/kfd` read goes through `host_path`. The gate now does, with the real node keeping a veto exactly as the dash pre-flight does, and the module doc lists it. rocmd kept its own unprotected `run_command_with_timeout` for `run_rocm_capture_for_paths`, reachable from the MCP bridge: the same pipe-fill deadlock fixed everywhere else, and its timeout error quoted the child's whole output. Deleted for the shared helper. Also: `amd_smi_preflight_ok` was dead public API describing a call graph the code did not have -- deleted, docs repointed; three doc comments that contradicted the rewritten `run_command_with_timeout` corrected; the gate doc and decline message narrowed to what a read-open actually detects (inaccessible, not unhealthy); `drain_on_thread` uses `thread::Builder` so a thread-creation failure cannot panic out of the daemon; the e2e KNOWN GAP note corrected where its premises no longer hold. New tests, each confirmed to fail without its fix: - a_failing_amd_smi_cannot_paste_its_whole_output_into_the_error - the_public_amd_smi_entry_point_gates_on_the_real_kfd_device - the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto - the_kfd_preflight_waves_through_wsl_hosts_only_when_the_gpu_is_reachable Signed-off-by: Roman Sirokov --- apps/rocmd/src/common.rs | 45 +-- crates/rocm-core/src/examine.rs | 5 +- crates/rocm-core/src/hardware_root.rs | 22 +- crates/rocm-core/src/lib.rs | 340 ++++++++++++++---- .../features/model_serving.feature | 66 ++-- 5 files changed, 334 insertions(+), 144 deletions(-) diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index ca65f58773..f4058a0b37 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -5,9 +5,8 @@ use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, - CodexBridgeSnapshot, ExamineSummary, RocmCliConfig, amd_smi_json, command_failure_detail, - daemon_binary_path, default_engine_for_platform, format_host_port, - load_recent_automation_events, unix_time_millis, + CodexBridgeSnapshot, ExamineSummary, RocmCliConfig, amd_smi_json, daemon_binary_path, + default_engine_for_platform, format_host_port, load_recent_automation_events, unix_time_millis, }; #[cfg(test)] use rocm_engine_protocol::EnginePluginDescriptor; @@ -163,37 +162,6 @@ pub(crate) struct CommandCapture { pub(crate) stderr: String, } -pub(crate) fn run_command_with_timeout( - mut command: ProcessCommand, - timeout: Duration, -) -> Result { - let mut child = command.spawn().context("failed to spawn child process")?; - let started = std::time::Instant::now(); - loop { - if child - .try_wait() - .context("failed to poll child process")? - .is_some() - { - return child - .wait_with_output() - .context("failed to collect child process output"); - } - if started.elapsed() >= timeout { - let _ = child.kill(); - let output = child - .wait_with_output() - .context("failed to collect timed-out child process output")?; - bail!( - "process exceeded {}s timeout: {}", - timeout.as_secs(), - command_failure_detail(&output) - ); - } - thread::sleep(Duration::from_millis(50)); - } -} - pub(crate) fn run_rocm_capture(args: &[&str]) -> Result { let paths = AppPaths::discover()?; run_rocm_capture_for_paths(&paths, args, Duration::from_mins(2)) @@ -214,7 +182,14 @@ pub(crate) fn run_rocm_capture_for_paths( .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let output = run_command_with_timeout(command, timeout) + // The shared helper, not a local poll-and-`wait_with_output` loop: this path + // is reached from the MCP bridge's examine/update/sandbox-check handlers, + // where `rocm` can easily print more than the 64 KiB pipe buffer, and the + // local copy reported that as a timeout it had itself caused. It also quoted + // the whole of the child's output into that timeout error, which is the + // unbounded-note defect `fix/rocmai-448-drain-child-pipes` reproduced from + // the daemon's JSONL logs; the shared helper quotes none of it. + let output = rocm_core::run_command_with_timeout(command, timeout) .with_context(|| format!("failed to run {}", rocm_binary.display()))?; Ok(CommandCapture { argv: std::iter::once(rocm_binary.display().to_string()) diff --git a/crates/rocm-core/src/examine.rs b/crates/rocm-core/src/examine.rs index bad9aa6058..c06c2bf8eb 100644 --- a/crates/rocm-core/src/examine.rs +++ b/crates/rocm-core/src/examine.rs @@ -2762,7 +2762,10 @@ fn managed_runtime_roots() -> Vec<(std::path::PathBuf, Vec)> /// Truncate `value` to at most `max_chars` characters, appending a marker when /// truncated. Slices on char boundaries (matching Python's `value[:n]`); a byte /// slice would panic when the cut lands inside a multibyte character. -fn truncate_to_chars(value: String, max_chars: usize) -> String { +/// +/// `pub(crate)` for the `amd-smi` failure quote in `lib.rs`, which bounds raw +/// child output for the same reason the examination bounds captured text. +pub(crate) fn truncate_to_chars(value: String, max_chars: usize) -> String { if value.chars().count() > max_chars { let truncated: String = value.chars().take(max_chars).collect(); format!("{truncated}...[truncated]") diff --git a/crates/rocm-core/src/hardware_root.rs b/crates/rocm-core/src/hardware_root.rs index 4fd460f870..e90eea891b 100644 --- a/crates/rocm-core/src/hardware_root.rs +++ b/crates/rocm-core/src/hardware_root.rs @@ -39,14 +39,20 @@ //! machine. Process liveness under `/proc/` and `/dev/shm` sizing are //! about this process's environment, not the hardware. //! -//! Routed, but with the real device keeping a veto: the `rocm dash` `amd-smi` -//! pre-flight. It decides whether a real `amd-smi` process may start, so -//! `apps/rocm/src/dash.rs` passes `host_path("/dev/kfd")` to the dash daemon -//! and the collector requires both that path and the real `/dev/kfd` to be -//! readable. A simulated root can hide the GPU from the dashboard but cannot -//! start `amd-smi` on a machine without one. The WSL reachability verdict -//! still substitutes for the whole device check, as it always has, so the veto -//! does not cover it. +//! Routed, but with the real device keeping a veto: the two `amd-smi` +//! pre-flights. Each decides whether a real `amd-smi` process may start, so +//! each requires `host_path("/dev/kfd")` *and* the real `/dev/kfd` to be +//! readable. A simulated root can hide the GPU from them, but cannot start +//! `amd-smi` on a machine without one. They are: +//! +//! - the `rocm dash` collector's, where `apps/rocm/src/dash.rs` passes the +//! re-rooted path down to the dash daemon — those crates deliberately do not +//! depend on this one, so the caller resolves it; +//! - the synchronous one behind `crate::amd_smi_json`, which `rocm serve` and +//! `rocmd` reach and which calls [`host_path`] itself. +//! +//! The WSL reachability verdict still substitutes for the whole device check in +//! both, as it always has, so the veto does not cover it. //! //! In a normal build [`host_path`] is the identity: the path is returned as //! written and the probes read the real host. Only a build compiled with the diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 598cc69e48..ca405b2460 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -4487,48 +4487,70 @@ fn resolve_amd_smi_binary_in_home(home_dir: Option<&Path>) -> OsString { /// [`crate::examine::WslFacts`]). const KFD_DEVICE: &str = "/dev/kfd"; -/// Whether it is safe to launch `amd-smi` on this host. +/// Whether `/dev/kfd` can be opened for reading — the pre-flight every +/// `amd-smi` launch goes through, and it is **mandatory** before every launch. /// -/// On Linux this is a readability check on `/dev/kfd`, and it is **mandatory** -/// before every launch. Against a kernel fusion driver in a bad state — -/// partially installed, version-mismatched, wedged after a GPU fault, or passed -/// into a container from an unhealthy host — `amd-smi` blocks in uninterruptible -/// kernel sleep. No signal escapes that state, so the process survives both -/// Ctrl-C and `SIGKILL`, and a timeout cannot rescue it either: killing a -/// D-state child is a no-op and the `wait` that follows blocks alongside it. -/// Declining to launch is the only mitigation. +/// What it actually detects is **inaccessible**, not **unhealthy**. `kfd_open` +/// refuses when the node is absent, when permission is denied, when the driver +/// exposes no KFD GPU node, and while a reset is in progress; a driver that is +/// merely version-mismatched or wedged after a fault still opens. Those are the +/// states in which `amd-smi` blocks in uninterruptible kernel sleep — no signal +/// escapes, so the process survives both Ctrl-C and `SIGKILL`, and no timeout +/// rescues it, because killing a D-state child is a no-op. This gate removes +/// the reachable half of that hazard; it does not claim to see the rest. /// -/// Off Linux there is no such device and no such kernel state, so this returns -/// `true` without probing anything. That gate is load-bearing rather than -/// pedantic: `amd-smi.exe` is a supported Windows binary (see +/// Off Linux there is no such device and no such kernel state, so the check is +/// `true` without probing anything. That wave-through is load-bearing rather +/// than pedantic: `amd-smi.exe` is a supported Windows binary (see /// [`managed_sdk_tool_path`]), so a check that simply failed to find `/dev/kfd` /// there would silently disable GPU detection on every Windows host. /// -/// WSL needs the same wave-through for the same reason, and cannot get it from -/// `cfg`: it compiles as Linux yet has no `/dev/kfd` (see [`KFD_DEVICE`]), so -/// the real open would fail on every WSL host and decline every launch. WSL is -/// supported here — it has its own diagnosis catalog and its own GPU E2E lane — -/// and both `serve` probes discard the error with `.ok()?`, so that would be a -/// silent loss of GPU detection rather than a visible failure. The D-state hazard -/// this gate exists for is an `amdgpu`/KFD one, which is precisely the stack WSL -/// does not load. -#[must_use] -pub fn amd_smi_preflight_ok() -> bool { - kfd_readable(Path::new(KFD_DEVICE)) +/// WSL needs a wave-through for the same reason and cannot get it from `cfg`: +/// it compiles as Linux yet has no `/dev/kfd` (see [`KFD_DEVICE`]), so the real +/// open fails on every WSL host. But WSL identity alone is not the condition. +/// `b4f98bc2` replaced exactly such a bare identity check in the dashboard +/// pre-flight with [`has_usable_amd_gpu`], because `/dev/dxg` is vendor-agnostic +/// and a WSL host whose ROCDXG plumbing is not ready is one where `amd-smi` +/// still hangs — a different hazard from the KFD one, which an "is this WSL" +/// test cannot tell apart from a healthy host. So the substitution is +/// `is_wsl_host() && has_usable_amd_gpu()`, the same conjunction +/// `apps/rocm/src/dash.rs::gpu_reachable_for_preflight` feeds the dashboard, and +/// `serve`/`rocmd`/the dashboard keep the single cross-surface verdict that +/// commit established. On bare metal `is_wsl_host()` short-circuits, so +/// [`has_usable_amd_gpu`] is never consulted and behaviour is a plain +/// `/dev/kfd` open. +/// +/// Hardware-probe routing (see [`crate::hardware_root`]): like the dashboard +/// pre-flight, this reads `host_path("/dev/kfd")` *and* keeps the real node's +/// veto. A simulated E2E root can therefore make the gate decline on a machine +/// that has a GPU, but can never make it allow a launch on a machine that does +/// not. +#[cfg(target_os = "linux")] +fn kfd_readable(device: &Path) -> bool { + kfd_readable_with( + // `&&` short-circuits: `has_usable_amd_gpu` runs on WSL only. + is_wsl_host() && has_usable_amd_gpu(), + device, + &host_path(device), + ) } +/// [`kfd_readable`] with its two non-syscall inputs injected, so the +/// wave-through and the simulated-root routing can both be pinned from a test +/// on an ordinary Linux host rather than only on WSL or under an E2E root. +/// +/// `wsl_gpu_reachable` may only ever be the WSL conjunction described on +/// [`kfd_readable`]; it substitutes for the device open entirely, so widening it +/// to any fail-open verdict would disable the gate on bare metal. #[cfg(target_os = "linux")] -fn kfd_readable(device: &Path) -> bool { - kfd_readable_with(is_wsl_host(), device) +fn kfd_readable_with(wsl_gpu_reachable: bool, real_device: &Path, host_device: &Path) -> bool { + wsl_gpu_reachable + || (device_readable(real_device) + && (host_device == real_device || device_readable(host_device))) } -/// [`kfd_readable`] with the WSL determination injected, so the wave-through can -/// be pinned from a test on an ordinary Linux host rather than only on WSL. #[cfg(target_os = "linux")] -fn kfd_readable_with(is_wsl: bool, device: &Path) -> bool { - if is_wsl { - return true; - } +fn device_readable(device: &Path) -> bool { fs::OpenOptions::new().read(true).open(device).is_ok() } @@ -4537,8 +4559,27 @@ fn kfd_readable(_device: &Path) -> bool { true } +/// How much of a failed `amd-smi`'s own output the error message quotes. +/// +/// The bound is not cosmetic. `rocmd` renders this whole error chain into +/// `CodexBridgeGpuSnapshot.note` and the GPU-metrics watcher copies that note +/// into the automation-event and audit JSONL files on every poll — files whose +/// append helpers rewrite the whole file each time. Unbounded, one `amd-smi` +/// that prints a payload and then fails grows both logs by that much per tick, +/// and every subsequent append costs more than the last. A 151K-character JSONL +/// line was reproduced from exactly that poller. +/// +/// Deliberately the same name and value as the `MAX_QUOTED_OUTPUT_CHARS` in +/// `fix/rocmai-448-drain-child-pipes` (PR #526), which bounds the same quote for +/// the same reason one layer down, in its `run_with_timeout`. The two branches +/// are building overlapping subprocess-timeout helpers; keeping the bound +/// identical means converging on one of them is a deletion rather than a +/// behaviour change. Sized to still hold a real stderr backtrace: the point is +/// to stop a payload-sized quote, not to shorten diagnostics. +const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; + /// Run `amd-smi ` and parse its stdout as JSON, declining to launch at all -/// unless [`amd_smi_preflight_ok`] passes. +/// unless the `/dev/kfd` pre-flight in [`kfd_readable`] passes. /// /// This is the supported way to launch `amd-smi` from a synchronous caller. The /// pre-flight has to happen at every launch site or it protects nothing, and @@ -4546,10 +4587,10 @@ fn kfd_readable(_device: &Path) -> bool { /// added. /// /// `timeout` is defence in depth, not the mitigation: see -/// [`run_command_with_timeout`], which kills the child and then waits for it, -/// neither of which dislodges a process in uninterruptible sleep. It bounds the -/// ordinary failures — a slow or wedged-but-signalable `amd-smi` — and nothing -/// more. +/// [`run_command_with_timeout`], which kills the child and hands the `wait` to a +/// detached reaper — neither of which dislodges a process in uninterruptible +/// sleep. It bounds the ordinary failures — a slow or wedged-but-signalable +/// `amd-smi` — and it bounds this call's own return, not the child's lifetime. pub fn amd_smi_json(args: &[&str], timeout: Duration) -> Result { amd_smi_json_with( &resolve_amd_smi_binary(), @@ -4567,8 +4608,9 @@ fn amd_smi_json_with( ) -> Result { if !kfd_readable(kfd_device) { bail!( - "declined to launch `amd-smi {}`: {} is not readable, and amd-smi can block \ - unkillably in uninterruptible sleep against a kernel driver in that state", + "declined to launch `amd-smi {}`: {} could not be opened for reading (absent, \ + permission denied, no KFD GPU node, or a reset in progress), and amd-smi can \ + block unkillably in uninterruptible sleep against a device in that state", args.join(" "), kfd_device.display() ); @@ -4586,18 +4628,26 @@ fn amd_smi_json_with( if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + // Truncated, not dropped. This message reaches `rocmd`'s snapshot note + // and from there the JSONL logs, so the quote has to be bounded at the + // point that knows these characters are raw child output — see + // [`MAX_QUOTED_OUTPUT_CHARS`]. The head is where a failing `amd-smi` + // says what went wrong, and the command is named before it either way. bail!( "amd-smi {} failed: {}", args.join(" "), - if stderr.is_empty() { - if stdout.is_empty() { - format!("exit status {}", output.status) + crate::examine::truncate_to_chars( + if stderr.is_empty() { + if stdout.is_empty() { + format!("exit status {}", output.status) + } else { + stdout + } } else { - stdout - } - } else { - stderr - } + stderr + }, + MAX_QUOTED_OUTPUT_CHARS, + ) ); } @@ -4610,8 +4660,8 @@ fn amd_smi_json_with( /// /// Bounds signalable children only. A child in uninterruptible sleep ignores /// the `kill`, so callers that can reach one must decline to spawn it in the -/// first place — see [`amd_smi_preflight_ok`]. This function at least returns -/// to its caller in that case rather than joining the child in its wait. +/// first place — see the pre-flight on [`amd_smi_json`]. This function at least +/// returns to its caller in that case rather than joining the child in its wait. /// /// On timeout the calling thread returns without waiting on the child; the /// `wait` is handed to a detached reaper thread instead, because @@ -4630,8 +4680,8 @@ pub fn run_command_with_timeout( // (64 KiB on Linux) blocks mid-write and never exits, so a merely verbose // command would be reported as having hung. `amd-smi metric --json` on a // multi-GPU host clears 64 KiB comfortably. - let stdout = child.stdout.take().map(drain_on_thread); - let stderr = child.stderr.take().map(drain_on_thread); + let stdout = child.stdout.take().and_then(drain_on_thread); + let stderr = child.stderr.take().and_then(drain_on_thread); let started = Instant::now(); loop { if let Some(status) = child.try_wait().context("failed to poll child process")? { @@ -4683,15 +4733,19 @@ pub fn run_command_with_timeout( /// Does not guarantee: /// /// - That the child dies. A process in uninterruptible sleep ignores `SIGKILL`, -/// so its reaper parks until the kernel lets it go, possibly forever. The -/// resource leaked in that case is a blocked thread rather than a zombie PID -/// — no worse than what it replaces, and better than blocking the caller, but -/// still a leak. The only real mitigation is declining to spawn against a -/// suspect device: see [`amd_smi_preflight_ok`]. +/// so its reaper parks until the kernel lets it go, possibly forever — and so +/// do both reader threads, which only end at EOF on their pipes. The resources +/// leaked in that case are three parked threads rather than a zombie PID: +/// better than blocking the caller, but still a leak, and a larger one than +/// the single blocked caller thread the pre-rewrite code would have had. The +/// only real mitigation is declining to spawn against a suspect device: see +/// the pre-flight on [`amd_smi_json`]. /// - That a reaper always exists. If the thread cannot be spawned, the child is -/// dropped and left as a zombie, which is the pre-existing failure mode and -/// strictly better than propagating an error from a path that is already -/// reporting one. +/// dropped and left as a zombie. That is strictly better than propagating an +/// error from a path that is already reporting one — but it is *not* what the +/// pre-rewrite code did: that waited synchronously after the `kill` and so +/// left no zombie, at the cost of blocking the caller in the D-state case +/// this function exists to return from. /// /// Reaper threads do not accumulate under normal operation: one is created per /// timeout, and each one exits as soon as its child does. Accumulation needs a @@ -4705,12 +4759,23 @@ fn reap_in_background(mut child: std::process::Child) { }); } -fn drain_on_thread(mut pipe: R) -> thread::JoinHandle> { - thread::spawn(move || { - let mut buffer = Vec::new(); - let _ = pipe.read_to_end(&mut buffer); - buffer - }) +/// Start draining `pipe` on its own thread, or return `None` if no thread can be +/// started. +/// +/// `thread::Builder` rather than `thread::spawn` for the same reason +/// [`reap_in_background`] uses it: a panic here would unwind out of a daemon +/// with the child already spawned and nobody left to kill or reap it. Failing to +/// start a reader instead costs that stream's output and leaves the timeout to +/// bound the call, which is survivable. +fn drain_on_thread(mut pipe: R) -> Option>> { + thread::Builder::new() + .name("rocm-drain-child-pipe".to_owned()) + .spawn(move || { + let mut buffer = Vec::new(); + let _ = pipe.read_to_end(&mut buffer); + buffer + }) + .ok() } fn join_drained(reader: Option>>) -> Vec { @@ -6580,7 +6645,6 @@ mod tests { #[cfg(not(target_os = "linux"))] fn the_kfd_preflight_waves_through_hosts_that_have_no_kfd() { assert!(kfd_readable(Path::new("/definitely/not/a/device/node"))); - assert!(amd_smi_preflight_ok()); } /// WSL has no `/dev/kfd` either, but unlike Windows it cannot be waved @@ -6590,20 +6654,152 @@ mod tests { /// `.ok()?`. Same wave-through as the off-Linux case above, decided at /// runtime instead. /// - /// The bare-metal assertion is the control: without it this would still - /// pass with the wave-through deleted. + /// The *conjunction* is what is pinned, not WSL identity: a WSL host whose + /// ROCDXG plumbing is not ready is one where `amd-smi` still hangs, which is + /// why `b4f98bc2` replaced a bare identity check with `has_usable_amd_gpu()` + /// in the dashboard pre-flight. The unready-WSL case below is the one that + /// distinguishes this from that mistake; the bare-metal case is the control + /// that stops the whole test passing with the gate deleted. #[test] #[cfg(target_os = "linux")] - fn the_kfd_preflight_waves_through_wsl_hosts() { + fn the_kfd_preflight_waves_through_wsl_hosts_only_when_the_gpu_is_reachable() { let absent = Path::new("/definitely/not/a/device/node"); assert!( - kfd_readable_with(true, absent), - "WSL has no /dev/kfd, so its absence must not decline the launch" + kfd_readable_with(true, absent, absent), + "a ready WSL host has no /dev/kfd, so its absence must not decline the launch" + ); + assert!( + !kfd_readable_with(false, absent, absent), + "an unready WSL host — or a bare-metal one with no readable KFD — must be declined" + ); + } + + /// The simulated-root routing, which `kfd_readable` applies to the real + /// `/dev/kfd` on the production path. + /// + /// Both halves are load-bearing and they fail in opposite directions. A + /// simulated root that hides the device must decline (so an E2E lane can + /// manufacture the declined branch on a machine that does have a GPU), and + /// a simulated root that *invents* a device must not allow (so no test hook + /// can ever start a real `amd-smi` on a machine that has no GPU — the veto + /// `hardware_root`'s module contract requires of this probe). + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto() -> Result<()> { + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-root-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let present = temp_root.join("present-kfd"); + fs::write(&present, b"")?; + let absent = temp_root.join("absent-kfd"); + + let real_present_root_absent = kfd_readable_with(false, &present, &absent); + let real_absent_root_present = kfd_readable_with(false, &absent, &present); + let both_present = kfd_readable_with(false, &present, &present); + let _ = fs::remove_dir_all(&temp_root); + + assert!( + !real_present_root_absent, + "a simulated root that hides the device must decline the launch" + ); + assert!( + !real_absent_root_present, + "a simulated root must never be able to start amd-smi on a host with no real device" + ); + assert!( + both_present, + "the ordinary case must still pass, or the two above prove nothing" + ); + Ok(()) + } + + /// Pins the production wiring the injected helpers cannot: that + /// [`amd_smi_json`] gates on [`KFD_DEVICE`] itself. + /// + /// Every other gate test reaches `amd_smi_json_with` with a device of its + /// own choosing, so swapping the constant out of the public wrapper — which + /// disables the gate for every production caller — leaves them all green. + /// This compares the wrapper's observable verdict against the gate read + /// directly, so the two can only agree while the wrapper passes the real + /// device through. + /// + /// Non-vacuous on any host with no readable `/dev/kfd` (every non-GPU CI + /// runner): there the gate says "decline" and a wrapper that no longer + /// consults it does not. On a GPU host both sides say "allow" and this + /// asserts only that no spurious decline appeared. + #[test] + #[cfg(target_os = "linux")] + fn the_public_amd_smi_entry_point_gates_on_the_real_kfd_device() { + let gate_allows = kfd_readable(Path::new(KFD_DEVICE)); + + // A deliberately bogus subcommand: if the gate allows the launch this + // still fails, but with a *launch* error rather than a decline, and it + // is the decline that is under test. The timeout is short because the + // pre-flight has already passed by the time anything is spawned. + let error = amd_smi_json(&["--rocm-cli-gate-probe"], Duration::from_secs(5)).err(); + let declined = error.is_some_and(|error| { + let message = format!("{error:#}"); + message.contains("declined to launch") && message.contains(KFD_DEVICE) + }); + + assert_eq!( + declined, !gate_allows, + "amd_smi_json must decline exactly when the gate on {KFD_DEVICE} does; a wrapper \ + that passes some other path declines on neither host" + ); + } + + /// A failing `amd-smi` that prints a payload must not paste the payload into + /// the error. `rocmd` puts this message in its snapshot note and the + /// GPU-metrics watcher copies that note into rewrite-the-whole-file JSONL + /// logs on every poll, which is how a 151K-character log line was produced. + #[test] + #[cfg(target_os = "linux")] + fn a_failing_amd_smi_cannot_paste_its_whole_output_into_the_error() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if is_wsl_host() { + return Ok(()); + } + + const PAYLOAD_CHARS: usize = 200_000; + + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-amd-smi-quote-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let fake_amd_smi = temp_root.join("amd-smi"); + fs::write( + &fake_amd_smi, + format!("#!/bin/sh\nyes x | head -c {PAYLOAD_CHARS} | tr -d '\\n' >&2\nexit 1\n"), + )?; + fs::set_permissions(&fake_amd_smi, fs::Permissions::from_mode(0o755))?; + let readable_device = temp_root.join("stand-in-kfd"); + fs::write(&readable_device, b"")?; + + let error = amd_smi_json_with( + fake_amd_smi.as_os_str(), + &readable_device, + &["list", "--json"], + Duration::from_secs(30), + ) + .expect_err("a child that exits 1 must fail the call"); + let message = format!("{error:#}"); + let _ = fs::remove_dir_all(&temp_root); + + assert!( + message.chars().count() < PAYLOAD_CHARS / 10, + "the failure message quoted {} characters of child output unbounded", + message.chars().count() ); assert!( - !kfd_readable_with(false, absent), - "a bare-metal host with no readable KFD must still be declined" + message.contains("amd-smi list --json failed"), + "the bound must not cost the message its subject: {message}" ); + assert!( + message.contains("[truncated]"), + "an over-long quote must say it was cut, not look like the whole story: {message}" + ); + Ok(()) } #[test] diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index d0d839f20a..91d34e5b73 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -323,7 +323,9 @@ Feature: Model serving And running that command actually stops the service # KNOWN GAP — the `/dev/kfd` pre-flight on the serve probes (EAI-8457) has no - # scenario here, and cannot have one on any lane this suite runs. + # scenario here yet. It is now reachable, which it was not when this note was + # first written; what follows records what changed rather than arguing the + # behaviour is untestable, because it no longer is. # # The behaviour: on a Linux host where `/dev/kfd` cannot be opened, `--gpu # auto` no longer launches `amd-smi` at all (`detect_gpu_count`, @@ -332,34 +334,42 @@ Feature: Model serving # AGENTS.md asks for a scenario rather than only the `rocm-core` unit tests # that pin `kfd_readable` and `amd_smi_json`. # - # Why no lane can carry it. The premise needs a host that is BOTH GPU-present - # by the KFD/DRM sysfs rule AND unable to open `/dev/kfd` — the real shape of - # a container started without `--device /dev/kfd`, or a user outside the - # render group, where world-readable sysfs still enumerates the device that - # `open(2)` then refuses. No lane is that host: + # The premise needs a host that is BOTH GPU-present by the KFD/DRM sysfs rule + # AND unable to open `/dev/kfd` — the real shape of a container started + # without `--device /dev/kfd`, or a user outside the render group, where + # world-readable sysfs still enumerates the device that `open(2)` then + # refuses. No lane is naturally that host, but two earlier claims about why it + # could never be manufactured were wrong and are corrected here: # - # - the no-GPU mock lane never reaches either probe. `usable_amd_gpu_indices` - # answers an authoritative empty set there and `serve` fails fast on it - # first — that is serve-13 above, which passes on every PR and is the - # evidence for this claim. Code after that bail is unreachable on the lane. - # - `@requires-gpu` lanes have a readable `/dev/kfd` by construction; every - # other GPU scenario depends on it. The declined branch never fires. - # - `@requires-wsl` exercises the wave-through (the gate is skipped on WSL), - # which is the opposite branch. + # - "the no-GPU mock lane never reaches either probe" is false. It holds for + # serve-13's plain path, where `usable_amd_gpu_indices` answers an + # authoritative empty set and `serve` bails first — but + # `ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE` (set by the mock lane, see + # `serving_steps.rs`) skips exactly that bail in `serve_cmd.rs`, and + # `gpu_vram_usage()` is called below it. `host_accelerator_memory` in + # `main.rs` reaches the same probe from `diagnose --model` with no GPU + # guard at all. The declined branch therefore does run on the mock lane. + # - "`KFD_DEVICE` is a compile-time constant with no path override" is now + # false too: the gate is routed through `rocm_core::host_path`, so an + # `e2e-test-hooks` build with `ROCM_CLI_TEST_HOST_ROOT` set can present a + # simulated host whose KFD topology says GPU-present and whose `/dev/kfd` + # is absent. The real device keeps its veto, so this can only manufacture + # the *declined* branch, never a spurious allow. # - # Nor can the harness manufacture the premise: `KFD_DEVICE` is a compile-time - # constant in `rocm-core` with no path override, the node is root-owned so the - # harness can neither remove it nor make it unreadable, and supplementary - # group membership cannot be dropped without privilege. A scenario planting a - # stub `amd-smi` under `ROCM_CLI_DATA_DIR` would still take the *allowed* - # branch on a GPU lane and never run at all on the mock lane, so it would be - # green without exercising anything — the vacuity the rule exists to prevent. + # What is still true: `@requires-gpu` lanes have a readable `/dev/kfd` by + # construction, and `@requires-wsl` exercises the wave-through, which is the + # opposite branch. So the scenario belongs on the mock lane, built from the + # simulated root plus a hanging stub `amd-smi` — `serve` and `diagnose` must + # complete promptly where before this change they would have hung on it. # - # Covered instead at unit level in `rocm-core`, each confirmed non-vacuous by - # reverting the fix: `the_kfd_gate_stops_amd_smi_before_it_is_spawned` asserts - # the gate stops the *spawn* (a fake `amd-smi` leaves a marker file that must - # not appear), with the same fixture re-run against a readable device to prove - # the fake would otherwise have executed. + # Covered meanwhile at unit level in `rocm-core`, each confirmed non-vacuous + # by mutation: `the_kfd_gate_stops_amd_smi_before_it_is_spawned` asserts the + # gate stops the *spawn* (a fake `amd-smi` leaves a marker file that must not + # appear), with the same fixture re-run against a readable device to prove the + # fake would otherwise have executed; + # `the_public_amd_smi_entry_point_gates_on_the_real_kfd_device` pins that the + # production wrapper gates on `/dev/kfd` itself; and + # `the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto` pins + # the routing the scenario above would depend on. # - # Retire this note if a lane ever gains a container with sysfs GPU topology - # and no `/dev/kfd` passed through, which would make the premise reachable. + # Retire this note when that mock-lane scenario lands. From 676ae1ba99390d51651db5895e3e504497f896d9 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 10:21:31 +0300 Subject: [PATCH 09/32] test(core): make the gate-wiring test non-vacuous under a simulated host root On a machine that has a readable /dev/kfd the comparison alone is vacuous: both the gate and a wrapper passing some other readable path say "allow". Under an e2e-test-hooks build an empty simulated root has no dev/kfd, so the gate declines on every host while the obvious mutant (Path::new("/")) re-roots to the root directory and opens. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index ca405b2460..1f2e53dc89 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -6723,13 +6723,29 @@ mod tests { /// directly, so the two can only agree while the wrapper passes the real /// device through. /// - /// Non-vacuous on any host with no readable `/dev/kfd` (every non-GPU CI - /// runner): there the gate says "decline" and a wrapper that no longer - /// consults it does not. On a GPU host both sides say "allow" and this - /// asserts only that no spurious decline appeared. + /// Non-vacuous on *every* host in an `e2e-test-hooks` build: the empty + /// simulated root below has no `dev/kfd`, so the gate declines whatever the + /// machine has, while a wrapper passing any other path — `/` being the + /// obvious mutant — re-roots to the root directory itself, which opens. + /// Without the feature it is non-vacuous on any host with no readable + /// `/dev/kfd` (every non-GPU CI runner) and asserts only the absence of a + /// spurious decline on a GPU host. #[test] #[cfg(target_os = "linux")] fn the_public_amd_smi_entry_point_gates_on_the_real_kfd_device() { + #[cfg(feature = "e2e-test-hooks")] + let _simulated_host = { + let guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let root = tempfile::tempdir().expect("a simulated host root"); + let variable = crate::test_env::RestoredEnvVar::set( + crate::hardware_root::TEST_HOST_ROOT_ENV, + root.path(), + ); + (guard, variable, root) + }; + let gate_allows = kfd_readable(Path::new(KFD_DEVICE)); // A deliberately bogus subcommand: if the gate allows the launch this From 22ba7ed1956179575208a583b44bbf1c2dd14013 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 10:22:52 +0300 Subject: [PATCH 10/32] test(core): serialise the device-gate tests on the simulated-root lock The gate now resolves its device through host_path, so an e2e-test-hooks build in which one test sets ROCM_CLI_TEST_HOST_ROOT re-roots another test's tempdir device out from under it. Same lock fix.rs already holds for the same reason. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 1f2e53dc89..d9abb17ba7 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -6444,6 +6444,21 @@ mod tests { Ok(()) } + /// Serialise a test that reads a device node through [`kfd_readable`]. + /// + /// The gate resolves its device through `host_path`, so in an + /// `e2e-test-hooks` build another test setting `ROCM_CLI_TEST_HOST_ROOT` + /// process-wide would re-root this test's own tempdir device out from under + /// it, and the gate would decline a device the test had just created. This + /// is the same lock `fix.rs` holds for the same reason; without the feature + /// it is uncontended and costs nothing. + #[cfg(target_os = "linux")] + fn host_root_guard() -> std::sync::MutexGuard<'static, ()> { + crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + /// The gate has to stop the *spawn*, not merely report a failure after the /// fact: the launch that already happened is the launch that hangs. The /// fake `amd-smi` leaves a marker file, so its absence is direct evidence @@ -6464,6 +6479,7 @@ mod tests { if is_wsl_host() { return Ok(()); } + let _host_root = host_root_guard(); let temp_root = std::env::temp_dir().join(format!("rocm-cli-kfd-gate-{}", unix_time_millis())); @@ -6686,6 +6702,7 @@ mod tests { #[test] #[cfg(target_os = "linux")] fn the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto() -> Result<()> { + let _host_root = host_root_guard(); let temp_root = std::env::temp_dir().join(format!("rocm-cli-kfd-root-{}", unix_time_millis())); fs::create_dir_all(&temp_root)?; @@ -6733,17 +6750,15 @@ mod tests { #[test] #[cfg(target_os = "linux")] fn the_public_amd_smi_entry_point_gates_on_the_real_kfd_device() { + let _host_root = host_root_guard(); #[cfg(feature = "e2e-test-hooks")] let _simulated_host = { - let guard = crate::hardware_root::HOST_ROOT_TEST_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); let root = tempfile::tempdir().expect("a simulated host root"); let variable = crate::test_env::RestoredEnvVar::set( crate::hardware_root::TEST_HOST_ROOT_ENV, root.path(), ); - (guard, variable, root) + (variable, root) }; let gate_allows = kfd_readable(Path::new(KFD_DEVICE)); @@ -6778,6 +6793,8 @@ mod tests { return Ok(()); } + let _host_root = host_root_guard(); + const PAYLOAD_CHARS: usize = 200_000; let temp_root = From a0a791b41c9de52cd699ea39cd31074750b38fb3 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 10:51:54 +0300 Subject: [PATCH 11/32] fix(core): hoist a test const above the statements clippy objects to `clippy::items_after_statements` rejects `const PAYLOAD_CHARS` declared part-way through the test body: items exist from the start of the scope regardless, so declaring one mid-function reads as if it did not. Caught by the full workspace clippy run, which the agent that wrote this test did not reach before stalling. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index d9abb17ba7..3f840e3e71 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -6789,14 +6789,14 @@ mod tests { fn a_failing_amd_smi_cannot_paste_its_whole_output_into_the_error() -> Result<()> { use std::os::unix::fs::PermissionsExt; + const PAYLOAD_CHARS: usize = 200_000; + if is_wsl_host() { return Ok(()); } let _host_root = host_root_guard(); - const PAYLOAD_CHARS: usize = 200_000; - let temp_root = std::env::temp_dir().join(format!("rocm-cli-amd-smi-quote-{}", unix_time_millis())); fs::create_dir_all(&temp_root)?; From 9162d2b81af90b72a75441cb254f765733b99aad Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 10:59:55 +0300 Subject: [PATCH 12/32] test(core): spell the host-root lock so the env-mutation guard sees it `xtask`'s env-mutation contract failed on this test: crates/rocm-core/src/lib.rs:6757: RestoredEnvVar::set The lock was already held -- `host_root_guard()` on the line above takes HOST_ROOT_TEST_LOCK -- but that guard matches TEXT, recognising a serializer by the `_TEST_LOCK` suffix or the name `ScopedTestEnv`. The helper's name carries neither, so a correctly serialized test read as an unguarded mutation. This is the only test here that both takes the lock and calls `RestoredEnvVar::set`, which is why the other three using the helper pass. Acquire the same lock by name instead, with a comment so the next person does not "simplify" it back to the helper. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 3f840e3e71..5a984273ec 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -6750,7 +6750,14 @@ mod tests { #[test] #[cfg(target_os = "linux")] fn the_public_amd_smi_entry_point_gates_on_the_real_kfd_device() { - let _host_root = host_root_guard(); + // Acquired by name rather than through `host_root_guard()`: this is the + // one test here that also calls `RestoredEnvVar::set`, and the + // env-mutation contract guard matches TEXT -- it recognises a + // serializer by the `_TEST_LOCK` suffix, which the helper's name hides. + // Same lock, spelled so the guard can see it. + let _host_root = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); #[cfg(feature = "e2e-test-hooks")] let _simulated_host = { let root = tempfile::tempdir().expect("a simulated host root"); From d84c1571371bd2998363feff4522bef521a1bc93 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 15:27:04 +0300 Subject: [PATCH 13/32] fix(core): make the off-Linux KFD wave-through a const fn The Windows lane builds with `-D warnings` and runs clippy, so `clippy::missing_const_for_fn` turned the `cfg(not(target_os = "linux"))` stub into a hard compile error there. The Linux body opens a device and cannot be const, so the lint only ever fires on the other branch of the pair -- which no Linux-side gate can catch. error: this could be a `const fn` --> crates\rocm-core\src\lib.rs:4619:1 Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 5a984273ec..ac1f714525 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -4554,8 +4554,16 @@ fn device_readable(device: &Path) -> bool { fs::OpenOptions::new().read(true).open(device).is_ok() } +/// Off-Linux counterpart of [`kfd_readable`]: see that function's docs for why +/// the wave-through is load-bearing rather than a stub. +/// +/// `const` because it has to be. The Windows lane builds with `-D warnings` and +/// runs clippy, so `clippy::missing_const_for_fn` makes this a hard compile +/// error there while the Linux lane — where the real body opens a device and +/// cannot be `const` — never sees the lint. A `cfg`-selected pair of functions +/// is linted per branch, not per name. #[cfg(not(target_os = "linux"))] -fn kfd_readable(_device: &Path) -> bool { +const fn kfd_readable(_device: &Path) -> bool { true } From 62d0786e41bf1eeb27591632ee4d26600f0b2367 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 16:24:54 +0300 Subject: [PATCH 14/32] fix(core): restore the io::Read import the lib.rs extraction took with it Merging main's `process.rs`/`net.rs`/`paths.rs` extraction (#599) into this branch is a textbook silent semantic conflict: main deleted `use std::io::Read` from `lib.rs` along with the code that needed it, this branch added `drain_on_thread` to the same file, and the two edits never touched adjacent lines -- so git merged them without a marker and left a module that cannot compile. error[E0405]: cannot find trait `Read` in this scope --> crates/rocm-core/src/lib.rs:2869:23 `run_command_with_timeout` and its drain threads are not `cfg`-gated, so the import is live on every target and cannot trip the Windows lane's `-D warnings` unused-import check. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 4cf849722e..5e77cf9616 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -7,13 +7,16 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs; +// `Read`/`Command`/`Stdio`/`thread`/`Instant` are this branch's, not survivors +// of the `process.rs` extraction (#599): `run_command_with_timeout` and its +// drain and reaper threads still live in this module, so the `Read` bound that +// `drain_on_thread` needs has to come back even though the extraction took the +// import with it. The `windows_sys` block that used to sit here left with that +// extraction and is deliberately not kept — nothing in this file uses it any +// more, and an unused import is a hard error on the Windows lane's +// `-D warnings` build. +use std::io::Read; use std::path::{Path, PathBuf}; -// `Command`/`Stdio`/`thread`/`Instant` are this branch's, not survivors of the -// `process.rs` extraction (#599): `run_command_with_timeout` and its drain and -// reaper threads still live in this module. The `windows_sys` block that used -// to sit here left with that extraction and is deliberately not kept — nothing -// in this file uses it any more, and an unused import is a hard error on the -// Windows lane's `-D warnings` build. use std::process::{Command, Stdio}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; From a75a68e1d7c66ac8cf6dbc386e3899b9009f2e7b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 13 Oct 2026 16:30:48 +0300 Subject: [PATCH 15/32] test(core): plant the fake amd-smi through the ETXTBSY-safe writer Both new amd-smi tests wrote their stand-in binary with `fs::write` plus `set_permissions` and then executed it. Main's #649 landed `fake_executable::write_fake_executable` precisely because that pattern races every other test thread in the same process: a parallel `fork` inherits the writer's fd and the kernel refuses to `exec` a file anyone still holds open for writing. The merge brought the helper in but could not retrofit these call sites, and the race duly fired on a 112-core run: ---- tests::the_kfd_gate_stops_amd_smi_before_it_is_spawned stdout ---- a readable device must let the launch through: 0: failed to spawn child process 1: Text file busy (os error 26) Routing both plants through the helper makes the gate test deterministic again; `cargo test -p rocm-core --lib` now passes three runs in a row where it failed one in two before. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 5e77cf9616..7389a275e5 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -3162,8 +3162,6 @@ mod tests { #[test] #[cfg(target_os = "linux")] fn the_kfd_gate_stops_amd_smi_before_it_is_spawned() -> Result<()> { - use std::os::unix::fs::PermissionsExt; - // WSL compiles as Linux but is waved through deliberately (see // `the_kfd_preflight_waves_through_wsl_hosts`), so there is no gate to // exercise on such a host and the blocked case could never hold. @@ -3177,11 +3175,13 @@ mod tests { fs::create_dir_all(&temp_root)?; let marker = temp_root.join("amd-smi-ran"); let fake_amd_smi = temp_root.join("amd-smi"); - fs::write( + // Not `fs::write` + `set_permissions`: this process is about to execute + // the file, and a write fd leaked into a parallel test's fork would make + // the first run fail with `ETXTBSY`. See `fake_executable`. + crate::fake_executable::write_fake_executable( &fake_amd_smi, format!("#!/bin/sh\ntouch '{}'\necho '{{}}'\n", marker.display()), )?; - fs::set_permissions(&fake_amd_smi, fs::Permissions::from_mode(0o755))?; let readable_device = temp_root.join("stand-in-kfd"); fs::write(&readable_device, b"")?; @@ -3485,8 +3485,6 @@ mod tests { #[test] #[cfg(target_os = "linux")] fn a_failing_amd_smi_cannot_paste_its_whole_output_into_the_error() -> Result<()> { - use std::os::unix::fs::PermissionsExt; - const PAYLOAD_CHARS: usize = 200_000; if is_wsl_host() { @@ -3499,11 +3497,12 @@ mod tests { std::env::temp_dir().join(format!("rocm-cli-amd-smi-quote-{}", unix_time_millis())); fs::create_dir_all(&temp_root)?; let fake_amd_smi = temp_root.join("amd-smi"); - fs::write( + // See the sibling gate test: planting an executable this process then + // runs has to go through `fake_executable` or it races `ETXTBSY`. + crate::fake_executable::write_fake_executable( &fake_amd_smi, format!("#!/bin/sh\nyes x | head -c {PAYLOAD_CHARS} | tr -d '\\n' >&2\nexit 1\n"), )?; - fs::set_permissions(&fake_amd_smi, fs::Permissions::from_mode(0o755))?; let readable_device = temp_root.join("stand-in-kfd"); fs::write(&readable_device, b"")?; From 983e11dd559629eea5715e55e79760d7041318c1 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 09:50:57 +0300 Subject: [PATCH 16/32] fix(rocmd): route the sandbox timeout through the consolidated helper Addresses review on PR #447. `apps/rocmd/src/sandbox.rs::run_process_with_timeout` was a third copy of the poll-and-kill loop from the same split (e760d9a5) that produced the amd-smi ones, and carried the same defect: it polled `try_wait` with both pipes undrained, so a sandbox child that filled the 64 KiB pipe buffer blocked mid-write and was then reported as having exceeded its timeout. Route it through `rocm_core::run_command_with_timeout`, which drains both pipes on their own threads and reaps a killed child on a detached thread. Bound the gate's own `/dev/kfd` open so `kfd_readable`'s safety no longer rests on the unverifiable assumption that `kfd_open` always returns promptly. An open that overruns `KFD_OPEN_TIMEOUT` reads as "not readable", which is the declining answer, so the unanticipated case fails in the safe direction. Pinned by a test against a FIFO with no writer, whose `open` really does block in the kernel. Deduplicate the stderr-then-stdout precedence `amd_smi_json_with` had inlined into `command_failure_detail_or_else`, which `command_failure_detail` now delegates to; only the both-blank fallback differs between the two callers. Document the `serve` worst-case probe latency (two sequential 10s bounds, replacing two unbounded `.output()` calls) and why `rocmd`'s 2s bound on the same helper is deliberately shorter. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 22 +++++ apps/rocmd/src/common.rs | 10 +++ apps/rocmd/src/sandbox.rs | 54 +++++------ crates/rocm-core/src/lib.rs | 172 +++++++++++++++++++++++++++++++++--- 4 files changed, 215 insertions(+), 43 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 99b455121d..d839c0cac9 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -14586,6 +14586,28 @@ fn select_auto_gpu_index( /// alone without them — so a stalled `amd-smi` must not hold up a launch. This /// bounds the ordinary stalls only; the unkillable one is prevented by the /// pre-flight inside [`rocm_core::amd_smi_json`], not by any timeout. +/// +/// Used by `serve` only — [`gpu_vram_usage_amd_smi`] and [`detect_gpu_count`] +/// are its two consumers and both are reached solely from `serve_cmd::serve`. +/// `rocmd`'s watcher has its own, shorter `AMD_SMI_PROBE_TIMEOUT` (2s) for the +/// same helper: that one fires on a polling cadence where a stalled probe costs +/// a missed telemetry tick and nothing else, so it can afford to give up early, +/// while a serve launch gets exactly one chance to learn the GPU layout and +/// giving up early there degrades the placement decision itself. +/// +/// Worst-case latency, deliberately accepted rather than parallelised: `serve` +/// runs the two probes sequentially — `gpu_vram_usage` before the launch lock, +/// `detect_gpu_count` either in the `--gpu ` validation or inside the +/// lock for `--gpu auto` — so a host where `amd-smi` passes the KFD gate and +/// then stalls both times adds up to ~20s to an interactive launch. Before this +/// bound existed both probes called `.output()` with no timeout at all, so that +/// same host hung indefinitely; 20s is the ceiling replacing "forever", not a +/// new cost. Running them concurrently is not free: `detect_gpu_count` is +/// invoked inside [`select_gpu_indices_under_launch_lock`] precisely so the +/// cpu-only and pinned paths never pay for the subprocess, and hoisting it out +/// to overlap with the VRAM probe would make every `rocm serve --cpu-only` spawn +/// an `amd-smi` it has no use for. The stall is rare, the extra spawn would be +/// unconditional. const AMD_SMI_SERVE_PROBE_TIMEOUT: Duration = Duration::from_secs(10); /// Best-effort per-GPU VRAM occupancy. Prefers `amd-smi metric --json`; when diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index f4058a0b37..53e1fb14b4 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -25,6 +25,16 @@ use std::process::{Command as ProcessCommand, Stdio}; use std::thread; use std::time::Duration; +/// How long `rocmd` waits on an `amd-smi` probe before giving up on it. +/// +/// Deliberately 5x shorter than `serve`'s `AMD_SMI_SERVE_PROBE_TIMEOUT` (10s), +/// even though both now reach the same `rocm_core::amd_smi_json`. The two pay +/// different prices for giving up early. This one runs on the GPU-telemetry +/// watcher's polling cadence, where a probe that is abandoned costs one stale +/// tick and the next poll retries moments later — so a long wait buys nothing +/// and keeps a watcher thread parked on an unhealthy host. `serve` gets exactly +/// one shot at learning the GPU layout before placing a model, and abandoning +/// it degrades the placement decision for the whole lifetime of that service. const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); pub(crate) fn build_bridge_snapshot(paths: &AppPaths) -> Result { diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 20e3811797..c856879e11 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -9,8 +9,6 @@ use crate::persistence::load_managed_services; use crate::service::stop_managed_service; use crate::watchers::restart_managed_service; use anyhow::{Context, Result, bail}; -#[cfg(target_os = "linux")] -use rocm_core::command_failure_detail; use rocm_core::{ AppPaths, AuditEventRecord, ExamineSummary, ModelRecipeArtifactRecord, append_audit_event, model_artifact_cache_status, resolve_model_recipe_artifact, unix_time_millis, @@ -943,41 +941,37 @@ fn append_sandbox_tool_command_args( } } +/// Run the bubblewrap child under a timeout, via the one shared helper. +/// +/// This used to be a third hand-rolled copy of the same poll-and-kill loop, from +/// the same split (`e760d9a5`) that produced the amd-smi ones — and it carried +/// the same defect: it polled `try_wait` while both pipes stayed undrained, so a +/// sandbox child that filled the 64 KiB pipe buffer blocked mid-write and was +/// then reported as having exceeded its timeout. A verbose child, misdiagnosed +/// as a hang. `rocm_core::run_command_with_timeout` drains both pipes on their +/// own threads and hands the `wait` of a killed child to a detached reaper, so +/// routing through it fixes that here too and leaves one implementation to +/// maintain. +/// +/// The stdio setup stays at this call site because the helper does not impose +/// one: [`rocm_core::amd_smi_json`] configures its own the same way. +/// +/// Two deliberate message changes come with the move. The helper's timeout error +/// is the generic `process exceeded s timeout`, so the "sandbox process" +/// framing is restored as context rather than baked into the bail. And it no +/// longer quotes the timed-out child's partial output: the helper refuses to +/// join the reader threads on the timeout path precisely because an unkillable +/// child would block them, which is the hang the timeout exists to bound. #[cfg(target_os = "linux")] fn run_process_with_timeout( mut command: ProcessCommand, timeout: Duration, ) -> Result { - let mut child = command + command .stdin(Stdio::null()) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn sandbox process")?; - let started = std::time::Instant::now(); - loop { - if child - .try_wait() - .context("failed to poll sandbox process")? - .is_some() - { - return child - .wait_with_output() - .context("failed to collect sandbox process output"); - } - if started.elapsed() >= timeout { - let _ = child.kill(); - let output = child - .wait_with_output() - .context("failed to collect timed-out sandbox process output")?; - bail!( - "sandbox process exceeded {}s timeout: {}", - timeout.as_secs(), - command_failure_detail(&output) - ); - } - thread::sleep(Duration::from_millis(50)); - } + .stderr(Stdio::piped()); + rocm_core::run_command_with_timeout(command, timeout).context("sandbox process failed") } #[cfg(target_os = "linux")] diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index e9c1a3fc69..d9e033ebab 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -199,6 +199,22 @@ pub fn require_nonempty(value: &str, field_name: &str) -> Result<()> { /// captured output: stderr if non-blank after trimming, else stdout, else a /// literal "no output"; the result is trimmed. pub fn command_failure_detail(output: &std::process::Output) -> String { + command_failure_detail_or_else(output, || "no output".to_owned()) +} + +/// The stderr-then-stdout precedence behind [`command_failure_detail`], with the +/// both-blank fallback left to the caller. +/// +/// Exists so the precedence rule lives in exactly one place. [`amd_smi_json`] +/// needs the same ordering but a different last resort — `exit status ` +/// rather than `no output`, because a silent amd-smi's exit code is the only +/// diagnostic left — and inlining the two `is_empty` checks there to get it put +/// the same rule in two places. `fallback` is a closure because formatting an +/// `ExitStatus` is wasted work on the overwhelmingly common non-blank path. +pub fn command_failure_detail_or_else( + output: &std::process::Output, + fallback: impl FnOnce() -> String, +) -> String { let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); if !stderr.is_empty() { return stderr; @@ -207,7 +223,7 @@ pub fn command_failure_detail(output: &std::process::Output) -> String { if !stdout.is_empty() { return stdout; } - "no output".to_owned() + fallback() } #[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq)] @@ -1867,6 +1883,12 @@ const KFD_DEVICE: &str = "/dev/kfd"; /// rescues it, because killing a D-state child is a no-op. This gate removes /// the reachable half of that hazard; it does not claim to see the rest. /// +/// The probe does not assume `kfd_open` returns promptly. An `open` that blocks +/// — whatever the reason, including a state this gate has not anticipated — +/// is bounded by [`KFD_OPEN_TIMEOUT`] and read as "not readable", so the gate +/// declines rather than inheriting the stall it exists to prevent. See +/// [`device_readable_within`]. +/// /// Off Linux there is no such device and no such kernel state, so the check is /// `true` without probing anything. That wave-through is load-bearing rather /// than pedantic: `amd-smi.exe` is a supported Windows binary (see @@ -1917,9 +1939,69 @@ fn kfd_readable_with(wsl_gpu_reachable: bool, real_device: &Path, host_device: & && (host_device == real_device || device_readable(host_device))) } +/// How long the pre-flight waits for the device `open` itself before giving up +/// on it and declining the launch. +/// +/// The gate exists to avoid spawning `amd-smi` against a device that can park a +/// process in uninterruptible sleep. Performing the probe with a plain blocking +/// `open` would make the gate's own safety rest on an *assumption* — that +/// `kfd_open` always returns promptly, and that only amd-smi's later ioctls can +/// enter D-state — and that assumption cannot be verified against a +/// reset-in-progress device from any test. If it were ever wrong, the gate +/// would have relocated the hazard into a synchronous call with *less* +/// protection than the spawn it replaced, since there is no child to kill. +/// +/// Bounding the open removes the need to be right about that: a `/dev/kfd` open +/// that has not returned in this long is treated as "not readable", which is the +/// declining answer, so the unverifiable case fails in the safe direction. Short +/// because a healthy `kfd_open` is a handful of microseconds and the only thing +/// a longer wait buys is a longer stall on a host that is already unhealthy. +#[cfg(target_os = "linux")] +const KFD_OPEN_TIMEOUT: Duration = Duration::from_secs(2); + +/// Whether `device` can be opened for reading, giving up after +/// [`KFD_OPEN_TIMEOUT`]. #[cfg(target_os = "linux")] fn device_readable(device: &Path) -> bool { - fs::OpenOptions::new().read(true).open(device).is_ok() + device_readable_within(device, KFD_OPEN_TIMEOUT) +} + +/// [`device_readable`] with the bound injected, so a test can pin the timeout +/// path against a device that really does block in `open` (a FIFO with no +/// writer) without stalling the suite for the production bound. +/// +/// The `open` runs on its own thread and the caller waits on a channel, because +/// there is no portable way to interrupt a blocking `open` once it has entered +/// the kernel. A probe that overruns leaves that thread parked — the same +/// trade-off [`reap_in_background`] documents, and for the same reason: one +/// leaked thread is strictly cheaper than a blocked caller, and a device that +/// can park a thread forever is exactly the device this whole gate exists to +/// refuse. The opened handle is dropped on the probe thread, so a late success +/// closes its own descriptor rather than leaking one. +/// +/// If no thread can be spawned the probe falls back to a direct blocking open: +/// that is the pre-existing behaviour, and refusing every launch because the +/// process is out of threads would be a worse failure than the one being +/// guarded against. +#[cfg(target_os = "linux")] +fn device_readable_within(device: &Path, timeout: Duration) -> bool { + let (sender, receiver) = std::sync::mpsc::channel(); + let probe_path = device.to_path_buf(); + let spawned = thread::Builder::new() + .name("rocm-kfd-open-probe".to_owned()) + .spawn(move || { + let readable = fs::OpenOptions::new().read(true).open(&probe_path).is_ok(); + // The receiver is gone whenever the probe overran; dropping the + // send result is the whole handling. + let _ = sender.send(readable); + }) + .is_ok(); + if !spawned { + return fs::OpenOptions::new().read(true).open(device).is_ok(); + } + // Both error arms — timeout and a probe thread that died without sending — + // mean "no readable answer", which the gate must read as not readable. + receiver.recv_timeout(timeout).unwrap_or(false) } /// Off-Linux counterpart of [`kfd_readable`]: see that function's docs for why @@ -2002,26 +2084,24 @@ fn amd_smi_json_with( .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); // Truncated, not dropped. This message reaches `rocmd`'s snapshot note // and from there the JSONL logs, so the quote has to be bounded at the // point that knows these characters are raw child output — see // [`MAX_QUOTED_OUTPUT_CHARS`]. The head is where a failing `amd-smi` // says what went wrong, and the command is named before it either way. + // + // The stderr-then-stdout precedence is `command_failure_detail`'s, taken + // from the shared helper rather than re-spelled here; only the + // both-blank fallback differs, naming the exit status instead of the + // generic "no output". bail!( "amd-smi {} failed: {}", args.join(" "), crate::examine::truncate_to_chars( - if stderr.is_empty() { - if stdout.is_empty() { - format!("exit status {}", output.status) - } else { - stdout - } - } else { - stderr - }, + command_failure_detail_or_else(&output, || format!( + "exit status {}", + output.status + )), MAX_QUOTED_OUTPUT_CHARS, ) ); @@ -2695,6 +2775,72 @@ mod tests { Ok(()) } + /// The gate's own probe must not be able to hang. + /// + /// `kfd_readable` was reviewed as resting on an unverified assumption: that + /// `open("/dev/kfd")` always returns promptly, so that only amd-smi's later + /// ioctls can enter uninterruptible sleep. If that were ever wrong the gate + /// would have moved the hazard into a synchronous call with *less* + /// protection than the spawn it replaced — no child to kill, no timeout. + /// Rather than assert the assumption, the probe is bounded; this pins that. + /// + /// A FIFO is the fixture that makes the blocking case real rather than + /// simulated: `open(O_RDONLY)` on a FIFO with no writer blocks in the kernel + /// indefinitely, which is the observable shape of the hazard. The deadline + /// assertion is what distinguishes a bounded probe from an unbounded one — + /// without it, deleting the timeout would hang the suite rather than fail + /// this test, and a hang is not a readable failure. The readable control in + /// the same test stops the whole thing passing vacuously, e.g. if + /// `device_readable_within` were mutated to return `false` unconditionally. + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_probe_declines_a_device_whose_open_blocks_instead_of_hanging() -> Result<()> { + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-open-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let fifo = temp_root.join("blocking-kfd"); + let fifo_c = std::ffi::CString::new(fifo.as_os_str().as_encoded_bytes()) + .context("fifo path contained an interior NUL")?; + // SAFETY: `fifo_c` is a NUL-terminated path that lives across the call, + // and `mkfifo` only creates a node at it. + let made = unsafe { libc::mkfifo(fifo_c.as_ptr(), 0o644) }; + assert_eq!( + made, + 0, + "could not create the FIFO fixture: {}", + std::io::Error::last_os_error() + ); + + let probe_timeout = Duration::from_millis(300); + let started = Instant::now(); + let blocked = device_readable_within(&fifo, probe_timeout); + let elapsed = started.elapsed(); + + let readable = temp_root.join("ordinary-kfd"); + fs::write(&readable, b"")?; + let allowed = device_readable_within(&readable, probe_timeout); + // The overrun probe's thread is still parked in `open` and stays that + // way until this test binary exits — deliberately, and the same leak + // `device_readable_within` documents. Unlinking the FIFO does not + // release it, and nothing here needs it released. + let _ = fs::remove_dir_all(&temp_root); + + assert!( + !blocked, + "a device whose open never returns must read as not readable, so the gate declines" + ); + assert!( + elapsed < probe_timeout * 10, + "the probe returned after {elapsed:?}, so it is not bounded by its {probe_timeout:?} \ + timeout — an unbounded open relocates the D-state hang into the gate itself" + ); + assert!( + allowed, + "an ordinary readable device must still pass, or the blocked case proves nothing" + ); + Ok(()) + } + /// Pins the production wiring the injected helpers cannot: that /// [`amd_smi_json`] gates on [`KFD_DEVICE`] itself. /// From ed0d467bcfc2c696b23d2d1ecda99b9f6db97996 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 09:55:15 +0300 Subject: [PATCH 17/32] fix(core): create the blocking-open fixture without unsafe The workspace lints deny `unsafe_code`, so `libc::mkfifo` cannot be called directly; shell out to `mkfifo(1)` and skip when it is absent. Also narrow sandbox.rs's `std::thread` import to `cfg(test)`: the Linux timeout no longer runs its own sleep-poll loop. Signed-off-by: Roman Sirokov --- apps/rocmd/src/sandbox.rs | 4 +++- crates/rocm-core/src/lib.rs | 27 ++++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index c856879e11..3038c5af3b 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -23,7 +23,9 @@ use std::io::{self, Write}; use std::path::{Path, PathBuf}; #[cfg(target_os = "linux")] use std::process::{Command as ProcessCommand, Stdio}; -#[cfg(any(test, target_os = "linux"))] +// Test-only now: the Linux sandbox timeout no longer runs its own sleep-poll +// loop, it delegates to `rocm_core::run_command_with_timeout`. +#[cfg(test)] use std::thread; use std::time::Duration; diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index d9e033ebab..8667cd2185 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -2792,6 +2792,12 @@ mod tests { /// this test, and a hang is not a readable failure. The readable control in /// the same test stops the whole thing passing vacuously, e.g. if /// `device_readable_within` were mutated to return `false` unconditionally. + /// + /// The FIFO is created by shelling out to `mkfifo(1)` rather than calling + /// `libc::mkfifo`: the workspace lints deny `unsafe_code`, and one test + /// fixture does not justify an exception. The tool is coreutils, so it is + /// present wherever this test runs; a host without it skips rather than + /// fails, since a missing fixture is not evidence about the probe. #[test] #[cfg(target_os = "linux")] fn the_kfd_probe_declines_a_device_whose_open_blocks_instead_of_hanging() -> Result<()> { @@ -2799,17 +2805,16 @@ mod tests { std::env::temp_dir().join(format!("rocm-cli-kfd-open-{}", unix_time_millis())); fs::create_dir_all(&temp_root)?; let fifo = temp_root.join("blocking-kfd"); - let fifo_c = std::ffi::CString::new(fifo.as_os_str().as_encoded_bytes()) - .context("fifo path contained an interior NUL")?; - // SAFETY: `fifo_c` is a NUL-terminated path that lives across the call, - // and `mkfifo` only creates a node at it. - let made = unsafe { libc::mkfifo(fifo_c.as_ptr(), 0o644) }; - assert_eq!( - made, - 0, - "could not create the FIFO fixture: {}", - std::io::Error::last_os_error() - ); + let made = Command::new("mkfifo") + .arg(&fifo) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + if !matches!(&made, Ok(status) if status.success()) { + let _ = fs::remove_dir_all(&temp_root); + return Ok(()); + } let probe_timeout = Duration::from_millis(300); let started = Instant::now(); From 2b42e9e39d338f9cbeb2ab6a1ea284fbb41c2baf Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 10:06:48 +0300 Subject: [PATCH 18/32] test(core): watch the blocking-open probe so an unbounded mutant fails, not hangs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calling the probe inline meant a mutant that drops the timeout from `device_readable_within` would block forever on the FIFO and hang the test binary — a CI timeout rather than a readable failure. Run it on a watched thread with a much looser deadline so that mutant reports the cause instead. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 8667cd2185..33f4c7d5f5 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -2817,9 +2817,22 @@ mod tests { } let probe_timeout = Duration::from_millis(300); - let started = Instant::now(); - let blocked = device_readable_within(&fifo, probe_timeout); - let elapsed = started.elapsed(); + // The probe runs on a thread the test watches with its own, much looser + // deadline, rather than being called inline. A mutant that drops the + // bound from `device_readable_within` would block forever on this FIFO; + // called inline that would hang the whole test binary, and a hang is a + // CI timeout rather than a readable failure. Watched, it fails here with + // a message naming the cause. + let (verdict_tx, verdict_rx) = std::sync::mpsc::channel(); + let probe_fifo = fifo.clone(); + thread::Builder::new() + .name("kfd-open-probe-test".to_owned()) + .spawn(move || { + let _ = verdict_tx.send(device_readable_within(&probe_fifo, probe_timeout)); + }) + .context("could not start the watched probe thread")?; + let watchdog = probe_timeout * 20; + let blocked = verdict_rx.recv_timeout(watchdog); let readable = temp_root.join("ordinary-kfd"); fs::write(&readable, b"")?; @@ -2830,14 +2843,13 @@ mod tests { // release it, and nothing here needs it released. let _ = fs::remove_dir_all(&temp_root); - assert!( - !blocked, - "a device whose open never returns must read as not readable, so the gate declines" - ); - assert!( - elapsed < probe_timeout * 10, - "the probe returned after {elapsed:?}, so it is not bounded by its {probe_timeout:?} \ - timeout — an unbounded open relocates the D-state hang into the gate itself" + assert_eq!( + blocked, + Ok(false), + "a device whose open never returns must read as not readable within its \ + {probe_timeout:?} bound, so the gate declines; `Err` here means the probe had still \ + not returned after {watchdog:?} — an unbounded open relocates the D-state hang into \ + the gate itself" ); assert!( allowed, From 4d77efa6d1098f3a1aee173985b7cb25e233f995 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 10:16:23 +0300 Subject: [PATCH 19/32] test(core): move the fifo path into the probe thread instead of cloning it `clippy::redundant_clone`: the fixture path has no use after the spawn, so the clone was dead. The thread can take it by move. Caught by CI, not locally: the devbox run that cleared this branch was made against the merge commit, before this test existed. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 33f4c7d5f5..1d74adb671 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -2824,11 +2824,10 @@ mod tests { // CI timeout rather than a readable failure. Watched, it fails here with // a message naming the cause. let (verdict_tx, verdict_rx) = std::sync::mpsc::channel(); - let probe_fifo = fifo.clone(); thread::Builder::new() .name("kfd-open-probe-test".to_owned()) .spawn(move || { - let _ = verdict_tx.send(device_readable_within(&probe_fifo, probe_timeout)); + let _ = verdict_tx.send(device_readable_within(&fifo, probe_timeout)); }) .context("could not start the watched probe thread")?; let watchdog = probe_timeout * 20; From 1be50f495329647e52f83ac28b3e1ff4f8fed2c3 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 16 Oct 2026 17:35:18 +0300 Subject: [PATCH 20/32] test(e2e): reach both sides of the KFD pre-flight from a scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `/dev/kfd` pre-flight refuses to spawn `amd-smi` when the device cannot be opened. The per-PR E2E lane has no `/dev/kfd`, so every scenario that plants a stub `amd-smi` was refused before the stub ran: examine-24/25/26 (the pipe-drain regression tests from #526) started asserting against a decline instead of the behaviour they are about. `ROCM_CLI_TEST_HOST_ROOT` cannot open the gate. `kfd_readable_with` keeps the real node's veto on purpose, so a simulated root can make the gate decline on a machine that has a GPU but can never make it allow on one that does not -- the invariant that stops a simulated root launching `amd-smi` where there is nothing to talk to. So add `ROCM_E2E_KFD_DEVICE`, honoured only by an `e2e-test-hooks` build, which moves the device the gate opens. It is an override, not a bypass: the probe still runs, against the named path, and a shipped build has no override at all. Tagging the three scenarios `@requires-gpu` was the alternative and was rejected -- examine.feature says in terms that they are built to need no GPU so they run per PR, and a GPU tag would strand #526's regression tests on a nightly lane. The seam also makes the declined branch observable, which AGENTS.md §3 asks for and the PR had only unit tests for. examine-29 points the pre-flight at an absent device and asserts the bridge snapshot explains itself, names that device, and -- the load-bearing half -- that the stub `amd-smi`, which would have answered, never ran. Signed-off-by: Roman --- crates/rocm-core/src/lib.rs | 95 ++++++++++++- tests/e2e-cucumber/features/examine.feature | 31 +++++ .../features/model_serving.feature | 70 ++++------ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 126 ++++++++++++++++++ 4 files changed, 276 insertions(+), 46 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index b65f68fd96..b00a72621c 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -419,6 +419,47 @@ fn resolve_amd_smi_binary_in_home(home_dir: Option<&Path>) -> OsString { /// [`crate::examine::WslFacts`]). const KFD_DEVICE: &str = "/dev/kfd"; +/// Name of the E2E-only seam that moves the pre-flight's device node, so the +/// gate's two branches are both reachable from a Gherkin scenario. +/// +/// Honoured only by a build carrying the `e2e-test-hooks` feature, which no +/// released artefact does; see [`preflight_kfd_device`]. +#[cfg(feature = "e2e-test-hooks")] +pub(crate) const TEST_KFD_DEVICE_ENV: &str = "ROCM_E2E_KFD_DEVICE"; + +/// The device node [`amd_smi_json`] runs its pre-flight against. +/// +/// [`KFD_DEVICE`] in every shipped build. An `e2e-test-hooks` build lets +/// [`TEST_KFD_DEVICE_ENV`] name a different path, which is what makes the gate +/// testable from the outside at all. +/// +/// Why a seam rather than the existing [`crate::hardware_root`] re-rooting: +/// [`kfd_readable`] deliberately keeps the *real* node's veto, so a simulated +/// root can only ever make the gate decline, never allow. That asymmetry is the +/// invariant that stops a simulated root launching `amd-smi` on a machine with +/// no GPU, and it is correct — but it also means no simulated root can exercise +/// the *allowed* branch on a GPU-less runner, which is where the per-PR lane +/// runs. A path override sidesteps that without touching the invariant: the +/// production build has no override, so the veto and the real device are all +/// there is. +/// +/// Safe to be an override rather than a bypass. It relocates the probe; it does +/// not skip it. A build with the feature on and the variable unset — every unit +/// test in this crate, and every E2E scenario that does not set it — gates on +/// `/dev/kfd` exactly as production does. +#[cfg(feature = "e2e-test-hooks")] +fn preflight_kfd_device() -> std::path::PathBuf { + std::env::var_os(TEST_KFD_DEVICE_ENV) + .map_or_else(|| std::path::PathBuf::from(KFD_DEVICE), Into::into) +} + +/// Production counterpart of [`preflight_kfd_device`]: the constant, with no +/// environment consulted. See that function for why the seam exists. +#[cfg(not(feature = "e2e-test-hooks"))] +fn preflight_kfd_device() -> std::path::PathBuf { + std::path::PathBuf::from(KFD_DEVICE) +} + /// Whether `/dev/kfd` can be opened for reading — the pre-flight every /// `amd-smi` launch goes through, and it is **mandatory** before every launch. /// @@ -602,7 +643,10 @@ const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; pub fn amd_smi_json(args: &[&str], timeout: Duration) -> Result { amd_smi_json_with( &resolve_amd_smi_binary(), - Path::new(KFD_DEVICE), + // [`preflight_kfd_device`], not the constant: identical in every shipped + // build, and the one seam an E2E scenario can move so that both sides of + // the gate are reachable from the lane that runs per PR. + &preflight_kfd_device(), args, timeout, ) @@ -1170,6 +1214,55 @@ mod tests { ); } + /// The E2E path seam is an override, not a bypass: with the variable unset + /// the pre-flight device is `/dev/kfd` on every build, and with it set the + /// gate still *runs* — against the named path — rather than being skipped. + /// + /// Worth pinning because the seam's whole justification is that it cannot + /// weaken a shipped build. Both halves are load-bearing: a `preflight_kfd_device` + /// that read the variable unconditionally would fail the unset half here and + /// hand any process on the host a way to disable the gate, and one that + /// returned a readable path outright — the bypass this deliberately is not — + /// would fail the declining half. + #[test] + #[cfg(target_os = "linux")] + fn the_e2e_kfd_path_seam_moves_the_gate_without_disabling_it() { + // The same serializer the simulated-root tests take, rather than a + // second one: both mutate the environment the pre-flight reads, and one + // lock for the pair cannot deadlock against itself. + let _env = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + assert_eq!( + preflight_kfd_device(), + Path::new(KFD_DEVICE), + "with no seam set, every build must gate on the real device" + ); + + #[cfg(feature = "e2e-test-hooks")] + { + let scratch = tempfile::tempdir().expect("a scratch dir"); + let absent = scratch.path().join("no-such-kfd"); + let _seam = crate::test_env::RestoredEnvVar::set(TEST_KFD_DEVICE_ENV, &absent); + + assert_eq!( + preflight_kfd_device(), + absent, + "an e2e-test-hooks build must honour the seam" + ); + + let error = amd_smi_json(&["--rocm-cli-gate-probe"], Duration::from_secs(5)) + .expect_err("an unopenable seam path must decline the launch"); + let message = format!("{error:#}"); + assert!( + message.contains("declined to launch") + && message.contains(&absent.display().to_string()), + "the seam must move the gate, not remove it; got: {message}" + ); + } + } + /// A failing `amd-smi` that prints a payload must not paste the payload into /// the error. `rocmd` puts this message in its snapshot note and the /// GPU-metrics watcher copies that note into rewrite-the-whole-file JSONL diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 00c62c3f37..b063919deb 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -405,6 +405,37 @@ Feature: GPU detection and system inspection When the daemon gathers a bridge snapshot Then the snapshot keeps the first probe's GPUs and explains the second with a bounded note naming the timeout + # The other side of the three above, and the user-observable half of the + # `/dev/kfd` pre-flight (EAI-8457): where those three prove a healthy device + # still reaches amd-smi, this proves an unopenable one stops the launch from + # happening at all. On a host in that state amd-smi can park in + # uninterruptible sleep, where no signal and therefore no timeout reaches it, + # so "declined" is the only outcome that bounds the daemon's 60s telemetry + # poll. The snapshot degrades; the command still succeeds. + # + # Non-vacuity is the whole difficulty here, and it is carried in two places. + # The stub amd-smi would answer this probe perfectly and records that it ran, + # so the marker's absence is what distinguishes "refused before the spawn" + # from "spawned and failed" -- deleting the gate makes this scenario fail + # rather than merely changing a message. And the note must name the device + # this run pointed the pre-flight at: the lane that runs this per PR has no + # `/dev/kfd` of its own, so a build that ignored the seam would decline for + # the host's own reasons and otherwise look identical. + # + # The seam is `ROCM_E2E_KFD_DEVICE`, honoured only by an `e2e-test-hooks` + # build (`rocm_core::preflight_kfd_device`), and it moves the device the gate + # opens rather than skipping the gate. `ROCM_CLI_TEST_HOST_ROOT` cannot stand + # in for it: the gate keeps the real node's veto on purpose, so a simulated + # root can only ever make it decline -- which is this branch, but leaves the + # three scenarios above unable to reach amd-smi on a GPU-less runner at all. + # Linux-only, like its three siblings: the stub is a shell script, and the + # gate is a wave-through on every other platform. + @id:examine-telemetry-declines-an-unopenable-kfd-device @requires-os:linux + Scenario: examine-29 - An unopenable KFD device stops amd-smi being launched at all + Given the KFD device cannot be opened + When the daemon gathers a bridge snapshot + Then the snapshot says amd-smi was never launched and names the device + # `os-release(5)` says to read `/etc/os-release` and, if it does not exist, # fall back to `/usr/lib/os-release`. This plants a simulated host root with # only the fallback path populated, so a reader that stopped after the first diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index b0547eeb47..2497008fde 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -342,54 +342,34 @@ Feature: Model serving When the user lists recommended models Then the model list output contains only ASCII characters - # KNOWN GAP — the `/dev/kfd` pre-flight on the serve probes (EAI-8457) has no - # scenario here yet. It is now reachable, which it was not when this note was - # first written; what follows records what changed rather than arguing the - # behaviour is untestable, because it no longer is. + # The `/dev/kfd` pre-flight (EAI-8457) is covered by `examine-29` in + # `examine.feature`, not here, and this note records why the scenario lives + # there rather than alongside the serve probes it also guards. # - # The behaviour: on a Linux host where `/dev/kfd` cannot be opened, `--gpu - # auto` no longer launches `amd-smi` at all (`detect_gpu_count`, - # `gpu_vram_usage`), so GPU auto-selection degrades to service state alone - # where it previously attempted the launch. That is user-observable, so §3 of - # AGENTS.md asks for a scenario rather than only the `rocm-core` unit tests - # that pin `kfd_readable` and `amd_smi_json`. + # The behaviour: on a Linux host where the KFD device cannot be opened, + # `rocm_core::amd_smi_json` declines the launch outright, so `serve`'s + # `gpu_vram_usage` and `detect_gpu_count` degrade to service state alone where + # they previously attempted the spawn. The same chokepoint serves `rocmd`'s + # bridge snapshot, and that surface is the one a scenario can observe + # directly: the snapshot reports `amd_smi_available` and a note that says why, + # as JSON, in one command. `serve`'s probes discard their error with `.ok()?` + # by design, so the declined branch is visible there only as the absence of a + # GPU-aware placement -- which a GPU-less lane cannot tell apart from having + # no GPU. One scenario on the observable surface beats a weaker one here. # - # The premise needs a host that is BOTH GPU-present by the KFD/DRM sysfs rule - # AND unable to open `/dev/kfd` — the real shape of a container started - # without `--device /dev/kfd`, or a user outside the render group, where - # world-readable sysfs still enumerates the device that `open(2)` then - # refuses. No lane is naturally that host, but two earlier claims about why it - # could never be manufactured were wrong and are corrected here: + # Reaching either branch needs the `e2e-test-hooks` seam + # `ROCM_E2E_KFD_DEVICE` (`rocm_core::preflight_kfd_device`), which moves the + # device the gate opens. `ROCM_CLI_TEST_HOST_ROOT` cannot substitute: the gate + # keeps the real node's veto on purpose, so a simulated root can make it + # decline but never allow -- see + # `the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto`. # - # - "the no-GPU mock lane never reaches either probe" is false. It holds for - # serve-13's plain path, where `usable_amd_gpu_indices` answers an - # authoritative empty set and `serve` bails first — but - # `ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE` (set by the mock lane, see - # `serving_steps.rs`) skips exactly that bail in `serve_cmd.rs`, and - # `gpu_vram_usage()` is called below it. `host_accelerator_memory` in - # `main.rs` reaches the same probe from `diagnose --model` with no GPU - # guard at all. The declined branch therefore does run on the mock lane. - # - "`KFD_DEVICE` is a compile-time constant with no path override" is now - # false too: the gate is routed through `rocm_core::host_path`, so an - # `e2e-test-hooks` build with `ROCM_CLI_TEST_HOST_ROOT` set can present a - # simulated host whose KFD topology says GPU-present and whose `/dev/kfd` - # is absent. The real device keeps its veto, so this can only manufacture - # the *declined* branch, never a spurious allow. - # - # What is still true: `@requires-gpu` lanes have a readable `/dev/kfd` by - # construction, and `@requires-wsl` exercises the wave-through, which is the - # opposite branch. So the scenario belongs on the mock lane, built from the - # simulated root plus a hanging stub `amd-smi` — `serve` and `diagnose` must - # complete promptly where before this change they would have hung on it. - # - # Covered meanwhile at unit level in `rocm-core`, each confirmed non-vacuous - # by mutation: `the_kfd_gate_stops_amd_smi_before_it_is_spawned` asserts the - # gate stops the *spawn* (a fake `amd-smi` leaves a marker file that must not + # Also covered at unit level in `rocm-core`, each confirmed non-vacuous by + # mutation: `the_kfd_gate_stops_amd_smi_before_it_is_spawned` asserts the gate + # stops the *spawn* (a fake `amd-smi` leaves a marker file that must not # appear), with the same fixture re-run against a readable device to prove the # fake would otherwise have executed; # `the_public_amd_smi_entry_point_gates_on_the_real_kfd_device` pins that the - # production wrapper gates on `/dev/kfd` itself; and - # `the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto` pins - # the routing the scenario above would depend on. - # - # Retire this note when that mock-lane scenario lands. + # production wrapper gates on the real device; and + # `the_e2e_kfd_path_seam_moves_the_gate_without_disabling_it` pins that the + # seam above relocates the probe instead of skipping it. diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 95f0344aaf..102a3bff7d 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -40,6 +40,18 @@ const STUB_MARKET_NAME: &str = "stub-accelerator"; /// otherwise, per `command_failure_detail` — not stdout unconditionally. const STUB_FIRST_GPU_MARKET_NAME: &str = "stub-accelerator-gpu0"; +/// The `e2e-test-hooks` seam that names the device node `rocm_core`'s `amd-smi` +/// pre-flight opens (EAI-8457). +/// +/// Spelled out rather than imported: `rocm_core` exports it only to its own +/// crate, because a `pub` constant would read as an invitation for production +/// code to consult it. The harness has to agree with that string by hand, and +/// the scenario that points it at an absent path is what notices if it drifts: +/// it asserts the decline names *that* path, so a build that ignored the +/// variable would decline naming `/dev/kfd` and fail, rather than passing on a +/// gate that declined for the host's own reasons. +const KFD_DEVICE_SEAM: &str = "ROCM_E2E_KFD_DEVICE"; + /// Where the `amd-smi` stub and its payload live inside the isolated root. /// /// Derived rather than carried on the World so the Given and the When agree on @@ -1712,6 +1724,44 @@ async fn stub_amd_smi_that_stalls_only_on_monitor(world: &mut E2eWorld) { ); } +/// Where the stub records that it ran, so the decline can be asserted as "never +/// spawned" rather than merely "reported unavailable". +fn stub_ran_marker(world: &E2eWorld) -> PathBuf { + stub_bin_dir(world).join("amd-smi-ran") +} + +#[given("the KFD device cannot be opened")] +async fn kfd_device_cannot_be_opened(world: &mut E2eWorld) { + // A stub that *would* answer perfectly well, and a marker that records it + // running. The point of the gate is that neither ever happens: a scenario + // whose stub could not have answered anyway would pass against a gate that + // had been deleted. + let payload = write_stub_payload(world); + let marker = stub_ran_marker(world); + write_amd_smi_stub( + world, + &format!( + "touch {marker}\nexec cat {payload}", + marker = marker.display(), + payload = payload.display() + ), + ); + + // Inside the isolated root and never created. The decline must name this + // path, which is what proves the daemon read the seam rather than falling + // back to `/dev/kfd` -- a host-dependent answer that would make the + // scenario pass for the wrong reason on a lane with no GPU. + let absent = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("absent-kfd-device"); + world + .command_env + .push((KFD_DEVICE_SEAM, absent.into_os_string())); +} + #[when("the daemon gathers a bridge snapshot")] async fn daemon_gathers_bridge_snapshot(world: &mut E2eWorld) { let rocmd = crate::rocmd_binary(); @@ -1750,6 +1800,28 @@ async fn daemon_gathers_bridge_snapshot(world: &mut E2eWorld) { .expect("failed to build the stub PATH"); command.env("PATH", path); + // The `/dev/kfd` pre-flight in `rocm_core::amd_smi_json` (EAI-8457) refuses + // to spawn `amd-smi` at all when the device cannot be opened, and the lane + // that runs these scenarios per PR has no `/dev/kfd`. Without a seam the + // stub would never be reached and all three scenarios would assert against + // a decline instead of the pipe-drain behaviour they are about. + // + // `ROCM_CLI_TEST_HOST_ROOT` cannot do this job: the gate deliberately keeps + // the real node's veto, so a simulated root can only ever make it decline. + // The `e2e-test-hooks` path seam can, and it moves the gate rather than + // disabling it -- the daemon still opens the file named here before it + // spawns anything, which is why a plain readable file is enough to stand in + // for a healthy device. The companion scenario points the same variable at + // an absent path to exercise the other branch. + let kfd_stand_in = isolated_home.join("kfd-stand-in"); + std::fs::write(&kfd_stand_in, b"").expect("failed to plant the stand-in KFD node"); + command.env(KFD_DEVICE_SEAM, &kfd_stand_in); + // Applied after the default above so a scenario that pushed its own value + // -- the declined branch does -- wins over it. + for (key, value) in &world.command_env { + command.env(key, value); + } + let output = command .output() .expect("failed to run rocmd bridge-snapshot"); @@ -1868,6 +1940,60 @@ async fn assert_stalled_probe_note_is_bounded(world: &mut E2eWorld) { ); } +#[then("the snapshot says amd-smi was never launched and names the device")] +async fn assert_snapshot_declined_the_launch(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no rocmd exit status recorded"); + let stdout = world.cli_output.as_ref().expect("no rocmd output recorded"); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + // A declined probe degrades the GPU section the same way a timed-out one + // does; it must not fail the command. + assert_eq!(rc, 0, "rocmd bridge-snapshot failed:\n{stdout}\n{stderr}"); + + let snapshot: serde_json::Value = serde_json::from_str(stdout) + .unwrap_or_else(|error| panic!("bridge-snapshot did not emit JSON: {error}\n{stderr}")); + let gpu = &snapshot["gpu"]; + + assert_eq!( + gpu["amd_smi_available"], + serde_json::Value::Bool(false), + "a launch that was refused cannot have reported a GPU: {gpu}" + ); + let note = gpu["note"] + .as_str() + .unwrap_or_else(|| panic!("a degraded snapshot must explain itself: {gpu}")); + assert!( + note.contains("declined to launch"), + "the note must say the launch was refused, not blame the binary or a timeout: {note:?}" + ); + + // The device this run was pointed at, not `/dev/kfd`. On a lane with no GPU + // the gate declines either way, so without this the scenario would pass + // against a build that ignored the seam entirely -- and would then be + // asserting the runner's own missing device rather than the gate. + let expected = world + .command_env + .iter() + .find(|(key, _)| *key == KFD_DEVICE_SEAM) + .map(|(_, value)| value.to_string_lossy().into_owned()) + .expect("the scenario must have pointed the pre-flight at an absent device"); + assert!( + note.contains(&expected), + "the note must name the device the pre-flight actually probed ({expected}): {note:?}" + ); + + // The load-bearing half: the gate's claim is that nothing is spawned, which + // a note alone does not establish. The stub would have answered this probe + // perfectly, so the marker's absence is the only evidence that the refusal + // happened before the launch rather than after a failed one. + let marker = stub_ran_marker(world); + assert!( + !marker.exists(), + "the pre-flight must refuse before the spawn, but the stub amd-smi ran and left \ + {}", + marker.display() + ); +} + #[then( "the snapshot keeps the first probe's GPUs and explains the second with a bounded note naming the timeout" )] From 3c4d6c2b8538696b730c0585b93823343d6f486f Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 16 Oct 2026 17:38:58 +0300 Subject: [PATCH 21/32] test(e2e): declare the KFD pre-flight scenario last in examine.feature `feature_naming` requires the per-feature scenario index to follow declaration order, so a new examine-29 cannot sit beside examine-26. Signed-off-by: Roman --- tests/e2e-cucumber/features/examine.feature | 58 +++++++++++---------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index b063919deb..f039346c8e 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -405,10 +405,36 @@ Feature: GPU detection and system inspection When the daemon gathers a bridge snapshot Then the snapshot keeps the first probe's GPUs and explains the second with a bounded note naming the timeout - # The other side of the three above, and the user-observable half of the + # `os-release(5)` says to read `/etc/os-release` and, if it does not exist, + # fall back to `/usr/lib/os-release`. This plants a simulated host root with + # only the fallback path populated, so a reader that stopped after the first + # candidate would see no file at all and report an unknown distribution. + @id:examine-falls-back-to-usr-lib-os-release @requires-os:linux + Scenario: examine-27 - The inspection falls back to /usr/lib/os-release when /etc/os-release is absent + Given a machine whose release file exists only at /usr/lib/os-release + When the user inspects that machine in machine-readable form + Then the inspection reports the distribution "ubuntu" version "24.04" named "Ubuntu 24.04 LTS" + And the inspection reports no trouble reading the release file + + # Before this change, a release file that exists but cannot be read (not + # absent, so no fallback candidate is tried either) silently produced an + # empty, unreported distro with no failure recorded. It now reports the + # file as itself, distinctly from both an unknown distribution and a + # malformed line it never reached. A directory in place of the file is a + # non-`NotFound` read error that needs no root privilege, unlike a + # mode-000 file, which CI running as root would read anyway. + @id:examine-degrades-on-unreadable-release-file @requires-os:linux + Scenario: examine-28 - An unreadable release file degrades the verdict on a machine with a GPU + Given a GPU machine whose release file cannot be read + When the user inspects that machine in machine-readable form + Then the inspection's verdict is degraded, and its only failure says the release file could not be read + + # The other side of examine-24/25/26, and the user-observable half of the # `/dev/kfd` pre-flight (EAI-8457): where those three prove a healthy device # still reaches amd-smi, this proves an unopenable one stops the launch from - # happening at all. On a host in that state amd-smi can park in + # happening at all. (Declared here at the end rather than beside them because + # the scenario index is per-feature and in declaration order.) On a host in + # that state amd-smi can park in # uninterruptible sleep, where no signal and therefore no timeout reaches it, # so "declined" is the only outcome that bounds the daemon's 60s telemetry # poll. The snapshot degrades; the command still succeeds. @@ -426,8 +452,8 @@ Feature: GPU detection and system inspection # build (`rocm_core::preflight_kfd_device`), and it moves the device the gate # opens rather than skipping the gate. `ROCM_CLI_TEST_HOST_ROOT` cannot stand # in for it: the gate keeps the real node's veto on purpose, so a simulated - # root can only ever make it decline -- which is this branch, but leaves the - # three scenarios above unable to reach amd-smi on a GPU-less runner at all. + # root can only ever make it decline -- which is this branch, but leaves + # examine-24/25/26 unable to reach amd-smi on a GPU-less runner at all. # Linux-only, like its three siblings: the stub is a shell script, and the # gate is a wave-through on every other platform. @id:examine-telemetry-declines-an-unopenable-kfd-device @requires-os:linux @@ -435,27 +461,3 @@ Feature: GPU detection and system inspection Given the KFD device cannot be opened When the daemon gathers a bridge snapshot Then the snapshot says amd-smi was never launched and names the device - - # `os-release(5)` says to read `/etc/os-release` and, if it does not exist, - # fall back to `/usr/lib/os-release`. This plants a simulated host root with - # only the fallback path populated, so a reader that stopped after the first - # candidate would see no file at all and report an unknown distribution. - @id:examine-falls-back-to-usr-lib-os-release @requires-os:linux - Scenario: examine-27 - The inspection falls back to /usr/lib/os-release when /etc/os-release is absent - Given a machine whose release file exists only at /usr/lib/os-release - When the user inspects that machine in machine-readable form - Then the inspection reports the distribution "ubuntu" version "24.04" named "Ubuntu 24.04 LTS" - And the inspection reports no trouble reading the release file - - # Before this change, a release file that exists but cannot be read (not - # absent, so no fallback candidate is tried either) silently produced an - # empty, unreported distro with no failure recorded. It now reports the - # file as itself, distinctly from both an unknown distribution and a - # malformed line it never reached. A directory in place of the file is a - # non-`NotFound` read error that needs no root privilege, unlike a - # mode-000 file, which CI running as root would read anyway. - @id:examine-degrades-on-unreadable-release-file @requires-os:linux - Scenario: examine-28 - An unreadable release file degrades the verdict on a machine with a GPU - Given a GPU machine whose release file cannot be read - When the user inspects that machine in machine-readable form - Then the inspection's verdict is degraded, and its only failure says the release file could not be read From c1ad263d7a6beca1334409db625414ef28297eee Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 16 Oct 2026 17:41:24 +0300 Subject: [PATCH 22/32] docs(serve): stop AMD_SMI_SERVE_PROBE_TIMEOUT calling itself serve-only `gpu_vram_usage_amd_smi` is also reached from `host_accelerator_memory`, and so from the `--model` readiness path, not only from `serve`. The budget is still the right one there -- same question, same one-shot cost -- but the doc said something untrue about where it is spent. Signed-off-by: Roman --- apps/rocm/src/main.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index fa91fc1fcb..cdc50258b7 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -10776,8 +10776,16 @@ fn select_auto_gpu_index( /// bounds the ordinary stalls only; the unkillable one is prevented by the /// pre-flight inside [`rocm_core::amd_smi_json`], not by any timeout. /// -/// Used by `serve` only — [`gpu_vram_usage_amd_smi`] and [`detect_gpu_count`] -/// are its two consumers and both are reached solely from `serve_cmd::serve`. +/// Named for `serve` because that is the path the budget is sized for, not +/// because it is the only one that spends it. [`detect_gpu_count`] is reached +/// solely from `serve_cmd::serve`, but [`gpu_vram_usage_amd_smi`] is also +/// reached from [`host_accelerator_memory`], and so from +/// `assess_model_with_host_paths` on the `--model` readiness path. That path +/// wants the same answer for the same reason — how much VRAM this host has, to +/// decide what it can run — so it wants the same budget; the one-shot argument +/// below applies to it unchanged. It is one probe there rather than two, so the +/// worst case is 10s, not 20s. +/// /// `rocmd`'s watcher has its own, shorter `AMD_SMI_PROBE_TIMEOUT` (2s) for the /// same helper: that one fires on a polling cadence where a stalled probe costs /// a missed telemetry tick and nothing else, so it can afford to give up early, From 741852a8dac7223595ebf80ea8b862fe1ee3569b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 14:28:03 +0200 Subject: [PATCH 23/32] fix(dash): bound the dashboard collector's /dev/kfd probe too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rocm-dash-collectors` runs its own pre-flight before spawning `amd-smi` for the dashboard's GPU metrics, on the same `/dev/kfd` and for the same reason as the CLI launch sites. It used a plain `OpenOptions::open(..).is_ok()`, so it stayed the one launch site that a wedged device could still block indefinitely — the exact hazard the rest of EAI-8457 removes. It now shares `rocm_core::device_readable_bounded`, which carries the 2s bound and the in-flight probe budget with it, instead of restating them. A second copy of that primitive would be free to drift from the gate it belongs to. That needs a new first-party edge, `rocm-dash-collectors -> rocm-core`, so `ALLOWLIST` gains it and the module doc records why. The invariant that forbids this edge binds `rocm-dash-tui`, the presentation layer, which makes no such call and keeps the stricter rule. Signed-off-by: Roman Sirokov --- Cargo.lock | 1 + crates/rocm-core/src/amd_smi.rs | 30 ++++++++++++++++++++++ crates/rocm-core/src/lib.rs | 2 +- crates/rocm-dash-collectors/Cargo.toml | 1 + crates/rocm-dash-collectors/src/amd_smi.rs | 9 ++++++- xtask/src/crate_edges.rs | 9 +++++++ 6 files changed, 50 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4baf28229..5bbb81c862 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3748,6 +3748,7 @@ dependencies = [ "csv", "regex", "reqwest 0.12.28", + "rocm-core", "rocm-dash-core", "serde", "serde_json", diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index 87bda37381..ec35a3a7c5 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -399,6 +399,36 @@ fn device_readable_within_budget( receiver.recv_timeout(timeout).unwrap_or(false) } +/// A read probe of `device` that cannot park the caller. +/// +/// This is the bounded open this module already uses for its own `/dev/kfd` +/// pre-flight, exposed for the other crate that gates an `amd-smi` launch on +/// the same device. +/// +/// `rocm-dash-collectors` runs its own pre-flight before spawning `amd-smi` for +/// the dashboard's GPU metrics, on the same `/dev/kfd` and for the same reason. +/// It used a plain `OpenOptions::open(..).is_ok()`, so the one launch site not +/// routed through this module could still block indefinitely on a wedged device +/// — the exact hazard the gate removes everywhere else. Sharing the probe rather +/// than restating it there keeps the bound, the budget, and the reasoning about +/// why both failure branches decline in one place; a second copy would be free +/// to drift away from all three. +/// +/// Off Linux this is a direct open, matching [`kfd_readable`]'s wave-through: +/// there is no `/dev/kfd` to wedge, and the bound would buy nothing. +pub fn device_readable_bounded(device: &Path) -> bool { + #[cfg(target_os = "linux")] + { + device_readable(device) + } + #[cfg(not(target_os = "linux"))] + { + // Fully qualified: the `fs` import is Linux-gated, since the bounded + // probe is the only other user of it. + std::fs::OpenOptions::new().read(true).open(device).is_ok() + } +} + /// Off-Linux counterpart of [`kfd_readable`]: see that function's docs for why /// the wave-through is load-bearing rather than a stub. /// diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index b74da06a94..bedbd28b57 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -54,7 +54,7 @@ mod test_env; mod test_support; pub mod uv; pub mod windows_driver; -pub use amd_smi::{amd_smi_json, resolve_amd_smi_binary}; +pub use amd_smi::{amd_smi_json, device_readable_bounded, resolve_amd_smi_binary}; pub use automation::{ AuditEventRecord, AutomationEventRecord, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, BuiltinWatcherSpec, EVENT_LOG_ROTATE_BYTES, diff --git a/crates/rocm-dash-collectors/Cargo.toml b/crates/rocm-dash-collectors/Cargo.toml index da6c5c6312..a6557afe7b 100644 --- a/crates/rocm-dash-collectors/Cargo.toml +++ b/crates/rocm-dash-collectors/Cargo.toml @@ -12,6 +12,7 @@ workspace = true [dependencies] rocm-dash-core = { path = "../rocm-dash-core" } +rocm-core = { path = "../rocm-core" } serde = { version = "1", features = ["derive"] } serde_json = "1" csv = "1" diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 564c57dea9..3215d70a0e 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -198,8 +198,15 @@ fn preflight_passes( && (host_kfd_device == real_kfd_device || device_accessible(host_kfd_device))) } +/// Shared with `rocm-core`'s `/dev/kfd` pre-flight rather than restated here. +/// +/// This was a plain `OpenOptions::open(..).is_ok()`, which left the dashboard +/// collector as the one `amd-smi` launch site that could still block forever on +/// a wedged device after EAI-8457 bounded every other one. The shared probe +/// carries the 2s bound and the in-flight budget with it, so this site cannot +/// drift away from the gate it is part of. fn device_accessible(path: &Path) -> bool { - std::fs::OpenOptions::new().read(true).open(path).is_ok() + rocm_core::device_readable_bounded(path) } fn val_u64(v: Option<&Value>) -> Option { diff --git a/xtask/src/crate_edges.rs b/xtask/src/crate_edges.rs index d13c978e17..6b9b7f4975 100644 --- a/xtask/src/crate_edges.rs +++ b/xtask/src/crate_edges.rs @@ -14,6 +14,14 @@ //! leaf crate other subsystems build on. //! 3. `rocm-dash-tui` must not depend on `rocm-core`. //! +//! Invariant 3 binds the presentation layer only. `rocm-dash-collectors` *does* +//! depend on `rocm-core` (EAI-8457): it spawns `amd-smi` for the dashboard's +//! GPU metrics and gates that spawn on the same `/dev/kfd` device, so it shares +//! `rocm_core::device_readable_bounded` rather than restating the bounded open +//! and its probe budget. A second copy of that primitive would be free to drift +//! from the gate it belongs to. The TUI makes no such call and keeps the +//! stricter rule. +//! //! A literal cycle check would be a no-op: Cargo's resolver already rejects //! dependency cycles among normal/build dependencies. The value here is //! entirely in the **allowlist**: every first-party normal/build edge must be @@ -55,6 +63,7 @@ const ALLOWLIST: &[(&str, &str)] = &[ ("rocmd", "rocm-core"), ("rocmd", "rocm-engine-protocol"), ("rocm-engine-protocol", "rocm-core"), + ("rocm-dash-collectors", "rocm-core"), ("rocm-dash-collectors", "rocm-dash-core"), ("rocm-dash-daemon", "rocm-dash-core"), ("rocm-dash-daemon", "rocm-dash-collectors"), From 472efacc9b61b2172e39e118bb243f0b5550bb94 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 15:47:18 +0200 Subject: [PATCH 24/32] docs(amd-smi): document the KFD seam and examine-31, and state the gate's real scope docs/testing.md had no entry for the `ROCM_E2E_KFD_DEVICE` seam or scenario examine-31, which the repo guidelines ask for in the same change. The gate declines on a failed `open` as well as on one that never returns, and the first class cannot cause the D-state hang the rationale described. Keep the fail-safe behaviour -- narrowing it would mean deciding per errno which kernel states are safe to launch into, an unverifiable list of exactly the kind the bounded open exists to avoid resting on -- and say so, including what the broader refusal costs `detect_gpu_count` and what answers in its place. Prune the doc comments to the invariants: drop the review-round narration, the "this used to be X" archaeology and the two citations of a commit hash that does not exist in this repository. Signed-off-by: Roman Sirokov --- apps/rocm/src/gpu_policy.rs | 10 +- crates/rocm-core/src/amd_smi.rs | 405 ++++++++++----------- crates/rocm-dash-collectors/src/amd_smi.rs | 9 +- docs/architecture.md | 2 +- docs/testing.md | 74 ++++ 5 files changed, 277 insertions(+), 223 deletions(-) diff --git a/apps/rocm/src/gpu_policy.rs b/apps/rocm/src/gpu_policy.rs index 3e0c36f1e6..a5b44bbdfd 100644 --- a/apps/rocm/src/gpu_policy.rs +++ b/apps/rocm/src/gpu_policy.rs @@ -509,8 +509,7 @@ fn select_auto_gpu_index( /// lock, `detect_gpu_count` either in the `--gpu ` validation or inside /// the lock for `--gpu auto` — so a Linux host where the KFD open hangs *and* /// `amd-smi` then stalls adds up to ~24s to an interactive launch (2 × (2s + -/// 10s)); the single-probe `--model` readiness path adds up to ~12s. Both -/// figures used to be quoted as 10s and 20s, which omitted the pre-flight. +/// 10s)); the single-probe `--model` readiness path adds up to ~12s. /// /// The pre-flight's 2s is only spent on a host whose device open is actually /// hanging, which is also the host on which the probe then declines and the @@ -839,6 +838,13 @@ fn busy_gpu_indices(paths: &AppPaths) -> Vec { /// Best-effort count of local AMD GPUs via `amd-smi`. Returns `None` when /// amd-smi is unavailable or its output cannot be parsed (callers then fall /// back to conservative defaults). +/// +/// Also `None` whenever the `/dev/kfd` pre-flight declines the launch, which it +/// does for an unreachable device and not only for one that would hang — so a +/// host outside the render group lands here, with no sysfs fallback of the kind +/// [`gpu_vram_usage`] has. That is deliberate and is not a loss of GPU +/// detection; `rocm_core`'s `kfd_readable` documents why, and what answers in +/// this count's place. pub(crate) fn detect_gpu_count() -> Option { let value = rocm_core::amd_smi_json(&["list", "--json"], AMD_SMI_SERVE_PROBE_TIMEOUT).ok()?; let count = value.as_array().map(Vec::len)?; diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index ec35a3a7c5..c54a52035d 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -8,10 +8,9 @@ //! recorded in the registry, then falls back to the home-directory venv and //! legacy locations, then to the bare `amd-smi` name for a `PATH` lookup. //! -//! Launching goes through [`amd_smi_json`], which is the one supported way for -//! a synchronous caller to start the binary: it runs the `/dev/kfd` pre-flight -//! that keeps `amd-smi` away from a device that can park a process in -//! uninterruptible sleep, and the pre-flight protects nothing unless every +//! Launching goes through [`amd_smi_json`], the one supported way for a +//! synchronous caller to start the binary: it runs the `/dev/kfd` pre-flight in +//! `kfd_readable` first, and that pre-flight protects nothing unless every //! launch site goes through it. use std::ffi::{OsStr, OsString}; @@ -106,12 +105,10 @@ fn resolve_amd_smi_binary_in_home(home_dir: Option<&Path>) -> OsString { "amd-smi".into() } -/// Device node of the AMD kernel fusion driver. Bare-metal Linux only: no other -/// platform exposes one, and WSL — which *is* `target_os = "linux"` — is not an -/// exception to hand-wave past. It reaches the GPU through `/dev/dxg` and the -/// Windows host driver and has no `/dev/kfd` at all, which is why the WSL -/// diagnosis catalog skips every bare-metal driver check (see `docs/wsl.md` and -/// [`crate::examine::WslFacts`]). +/// Device node of the AMD kernel fusion driver. Bare-metal Linux only. WSL +/// compiles as `target_os = "linux"` but reaches the GPU through `/dev/dxg` and +/// the Windows host driver, and has no `/dev/kfd` at all — see `docs/wsl.md` +/// and [`crate::examine::WslFacts`]. const KFD_DEVICE: &str = "/dev/kfd"; /// Name of the E2E-only seam that moves the pre-flight's device node, so the @@ -128,20 +125,18 @@ pub(crate) const TEST_KFD_DEVICE_ENV: &str = "ROCM_E2E_KFD_DEVICE"; /// [`TEST_KFD_DEVICE_ENV`] name a different path, which is what makes the gate /// testable from the outside at all. /// -/// Why a seam rather than the existing [`crate::hardware_root`] re-rooting: -/// [`kfd_readable`] deliberately keeps the *real* node's veto, so a simulated -/// root can only ever make the gate decline, never allow. That asymmetry is the -/// invariant that stops a simulated root launching `amd-smi` on a machine with -/// no GPU, and it is correct — but it also means no simulated root can exercise -/// the *allowed* branch on a GPU-less runner, which is where the per-PR lane -/// runs. A path override sidesteps that without touching the invariant: the -/// production build has no override, so the veto and the real device are all -/// there is. +/// A path seam rather than the existing [`crate::hardware_root`] re-rooting, +/// because the two cannot do the same job here: [`kfd_readable`] keeps the real +/// node's veto on purpose, so a simulated root can only ever make the gate +/// decline, never allow. That is the invariant stopping a test hook from +/// launching `amd-smi` on a machine with no GPU — and it also means no +/// simulated root can reach the *allowed* branch on the GPU-less lane that runs +/// per PR. /// -/// Safe to be an override rather than a bypass. It relocates the probe; it does -/// not skip it. A build with the feature on and the variable unset — every unit -/// test in this crate, and every E2E scenario that does not set it — gates on -/// `/dev/kfd` exactly as production does. +/// An override, not a bypass: it relocates the probe and does not skip it. With +/// the feature on and the variable unset — every unit test here, and every E2E +/// scenario that does not set it — the gate is `/dev/kfd` exactly as in +/// production. #[cfg(feature = "e2e-test-hooks")] fn preflight_kfd_device() -> PathBuf { std::env::var_os(TEST_KFD_DEVICE_ENV).map_or_else(|| PathBuf::from(KFD_DEVICE), Into::into) @@ -157,48 +152,70 @@ fn preflight_kfd_device() -> PathBuf { /// Whether `/dev/kfd` can be opened for reading — the pre-flight every /// `amd-smi` launch goes through, and it is **mandatory** before every launch. /// -/// What it actually detects is **inaccessible**, not **unhealthy**. `kfd_open` -/// refuses when the node is absent, when permission is denied, when the driver -/// exposes no KFD GPU node, and while a reset is in progress; a driver that is -/// merely version-mismatched or wedged after a fault still opens. Those are the -/// states in which `amd-smi` blocks in uninterruptible kernel sleep — no signal -/// escapes, so the process survives both Ctrl-C and `SIGKILL`, and no timeout -/// rescues it, because killing a D-state child is a no-op. This gate removes -/// the reachable half of that hazard; it does not claim to see the rest. +/// # What the predicate is /// -/// The probe does not assume `kfd_open` returns promptly. An `open` that blocks -/// — whatever the reason, including a state this gate has not anticipated — -/// is bounded by [`KFD_OPEN_TIMEOUT`] and read as "not readable", so the gate -/// declines rather than inheriting the stall it exists to prevent. See -/// [`device_readable_within_budget`]. +/// "This process can reach the KFD", not "`amd-smi` would hang". The two are +/// different sets, and declining the difference is the intended trade: /// -/// Off Linux there is no such device and no such kernel state, so the check is -/// `true` without probing anything. That wave-through is load-bearing rather -/// than pedantic: `amd-smi.exe` is a supported Windows binary (see -/// [`crate::managed_runtime::managed_sdk_tool_path`]), so a check that simply -/// failed to find `/dev/kfd` there would silently disable GPU detection on -/// every Windows host. +/// * An open that **never returns** is the hazard itself — `amd-smi` against +/// such a device blocks in uninterruptible kernel sleep, where no signal +/// escapes, so neither Ctrl-C nor `SIGKILL` nor any caller's timeout reaches +/// it. [`KFD_OPEN_TIMEOUT`] bounds the probe and reads the overrun as "not +/// readable", so the gate declines instead of inheriting the stall. +/// * An open that **fails with an errno** — `ENOENT` for an absent node, +/// `EACCES` outside the render group, whatever a driver mid-reset returns — +/// cannot park anything by itself. The gate declines these too. /// -/// WSL needs a wave-through for the same reason and cannot get it from `cfg`: -/// it compiles as Linux yet has no `/dev/kfd` (see [`KFD_DEVICE`]), so the real -/// open fails on every WSL host. But WSL identity alone is not the condition. -/// `b4f98bc2` replaced exactly such a bare identity check in the dashboard -/// pre-flight with [`has_usable_amd_gpu`], because `/dev/dxg` is vendor-agnostic -/// and a WSL host whose ROCDXG plumbing is not ready is one where `amd-smi` -/// still hangs — a different hazard from the KFD one, which an "is this WSL" -/// test cannot tell apart from a healthy host. So the substitution is +/// The second class is broader than the hazard on purpose. Narrowing it means +/// deciding per errno which kernel states are safe to launch into, and that +/// list cannot be checked against a resetting or wedged device by any test this +/// repo can run; a gate that declined only on the timeout would be assuming +/// every errno it had not anticipated is harmless. Not resting on an assumption +/// of that shape is the whole reason the open is bounded in the first place +/// (see [`KFD_OPEN_TIMEOUT`]). A refusal is recoverable and names what it +/// refused; a hang is neither. +/// +/// # What the broad refusal costs +/// +/// On a host outside the render group `amd-smi` might well have answered — much +/// of its inventory comes from sysfs — so `gpu_policy::detect_gpu_count` now +/// returns `None` there and, unlike `gpu_vram_usage`, has no sysfs fallback of +/// its own. That is a real narrowing. It is not a loss of GPU detection: on the +/// hosts this branch can fire on, `serve` resolves ordinals from +/// [`crate::usable_amd_gpu_indices`], which counts KFD topology nodes under +/// `/sys/class/kfd` and AMD DRM cards under `/sys/class/drm` — both +/// world-readable, neither needing the device node open — and +/// `validate_pinned_gpu_index` prefers that set outright. What is lost is the +/// secondary count `--gpu auto` ranks with, so on a *multi*-GPU host in that +/// state auto-selection returns no selection and the engine's own device probe +/// picks. `rocm diagnose` already names the host, as `fix-4-render-group`. +/// +/// # Platforms with no KFD +/// +/// Off Linux the check is `true` without probing. Load-bearing rather than +/// pedantic: `amd-smi.exe` is a supported Windows binary (see +/// [`crate::managed_runtime::managed_sdk_tool_path`]), so reading an absent +/// `/dev/kfd` as "unsafe" would silently disable GPU detection on every Windows +/// host. +/// +/// WSL needs the same wave-through and cannot get it from `cfg`: it compiles as +/// Linux yet has no `/dev/kfd` (see [`KFD_DEVICE`]), so the real open fails on +/// every WSL host. WSL identity alone is not the condition, though — `/dev/dxg` +/// is vendor-agnostic, and a WSL host whose ROCDXG plumbing is not ready is one +/// where `amd-smi` still hangs, which a bare "is this WSL" test cannot tell +/// apart from a healthy one. The condition is therefore /// `is_wsl_host() && has_usable_amd_gpu()`, the same conjunction -/// `apps/rocm/src/dash.rs::gpu_reachable_for_preflight` feeds the dashboard, and -/// `serve`/`rocmd`/the dashboard keep the single cross-surface verdict that -/// commit established. On bare metal `is_wsl_host()` short-circuits, so -/// [`has_usable_amd_gpu`] is never consulted and behaviour is a plain +/// `apps/rocm/src/dash.rs::gpu_reachable_for_preflight` feeds the dashboard, so +/// `serve`, `rocmd` and the dashboard keep one cross-surface verdict. On bare +/// metal `is_wsl_host()` short-circuits and the behaviour is a plain /// `/dev/kfd` open. /// -/// Hardware-probe routing (see [`crate::hardware_root`]): like the dashboard -/// pre-flight, this reads `host_path("/dev/kfd")` *and* keeps the real node's -/// veto. A simulated E2E root can therefore make the gate decline on a machine -/// that has a GPU, but can never make it allow a launch on a machine that does -/// not. +/// # Hardware-probe routing +/// +/// Like the dashboard pre-flight, this reads `host_path("/dev/kfd")` *and* +/// keeps the real node's veto (see [`crate::hardware_root`]). A simulated E2E +/// root can make the gate decline on a machine that has a GPU, but can never +/// make it allow a launch on a machine that does not. #[cfg(target_os = "linux")] fn kfd_readable(device: &Path) -> bool { kfd_readable_with( @@ -226,20 +243,18 @@ fn kfd_readable_with(wsl_gpu_reachable: bool, real_device: &Path, host_device: & /// How long the pre-flight waits for the device `open` itself before giving up /// on it and declining the launch. /// -/// The gate exists to avoid spawning `amd-smi` against a device that can park a -/// process in uninterruptible sleep. Performing the probe with a plain blocking -/// `open` would make the gate's own safety rest on an *assumption* — that -/// `kfd_open` always returns promptly, and that only amd-smi's later ioctls can -/// enter D-state — and that assumption cannot be verified against a -/// reset-in-progress device from any test. If it were ever wrong, the gate -/// would have relocated the hazard into a synchronous call with *less* -/// protection than the spawn it replaced, since there is no child to kill. +/// A plain blocking `open` would make the gate's own safety rest on an +/// assumption — that `kfd_open` always returns promptly, so only `amd-smi`'s +/// later ioctls can enter D-state — which cannot be checked against a +/// reset-in-progress device by any test here. Were it wrong, the gate would +/// have moved the hazard into a synchronous call with *less* protection than +/// the spawn it replaced, there being no child to kill. /// -/// Bounding the open removes the need to be right about that: a `/dev/kfd` open -/// that has not returned in this long is treated as "not readable", which is the -/// declining answer, so the unverifiable case fails in the safe direction. Short -/// because a healthy `kfd_open` is a handful of microseconds and the only thing -/// a longer wait buys is a longer stall on a host that is already unhealthy. +/// Bounding the open removes the need to be right about it: an open that has +/// not returned in this long reads as "not readable", the declining answer, so +/// the unverifiable case fails safe. Short, because a healthy `kfd_open` takes +/// microseconds and a longer wait only buys a longer stall on a host that is +/// already unhealthy. /// /// **It is additive to every [`amd_smi_json`] `timeout`, not inside it.** The /// pre-flight finishes before anything is spawned, so a caller that passes a @@ -253,16 +268,15 @@ const KFD_OPEN_TIMEOUT: Duration = Duration::from_secs(2); /// starts declining outright. /// /// Every probe that overruns [`KFD_OPEN_TIMEOUT`] leaves a thread blocked in -/// `open` until the kernel lets it go, which against the wedged device this gate -/// exists to refuse may be never. `rocmd`'s telemetry watcher and `serve`'s +/// `open` until the kernel lets it go, which against the wedged device this +/// gate exists to refuse may be never. `rocmd`'s telemetry watcher and `serve`'s /// probes both poll, so without a cap a host stuck in that state accumulates one /// parked thread per poll until the process runs out of its thread budget. /// -/// Small on purpose. The only thing a larger budget buys is more parked threads -/// on a host that is already answering "not readable"; a healthy `kfd_open` -/// returns in microseconds, so even one slot is enough for every caller this -/// crate has, and 8 leaves headroom for concurrent callers without ever -/// approaching a thread limit. +/// Small on purpose: a larger budget only buys more parked threads on a host +/// already answering "not readable". A healthy `kfd_open` returns in +/// microseconds, so one slot would serve every caller this crate has; 8 leaves +/// headroom for concurrent ones without approaching a thread limit. #[cfg(target_os = "linux")] const MAX_INFLIGHT_KFD_PROBES: usize = 8; @@ -340,33 +354,24 @@ fn device_readable(device: &Path) -> bool { /// the kernel. A probe that overruns leaves that thread parked — the same /// trade-off [`crate::process::run_with_timeout`] makes when it abandons rather /// than joins its readers, and for the same reason: one leaked thread is -/// strictly cheaper than a blocked caller, and a device that can park a thread -/// forever is exactly the device this whole gate exists to refuse. The opened -/// handle is dropped on the probe thread, so a late success closes its own -/// descriptor rather than leaking one. +/// cheaper than a blocked caller. The opened handle is dropped on the probe +/// thread, so a late success closes its own descriptor rather than leaking one. /// /// # Why both failure branches decline /// -/// The parked threads are bounded by `budget`, and a probe that cannot claim a -/// slot returns `false` without opening anything. A thread that cannot be -/// spawned at all returns `false` too. -/// -/// Both of those used to fall back to a direct blocking `open` on the calling -/// thread, on the reasoning that refusing a launch because the process was out -/// of threads would be a worse failure than the one being guarded against. That -/// reasoning does not survive contact with the escalation it has to handle. The -/// states in which this process runs out of threads are not independent of the -/// state this gate refuses: the way a thread budget is exhausted *here* is -/// precisely one parked `open` per poll against a wedged device, so the fallback -/// fires exactly when the device is at its most dangerous, and performs the -/// unbounded blocking `open` the bound exists to prevent — on the caller's own -/// thread, where there is nothing left to kill. A refusal is recoverable and -/// visible: `amd_smi_json` says it declined and names the device, the GPU-aware -/// paths degrade to service state alone, and the next poll on a recovered host -/// finds its slots freed and allows the launch again. A hang is neither. +/// A probe that cannot claim a `budget` slot, and one whose thread cannot be +/// spawned at all, both answer `false` rather than falling back to a direct +/// `open` on the calling thread. Neither failure is independent of the state +/// this gate refuses: the way threads run out *here* is one parked `open` per +/// poll against a wedged device, so such a fallback would fire exactly when the +/// device is most dangerous and perform the unbounded blocking `open` the bound +/// exists to prevent — on the caller's own thread, with nothing left to kill. /// -/// So both branches answer `false` — "not readable" — which is the same -/// declining answer the timeout gives, for the same reason. +/// Refusing instead is recoverable and visible: [`amd_smi_json`] says it +/// declined and names the device, GPU-aware callers degrade to service state +/// alone, and the next poll on a recovered host finds its slots freed and +/// allows the launch again. So both branches give the same `false` the timeout +/// gives, for the same reason. #[cfg(target_os = "linux")] fn device_readable_within_budget( device: &Path, @@ -401,18 +406,11 @@ fn device_readable_within_budget( /// A read probe of `device` that cannot park the caller. /// -/// This is the bounded open this module already uses for its own `/dev/kfd` -/// pre-flight, exposed for the other crate that gates an `amd-smi` launch on -/// the same device. -/// -/// `rocm-dash-collectors` runs its own pre-flight before spawning `amd-smi` for -/// the dashboard's GPU metrics, on the same `/dev/kfd` and for the same reason. -/// It used a plain `OpenOptions::open(..).is_ok()`, so the one launch site not -/// routed through this module could still block indefinitely on a wedged device -/// — the exact hazard the gate removes everywhere else. Sharing the probe rather -/// than restating it there keeps the bound, the budget, and the reasoning about -/// why both failure branches decline in one place; a second copy would be free -/// to drift away from all three. +/// The same bounded open this module uses for its own `/dev/kfd` pre-flight, +/// exposed for `rocm-dash-collectors`, which gates the dashboard's GPU-metrics +/// `amd-smi` launch on the same device. Shared rather than restated there so +/// the bound, the budget and the reasoning on `device_readable_within_budget` +/// stay in one place; a second copy would be free to drift from all three. /// /// Off Linux this is a direct open, matching [`kfd_readable`]'s wave-through: /// there is no `/dev/kfd` to wedge, and the bound would buy nothing. @@ -445,25 +443,24 @@ const fn kfd_readable(_device: &Path) -> bool { /// Run `amd-smi ` and parse its stdout as JSON, declining to launch at all /// unless the `/dev/kfd` pre-flight in [`kfd_readable`] passes. /// -/// This is the supported way to launch `amd-smi` from a synchronous caller. The +/// The supported way to launch `amd-smi` from a synchronous caller. The /// pre-flight has to happen at every launch site or it protects nothing, and -/// routing the launch through one function is what keeps that true as sites are -/// added. +/// routing every launch through one function is what keeps that true as sites +/// are added. /// -/// `timeout` is defence in depth, not the mitigation: see -/// [`crate::process::run_with_timeout`], which kills the timed-out child and -/// waits on it — neither of which dislodges a process in uninterruptible sleep, -/// and the `wait` of an undislodgeable one does not return either. It bounds -/// the ordinary failures — a slow or wedged-but-signalable `amd-smi` — and the -/// pre-flight above, not the timeout, is what keeps the D-state case away. +/// `timeout` is defence in depth, not the mitigation: [`crate::process::run_with_timeout`] +/// kills the timed-out child and waits on it, and neither dislodges a process +/// in uninterruptible sleep — nor does the `wait` on one return. It bounds the +/// ordinary failures (a slow or wedged-but-signalable `amd-smi`); the +/// pre-flight is what keeps the D-state case away. /// /// # Latency /// -/// `timeout` is not the worst case for the call. The pre-flight runs first and -/// is bounded separately by [`KFD_OPEN_TIMEOUT`] (2s on Linux), so the worst -/// case is `KFD_OPEN_TIMEOUT + timeout` — 12s for a 10s `timeout`, 4s for the -/// 2s one `rocmd`'s watcher passes. A caller that documents its own worst case -/// has to add the pre-flight's bound to the one it passes here. +/// `timeout` is not the worst case. The pre-flight runs first and is bounded +/// separately by [`KFD_OPEN_TIMEOUT`] (2s on Linux), so the worst case is +/// `KFD_OPEN_TIMEOUT + timeout` — 12s for a 10s `timeout`, 4s for the 2s one +/// `rocmd`'s watcher passes. A caller documenting its own worst case has to add +/// the pre-flight's bound to the one it passes here. pub fn amd_smi_json(args: &[&str], timeout: Duration) -> Result { amd_smi_json_with( &resolve_amd_smi_binary(), @@ -485,8 +482,9 @@ fn amd_smi_json_with( if !kfd_readable(kfd_device) { bail!( "declined to launch `amd-smi {}`: {} could not be opened for reading (absent, \ - permission denied, no KFD GPU node, or a reset in progress), and amd-smi can \ - block unkillably in uninterruptible sleep against a device in that state", + permission denied, no KFD GPU node, or a reset in progress); the pre-flight \ + refuses an unreachable device because amd-smi can block unkillably in \ + uninterruptible sleep against one whose open never returns", args.join(" "), kfd_device.display() ); @@ -543,9 +541,8 @@ mod tests { /// The gate resolves its device through `host_path`, so in an /// `e2e-test-hooks` build another test setting `ROCM_CLI_TEST_HOST_ROOT` /// process-wide would re-root this test's own tempdir device out from under - /// it, and the gate would decline a device the test had just created. This - /// is the same lock `fix.rs` holds for the same reason; without the feature - /// it is uncontended and costs nothing. + /// it. Same lock `fix.rs` takes, for the same reason; without the feature it + /// is uncontended and costs nothing. #[cfg(target_os = "linux")] fn host_root_guard() -> std::sync::MutexGuard<'static, ()> { crate::hardware_root::HOST_ROOT_TEST_LOCK @@ -553,22 +550,20 @@ mod tests { .unwrap_or_else(std::sync::PoisonError::into_inner) } - /// The gate has to stop the *spawn*, not merely report a failure after the - /// fact: the launch that already happened is the launch that hangs. The - /// fake `amd-smi` leaves a marker file, so its absence is direct evidence + /// The gate has to stop the *spawn*, not merely report a failure after it: + /// the launch that already happened is the launch that hangs. The fake + /// `amd-smi` leaves a marker file, so its absence is direct evidence /// nothing ran — and running the same fixture against a readable device /// proves the fake would otherwise have executed, which is what stops the - /// blocked case from passing vacuously. + /// blocked case passing vacuously. /// - /// Linux-only because `kfd_readable` is deliberately a no-op elsewhere; - /// off Linux there is no gate to exercise. + /// Linux-only: off Linux `kfd_readable` is deliberately a no-op, so there + /// is no gate to exercise. #[test] #[cfg(target_os = "linux")] fn the_kfd_gate_stops_amd_smi_before_it_is_spawned() -> Result<()> { - // WSL compiles as Linux but is waved through deliberately (see - // `the_kfd_preflight_waves_through_wsl_hosts_only_when_the_gpu_is_reachable`), - // so there is no gate to exercise on such a host and the blocked case - // could never hold. + // WSL is waved through deliberately, so the blocked case cannot hold + // there. if is_wsl_host() { return Ok(()); } @@ -634,19 +629,16 @@ mod tests { assert!(kfd_readable(Path::new("/definitely/not/a/device/node"))); } - /// WSL has no `/dev/kfd` either, but unlike Windows it cannot be waved - /// through by `cfg`: it compiles as Linux, so the real open runs, always - /// fails, and declines every routed `amd-smi` launch on a supported - /// platform — silently, since both `serve` probes drop the error with - /// `.ok()?`. Same wave-through as the off-Linux case above, decided at - /// runtime instead. + /// WSL has no `/dev/kfd` either, but unlike Windows cannot be waved through + /// by `cfg`: it compiles as Linux, so the real open runs, always fails, and + /// would decline every routed `amd-smi` launch on a supported platform — + /// silently, since both `serve` probes drop the error with `.ok()?`. /// /// The *conjunction* is what is pinned, not WSL identity: a WSL host whose - /// ROCDXG plumbing is not ready is one where `amd-smi` still hangs, which is - /// why `b4f98bc2` replaced a bare identity check with `has_usable_amd_gpu()` - /// in the dashboard pre-flight. The unready-WSL case below is the one that - /// distinguishes this from that mistake; the bare-metal case is the control - /// that stops the whole test passing with the gate deleted. + /// ROCDXG plumbing is not ready is one where `amd-smi` still hangs, so a + /// bare identity check would wave it through. The unready-WSL case + /// distinguishes the two; the bare-metal case is the control that stops the + /// test passing with the gate deleted. #[test] #[cfg(target_os = "linux")] fn the_kfd_preflight_waves_through_wsl_hosts_only_when_the_gpu_is_reachable() { @@ -701,35 +693,22 @@ mod tests { Ok(()) } - /// The gate's own probe must not be able to hang. - /// - /// `kfd_readable` was reviewed as resting on an unverified assumption: that - /// `open("/dev/kfd")` always returns promptly, so that only amd-smi's later - /// ioctls can enter uninterruptible sleep. If that were ever wrong the gate - /// would have moved the hazard into a synchronous call with *less* - /// protection than the spawn it replaced — no child to kill, no timeout. - /// Rather than assert the assumption, the probe is bounded; this pins that. - /// - /// A FIFO is the fixture that makes the blocking case real rather than - /// simulated: `open(O_RDONLY)` on a FIFO with no writer blocks in the kernel - /// indefinitely, which is the observable shape of the hazard. The deadline - /// assertion is what distinguishes a bounded probe from an unbounded one — - /// without it, deleting the timeout would hang the suite rather than fail - /// this test, and a hang is not a readable failure. The readable control in - /// the same test stops the whole thing passing vacuously, e.g. if - /// `device_readable_within_budget` were mutated to return `false` - /// unconditionally. + /// The gate's own probe must not be able to hang: an unbounded `open` would + /// move the D-state hazard into a synchronous call with *less* protection + /// than the spawn it replaced — no child to kill, no timeout. /// - /// Runs against a budget of its own rather than [`KFD_PROBE_BUDGET`]: it - /// deliberately parks a probe thread for the lifetime of the test binary, - /// and spending a production slot would make every later test in this - /// process share a quietly smaller budget. + /// A FIFO makes the blocking case real rather than simulated: + /// `open(O_RDONLY)` on one with no writer blocks in the kernel + /// indefinitely, the observable shape of the hazard. The deadline assertion + /// is what distinguishes a bounded probe from an unbounded one — without + /// it, deleting the timeout would hang the suite instead of failing here, + /// and a hang is not a readable failure. The readable control stops the test + /// passing vacuously against a probe mutated to return `false` always. /// - /// The FIFO is created by shelling out to `mkfifo(1)` rather than calling - /// `libc::mkfifo`: the workspace lints deny `unsafe_code`, and one test - /// fixture does not justify an exception. The tool is coreutils, so it is - /// present wherever this test runs; a host without it skips rather than - /// fails, since a missing fixture is not evidence about the probe. + /// Uses a budget of its own rather than [`KFD_PROBE_BUDGET`]: it parks a + /// probe thread for the lifetime of the test binary, and spending a + /// production slot would quietly shrink the budget for every later test in + /// this process. #[test] #[cfg(target_os = "linux")] fn the_kfd_probe_declines_a_device_whose_open_blocks_instead_of_hanging() -> Result<()> { @@ -744,12 +723,10 @@ mod tests { }; let probe_timeout = Duration::from_millis(300); - // The probe runs on a thread the test watches with its own, much looser - // deadline, rather than being called inline. A mutant that drops the - // bound from `device_readable_within_budget` would block forever on this - // FIFO; called inline that would hang the whole test binary, and a hang - // is a CI timeout rather than a readable failure. Watched, it fails here - // with a message naming the cause. + // Watched on its own thread with a much looser deadline rather than + // called inline: a mutant dropping the bound would block forever on this + // FIFO, and called inline that is a CI timeout rather than a failure + // naming its cause. let (verdict_tx, verdict_rx) = std::sync::mpsc::channel(); thread::Builder::new() .name("kfd-open-probe-test".to_owned()) @@ -788,22 +765,21 @@ mod tests { /// Parked probes are bounded, and a probe that cannot claim a slot declines /// rather than opening anything on the calling thread. /// - /// The escalation this pins is the one the review named: against a wedged - /// device every poll leaves a thread parked in `open`, so an unbounded probe - /// accumulates them until the process runs out of threads, at which point a - /// fallback to a direct blocking open would reinstate exactly the hang the - /// gate exists to refuse. + /// The escalation: against a wedged device every poll leaves a thread + /// parked in `open`, so an uncapped probe accumulates them until the process + /// runs out of threads — at which point a fallback to a direct blocking open + /// would reinstate exactly the hang the gate exists to refuse. /// - /// Non-vacuous by construction. The device the capped probe is pointed at is - /// an ordinary readable file, so an unbounded implementation — or one whose - /// exhausted branch fell back to a blocking open — answers `true`. Only a - /// probe that refuses on an exhausted budget answers `false`. The control - /// re-runs the same device against a budget with a free slot and must get - /// `true`, so a mutant returning `false` unconditionally fails too. + /// Non-vacuous by construction. The capped probe is pointed at an ordinary + /// readable file, so an uncapped implementation — or one whose exhausted + /// branch fell back to a blocking open — answers `true`; only a probe that + /// refuses on an exhausted budget answers `false`. The control re-runs the + /// same device against a budget with a free slot and must get `true`, so a + /// mutant returning `false` always fails too. /// - /// Uses a budget of its own, with a limit of one: the single parked FIFO - /// probe below spends it, and it is never freed, so sharing the production - /// budget would silently shrink it for every other test in this process. + /// Uses budgets of its own, limit one: the parked FIFO probe spends the + /// slot and never frees it, so sharing the production budget would silently + /// shrink it for every other test in this process. #[test] #[cfg(target_os = "linux")] fn the_kfd_probe_budget_declines_instead_of_parking_threads_without_bound() -> Result<()> { @@ -848,9 +824,9 @@ mod tests { /// A FIFO under `root`, or `None` on a host with no `mkfifo(1)`. /// - /// Shelled out rather than calling `libc::mkfifo`: the workspace lints deny - /// `unsafe_code` and one test fixture does not justify an exception. A - /// missing coreutils is not evidence about the probe, so the caller skips. + /// Shelled out rather than calling `libc::mkfifo`, because the workspace + /// lints deny `unsafe_code`. A missing coreutils is not evidence about the + /// probe, so the caller skips rather than fails. #[cfg(target_os = "linux")] fn make_fifo(root: &Path, name: &str) -> Option { let fifo = root.join(name); @@ -869,17 +845,16 @@ mod tests { /// Every other gate test reaches `amd_smi_json_with` with a device of its /// own choosing, so swapping the constant out of the public wrapper — which /// disables the gate for every production caller — leaves them all green. - /// This compares the wrapper's observable verdict against the gate read - /// directly, so the two can only agree while the wrapper passes the real + /// Comparing the wrapper's observable verdict against the gate read + /// directly means the two can only agree while the wrapper passes the real /// device through. /// /// Non-vacuous on *every* host in an `e2e-test-hooks` build: the empty - /// simulated root below has no `dev/kfd`, so the gate declines whatever the + /// simulated root has no `dev/kfd`, so the gate declines whatever the /// machine has, while a wrapper passing any other path — `/` being the /// obvious mutant — re-roots to the root directory itself, which opens. /// Without the feature it is non-vacuous on any host with no readable - /// `/dev/kfd` (every non-GPU CI runner) and asserts only the absence of a - /// spurious decline on a GPU host. + /// `/dev/kfd` (every non-GPU CI runner). #[test] #[cfg(target_os = "linux")] fn the_public_amd_smi_entry_point_gates_on_the_real_kfd_device() { @@ -924,12 +899,12 @@ mod tests { /// the pre-flight device is `/dev/kfd` on every build, and with it set the /// gate still *runs* — against the named path — rather than being skipped. /// - /// Worth pinning because the seam's whole justification is that it cannot - /// weaken a shipped build. Both halves are load-bearing: a `preflight_kfd_device` - /// that read the variable unconditionally would fail the unset half here and - /// hand any process on the host a way to disable the gate, and one that - /// returned a readable path outright — the bypass this deliberately is not — - /// would fail the declining half. + /// The seam's whole justification is that it cannot weaken a shipped build, + /// and both halves are load-bearing: a `preflight_kfd_device` reading the + /// variable unconditionally would fail the unset half and hand any process + /// on the host a way to disable the gate, and one returning a readable path + /// outright — the bypass this deliberately is not — would fail the + /// declining half. #[test] #[cfg(target_os = "linux")] fn the_e2e_kfd_path_seam_moves_the_gate_without_disabling_it() { diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 3215d70a0e..5aacb222ce 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -200,11 +200,10 @@ fn preflight_passes( /// Shared with `rocm-core`'s `/dev/kfd` pre-flight rather than restated here. /// -/// This was a plain `OpenOptions::open(..).is_ok()`, which left the dashboard -/// collector as the one `amd-smi` launch site that could still block forever on -/// a wedged device after EAI-8457 bounded every other one. The shared probe -/// carries the 2s bound and the in-flight budget with it, so this site cannot -/// drift away from the gate it is part of. +/// A plain `OpenOptions::open(..).is_ok()` would leave the dashboard collector +/// as the one `amd-smi` launch site that can still block forever on a wedged +/// device. The shared probe carries the 2s bound and the in-flight budget with +/// it, so this site cannot drift away from the gate it is part of. fn device_accessible(path: &Path) -> bool { rocm_core::device_readable_bounded(path) } diff --git a/docs/architecture.md b/docs/architecture.md index 439efc7b7c..eafb9c62fb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,7 +98,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): -- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count` and `rocmd`'s `common.rs::capture_amd_smi_json`; the `rocm dash` collectors keep their own pre-flight, since those crates deliberately do not depend on this one. +- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count` and `rocmd`'s `common.rs::capture_amd_smi_json`. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151, Phase 7d) the built-in automation-watcher catalog, automation-daemon runtime state, and its on-disk event/proposal/audit records — see the module's own doc comment for what it owns. [`config.rs`](../crates/rocm-core/src/config.rs) and this module depend on each other — `config.rs` imports `BuiltinWatcherSpec` from here, and this module imports `WatcherMode` from `config.rs` — a back-edge the split introduces without eliminating; `WatcherMode` stays with the persisted config because it is part of `RocmCliConfig`'s own on-disk shape, not an automation-daemon concern. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. diff --git a/docs/testing.md b/docs/testing.md index 782e892e75..203ac3d981 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2127,6 +2127,80 @@ for them, including for the fallback; their parsing is covered by the unit tests above, and the fallback itself by `hardware_root`'s own unit tests, since both now read through the same helper with an identity resolver. +## amd-smi KFD pre-flight + +Every synchronous `amd-smi` launch goes through +[`rocm_core::amd_smi_json`](../crates/rocm-core/src/amd_smi.rs), which first +tries to open `/dev/kfd` for reading and declines the launch outright when it +cannot. Against a device in the wrong kernel state `amd-smi` blocks in +uninterruptible sleep, where no signal and therefore no caller timeout reaches +it, so declining is the only outcome that bounds `rocmd`'s telemetry poll and +`serve`'s GPU probes. The open is itself bounded (2s) and the number of probes +that may be parked at once is capped (8), so the gate's own probe cannot inherit +the stall it exists to refuse. The dashboard collector +([amd_smi.rs](../crates/rocm-dash-collectors/src/amd_smi.rs)) gates its own +launch on the same device through the same shared probe, +`rocm_core::device_readable_bounded`. + +The gate declines on *any* unreachable device, including the fast errno cases +(absent node, permission denied) that cannot hang anything. That is wider than +the hazard and deliberate; `kfd_readable`'s doc comment says why, and names what +`serve` still answers from on such a host. + +```bash +cargo test -p rocm-core --lib amd_smi +cargo test -p rocm-core --lib --features e2e-test-hooks -- the_e2e_kfd_path_seam_moves_the_gate_without_disabling_it the_public_amd_smi_entry_point_gates_on_the_real_kfd_device +``` + +The unit tests cover the gate in isolation: that it stops the *spawn* rather +than reporting a failure afterwards (a stub `amd-smi` leaves a marker file, and +its absence is the evidence), that the probe declines rather than hangs against +a device whose `open` blocks (a FIFO with no writer), that the in-flight budget +declines rather than parking threads without bound, that the WSL and off-Linux +wave-throughs hold, and that the public entry point gates on `/dev/kfd` itself +rather than on some other path. + +**The `ROCM_E2E_KFD_DEVICE` seam.** `rocm_core::preflight_kfd_device` reads this +variable and uses the path it names as the device the pre-flight opens. Only a +build carrying the `e2e-test-hooks` feature honours it; a released artefact has +no override compiled in and always gates on `/dev/kfd`. It moves the gate rather +than removing it, so a scenario pointing it at an unopenable path gets the +declining branch and one pointing it at an ordinary file gets the allowed +branch, while with the variable unset the behaviour is production's on every +build. + +`ROCM_CLI_TEST_HOST_ROOT` cannot stand in for it. The gate keeps the real +`/dev/kfd`'s veto on purpose, so a simulated root can only ever make it decline, +never allow — which is what stops a test hook starting a real `amd-smi` on a +machine with no GPU, and also what makes the *allowed* branch unreachable under +a simulated root on the GPU-less lane that runs per PR. + +`examine-31` in +[examine.feature](../tests/e2e-cucumber/features/examine.feature) covers the +declining branch end to end. It plants a stub `amd-smi` that would answer the +telemetry probe perfectly and records that it ran, points the seam at a path +inside the scenario's isolated root that is never created, and checks that +`rocmd`'s bridge snapshot still exits 0, reports `amd_smi_available: false`, +carries a note saying the launch was *declined*, and names the device **this +run** pointed the pre-flight at. Both of the last two are what make it +non-vacuous: without the marker the scenario would pass against a gate that had +been deleted and merely failed the spawn, and without the named device it would +pass against a build that ignored the seam, since the GPU-less lane has no +readable `/dev/kfd` of its own either. Linux-only, like its `examine-24` to +`examine-26` telemetry siblings: the stub is a shell script, and the gate is a +wave-through on every other platform. + +```bash +cargo xtask e2e -- -n 'examine-31 ' +``` + +No scenario drives the `rocm serve` / `rocm model` call sites through the gate. +Those probes discard the decline with `.ok()?` and degrade to +service-state-only GPU selection, so there is no user-visible output for a +scenario to assert on; what they do with a missing count is covered by +`gpu_policy.rs`'s unit tests, and the single chokepoint above is what keeps a +new launch site from bypassing the pre-flight in the first place. + ## Windows Runtime Preflight On Windows, `rocm diagnose` compares the Visual C++ runtime in `System32` From 57071f6effdf64f7aab0dbfbde9ef0d5d5d87d4a Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 3 Nov 2026 00:00:15 +0200 Subject: [PATCH 25/32] fix(dash,core): keep the bounded KFD probe off the async worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 3 on #447. - `AmdSmiCollector::new` reached `device_accessible` -> the bounded open from an `async fn`, parking a tokio worker for up to the 2s `KFD_OPEN_TIMEOUT` per probe — ~4s under a simulated host root, where there are two. The pre-flight now runs through `preflight_passes_off_thread`, which short-circuits the two non-device branches before the thread hop and puts the rest on `spawn_blocking`. A `JoinError` reads as "not readable", matching `device_readable_within_budget`: every branch that cannot get an answer declines rather than opening on the caller's thread. - The timeout and budget tests returned `Ok(())` when `mkfifo(1)` was missing, and on WSL, which is indistinguishable from a pass for a test that asserted nothing. Both now announce the skip through a shared `skip_notice`. - `command_failure_detail_or_else` is `pub(crate)`; its only caller outside its module is `amd_smi_json`, in this same crate. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/amd_smi.rs | 28 ++++- crates/rocm-core/src/lib.rs | 7 +- crates/rocm-dash-collectors/src/amd_smi.rs | 134 ++++++++++++++++++++- 3 files changed, 164 insertions(+), 5 deletions(-) diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index c54a52035d..97e325da6d 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -565,6 +565,7 @@ mod tests { // WSL is waved through deliberately, so the blocked case cannot hold // there. if is_wsl_host() { + skip_notice("this is a WSL host, which the KFD gate waves through by design"); return Ok(()); } let _host_root = host_root_guard(); @@ -826,7 +827,9 @@ mod tests { /// /// Shelled out rather than calling `libc::mkfifo`, because the workspace /// lints deny `unsafe_code`. A missing coreutils is not evidence about the - /// probe, so the caller skips rather than fails. + /// probe, so the caller skips rather than fails — but it says so on stderr + /// (see [`skip_notice`]), because a silent skip here is a green result for + /// a test that checked nothing. #[cfg(target_os = "linux")] fn make_fifo(root: &Path, name: &str) -> Option { let fifo = root.join(name); @@ -836,7 +839,25 @@ mod tests { .stdout(Stdio::null()) .stderr(Stdio::null()) .status(); - matches!(&made, Ok(status) if status.success()).then_some(fifo) + if matches!(&made, Ok(status) if status.success()) { + return Some(fifo); + } + skip_notice("no usable mkfifo(1), so no device whose open really blocks can be built here"); + None + } + + /// Announce that a test gave up without asserting its subject. + /// + /// Every conditional skip in this module goes through here. The tests that + /// have one are the ones that need a host feature (`mkfifo(1)`) or a host + /// *shape* (bare-metal Linux rather than WSL) to have anything to assert; + /// where that is missing they return `Ok(())`, which is indistinguishable + /// from a pass unless something says otherwise. `cargo test` shows this on + /// a failing run and under `--nocapture`; matches the `skip:` form already + /// used in `rocm-dash-daemon`'s server and end-to-end tests. + #[cfg(target_os = "linux")] + fn skip_notice(reason: &str) { + eprintln!("skip: {reason}; this assertion did NOT run"); } /// Pins the production wiring the injected helpers cannot: that @@ -954,6 +975,9 @@ mod tests { const PAYLOAD_CHARS: usize = 200_000; if is_wsl_host() { + // The gate waves WSL through, so the fake amd-smi would run and + // this would stop testing the quote bound rather than the gate. + skip_notice("this is a WSL host, where the gate does not reach the launch path"); return Ok(()); } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index f94bee84d6..0bb4fb5661 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -242,7 +242,12 @@ pub fn command_failure_detail(output: &std::process::Output) -> String { /// diagnostic left — and inlining the two `is_empty` checks there to get it put /// the same rule in two places. `fallback` is a closure because formatting an /// `ExitStatus` is wasted work on the overwhelmingly common non-blank path. -pub fn command_failure_detail_or_else( +/// +/// `pub(crate)`, not `pub`: the only caller outside this module is +/// [`amd_smi_json`], also in this crate. External callers that want the +/// precedence rule want [`command_failure_detail`], which is the public pair of +/// this and carries the fallback they would otherwise have to supply. +pub(crate) fn command_failure_detail_or_else( output: &std::process::Output, fallback: impl FnOnce() -> String, ) -> String { diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 5aacb222ce..e3843c75f3 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -94,12 +94,14 @@ impl AmdSmiCollector { gpu_reachable: bool, host_kfd_device: &Path, ) -> Option { - if !preflight_passes( + if !preflight_passes_off_thread( skip_device_preflight, gpu_reachable, Path::new(KFD_DEVICE), host_kfd_device, - ) { + ) + .await + { return None; } let me = Self { @@ -165,6 +167,46 @@ impl AmdSmiCollector { } } +/// [`preflight_passes`] moved off the async worker thread. +/// +/// The device branch of the pre-flight blocks: `device_accessible` waits on a +/// bounded `open`, up to `rocm_core`'s 2s `KFD_OPEN_TIMEOUT` per probe, and +/// under a simulated host root there are two probes, so ~4s. The only caller is +/// an `async fn` reached from a `tokio::spawn` in `rocm-dash-daemon`'s run +/// loop, and AGENTS.md puts blocking work on the blocking pool rather than on +/// an async worker. +/// +/// Deliberately no `timeout` wrapped around the `spawn_blocking`: that would +/// drop only the `JoinHandle` and leave the blocking thread running, and the +/// probe is already bounded from the inside, which is the bound that matters. +/// +/// The two non-device branches short-circuit *before* the thread hop, so a +/// caller that skips the pre-flight or already knows the GPU is reachable pays +/// nothing and -- relevant to the `#[doc(hidden)]` skip entry point -- never +/// needs a runtime with a blocking pool at all. +/// +/// A `JoinError` reads as "not readable", matching +/// `device_readable_within_budget`: every branch that fails to get an answer +/// declines rather than falling back to a blocking open on the caller's thread. +async fn preflight_passes_off_thread( + skip_device_preflight: bool, + gpu_reachable: bool, + real_kfd_device: &Path, + host_kfd_device: &Path, +) -> bool { + if skip_device_preflight || gpu_reachable { + return true; + } + // Owned: `spawn_blocking` needs a `'static` closure. + let real_kfd_device = real_kfd_device.to_path_buf(); + let host_kfd_device = host_kfd_device.to_path_buf(); + tokio::task::spawn_blocking(move || { + preflight_passes(false, false, &real_kfd_device, &host_kfd_device) + }) + .await + .unwrap_or(false) +} + /// The device pre-flight passes if the caller already skipped it, already /// knows the GPU is reachable by some other means (e.g. WSL, via /// `rocm_core::has_usable_amd_gpu()`), or the bare-metal device node itself is @@ -491,6 +533,94 @@ mod tests { ); } + /// A FIFO under `root`, or `None` (with a visible skip notice) on a host + /// with no `mkfifo(1)`. + /// + /// Shelled out rather than calling `libc::mkfifo`, because the workspace + /// lints deny `unsafe_code` — the same reason, and the same shape, as + /// `rocm_core::amd_smi`'s test helper of this name. + #[cfg(target_os = "linux")] + fn make_fifo(root: &Path, name: &str) -> Option { + let fifo = root.join(name); + let made = std::process::Command::new("mkfifo") + .arg(&fifo) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + if matches!(&made, Ok(status) if status.success()) { + return Some(fifo); + } + eprintln!( + "skip: no usable mkfifo(1), so the off-thread pre-flight check cannot be exercised \ + here; this assertion did NOT run" + ); + None + } + + /// The blocking pre-flight must not be run on the async worker. + /// + /// `device_readable_bounded` parks its caller for up to `rocm_core`'s 2s + /// `KFD_OPEN_TIMEOUT`, and the only production caller is an `async fn` + /// reached from `rocm-dash-daemon`'s `tokio::spawn`. A single-worker + /// (`current_thread`) runtime is what makes the difference observable: a + /// ticker task can only make progress while the pre-flight is *not* + /// occupying the one worker, so an inline `preflight_passes` — the shape + /// this had before — records zero ticks, while the `spawn_blocking` version + /// lets the ticker run for the whole ~2s probe. + /// + /// The device is a FIFO with no writer, the same real blocking `open` the + /// `rocm-core` probe tests use: against an ordinary missing path the open + /// fails instantly and the test could pass without proving anything. + /// + /// Linux-only: off Linux `device_readable_bounded` is a direct unbounded + /// open, which on this FIFO would never return at all. + #[tokio::test(flavor = "current_thread")] + #[cfg(target_os = "linux")] + async fn the_device_preflight_does_not_run_on_the_async_worker() { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let dir = tempfile::tempdir().unwrap(); + let Some(fifo) = make_fifo(dir.path(), "blocking-kfd") else { + return; + }; + + let ticks = Arc::new(AtomicUsize::new(0)); + let ticker_ticks = Arc::clone(&ticks); + let ticker = tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_millis(20)).await; + ticker_ticks.fetch_add(1, Ordering::Relaxed); + } + }); + + let passed = preflight_passes_off_thread(false, false, &fifo, &fifo).await; + ticker.abort(); + let observed = ticks.load(Ordering::Relaxed); + + assert!( + !passed, + "a device whose open never returns must fail the pre-flight, or the timing \ + assertion below proves nothing" + ); + assert!( + observed >= 5, + "the async worker was parked by the pre-flight: a 20ms ticker got {observed} ticks \ + during a ~2s blocking probe, so the probe is not on the blocking pool" + ); + } + + /// The two non-device branches answer without a thread hop, so they stay + /// usable on a runtime with no blocking pool — notably the `#[doc(hidden)]` + /// skip entry point the daemon integration tests use. + #[tokio::test(flavor = "current_thread")] + async fn the_short_circuit_branches_need_no_blocking_pool() { + let missing = Path::new("/nonexistent/kfd"); + assert!(preflight_passes_off_thread(true, false, missing, missing).await); + assert!(preflight_passes_off_thread(false, true, missing, missing).await); + } + const SAMPLE_METRIC: &str = r#"{ "gpu_data": [ { From b868d1253d7f91eae67dfec7857b04d1149b7e81 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 3 Nov 2026 12:55:39 +0200 Subject: [PATCH 26/32] docs(core,dash): stop claiming the dash crates have no rocm-core edge This PR added `rocm-dash-collectors -> rocm-core` so the dashboard's `amd-smi` launch could share the bounded `/dev/kfd` probe, which made six doc comments false as written. The invariant that survives binds only the presentation layer: `rocm-dash-tui` has no edge to `rocm-core` and `crate_edges.rs` still forbids it one. The collectors crate does have one, and says what for. `runner.rs` and the collector's own docs additionally explained the threading of `gpu_reachable`/`host_kfd_device` by the missing dependency, which is no longer the reason: the verdict is threaded through because `serve`, `examine` and the dashboard must not reach it independently, not because the collector could not compute it. Signed-off-by: Roman Sirokov --- apps/rocm/src/chat_host_facts.rs | 9 ++++++--- apps/rocm/src/dash.rs | 4 ++-- crates/rocm-core/src/hardware_root.rs | 10 ++++++++-- crates/rocm-core/src/service_record.rs | 5 +++-- crates/rocm-dash-collectors/src/amd_smi.rs | 13 ++++++++----- crates/rocm-dash-daemon/src/runner.rs | 11 +++++++---- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/apps/rocm/src/chat_host_facts.rs b/apps/rocm/src/chat_host_facts.rs index 84dc740a75..d6a2d8b047 100644 --- a/apps/rocm/src/chat_host_facts.rs +++ b/apps/rocm/src/chat_host_facts.rs @@ -17,9 +17,12 @@ //! fast public getters (`runtime_os_name`, `is_wsl_host`, //! `detect_host_gpu_summary`) and never `ExamineSummary::gather()`, whose //! Windows path chains several process-spawning inventory queries. -//! - **Plain data.** The dash crates carry no `rocm-core` dependency; the bin -//! detects and renders, and the rendered `String` rides down through -//! `ResolvedArgs`. +//! - **Plain data.** `rocm-dash-tui`, which owns `ResolvedArgs`, carries no +//! `rocm-core` dependency and is forbidden one by `xtask/src/crate_edges.rs`. +//! So the bin detects and renders, and the rendered `String` rides down +//! through `ResolvedArgs`. (`rocm-dash-collectors` did gain a `rocm-core` +//! edge in EAI-8457, for the bounded `/dev/kfd` probe; nothing on this path +//! runs there.) //! //! Rendering is separated from detection ([`HostFacts::render`] is pure) so the //! wording is unit-testable without a machine. diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 891255c1d7..284ab8baa2 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -184,8 +184,8 @@ fn anthropic_api_key_for_dash() -> Option { } /// Adapt the built-in `rocm-core` model recipes into the TUI-local summaries the -/// serve-wizard picker consumes (the bin owns the `rocm-core` dependency so the -/// dash crates stay free of it). +/// serve-wizard picker consumes (the bin owns the `rocm-core` dependency so +/// `rocm-dash-tui`, which `crate_edges.rs` still forbids one, stays free of it). fn model_recipe_summaries() -> Vec { builtin_model_recipes() .into_iter() diff --git a/crates/rocm-core/src/hardware_root.rs b/crates/rocm-core/src/hardware_root.rs index eeaa696acd..c90014f85a 100644 --- a/crates/rocm-core/src/hardware_root.rs +++ b/crates/rocm-core/src/hardware_root.rs @@ -49,8 +49,14 @@ //! `amd-smi` on a machine without one. They are: //! //! - the `rocm dash` collector's, where `apps/rocm/src/dash.rs` passes the -//! re-rooted path down to the dash daemon — those crates deliberately do not -//! depend on this one, so the caller resolves it; +//! re-rooted path down to the dash daemon. `rocm-dash-collectors` does depend +//! on this crate (EAI-8457) and shares [`crate::device_readable_bounded`], but +//! it is still *handed* the re-rooted path rather than calling [`host_path`] +//! itself: the path has to come from the same resolution `apps/rocm` used for +//! its own probes, and the daemon that carries it in between +//! (`rocm-dash-daemon`) has no dependency on this crate to resolve it with. +//! The presentation layer, `rocm-dash-tui`, has no edge to this crate at all +//! and is the one the `crate_edges.rs` allowlist still forbids; //! - the synchronous one behind `crate::amd_smi_json`, which `rocm serve` and //! `rocmd` reach and which calls [`host_path`] itself. //! diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 37bb07d945..f7994c6c86 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -11,8 +11,9 @@ //! on-disk format at runtime, through its own `ServiceRecord` mirror struct //! in `registry.rs` rather than this type, and the serde test guards that //! contract. `rocm-engine-protocol` mentions `ManagedServiceRecord` solely in -//! a doc comment — `rocm-dash-core` and `rocm-dash-collectors` do not depend -//! on `rocm-core` at all. +//! a doc comment. `rocm-dash-core` does not depend on `rocm-core` at all; +//! `rocm-dash-collectors` does since EAI-8457, but only for the shared bounded +//! `/dev/kfd` probe, and it never reads this on-disk format either. use anyhow::{Context, Result}; use serde::Serialize; diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index e3843c75f3..0f11fe5dd3 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -51,11 +51,14 @@ impl AmdSmiCollector { /// `gpu_reachable` lets the pre-flight pass without a readable `/dev/kfd` /// (e.g. on WSL, where the real signal is `rocm_core::has_usable_amd_gpu()` /// rather than any single device node this crate could probe directly). - /// This crate deliberately does not depend on `rocm-core` to compute that - /// verdict itself; the caller (`apps/rocm`, which already depends on it) - /// threads it through instead, so on WSL, `serve`/`examine`/the dashboard - /// never disagree about whether a host's GPU is usable. On bare metal the - /// caller always passes `false`, so this is still a plain `/dev/kfd` check. + /// This crate does depend on `rocm-core` (EAI-8457, for the shared bounded + /// device probe below), so computing the verdict here would compile — it is + /// still threaded in from the caller (`apps/rocm`) on purpose. The verdict + /// is a *decision about this host* that `serve`, `examine` and the dashboard + /// must not reach independently; recomputing it in the collector is how the + /// three come to disagree about whether a host's GPU is usable, not how they + /// stay in step. On bare metal the caller always passes `false`, so this is + /// still a plain `/dev/kfd` check. /// /// `host_kfd_device` is where the caller's hardware probes put `/dev/kfd` /// (`rocm_core::host_path`'s answer, threaded in for the same reason as diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index aeb554b940..e610903d06 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -88,10 +88,13 @@ pub struct RunnerOptions { /// Precomputed GPU-reachability verdict (from `rocm_core::has_usable_amd_gpu`) /// that lets the amd-smi device pre-flight pass without a readable /// `/dev/kfd` — the WSL case, where that verdict is the same one `serve` - /// and `examine` already act on. `rocm-dash-daemon`/`rocm-dash-collectors` - /// deliberately don't depend on `rocm-core` to compute this themselves; - /// the caller (`apps/rocm`) does and passes the answer through. `false` - /// (the default) preserves the bare-metal-only `/dev/kfd` check. + /// and `examine` already act on. This crate has no `rocm-core` dependency + /// to compute it with; `rocm-dash-collectors`, which does have one since + /// EAI-8457, still takes it threaded through rather than recomputing it, so + /// the dashboard cannot reach a different verdict about the same host than + /// `serve` and `examine` did. The caller (`apps/rocm`) computes it once and + /// passes the answer through both. `false` (the default) preserves the + /// bare-metal-only `/dev/kfd` check. pub amd_smi_gpu_reachable: bool, /// Where the caller's hardware probes put `/dev/kfd`. /// From 612d35e5d2d51de531382310fcc0bb07a048933a Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 3 Nov 2026 12:56:22 +0200 Subject: [PATCH 27/32] test(amd-smi): fail the build when a launch site skips the KFD pre-flight The pre-flight protects nothing unless every `amd-smi` launch goes through it, and nothing in the tree failed when one stopped: reverting either `gpu_policy.rs` probe to `Command::new(resolve_amd_smi_binary())` left the whole suite green, because the regression is the absence of a call and no behavioural test can observe that. So guard the ingredient. The managed SDK ships `amd-smi` inside the runtime wheel's bin directory rather than on `PATH`, so `resolve_amd_smi_binary` is the only way to find a binary to launch; a static contract test now fails any file outside a two-entry, justified allowlist that calls it. Bidirectional like the crate-edge allowlist, so an exemption cannot outlive its reason. Verified by mutation: a raw spawn in `detect_gpu_count` fails `only_preflighted_sites_resolve_the_amd_smi_binary` by name. It is a source-text scan, not a call-graph analysis, and the module doc says so rather than overclaiming. Signed-off-by: Roman Sirokov --- xtask/src/amd_smi_launch_contract.rs | 329 +++++++++++++++++++++++++++ xtask/src/main.rs | 1 + 2 files changed, 330 insertions(+) create mode 100644 xtask/src/amd_smi_launch_contract.rs diff --git a/xtask/src/amd_smi_launch_contract.rs b/xtask/src/amd_smi_launch_contract.rs new file mode 100644 index 0000000000..8f19d52036 --- /dev/null +++ b/xtask/src/amd_smi_launch_contract.rs @@ -0,0 +1,329 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: only a pre-flighted call site may resolve the +//! `amd-smi` binary. +//! +//! Structural guard for EAI-8457. The `/dev/kfd` pre-flight exists because +//! `amd-smi` can park in uninterruptible sleep (D-state) against a device whose +//! `open` never returns, where no signal and therefore no caller timeout +//! reaches it. A gate like that protects nothing unless *every* launch site +//! goes through it, and before this file nothing in the tree failed if one +//! stopped: `apps/rocm/src/gpu_policy.rs`'s two probes could revert to +//! `Command::new(rocm_core::resolve_amd_smi_binary())` and the whole suite +//! would stay green, because the regression is the *absence* of a call, which +//! no behavioural test can observe. +//! +//! So the guard is on the ingredient rather than the behaviour. A launch needs +//! a binary to launch, and [`RESOLVER`] is the only way to find one — the +//! managed ROCm SDK ships `amd-smi` inside the runtime wheel's bin directory +//! rather than on `PATH`, so a hand-rolled `Command::new("amd-smi")` finds +//! nothing on the hosts this product targets. Pinning who may *call* the +//! resolver therefore pins who may launch, and a new launch site has to either +//! route through `rocm_core::amd_smi_json` (which pre-flights) or add itself to +//! [`ALLOWED_CALLERS`] in a reviewed diff that says why it is safe. That review +//! is the entire point, exactly as it is for +//! [`crate::crate_edges::ALLOWLIST`]. +//! +//! What this does **not** claim: it is a source-text scan, not a call-graph +//! analysis. It cannot see a resolver call reached through a `fn` pointer, a +//! re-export under a different name, or a path assembled at runtime, and it +//! says nothing about whether an allowlisted caller then does the right thing +//! with what it gets. It closes the cheap, likely regression — somebody adding +//! a third probe and spawning it directly, the way both existing probes were +//! written before EAI-8457 — and nothing more. +//! +//! Bidirectional, like the crate-edge allowlist: an entry with no matching call +//! fails too. A site that stops launching `amd-smi` must lose its entry, or the +//! exemption outlives the reason for it and the next direct spawn lands in an +//! already-blessed file. + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + /// The call this guard tracks, matched as a call rather than a mention. + /// + /// The trailing `(` is load-bearing in both directions. It keeps prose out + /// — several modules name `resolve_amd_smi_binary` in a doc comment to say + /// where the path comes from, and none of them launches anything — and it + /// keeps the two private helpers out too, since + /// `resolve_amd_smi_binary_in_registry(` and `resolve_amd_smi_binary_in_home(` + /// do not contain this string. Those two are `rocm-core`-private and + /// unreachable from another crate anyway. + const RESOLVER: &str = "resolve_amd_smi_binary("; + + /// Every file that may call [`RESOLVER`], with the reason it is allowed to. + /// + /// Paths are repo-relative and `/`-separated; the comparison normalises + /// Windows separators so the guard gives the same verdict on every lane. + const ALLOWED_CALLERS: &[(&str, &str)] = &[ + ( + "crates/rocm-core/src/amd_smi.rs", + "defines the resolver, and calls it from `amd_smi_json` — the one \ + synchronous launch path, and the only place the `/dev/kfd` pre-flight runs", + ), + ( + "apps/rocm/src/dash.rs", + "resolves the path for the dashboard collector rather than launching anything \ + itself; `rocm-dash-collectors` runs its own pre-flight (`preflight_passes`) on \ + the same device, through the same shared `rocm_core::device_readable_bounded` \ + probe, before it spawns", + ), + ]; + + fn repo_root() -> PathBuf { + // CARGO_MANIFEST_DIR is the xtask/ crate dir; its parent is the repo + // root (same idiom as env_mutation_contract::repo_root). + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + /// Every `.rs` file in the repository, as repo-relative `/`-separated + /// paths. + /// + /// A whole-tree walk rather than a list of source directories, so a crate + /// added under a new top-level directory is covered without anyone + /// remembering to extend this. `target/` (generated and vendored code) and + /// dot-directories (which include the `.claude/worktrees` checkouts of + /// other branches) are skipped, so the verdict never depends on state + /// outside the commit under test — the same two exclusions, for the same + /// two reasons, as `env_mutation_contract`. + fn workspace_rust_sources(root: &Path) -> Vec { + let mut found = Vec::new(); + collect_rust_files(root, root, &mut found); + assert!( + !found.is_empty(), + "found no Rust sources to scan -- the walk is broken, not the tree" + ); + found.sort(); + found + } + + fn collect_rust_files(root: &Path, dir: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let skip = path + .file_name() + .and_then(|name| name.to_str()) + .is_none_or(|name| name == "target" || name.starts_with('.')); + if skip { + continue; + } + collect_rust_files(root, &path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") + && let Ok(relative) = path.strip_prefix(root) + { + found.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } + + /// `line` with any `//` comment tail removed. + /// + /// Without this the guard reports prose. A doc comment is free to write + /// `resolve_amd_smi_binary()` with the parens — the natural way to name a + /// function in English — and that is a description of a call, not one. + /// + /// Deliberately crude: it does not track string literals, so a `//` inside + /// one truncates the line early and could hide a real call after it. That + /// direction fails *open* for a pathological line no file in this tree has, + /// and the alternative (a Rust lexer in a contract test) costs more than + /// the risk. Block comments are not stripped for the same reason; rustfmt + /// renders every doc comment in this tree as `///`. + fn strip_comment(line: &str) -> &str { + match line.find("//") { + Some(at) => &line[..at], + None => line, + } + } + + /// This file, which the scan must skip. + /// + /// Not an exemption like [`ALLOWED_CALLERS`]: this file spawns nothing. Its + /// fixtures quote `resolve_amd_smi_binary(` as *data* — a guard that cannot + /// recognise the thing it guards against is a guard nobody can test — and a + /// text scan has no way to tell a fixture string from a call. Excluding + /// exactly one path, by name, is narrower than teaching [`strip_comment`] + /// about string literals, and leaves no room for a real launch site to hide + /// behind the same trick. + const SELF_PATH: &str = "xtask/src/amd_smi_launch_contract.rs"; + + /// Files that really call [`RESOLVER`], ignoring mentions in comments and + /// this file's own fixtures. + fn files_calling_resolver(root: &Path) -> Vec { + workspace_rust_sources(root) + .into_iter() + .filter(|relative| relative != SELF_PATH) + .filter(|relative| { + std::fs::read_to_string(root.join(relative)).is_ok_and(|text| { + text.lines() + .any(|line| strip_comment(line).contains(RESOLVER)) + }) + }) + .collect() + } + + /// Check a set of calling files against [`ALLOWED_CALLERS`] in both + /// directions, returning every violation found. + fn check(calling: &[String]) -> Vec { + let unexpected = calling + .iter() + .filter(|file| !ALLOWED_CALLERS.iter().any(|(allowed, _)| allowed == file)) + .map(|file| { + format!( + "`{file}` calls `{RESOLVER}..)`, which is how an `amd-smi` launch site is \ + built. Route the launch through `rocm_core::amd_smi_json`, which runs the \ + mandatory `/dev/kfd` pre-flight, or add `{file}` to ALLOWED_CALLERS in \ + xtask/src/amd_smi_launch_contract.rs with the reason the site is safe \ + without it" + ) + }); + let stale = ALLOWED_CALLERS + .iter() + .filter(|(allowed, _)| !calling.iter().any(|file| file == allowed)) + .map(|(allowed, _)| { + format!( + "ALLOWED_CALLERS entry `{allowed}` is stale: it no longer calls \ + `{RESOLVER}..)`. Remove it, or the exemption outlives its reason and the \ + next direct spawn lands in an already-blessed file" + ) + }); + unexpected.chain(stale).collect() + } + + #[test] + fn only_preflighted_sites_resolve_the_amd_smi_binary() { + let root = repo_root(); + let violations = check(&files_calling_resolver(&root)); + assert!( + violations.is_empty(), + "the `/dev/kfd` pre-flight only protects the launch sites that go through it \ + (EAI-8457):\n{}", + violations.join("\n") + ); + } + + #[test] + fn every_allowed_caller_carries_a_reason() { + // An entry with an empty reason is the failure mode this guard exists + // to prevent, written into the guard: a red run "fixed" by pasting a + // path into the list instead of routing the launch through the gate. + for (file, reason) in ALLOWED_CALLERS { + assert!( + reason.len() > 40, + "ALLOWED_CALLERS entry `{file}` has no real justification: {reason:?}" + ); + } + } + + #[test] + fn a_new_direct_resolver_call_fails() { + let mut calling: Vec = ALLOWED_CALLERS + .iter() + .map(|(file, _)| (*file).to_owned()) + .collect(); + calling.push("apps/rocm/src/gpu_policy.rs".to_owned()); + let violations = check(&calling); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("apps/rocm/src/gpu_policy.rs")); + assert!(violations[0].contains("amd_smi_json")); + } + + #[test] + fn an_allowed_caller_that_stops_calling_fails() { + let calling: Vec = ALLOWED_CALLERS + .iter() + .skip(1) + .map(|(file, _)| (*file).to_owned()) + .collect(); + let violations = check(&calling); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("is stale")); + assert!(violations[0].contains(ALLOWED_CALLERS[0].0)); + } + + #[test] + fn a_mention_in_a_comment_is_not_a_call() { + // Both shapes real files in this tree use. The second is why + // `strip_comment` exists at all: parens around the name read as a call + // to a plain `contains`. + assert!( + !strip_comment(" /// (via `rocm_core::resolve_amd_smi_binary`)").contains(RESOLVER) + ); + assert!( + !strip_comment(" // none of the paths resolve_amd_smi_binary() searches") + .contains(RESOLVER) + ); + assert!(strip_comment(" let binary = resolve_amd_smi_binary();").contains(RESOLVER)); + // Code before a trailing comment still counts. + assert!( + strip_comment(" let b = resolve_amd_smi_binary(); // resolved here") + .contains(RESOLVER) + ); + } + + #[test] + fn the_private_resolver_helpers_are_not_matched() { + // `rocm-core`-private, unreachable from another crate, and each is a + // lookup in one location rather than the whole resolution. Matching + // them would make every `rocm-core` test that exercises path + // resolution a launch site. + for helper in [ + " let r = resolve_amd_smi_binary_in_registry(&dir);", + " let r = resolve_amd_smi_binary_in_home(Some(dir));", + " fn resolve_amd_smi_binary_prefers_home_rocm_venv_path() {", + ] { + assert!(!strip_comment(helper).contains(RESOLVER), "{helper}"); + } + } + + #[test] + fn the_self_exclusion_names_a_file_that_exists_and_would_otherwise_be_reported() { + // Two ways `SELF_PATH` could go quietly wrong: a rename leaves it + // pointing at nothing (so the exclusion silently stops applying, and + // this contract starts reporting itself again on the next fixture + // added), or the fixtures are removed and the exclusion is now + // suppressing a file that would pass anyway -- at which point the + // exclusion should go, not linger as a hole. + let root = repo_root(); + let text = std::fs::read_to_string(root.join(SELF_PATH)) + .unwrap_or_else(|e| panic!("SELF_PATH `{SELF_PATH}` does not name a real file: {e}")); + assert!( + text.lines() + .any(|line| strip_comment(line).contains(RESOLVER)), + "`{SELF_PATH}` no longer quotes `{RESOLVER}..)` outside a comment, so the \ + self-exclusion is suppressing nothing and should be deleted" + ); + } + + #[test] + fn the_walk_reaches_every_first_party_crate_directory() { + // Without this the contract could pass by scanning nothing: a walk + // that silently stopped at the repo root would report no violations + // forever. Names a file in each top-level source directory a launch + // site could plausibly be added to. + let files = workspace_rust_sources(&repo_root()); + for expected in [ + "apps/rocm/src/gpu_policy.rs", + "apps/rocmd/src/host_snapshot.rs", + "crates/rocm-core/src/amd_smi.rs", + "crates/rocm-dash-collectors/src/amd_smi.rs", + "crates/rocm-dash-daemon/src/runner.rs", + "xtask/src/crate_edges.rs", + ] { + assert!( + files.iter().any(|file| file == expected), + "the source walk missed `{expected}`, so the contract is scanning less than \ + the tree" + ); + } + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 6c2aa8fd05..87a128efc5 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -11,6 +11,7 @@ //! definition and dispatch. Run via the workspace alias `cargo xtask `. mod affected; +mod amd_smi_launch_contract; mod ascii_output_contract; mod catalog; mod coverage; From a3e6e844f8492b1383e6845db3198b7208a79fe0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 3 Nov 2026 12:57:03 +0200 Subject: [PATCH 28/32] fix(serve): say why the amd-smi probe declined, and pin the quoted latencies Both serve-path probes return `Option`, so a decline had nowhere to go and was dropped with `.ok()?`. That silenced the two cases a user wants: an `amd-smi` that ran and failed, and a false decline when the 2s open bound trips on a slow but healthy host. `amd_smi_probe_json` now logs the full error chain at `debug` -- not `warn`, because on a GPU-less host it is the expected answer on every `serve` -- so "why did `--gpu auto` not rank my GPUs?" is answerable from `RUST_LOG=debug`. The gate's 2s bound and 8-slot budget are restated as literal numbers in five places, two of them as derived figures (caller timeout + 2s), none of which can see the constant. A tripwire test now fails on a change to either and names every restatement to update. Also state plainly, in `kfd_readable`'s docs and `docs/testing.md`, what the gate is not: it proves only that `open(/dev/kfd)` returned, while the hang is attributed to amd-smi's later ioctls, and no unhealthy-driver run backs it up. It is a mitigation heuristic. And the decline is broader than the ticket -- an absent node and EACCES decline too, and `detect_gpu_count` has no sysfs fallback. Signed-off-by: Roman Sirokov --- apps/rocm/src/gpu_policy.rs | 40 +++++++++++++++++++++-- crates/rocm-core/src/amd_smi.rs | 48 +++++++++++++++++++++++++++ docs/architecture.md | 4 +-- docs/testing.md | 58 ++++++++++++++++++++++++++++++--- 4 files changed, 141 insertions(+), 9 deletions(-) diff --git a/apps/rocm/src/gpu_policy.rs b/apps/rocm/src/gpu_policy.rs index a5b44bbdfd..c7530f95e4 100644 --- a/apps/rocm/src/gpu_policy.rs +++ b/apps/rocm/src/gpu_policy.rs @@ -576,11 +576,47 @@ fn simulated_low_vram_usage() -> Option> { /// The `amd-smi` half of [`gpu_vram_usage`], routed through the guarded /// launcher so a missing `/dev/kfd` declines the spawn instead of stalling. fn gpu_vram_usage_amd_smi() -> Option> { - let value = rocm_core::amd_smi_json(&["metric", "--json"], AMD_SMI_SERVE_PROBE_TIMEOUT).ok()?; + let value = amd_smi_probe_json(&["metric", "--json"], "per-GPU VRAM occupancy")?; let rows = parse_gpu_vram_usage(&value); if rows.is_empty() { None } else { Some(rows) } } +/// Run one advisory `serve`-path `amd-smi` probe, logging the error on the way +/// to the `None` both callers want. +/// +/// Both probes degrade rather than fail, so neither has anywhere to *return* an +/// error: the caller's contract is `Option`, and a decline is a normal, expected +/// answer on a GPU-less host. That made the whole class silent, including the +/// two cases a user would want to know about — the pre-flight refusing a device +/// that is merely slow rather than wedged (the 2s `KFD_OPEN_TIMEOUT` is a +/// heuristic bound, and a loaded host can trip it while being perfectly +/// healthy), and an `amd-smi` that ran and failed. "Why did `--gpu auto` not +/// rank my GPUs?" had no answer anywhere. +/// +/// `debug!` and not `warn!`: on a GPU-less host this fires on every `serve`, +/// where it is the expected outcome and not a problem, so at `warn` it would be +/// noise that trains people to ignore the level. `RUST_LOG=debug`, or the CLI's +/// own log file, is where someone looking for this goes. +/// +/// `{error:#}` rather than `to_string()`, matching `rocmd`'s bridge-snapshot +/// note: the outermost context says "failed to launch amd-smi ...", which names +/// the wrong thing when what happened was a pre-flight decline or a timeout. +/// The causes are the whole value of the line. +fn amd_smi_probe_json(args: &[&str], purpose: &str) -> Option { + match rocm_core::amd_smi_json(args, AMD_SMI_SERVE_PROBE_TIMEOUT) { + Ok(value) => Some(value), + Err(error) => { + tracing::debug!( + probe = purpose, + args = args.join(" "), + error = format!("{error:#}"), + "amd-smi probe declined or failed; GPU selection degrades to service state alone" + ); + None + } + } +} + /// Fallback per-GPU VRAM occupancy read straight from the amdgpu DRM sysfs /// counters (`/sys/class/drm/card*/device/mem_info_vram_{total,used}`), used when /// amd-smi is not installed. Linux-only; other platforms expose no such @@ -846,7 +882,7 @@ fn busy_gpu_indices(paths: &AppPaths) -> Vec { /// detection; `rocm_core`'s `kfd_readable` documents why, and what answers in /// this count's place. pub(crate) fn detect_gpu_count() -> Option { - let value = rocm_core::amd_smi_json(&["list", "--json"], AMD_SMI_SERVE_PROBE_TIMEOUT).ok()?; + let value = amd_smi_probe_json(&["list", "--json"], "local AMD GPU count")?; let count = value.as_array().map(Vec::len)?; if count == 0 { None } else { Some(count) } } diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index 97e325da6d..126248c6d2 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -175,6 +175,20 @@ fn preflight_kfd_device() -> PathBuf { /// (see [`KFD_OPEN_TIMEOUT`]). A refusal is recoverable and names what it /// refused; a hang is neither. /// +/// # What this does not prove +/// +/// It is a **mitigation heuristic, not a proof**. All this establishes is that +/// `open("/dev/kfd")` returned inside [`KFD_OPEN_TIMEOUT`]. The hang it exists +/// to prevent is attributed to `amd-smi`'s *later* ioctls against a wedged +/// device, so a device that opens and then wedges on the next ioctl is outside +/// what the gate can see, and the 2s bound is chosen rather than measured. No +/// run against a genuinely unhealthy driver backs any of this up: the lane that +/// runs these tests has no `amd-smi` and no GPU, and the FIFO fixtures model a +/// blocking `open`, the closest reachable analogue rather than the real +/// failure. What the gate buys is that the hazard's common precondition — a +/// device the kernel will not hand out — refuses the spawn, instead of betting +/// on a timeout that cannot reach a process in uninterruptible sleep. +/// /// # What the broad refusal costs /// /// On a host outside the render group `amd-smi` might well have answered — much @@ -550,6 +564,40 @@ mod tests { .unwrap_or_else(std::sync::PoisonError::into_inner) } + /// The pre-flight's bound and budget are quoted as literal numbers in + /// several places that cannot see this constant, and in at least two of + /// them the quoted figure is *derived* — the caller's own timeout plus this + /// one. Changing [`KFD_OPEN_TIMEOUT`] without touching those leaves prose + /// that confidently states the wrong worst case, and nothing else in the + /// tree notices. This test is the tripwire: it fails on the change, and its + /// message is the list of what to update. + /// + /// Not a claim that 2s and 8 are the right values — both are judgement + /// calls, argued where they are declared. It only makes changing them cost + /// a deliberate pass over every restatement. + #[test] + #[cfg(target_os = "linux")] + fn the_probe_bound_and_budget_are_restated_in_these_places() { + assert_eq!( + KFD_OPEN_TIMEOUT, + Duration::from_secs(2), + "the pre-flight's open bound changed. Figures derived from it are restated in: \ + this module's `KFD_OPEN_TIMEOUT` and `amd_smi_json` docs; \ + `apps/rocm/src/gpu_policy.rs::AMD_SMI_SERVE_PROBE_TIMEOUT` (one probe 2+10=12s, \ + two sequential probes ~24s, `--model` readiness ~12s); \ + `apps/rocmd/src/host_snapshot.rs::AMD_SMI_PROBE_TIMEOUT` (2+2=4s); \ + `crates/rocm-dash-collectors/src/amd_smi.rs`'s \ + `the_device_preflight_does_not_run_on_the_async_worker` doc; and \ + the `amd-smi KFD pre-flight` section of `docs/testing.md`. \ + Update every one of them, then update this assertion." + ); + assert_eq!( + MAX_INFLIGHT_KFD_PROBES, 8, + "the in-flight probe budget changed; it is restated in this module's own docs and \ + in the `amd-smi KFD pre-flight` section of `docs/testing.md`" + ); + } + /// The gate has to stop the *spawn*, not merely report a failure after it: /// the launch that already happened is the launch that hangs. The fake /// `amd-smi` leaves a marker file, so its absence is direct evidence diff --git a/docs/architecture.md b/docs/architecture.md index fbc899ec00..0c33565b47 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -46,7 +46,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation (Phase 6f, ROCMAI-208, folding Phase 6f/ROCMAI-86 closed): the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`), the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`), and the model-catalog rendering behind `rocm model` (alias `models`) and `rocm model --verbose` (`model_catalog_platform_heading`, `render_model_registry_text_with_context_and_host`, `model_recipe_memory_label`, `model_recipe_gpu_fit_label`, `render_model_registry_verbose_text_with_context_and_host`, `model_catalog_header`, `append_model_recipe_registry_source`, `append_model_recipe_metadata_lines`, `append_model_engine_recipe_settings_lines`, `format_list_or_none`, `format_string_map_or_none`, `append_model_artifact_lines`, `append_artifact_source_policy_lines`, `artifact_source_policy_label`, `format_bytes`, `append_model_host_ram_fit_lines`, `append_model_fit_lines`, `append_manual_alternative_lines`, `manual_alternative_recommendations`, `recipe_is_manual_fit`, `append_model_engine_support_lines`, `recipe_display_ref`, `preferred_engine_action_target`, `model_registry_reason`, `format_gib`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`, `engine_runtime_is_native_windows_unsupported`). Two items physically interleaved in this span in `main.rs` moved with it rather than staying behind, since neither has a caller outside this cluster: the `DEFAULT_ENGINE_MARKER` constant (shared by both engine-inventory renderers so the two copies cannot drift apart) and `model_registry_adapter_availability_note` (read only from `append_model_engine_support_lines` and this module's own test); both stay private. `render_config_text` and `telemetry_policy_summary` sit immediately adjacent to this cluster's end boundary in `main.rs` but are explicitly out of scope — a gap already flagged on this epic and on ROCMAI-179 ("Phase 8: validate module-split sufficiency") — and stay untouched. `dispatch()` calls directly into this module at two points — `render_model_registry_verbose_text_with_context_and_host` and `render_model_registry_text_with_context_and_host` — and `main.rs`'s `examine()`, which the `Command::Examine` arm calls, calls `examine_human_report`, all re-imported via `use crate::examine_report_text::{...};`; `main.rs`'s own `launch_default` calls `render_launch_summary` directly, also re-imported there. `render_examine_text`, `render_examine_text_with_paths`, and `render_model_registry_verbose_text_with_context_and_host` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `render_engine_inventory_text_with_paths` is also read from `chat.rs` and from `engines_cmd.rs`'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build; `format_bytes` is also `pub(crate)`, read from [`storage.rs`](../apps/rocm/src/storage.rs)'s cache-size reporting and [`automations.rs`](../apps/rocm/src/automations.rs)'s plain-language summaries. Every other item in this list has no caller outside this module and stays private. This cluster also reaches three items that stayed behind in `main.rs` because they already had callers outside it: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)), `engine_ruled_out_by_platform` (also read from `main.rs`'s `assess_model_for_host`), and `missing_packaged_engine_reason` (already noted in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s entry above as having callers outside that cluster, plus its own test — this cluster is one of them) — all three reached via `use crate::{...};`. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`, the same way other mechanically relocated modules do. Four tests interleaved among this cluster's tests in `main.rs`'s old shared `mod tests` stayed behind because their primary subject is unrelated: `missing_packaged_engine_reason`'s own test, two CLI-lifecycle-audit-log tests, and the whole-command-tree ASCII-help walk. - [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) — mechanical relocation, dispatch-adjacent, owns a `pub(crate)` type (as of Phase 6e's follow-up, ROCMAI-726; see the Phase 6a variant note above for the owned-types pattern). Holds the freeform natural-language planner's plan-*execution*/consent half that [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) below deliberately left out: `validate_freeform_execution_action()`, `prepare_freeform_execution()`, `apply_freeform_execution_consent()`, `render_freeform_execution_header()`, and the `pub(crate)` `FreeformExecution` struct that `prepare_freeform_execution` returns and `render_freeform_execution_header` consumes. `main.rs`'s `execute_freeform_next_action()` and `run_freeform()` stay in `main.rs` — both are dispatch-level, not part of deciding whether a plan's next action is safe to execute — and `execute_freeform_next_action()` calls in via the unqualified re-import `use crate::freeform_execution::{prepare_freeform_execution, render_freeform_execution_header};` (`run_freeform()` only calls `execute_freeform_next_action()`, never this module directly), matching [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s dispatch-adjacent form rather than `freeform_planner.rs`'s own qualified-path form. `prepare_freeform_execution` takes the already-built `plan` by value rather than rebuilding it from `request`/`paths`/`config` (ROCMAI-725's single-build guarantee, inherited unchanged from `main.rs`); `prepare_freeform_execution`/`render_freeform_execution_header` are `pub(crate)` for that cross-module call; `FreeformExecution` was already `pub(crate)` (read directly by `main.rs` and this module's own tests); `validate_freeform_execution_action`/`apply_freeform_execution_consent` have no caller outside this module and stay module-private, unchanged from before the move. The cluster's six regression tests moved out of `main.rs`'s shared `mod tests` into this module's own `mod tests`, inheriting the qualified `freeform_planner::` path they already used to reach `freeform_planner.rs`'s items as `main.rs`'s former tests — this module reaches in from outside using the same qualified path `main.rs` and `providers.rs`'s tests use (unlike `chat.rs`/`comfyui.rs`, which import it unqualified). - [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction`, `StructuredRequestPlan` and `PlannedToolCall`, which are `pub(crate)`), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow ([`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `render_launch_summary` → `host_default_engine` is another call into this module, unrelated to planning: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, and its `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` directly as a no-CPU-fallback safety check on provider-proposed `serve` args. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context`, `provider_planner_response_to_plan`, `format_structured_tool_call`, and `serve_args_request_cpu_device` are `pub(crate)` for these reasons, as are `infer_engine_from_request` (read from `chat.rs`) and `plan_driver_request` (read from [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s test module); `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller. -- [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`). Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`main.rs`](../apps/rocm/src/main.rs) (`classify_accelerator_memory`, `host_accelerator_memory`, the `dispatch()` `Command::EngineServeHttp` arm, and `collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`services_text.rs`](../apps/rocm/src/services_text.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `main.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`, which stayed) still needs them too. +- [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`). Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`main.rs`](../apps/rocm/src/main.rs) (`classify_accelerator_memory`, `host_accelerator_memory`, the `dispatch()` `Command::EngineServeHttp` arm, and `collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`services_text.rs`](../apps/rocm/src/services_text.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, `amd_smi_probe_json` (the shared wrapper both `amd-smi` probes call, which logs the decline the `Option` contract would otherwise swallow), and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `main.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`, which stayed) still needs them too. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. - [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`main.rs`](../apps/rocm/src/main.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning and the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) — full domain extraction. @@ -108,7 +108,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): -- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count` and `rocmd`'s `common.rs::capture_amd_smi_json`. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. +- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `debug`), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151, Phase 7d) the built-in automation-watcher catalog, automation-daemon runtime state, and its on-disk event/proposal/audit records — see the module's own doc comment for what it owns. [`config.rs`](../crates/rocm-core/src/config.rs) and this module depend on each other — `config.rs` imports `BuiltinWatcherSpec` from here, and this module imports `WatcherMode` from `config.rs` — a back-edge the split introduces without eliminating; `WatcherMode` stays with the persisted config because it is part of `RocmCliConfig`'s own on-disk shape, not an automation-daemon concern. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. diff --git a/docs/testing.md b/docs/testing.md index 6b89087ea0..259b03eef4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2372,11 +2372,59 @@ cargo xtask e2e -- -n 'examine-31 ' ``` No scenario drives the `rocm serve` / `rocm model` call sites through the gate. -Those probes discard the decline with `.ok()?` and degrade to -service-state-only GPU selection, so there is no user-visible output for a -scenario to assert on; what they do with a missing count is covered by -`gpu_policy.rs`'s unit tests, and the single chokepoint above is what keeps a -new launch site from bypassing the pre-flight in the first place. +Those probes degrade to service-state-only GPU selection rather than failing, +so there is no user-visible *output* for a scenario to assert on; what they do +with a missing count is covered by `gpu_policy.rs`'s unit tests. The decline +itself is no longer discarded — `gpu_policy.rs`'s `amd_smi_probe_json` logs it +at `debug` with the full error chain, which is what makes "why did `--gpu auto` +not rank my GPUs?" answerable from `RUST_LOG=debug` or the CLI log file. It is +`debug` and not `warn` because on a GPU-less host it fires on every `serve`, +where it is the expected answer and not a fault. + +### Keeping new launch sites inside the gate + +A chokepoint only holds while every launch site uses it, and nothing in a +behavioural test suite fails when one stops: the regression is the *absence* of +a call. [amd_smi_launch_contract.rs](../xtask/src/amd_smi_launch_contract.rs) is +the structural guard for that. It scans every `.rs` file in the tree (comments +stripped) for calls to `rocm_core::resolve_amd_smi_binary` — the only way to +find the binary, since the managed SDK ships `amd-smi` inside the runtime +wheel's bin directory rather than on `PATH` — and fails unless the calling file +is one of two allowlisted, justified sites. Reverting either `gpu_policy.rs` +probe to a raw `Command::new(resolve_amd_smi_binary())` now fails this test by +name. It is bidirectional like the crate-edge allowlist: an entry whose file +stops calling the resolver is stale and fails too. + +```bash +cargo test -p xtask amd_smi_launch_contract +``` + +It is a source-text scan, not a call-graph analysis: a resolver reached through +a function pointer or a re-export under another name is outside what it can +see. It closes the cheap, likely regression and claims nothing more. + +### What the gate is, and is not + +**It is a mitigation heuristic, not a proof.** The gate establishes exactly one +thing: that `open("/dev/kfd")` returned within 2s. The D-state hang it exists to +prevent is attributed to `amd-smi`'s *later* `ioctl`s against a wedged device, +not to the open — so a device whose open succeeds and whose next ioctl wedges is +not covered, and the 2s bound is a chosen number rather than a measured one. No +run against a genuinely unhealthy driver was performed for this work: the GPU +lane has no `amd-smi` at all, and the FIFO fixtures model a blocking `open`, +which is the closest reachable analogue rather than the real failure. What the +gate does buy is that the common precondition for the hazard — a device the +kernel will not hand out — refuses the spawn instead of betting on a timeout +that cannot reach a process in uninterruptible sleep. + +**The decline is broader than the hazard, and broader than the ticket.** Any +unreachable `/dev/kfd` declines, including the fast errno cases that cannot hang +anything: an absent node (EAI-8457 states the scenario does not apply with no +`/dev/kfd`) and `EACCES` for a user outside the `render`/`video` groups. +`gpu_vram_usage` still falls back to the DRM sysfs counters on such a host, but +`detect_gpu_count` has **no** sysfs fallback and returns `None`, so `--gpu auto` +there ranks from the sysfs VRAM rows alone. `kfd_readable`'s doc comment carries +the same statement next to the code. ## Windows Runtime Preflight From bc8134112709539eb15cc23664d6719d2e7f40f1 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 4 Nov 2026 18:07:04 +0200 Subject: [PATCH 29/32] fix(dash): cfg-split device_accessible so the Windows lane builds `device_readable_bounded` waves through with a literal off Linux, which makes this wrapper `const`-able there, and `clippy::missing_const_for_fn` is a hard error on the Windows lane's `-D warnings` build. Split into a Linux body that opens the device and a non-Linux `const fn`, the same shape `kfd_readable` and `device_readable_bounded` already use. Caught by CI's Windows job, not locally: the lint only fires on the branch the Mac and the Linux box never compile. Verified this time by running clippy over every crate that does build on Darwin. Signed-off-by: Roman Sirokov --- crates/rocm-dash-collectors/src/amd_smi.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 0f11fe5dd3..a3251a862c 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -249,10 +249,24 @@ fn preflight_passes( /// as the one `amd-smi` launch site that can still block forever on a wedged /// device. The shared probe carries the 2s bound and the in-flight budget with /// it, so this site cannot drift away from the gate it is part of. +/// +/// `cfg`-split for the same reason `rocm_core::device_readable_bounded` is: +/// off Linux that probe waves through with a literal, so this wrapper becomes +/// `const`-able and `clippy::missing_const_for_fn` is a hard error on the +/// Windows lane's `-D warnings` build, while the Linux body opens a device and +/// cannot be `const`. A `cfg`-selected pair is linted per branch, not per name. +#[cfg(target_os = "linux")] fn device_accessible(path: &Path) -> bool { rocm_core::device_readable_bounded(path) } +/// Off-Linux counterpart of [`device_accessible`]: see that function for why +/// the pair is split rather than written once. +#[cfg(not(target_os = "linux"))] +const fn device_accessible(_path: &Path) -> bool { + true +} + fn val_u64(v: Option<&Value>) -> Option { v?.as_u64() .or_else(|| v?.as_f64().map(|x| x.round() as u64)) From 980d0a2ca685e04a763b35893c87aa2b42d3160c Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 4 Nov 2026 20:16:39 +0200 Subject: [PATCH 30/32] fix(amd-smi): keep the WSL short-circuit, and stop the docs saying debug Review round on #447. - Extracting the WSL condition into `wsl_wave_through` turned a short-circuit into two eager argument evaluations, so `has_usable_amd_gpu` ran on every Linux launch instead of on WSL only. The probe is passed unevaluated now, which restores the original behaviour and keeps the condition testable. The comment that still claimed `&&` short-circuited was describing code that no longer did. - `docs/architecture.md` and `docs/testing.md` still said the discarded decline is logged at `debug`; it is `info!`, deliberately, because the default filter is `warn,rocm=info`. Both now say so and why. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/amd_smi.rs | 28 +++++++++++++++++++--------- docs/architecture.md | 2 +- docs/testing.md | 11 +++++++---- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index 6f0c3ed605..3c6802fa84 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -233,8 +233,12 @@ fn preflight_kfd_device() -> PathBuf { #[cfg(target_os = "linux")] fn kfd_readable(device: &Path) -> bool { kfd_readable_with( - // `&&` short-circuits: `has_usable_amd_gpu` runs on WSL only. - wsl_wave_through(is_wsl_host(), has_usable_amd_gpu()), + // Second argument is a closure, not a value: extracting this condition + // into a named function turned a short-circuit into two eager + // evaluations, so `has_usable_amd_gpu` was running on every Linux + // launch rather than on WSL only. Passing the probe unevaluated keeps + // the original behaviour and still lets the condition be tested. + wsl_wave_through(is_wsl_host(), has_usable_amd_gpu), device, &host_path(device), ) @@ -246,14 +250,17 @@ fn kfd_readable(device: &Path) -> bool { /// [`kfd_readable_with`] with literal booleans, which says nothing about how /// the caller computes that argument — `&&` could become `||`, or /// `has_usable_amd_gpu()` could be dropped entirely, and every one of them -/// would stay green on a GPU-less lane. Both conjuncts are load-bearing in +/// would stay green on a GPU-less lane. The GPU probe is taken unevaluated so +/// the short-circuit survives the extraction: on bare metal `is_wsl` is false +/// and `has_usable_amd_gpu` must not run at all. Both conjuncts are +/// load-bearing in /// opposite directions: without `is_wsl_host` a bare-metal host with a GPU /// would skip the device check the gate exists for, and without /// `has_usable_amd_gpu` an unready WSL host would be waved through to an /// `amd-smi` that cannot work. #[cfg(target_os = "linux")] -const fn wsl_wave_through(is_wsl: bool, has_usable_gpu: bool) -> bool { - is_wsl && has_usable_gpu +fn wsl_wave_through(is_wsl: bool, has_usable_gpu: impl FnOnce() -> bool) -> bool { + is_wsl && has_usable_gpu() } /// [`kfd_readable`] with its two non-syscall inputs injected, so the @@ -733,18 +740,21 @@ mod tests { #[test] #[cfg(target_os = "linux")] fn the_wsl_wave_through_needs_both_a_wsl_host_and_a_reachable_gpu() { - assert!(wsl_wave_through(true, true), "ready WSL host waves through"); assert!( - !wsl_wave_through(true, false), + wsl_wave_through(true, || true), + "ready WSL host waves through" + ); + assert!( + !wsl_wave_through(true, || false), "a WSL host with no reachable GPU must not wave through: amd-smi \ cannot work there, and `||` would allow it" ); assert!( - !wsl_wave_through(false, true), + !wsl_wave_through(false, || true), "bare metal must reach the device check even with a usable GPU, or \ the gate this whole module exists for is skipped" ); - assert!(!wsl_wave_through(false, false)); + assert!(!wsl_wave_through(false, || false)); } /// The simulated-root routing, which `kfd_readable` applies to the real diff --git a/docs/architecture.md b/docs/architecture.md index 48cca1df6d..6aec2b34d7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -136,7 +136,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): -- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `debug`), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. +- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `info`, since the default filter is `warn,rocm=info` and a decline on a healthy host is otherwise indistinguishable from "no GPU"), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151, Phase 7d) the built-in automation-watcher catalog, automation-daemon runtime state, and its on-disk event/proposal/audit records — see the module's own doc comment for what it owns. [`config.rs`](../crates/rocm-core/src/config.rs) and this module depend on each other — `config.rs` imports `BuiltinWatcherSpec` from here, and this module imports `WatcherMode` from `config.rs` — a back-edge the split introduces without eliminating; `WatcherMode` stays with the persisted config because it is part of `RocmCliConfig`'s own on-disk shape, not an automation-daemon concern. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. diff --git a/docs/testing.md b/docs/testing.md index 886b907990..e714f1db4c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2723,10 +2723,13 @@ Those probes degrade to service-state-only GPU selection rather than failing, so there is no user-visible *output* for a scenario to assert on; what they do with a missing count is covered by `gpu_policy.rs`'s unit tests. The decline itself is no longer discarded — `gpu_policy.rs`'s `amd_smi_probe_json` logs it -at `debug` with the full error chain, which is what makes "why did `--gpu auto` -not rank my GPUs?" answerable from `RUST_LOG=debug` or the CLI log file. It is -`debug` and not `warn` because on a GPU-less host it fires on every `serve`, -where it is the expected answer and not a fault. +at `info` with the full error chain, which is what makes "why did `--gpu auto` +not rank my GPUs?" answerable without re-running. It is not `warn`, because on +a GPU-less host it fires on every `serve`, where it is the expected answer and +not a fault; and not `debug`, because the default filter is `warn,rocm=info`, +so a `debug` line is invisible unless the user already suspected this — the +wrong way round for the case that needs it, where the 2s open bound trips on a +slow but healthy host and the decline is indistinguishable from "no GPU". ### Keeping new launch sites inside the gate From 1e091a3fb266f53dd0e39b54e33036479677f0a1 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 5 Nov 2026 11:11:10 +0200 Subject: [PATCH 31/32] fix(amd-smi): stop device_readable_bounded waving through off Linux The previous round made the off-Linux branch return `true`, on the reasoning that it should mirror `kfd_readable`'s wave-through as its doc claimed. That was the wrong half to fix. The two answer different questions: `kfd_readable` gates this crate's own `amd-smi` launch, which off Linux has no `/dev/kfd` to reason about; `device_readable_bounded` is the shared probe `rocm-dash-collectors` calls for a pre-flight that is expected to decline a missing device on every platform. Windows CI caught it: `missing_kfd_and_no_reachability_fails_preflight` and `real_kfd_keeps_a_veto_over_the_host_root_view` both went green while losing the veto they exist to protect. The off-Linux branch is a direct open again and the doc now says why it does not wave through. That also removes the `const`-able body, so the `missing_const_for_fn` split added for it is gone from both crates. Both tests verified passing on Darwin, which compiles the same non-Linux branch Windows does. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/amd_smi.rs | 27 ++++++++++++++-------- crates/rocm-dash-collectors/src/amd_smi.rs | 14 ----------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index 3c6802fa84..8d9504ad1d 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -456,18 +456,25 @@ pub fn device_readable_bounded(device: &Path) -> bool { device_readable(device) } -/// Off-Linux counterpart of [`device_readable_bounded`]: waves through, like -/// [`kfd_readable`] does, because there is no `/dev/kfd` on these platforms to -/// decline about. +/// Off-Linux counterpart of [`device_readable_bounded`]: a direct open, with no +/// bound and no wave-through. /// -/// `const` because it has to be, and split for the same reason `kfd_readable` -/// is: the Windows lane builds with `-D warnings`, so -/// `clippy::missing_const_for_fn` makes a non-`const` body that only returns a -/// literal a hard error there, while the Linux body opens a device and cannot -/// be `const`. A `cfg`-selected pair is linted per branch, not per name. +/// It does **not** mirror [`kfd_readable`]'s off-Linux wave-through, and an +/// earlier revision that made it do so was wrong. The two answer different +/// questions. `kfd_readable` gates *this crate's* `amd-smi` launch, which off +/// Linux has no `/dev/kfd` to reason about, so waving through is right there. +/// This function is the shared probe `rocm-dash-collectors` calls for its own +/// pre-flight, and that pre-flight is expected to decline a missing device on +/// every platform — `missing_kfd_and_no_reachability_fails_preflight` and +/// `real_kfd_keeps_a_veto_over_the_host_root_view` assert exactly that, and a +/// wave-through turns both green while removing the veto they exist to protect. +/// +/// The bound is Linux-only because the hazard is: `/dev/kfd` is what can park a +/// process in uninterruptible sleep. An ordinary file open elsewhere returns or +/// errors. #[cfg(not(target_os = "linux"))] -pub const fn device_readable_bounded(_device: &Path) -> bool { - true +pub fn device_readable_bounded(device: &Path) -> bool { + std::fs::OpenOptions::new().read(true).open(device).is_ok() } /// Off-Linux counterpart of [`kfd_readable`]: see that function's docs for why diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index b586f51bcb..ac5446768a 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -268,24 +268,10 @@ fn preflight_passes( /// as the one `amd-smi` launch site that can still block forever on a wedged /// device. The shared probe carries the 2s bound and the in-flight budget with /// it, so this site cannot drift away from the gate it is part of. -/// -/// `cfg`-split for the same reason `rocm_core::device_readable_bounded` is: -/// off Linux that probe waves through with a literal, so this wrapper becomes -/// `const`-able and `clippy::missing_const_for_fn` is a hard error on the -/// Windows lane's `-D warnings` build, while the Linux body opens a device and -/// cannot be `const`. A `cfg`-selected pair is linted per branch, not per name. -#[cfg(target_os = "linux")] fn device_accessible(path: &Path) -> bool { rocm_core::device_readable_bounded(path) } -/// Off-Linux counterpart of [`device_accessible`]: see that function for why -/// the pair is split rather than written once. -#[cfg(not(target_os = "linux"))] -const fn device_accessible(_path: &Path) -> bool { - true -} - fn val_u64(v: Option<&Value>) -> Option { v?.as_u64() .or_else(|| v?.as_f64().map(|x| x.round() as u64)) From 2d2b8f0dae1bc36967c82f592c8145779b5bd899 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 6 Nov 2026 10:22:24 +0200 Subject: [PATCH 32/32] docs(amd-smi): put the probe sentence in the right bullet, and stop two claims overreaching Review round on #447. - The `AMD_SMI_SERVE_PROBE_TIMEOUT` sentence had drifted into `architecture.md`'s `freeform_planner.rs` bullet across several merge resolutions. Both probes live in `gpu_policy.rs`; it sits there now, next to `detect_gpu_count`. - `amd_smi.rs` said that losing the amd-smi count leaves `--gpu auto` with no selection. It does not: `effective_gpu_count` falls through to the DRM sysfs VRAM rows, which is what `docs/testing.md` already said. The code was right and this doc was wrong, so the doc now matches and names the function. - `docs/testing.md` claimed unit tests cover the rewritten call sites. They cover what those sites do with a *missing count* (`effective_gpu_count_prefers_amd_smi_then_falls_back_to_vram_rows`); that they route through the chokepoint at all is guarded only by the xtask contract scan. Both halves are now stated separately. - examine-31 was `@requires-os:linux`, but WSL2 reports that os_family and `kfd_readable` waves through there on a reachable GPU, so the decline it asserts never happens. Now `@requires-bare-metal` too. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/amd_smi.rs | 7 ++++--- docs/architecture.md | 4 ++-- docs/testing.md | 13 +++++++++++-- tests/e2e-cucumber/features/examine.feature | 7 ++++++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index 8d9504ad1d..9947c8d331 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -200,9 +200,10 @@ fn preflight_kfd_device() -> PathBuf { /// `/sys/class/kfd` and AMD DRM cards under `/sys/class/drm` — both /// world-readable, neither needing the device node open — and /// `validate_pinned_gpu_index` prefers that set outright. What is lost is the -/// secondary count `--gpu auto` ranks with, so on a *multi*-GPU host in that -/// state auto-selection returns no selection and the engine's own device probe -/// picks. `rocm diagnose` already names the host, as `fix-4-render-group`. +/// `amd-smi` device count `--gpu auto` would rank with; `effective_gpu_count` +/// (`apps/rocm/src/gpu_policy.rs`) falls through to the DRM sysfs VRAM rows, so +/// auto-selection still ranks, over those rows alone rather than over nothing. +/// `rocm diagnose` already names the host, as `fix-4-render-group`. /// /// # Platforms with no KFD /// diff --git a/docs/architecture.md b/docs/architecture.md index 26168f8d1d..350d2e42e7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -56,8 +56,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it outside its own tests. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. One test here, `linux_only_engine_runtime_status_is_explicit_on_native_windows`, also calls `model_registry_adapter_availability_note` from `model_catalog_text.rs`, which is `pub(crate)` for that reason; its primary subject is `engine_runtime_status_label`, so it stays. - [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) — mechanical relocation, no owned types: the answers a freeform request can get without running a command — `render_freeform_read_only_answer`, which answers a single approval-free `examine`, `comfyui status` or `comfyui logs` action, and `render_freeform_examine_answer`, the plain-language ROCm status it prints for `examine`. `run_freeform` in [`main.rs`](../apps/rocm/src/main.rs) is the only caller of the first, which is `pub(crate)` and re-imported there; the second is private. Besides `chat.rs` (`any_substring`, re-exported there from [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs)), it depends on `comfyui.rs`, `runtimes_cmd.rs` and `therock.rs`, none of which calls back into it. No unit test calls either function directly. - [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) — mechanical relocation, dispatch-adjacent, owns a `pub(crate)` type (as of Phase 6e's follow-up, ROCMAI-726; see the Phase 6a variant note above for the owned-types pattern). Holds the freeform natural-language planner's plan-*execution*/consent half that [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) below deliberately left out: `validate_freeform_execution_action()`, `prepare_freeform_execution()`, `apply_freeform_execution_consent()`, `render_freeform_execution_header()`, and the `pub(crate)` `FreeformExecution` struct that `prepare_freeform_execution` returns and `render_freeform_execution_header` consumes. `main.rs`'s `execute_freeform_next_action()` and `run_freeform()` stay in `main.rs` — both are dispatch-level, not part of deciding whether a plan's next action is safe to execute — and `execute_freeform_next_action()` calls in via the unqualified re-import `use crate::freeform_execution::{prepare_freeform_execution, render_freeform_execution_header};` (`run_freeform()` only calls `execute_freeform_next_action()`, never this module directly), matching [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s dispatch-adjacent form rather than `freeform_planner.rs`'s own qualified-path form. `prepare_freeform_execution` takes the already-built `plan` by value rather than rebuilding it from `request`/`paths`/`config` (ROCMAI-725's single-build guarantee, inherited unchanged from `main.rs`); `prepare_freeform_execution`/`render_freeform_execution_header` are `pub(crate)` for that cross-module call; `FreeformExecution` was already `pub(crate)` (read directly by `main.rs` and this module's own tests); `validate_freeform_execution_action`/`apply_freeform_execution_consent` have no caller outside this module and stay module-private, unchanged from before the move. The cluster's six regression tests moved out of `main.rs`'s shared `mod tests` into this module's own `mod tests`, inheriting the qualified `freeform_planner::` path they already used to reach `freeform_planner.rs`'s items as `main.rs`'s former tests — this module reaches in from outside using the same qualified path `main.rs` and `providers.rs`'s tests use (unlike `chat.rs`/`comfyui.rs`, which import it unqualified). -- [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction`, `StructuredRequestPlan` and `PlannedToolCall`, which are `pub(crate)`, and `ProviderPlannerToolCall`, `pub(crate)` together with `validate_provider_planner_tool_call` and `planner_failure_note` so [`chat.rs`](../apps/rocm/src/chat.rs)'s command-preview property tests can drive the planner's guards), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow ([`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `render_launch_summary` → `host_default_engine` is another call into this module, unrelated to planning: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, while [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs)'s `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` from [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) directly as a no-CPU-fallback safety check on provider-proposed `serve` args; `validate_provider_planner_tool_call` calls that same function in its own `serve` branch, and hands an `install sdk` call to `tool_policy::validate_chat_rocm_command_safety`. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context` and `provider_planner_response_to_plan` are `pub(crate)` for these reasons, as is `plan_driver_request` (read from [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs)'s test module); `format_structured_tool_call` lives in [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) and is re-exported here with `pub(crate) use`, so the callers above keep their `freeform_planner::` path, while the planner's own argv lookups call `rocm_argv`, its phrase and engine readers call [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) and its safety check calls `tool_policy.rs`, directly: the planner no longer imports `chat`, and only `chat` imports the planner (`plan_freeform_request_with_context`); `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller. -- [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`). `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. +- [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction`, `StructuredRequestPlan` and `PlannedToolCall`, which are `pub(crate)`, and `ProviderPlannerToolCall`, `pub(crate)` together with `validate_provider_planner_tool_call` and `planner_failure_note` so [`chat.rs`](../apps/rocm/src/chat.rs)'s command-preview property tests can drive the planner's guards), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow ([`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `render_launch_summary` → `host_default_engine` is another call into this module, unrelated to planning: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, while [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs)'s `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` from [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) directly as a no-CPU-fallback safety check on provider-proposed `serve` args; `validate_provider_planner_tool_call` calls that same function in its own `serve` branch, and hands an `install sdk` call to `tool_policy::validate_chat_rocm_command_safety`. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context` and `provider_planner_response_to_plan` are `pub(crate)` for these reasons, as is `plan_driver_request` (read from [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs)'s test module); `format_structured_tool_call` lives in [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) and is re-exported here with `pub(crate) use`, so the callers above keep their `freeform_planner::` path, while the planner's own argv lookups call `rocm_argv`, its phrase and engine readers call [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) and its safety check calls `tool_policy.rs`, directly: the planner no longer imports `chat`, and only `chat` imports the planner (`plan_freeform_request_with_context`); `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller. +- [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`).Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. - [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm install`'s command handler (`install()`, for the `sdk` and `driver` targets), the `--version`/`--build-date` selector parsing and display it uses (`therock_install_version_selector`, `therock_install_version_selector_display`) and `sdk_install_request`, which maps the parsed `install sdk` arguments onto a `therock::SdkInstallRequest` and moved here from [`chat.rs`](../apps/rocm/src/chat.rs) with its two tests. `InstallTarget` and `InstallFormat` live in [`cli.rs`](../apps/rocm/src/cli.rs). `install()` is `pub(crate)` and re-imported in `main.rs` for `dispatch()`; `therock_install_version_selector` is `pub(crate)` too, and `chat.rs`'s `render_install_sdk_dry_run_for_args` reaches it as `crate::therock_install_version_selector` through the same kind of re-import, which is why `sdk_install_request` lives here and not in `chat.rs`: this module does not depend on `chat.rs`. `sdk_install_request` keeps the `pub(crate)` it had, although only `install()` and the tests call it now. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. - [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. diff --git a/docs/testing.md b/docs/testing.md index cfeddd02df..b3242c38b4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2736,8 +2736,17 @@ cargo xtask e2e -- -n 'examine-31 ' No scenario drives the `rocm serve` / `rocm model` call sites through the gate. Those probes degrade to service-state-only GPU selection rather than failing, -so there is no user-visible *output* for a scenario to assert on; what they do -with a missing count is covered by `gpu_policy.rs`'s unit tests. The decline +so there is no user-visible *output* for a scenario to assert on. + +Be precise about what does cover them, because the two halves have different +guards. What the call sites do **with a missing count** is covered by +`gpu_policy.rs`'s unit tests — `effective_gpu_count_prefers_amd_smi_then_falls_back_to_vram_rows` +pins the `None` fall-through. That the call sites **route through the +chokepoint at all** is *not* covered by any unit test: it is guarded by +`xtask/src/amd_smi_launch_contract.rs`, a static text scan that fails the build +when a file outside its allowlist calls `resolve_amd_smi_binary`. A rewrite that +kept the count behaviour but spawned `amd-smi` directly would be caught by that +scan and by nothing else. The decline itself is no longer discarded — `gpu_policy.rs`'s `amd_smi_probe_json` logs it at `info` with the full error chain, which is what makes "why did `--gpu auto` not rank my GPUs?" answerable without re-running. It is not `warn`, because on diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index e776f08a1a..54fcf5c51b 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -498,7 +498,12 @@ Feature: GPU detection and system inspection # examine-24/25/26 unable to reach amd-smi on a GPU-less runner at all. # Linux-only, like its three siblings: the stub is a shell script, and the # gate is a wave-through on every other platform. - @id:examine-telemetry-declines-an-unopenable-kfd-device @requires-os:linux + # `@requires-bare-metal`, not just `@requires-os:linux`: WSL2 reports an + # os_family of `linux`, so the Linux tag matches there — but on a WSL host + # with a usable GPU `kfd_readable`'s wave-through short-circuits the device + # open entirely, and the decline this scenario asserts never happens. The + # premise is a host where the `/dev/kfd` open is what decides. + @id:examine-telemetry-declines-an-unopenable-kfd-device @requires-os:linux @requires-bare-metal Scenario: examine-31 - An unopenable KFD device stops amd-smi being launched at all Given the KFD device cannot be opened When the daemon gathers a bridge snapshot