diff --git a/apps/rocm/src/chat_host_facts.rs b/apps/rocm/src/chat_host_facts.rs index 74d985a994..6797aade50 100644 --- a/apps/rocm/src/chat_host_facts.rs +++ b/apps/rocm/src/chat_host_facts.rs @@ -17,9 +17,12 @@ //! fast public getters (`runtime_os_name`, `is_wsl_host`, //! `detect_host_gpu_summary`) and never `ExamineSummary::gather()`, whose //! Windows path chains several process-spawning inventory queries. -//! - **Plain data.** The dash crates carry no `rocm-core` dependency; the bin -//! detects and renders, and the rendered `String` rides down through -//! `ResolvedArgs`. +//! - **Plain data.** `rocm-dash-tui`, which owns `ResolvedArgs`, carries no +//! `rocm-core` dependency and is forbidden one by `xtask/src/crate_edges.rs`. +//! So the bin detects and renders, and the rendered `String` rides down +//! through `ResolvedArgs`. (`rocm-dash-collectors` did gain a `rocm-core` +//! edge in EAI-8457, for the bounded `/dev/kfd` probe; nothing on this path +//! runs there.) //! //! Rendering is separated from detection ([`HostFacts::render`] is pure) so the //! wording is unit-testable without a machine. diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 304e9faa29..28346a4c05 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -184,8 +184,8 @@ fn anthropic_api_key_for_dash() -> Option { } /// Adapt the built-in `rocm-core` model recipes into the TUI-local summaries the -/// serve-wizard picker consumes (the bin owns the `rocm-core` dependency so the -/// dash crates stay free of it). +/// serve-wizard picker consumes (the bin owns the `rocm-core` dependency so +/// `rocm-dash-tui`, which `crate_edges.rs` still forbids one, stays free of it). fn model_recipe_summaries() -> Vec { builtin_model_recipes() .into_iter() diff --git a/apps/rocm/src/gpu_policy.rs b/apps/rocm/src/gpu_policy.rs index 448acef02f..81bef34323 100644 --- a/apps/rocm/src/gpu_policy.rs +++ b/apps/rocm/src/gpu_policy.rs @@ -67,7 +67,7 @@ use std::fs; #[cfg(any(target_os = "linux", test))] use std::path::Path; -use std::process::{Command as ProcessCommand, Stdio}; +use std::time::Duration; use anyhow::{Result, bail}; use rocm_core::AppPaths; @@ -640,6 +640,64 @@ fn select_auto_gpu_index( vec![reported[0]] } +/// How long the `serve` path waits on `amd-smi` before giving up on it. Both +/// probes below are advisory — GPU auto-selection degrades to service state +/// alone without them — so a stalled `amd-smi` must not hold up a launch. This +/// bounds the ordinary stalls only; the unkillable one is prevented by the +/// pre-flight inside [`rocm_core::amd_smi_json`], not by any timeout. +/// +/// Named for `serve` because that is the path the budget is sized for, not +/// because it is the only one that spends it. [`detect_gpu_count`] is reached +/// solely from `serve_cmd::serve`, but [`gpu_vram_usage_amd_smi`] is also +/// reached from `main.rs`'s `host_accelerator_memory`, and so from +/// `assess_model_with_host_paths` on the `--model` readiness path. That path +/// wants the same answer for the same reason — how much VRAM this host has, to +/// decide what it can run — so it wants the same budget; the one-shot argument +/// below applies to it unchanged. It is one probe there rather than two, so its +/// worst case is one probe's, not two. +/// +/// `rocmd`'s watcher has its own, shorter `AMD_SMI_PROBE_TIMEOUT` (2s) for the +/// same helper: that one fires on a polling cadence where a stalled probe costs +/// a missed telemetry tick and nothing else, so it can afford to give up early, +/// while a serve launch gets exactly one chance to learn the GPU layout and +/// giving up early there degrades the placement decision itself. +/// +/// # Worst-case latency, deliberately accepted rather than parallelised +/// +/// This value is not the whole wait. `rocm_core::amd_smi_json` runs its +/// `/dev/kfd` pre-flight *before* it spawns anything, and that open is bounded +/// separately by `rocm_core`'s `KFD_OPEN_TIMEOUT` (2s on Linux, no probe at all +/// elsewhere). So one probe costs up to 12s on Linux, not 10s. +/// +/// `serve` runs the two probes sequentially — `gpu_vram_usage` before the launch +/// lock, `detect_gpu_count` either in the `--gpu ` validation or inside +/// the lock for `--gpu auto` — so a Linux host where the KFD open hangs *and* +/// `amd-smi` then stalls adds up to ~24s to an interactive launch (2 × (2s + +/// 10s)); the single-probe `--model` readiness path adds up to ~12s. +/// +/// The pre-flight's 2s is only spent on a host whose device open is actually +/// hanging, which is also the host on which the probe then declines and the +/// `amd-smi` 10s is *not* spent — so the two bounds are additive in the formula +/// but rarely both paid in practice. Quoted additively anyway: a ceiling that +/// assumes the good case is not a ceiling. +/// +/// Before this bound existed both probes called `.output()` with no timeout at +/// all, so a stalling host hung indefinitely; ~24s is the ceiling replacing +/// "forever", not a new cost. +/// +/// It is a ceiling on *this* hazard, not on every one. It holds when the +/// `open` is what hangs, which is what the pre-flight bounds. If `amd-smi` +/// gets past the open and a later ioctl wedges it into uninterruptible sleep, +/// `run_with_timeout`'s post-kill `wait` does not return either, and no number +/// quoted here applies — the pre-flight is a mitigation for the reachable +/// case, not a proof about the whole call. Running them concurrently is not free: +/// `detect_gpu_count` is invoked inside [`select_gpu_indices_under_launch_lock`] +/// precisely so the cpu-only and pinned paths never pay for the subprocess, and +/// hoisting it out to overlap with the VRAM probe would make every +/// `rocm serve --cpu-only` spawn an `amd-smi` it has no use for. The stall is +/// rare, the extra spawn would be unconditional. +const AMD_SMI_SERVE_PROBE_TIMEOUT: Duration = Duration::from_secs(10); + /// Best-effort per-GPU VRAM occupancy. Prefers `amd-smi metric --json`; when /// amd-smi is not installed (e.g. a shared node or container that ships no /// amd-smi — this never tries `rocm-smi`) it falls back to the amdgpu DRM sysfs @@ -686,26 +744,57 @@ fn simulated_low_vram_usage() -> Option> { }]) } -/// Per-GPU VRAM occupancy via `amd-smi metric --json`. Returns `None` when -/// amd-smi is unavailable or its output cannot be parsed. +/// The `amd-smi` half of [`gpu_vram_usage`], routed through the guarded +/// launcher so a missing `/dev/kfd` declines the spawn instead of stalling. fn gpu_vram_usage_amd_smi() -> Option> { - let binary = rocm_core::resolve_amd_smi_binary(); - let output = ProcessCommand::new(&binary) - .arg("metric") - .arg("--json") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .output() - .ok()?; - if !output.status.success() { - return None; - } - let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?; + let value = amd_smi_probe_json(&["metric", "--json"], "per-GPU VRAM occupancy")?; let rows = parse_gpu_vram_usage(&value); if rows.is_empty() { None } else { Some(rows) } } +/// Run one advisory `serve`-path `amd-smi` probe, logging the error on the way +/// to the `None` both callers want. +/// +/// Both probes degrade rather than fail, so neither has anywhere to *return* an +/// error: the caller's contract is `Option`, and a decline is a normal, expected +/// answer on a GPU-less host. That made the whole class silent, including the +/// two cases a user would want to know about — the pre-flight refusing a device +/// that is merely slow rather than wedged (the 2s `KFD_OPEN_TIMEOUT` is a +/// heuristic bound, and a loaded host can trip it while being perfectly +/// healthy), and an `amd-smi` that ran and failed. "Why did `--gpu auto` not +/// rank my GPUs?" had no answer anywhere. +/// +/// `info!` and not `warn!` or `debug!`. Not `warn!`: on a GPU-less host this +/// fires on every `serve`, where it is the expected outcome and not a problem, +/// so at `warn` it would be noise that trains people to ignore the level. Not +/// `debug!` either, which is where this started: the default filter is +/// `warn,rocm=info`, so a `debug!` line in this crate is invisible unless +/// someone already suspected it and re-ran with `RUST_LOG`. That is the wrong +/// way round for the case that actually needs explaining — the 2s open bound +/// tripping on a slow but healthy host, or an exhausted probe budget — where +/// the decline is indistinguishable from "no GPU" and the user has no reason +/// to go looking. `rocm=info` is on by default, so the reason is there the +/// first time. +/// +/// `{error:#}` rather than `to_string()`, matching `rocmd`'s bridge-snapshot +/// note: the outermost context says "failed to launch amd-smi ...", which names +/// the wrong thing when what happened was a pre-flight decline or a timeout. +/// The causes are the whole value of the line. +fn amd_smi_probe_json(args: &[&str], purpose: &str) -> Option { + match rocm_core::amd_smi_json(args, AMD_SMI_SERVE_PROBE_TIMEOUT) { + Ok(value) => Some(value), + Err(error) => { + tracing::info!( + probe = purpose, + args = args.join(" "), + error = format!("{error:#}"), + "amd-smi probe declined or failed; GPU selection degrades to service state alone" + ); + None + } + } +} + /// Fallback per-GPU VRAM occupancy read straight from the amdgpu DRM sysfs /// counters (`/sys/class/drm/card*/device/mem_info_vram_{total,used}`), used when /// amd-smi is not installed. Linux-only; other platforms expose no such @@ -963,20 +1052,15 @@ fn busy_gpu_indices(paths: &AppPaths) -> Vec { /// Best-effort count of local AMD GPUs via `amd-smi`. Returns `None` when /// amd-smi is unavailable or its output cannot be parsed (callers then fall /// back to conservative defaults). +/// +/// Also `None` whenever the `/dev/kfd` pre-flight declines the launch, which it +/// does for an unreachable device and not only for one that would hang — so a +/// host outside the render group lands here, with no sysfs fallback of the kind +/// [`gpu_vram_usage`] has. That is deliberate and is not a loss of GPU +/// detection; `rocm_core`'s `kfd_readable` documents why, and what answers in +/// this count's place. pub(crate) fn detect_gpu_count() -> Option { - let binary = rocm_core::resolve_amd_smi_binary(); - let output = ProcessCommand::new(&binary) - .arg("list") - .arg("--json") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .output() - .ok()?; - if !output.status.success() { - return None; - } - let value: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?; + let value = amd_smi_probe_json(&["list", "--json"], "local AMD GPU count")?; let count = value.as_array().map(Vec::len)?; if count == 0 { None } else { Some(count) } } diff --git a/apps/rocmd/src/host_snapshot.rs b/apps/rocmd/src/host_snapshot.rs index 93ce12edbc..a1f39f3aa7 100644 --- a/apps/rocmd/src/host_snapshot.rs +++ b/apps/rocmd/src/host_snapshot.rs @@ -2,16 +2,31 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result}; +use anyhow::Result; use rocm_core::{ AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, - CodexBridgeSnapshot, ExamineSummary, RocmCliConfig, default_engine_for_platform, - load_recent_automation_events, resolve_amd_smi_binary, unix_time_millis, + CodexBridgeSnapshot, ExamineSummary, RocmCliConfig, amd_smi_json, default_engine_for_platform, + load_recent_automation_events, unix_time_millis, }; use serde_json::Value; -use std::process::Command as ProcessCommand; use std::time::Duration; +/// How long `rocmd` waits on an `amd-smi` probe before giving up on it. +/// +/// Deliberately 5x shorter than `serve`'s `AMD_SMI_SERVE_PROBE_TIMEOUT` (10s), +/// even though both now reach the same `rocm_core::amd_smi_json`. The two pay +/// different prices for giving up early. This one runs on the GPU-telemetry +/// watcher's polling cadence, where a probe that is abandoned costs one stale +/// tick and the next poll retries moments later — so a long wait buys nothing +/// and keeps a watcher thread parked on an unhealthy host. `serve` gets exactly +/// one shot at learning the GPU layout before placing a model, and abandoning +/// it degrades the placement decision for the whole lifetime of that service. +/// +/// Not the whole wait: `amd_smi_json` runs its `/dev/kfd` pre-flight before it +/// spawns anything, bounded separately by `rocm_core`'s `KFD_OPEN_TIMEOUT` (2s +/// on Linux). A poll therefore costs up to 4s on a host whose device open also +/// hangs. Still inside the watcher's 60s cadence, so the tick is late rather +/// than skipped. const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); pub(crate) fn build_bridge_snapshot(paths: &AppPaths) -> Result { @@ -43,7 +58,11 @@ fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { // Every failure `run_with_timeout` raises carries its cause the // same way, since all of them arrive under that one context: // the timeout, the failure to spawn, and the poll and reap - // failures of its wait loop. Spawn is the case on a host where + // failures of its wait loop. The `/dev/kfd` pre-flight in + // `amd_smi_json` is the one failure that arrives *without* it, + // because it declines before there is a launch to contextualise + // -- its own message is self-describing and reaches the note + // intact. Spawn is the case on a host where // none of the paths `resolve_amd_smi_binary` searches holds an // `amd-smi`, down to the bare name on `PATH`; the cause is then // the OS's own "not found" text, which is not the same string @@ -57,7 +76,8 @@ fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { // The note is the only account of any of this that reaches the // bridge snapshot and the automation watcher, so it carries the // causes too -- bounded, because it is persisted; see - // `run_with_timeout`'s `MAX_QUOTED_OUTPUT_CHARS`. + // `MAX_QUOTED_OUTPUT_CHARS`, which `amd_smi_json` and + // `run_with_timeout` share for their two quotes. amd_smi_available: false, static_snapshot: None, monitor_snapshot: None, @@ -109,32 +129,11 @@ pub(crate) fn gather_gpu_snapshot_for_config(config: &RocmCliConfig) -> CodexBri } fn capture_amd_smi_json(args: &[&str]) -> Result { - let amd_smi_binary = resolve_amd_smi_binary(); - let mut command = ProcessCommand::new(&amd_smi_binary); - // No stdio here: `run_with_timeout` sets null stdin and piped stdout/stderr - // itself, since capturing both is its contract. - command.args(args); - let output = rocm_core::process::run_with_timeout(command, AMD_SMI_PROBE_TIMEOUT, "process") - .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - anyhow::bail!( - "amd-smi {} failed: {}", - args.join(" "), - if !stderr.is_empty() { - stderr - } else if !stdout.is_empty() { - stdout - } else { - format!("exit status {}", output.status) - } - ); - } - - serde_json::from_slice(&output.stdout) - .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) + // `rocm_core::amd_smi_json`, not a local spawn: it is the one launch site + // that runs the `/dev/kfd` pre-flight, and the pre-flight protects nothing + // unless every launch goes through it. The spawn, the timeout, the + // non-zero-exit quote and the JSON parse all live there now. + amd_smi_json(args, AMD_SMI_PROBE_TIMEOUT) } pub(crate) fn bridge_engine_inventory() -> Vec { diff --git a/crates/rocm-core/src/amd_smi.rs b/crates/rocm-core/src/amd_smi.rs index 3299d66460..9947c8d331 100644 --- a/crates/rocm-core/src/amd_smi.rs +++ b/crates/rocm-core/src/amd_smi.rs @@ -2,20 +2,46 @@ // // SPDX-License-Identifier: MIT -//! Locating the `amd-smi` binary. +//! Locating and launching the `amd-smi` binary. //! -//! Prefers the binary shipped in the newest managed ROCm SDK runtime recorded -//! in the registry, then falls back to the home-directory venv and legacy -//! locations, then to the bare `amd-smi` name for a `PATH` lookup. +//! Location prefers the binary shipped in the newest managed ROCm SDK runtime +//! recorded in the registry, then falls back to the home-directory venv and +//! legacy locations, then to the bare `amd-smi` name for a `PATH` lookup. +//! +//! Launching goes through [`amd_smi_json`], the one supported way for a +//! synchronous caller to start the binary: it runs the `/dev/kfd` pre-flight in +//! `kfd_readable` first, and that pre-flight protects nothing unless every +//! launch site goes through it. -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; +#[cfg(target_os = "linux")] +use std::fs; use std::path::{Path, PathBuf}; +// `Command` is this module's own — [`amd_smi_json_with`] builds one — but +// nothing here runs it: the wait, the kill and both pipe drains belong to +// [`crate::process::run_with_timeout`], which is why no production `Stdio` +// import comes with it (the helper sets the child's stdio itself, and its docs +// say callers should not). `Stdio` and `thread` are left to the KFD open probe +// and its Linux-only test, hence the `cfg`s: an unused import is a hard error +// on the Windows lane's `-D warnings` build. +use std::process::Command; +#[cfg(all(test, target_os = "linux"))] +use std::process::Stdio; +#[cfg(target_os = "linux")] +use std::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(target_os = "linux")] +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; use crate::managed_runtime::{ TheRockFamilyManifest, managed_sdk_tool_path, managed_therock_environment_records, sort_records_newest_install_first, }; use crate::{default_data_dir, runtime_home_dir}; +#[cfg(target_os = "linux")] +use crate::{has_usable_amd_gpu, host_path, is_wsl_host}; pub fn resolve_amd_smi_binary() -> OsString { if let Some(path) = default_data_dir() @@ -79,6 +105,479 @@ fn resolve_amd_smi_binary_in_home(home_dir: Option<&Path>) -> OsString { "amd-smi".into() } +/// Device node of the AMD kernel fusion driver. Bare-metal Linux only. WSL +/// compiles as `target_os = "linux"` but reaches the GPU through `/dev/dxg` and +/// the Windows host driver, and has no `/dev/kfd` at all — see `docs/wsl.md` +/// and [`crate::examine::WslFacts`]. +const KFD_DEVICE: &str = "/dev/kfd"; + +/// Name of the E2E-only seam that moves the pre-flight's device node, so the +/// gate's two branches are both reachable from a Gherkin scenario. +/// +/// Honoured only by a build carrying the `e2e-test-hooks` feature, which no +/// released artefact does; see [`preflight_kfd_device`]. +#[cfg(feature = "e2e-test-hooks")] +pub(crate) const TEST_KFD_DEVICE_ENV: &str = "ROCM_E2E_KFD_DEVICE"; + +/// The device node [`amd_smi_json`] runs its pre-flight against. +/// +/// [`KFD_DEVICE`] in every shipped build. An `e2e-test-hooks` build lets +/// [`TEST_KFD_DEVICE_ENV`] name a different path, which is what makes the gate +/// testable from the outside at all. +/// +/// A path seam rather than the existing [`crate::hardware_root`] re-rooting, +/// because the two cannot do the same job here: [`kfd_readable`] keeps the real +/// node's veto on purpose, so a simulated root can only ever make the gate +/// decline, never allow. That is the invariant stopping a test hook from +/// launching `amd-smi` on a machine with no GPU — and it also means no +/// simulated root can reach the *allowed* branch on the GPU-less lane that runs +/// per PR. +/// +/// An override, not a bypass: it relocates the probe and does not skip it. With +/// the feature on and the variable unset — every unit test here, and every E2E +/// scenario that does not set it — the gate is `/dev/kfd` exactly as in +/// production. +#[cfg(feature = "e2e-test-hooks")] +fn preflight_kfd_device() -> PathBuf { + std::env::var_os(TEST_KFD_DEVICE_ENV).map_or_else(|| PathBuf::from(KFD_DEVICE), Into::into) +} + +/// Production counterpart of [`preflight_kfd_device`]: the constant, with no +/// environment consulted. See that function for why the seam exists. +#[cfg(not(feature = "e2e-test-hooks"))] +fn preflight_kfd_device() -> PathBuf { + PathBuf::from(KFD_DEVICE) +} + +/// Whether `/dev/kfd` can be opened for reading — the pre-flight every +/// `amd-smi` launch goes through, and it is **mandatory** before every launch. +/// +/// # What the predicate is +/// +/// "This process can reach the KFD", not "`amd-smi` would hang". The two are +/// different sets, and declining the difference is the intended trade: +/// +/// * An open that **never returns** is the hazard itself — `amd-smi` against +/// such a device blocks in uninterruptible kernel sleep, where no signal +/// escapes, so neither Ctrl-C nor `SIGKILL` nor any caller's timeout reaches +/// it. [`KFD_OPEN_TIMEOUT`] bounds the probe and reads the overrun as "not +/// readable", so the gate declines instead of inheriting the stall. +/// * An open that **fails with an errno** — `ENOENT` for an absent node, +/// `EACCES` outside the render group, whatever a driver mid-reset returns — +/// cannot park anything by itself. The gate declines these too. +/// +/// The second class is broader than the hazard on purpose. Narrowing it means +/// deciding per errno which kernel states are safe to launch into, and that +/// list cannot be checked against a resetting or wedged device by any test this +/// repo can run; a gate that declined only on the timeout would be assuming +/// every errno it had not anticipated is harmless. Not resting on an assumption +/// of that shape is the whole reason the open is bounded in the first place +/// (see [`KFD_OPEN_TIMEOUT`]). A refusal is recoverable and names what it +/// refused; a hang is neither. +/// +/// # What this does not prove +/// +/// It is a **mitigation heuristic, not a proof**. All this establishes is that +/// `open("/dev/kfd")` returned inside [`KFD_OPEN_TIMEOUT`]. The hang it exists +/// to prevent is attributed to `amd-smi`'s *later* ioctls against a wedged +/// device, so a device that opens and then wedges on the next ioctl is outside +/// what the gate can see, and the 2s bound is chosen rather than measured. No +/// run against a genuinely unhealthy driver backs any of this up: the lane that +/// runs these tests has no `amd-smi` and no GPU, and the FIFO fixtures model a +/// blocking `open`, the closest reachable analogue rather than the real +/// failure. What the gate buys is that the hazard's common precondition — a +/// device the kernel will not hand out — refuses the spawn, instead of betting +/// on a timeout that cannot reach a process in uninterruptible sleep. +/// +/// # What the broad refusal costs +/// +/// On a host outside the render group `amd-smi` might well have answered — much +/// of its inventory comes from sysfs — so `gpu_policy::detect_gpu_count` now +/// returns `None` there and, unlike `gpu_vram_usage`, has no sysfs fallback of +/// its own. That is a real narrowing. It is not a loss of GPU detection: on the +/// hosts this branch can fire on, `serve` resolves ordinals from +/// [`crate::usable_amd_gpu_indices`], which counts KFD topology nodes under +/// `/sys/class/kfd` and AMD DRM cards under `/sys/class/drm` — both +/// world-readable, neither needing the device node open — and +/// `validate_pinned_gpu_index` prefers that set outright. What is lost is the +/// `amd-smi` device count `--gpu auto` would rank with; `effective_gpu_count` +/// (`apps/rocm/src/gpu_policy.rs`) falls through to the DRM sysfs VRAM rows, so +/// auto-selection still ranks, over those rows alone rather than over nothing. +/// `rocm diagnose` already names the host, as `fix-4-render-group`. +/// +/// # Platforms with no KFD +/// +/// Off Linux the check is `true` without probing. Load-bearing rather than +/// pedantic: `amd-smi.exe` is a supported Windows binary (see +/// [`crate::managed_runtime::managed_sdk_tool_path`]), so reading an absent +/// `/dev/kfd` as "unsafe" would silently disable GPU detection on every Windows +/// host. +/// +/// WSL needs the same wave-through and cannot get it from `cfg`: it compiles as +/// Linux yet has no `/dev/kfd` (see [`KFD_DEVICE`]), so the real open fails on +/// every WSL host. WSL identity alone is not the condition, though — `/dev/dxg` +/// is vendor-agnostic, and a WSL host whose ROCDXG plumbing is not ready is one +/// where `amd-smi` still hangs, which a bare "is this WSL" test cannot tell +/// apart from a healthy one. The condition is therefore +/// `is_wsl_host() && has_usable_amd_gpu()`, the same conjunction +/// `apps/rocm/src/dash.rs::gpu_reachable_for_preflight` feeds the dashboard, so +/// `serve`, `rocmd` and the dashboard keep one cross-surface verdict. On bare +/// metal `is_wsl_host()` short-circuits and the behaviour is a plain +/// `/dev/kfd` open. +/// +/// # Hardware-probe routing +/// +/// Like the dashboard pre-flight, this reads `host_path("/dev/kfd")` *and* +/// keeps the real node's veto (see [`crate::hardware_root`]). A simulated E2E +/// root can make the gate decline on a machine that has a GPU, but can never +/// make it allow a launch on a machine that does not. +#[cfg(target_os = "linux")] +fn kfd_readable(device: &Path) -> bool { + kfd_readable_with( + // Second argument is a closure, not a value: extracting this condition + // into a named function turned a short-circuit into two eager + // evaluations, so `has_usable_amd_gpu` was running on every Linux + // launch rather than on WSL only. Passing the probe unevaluated keeps + // the original behaviour and still lets the condition be tested. + wsl_wave_through(is_wsl_host(), has_usable_amd_gpu), + device, + &host_path(device), + ) +} + +/// The WSL wave-through's own condition: a WSL host **and** a reachable GPU. +/// +/// Extracted so it can be pinned by a truth table. The tests below drive +/// [`kfd_readable_with`] with literal booleans, which says nothing about how +/// the caller computes that argument — `&&` could become `||`, or +/// `has_usable_amd_gpu()` could be dropped entirely, and every one of them +/// would stay green on a GPU-less lane. The GPU probe is taken unevaluated so +/// the short-circuit survives the extraction: on bare metal `is_wsl` is false +/// and `has_usable_amd_gpu` must not run at all. Both conjuncts are +/// load-bearing in +/// opposite directions: without `is_wsl_host` a bare-metal host with a GPU +/// would skip the device check the gate exists for, and without +/// `has_usable_amd_gpu` an unready WSL host would be waved through to an +/// `amd-smi` that cannot work. +#[cfg(target_os = "linux")] +fn wsl_wave_through(is_wsl: bool, has_usable_gpu: impl FnOnce() -> bool) -> bool { + is_wsl && has_usable_gpu() +} + +/// [`kfd_readable`] with its two non-syscall inputs injected, so the +/// wave-through and the simulated-root routing can both be pinned from a test +/// on an ordinary Linux host rather than only on WSL or under an E2E root. +/// +/// `wsl_gpu_reachable` may only ever be the WSL conjunction described on +/// [`kfd_readable`]; it substitutes for the device open entirely, so widening it +/// to any fail-open verdict would disable the gate on bare metal. +#[cfg(target_os = "linux")] +fn kfd_readable_with(wsl_gpu_reachable: bool, real_device: &Path, host_device: &Path) -> bool { + wsl_gpu_reachable + || (device_readable(real_device) + && (host_device == real_device || device_readable(host_device))) +} + +/// How long the pre-flight waits for the device `open` itself before giving up +/// on it and declining the launch. +/// +/// A plain blocking `open` would make the gate's own safety rest on an +/// assumption — that `kfd_open` always returns promptly, so only `amd-smi`'s +/// later ioctls can enter D-state — which cannot be checked against a +/// reset-in-progress device by any test here. Were it wrong, the gate would +/// have moved the hazard into a synchronous call with *less* protection than +/// the spawn it replaced, there being no child to kill. +/// +/// Bounding the open removes the need to be right about it: an open that has +/// not returned in this long reads as "not readable", the declining answer, so +/// the unverifiable case fails safe. Short, because a healthy `kfd_open` takes +/// microseconds and a longer wait only buys a longer stall on a host that is +/// already unhealthy. +/// +/// **It is additive to every [`amd_smi_json`] `timeout`, not inside it.** The +/// pre-flight finishes before anything is spawned, so a caller that passes a +/// 10s timeout can wait up to 12s in the worst case. Callers that document a +/// worst-case latency have to include this; see +/// `apps/rocm/src/gpu_policy.rs::AMD_SMI_SERVE_PROBE_TIMEOUT`. +#[cfg(target_os = "linux")] +const KFD_OPEN_TIMEOUT: Duration = Duration::from_secs(2); + +/// How many device-open probes may be parked at once before the pre-flight +/// starts declining outright. +/// +/// Every probe that overruns [`KFD_OPEN_TIMEOUT`] leaves a thread blocked in +/// `open` until the kernel lets it go, which against the wedged device this +/// gate exists to refuse may be never. `rocmd`'s telemetry watcher and `serve`'s +/// probes both poll, so without a cap a host stuck in that state accumulates one +/// parked thread per poll until the process runs out of its thread budget. +/// +/// Small on purpose: a larger budget only buys more parked threads on a host +/// already answering "not readable". A healthy `kfd_open` returns in +/// microseconds, so one slot would serve every caller this crate has; 8 leaves +/// headroom for concurrent ones without approaching a thread limit. +#[cfg(target_os = "linux")] +const MAX_INFLIGHT_KFD_PROBES: usize = 8; + +/// The process-wide budget every production probe claims from. +#[cfg(target_os = "linux")] +static KFD_PROBE_BUDGET: ProbeBudget = ProbeBudget::new(MAX_INFLIGHT_KFD_PROBES); + +/// A cap on concurrently parked device-open probes. +/// +/// A counter rather than a `Semaphore` because the release is not scoped to the +/// caller: a slot is held by the probe *thread*, which outlives the call that +/// started it whenever the probe overran, and is freed if and when the kernel +/// finally returns from `open`. So a host that recovers goes back to its full +/// budget without anything having to notice, while one that never does keeps +/// answering from the declining branch rather than spawning threads forever. +#[cfg(target_os = "linux")] +struct ProbeBudget { + in_flight: AtomicUsize, + limit: usize, +} + +#[cfg(target_os = "linux")] +impl ProbeBudget { + const fn new(limit: usize) -> Self { + Self { + in_flight: AtomicUsize::new(0), + limit, + } + } + + /// Take one slot, or `None` when the budget is already spent. + /// + /// `fetch_update` rather than `fetch_add` + compensating decrement: the + /// latter is momentarily over the limit, which two callers racing at the + /// boundary can both observe. + fn claim(&'static self) -> Option { + self.in_flight + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |in_flight| { + (in_flight < self.limit).then_some(in_flight + 1) + }) + .ok() + .map(|_| ProbeSlot(self)) + } +} + +/// One claimed slot of a [`ProbeBudget`], released on drop. +/// +/// Moved *into* the probe thread so the release happens when the `open` returns +/// rather than when the caller stops waiting — the whole point of the cap is to +/// count threads that are still parked. +#[cfg(target_os = "linux")] +struct ProbeSlot(&'static ProbeBudget); + +#[cfg(target_os = "linux")] +impl Drop for ProbeSlot { + fn drop(&mut self) { + self.0.in_flight.fetch_sub(1, Ordering::Release); + } +} + +/// Whether `device` can be opened for reading, giving up after +/// [`KFD_OPEN_TIMEOUT`]. +#[cfg(target_os = "linux")] +fn device_readable(device: &Path) -> bool { + device_readable_within_budget(device, KFD_OPEN_TIMEOUT, &KFD_PROBE_BUDGET) +} + +/// [`device_readable`] with the bound and the budget injected, so a test can pin +/// the timeout path against a device that really does block in `open` (a FIFO +/// with no writer) without stalling the suite for the production bound, and can +/// exhaust a budget of its own without spending the production one. +/// +/// The `open` runs on its own thread and the caller waits on a channel, because +/// there is no portable way to interrupt a blocking `open` once it has entered +/// the kernel. A probe that overruns leaves that thread parked — the same +/// trade-off [`crate::process::run_with_timeout`] makes when it abandons rather +/// than joins its readers, and for the same reason: one leaked thread is +/// cheaper than a blocked caller. The opened handle is dropped on the probe +/// thread, so a late success closes its own descriptor rather than leaking one. +/// +/// # Why both failure branches decline +/// +/// A probe that cannot claim a `budget` slot, and one whose thread cannot be +/// spawned at all, both answer `false` rather than falling back to a direct +/// `open` on the calling thread. Neither failure is independent of the state +/// this gate refuses: the way threads run out *here* is one parked `open` per +/// poll against a wedged device, so such a fallback would fire exactly when the +/// device is most dangerous and perform the unbounded blocking `open` the bound +/// exists to prevent — on the caller's own thread, with nothing left to kill. +/// +/// Refusing instead is recoverable and visible: [`amd_smi_json`] says it +/// declined and names the device, GPU-aware callers degrade to service state +/// alone, and the next poll on a recovered host finds its slots freed and +/// allows the launch again. So both branches give the same `false` the timeout +/// gives, for the same reason. +#[cfg(target_os = "linux")] +fn device_readable_within_budget( + device: &Path, + timeout: Duration, + budget: &'static ProbeBudget, +) -> bool { + let Some(slot) = budget.claim() else { + return false; + }; + let (sender, receiver) = std::sync::mpsc::channel(); + let probe_path = device.to_path_buf(); + let spawned = thread::Builder::new() + .name("rocm-kfd-open-probe".to_owned()) + .spawn(move || { + // Held until the `open` returns, however long that is, so the budget + // counts threads that are still parked rather than calls still + // waiting. A failed spawn drops the closure, and with it the slot. + let _slot = slot; + let readable = fs::OpenOptions::new().read(true).open(&probe_path).is_ok(); + // The receiver is gone whenever the probe overran; dropping the + // send result is the whole handling. + let _ = sender.send(readable); + }) + .is_ok(); + if !spawned { + return false; + } + // Both error arms — timeout and a probe thread that died without sending — + // mean "no readable answer", which the gate must read as not readable. + receiver.recv_timeout(timeout).unwrap_or(false) +} + +/// A read probe of `device` that cannot park the caller. +/// +/// The same bounded open this module uses for its own `/dev/kfd` pre-flight, +/// exposed for `rocm-dash-collectors`, which gates the dashboard's GPU-metrics +/// `amd-smi` launch on the same device. Shared rather than restated there so +/// the bound, the budget and the reasoning on `device_readable_within_budget` +/// stay in one place; a second copy would be free to drift from all three. +/// +/// Off Linux this is a direct open, matching [`kfd_readable`]'s wave-through: +/// there is no `/dev/kfd` to wedge, and the bound would buy nothing. +#[cfg(target_os = "linux")] +pub fn device_readable_bounded(device: &Path) -> bool { + device_readable(device) +} + +/// Off-Linux counterpart of [`device_readable_bounded`]: a direct open, with no +/// bound and no wave-through. +/// +/// It does **not** mirror [`kfd_readable`]'s off-Linux wave-through, and an +/// earlier revision that made it do so was wrong. The two answer different +/// questions. `kfd_readable` gates *this crate's* `amd-smi` launch, which off +/// Linux has no `/dev/kfd` to reason about, so waving through is right there. +/// This function is the shared probe `rocm-dash-collectors` calls for its own +/// pre-flight, and that pre-flight is expected to decline a missing device on +/// every platform — `missing_kfd_and_no_reachability_fails_preflight` and +/// `real_kfd_keeps_a_veto_over_the_host_root_view` assert exactly that, and a +/// wave-through turns both green while removing the veto they exist to protect. +/// +/// The bound is Linux-only because the hazard is: `/dev/kfd` is what can park a +/// process in uninterruptible sleep. An ordinary file open elsewhere returns or +/// errors. +#[cfg(not(target_os = "linux"))] +pub fn device_readable_bounded(device: &Path) -> bool { + std::fs::OpenOptions::new().read(true).open(device).is_ok() +} + +/// Off-Linux counterpart of [`kfd_readable`]: see that function's docs for why +/// the wave-through is load-bearing rather than a stub. +/// +/// `const` because it has to be. The Windows lane builds with `-D warnings` and +/// runs clippy, so `clippy::missing_const_for_fn` makes this a hard compile +/// error there while the Linux lane — where the real body opens a device and +/// cannot be `const` — never sees the lint. A `cfg`-selected pair of functions +/// is linted per branch, not per name. +#[cfg(not(target_os = "linux"))] +const fn kfd_readable(_device: &Path) -> bool { + true +} + +/// Run `amd-smi ` and parse its stdout as JSON, declining to launch at all +/// unless the `/dev/kfd` pre-flight in [`kfd_readable`] passes. +/// +/// The supported way to launch `amd-smi` from a synchronous caller. The +/// pre-flight has to happen at every launch site or it protects nothing, and +/// routing every launch through one function is what keeps that true as sites +/// are added. +/// +/// `timeout` is defence in depth, not the mitigation: [`crate::process::run_with_timeout`] +/// kills the timed-out child and waits on it, and neither dislodges a process +/// in uninterruptible sleep — nor does the `wait` on one return. It bounds the +/// ordinary failures (a slow or wedged-but-signalable `amd-smi`); the +/// pre-flight is what keeps the D-state case away. +/// +/// # Latency +/// +/// `timeout` is not the worst case. The pre-flight runs first and is bounded +/// separately by [`KFD_OPEN_TIMEOUT`] (2s on Linux), so the worst case is +/// `KFD_OPEN_TIMEOUT + timeout` — 12s for a 10s `timeout`, 4s for the 2s one +/// `rocmd`'s watcher passes. A caller documenting its own worst case has to add +/// the pre-flight's bound to the one it passes here. +pub fn amd_smi_json(args: &[&str], timeout: Duration) -> Result { + amd_smi_json_with( + &resolve_amd_smi_binary(), + // [`preflight_kfd_device`], not the constant: identical in every shipped + // build, and the one seam an E2E scenario can move so that both sides of + // the gate are reachable from the lane that runs per PR. + &preflight_kfd_device(), + args, + timeout, + ) +} + +fn amd_smi_json_with( + binary: &OsStr, + kfd_device: &Path, + args: &[&str], + timeout: Duration, +) -> Result { + if !kfd_readable(kfd_device) { + bail!( + "declined to launch `amd-smi {}`: {} could not be opened for reading (absent, \ + permission denied, no KFD GPU node, or a reset in progress); the pre-flight \ + refuses an unreachable device because amd-smi can block unkillably in \ + uninterruptible sleep against one whose open never returns", + args.join(" "), + kfd_device.display() + ); + } + + let mut command = Command::new(binary); + // No stdio here: `run_with_timeout` sets null stdin and piped stdout/stderr + // itself, since capturing both is its contract, and its docs say callers + // should not set them. + command.args(args); + let output = crate::process::run_with_timeout(command, timeout, "amd-smi") + .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; + + if !output.status.success() { + // Truncated, not dropped. This message reaches `rocmd`'s snapshot note + // and from there the JSONL logs, so the quote has to be bounded at the + // point that knows these characters are raw child output — the same + // `MAX_QUOTED_OUTPUT_CHARS` `run_with_timeout` bounds its own quote + // with, shared rather than re-declared so the two cannot drift. The + // head is where a failing `amd-smi` says what went wrong, and the + // command is named before it either way. + // + // The stderr-then-stdout precedence is `command_failure_detail`'s, taken + // from the shared helper rather than re-spelled here; only the + // both-blank fallback differs, naming the exit status instead of the + // generic "no output". `ExitStatus`'s own `Display` already reads + // "exit status: 1" (and "exit code: 1" on Windows), so prefixing it + // again produced "exit status exit status: 1". + bail!( + "amd-smi {} failed: {}", + args.join(" "), + crate::probe_exec::truncate_to_chars( + crate::command_failure_detail_or_else(&output, || output.status.to_string()), + crate::process::MAX_QUOTED_OUTPUT_CHARS, + ) + ); + } + + serde_json::from_slice(&output.stdout) + .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) +} + #[cfg(test)] mod tests { use super::*; @@ -87,6 +586,608 @@ mod tests { use anyhow::Result; use std::fs; + /// Serialise a test that reads a device node through [`kfd_readable`]. + /// + /// The gate resolves its device through `host_path`, so in an + /// `e2e-test-hooks` build another test setting `ROCM_CLI_TEST_HOST_ROOT` + /// process-wide would re-root this test's own tempdir device out from under + /// it. Same lock `fix.rs` takes, for the same reason; without the feature it + /// is uncontended and costs nothing. + #[cfg(target_os = "linux")] + fn host_root_guard() -> std::sync::MutexGuard<'static, ()> { + crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// The pre-flight's bound and budget are quoted as literal numbers in + /// several places that cannot see this constant, and in at least two of + /// them the quoted figure is *derived* — the caller's own timeout plus this + /// one. Changing [`KFD_OPEN_TIMEOUT`] without touching those leaves prose + /// that confidently states the wrong worst case, and nothing else in the + /// tree notices. This test is the tripwire: it fails on the change, and its + /// message is the list of what to update. + /// + /// Not a claim that 2s and 8 are the right values — both are judgement + /// calls, argued where they are declared. It only makes changing them cost + /// a deliberate pass over every restatement. + #[test] + #[cfg(target_os = "linux")] + fn the_probe_bound_and_budget_are_restated_in_these_places() { + assert_eq!( + KFD_OPEN_TIMEOUT, + Duration::from_secs(2), + "the pre-flight's open bound changed. Figures derived from it are restated in: \ + this module's `KFD_OPEN_TIMEOUT` and `amd_smi_json` docs; \ + `apps/rocm/src/gpu_policy.rs::AMD_SMI_SERVE_PROBE_TIMEOUT` (one probe 2+10=12s, \ + two sequential probes ~24s, `--model` readiness ~12s); \ + `apps/rocmd/src/host_snapshot.rs::AMD_SMI_PROBE_TIMEOUT` (2+2=4s); \ + `crates/rocm-dash-collectors/src/amd_smi.rs`'s \ + `the_device_preflight_does_not_run_on_the_async_worker` doc; and \ + the `amd-smi KFD pre-flight` section of `docs/testing.md`. \ + Update every one of them, then update this assertion." + ); + assert_eq!( + MAX_INFLIGHT_KFD_PROBES, 8, + "the in-flight probe budget changed; it is restated in this module's own docs and \ + in the `amd-smi KFD pre-flight` section of `docs/testing.md`" + ); + } + + /// The gate has to stop the *spawn*, not merely report a failure after it: + /// the launch that already happened is the launch that hangs. The fake + /// `amd-smi` leaves a marker file, so its absence is direct evidence + /// nothing ran — and running the same fixture against a readable device + /// proves the fake would otherwise have executed, which is what stops the + /// blocked case passing vacuously. + /// + /// Linux-only: off Linux `kfd_readable` is deliberately a no-op, so there + /// is no gate to exercise. + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_gate_stops_amd_smi_before_it_is_spawned() -> Result<()> { + // WSL is waved through deliberately, so the blocked case cannot hold + // there. + if is_wsl_host() { + skip_notice("this is a WSL host, which the KFD gate waves through by design"); + return Ok(()); + } + let _host_root = host_root_guard(); + + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-gate-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let marker = temp_root.join("amd-smi-ran"); + let fake_amd_smi = temp_root.join("amd-smi"); + // Not `fs::write` + `set_permissions`: this process is about to execute + // the file, and a write fd leaked into a parallel test's fork would make + // the first run fail with `ETXTBSY`. See `fake_executable`. + crate::fake_executable::write_fake_executable( + &fake_amd_smi, + format!("#!/bin/sh\ntouch '{}'\necho '{{}}'\n", marker.display()), + )?; + let readable_device = temp_root.join("stand-in-kfd"); + fs::write(&readable_device, b"")?; + + let timeout = Duration::from_secs(10); + let args = ["list", "--json"]; + + let blocked = amd_smi_json_with( + fake_amd_smi.as_os_str(), + &temp_root.join("absent-kfd"), + &args, + timeout, + ); + let ran_while_blocked = marker.exists(); + + let allowed = amd_smi_json_with(fake_amd_smi.as_os_str(), &readable_device, &args, timeout); + let ran_while_allowed = marker.exists(); + + let _ = fs::remove_dir_all(&temp_root); + + assert!( + blocked.is_err(), + "an unreadable KFD device must stop the launch" + ); + assert!( + !ran_while_blocked, + "amd-smi was spawned even though the pre-flight failed" + ); + assert!( + allowed.is_ok(), + "a readable device must let the launch through: {allowed:?}" + ); + assert!( + ran_while_allowed, + "the fake amd-smi never ran at all, so the blocked case proves nothing" + ); + Ok(()) + } + + /// No host off Linux has a `/dev/kfd`, so the pre-flight must wave the + /// launch through instead of reading the absent node as "unsafe". + /// `amd-smi.exe` is a supported Windows binary, so treating its absence as + /// a failure would disable GPU detection on every Windows host. + #[test] + #[cfg(not(target_os = "linux"))] + fn the_kfd_preflight_waves_through_hosts_that_have_no_kfd() { + assert!(kfd_readable(Path::new("/definitely/not/a/device/node"))); + } + + /// WSL has no `/dev/kfd` either, but unlike Windows cannot be waved through + /// by `cfg`: it compiles as Linux, so the real open runs, always fails, and + /// would decline every routed `amd-smi` launch on a supported platform — + /// silently, since both `serve` probes drop the error with `.ok()?`. + /// + /// The *conjunction* is what is pinned, not WSL identity: a WSL host whose + /// ROCDXG plumbing is not ready is one where `amd-smi` still hangs, so a + /// bare identity check would wave it through. The unready-WSL case + /// distinguishes the two; the bare-metal case is the control that stops the + /// test passing with the gate deleted. + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_preflight_waves_through_wsl_hosts_only_when_the_gpu_is_reachable() { + let absent = Path::new("/definitely/not/a/device/node"); + assert!( + kfd_readable_with(true, absent, absent), + "a ready WSL host has no /dev/kfd, so its absence must not decline the launch" + ); + assert!( + !kfd_readable_with(false, absent, absent), + "an unready WSL host — or a bare-metal one with no readable KFD — must be declined" + ); + } + + /// The conjunction `kfd_readable` hands to the wave-through, as a truth + /// table. + /// + /// The test above drives `kfd_readable_with` with literal booleans, so it + /// cannot see how that argument is produced: `&&` to `||`, or dropping + /// `has_usable_amd_gpu()`, leaves it green on a GPU-less lane. Only the + /// `(true, false)` and `(false, true)` rows here fail for those mutations. + #[test] + #[cfg(target_os = "linux")] + fn the_wsl_wave_through_needs_both_a_wsl_host_and_a_reachable_gpu() { + assert!( + wsl_wave_through(true, || true), + "ready WSL host waves through" + ); + assert!( + !wsl_wave_through(true, || false), + "a WSL host with no reachable GPU must not wave through: amd-smi \ + cannot work there, and `||` would allow it" + ); + assert!( + !wsl_wave_through(false, || true), + "bare metal must reach the device check even with a usable GPU, or \ + the gate this whole module exists for is skipped" + ); + assert!(!wsl_wave_through(false, || false)); + } + + /// The simulated-root routing, which `kfd_readable` applies to the real + /// `/dev/kfd` on the production path. + /// + /// Both halves are load-bearing and they fail in opposite directions. A + /// simulated root that hides the device must decline (so an E2E lane can + /// manufacture the declined branch on a machine that does have a GPU), and + /// a simulated root that *invents* a device must not allow (so no test hook + /// can ever start a real `amd-smi` on a machine that has no GPU — the veto + /// `hardware_root`'s module contract requires of this probe). + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto() -> Result<()> { + let _host_root = host_root_guard(); + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-root-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let present = temp_root.join("present-kfd"); + fs::write(&present, b"")?; + let absent = temp_root.join("absent-kfd"); + + let real_present_root_absent = kfd_readable_with(false, &present, &absent); + let real_absent_root_present = kfd_readable_with(false, &absent, &present); + let both_present = kfd_readable_with(false, &present, &present); + let _ = fs::remove_dir_all(&temp_root); + + assert!( + !real_present_root_absent, + "a simulated root that hides the device must decline the launch" + ); + assert!( + !real_absent_root_present, + "a simulated root must never be able to start amd-smi on a host with no real device" + ); + assert!( + both_present, + "the ordinary case must still pass, or the two above prove nothing" + ); + Ok(()) + } + + /// The gate's own probe must not be able to hang: an unbounded `open` would + /// move the D-state hazard into a synchronous call with *less* protection + /// than the spawn it replaced — no child to kill, no timeout. + /// + /// A FIFO makes the blocking case real rather than simulated: + /// `open(O_RDONLY)` on one with no writer blocks in the kernel + /// indefinitely, the observable shape of the hazard. The deadline assertion + /// is what distinguishes a bounded probe from an unbounded one — without + /// it, deleting the timeout would hang the suite instead of failing here, + /// and a hang is not a readable failure. The readable control stops the test + /// passing vacuously against a probe mutated to return `false` always. + /// + /// Uses a budget of its own rather than [`KFD_PROBE_BUDGET`]: it parks a + /// probe thread for the lifetime of the test binary, and spending a + /// production slot would quietly shrink the budget for every later test in + /// this process. + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_probe_declines_a_device_whose_open_blocks_instead_of_hanging() -> Result<()> { + static BUDGET: ProbeBudget = ProbeBudget::new(MAX_INFLIGHT_KFD_PROBES); + + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-open-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let Some(fifo) = make_fifo(&temp_root, "blocking-kfd") else { + let _ = fs::remove_dir_all(&temp_root); + return Ok(()); + }; + + let probe_timeout = Duration::from_millis(300); + // Watched on its own thread with a much looser deadline rather than + // called inline: a mutant dropping the bound would block forever on this + // FIFO, and called inline that is a CI timeout rather than a failure + // naming its cause. + let (verdict_tx, verdict_rx) = std::sync::mpsc::channel(); + thread::Builder::new() + .name("kfd-open-probe-test".to_owned()) + .spawn(move || { + let _ = + verdict_tx.send(device_readable_within_budget(&fifo, probe_timeout, &BUDGET)); + }) + .context("could not start the watched probe thread")?; + let watchdog = probe_timeout * 20; + let blocked = verdict_rx.recv_timeout(watchdog); + + let readable = temp_root.join("ordinary-kfd"); + fs::write(&readable, b"")?; + let allowed = device_readable_within_budget(&readable, probe_timeout, &BUDGET); + // The overrun probe's thread is still parked in `open` and stays that + // way until this test binary exits — deliberately, and the same leak + // `device_readable_within_budget` documents. Unlinking the FIFO does not + // release it, and nothing here needs it released. + let _ = fs::remove_dir_all(&temp_root); + + assert_eq!( + blocked, + Ok(false), + "a device whose open never returns must read as not readable within its \ + {probe_timeout:?} bound, so the gate declines; `Err` here means the probe had still \ + not returned after {watchdog:?} — an unbounded open relocates the D-state hang into \ + the gate itself" + ); + assert!( + allowed, + "an ordinary readable device must still pass, or the blocked case proves nothing" + ); + Ok(()) + } + + /// Parked probes are bounded, and a probe that cannot claim a slot declines + /// rather than opening anything on the calling thread. + /// + /// The escalation: against a wedged device every poll leaves a thread + /// parked in `open`, so an uncapped probe accumulates them until the process + /// runs out of threads — at which point a fallback to a direct blocking open + /// would reinstate exactly the hang the gate exists to refuse. + /// + /// Non-vacuous by construction. The capped probe is pointed at an ordinary + /// readable file, so an uncapped implementation — or one whose exhausted + /// branch fell back to a blocking open — answers `true`; only a probe that + /// refuses on an exhausted budget answers `false`. The control re-runs the + /// same device against a budget with a free slot and must get `true`, so a + /// mutant returning `false` always fails too. + /// + /// Uses budgets of its own, limit one: the parked FIFO probe spends the + /// slot and never frees it, so sharing the production budget would silently + /// shrink it for every other test in this process. + #[test] + #[cfg(target_os = "linux")] + fn the_kfd_probe_budget_declines_instead_of_parking_threads_without_bound() -> Result<()> { + static ONE_SLOT: ProbeBudget = ProbeBudget::new(1); + static SPARE: ProbeBudget = ProbeBudget::new(1); + + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-budget-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let Some(fifo) = make_fifo(&temp_root, "blocking-kfd") else { + let _ = fs::remove_dir_all(&temp_root); + return Ok(()); + }; + let readable = temp_root.join("ordinary-kfd"); + fs::write(&readable, b"")?; + + let probe_timeout = Duration::from_millis(300); + // Spends the only slot and never gives it back: the thread stays parked + // in `open` on a FIFO with no writer, which is precisely the state the + // cap is counting. + assert!( + !device_readable_within_budget(&fifo, probe_timeout, &ONE_SLOT), + "a blocking open must read as not readable" + ); + + let with_budget_spent = device_readable_within_budget(&readable, probe_timeout, &ONE_SLOT); + let with_a_free_slot = device_readable_within_budget(&readable, probe_timeout, &SPARE); + let _ = fs::remove_dir_all(&temp_root); + + assert!( + !with_budget_spent, + "with every probe slot held by a parked open, the pre-flight must decline rather \ + than open on the calling thread — that fallback is the hang this gate refuses" + ); + assert!( + with_a_free_slot, + "the same readable device must pass when a slot is free, or the decline above \ + proves nothing about the budget" + ); + Ok(()) + } + + /// A probe that returns must give its slot back — eventually. + /// + /// Nothing else asserts this. The budget test above only shows that an + /// exhausted budget declines; a `ProbeSlot::drop` that forgot to decrement + /// would satisfy it too, and would then decline every probe after the first + /// eight for the life of the process — the gate silently becoming "no GPU, + /// ever". + /// + /// "Eventually" is the honest invariant, and writing this test is what + /// showed why. The slot is released when the probe *thread* drops it, not + /// when the caller returns: `device_readable_within_budget` returns as soon + /// as `recv_timeout` has the answer, and the thread may not have unwound + /// yet. A first draft reused a one-slot budget back-to-back and failed on + /// the second call for exactly that reason — a real race, not a flake. The + /// bound below is therefore a poll, and the budget is sized so the test + /// cannot depend on the timing it just discovered. + #[test] + #[cfg(target_os = "linux")] + fn a_completed_probe_frees_its_slot() -> Result<()> { + static REUSED: ProbeBudget = ProbeBudget::new(1); + + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-kfd-slot-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let readable = temp_root.join("readable"); + fs::write(&readable, b"")?; + + assert!( + device_readable_within_budget(&readable, KFD_OPEN_TIMEOUT, &REUSED), + "a readable file must probe true" + ); + + // Poll rather than assert immediately: release happens on the probe + // thread's unwind. A `drop` that never decrements never reaches zero, + // which is the mutation this exists to catch. + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while REUSED.in_flight.load(Ordering::Acquire) != 0 { + assert!( + std::time::Instant::now() < deadline, + "the slot was never released: in_flight stayed at {}", + REUSED.in_flight.load(Ordering::Acquire) + ); + thread::sleep(Duration::from_millis(10)); + } + + // The budget is genuinely reusable once released, which is the point. + assert!( + device_readable_within_budget(&readable, KFD_OPEN_TIMEOUT, &REUSED), + "the one slot must serve a second probe after the first released" + ); + + let _ = fs::remove_dir_all(&temp_root); + Ok(()) + } + + /// A FIFO under `root`, or `None` on a host with no `mkfifo(1)`. + /// + /// Shelled out rather than calling `libc::mkfifo`, because the workspace + /// lints deny `unsafe_code`. A missing coreutils is not evidence about the + /// probe, so the caller skips rather than fails — but it says so on stderr + /// (see [`skip_notice`]), because a silent skip here is a green result for + /// a test that checked nothing. + #[cfg(target_os = "linux")] + fn make_fifo(root: &Path, name: &str) -> Option { + let fifo = root.join(name); + let made = Command::new("mkfifo") + .arg(&fifo) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + if matches!(&made, Ok(status) if status.success()) { + return Some(fifo); + } + skip_notice("no usable mkfifo(1), so no device whose open really blocks can be built here"); + None + } + + /// Announce that a test gave up without asserting its subject. + /// + /// Every conditional skip in this module goes through here. The tests that + /// have one are the ones that need a host feature (`mkfifo(1)`) or a host + /// *shape* (bare-metal Linux rather than WSL) to have anything to assert; + /// where that is missing they return `Ok(())`, which is indistinguishable + /// from a pass unless something says otherwise. `cargo test` shows this on + /// a failing run and under `--nocapture`; matches the `skip:` form already + /// used in `rocm-dash-daemon`'s server and end-to-end tests. + #[cfg(target_os = "linux")] + fn skip_notice(reason: &str) { + eprintln!("skip: {reason}; this assertion did NOT run"); + } + + /// Pins the production wiring the injected helpers cannot: that + /// [`amd_smi_json`] gates on [`KFD_DEVICE`] itself. + /// + /// Every other gate test reaches `amd_smi_json_with` with a device of its + /// own choosing, so swapping the constant out of the public wrapper — which + /// disables the gate for every production caller — leaves them all green. + /// Comparing the wrapper's observable verdict against the gate read + /// directly means the two can only agree while the wrapper passes the real + /// device through. + /// + /// Non-vacuous on *every* host in an `e2e-test-hooks` build: the empty + /// simulated root has no `dev/kfd`, so the gate declines whatever the + /// machine has, while a wrapper passing any other path — `/` being the + /// obvious mutant — re-roots to the root directory itself, which opens. + /// Without the feature it is non-vacuous on any host with no readable + /// `/dev/kfd` (every non-GPU CI runner). + #[test] + #[cfg(target_os = "linux")] + fn the_public_amd_smi_entry_point_gates_on_the_real_kfd_device() { + // Acquired by name rather than through `host_root_guard()`: this is the + // one test here that also calls `RestoredEnvVar::set`, and the + // env-mutation contract guard matches TEXT -- it recognises a + // serializer by the `_TEST_LOCK` suffix, which the helper's name hides. + // Same lock, spelled so the guard can see it. + let _host_root = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + #[cfg(feature = "e2e-test-hooks")] + let _simulated_host = { + let root = tempfile::tempdir().expect("a simulated host root"); + let variable = crate::test_env::RestoredEnvVar::set( + crate::hardware_root::TEST_HOST_ROOT_ENV, + root.path(), + ); + (variable, root) + }; + + let gate_allows = kfd_readable(Path::new(KFD_DEVICE)); + + // A deliberately bogus subcommand: if the gate allows the launch this + // still fails, but with a *launch* error rather than a decline, and it + // is the decline that is under test. The timeout is short because the + // pre-flight has already passed by the time anything is spawned. + let error = amd_smi_json(&["--rocm-cli-gate-probe"], Duration::from_secs(5)).err(); + let declined = error.is_some_and(|error| { + let message = format!("{error:#}"); + message.contains("declined to launch") && message.contains(KFD_DEVICE) + }); + + assert_eq!( + declined, !gate_allows, + "amd_smi_json must decline exactly when the gate on {KFD_DEVICE} does; a wrapper \ + that passes some other path declines on neither host" + ); + } + + /// The E2E path seam is an override, not a bypass: with the variable unset + /// the pre-flight device is `/dev/kfd` on every build, and with it set the + /// gate still *runs* — against the named path — rather than being skipped. + /// + /// The seam's whole justification is that it cannot weaken a shipped build, + /// and both halves are load-bearing: a `preflight_kfd_device` reading the + /// variable unconditionally would fail the unset half and hand any process + /// on the host a way to disable the gate, and one returning a readable path + /// outright — the bypass this deliberately is not — would fail the + /// declining half. + #[test] + #[cfg(target_os = "linux")] + fn the_e2e_kfd_path_seam_moves_the_gate_without_disabling_it() { + // The same serializer the simulated-root tests take, rather than a + // second one: both mutate the environment the pre-flight reads, and one + // lock for the pair cannot deadlock against itself. + let _env = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + assert_eq!( + preflight_kfd_device(), + Path::new(KFD_DEVICE), + "with no seam set, every build must gate on the real device" + ); + + #[cfg(feature = "e2e-test-hooks")] + { + let scratch = tempfile::tempdir().expect("a scratch dir"); + let absent = scratch.path().join("no-such-kfd"); + let _seam = crate::test_env::RestoredEnvVar::set(TEST_KFD_DEVICE_ENV, &absent); + + assert_eq!( + preflight_kfd_device(), + absent, + "an e2e-test-hooks build must honour the seam" + ); + + let error = amd_smi_json(&["--rocm-cli-gate-probe"], Duration::from_secs(5)) + .expect_err("an unopenable seam path must decline the launch"); + let message = format!("{error:#}"); + assert!( + message.contains("declined to launch") + && message.contains(&absent.display().to_string()), + "the seam must move the gate, not remove it; got: {message}" + ); + } + } + + /// A failing `amd-smi` that prints a payload must not paste the payload into + /// the error. `rocmd` puts this message in its snapshot note and the + /// GPU-metrics watcher copies that note into rewrite-the-whole-file JSONL + /// logs on every poll, which is how a 151K-character log line was produced. + #[test] + #[cfg(target_os = "linux")] + fn a_failing_amd_smi_cannot_paste_its_whole_output_into_the_error() -> Result<()> { + const PAYLOAD_CHARS: usize = 200_000; + + if is_wsl_host() { + // The gate waves WSL through, so the fake amd-smi would run and + // this would stop testing the quote bound rather than the gate. + skip_notice("this is a WSL host, where the gate does not reach the launch path"); + return Ok(()); + } + + let _host_root = host_root_guard(); + + let temp_root = + std::env::temp_dir().join(format!("rocm-cli-amd-smi-quote-{}", unix_time_millis())); + fs::create_dir_all(&temp_root)?; + let fake_amd_smi = temp_root.join("amd-smi"); + // See the sibling gate test: planting an executable this process then + // runs has to go through `fake_executable` or it races `ETXTBSY`. + crate::fake_executable::write_fake_executable( + &fake_amd_smi, + format!("#!/bin/sh\nyes x | head -c {PAYLOAD_CHARS} | tr -d '\\n' >&2\nexit 1\n"), + )?; + let readable_device = temp_root.join("stand-in-kfd"); + fs::write(&readable_device, b"")?; + + let error = amd_smi_json_with( + fake_amd_smi.as_os_str(), + &readable_device, + &["list", "--json"], + Duration::from_secs(30), + ) + .expect_err("a child that exits 1 must fail the call"); + let message = format!("{error:#}"); + let _ = fs::remove_dir_all(&temp_root); + + assert!( + message.chars().count() < PAYLOAD_CHARS / 10, + "the failure message quoted {} characters of child output unbounded", + message.chars().count() + ); + assert!( + message.contains("amd-smi list --json failed"), + "the bound must not cost the message its subject: {message}" + ); + assert!( + message.contains("[truncated]"), + "an over-long quote must say it was cut, not look like the whole story: {message}" + ); + Ok(()) + } + #[test] fn resolve_amd_smi_binary_prefers_home_rocm_venv_path() -> Result<()> { let temp_root = diff --git a/crates/rocm-core/src/hardware_root.rs b/crates/rocm-core/src/hardware_root.rs index 86a5e3d9e2..c90014f85a 100644 --- a/crates/rocm-core/src/hardware_root.rs +++ b/crates/rocm-core/src/hardware_root.rs @@ -42,14 +42,26 @@ //! machine. Process liveness under `/proc/` and `/dev/shm` sizing are //! about this process's environment, not the hardware. //! -//! Routed, but with the real device keeping a veto: the `rocm dash` `amd-smi` -//! pre-flight. It decides whether a real `amd-smi` process may start, so -//! `apps/rocm/src/dash.rs` passes `host_path("/dev/kfd")` to the dash daemon -//! and the collector requires both that path and the real `/dev/kfd` to be -//! readable. A simulated root can hide the GPU from the dashboard but cannot -//! start `amd-smi` on a machine without one. The WSL reachability verdict -//! still substitutes for the whole device check, as it always has, so the veto -//! does not cover it. +//! Routed, but with the real device keeping a veto: the two `amd-smi` +//! pre-flights. Each decides whether a real `amd-smi` process may start, so +//! each requires `host_path("/dev/kfd")` *and* the real `/dev/kfd` to be +//! readable. A simulated root can hide the GPU from them, but cannot start +//! `amd-smi` on a machine without one. They are: +//! +//! - the `rocm dash` collector's, where `apps/rocm/src/dash.rs` passes the +//! re-rooted path down to the dash daemon. `rocm-dash-collectors` does depend +//! on this crate (EAI-8457) and shares [`crate::device_readable_bounded`], but +//! it is still *handed* the re-rooted path rather than calling [`host_path`] +//! itself: the path has to come from the same resolution `apps/rocm` used for +//! its own probes, and the daemon that carries it in between +//! (`rocm-dash-daemon`) has no dependency on this crate to resolve it with. +//! The presentation layer, `rocm-dash-tui`, has no edge to this crate at all +//! and is the one the `crate_edges.rs` allowlist still forbids; +//! - the synchronous one behind `crate::amd_smi_json`, which `rocm serve` and +//! `rocmd` reach and which calls [`host_path`] itself. +//! +//! The WSL reachability verdict still substitutes for the whole device check in +//! both, as it always has, so the veto does not cover it. //! //! In a normal build [`host_path`] is the identity: the path is returned as //! written and the probes read the real host. Only a build compiled with the diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 69e2da9599..3fa5d8e971 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -62,7 +62,7 @@ mod test_support; pub mod uv; pub mod vllm_oom; pub mod windows_driver; -pub use amd_smi::resolve_amd_smi_binary; +pub use amd_smi::{amd_smi_json, device_readable_bounded, resolve_amd_smi_binary}; pub use automation::{ AuditEventRecord, AutomationEventRecord, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, EVENT_LOG_ROTATE_BYTES, EVENT_LOG_ROTATED_GENERATIONS, @@ -259,6 +259,27 @@ pub fn require_nonempty(value: &str, field_name: &str) -> Result<()> { /// captured output: stderr if non-blank after trimming, else stdout, else a /// literal "no output"; the result is trimmed. pub fn command_failure_detail(output: &std::process::Output) -> String { + command_failure_detail_or_else(output, || "no output".to_owned()) +} + +/// The stderr-then-stdout precedence behind [`command_failure_detail`], with the +/// both-blank fallback left to the caller. +/// +/// Exists so the precedence rule lives in exactly one place. [`amd_smi_json`] +/// needs the same ordering but a different last resort — `exit status ` +/// rather than `no output`, because a silent amd-smi's exit code is the only +/// diagnostic left — and inlining the two `is_empty` checks there to get it put +/// the same rule in two places. `fallback` is a closure because formatting an +/// `ExitStatus` is wasted work on the overwhelmingly common non-blank path. +/// +/// `pub(crate)`, not `pub`: the only caller outside this module is +/// [`amd_smi_json`], also in this crate. External callers that want the +/// precedence rule want [`command_failure_detail`], which is the public pair of +/// this and carries the fallback they would otherwise have to supply. +pub(crate) fn command_failure_detail_or_else( + output: &std::process::Output, + fallback: impl FnOnce() -> String, +) -> String { let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); if !stderr.is_empty() { return stderr; @@ -267,7 +288,7 @@ pub fn command_failure_detail(output: &std::process::Output) -> String { if !stdout.is_empty() { return stdout; } - "no output".to_owned() + fallback() } /// The `--gpu-memory-utilization` workaround for a shared/busy GPU. diff --git a/crates/rocm-core/src/probe_exec.rs b/crates/rocm-core/src/probe_exec.rs index 3d6d206ed1..86f807cfa4 100644 --- a/crates/rocm-core/src/probe_exec.rs +++ b/crates/rocm-core/src/probe_exec.rs @@ -164,8 +164,12 @@ pub(crate) const MEDIUM: Duration = Duration::from_secs(8); /// truncated. Slices on char boundaries (matching Python's `value[:n]`); a byte /// slice would panic when the cut lands inside a multibyte character. /// -/// `pub(crate)` because [`crate::process`] bounds its timeout quote with the -/// same marker, and a second copy here would be free to drift from it. +/// `pub(crate)` because two other places bound a quote with the same marker and +/// a second copy here would be free to drift from them: [`crate::process`] for +/// its timeout quote, and [`crate::amd_smi`] for the `amd-smi` failure quote, +/// which bounds raw child output for the same reason the examination bounds +/// captured text. Those two also share one character budget, declared once in +/// [`crate::process`]. /// (`host_gpu/diagnostics.rs`'s `truncate_diagnostic_line` is a different function with a /// different marker — plain `...` — not a third copy of this one.) pub(crate) fn truncate_to_chars(value: String, max_chars: usize) -> String { diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index d14f6f939f..374edcef0a 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -662,23 +662,25 @@ const DRAIN_GRACE: Duration = Duration::from_millis(200); /// only as an opaque error, so this is the only place that can still tell which /// of its characters came from the child. All four callers inherit it. /// -/// It bounds this message and nothing else. A caller that quotes the `Output` -/// handed back to it is quoting raw child output of its own, unbounded and -/// persisted the same way -- `capture_amd_smi_json`'s non-zero-exit `bail!` and -/// the daemon's `parse_sandbox_child_output` both do. Both predate this -/// constant and are outside its scope; of the two, #623 tracks the -/// `capture_amd_smi_json` quote. +/// It bounds this message by itself. A caller that quotes the `Output` handed +/// back to it is quoting raw child output of its own, persisted the same way, +/// so it has to apply a bound too -- which is why this is `pub(crate)` rather +/// than private: [`crate::amd_smi::amd_smi_json`] bounds its non-zero-exit +/// quote (the one #623 tracked) with *this* constant instead of declaring a +/// second one free to drift from it. The daemon's +/// `parse_sandbox_child_output` still predates both and is outside this scope. /// /// The bound matters because the message does not stop at a terminal. `rocmd` /// renders the whole error chain into the bridge snapshot's `note`, and the /// GPU-metrics watcher copies that note into the automation-event and audit /// JSONL files once a minute — stores that are never rotated or capped. /// Unbounded, a child that prints a megabyte and then stalls past its -/// timeout grows both logs by that much per minute, forever. +/// timeout grows both logs by that much per minute, forever. A 151K-character +/// JSONL line was reproduced from exactly that poller. /// /// Sized to still hold a real stderr backtrace: the point is to stop a /// payload-sized quote, not to shorten diagnostics. -const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; +pub(crate) const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; /// One captured stream: the bytes so far, whether its reader saw EOF, and the /// read error that stopped it, if one did. diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 32dd2a88f4..9b03c98e31 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -11,8 +11,9 @@ //! on-disk format at runtime, through its own `ServiceRecord` mirror struct //! in `registry.rs` rather than this type, and the serde test guards that //! contract. `rocm-engine-protocol` mentions `ManagedServiceRecord` solely in -//! a doc comment — `rocm-dash-core` and `rocm-dash-collectors` do not depend -//! on `rocm-core` at all. +//! a doc comment. `rocm-dash-core` does not depend on `rocm-core` at all; +//! `rocm-dash-collectors` does since EAI-8457, but only for the shared bounded +//! `/dev/kfd` probe, and it never reads this on-disk format either. use anyhow::{Context, Result}; use serde::Serialize; diff --git a/crates/rocm-dash-collectors/Cargo.toml b/crates/rocm-dash-collectors/Cargo.toml index a0033cdc82..f8e7a826eb 100644 --- a/crates/rocm-dash-collectors/Cargo.toml +++ b/crates/rocm-dash-collectors/Cargo.toml @@ -12,6 +12,7 @@ workspace = true [dependencies] rocm-dash-core = { path = "../rocm-dash-core" } +rocm-core = { path = "../rocm-core" } serde = { version = "1", features = ["derive"] } serde_json = "1" csv = "1" diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 5c36933d76..0d258952e1 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -51,11 +51,14 @@ impl AmdSmiCollector { /// `gpu_reachable` lets the pre-flight pass without a readable `/dev/kfd` /// (e.g. on WSL, where the real signal is `rocm_core::has_usable_amd_gpu()` /// rather than any single device node this crate could probe directly). - /// This crate deliberately does not depend on `rocm-core` to compute that - /// verdict itself; the caller (`apps/rocm`, which already depends on it) - /// threads it through instead, so on WSL, `serve`/`examine`/the dashboard - /// never disagree about whether a host's GPU is usable. On bare metal the - /// caller always passes `false`, so this is still a plain `/dev/kfd` check. + /// This crate does depend on `rocm-core` (EAI-8457, for the shared bounded + /// device probe below), so computing the verdict here would compile — it is + /// still threaded in from the caller (`apps/rocm`) on purpose. The verdict + /// is a *decision about this host* that `serve`, `examine` and the dashboard + /// must not reach independently; recomputing it in the collector is how the + /// three come to disagree about whether a host's GPU is usable, not how they + /// stay in step. On bare metal the caller always passes `false`, so this is + /// still a plain `/dev/kfd` check. /// /// `host_kfd_device` is where the caller's hardware probes put `/dev/kfd` /// (`rocm_core::host_path`'s answer, threaded in for the same reason as @@ -94,12 +97,14 @@ impl AmdSmiCollector { gpu_reachable: bool, host_kfd_device: &Path, ) -> Option { - if !preflight_passes( + if !preflight_passes_off_thread( skip_device_preflight, gpu_reachable, Path::new(KFD_DEVICE), host_kfd_device, - ) { + ) + .await + { return None; } let me = Self { @@ -191,6 +196,46 @@ impl AmdSmiCollector { } } +/// [`preflight_passes`] moved off the async worker thread. +/// +/// The device branch of the pre-flight blocks: `device_accessible` waits on a +/// bounded `open`, up to `rocm_core`'s 2s `KFD_OPEN_TIMEOUT` per probe, and +/// under a simulated host root there are two probes, so ~4s. The only caller is +/// an `async fn` reached from a `tokio::spawn` in `rocm-dash-daemon`'s run +/// loop, and AGENTS.md puts blocking work on the blocking pool rather than on +/// an async worker. +/// +/// Deliberately no `timeout` wrapped around the `spawn_blocking`: that would +/// drop only the `JoinHandle` and leave the blocking thread running, and the +/// probe is already bounded from the inside, which is the bound that matters. +/// +/// The two non-device branches short-circuit *before* the thread hop, so a +/// caller that skips the pre-flight or already knows the GPU is reachable pays +/// nothing and -- relevant to the `#[doc(hidden)]` skip entry point -- never +/// needs a runtime with a blocking pool at all. +/// +/// A `JoinError` reads as "not readable", matching +/// `device_readable_within_budget`: every branch that fails to get an answer +/// declines rather than falling back to a blocking open on the caller's thread. +async fn preflight_passes_off_thread( + skip_device_preflight: bool, + gpu_reachable: bool, + real_kfd_device: &Path, + host_kfd_device: &Path, +) -> bool { + if skip_device_preflight || gpu_reachable { + return true; + } + // Owned: `spawn_blocking` needs a `'static` closure. + let real_kfd_device = real_kfd_device.to_path_buf(); + let host_kfd_device = host_kfd_device.to_path_buf(); + tokio::task::spawn_blocking(move || { + preflight_passes(false, false, &real_kfd_device, &host_kfd_device) + }) + .await + .unwrap_or(false) +} + /// The device pre-flight passes if the caller already skipped it, already /// knows the GPU is reachable by some other means (e.g. WSL, via /// `rocm_core::has_usable_amd_gpu()`), or the bare-metal device node itself is @@ -224,8 +269,14 @@ fn preflight_passes( && (host_kfd_device == real_kfd_device || device_accessible(host_kfd_device))) } +/// Shared with `rocm-core`'s `/dev/kfd` pre-flight rather than restated here. +/// +/// A plain `OpenOptions::open(..).is_ok()` would leave the dashboard collector +/// as the one `amd-smi` launch site that can still block forever on a wedged +/// device. The shared probe carries the 2s bound and the in-flight budget with +/// it, so this site cannot drift away from the gate it is part of. fn device_accessible(path: &Path) -> bool { - std::fs::OpenOptions::new().read(true).open(path).is_ok() + rocm_core::device_readable_bounded(path) } fn val_u64(v: Option<&Value>) -> Option { @@ -584,6 +635,94 @@ mod tests { assert_eq!(m.get("gpu-1"), Some(&600.0), "N/A falls through too"); } + /// A FIFO under `root`, or `None` (with a visible skip notice) on a host + /// with no `mkfifo(1)`. + /// + /// Shelled out rather than calling `libc::mkfifo`, because the workspace + /// lints deny `unsafe_code` — the same reason, and the same shape, as + /// `rocm_core::amd_smi`'s test helper of this name. + #[cfg(target_os = "linux")] + fn make_fifo(root: &Path, name: &str) -> Option { + let fifo = root.join(name); + let made = std::process::Command::new("mkfifo") + .arg(&fifo) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + if matches!(&made, Ok(status) if status.success()) { + return Some(fifo); + } + eprintln!( + "skip: no usable mkfifo(1), so the off-thread pre-flight check cannot be exercised \ + here; this assertion did NOT run" + ); + None + } + + /// The blocking pre-flight must not be run on the async worker. + /// + /// `device_readable_bounded` parks its caller for up to `rocm_core`'s 2s + /// `KFD_OPEN_TIMEOUT`, and the only production caller is an `async fn` + /// reached from `rocm-dash-daemon`'s `tokio::spawn`. A single-worker + /// (`current_thread`) runtime is what makes the difference observable: a + /// ticker task can only make progress while the pre-flight is *not* + /// occupying the one worker, so an inline `preflight_passes` — the shape + /// this had before — records zero ticks, while the `spawn_blocking` version + /// lets the ticker run for the whole ~2s probe. + /// + /// The device is a FIFO with no writer, the same real blocking `open` the + /// `rocm-core` probe tests use: against an ordinary missing path the open + /// fails instantly and the test could pass without proving anything. + /// + /// Linux-only: off Linux `device_readable_bounded` is a direct unbounded + /// open, which on this FIFO would never return at all. + #[tokio::test(flavor = "current_thread")] + #[cfg(target_os = "linux")] + async fn the_device_preflight_does_not_run_on_the_async_worker() { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let dir = tempfile::tempdir().unwrap(); + let Some(fifo) = make_fifo(dir.path(), "blocking-kfd") else { + return; + }; + + let ticks = Arc::new(AtomicUsize::new(0)); + let ticker_ticks = Arc::clone(&ticks); + let ticker = tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_millis(20)).await; + ticker_ticks.fetch_add(1, Ordering::Relaxed); + } + }); + + let passed = preflight_passes_off_thread(false, false, &fifo, &fifo).await; + ticker.abort(); + let observed = ticks.load(Ordering::Relaxed); + + assert!( + !passed, + "a device whose open never returns must fail the pre-flight, or the timing \ + assertion below proves nothing" + ); + assert!( + observed >= 5, + "the async worker was parked by the pre-flight: a 20ms ticker got {observed} ticks \ + during a ~2s blocking probe, so the probe is not on the blocking pool" + ); + } + + /// The two non-device branches answer without a thread hop, so they stay + /// usable on a runtime with no blocking pool — notably the `#[doc(hidden)]` + /// skip entry point the daemon integration tests use. + #[tokio::test(flavor = "current_thread")] + async fn the_short_circuit_branches_need_no_blocking_pool() { + let missing = Path::new("/nonexistent/kfd"); + assert!(preflight_passes_off_thread(true, false, missing, missing).await); + assert!(preflight_passes_off_thread(false, true, missing, missing).await); + } + const SAMPLE_METRIC: &str = r#"{ "gpu_data": [ { diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 7d526b9577..2558dfd20d 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -90,10 +90,13 @@ pub struct RunnerOptions { /// Precomputed GPU-reachability verdict (from `rocm_core::has_usable_amd_gpu`) /// that lets the amd-smi device pre-flight pass without a readable /// `/dev/kfd` — the WSL case, where that verdict is the same one `serve` - /// and `examine` already act on. `rocm-dash-daemon`/`rocm-dash-collectors` - /// deliberately don't depend on `rocm-core` to compute this themselves; - /// the caller (`apps/rocm`) does and passes the answer through. `false` - /// (the default) preserves the bare-metal-only `/dev/kfd` check. + /// and `examine` already act on. This crate has no `rocm-core` dependency + /// to compute it with; `rocm-dash-collectors`, which does have one since + /// EAI-8457, still takes it threaded through rather than recomputing it, so + /// the dashboard cannot reach a different verdict about the same host than + /// `serve` and `examine` did. The caller (`apps/rocm`) computes it once and + /// passes the answer through both. `false` (the default) preserves the + /// bare-metal-only `/dev/kfd` check. pub amd_smi_gpu_reachable: bool, /// Where the caller's hardware probes put `/dev/kfd`. /// diff --git a/docs/architecture.md b/docs/architecture.md index 724bc81c0f..350d2e42e7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,7 +57,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) — mechanical relocation, no owned types: the answers a freeform request can get without running a command — `render_freeform_read_only_answer`, which answers a single approval-free `examine`, `comfyui status` or `comfyui logs` action, and `render_freeform_examine_answer`, the plain-language ROCm status it prints for `examine`. `run_freeform` in [`main.rs`](../apps/rocm/src/main.rs) is the only caller of the first, which is `pub(crate)` and re-imported there; the second is private. Besides `chat.rs` (`any_substring`, re-exported there from [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs)), it depends on `comfyui.rs`, `runtimes_cmd.rs` and `therock.rs`, none of which calls back into it. No unit test calls either function directly. - [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) — mechanical relocation, dispatch-adjacent, owns a `pub(crate)` type (as of Phase 6e's follow-up, ROCMAI-726; see the Phase 6a variant note above for the owned-types pattern). Holds the freeform natural-language planner's plan-*execution*/consent half that [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) below deliberately left out: `validate_freeform_execution_action()`, `prepare_freeform_execution()`, `apply_freeform_execution_consent()`, `render_freeform_execution_header()`, and the `pub(crate)` `FreeformExecution` struct that `prepare_freeform_execution` returns and `render_freeform_execution_header` consumes. `main.rs`'s `execute_freeform_next_action()` and `run_freeform()` stay in `main.rs` — both are dispatch-level, not part of deciding whether a plan's next action is safe to execute — and `execute_freeform_next_action()` calls in via the unqualified re-import `use crate::freeform_execution::{prepare_freeform_execution, render_freeform_execution_header};` (`run_freeform()` only calls `execute_freeform_next_action()`, never this module directly), matching [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s dispatch-adjacent form rather than `freeform_planner.rs`'s own qualified-path form. `prepare_freeform_execution` takes the already-built `plan` by value rather than rebuilding it from `request`/`paths`/`config` (ROCMAI-725's single-build guarantee, inherited unchanged from `main.rs`); `prepare_freeform_execution`/`render_freeform_execution_header` are `pub(crate)` for that cross-module call; `FreeformExecution` was already `pub(crate)` (read directly by `main.rs` and this module's own tests); `validate_freeform_execution_action`/`apply_freeform_execution_consent` have no caller outside this module and stay module-private, unchanged from before the move. The cluster's six regression tests moved out of `main.rs`'s shared `mod tests` into this module's own `mod tests`, inheriting the qualified `freeform_planner::` path they already used to reach `freeform_planner.rs`'s items as `main.rs`'s former tests — this module reaches in from outside using the same qualified path `main.rs` and `providers.rs`'s tests use (unlike `chat.rs`/`comfyui.rs`, which import it unqualified). - [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction`, `StructuredRequestPlan` and `PlannedToolCall`, which are `pub(crate)`, and `ProviderPlannerToolCall`, `pub(crate)` together with `validate_provider_planner_tool_call` and `planner_failure_note` so [`chat.rs`](../apps/rocm/src/chat.rs)'s command-preview property tests can drive the planner's guards), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow ([`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `render_launch_summary` → `host_default_engine` is another call into this module, unrelated to planning: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, while [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs)'s `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` from [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) directly as a no-CPU-fallback safety check on provider-proposed `serve` args; `validate_provider_planner_tool_call` calls that same function in its own `serve` branch, and hands an `install sdk` call to `tool_policy::validate_chat_rocm_command_safety`. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context` and `provider_planner_response_to_plan` are `pub(crate)` for these reasons, as is `plan_driver_request` (read from [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs)'s test module); `format_structured_tool_call` lives in [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) and is re-exported here with `pub(crate) use`, so the callers above keep their `freeform_planner::` path, while the planner's own argv lookups call `rocm_argv`, its phrase and engine readers call [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) and its safety check calls `tool_policy.rs`, directly: the planner no longer imports `chat`, and only `chat` imports the planner (`plan_freeform_request_with_context`); `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller. -- [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`). `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. +- [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`).Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. - [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm install`'s command handler (`install()`, for the `sdk` and `driver` targets), the `--version`/`--build-date` selector parsing and display it uses (`therock_install_version_selector`, `therock_install_version_selector_display`) and `sdk_install_request`, which maps the parsed `install sdk` arguments onto a `therock::SdkInstallRequest` and moved here from [`chat.rs`](../apps/rocm/src/chat.rs) with its two tests. `InstallTarget` and `InstallFormat` live in [`cli.rs`](../apps/rocm/src/cli.rs). `install()` is `pub(crate)` and re-imported in `main.rs` for `dispatch()`; `therock_install_version_selector` is `pub(crate)` too, and `chat.rs`'s `render_install_sdk_dry_run_for_args` reaches it as `crate::therock_install_version_selector` through the same kind of re-import, which is why `sdk_install_request` lives here and not in `chat.rs`: this module does not depend on `chat.rs`. `sdk_install_request` keeps the `pub(crate)` it had, although only `install()` and the tests call it now. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. - [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. @@ -159,7 +159,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): -- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups; invoking `amd-smi` stays with its callers. +- [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `info`, since the default filter is `warn,rocm=info` and a decline on a healthy host is otherwise indistinguishable from "no GPU"), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151) automation-daemon runtime state and its on-disk event/proposal/audit records, and the policy applied to each store (which are rotated, and the proposal compaction) — see the module's own doc comment for what it owns. The generic JSONL machinery under those stores lives in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs). The dependency runs one way: this module reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), and [`config.rs`](../crates/rocm-core/src/config.rs) imports nothing from here, because the built-in watcher catalog (whose entries carry a default `WatcherMode`, and which `RocmCliConfig::watcher_enabled` and `effective_watcher_mode` take as an argument) lives in [`config.rs`](../crates/rocm-core/src/config.rs) beside `WatcherMode`. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. diff --git a/docs/testing.md b/docs/testing.md index eb9f36502e..b3242c38b4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2667,6 +2667,140 @@ for them, including for the fallback; their parsing is covered by the unit tests above, and the fallback itself by `hardware_root`'s own unit tests, since both now read through the same helper with an identity resolver. +## amd-smi KFD pre-flight + +Every synchronous `amd-smi` launch goes through +[`rocm_core::amd_smi_json`](../crates/rocm-core/src/amd_smi.rs), which first +tries to open `/dev/kfd` for reading and declines the launch outright when it +cannot. Against a device in the wrong kernel state `amd-smi` blocks in +uninterruptible sleep, where no signal and therefore no caller timeout reaches +it, so declining is the only outcome that bounds `rocmd`'s telemetry poll and +`serve`'s GPU probes. The open is itself bounded (2s) and the number of probes +that may be parked at once is capped (8), so the gate's own probe cannot inherit +the stall it exists to refuse. The dashboard collector +([amd_smi.rs](../crates/rocm-dash-collectors/src/amd_smi.rs)) gates its own +launch on the same device through the same shared probe, +`rocm_core::device_readable_bounded`. + +The gate declines on *any* unreachable device, including the fast errno cases +(absent node, permission denied) that cannot hang anything. That is wider than +the hazard and deliberate; `kfd_readable`'s doc comment says why, and names what +`serve` still answers from on such a host. + +```bash +cargo test -p rocm-core --lib amd_smi +cargo test -p rocm-core --lib --features e2e-test-hooks -- the_e2e_kfd_path_seam_moves_the_gate_without_disabling_it the_public_amd_smi_entry_point_gates_on_the_real_kfd_device +``` + +The unit tests cover the gate in isolation: that it stops the *spawn* rather +than reporting a failure afterwards (a stub `amd-smi` leaves a marker file, and +its absence is the evidence), that the probe declines rather than hangs against +a device whose `open` blocks (a FIFO with no writer), that the in-flight budget +declines rather than parking threads without bound, that the WSL and off-Linux +wave-throughs hold, and that the public entry point gates on `/dev/kfd` itself +rather than on some other path. + +**The `ROCM_E2E_KFD_DEVICE` seam.** `rocm_core::preflight_kfd_device` reads this +variable and uses the path it names as the device the pre-flight opens. Only a +build carrying the `e2e-test-hooks` feature honours it; a released artefact has +no override compiled in and always gates on `/dev/kfd`. It moves the gate rather +than removing it, so a scenario pointing it at an unopenable path gets the +declining branch and one pointing it at an ordinary file gets the allowed +branch, while with the variable unset the behaviour is production's on every +build. + +`ROCM_CLI_TEST_HOST_ROOT` cannot stand in for it. The gate keeps the real +`/dev/kfd`'s veto on purpose, so a simulated root can only ever make it decline, +never allow — which is what stops a test hook starting a real `amd-smi` on a +machine with no GPU, and also what makes the *allowed* branch unreachable under +a simulated root on the GPU-less lane that runs per PR. + +`examine-31` in +[examine.feature](../tests/e2e-cucumber/features/examine.feature) covers the +declining branch end to end. It plants a stub `amd-smi` that would answer the +telemetry probe perfectly and records that it ran, points the seam at a path +inside the scenario's isolated root that is never created, and checks that +`rocmd`'s bridge snapshot still exits 0, reports `amd_smi_available: false`, +carries a note saying the launch was *declined*, and names the device **this +run** pointed the pre-flight at. Both of the last two are what make it +non-vacuous: without the marker the scenario would pass against a gate that had +been deleted and merely failed the spawn, and without the named device it would +pass against a build that ignored the seam, since the GPU-less lane has no +readable `/dev/kfd` of its own either. Linux-only, like its `examine-24` to +`examine-26` telemetry siblings: the stub is a shell script, and the gate is a +wave-through on every other platform. + +```bash +cargo xtask e2e -- -n 'examine-31 ' +``` + +No scenario drives the `rocm serve` / `rocm model` call sites through the gate. +Those probes degrade to service-state-only GPU selection rather than failing, +so there is no user-visible *output* for a scenario to assert on. + +Be precise about what does cover them, because the two halves have different +guards. What the call sites do **with a missing count** is covered by +`gpu_policy.rs`'s unit tests — `effective_gpu_count_prefers_amd_smi_then_falls_back_to_vram_rows` +pins the `None` fall-through. That the call sites **route through the +chokepoint at all** is *not* covered by any unit test: it is guarded by +`xtask/src/amd_smi_launch_contract.rs`, a static text scan that fails the build +when a file outside its allowlist calls `resolve_amd_smi_binary`. A rewrite that +kept the count behaviour but spawned `amd-smi` directly would be caught by that +scan and by nothing else. The decline +itself is no longer discarded — `gpu_policy.rs`'s `amd_smi_probe_json` logs it +at `info` with the full error chain, which is what makes "why did `--gpu auto` +not rank my GPUs?" answerable without re-running. It is not `warn`, because on +a GPU-less host it fires on every `serve`, where it is the expected answer and +not a fault; and not `debug`, because the default filter is `warn,rocm=info`, +so a `debug` line is invisible unless the user already suspected this — the +wrong way round for the case that needs it, where the 2s open bound trips on a +slow but healthy host and the decline is indistinguishable from "no GPU". + +### Keeping new launch sites inside the gate + +A chokepoint only holds while every launch site uses it, and nothing in a +behavioural test suite fails when one stops: the regression is the *absence* of +a call. [amd_smi_launch_contract.rs](../xtask/src/amd_smi_launch_contract.rs) is +the structural guard for that. It scans every `.rs` file in the tree (comments +stripped) for calls to `rocm_core::resolve_amd_smi_binary` — the only way to +find the binary, since the managed SDK ships `amd-smi` inside the runtime +wheel's bin directory rather than on `PATH` — and fails unless the calling file +is one of two allowlisted, justified sites. Reverting either `gpu_policy.rs` +probe to a raw `Command::new(resolve_amd_smi_binary())` now fails this test by +name. It is bidirectional like the crate-edge allowlist: an entry whose file +stops calling the resolver is stale and fails too. + +```bash +cargo test -p xtask amd_smi_launch_contract +``` + +It is a source-text scan, not a call-graph analysis: a resolver reached through +a function pointer or a re-export under another name is outside what it can +see. It closes the cheap, likely regression and claims nothing more. + +### What the gate is, and is not + +**It is a mitigation heuristic, not a proof.** The gate establishes exactly one +thing: that `open("/dev/kfd")` returned within 2s. The D-state hang it exists to +prevent is attributed to `amd-smi`'s *later* `ioctl`s against a wedged device, +not to the open — so a device whose open succeeds and whose next ioctl wedges is +not covered, and the 2s bound is a chosen number rather than a measured one. No +run against a genuinely unhealthy driver was performed for this work: the GPU +lane has no `amd-smi` at all, and the FIFO fixtures model a blocking `open`, +which is the closest reachable analogue rather than the real failure. What the +gate does buy is that the common precondition for the hazard — a device the +kernel will not hand out — refuses the spawn instead of betting on a timeout +that cannot reach a process in uninterruptible sleep. + +**The decline is broader than the hazard, and broader than the ticket.** Any +unreachable `/dev/kfd` declines, including the fast errno cases that cannot hang +anything: an absent node (EAI-8457 states the scenario does not apply with no +`/dev/kfd`) and `EACCES` for a user outside the `render`/`video` groups. +`gpu_vram_usage` still falls back to the DRM sysfs counters on such a host, but +`detect_gpu_count` has **no** sysfs fallback and returns `None`, so `--gpu auto` +there ranks from the sysfs VRAM rows alone. `kfd_readable`'s doc comment carries +the same statement next to the code. + ## Windows Runtime Preflight On Windows, `rocm diagnose` compares the Visual C++ runtime in `System32` diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 4de33dc7b8..54fcf5c51b 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -470,3 +470,41 @@ Feature: GPU detection and system inspection Given a machine with a ROCm install that was not set up by the CLI When the user inspects the system in machine-readable form Then the inspection reports a current install-method name + + # The other side of examine-24/25/26, and the user-observable half of the + # `/dev/kfd` pre-flight (EAI-8457): where those three prove a healthy device + # still reaches amd-smi, this proves an unopenable one stops the launch from + # happening at all. (Declared here at the end rather than beside them because + # the scenario index is per-feature and in declaration order.) On a host in + # that state amd-smi can park in + # uninterruptible sleep, where no signal and therefore no timeout reaches it, + # so "declined" is the only outcome that bounds the daemon's 60s telemetry + # poll. The snapshot degrades; the command still succeeds. + # + # Non-vacuity is the whole difficulty here, and it is carried in two places. + # The stub amd-smi would answer this probe perfectly and records that it ran, + # so the marker's absence is what distinguishes "refused before the spawn" + # from "spawned and failed" -- deleting the gate makes this scenario fail + # rather than merely changing a message. And the note must name the device + # this run pointed the pre-flight at: the lane that runs this per PR has no + # `/dev/kfd` of its own, so a build that ignored the seam would decline for + # the host's own reasons and otherwise look identical. + # + # The seam is `ROCM_E2E_KFD_DEVICE`, honoured only by an `e2e-test-hooks` + # build (`rocm_core::preflight_kfd_device`), and it moves the device the gate + # opens rather than skipping the gate. `ROCM_CLI_TEST_HOST_ROOT` cannot stand + # in for it: the gate keeps the real node's veto on purpose, so a simulated + # root can only ever make it decline -- which is this branch, but leaves + # examine-24/25/26 unable to reach amd-smi on a GPU-less runner at all. + # Linux-only, like its three siblings: the stub is a shell script, and the + # gate is a wave-through on every other platform. + # `@requires-bare-metal`, not just `@requires-os:linux`: WSL2 reports an + # os_family of `linux`, so the Linux tag matches there — but on a WSL host + # with a usable GPU `kfd_readable`'s wave-through short-circuits the device + # open entirely, and the decline this scenario asserts never happens. The + # premise is a host where the `/dev/kfd` open is what decides. + @id:examine-telemetry-declines-an-unopenable-kfd-device @requires-os:linux @requires-bare-metal + Scenario: examine-31 - An unopenable KFD device stops amd-smi being launched at all + Given the KFD device cannot be opened + When the daemon gathers a bridge snapshot + Then the snapshot says amd-smi was never launched and names the device diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 539644f521..954490c68b 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -396,6 +396,34 @@ Feature: Model serving And the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + # The `/dev/kfd` pre-flight (EAI-8457) is covered by `examine-31` in + # `examine.feature`, not here, and this note records why the scenario lives + # there rather than alongside the serve probes it also guards. + # The behaviour: on a Linux host where the KFD device cannot be opened, + # `rocm_core::amd_smi_json` declines the launch outright, so `serve`'s + # `gpu_vram_usage` and `detect_gpu_count` degrade to service state alone where + # they previously attempted the spawn. The same chokepoint serves `rocmd`'s + # bridge snapshot, and that surface is the one a scenario can observe + # directly: the snapshot reports `amd_smi_available` and a note that says why, + # as JSON, in one command. `serve`'s probes discard their error with `.ok()?` + # by design, so the declined branch is visible there only as the absence of a + # GPU-aware placement -- which a GPU-less lane cannot tell apart from having + # no GPU. One scenario on the observable surface beats a weaker one here. + # Reaching either branch needs the `e2e-test-hooks` seam + # `ROCM_E2E_KFD_DEVICE` (`rocm_core::preflight_kfd_device`), which moves the + # device the gate opens. `ROCM_CLI_TEST_HOST_ROOT` cannot substitute: the gate + # keeps the real node's veto on purpose, so a simulated root can make it + # decline but never allow -- see + # `the_kfd_gate_honours_a_simulated_root_but_keeps_the_real_device_veto`. + # Also covered at unit level in `rocm-core`, each confirmed non-vacuous by + # mutation: `the_kfd_gate_stops_amd_smi_before_it_is_spawned` asserts the gate + # stops the *spawn* (a fake `amd-smi` leaves a marker file that must not + # appear), with the same fixture re-run against a readable device to prove the + # fake would otherwise have executed; + # `the_public_amd_smi_entry_point_gates_on_the_real_kfd_device` pins that the + # production wrapper gates on the real device; and + # `the_e2e_kfd_path_seam_moves_the_gate_without_disabling_it` pins that the + # seam above relocates the probe instead of skipping it. # Regression test for the reuse exemption (ROCMAI-419 review). Under the # default `gpu_required` policy a host with no usable AMD GPU refuses `rocm # serve` outright, before any engine work. Re-issuing `serve` against a managed diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index f3a948d5e3..17b8a97976 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -40,6 +40,18 @@ const STUB_MARKET_NAME: &str = "stub-accelerator"; /// otherwise, per `command_failure_detail` — not stdout unconditionally. const STUB_FIRST_GPU_MARKET_NAME: &str = "stub-accelerator-gpu0"; +/// The `e2e-test-hooks` seam that names the device node `rocm_core`'s `amd-smi` +/// pre-flight opens (EAI-8457). +/// +/// Spelled out rather than imported: `rocm_core` exports it only to its own +/// crate, because a `pub` constant would read as an invitation for production +/// code to consult it. The harness has to agree with that string by hand, and +/// the scenario that points it at an absent path is what notices if it drifts: +/// it asserts the decline names *that* path, so a build that ignored the +/// variable would decline naming `/dev/kfd` and fail, rather than passing on a +/// gate that declined for the host's own reasons. +const KFD_DEVICE_SEAM: &str = "ROCM_E2E_KFD_DEVICE"; + /// Where the `amd-smi` stub and its payload live inside the isolated root. /// /// Derived rather than carried on the World so the Given and the When agree on @@ -1751,6 +1763,44 @@ async fn stub_amd_smi_that_stalls_only_on_monitor(world: &mut E2eWorld) { ); } +/// Where the stub records that it ran, so the decline can be asserted as "never +/// spawned" rather than merely "reported unavailable". +fn stub_ran_marker(world: &E2eWorld) -> PathBuf { + stub_bin_dir(world).join("amd-smi-ran") +} + +#[given("the KFD device cannot be opened")] +async fn kfd_device_cannot_be_opened(world: &mut E2eWorld) { + // A stub that *would* answer perfectly well, and a marker that records it + // running. The point of the gate is that neither ever happens: a scenario + // whose stub could not have answered anyway would pass against a gate that + // had been deleted. + let payload = write_stub_payload(world); + let marker = stub_ran_marker(world); + write_amd_smi_stub( + world, + &format!( + "touch {marker}\nexec cat {payload}", + marker = marker.display(), + payload = payload.display() + ), + ); + + // Inside the isolated root and never created. The decline must name this + // path, which is what proves the daemon read the seam rather than falling + // back to `/dev/kfd` -- a host-dependent answer that would make the + // scenario pass for the wrong reason on a lane with no GPU. + let absent = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("absent-kfd-device"); + world + .command_env + .push((KFD_DEVICE_SEAM, absent.into_os_string())); +} + #[when("the daemon gathers a bridge snapshot")] async fn daemon_gathers_bridge_snapshot(world: &mut E2eWorld) { let rocmd = crate::rocmd_binary(); @@ -1789,6 +1839,28 @@ async fn daemon_gathers_bridge_snapshot(world: &mut E2eWorld) { .expect("failed to build the stub PATH"); command.env("PATH", path); + // The `/dev/kfd` pre-flight in `rocm_core::amd_smi_json` (EAI-8457) refuses + // to spawn `amd-smi` at all when the device cannot be opened, and the lane + // that runs these scenarios per PR has no `/dev/kfd`. Without a seam the + // stub would never be reached and all three scenarios would assert against + // a decline instead of the pipe-drain behaviour they are about. + // + // `ROCM_CLI_TEST_HOST_ROOT` cannot do this job: the gate deliberately keeps + // the real node's veto, so a simulated root can only ever make it decline. + // The `e2e-test-hooks` path seam can, and it moves the gate rather than + // disabling it -- the daemon still opens the file named here before it + // spawns anything, which is why a plain readable file is enough to stand in + // for a healthy device. The companion scenario points the same variable at + // an absent path to exercise the other branch. + let kfd_stand_in = isolated_home.join("kfd-stand-in"); + std::fs::write(&kfd_stand_in, b"").expect("failed to plant the stand-in KFD node"); + command.env(KFD_DEVICE_SEAM, &kfd_stand_in); + // Applied after the default above so a scenario that pushed its own value + // -- the declined branch does -- wins over it. + for (key, value) in &world.command_env { + command.env(key, value); + } + let output = command .output() .expect("failed to run rocmd bridge-snapshot"); @@ -1907,6 +1979,60 @@ async fn assert_stalled_probe_note_is_bounded(world: &mut E2eWorld) { ); } +#[then("the snapshot says amd-smi was never launched and names the device")] +async fn assert_snapshot_declined_the_launch(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no rocmd exit status recorded"); + let stdout = world.cli_output.as_ref().expect("no rocmd output recorded"); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + // A declined probe degrades the GPU section the same way a timed-out one + // does; it must not fail the command. + assert_eq!(rc, 0, "rocmd bridge-snapshot failed:\n{stdout}\n{stderr}"); + + let snapshot: serde_json::Value = serde_json::from_str(stdout) + .unwrap_or_else(|error| panic!("bridge-snapshot did not emit JSON: {error}\n{stderr}")); + let gpu = &snapshot["gpu"]; + + assert_eq!( + gpu["amd_smi_available"], + serde_json::Value::Bool(false), + "a launch that was refused cannot have reported a GPU: {gpu}" + ); + let note = gpu["note"] + .as_str() + .unwrap_or_else(|| panic!("a degraded snapshot must explain itself: {gpu}")); + assert!( + note.contains("declined to launch"), + "the note must say the launch was refused, not blame the binary or a timeout: {note:?}" + ); + + // The device this run was pointed at, not `/dev/kfd`. On a lane with no GPU + // the gate declines either way, so without this the scenario would pass + // against a build that ignored the seam entirely -- and would then be + // asserting the runner's own missing device rather than the gate. + let expected = world + .command_env + .iter() + .find(|(key, _)| *key == KFD_DEVICE_SEAM) + .map(|(_, value)| value.to_string_lossy().into_owned()) + .expect("the scenario must have pointed the pre-flight at an absent device"); + assert!( + note.contains(&expected), + "the note must name the device the pre-flight actually probed ({expected}): {note:?}" + ); + + // The load-bearing half: the gate's claim is that nothing is spawned, which + // a note alone does not establish. The stub would have answered this probe + // perfectly, so the marker's absence is the only evidence that the refusal + // happened before the launch rather than after a failed one. + let marker = stub_ran_marker(world); + assert!( + !marker.exists(), + "the pre-flight must refuse before the spawn, but the stub amd-smi ran and left \ + {}", + marker.display() + ); +} + #[then( "the snapshot keeps the first probe's GPUs and explains the second with a bounded note naming the timeout" )] diff --git a/xtask/src/amd_smi_launch_contract.rs b/xtask/src/amd_smi_launch_contract.rs new file mode 100644 index 0000000000..8f19d52036 --- /dev/null +++ b/xtask/src/amd_smi_launch_contract.rs @@ -0,0 +1,329 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: only a pre-flighted call site may resolve the +//! `amd-smi` binary. +//! +//! Structural guard for EAI-8457. The `/dev/kfd` pre-flight exists because +//! `amd-smi` can park in uninterruptible sleep (D-state) against a device whose +//! `open` never returns, where no signal and therefore no caller timeout +//! reaches it. A gate like that protects nothing unless *every* launch site +//! goes through it, and before this file nothing in the tree failed if one +//! stopped: `apps/rocm/src/gpu_policy.rs`'s two probes could revert to +//! `Command::new(rocm_core::resolve_amd_smi_binary())` and the whole suite +//! would stay green, because the regression is the *absence* of a call, which +//! no behavioural test can observe. +//! +//! So the guard is on the ingredient rather than the behaviour. A launch needs +//! a binary to launch, and [`RESOLVER`] is the only way to find one — the +//! managed ROCm SDK ships `amd-smi` inside the runtime wheel's bin directory +//! rather than on `PATH`, so a hand-rolled `Command::new("amd-smi")` finds +//! nothing on the hosts this product targets. Pinning who may *call* the +//! resolver therefore pins who may launch, and a new launch site has to either +//! route through `rocm_core::amd_smi_json` (which pre-flights) or add itself to +//! [`ALLOWED_CALLERS`] in a reviewed diff that says why it is safe. That review +//! is the entire point, exactly as it is for +//! [`crate::crate_edges::ALLOWLIST`]. +//! +//! What this does **not** claim: it is a source-text scan, not a call-graph +//! analysis. It cannot see a resolver call reached through a `fn` pointer, a +//! re-export under a different name, or a path assembled at runtime, and it +//! says nothing about whether an allowlisted caller then does the right thing +//! with what it gets. It closes the cheap, likely regression — somebody adding +//! a third probe and spawning it directly, the way both existing probes were +//! written before EAI-8457 — and nothing more. +//! +//! Bidirectional, like the crate-edge allowlist: an entry with no matching call +//! fails too. A site that stops launching `amd-smi` must lose its entry, or the +//! exemption outlives the reason for it and the next direct spawn lands in an +//! already-blessed file. + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + /// The call this guard tracks, matched as a call rather than a mention. + /// + /// The trailing `(` is load-bearing in both directions. It keeps prose out + /// — several modules name `resolve_amd_smi_binary` in a doc comment to say + /// where the path comes from, and none of them launches anything — and it + /// keeps the two private helpers out too, since + /// `resolve_amd_smi_binary_in_registry(` and `resolve_amd_smi_binary_in_home(` + /// do not contain this string. Those two are `rocm-core`-private and + /// unreachable from another crate anyway. + const RESOLVER: &str = "resolve_amd_smi_binary("; + + /// Every file that may call [`RESOLVER`], with the reason it is allowed to. + /// + /// Paths are repo-relative and `/`-separated; the comparison normalises + /// Windows separators so the guard gives the same verdict on every lane. + const ALLOWED_CALLERS: &[(&str, &str)] = &[ + ( + "crates/rocm-core/src/amd_smi.rs", + "defines the resolver, and calls it from `amd_smi_json` — the one \ + synchronous launch path, and the only place the `/dev/kfd` pre-flight runs", + ), + ( + "apps/rocm/src/dash.rs", + "resolves the path for the dashboard collector rather than launching anything \ + itself; `rocm-dash-collectors` runs its own pre-flight (`preflight_passes`) on \ + the same device, through the same shared `rocm_core::device_readable_bounded` \ + probe, before it spawns", + ), + ]; + + fn repo_root() -> PathBuf { + // CARGO_MANIFEST_DIR is the xtask/ crate dir; its parent is the repo + // root (same idiom as env_mutation_contract::repo_root). + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + /// Every `.rs` file in the repository, as repo-relative `/`-separated + /// paths. + /// + /// A whole-tree walk rather than a list of source directories, so a crate + /// added under a new top-level directory is covered without anyone + /// remembering to extend this. `target/` (generated and vendored code) and + /// dot-directories (which include the `.claude/worktrees` checkouts of + /// other branches) are skipped, so the verdict never depends on state + /// outside the commit under test — the same two exclusions, for the same + /// two reasons, as `env_mutation_contract`. + fn workspace_rust_sources(root: &Path) -> Vec { + let mut found = Vec::new(); + collect_rust_files(root, root, &mut found); + assert!( + !found.is_empty(), + "found no Rust sources to scan -- the walk is broken, not the tree" + ); + found.sort(); + found + } + + fn collect_rust_files(root: &Path, dir: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let skip = path + .file_name() + .and_then(|name| name.to_str()) + .is_none_or(|name| name == "target" || name.starts_with('.')); + if skip { + continue; + } + collect_rust_files(root, &path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") + && let Ok(relative) = path.strip_prefix(root) + { + found.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } + + /// `line` with any `//` comment tail removed. + /// + /// Without this the guard reports prose. A doc comment is free to write + /// `resolve_amd_smi_binary()` with the parens — the natural way to name a + /// function in English — and that is a description of a call, not one. + /// + /// Deliberately crude: it does not track string literals, so a `//` inside + /// one truncates the line early and could hide a real call after it. That + /// direction fails *open* for a pathological line no file in this tree has, + /// and the alternative (a Rust lexer in a contract test) costs more than + /// the risk. Block comments are not stripped for the same reason; rustfmt + /// renders every doc comment in this tree as `///`. + fn strip_comment(line: &str) -> &str { + match line.find("//") { + Some(at) => &line[..at], + None => line, + } + } + + /// This file, which the scan must skip. + /// + /// Not an exemption like [`ALLOWED_CALLERS`]: this file spawns nothing. Its + /// fixtures quote `resolve_amd_smi_binary(` as *data* — a guard that cannot + /// recognise the thing it guards against is a guard nobody can test — and a + /// text scan has no way to tell a fixture string from a call. Excluding + /// exactly one path, by name, is narrower than teaching [`strip_comment`] + /// about string literals, and leaves no room for a real launch site to hide + /// behind the same trick. + const SELF_PATH: &str = "xtask/src/amd_smi_launch_contract.rs"; + + /// Files that really call [`RESOLVER`], ignoring mentions in comments and + /// this file's own fixtures. + fn files_calling_resolver(root: &Path) -> Vec { + workspace_rust_sources(root) + .into_iter() + .filter(|relative| relative != SELF_PATH) + .filter(|relative| { + std::fs::read_to_string(root.join(relative)).is_ok_and(|text| { + text.lines() + .any(|line| strip_comment(line).contains(RESOLVER)) + }) + }) + .collect() + } + + /// Check a set of calling files against [`ALLOWED_CALLERS`] in both + /// directions, returning every violation found. + fn check(calling: &[String]) -> Vec { + let unexpected = calling + .iter() + .filter(|file| !ALLOWED_CALLERS.iter().any(|(allowed, _)| allowed == file)) + .map(|file| { + format!( + "`{file}` calls `{RESOLVER}..)`, which is how an `amd-smi` launch site is \ + built. Route the launch through `rocm_core::amd_smi_json`, which runs the \ + mandatory `/dev/kfd` pre-flight, or add `{file}` to ALLOWED_CALLERS in \ + xtask/src/amd_smi_launch_contract.rs with the reason the site is safe \ + without it" + ) + }); + let stale = ALLOWED_CALLERS + .iter() + .filter(|(allowed, _)| !calling.iter().any(|file| file == allowed)) + .map(|(allowed, _)| { + format!( + "ALLOWED_CALLERS entry `{allowed}` is stale: it no longer calls \ + `{RESOLVER}..)`. Remove it, or the exemption outlives its reason and the \ + next direct spawn lands in an already-blessed file" + ) + }); + unexpected.chain(stale).collect() + } + + #[test] + fn only_preflighted_sites_resolve_the_amd_smi_binary() { + let root = repo_root(); + let violations = check(&files_calling_resolver(&root)); + assert!( + violations.is_empty(), + "the `/dev/kfd` pre-flight only protects the launch sites that go through it \ + (EAI-8457):\n{}", + violations.join("\n") + ); + } + + #[test] + fn every_allowed_caller_carries_a_reason() { + // An entry with an empty reason is the failure mode this guard exists + // to prevent, written into the guard: a red run "fixed" by pasting a + // path into the list instead of routing the launch through the gate. + for (file, reason) in ALLOWED_CALLERS { + assert!( + reason.len() > 40, + "ALLOWED_CALLERS entry `{file}` has no real justification: {reason:?}" + ); + } + } + + #[test] + fn a_new_direct_resolver_call_fails() { + let mut calling: Vec = ALLOWED_CALLERS + .iter() + .map(|(file, _)| (*file).to_owned()) + .collect(); + calling.push("apps/rocm/src/gpu_policy.rs".to_owned()); + let violations = check(&calling); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("apps/rocm/src/gpu_policy.rs")); + assert!(violations[0].contains("amd_smi_json")); + } + + #[test] + fn an_allowed_caller_that_stops_calling_fails() { + let calling: Vec = ALLOWED_CALLERS + .iter() + .skip(1) + .map(|(file, _)| (*file).to_owned()) + .collect(); + let violations = check(&calling); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("is stale")); + assert!(violations[0].contains(ALLOWED_CALLERS[0].0)); + } + + #[test] + fn a_mention_in_a_comment_is_not_a_call() { + // Both shapes real files in this tree use. The second is why + // `strip_comment` exists at all: parens around the name read as a call + // to a plain `contains`. + assert!( + !strip_comment(" /// (via `rocm_core::resolve_amd_smi_binary`)").contains(RESOLVER) + ); + assert!( + !strip_comment(" // none of the paths resolve_amd_smi_binary() searches") + .contains(RESOLVER) + ); + assert!(strip_comment(" let binary = resolve_amd_smi_binary();").contains(RESOLVER)); + // Code before a trailing comment still counts. + assert!( + strip_comment(" let b = resolve_amd_smi_binary(); // resolved here") + .contains(RESOLVER) + ); + } + + #[test] + fn the_private_resolver_helpers_are_not_matched() { + // `rocm-core`-private, unreachable from another crate, and each is a + // lookup in one location rather than the whole resolution. Matching + // them would make every `rocm-core` test that exercises path + // resolution a launch site. + for helper in [ + " let r = resolve_amd_smi_binary_in_registry(&dir);", + " let r = resolve_amd_smi_binary_in_home(Some(dir));", + " fn resolve_amd_smi_binary_prefers_home_rocm_venv_path() {", + ] { + assert!(!strip_comment(helper).contains(RESOLVER), "{helper}"); + } + } + + #[test] + fn the_self_exclusion_names_a_file_that_exists_and_would_otherwise_be_reported() { + // Two ways `SELF_PATH` could go quietly wrong: a rename leaves it + // pointing at nothing (so the exclusion silently stops applying, and + // this contract starts reporting itself again on the next fixture + // added), or the fixtures are removed and the exclusion is now + // suppressing a file that would pass anyway -- at which point the + // exclusion should go, not linger as a hole. + let root = repo_root(); + let text = std::fs::read_to_string(root.join(SELF_PATH)) + .unwrap_or_else(|e| panic!("SELF_PATH `{SELF_PATH}` does not name a real file: {e}")); + assert!( + text.lines() + .any(|line| strip_comment(line).contains(RESOLVER)), + "`{SELF_PATH}` no longer quotes `{RESOLVER}..)` outside a comment, so the \ + self-exclusion is suppressing nothing and should be deleted" + ); + } + + #[test] + fn the_walk_reaches_every_first_party_crate_directory() { + // Without this the contract could pass by scanning nothing: a walk + // that silently stopped at the repo root would report no violations + // forever. Names a file in each top-level source directory a launch + // site could plausibly be added to. + let files = workspace_rust_sources(&repo_root()); + for expected in [ + "apps/rocm/src/gpu_policy.rs", + "apps/rocmd/src/host_snapshot.rs", + "crates/rocm-core/src/amd_smi.rs", + "crates/rocm-dash-collectors/src/amd_smi.rs", + "crates/rocm-dash-daemon/src/runner.rs", + "xtask/src/crate_edges.rs", + ] { + assert!( + files.iter().any(|file| file == expected), + "the source walk missed `{expected}`, so the contract is scanning less than \ + the tree" + ); + } + } +} diff --git a/xtask/src/crate_edges.rs b/xtask/src/crate_edges.rs index e3c828ca9d..d57fc388bc 100644 --- a/xtask/src/crate_edges.rs +++ b/xtask/src/crate_edges.rs @@ -14,6 +14,14 @@ //! leaf crate other subsystems build on. //! 3. `rocm-dash-tui` must not depend on `rocm-core`. //! +//! Invariant 3 binds the presentation layer only. `rocm-dash-collectors` *does* +//! depend on `rocm-core` (EAI-8457): it spawns `amd-smi` for the dashboard's +//! GPU metrics and gates that spawn on the same `/dev/kfd` device, so it shares +//! `rocm_core::device_readable_bounded` rather than restating the bounded open +//! and its probe budget. A second copy of that primitive would be free to drift +//! from the gate it belongs to. The TUI makes no such call and keeps the +//! stricter rule. +//! //! A literal cycle check would be a no-op: Cargo's resolver already rejects //! dependency cycles among normal/build dependencies. The value here is //! entirely in the **allowlist**: every first-party normal/build edge must be @@ -55,6 +63,7 @@ const ALLOWLIST: &[(&str, &str)] = &[ ("rocmd", "rocm-core"), ("rocmd", "rocm-engine-protocol"), ("rocm-engine-protocol", "rocm-core"), + ("rocm-dash-collectors", "rocm-core"), ("rocm-dash-collectors", "rocm-dash-core"), ("rocm-dash-daemon", "rocm-dash-core"), ("rocm-dash-daemon", "rocm-dash-collectors"), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index fc2f1737ef..adf1266fa9 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -11,6 +11,7 @@ //! definition and dispatch. Run via the workspace alias `cargo xtask `. mod affected; +mod amd_smi_launch_contract; mod ascii_output_contract; mod catalog; mod coverage;