diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index 923eb891e3..61d7e2b9ec 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -1275,6 +1275,11 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "rocm debug: command capture {stage} failed for {}: {detail}", "debug log line", ), + ( + "crates/rocm-core/src/host_gpu/capture.rs", + "rocm debug: command capture output from {} may be incomplete: {note}", + "debug log line", + ), ( "crates/rocm-dash-tui/src/ui/command_screen.rs", "rocm {}", diff --git a/apps/rocm/src/chat/mcp_bridge.rs b/apps/rocm/src/chat/mcp_bridge.rs index a4a7238f94..fa1f88097c 100644 --- a/apps/rocm/src/chat/mcp_bridge.rs +++ b/apps/rocm/src/chat/mcp_bridge.rs @@ -466,14 +466,14 @@ fn run_rocm_command_for_paths( _timeout: Duration, ) -> Result { match run_rocm_read_only_in_process(paths, args) { - Ok(stdout) => Ok(CommandCapture { - argv: std::iter::once("rocm".to_owned()) + Ok(stdout) => Ok(CommandCapture::from_text( + std::iter::once("rocm".to_owned()) .chain(args.iter().cloned()) .collect(), - exit_status: 0, + 0, stdout, - stderr: String::new(), - }), + String::new(), + )), Err(error) => Err(error).with_context(|| { format!( "read-only assistant command is not implemented in-process: {}", diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 9eaf6c8c4a..41fd001c40 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -865,12 +865,12 @@ mod tests { #[test] fn sandbox_check_updates_value_is_read_only_and_preserves_output() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release status=up_to_date\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release status=up_to_date\n".to_owned(), + String::new(), + )); assert_eq!( value.get("tool").and_then(Value::as_str), @@ -894,12 +894,12 @@ mod tests { #[test] fn sandbox_check_updates_value_marks_runtime_update_available() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), + String::new(), + )); assert_eq!( value.get("tool").and_then(Value::as_str), @@ -933,12 +933,12 @@ mod tests { #[test] fn sandbox_check_updates_value_marks_runtime_repair_available() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release-wheel-multi-arch-7-14-0 status=repair_available installed=7.14.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release-wheel-multi-arch-7-14-0 status=repair_available installed=7.14.0 latest=7.14.0\n".to_owned(), + String::new(), + )); assert_eq!( value.get("status").and_then(Value::as_str), @@ -961,12 +961,12 @@ mod tests { #[test] fn a_newer_version_outranks_a_composition_repair_in_the_watcher_report() { - let value = sandbox_check_updates_value(CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime old status=repair_available installed=7.14.0 latest=7.14.0\n runtime stale status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - }); + let value = sandbox_check_updates_value(CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime old status=repair_available installed=7.14.0 latest=7.14.0\n runtime stale status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), + String::new(), + )); assert_eq!( value.get("status").and_then(Value::as_str), @@ -976,18 +976,18 @@ mod tests { #[test] fn sandbox_driver_plan_value_is_read_only_and_preserves_output() { - let value = sandbox_driver_plan_value(CommandCapture { - argv: vec![ + let value = sandbox_driver_plan_value(CommandCapture::from_text( + vec![ "rocm".to_owned(), "install".to_owned(), "driver".to_owned(), "--dkms".to_owned(), "--dry-run".to_owned(), ], - exit_status: 0, - stdout: "driver install plan\n supported: true\n".to_owned(), - stderr: String::new(), - }); + 0, + "driver install plan\n supported: true\n".to_owned(), + String::new(), + )); assert_eq!( value.get("tool").and_then(Value::as_str), diff --git a/apps/rocmd/src/watchers/driver_upgrade.rs b/apps/rocmd/src/watchers/driver_upgrade.rs index 84967dd6d2..55bc78856d 100644 --- a/apps/rocmd/src/watchers/driver_upgrade.rs +++ b/apps/rocmd/src/watchers/driver_upgrade.rs @@ -231,18 +231,18 @@ mod tests { &event, |_paths| { Ok(crate::sandbox::sandbox_driver_plan_value( - rocm_core::CommandCapture { - argv: vec![ + rocm_core::CommandCapture::from_text( + vec![ "rocm".to_owned(), "install".to_owned(), "driver".to_owned(), "--dkms".to_owned(), "--dry-run".to_owned(), ], - exit_status: 0, - stdout: "driver install plan\n supported: true\n".to_owned(), - stderr: String::new(), - }, + 0, + "driver install plan\n supported: true\n".to_owned(), + String::new(), + ), )) }, )?; diff --git a/apps/rocmd/src/watchers/therock_update.rs b/apps/rocmd/src/watchers/therock_update.rs index e364c923d7..91f0304bee 100644 --- a/apps/rocmd/src/watchers/therock_update.rs +++ b/apps/rocmd/src/watchers/therock_update.rs @@ -254,12 +254,12 @@ mod tests { &event, |_paths| { Ok(crate::sandbox::sandbox_check_updates_value( - rocm_core::CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n managed runtimes: none\n".to_owned(), - stderr: String::new(), - }, + rocm_core::CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n managed runtimes: none\n".to_owned(), + String::new(), + ), )) }, )?; @@ -319,12 +319,12 @@ mod tests { &mut state, &event, |_paths| { - Ok(crate::sandbox::sandbox_check_updates_value(rocm_core::CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - })) + Ok(crate::sandbox::sandbox_check_updates_value(rocm_core::CommandCapture::from_text( + vec!["rocm".to_owned(), "update".to_owned()], + 0, + "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), + String::new(), + ))) }, )?; diff --git a/crates/rocm-core/src/command_capture.rs b/crates/rocm-core/src/command_capture.rs index 1b8c6bac37..5fb78a0169 100644 --- a/crates/rocm-core/src/command_capture.rs +++ b/crates/rocm-core/src/command_capture.rs @@ -2,11 +2,16 @@ // // SPDX-License-Identifier: MIT -//! Running the `rocm` CLI as a child process and capturing what it printed. +//! The result of running a command, and running the `rocm` CLI as a child +//! process and capturing what it printed. //! -//! Both apps do this — `rocmd`'s sandbox and MCP clusters, and `rocm`'s chat -//! tool bridge when it replays an approved mutating command — so both use the -//! one [`CommandCapture`], spawn wrapper and text formatter here. +//! [`CommandCapture`] is the one carrier of a command's result: the raw bytes +//! of both streams, a typed [`CommandOutcome`] and the status, for the host +//! probes the bounded executor ([`crate::process`]) runs as well as for the +//! `rocm` captures below. Both apps run `rocm` as a child — `rocmd`'s sandbox +//! and MCP clusters, and `rocm`'s chat tool bridge when it replays an approved +//! mutating command — so both use the one [`CommandCapture`], spawn wrapper and +//! text formatter here. //! //! The bounded-execution half — draining both pipes, the deadline, quoting a //! timed-out child's output — is [`run_with_timeout`]'s, not this module's. @@ -19,24 +24,169 @@ use crate::binary_paths::daemon_binary_path; use crate::process::run_with_timeout; use anyhow::{Context, Result}; +use std::io::ErrorKind; use std::path::Path; -use std::process::Command; +use std::process::{Command, ExitStatus, Output}; use std::time::Duration; -/// What a `rocm` command printed, and how it exited. +/// How a command ended, as far as the bounded executor +/// ([`crate::process::run_with_timeout`]'s core) can tell. /// -/// Usually filled in by [`run_rocm_capture_with_env`] from a real child. The -/// chat tool bridge also builds one by hand for read-only commands it answers -/// in-process, with no child at all, so the shape is "a command's result", not -/// proof that anything was spawned. +/// The variants are the ways a run can end that a caller handles differently: +/// a probe treats "not installed" quietly, "timed out" as a finding, and "exited +/// with a code" by reading the code. Telling a child killed by a signal from one +/// that exited with `1` is what the old integer `exit_status` could not do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CommandOutcome { + /// The program does not exist ([`ErrorKind::NotFound`] from the spawn). + NotFound, + /// The program could not be started for another reason, for example + /// [`ErrorKind::PermissionDenied`]. + SpawnFailed(ErrorKind), + /// The deadline passed and the child was killed. + TimedOut, + /// The executor could not poll or reap the child, killed it and gave up. + WaitFailed, + /// The child exited with this code. + Exited(i32), + /// The child ended without an exit code: killed by a signal. The number is + /// the signal where the platform reports one (Unix), else `None`. + Signaled(Option), +} + +impl CommandOutcome { + /// The outcome of a spawn that failed with `kind`. + #[must_use] + pub fn from_spawn_error(kind: ErrorKind) -> Self { + if kind == ErrorKind::NotFound { + Self::NotFound + } else { + Self::SpawnFailed(kind) + } + } + + /// The outcome of a child that was waited for and not timed out: an exit + /// code when there is one, else a signal. Takes the two parts rather than an + /// [`ExitStatus`] so every host can check it with the values a Windows or a + /// Unix status would give. + #[must_use] + pub const fn from_exit(code: Option, signal: Option) -> Self { + match code { + Some(code) => Self::Exited(code), + None => Self::Signaled(signal), + } + } + + /// The exit code when the child exited with one. + #[must_use] + pub const fn exit_code(self) -> Option { + match self { + Self::Exited(code) => Some(code), + _ => None, + } + } +} + +/// The outcome of a child that was waited for to the end of its life. +pub(crate) fn outcome_of_status(status: ExitStatus) -> CommandOutcome { + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + CommandOutcome::from_exit(status.code(), signal) +} + +/// What a command printed, and how it ended. +/// +/// Filled in by [`run_rocm_capture_with_env`] from a real child, and by the +/// bounded executor for the host probes. The chat tool bridge also builds one by +/// hand ([`Self::from_text`]) for read-only commands it answers in-process, with +/// no child at all (the daemon's tests build them the same way), so the shape is +/// "a command's result", not proof that anything was spawned. +/// +/// The text fields are the lossy form of the raw ones, kept because most callers +/// read text; a caller that must decode the bytes itself (UTF-16LE from +/// `wsl.exe`, or a strict UTF-8 check) reads `stdout_raw` and `stderr_raw`. #[derive(Debug)] pub struct CommandCapture { /// The program followed by its arguments. pub argv: Vec, - /// The exit code; `1` when a child had none (killed by a signal). + /// The exit code; `1` when there was none (a signal, a timeout, a program + /// that could not be started). Read [`Self::outcome`] to tell those apart. pub exit_status: i32, + /// Standard output, lossily decoded from [`Self::stdout_raw`]. pub stdout: String, + /// Standard error, lossily decoded from [`Self::stderr_raw`]. pub stderr: String, + /// Standard output as the child wrote it. On a timeout, whatever arrived + /// before the deadline and the drain grace. + pub stdout_raw: Vec, + /// Standard error as the child wrote it. + pub stderr_raw: Vec, + /// How the command ended. + pub outcome: CommandOutcome, + /// The status the operating system reported for a child that was reaped, for + /// a caller that prints it. `None` for a program that was never started or a + /// capture built by hand. + pub status: Option, + /// What the executor saw go wrong, in words: the spawn or wait error, or the + /// read error that cut a pipe short (the outcome is then still what the + /// child did, and the bytes are what arrived before the error). + pub detail: Option, +} + +impl CommandCapture { + /// A capture of a command that was answered without a child. + #[must_use] + pub fn from_text(argv: Vec, exit_status: i32, stdout: String, stderr: String) -> Self { + Self { + argv, + exit_status, + stdout_raw: stdout.clone().into_bytes(), + stderr_raw: stderr.clone().into_bytes(), + stdout, + stderr, + outcome: CommandOutcome::Exited(exit_status), + status: None, + detail: None, + } + } + + /// A capture of a finished child's [`Output`]. + fn from_output(argv: Vec, output: Output) -> Self { + let outcome = outcome_of_status(output.status); + Self::from_run( + argv, + outcome, + Some(output.status), + output.stdout, + output.stderr, + None, + ) + } + + /// The capture the bounded executor builds from what a run left behind. + #[must_use] + pub(crate) fn from_run( + argv: Vec, + outcome: CommandOutcome, + status: Option, + stdout_raw: Vec, + stderr_raw: Vec, + detail: Option, + ) -> Self { + Self { + argv, + exit_status: outcome.exit_code().unwrap_or(1), + stdout: String::from_utf8_lossy(&stdout_raw).into_owned(), + stderr: String::from_utf8_lossy(&stderr_raw).into_owned(), + stdout_raw, + stderr_raw, + outcome, + status, + detail, + } + } } /// Render a capture as the text shown to a tool caller: stdout alone when @@ -89,14 +239,12 @@ fn capture_program( } let output = run_with_timeout(command, timeout, "process") .with_context(|| format!("failed to run {}", program.display()))?; - Ok(CommandCapture { - argv: std::iter::once(program.display().to_string()) + Ok(CommandCapture::from_output( + std::iter::once(program.display().to_string()) .chain(args.iter().map(|value| (*value).to_owned())) .collect(), - exit_status: output.status.code().unwrap_or(1), - stdout: String::from_utf8_lossy(&output.stdout).into_owned(), - stderr: String::from_utf8_lossy(&output.stderr).into_owned(), - }) + output, + )) } #[cfg(test)] @@ -105,34 +253,34 @@ mod tests { #[test] fn command_capture_text_prefers_stdout_only_when_stderr_is_blank() { - let output = CommandCapture { - argv: vec!["rocm".to_owned()], - exit_status: 0, - stdout: " all good \n".to_owned(), - stderr: " \n".to_owned(), - }; + let output = CommandCapture::from_text( + vec!["rocm".to_owned()], + 0, + " all good \n".to_owned(), + " \n".to_owned(), + ); assert_eq!(command_capture_text(&output), "all good"); } #[test] fn command_capture_text_falls_back_to_stderr_when_stdout_is_blank() { - let output = CommandCapture { - argv: vec!["rocm".to_owned()], - exit_status: 1, - stdout: " ".to_owned(), - stderr: "boom".to_owned(), - }; + let output = CommandCapture::from_text( + vec!["rocm".to_owned()], + 1, + " ".to_owned(), + "boom".to_owned(), + ); assert_eq!(command_capture_text(&output), "stderr:\nboom"); } #[test] fn command_capture_text_joins_both_streams_when_present() { - let output = CommandCapture { - argv: vec!["rocm".to_owned()], - exit_status: 1, - stdout: "out".to_owned(), - stderr: "err".to_owned(), - }; + let output = CommandCapture::from_text( + vec!["rocm".to_owned()], + 1, + "out".to_owned(), + "err".to_owned(), + ); assert_eq!( command_capture_text(&output), "stdout:\nout\n\nstderr:\nerr" @@ -182,4 +330,164 @@ mod tests { assert_eq!(output.stdout.trim(), "marker-value"); assert_eq!(output.stderr.trim(), "oops"); } + + #[test] + fn an_exit_code_outranks_a_signal_and_no_code_is_a_signal() { + assert_eq!( + CommandOutcome::from_exit(Some(3), Some(9)), + CommandOutcome::Exited(3) + ); + assert_eq!( + CommandOutcome::from_exit(Some(0), None), + CommandOutcome::Exited(0) + ); + assert_eq!( + CommandOutcome::from_exit(None, Some(9)), + CommandOutcome::Signaled(Some(9)) + ); + assert_eq!( + CommandOutcome::from_exit(None, None), + CommandOutcome::Signaled(None), + "a status with neither (no signal number on Windows) is still not an exit" + ); + assert_eq!(CommandOutcome::Exited(3).exit_code(), Some(3)); + for outcome in [ + CommandOutcome::NotFound, + CommandOutcome::TimedOut, + CommandOutcome::WaitFailed, + CommandOutcome::Signaled(Some(9)), + CommandOutcome::SpawnFailed(ErrorKind::PermissionDenied), + ] { + assert_eq!(outcome.exit_code(), None, "{outcome:?}"); + } + } + + #[test] + fn only_a_missing_program_is_not_found() { + assert_eq!( + CommandOutcome::from_spawn_error(ErrorKind::NotFound), + CommandOutcome::NotFound + ); + assert_eq!( + CommandOutcome::from_spawn_error(ErrorKind::PermissionDenied), + CommandOutcome::SpawnFailed(ErrorKind::PermissionDenied) + ); + } + + #[test] + fn a_status_is_an_exit_code_or_a_signal() { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + // The raw wait status: the exit code in the second byte, or the + // signal number in the low bits. + assert_eq!( + outcome_of_status(ExitStatus::from_raw(3 << 8)), + CommandOutcome::Exited(3) + ); + assert_eq!( + outcome_of_status(ExitStatus::from_raw(9)), + CommandOutcome::Signaled(Some(9)) + ); + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt as _; + assert_eq!( + outcome_of_status(ExitStatus::from_raw(3)), + CommandOutcome::Exited(3) + ); + } + } + + #[test] + fn a_hand_built_capture_is_an_exit_whose_bytes_are_its_text() { + let capture = CommandCapture::from_text( + vec!["rocm".to_owned()], + 2, + "out".to_owned(), + "err".to_owned(), + ); + + assert_eq!(capture.outcome, CommandOutcome::Exited(2)); + assert_eq!(capture.exit_status, 2); + assert_eq!(capture.stdout_raw, b"out"); + assert_eq!(capture.stderr_raw, b"err"); + assert!(capture.status.is_none() && capture.detail.is_none()); + } + + #[test] + fn a_run_keeps_its_raw_bytes_and_reports_text_from_them_lossily() { + let capture = CommandCapture::from_run( + vec!["tool".to_owned()], + CommandOutcome::TimedOut, + None, + b"ok\xffdone".to_vec(), + Vec::new(), + None, + ); + + assert_eq!(capture.stdout_raw, b"ok\xffdone"); + assert_eq!(capture.stdout, "ok\u{FFFD}done"); + assert_eq!(capture.exit_status, 1, "no exit code to report"); + + let exited = CommandCapture::from_run( + Vec::new(), + CommandOutcome::Exited(7), + None, + Vec::new(), + Vec::new(), + None, + ); + assert_eq!(exited.exit_status, 7); + } + + /// Subprocess-runners pair of `docs/host-probing.md`: a child killed by a + /// signal has no exit code, so the legacy `exit_status` reads `1`, which is + /// also what a child that exits with `1` reads; the typed `outcome` is what + /// tells them apart. + #[cfg(unix)] + #[test] + fn a_child_killed_by_a_signal_reads_as_exit_status_1() { + let killed = capture_program( + Path::new("sh"), + &["-c", "echo partial; kill -9 $$"], + &[], + Duration::from_secs(10), + ) + .expect("a signal death is captured, not an error"); + let exited = capture_program( + Path::new("sh"), + &["-c", "echo partial; exit 1"], + &[], + Duration::from_secs(10), + ) + .expect("a non-zero exit is captured, not an error"); + + assert_eq!(killed.exit_status, 1); + assert_eq!(killed.stdout.trim(), "partial"); + assert_eq!( + killed.exit_status, exited.exit_status, + "the two cannot be told apart from the exit status" + ); + assert_eq!(killed.outcome, CommandOutcome::Signaled(Some(9))); + assert_eq!(exited.outcome, CommandOutcome::Exited(1)); + } + + /// A program that cannot be started is an error carrying the program, not a + /// capture, so a caller cannot tell "not installed" from a timeout except by + /// the text of the message. + #[test] + fn a_program_that_cannot_be_started_is_an_error_naming_it() { + let missing = Path::new("/nonexistent-dir-for-the-runner-pair/tool"); + let error = capture_program(missing, &[], &[], Duration::from_secs(10)) + .expect_err("a spawn failure is an error here"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains(&format!("failed to run {}", missing.display())), + "{rendered}" + ); + assert!(rendered.contains("failed to spawn process"), "{rendered}"); + } } diff --git a/crates/rocm-core/src/host_gpu/capture.rs b/crates/rocm-core/src/host_gpu/capture.rs index 7aef97001d..8fe6fbd91a 100644 --- a/crates/rocm-core/src/host_gpu/capture.rs +++ b/crates/rocm-core/src/host_gpu/capture.rs @@ -10,12 +10,12 @@ //! Windows system locations included). A missing, slow or failing tool reads as //! `None`, never as an error. +use crate::command_capture::{CommandCapture, CommandOutcome}; use crate::probe_exec::tool_path_candidates; -use std::io::Read; +use crate::process::run_bounded; use std::path::Path; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; +use std::process::Command; +use std::time::Duration; pub(super) const OPTIONAL_COMMAND_TIMEOUT: Duration = Duration::from_millis(1_500); @@ -38,77 +38,557 @@ pub(super) fn capture_optional_command_with_timeout( None } +/// One spelling of the program, run through [`run_bounded`], the executor every +/// host probe shares; [`optional_stdout`] says what the run is worth. fn capture_optional_command_candidate_with_timeout( program: &Path, args: &[&str], timeout: Duration, ) -> Option { - let mut child = match Command::new(program) - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .spawn() - { - Ok(child) => child, - Err(error) => { - debug_command_capture_failure(program, "spawn", &error.to_string()); - return None; - } - }; - let mut stdout_reader = child.stdout.take().map(|mut stdout| { - thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stdout.read_to_end(&mut bytes); - bytes - }) - }); - - let start = Instant::now(); - loop { - match child.try_wait() { - Ok(Some(status)) => { - let bytes = stdout_reader - .take() - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - if status.success() { - return String::from_utf8(bytes).ok(); - } - debug_command_capture_failure(program, "exit", &format!("status {status}")); - return None; - } - Ok(None) if start.elapsed() < timeout => { - thread::sleep(Duration::from_millis(25)); - } - Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - if let Some(reader) = stdout_reader.take() { - let _ = reader.join(); - } - debug_command_capture_failure(program, "timeout", "timed out"); - return None; - } - Err(_) => { - let _ = child.kill(); - let _ = child.wait(); - if let Some(reader) = stdout_reader.take() { - let _ = reader.join(); - } - debug_command_capture_failure(program, "wait", "failed to wait"); - return None; - } + candidate_with_debug(program, args, timeout, &mut debug_command_capture) +} + +/// [`capture_optional_command_candidate_with_timeout`] with the place its debug +/// lines go as an argument, so a test can read them. The sink is handed every +/// line; whether it is printed is its own decision. +fn candidate_with_debug( + program: &Path, + args: &[&str], + timeout: Duration, + debug: &mut impl FnMut(String), +) -> Option { + let mut command = Command::new(program); + command.args(args); + let capture = run_bounded(command, timeout, "process"); + if let Some(note) = incomplete_output_note(&capture) { + debug(format!( + "rocm debug: command capture output from {} may be incomplete: {note}", + program.display() + )); + } + match optional_stdout(capture) { + Ok(stdout) => stdout, + Err((stage, detail)) => { + debug(format!( + "rocm debug: command capture {stage} failed for {}: {detail}", + program.display() + )); + None } } } -fn debug_command_capture_failure(program: &Path, stage: &str, detail: &str) { +/// Why the stdout of a run that exited 0 may be shorter than the program wrote: +/// a descendant still writing when the executor stopped waiting for it, or a +/// pipe that failed to read. [`optional_stdout`] gives the bytes that arrived +/// either way, so this is the only place the shortfall shows. +fn incomplete_output_note(capture: &CommandCapture) -> Option<&str> { + match capture.outcome { + CommandOutcome::Exited(0) => capture.detail.as_deref(), + _ => None, + } +} + +/// What an optional capture makes of a finished run: the stdout of a program +/// that exited 0 and wrote valid UTF-8, and nothing for any other run. +/// +/// `Ok(None)` is a successful run whose output is not UTF-8, which reads as no +/// answer and is not worth a debug line. `Err` is a run that did not succeed, +/// with the stage and wording for that line. A pipe that failed to read part way +/// does not change the answer: the bytes that arrived are used, as they were +/// before the capture moved onto the shared executor. +fn optional_stdout(capture: CommandCapture) -> Result, (&'static str, String)> { + match capture.outcome { + CommandOutcome::Exited(0) => Ok(String::from_utf8(capture.stdout_raw).ok()), + CommandOutcome::Exited(_) | CommandOutcome::Signaled(_) => Err(( + "exit", + capture.status.map_or_else( + || "no status".to_owned(), + |status| format!("status {status}"), + ), + )), + CommandOutcome::TimedOut => Err(("timeout", "timed out".to_owned())), + CommandOutcome::WaitFailed => Err(("wait", "failed to wait".to_owned())), + CommandOutcome::NotFound | CommandOutcome::SpawnFailed(_) => Err(( + "spawn", + capture + .detail + .unwrap_or_else(|| "failed to start".to_owned()), + )), + } +} + +/// Print one debug line, when `ROCM_CLI_DEBUG_COMMAND_CAPTURE` asks for them. +fn debug_command_capture(line: String) { if !crate::runtime::env_flag("ROCM_CLI_DEBUG_COMMAND_CAPTURE") { return; } - eprintln!( - "rocm debug: command capture {stage} failed for {}: {detail}", - program.display() - ); + eprintln!("{line}"); +} + +/// The subprocess-runners pair of `docs/host-probing.md`: the three runners the +/// host probes use, which are the summary stack's +/// `capture_optional_command_with_timeout` (this file), the examination stack's +/// `run` over `run_raw` ([`crate::probe_exec`]), and `run_with_timeout` +/// ([`crate::process`]), with the first two now thin adapters over the +/// executor `run_with_timeout` shares. +/// +/// Every case runs one stand-in program through all three and pins what each +/// returns. The cases that stayed as the characterisation left them are the +/// adapters keeping their old answers; the ones that moved are the differences +/// the note lists, each named for the answer it now gives. Programs are planted +/// with [`crate::fake_executable::write_fake_executable`] and started by +/// absolute path, so no `PATH` is involved. Each case that leaves a descendant +/// holding a pipe gives it a short, fixed life, so a runner that waits for it is +/// slow and cannot hang the suite. +#[cfg(all(test, unix))] +mod subprocess_runners_pair { + use super::{candidate_with_debug, capture_optional_command_with_timeout}; + use crate::fake_executable::write_fake_executable; + use crate::probe_exec::{run, run_utf16le}; + use crate::process::run_with_timeout; + use std::os::unix::process::ExitStatusExt as _; + use std::path::Path; + use std::process::{Command, Output}; + use std::time::{Duration, Instant}; + + /// How long a descendant that keeps a pipe open lives. An unbounded reader + /// join waits this long; a bounded one does not. + const DESCENDANT_LIFE: &str = "2"; + + /// Under this a call did not wait for a [`DESCENDANT_LIFE`] descendant, and + /// over it a call did, with the margin on both sides for a loaded host. + const WAITED: Duration = Duration::from_millis(1_500); + + struct Fixture { + _dir: tempfile::TempDir, + program: String, + } + + /// Plant `body` as a shell script and return where it is. + fn fixture(body: &str) -> Fixture { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("fake-tool"); + write_fake_executable(&path, format!("#!/bin/sh\n{body}\n")).expect("plant the fake"); + let program = path.to_str().expect("a UTF-8 temp path").to_owned(); + Fixture { _dir: dir, program } + } + + fn timed(call: impl FnOnce() -> T) -> (T, Duration) { + let started = Instant::now(); + let value = call(); + (value, started.elapsed()) + } + + fn bounded(program: &str, timeout: Duration) -> anyhow::Result { + run_with_timeout(Command::new(program), timeout, "process") + } + + const LONG: Duration = Duration::from_secs(20); + + #[test] + fn a_program_that_succeeds_is_read_alike_by_all_three() { + let tool = fixture("printf 'out\\n'; printf 'err\\n' >&2; exit 0"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "out\n".to_owned(), "err\n".to_owned()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + Some("out\n".to_owned()), + "the optional capture keeps stdout and discards stderr" + ); + let output = bounded(&tool.program, LONG).expect("a success is not an error"); + assert!(output.status.success()); + assert_eq!(output.stdout, b"out\n"); + assert_eq!(output.stderr, b"err\n"); + } + + #[test] + fn a_non_zero_exit_keeps_its_output_for_run_but_reads_as_nothing_to_the_optional_capture() { + let tool = fixture("printf 'partial\\n'; printf 'bad\\n' >&2; exit 3"); + + assert_eq!( + run(&tool.program, &[], LONG), + (3, "partial\n".to_owned(), "bad\n".to_owned()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + None, + "the optional capture keeps stdout only on success" + ); + let output = bounded(&tool.program, LONG).expect("a non-zero exit is not an error"); + assert_eq!(output.status.code(), Some(3)); + assert_eq!(output.stdout, b"partial\n"); + assert_eq!(output.stderr, b"bad\n"); + } + + #[test] + fn a_program_that_cannot_be_started_is_127_empty_none_or_an_error() { + let missing = "/nonexistent-dir-for-the-runner-pair/tool"; + + assert_eq!(run(missing, &[], LONG), (127, String::new(), String::new())); + assert_eq!( + capture_optional_command_with_timeout(missing, &[], LONG), + None + ); + let error = bounded(missing, LONG).expect_err("a spawn failure is an error here"); + assert!( + format!("{error:#}").contains("failed to spawn process"), + "{error:#}" + ); + } + + #[test] + fn a_timeout_keeps_partial_output_for_run_and_quotes_it_for_the_bounded_executor() { + // `exec` so killing the shell at the deadline leaves no descendant on the + // pipe: this case is about what a timeout returns, not about the join. + let tool = fixture("printf 'before\\n'; exec sleep 30"); + // Room for a loaded host to start the shell and run its `printf` before + // the deadline, since the assertion is on what was printed by then. + let timeout = Duration::from_secs(3); + + assert_eq!( + run(&tool.program, &[], timeout), + (124, "before\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], timeout), + None + ); + let error = bounded(&tool.program, timeout).expect_err("a timeout is an error here"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("exceeded 3s timeout"), "{rendered}"); + assert!(rendered.contains("before"), "{rendered}"); + } + + /// The difference the note asks to be pinned first: the child prints `early` + /// and exits 0, and a descendant prints `late` half a second later, past the + /// 200 ms the executor gives the readers once the child has gone. All three + /// now return `early` alone; `run` and the optional capture used to join + /// their readers and return both lines. + #[test] + fn output_written_more_than_the_drain_grace_after_the_child_exits_is_lost_to_all_three() { + let tool = fixture("printf 'early\\n'; (sleep 0.5; printf 'late\\n') &"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "early\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + Some("early\n".to_owned()) + ); + let output = bounded(&tool.program, LONG).expect("the child exited 0"); + assert_eq!(output.stdout, b"early\n"); + } + + /// The other side of the grace: a descendant that writes 0.1 s after the + /// child exits is inside the 200 ms the readers are given, so every runner + /// still returns both lines. This is what keeps a probe whose tool leaves a + /// short-lived helper behind from losing its tail. + #[test] + fn output_written_within_the_drain_grace_after_the_child_exits_is_kept_by_all_three() { + let tool = fixture("printf 'early\\n'; (sleep 0.1; printf 'late\\n') &"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "early\nlate\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + Some("early\nlate\n".to_owned()) + ); + let output = bounded(&tool.program, LONG).expect("the child exited 0"); + assert_eq!(output.stdout, b"early\nlate\n"); + } + + /// A descendant holds both pipes for [`DESCENDANT_LIFE`] seconds after the + /// child exits 0. `run` and the optional capture used to join their readers + /// with no limit, so each call lasted as long as the descendant; now none of + /// the three waits for it, and each still returns what the child wrote. + #[test] + fn a_descendant_holding_the_pipe_no_longer_extends_run_or_the_optional_capture() { + let tool = fixture(&format!("printf 'ready\\n'; sleep {DESCENDANT_LIFE} &")); + + let (answer, elapsed) = timed(|| run(&tool.program, &[], LONG)); + assert_eq!(answer, (0, "ready\n".to_owned(), String::new())); + assert!(elapsed < WAITED, "run returned after {elapsed:?}"); + + let (answer, elapsed) = + timed(|| capture_optional_command_with_timeout(&tool.program, &[], LONG)); + assert_eq!(answer, Some("ready\n".to_owned())); + assert!( + elapsed < WAITED, + "the optional capture returned after {elapsed:?}" + ); + + let (answer, elapsed) = timed(|| bounded(&tool.program, LONG)); + assert_eq!(answer.expect("the child exited 0").stdout, b"ready\n"); + assert!(elapsed < WAITED, "the executor returned after {elapsed:?}"); + } + + /// The same descendant behind a child that is killed at its deadline: the + /// timeout used to bound nothing for the two older runners (issue 623), and + /// now bounds all three. `run` keeps the partial output and its `124`. + #[test] + fn a_descendant_holding_the_pipe_no_longer_outlasts_the_timeout_of_run_or_the_optional_capture() + { + let tool = fixture(&format!( + "printf 'start\\n'; sleep {DESCENDANT_LIFE} & wait" + )); + let timeout = Duration::from_millis(500); + + let (answer, elapsed) = timed(|| run(&tool.program, &[], timeout)); + // The rc and the time are the point here; what was printed before the + // deadline is `a_timeout_keeps_partial_output_for_run_...`'s, with room + // for a loaded host that this short deadline does not leave. + assert_eq!(answer.0, 124); + assert!(elapsed < WAITED, "run returned after {elapsed:?}"); + + let (answer, elapsed) = + timed(|| capture_optional_command_with_timeout(&tool.program, &[], timeout)); + assert_eq!(answer, None); + assert!( + elapsed < WAITED, + "the optional capture returned after {elapsed:?}" + ); + + let (answer, elapsed) = timed(|| bounded(&tool.program, timeout)); + assert!(answer.is_err(), "a timeout is an error here"); + assert!(elapsed < WAITED, "the executor returned after {elapsed:?}"); + } + + /// What the optional capture says on its debug channel for the same two + /// runs: a descendant that writes after the grace gives a short answer and a + /// line saying so, and a run that finished cleanly gives no line. The answer + /// and the line are read from one call, so the line cannot be claiming a + /// shortfall the answer does not have. + #[test] + fn a_short_answer_from_the_optional_capture_comes_with_a_debug_line_and_a_whole_one_does_not() { + let late = fixture("printf 'early\\n'; (sleep 0.5; printf 'late\\n') &"); + let mut lines = Vec::new(); + let answer = candidate_with_debug(Path::new(&late.program), &[], LONG, &mut |line| { + lines.push(line); + }); + assert_eq!(answer, Some("early\n".to_owned())); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!( + lines[0].starts_with("rocm debug: command capture output from ") + && lines[0].contains(&late.program) + && lines[0].contains("may be incomplete: output was still arriving"), + "{}", + lines[0] + ); + + let whole = fixture("printf 'early\\n'; (sleep 0.1; printf 'late\\n') &"); + let mut lines = Vec::new(); + let answer = candidate_with_debug(Path::new(&whole.program), &[], LONG, &mut |line| { + lines.push(line); + }); + assert_eq!(answer, Some("early\nlate\n".to_owned())); + assert!(lines.is_empty(), "{lines:?}"); + } + + /// A run that failed is described by the failure line alone, not also as an + /// incomplete answer: the second line would add nothing to the first. + #[test] + fn a_failed_run_gets_its_failure_line_and_no_incomplete_line() { + let tool = fixture("printf 'partial\\n'; (sleep 0.5; printf 'late\\n') & exit 3"); + let mut lines = Vec::new(); + let answer = candidate_with_debug(Path::new(&tool.program), &[], LONG, &mut |line| { + lines.push(line); + }); + assert_eq!(answer, None); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!( + lines[0].starts_with("rocm debug: command capture exit failed for "), + "{}", + lines[0] + ); + } + + /// A descendant that holds only stderr. The optional capture used to send + /// stderr to the null device, so such a descendant held nothing of its and + /// the call returned at once; it now pipes stderr like the others and gives + /// the descendant the drain grace. The answer is unchanged. The extra cost, + /// up to the 200 ms grace, is too small for a timing assertion to tell from + /// none, so the assertion is only that the call does not wait out the + /// descendant's life. `run` pipes stderr as it always did, and used to wait + /// the descendant out and return its late line. + #[test] + fn a_descendant_holding_only_stderr_costs_the_optional_capture_at_most_the_drain_grace() { + let tool = fixture(&format!( + "printf 'x\\n'; (exec >/dev/null; sleep {DESCENDANT_LIFE}; printf 'y\\n' >&2) &" + )); + + let (answer, elapsed) = + timed(|| capture_optional_command_with_timeout(&tool.program, &[], LONG)); + assert_eq!(answer, Some("x\n".to_owned())); + assert!(elapsed < WAITED, "returned after {elapsed:?}"); + + let (answer, elapsed) = timed(|| run(&tool.program, &[], LONG)); + assert_eq!(answer, (0, "x\n".to_owned(), String::new())); + assert!(elapsed < WAITED, "run returned after {elapsed:?}"); + } + + #[test] + fn output_that_is_not_utf8_is_lossy_for_run_none_for_the_optional_capture_and_raw_for_the_executor() + { + let tool = fixture("printf 'ok\\377done\\n'"); + + assert_eq!( + run(&tool.program, &[], LONG), + (0, "ok\u{FFFD}done\n".to_owned(), String::new()) + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + None, + "strict UTF-8: a stray byte reads as no answer" + ); + let output = bounded(&tool.program, LONG).expect("the child exited 0"); + assert_eq!(output.stdout, b"ok\xffdone\n"); + } + + #[test] + fn utf16le_output_is_decoded_by_declaration_from_the_raw_bytes() { + // `h` `i` as UTF-16LE, preceded by the byte order mark. + let tool = fixture("printf '\\377\\376h\\000i\\000'"); + + assert_eq!(run_utf16le(&tool.program, &[], LONG), (0, "hi".to_owned())); + } + + #[test] + fn a_program_killed_by_a_signal_has_no_exit_code() { + let tool = fixture("printf 'partial\\n'; kill -9 $$"); + + assert_eq!( + run(&tool.program, &[], LONG), + (-1, "partial\n".to_owned(), String::new()), + "run reports -1 where the status has no code" + ); + assert_eq!( + capture_optional_command_with_timeout(&tool.program, &[], LONG), + None + ); + let output = bounded(&tool.program, LONG).expect("a signal death is not an error"); + assert_eq!(output.status.code(), None); + assert_eq!(output.status.signal(), Some(9)); + assert_eq!(output.stdout, b"partial\n"); + } +} + +/// What the optional capture makes of each way a run can end. Runs on every +/// host, over captures built by hand, so the Windows job checks the translation +/// the real-process cases above cannot drive there. +#[cfg(test)] +mod optional_stdout_tests { + use super::{CommandCapture, CommandOutcome, incomplete_output_note, optional_stdout}; + + fn capture(outcome: CommandOutcome, stdout: &[u8], detail: Option<&str>) -> CommandCapture { + CommandCapture::from_run( + vec!["tool".to_owned()], + outcome, + None, + stdout.to_vec(), + b"ignored".to_vec(), + detail.map(str::to_owned), + ) + } + + #[test] + fn a_program_that_exited_0_gives_its_stdout_and_nothing_else() { + assert_eq!( + optional_stdout(capture(CommandOutcome::Exited(0), b"out\n", None)), + Ok(Some("out\n".to_owned())) + ); + } + + #[test] + fn a_program_that_exited_0_with_output_that_is_not_utf8_gives_no_answer_and_no_debug_line() { + assert_eq!( + optional_stdout(capture(CommandOutcome::Exited(0), b"ok\xffdone", None)), + Ok(None) + ); + } + + #[test] + fn a_pipe_that_failed_to_read_does_not_change_the_answer() { + // Kept as it was: the bytes that arrived are used, and the error is not + // an answer of its own. + assert_eq!( + optional_stdout(capture( + CommandOutcome::Exited(0), + b"partial", + Some("failed to read the output: broken pipe") + )), + Ok(Some("partial".to_owned())) + ); + } + + #[test] + fn every_other_way_a_run_ends_is_no_answer_with_its_stage_named() { + let cases = [ + (CommandOutcome::Exited(3), "exit"), + (CommandOutcome::Signaled(Some(9)), "exit"), + (CommandOutcome::TimedOut, "timeout"), + (CommandOutcome::WaitFailed, "wait"), + (CommandOutcome::NotFound, "spawn"), + ( + CommandOutcome::SpawnFailed(std::io::ErrorKind::PermissionDenied), + "spawn", + ), + ]; + for (outcome, stage) in cases { + // Stdout the child wrote before ending badly is not an answer. + let result = optional_stdout(capture(outcome, b"partial", Some("the os said no"))); + assert_eq!( + result.map_err(|(stage, _)| stage), + Err(stage), + "{outcome:?}" + ); + } + } + + #[test] + fn only_a_run_that_exited_0_with_something_cutting_its_output_short_is_called_incomplete() { + let note = "output was still arriving when the drain grace ended"; + assert_eq!( + incomplete_output_note(&capture(CommandOutcome::Exited(0), b"x", Some(note))), + Some(note) + ); + assert_eq!( + incomplete_output_note(&capture(CommandOutcome::Exited(0), b"x", None)), + None, + "a whole capture has nothing to report" + ); + // Every other ending already has a failure line of its own. + for outcome in [ + CommandOutcome::Exited(3), + CommandOutcome::Signaled(Some(9)), + CommandOutcome::TimedOut, + CommandOutcome::WaitFailed, + CommandOutcome::NotFound, + ] { + assert_eq!( + incomplete_output_note(&capture(outcome, b"x", Some(note))), + None, + "{outcome:?}" + ); + } + } + + #[test] + fn a_spawn_failure_reports_the_error_the_os_gave() { + let (stage, detail) = optional_stdout(capture( + CommandOutcome::NotFound, + b"", + Some("No such file or directory (os error 2)"), + )) + .expect_err("a program that cannot be started is no answer"); + + assert_eq!(stage, "spawn"); + assert_eq!(detail, "No such file or directory (os error 2)"); + } } diff --git a/crates/rocm-core/src/host_gpu/diagnostics.rs b/crates/rocm-core/src/host_gpu/diagnostics.rs index bc190d2f24..739eb34c5c 100644 --- a/crates/rocm-core/src/host_gpu/diagnostics.rs +++ b/crates/rocm-core/src/host_gpu/diagnostics.rs @@ -14,6 +14,8 @@ use super::windows_inventory::{ parse_windows_examine_inventory, parse_windows_pnputil_display_inventory, }; use super::wsl::{detect_wsl_windows_display_probe_text, is_wsl_host}; +#[cfg(any(windows, test))] +use crate::command_capture::{CommandCapture, CommandOutcome}; #[cfg(windows)] use crate::gfx_target::{ amd_pci_device_id_from_pnp_id, gfx_target_from_amd_marketing_name, @@ -21,17 +23,15 @@ use crate::gfx_target::{ }; #[cfg(windows)] use crate::probe_exec::tool_path_candidates; -use crate::{runtime_is_linux, runtime_is_windows, runtime_os_name}; -#[cfg(windows)] -use std::io::Read; #[cfg(windows)] -use std::path::PathBuf; -#[cfg(windows)] -use std::process::{Command, Stdio}; +use crate::process::run_bounded; +use crate::{runtime_is_linux, runtime_is_windows, runtime_os_name}; +#[cfg(any(windows, test))] +use std::path::{Path, PathBuf}; #[cfg(windows)] -use std::thread; +use std::process::Command; #[cfg(windows)] -use std::time::{Duration, Instant}; +use std::time::Duration; pub fn detect_host_gpu_diagnostics() -> String { use std::fmt::Write as _; @@ -186,7 +186,7 @@ fn append_windows_probe_diagnostics( } #[derive(Debug)] -#[cfg(windows)] +#[cfg(any(windows, test))] struct DiagnosticCommandResult { program: Option, status: Option, @@ -196,96 +196,79 @@ struct DiagnosticCommandResult { timed_out: bool, } +/// Run `program` under every spelling [`tool_path_candidates`] gives, through the +/// executor the other host probes share ([`run_bounded`]), and describe the +/// first run that started. #[cfg(windows)] fn capture_diagnostic_command( program: &str, args: &[&str], timeout: Duration, ) -> DiagnosticCommandResult { - let candidates = tool_path_candidates(program); + capture_diagnostic_with(program, &tool_path_candidates(program), |path| { + let mut command = Command::new(path); + command.args(args); + run_bounded(command, timeout, "process") + }) +} + +/// [`capture_diagnostic_command`] with the candidates and the runner passed in, +/// so a test on any host can say how each run ends. +/// +/// The next spelling is tried only when a program could not be started. Any run +/// that started is final: an exit (non-zero included) reports its status and +/// both streams, a timeout reports the output that arrived and says so, and a +/// failure to wait reports that, with no output. +#[cfg(any(windows, test))] +fn capture_diagnostic_with( + program: &str, + candidates: &[String], + mut run: impl FnMut(&Path) -> CommandCapture, +) -> DiagnosticCommandResult { let mut last_error = None; for candidate in candidates { - let path = PathBuf::from(&candidate); - let mut child = match Command::new(&path) - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - { - Ok(child) => child, - Err(error) => { - last_error = Some(format!("failed to launch {}: {error}", path.display())); - continue; + let path = PathBuf::from(candidate); + let capture = run(&path); + match capture.outcome { + CommandOutcome::NotFound | CommandOutcome::SpawnFailed(_) => { + last_error = Some(format!( + "failed to launch {}: {}", + path.display(), + capture.detail.as_deref().unwrap_or("unknown error") + )); } - }; - let stdout_reader = child.stdout.take().map(|mut stdout| { - thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stdout.read_to_end(&mut bytes); - bytes - }) - }); - let stderr_reader = child.stderr.take().map(|mut stderr| { - thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stderr.read_to_end(&mut bytes); - bytes - }) - }); - - let start = Instant::now(); - loop { - match child.try_wait() { - Ok(Some(status)) => { - let stdout = stdout_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - let stderr = stderr_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - return DiagnosticCommandResult { - program: Some(path), - status: Some(status.to_string()), - stdout: String::from_utf8_lossy(&stdout).into_owned(), - stderr: String::from_utf8_lossy(&stderr).into_owned(), - error: None, - timed_out: false, - }; - } - Ok(None) if start.elapsed() < timeout => { - thread::sleep(Duration::from_millis(25)); - } - Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - let stdout = stdout_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - let stderr = stderr_reader - .map(|reader| reader.join().unwrap_or_default()) - .unwrap_or_default(); - return DiagnosticCommandResult { - program: Some(path), - status: None, - stdout: String::from_utf8_lossy(&stdout).into_owned(), - stderr: String::from_utf8_lossy(&stderr).into_owned(), - error: None, - timed_out: true, - }; - } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return DiagnosticCommandResult { - program: Some(path), - status: None, - stdout: String::new(), - stderr: String::new(), - error: Some(format!("failed to wait: {error}")), - timed_out: false, - }; - } + CommandOutcome::Exited(_) | CommandOutcome::Signaled(_) => { + return DiagnosticCommandResult { + program: Some(path), + status: capture.status.map(|status| status.to_string()), + stdout: capture.stdout, + stderr: capture.stderr, + error: None, + timed_out: false, + }; + } + CommandOutcome::TimedOut => { + return DiagnosticCommandResult { + program: Some(path), + status: None, + stdout: capture.stdout, + stderr: capture.stderr, + error: None, + timed_out: true, + }; + } + CommandOutcome::WaitFailed => { + return DiagnosticCommandResult { + program: Some(path), + status: None, + stdout: String::new(), + stderr: String::new(), + error: Some(format!( + "failed to wait: {}", + capture.detail.as_deref().unwrap_or("unknown error") + )), + timed_out: false, + }; } } } @@ -330,3 +313,187 @@ fn truncate_diagnostic_line(line: &str, max_chars: usize) -> String { truncated } } + +/// The Windows-only diagnostic capture's choice of candidate and its report of +/// each way a run can end, driven through an injected runner so it runs on every +/// host, the Windows job included. +#[cfg(test)] +mod capture_diagnostic_tests { + use super::{CommandCapture, CommandOutcome, capture_diagnostic_with}; + use std::path::Path; + use std::process::ExitStatus; + + fn status(code: i32) -> ExitStatus { + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt as _; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt as _; + #[cfg(unix)] + let raw = code << 8; + #[cfg(windows)] + let raw = code.unsigned_abs(); + ExitStatus::from_raw(raw) + } + + fn candidates(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_owned()).collect() + } + + fn finished( + outcome: CommandOutcome, + code: Option, + detail: Option<&str>, + ) -> CommandCapture { + CommandCapture::from_run( + vec!["tool".to_owned()], + outcome, + code.map(status), + b"out\n".to_vec(), + b"err\n".to_vec(), + detail.map(str::to_owned), + ) + } + + /// A runner that gives each candidate the next scripted capture, and records + /// which candidates it was asked to start. + fn scripted( + mut script: Vec, + asked: &mut Vec, + ) -> impl FnMut(&Path) -> CommandCapture + '_ { + script.reverse(); + move |path| { + asked.push(path.display().to_string()); + script + .pop() + .expect("more candidates were started than scripted") + } + } + + #[test] + fn the_next_spelling_is_tried_only_when_a_program_could_not_be_started() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b", "c"]), + scripted( + vec![ + finished(CommandOutcome::NotFound, None, Some("no a")), + finished( + CommandOutcome::SpawnFailed(std::io::ErrorKind::PermissionDenied), + None, + Some("no b"), + ), + finished(CommandOutcome::Exited(0), Some(0), None), + ], + &mut asked, + ), + ); + + assert_eq!(asked, ["a", "b", "c"]); + assert_eq!(result.program.as_deref(), Some(Path::new("c"))); + assert_eq!(result.status, Some(status(0).to_string())); + assert_eq!( + (result.stdout.as_str(), result.stderr.as_str()), + ("out\n", "err\n") + ); + assert!(result.error.is_none() && !result.timed_out); + } + + #[test] + fn a_program_that_never_starts_reports_the_last_launch_error() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![ + finished(CommandOutcome::NotFound, None, Some("no a")), + finished(CommandOutcome::NotFound, None, Some("no b")), + ], + &mut asked, + ), + ); + + assert!(result.program.is_none() && result.status.is_none()); + assert_eq!(result.error.as_deref(), Some("failed to launch b: no b")); + assert!(result.stdout.is_empty() && !result.timed_out); + + let none = capture_diagnostic_with("tool", &[], |_| unreachable!("no candidate")); + assert_eq!(none.error.as_deref(), Some("tool was not found")); + } + + #[test] + fn a_non_zero_exit_is_final_and_reports_its_status_and_both_streams() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![finished(CommandOutcome::Exited(3), Some(3), None)], + &mut asked, + ), + ); + + assert_eq!(asked, ["a"], "a run that started must not be retried"); + assert_eq!(result.program.as_deref(), Some(Path::new("a"))); + assert_eq!(result.status, Some(status(3).to_string())); + #[cfg(windows)] + assert_eq!(result.status.as_deref(), Some("exit code: 3")); + assert_eq!( + (result.stdout.as_str(), result.stderr.as_str()), + ("out\n", "err\n") + ); + assert!(result.error.is_none() && !result.timed_out); + } + + #[test] + fn a_timeout_is_final_keeps_the_output_that_arrived_and_says_so() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![finished(CommandOutcome::TimedOut, Some(1), None)], + &mut asked, + ), + ); + + assert_eq!(asked, ["a"], "a hung program must not be started again"); + assert!(result.timed_out); + assert!( + result.status.is_none(), + "a killed child has no status to report" + ); + assert_eq!( + (result.stdout.as_str(), result.stderr.as_str()), + ("out\n", "err\n") + ); + assert!(result.error.is_none()); + } + + #[test] + fn a_failure_to_wait_is_final_and_reports_no_output() { + let mut asked = Vec::new(); + let result = capture_diagnostic_with( + "tool", + &candidates(&["a", "b"]), + scripted( + vec![finished( + CommandOutcome::WaitFailed, + None, + Some("the os said no"), + )], + &mut asked, + ), + ); + + assert_eq!(asked, ["a"]); + assert_eq!(result.program.as_deref(), Some(Path::new("a"))); + assert_eq!( + result.error.as_deref(), + Some("failed to wait: the os said no") + ); + assert!(result.stdout.is_empty() && result.stderr.is_empty()); + assert!(!result.timed_out && result.status.is_none()); + } +} diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index a8181620d0..36950d0c36 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -86,7 +86,9 @@ pub use binary_paths::{ daemon_binary_path, engine_binary_path, sibling_binary_path, sibling_binary_path_from, }; pub use codex_bridge::{CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot}; -pub use command_capture::{CommandCapture, command_capture_text, run_rocm_capture_with_env}; +pub use command_capture::{ + CommandCapture, CommandOutcome, command_capture_text, run_rocm_capture_with_env, +}; pub use config::{ AutomationsConfig, BuiltinWatcherSpec, DashboardConfig, DashboardDaemonConfig, DashboardTuiConfig, EngineUserConfig, ManagedToolConfig, PERMISSIONS_MODE_ASK, diff --git a/crates/rocm-core/src/probe_exec.rs b/crates/rocm-core/src/probe_exec.rs index 37dca2b437..e632612bd6 100644 --- a/crates/rocm-core/src/probe_exec.rs +++ b/crates/rocm-core/src/probe_exec.rs @@ -10,16 +10,17 @@ //! [`crate::examine`] so `fix`, `process` and `host_gpu` can reach them without //! importing the module that owns the `Examination` document. +use crate::command_capture::{CommandCapture, CommandOutcome}; +use crate::process::run_bounded; use crate::runtime_is_windows; use std::ffi::{OsStr, OsString}; -use std::io::Read; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; +use std::process::Command; +use std::time::Duration; -/// Run a command with a timeout. Returns `(rc, stdout, stderr)`. `rc` is `127` -/// when the program can't be spawned and `124` on timeout. +/// Run a command with a timeout. Returns `(rc, stdout, stderr)`. `rc` is the +/// exit code, `-1` for a child killed by a signal, `127` when the program can't +/// be spawned and `124` on timeout or when the child could not be waited for. pub(crate) fn run(program: &str, args: &[&str], timeout: Duration) -> (i32, String, String) { run_with_env(program, args, &[], timeout) } @@ -40,66 +41,36 @@ pub(crate) fn run_with_env( } /// [`run`] without the UTF-8 assumption, for output that is not UTF-8. +/// +/// A thin adapter over [`run_bounded`]: the executor drains both pipes as bytes +/// (so one stray byte cannot empty a capture, which would read downstream as +/// "the command printed nothing") and bounds the wait, and [`run_result`] turns +/// how the run ended into the `rc` this module has always returned. fn run_raw( program: &str, args: &[&str], envs: &[(String, OsString)], timeout: Duration, ) -> (i32, Vec, Vec) { - let Ok(mut child) = Command::new(program) + let mut command = Command::new(program); + command .args(args) - .envs(envs.iter().map(|(key, value)| (key, value))) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - else { - return (127, Vec::new(), Vec::new()); - }; - // Bytes, then a lossy conversion at the end. `read_to_string` FAILS on - // invalid UTF-8 and the error was discarded, so a single stray byte silently - // emptied the whole capture — which reads downstream as "the command printed - // nothing", not as "the output could not be decoded". - let stdout_handle = child.stdout.take().map(|mut stdout| { - thread::spawn(move || { - let mut buf = Vec::new(); - let _ = stdout.read_to_end(&mut buf); - buf - }) - }); - let stderr_handle = child.stderr.take().map(|mut stderr| { - thread::spawn(move || { - let mut buf = Vec::new(); - let _ = stderr.read_to_end(&mut buf); - buf - }) - }); - let deadline = Instant::now() + timeout; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break Some(status), - Ok(None) => { - if Instant::now() >= deadline { - let _ = child.kill(); - let _ = child.wait(); - break None; - } - thread::sleep(Duration::from_millis(20)); - } - Err(_) => break None, - } - }; - let stdout: Vec = stdout_handle - .and_then(|handle| handle.join().ok()) - .unwrap_or_default(); - let stderr: Vec = stderr_handle - .and_then(|handle| handle.join().ok()) - .unwrap_or_default(); - let rc = match status { - Some(status) => status.code().unwrap_or(-1), - None => 124, + .envs(envs.iter().map(|(key, value)| (key, value))); + run_result(run_bounded(command, timeout, "process")) +} + +/// `(rc, stdout, stderr)` of a finished run: the exit code when there is one, +/// `-1` for a child killed by a signal, `124` for a timeout or a failure to +/// wait, and `127` for a program that could not be started. The bytes are what +/// arrived, a timeout and a pipe that failed to read included. +fn run_result(capture: CommandCapture) -> (i32, Vec, Vec) { + let rc = match capture.outcome { + CommandOutcome::Exited(code) => code, + CommandOutcome::Signaled(_) => -1, + CommandOutcome::TimedOut | CommandOutcome::WaitFailed => 124, + CommandOutcome::NotFound | CommandOutcome::SpawnFailed(_) => 127, }; - (rc, stdout, stderr) + (rc, capture.stdout_raw, capture.stderr_raw) } /// Run a command whose output is UTF-16LE, as `wsl.exe`'s is. @@ -313,6 +284,58 @@ mod tests { ); } + fn capture_that(outcome: CommandOutcome, detail: Option<&str>) -> CommandCapture { + CommandCapture::from_run( + vec!["tool".to_owned()], + outcome, + None, + b"out".to_vec(), + b"err".to_vec(), + detail.map(str::to_owned), + ) + } + + #[test] + fn a_run_reads_as_the_rc_run_raw_has_always_returned() { + // Each outcome with the bytes it carries: the rc is the whole translation, + // and the bytes arrive whatever the outcome (a timeout's partial output + // included), as `run_raw` returned them after joining its readers. + let cases = [ + (CommandOutcome::Exited(0), 0), + (CommandOutcome::Exited(3), 3), + (CommandOutcome::Signaled(Some(9)), -1), + (CommandOutcome::Signaled(None), -1), + (CommandOutcome::TimedOut, 124), + (CommandOutcome::WaitFailed, 124), + (CommandOutcome::NotFound, 127), + ( + CommandOutcome::SpawnFailed(std::io::ErrorKind::PermissionDenied), + 127, + ), + ]; + for (outcome, rc) in cases { + assert_eq!( + run_result(capture_that(outcome, None)), + (rc, b"out".to_vec(), b"err".to_vec()), + "{outcome:?}" + ); + } + } + + #[test] + fn a_pipe_that_failed_to_read_does_not_change_what_run_returns() { + // The executor reports a read error in `detail` and keeps the bytes that + // arrived with the child's real status; `run_with_timeout` turns that into + // an error, and `run` has always discarded it (`let _ = read_to_end`). + assert_eq!( + run_result(capture_that( + CommandOutcome::Exited(0), + Some("failed to read the output: broken pipe") + )), + (0, b"out".to_vec(), b"err".to_vec()) + ); + } + #[test] fn env_truncation_is_char_safe_across_multibyte_boundary() { // 5000 'é' chars = 10000 bytes; byte 4000 lands inside a char, which a diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index f7744ba489..5751d17c74 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -10,19 +10,11 @@ //! processes — e.g. the daemon autostart path, the managed-launch GPU claim, //! a `uv` install, and each JSONL store's append/rewrite. //! -//! [`run_with_timeout`] is the bulk of the file: run a command to completion -//! with both streams captured and a real time bound. A thread per pipe drains -//! stdout and stderr from the moment the child starts, so output size cannot -//! stall a command into its own timeout, and a reader that cannot finish is -//! abandoned rather than joined, so a grandchild holding the pipe's write end -//! open cannot outlast the deadline either. The timeout error quotes a bounded -//! head of the child's own output, because that text reaches persisted -//! automation logs. `Capture`, `run_captured`, `wait_bounded`, `Waitable` and -//! `assemble_output` exist to serve it. -//! -//! It is not yet the only such helper in the tree; converging the remaining -//! capture helpers onto it is tracked in -//! [#623](https://github.com/ROCm/rocm-cli/issues/623). +//! The bounded subprocess executor — [`run_with_timeout`], which runs a command +//! to completion with both streams captured and a real time bound, and +//! [`run_bounded`], the same executor returning a typed capture — lives in the +//! private `bounded` submodule, split out to keep this file under the module +//! size limit, and is re-exported from here under its original paths. use crate::DEFAULT_LOCAL_HOST; use anyhow::{Context, Result, bail}; @@ -35,6 +27,9 @@ use std::net::TcpListener; #[cfg(windows)] use std::os::windows::ffi::OsStrExt; use std::path::{Path, PathBuf}; +use std::process::Command; +use std::thread; +use std::time::{Duration, Instant}; #[cfg(windows)] use windows_sys::Win32::System::Threading::{ CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, CREATE_UNICODE_ENVIRONMENT, @@ -44,13 +39,9 @@ use windows_sys::Win32::System::Threading::{ WaitForSingleObject, }; -use crate::probe_exec::truncate_to_chars; -use std::io::{ErrorKind, Read}; -use std::process::{Child, Command, ExitStatus, Output, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::thread; -use std::time::{Duration, Instant}; +mod bounded; +pub use bounded::run_with_timeout; +pub(crate) use bounded::{MAX_QUOTED_OUTPUT_CHARS, run_bounded}; #[cfg(windows)] pub fn spawn_detached_no_inherit( @@ -1013,422 +1004,6 @@ fn windows_environment_block(env_overrides: &[(&str, &Path)]) -> Vec { block } -/// How often the wait loop polls for the child's exit. -const POLL_INTERVAL: Duration = Duration::from_millis(50); - -/// How much a reader accumulates before publishing it to the shared buffer. -const READ_CHUNK: usize = 16 * 1024; - -/// How long the readers get to finish once the child is no longer running. -/// -/// Reaching this means something other than the child still holds the pipe's -/// write end — a grandchild the child left behind — so there is no EOF coming -/// and waiting longer would make `timeout` meaningless. -/// -/// For a child that genuinely finished the grace has only the tail to absorb: a -/// child cannot exit before writing everything, and the readers are started -/// before the wait loop. What they are not is *scheduled* before it — -/// `wait_bounded` polls `try_wait` once with no sleep in front of it, so a child -/// fast enough to exit between spawn and that first poll can be observed exited -/// while neither reader thread has run a single instruction. So the grace covers -/// reader start-up as well as the last buffer or two. 200ms absorbs both with -/// room to spare, including on the no-GPU e2e lane's 64 concurrent scenarios. -const DRAIN_GRACE: Duration = Duration::from_millis(200); - -/// How much of a timed-out child's own output the timeout error quotes. -/// -/// Bounded here rather than at any one caller because the quote is assembled -/// here: the timeout message is this function's own text, and a caller sees it -/// only as an opaque error, so this is the only place that can still tell which -/// of its characters came from the child. All four callers inherit it. -/// -/// It bounds this message by itself. A caller that quotes the `Output` handed -/// back to it is quoting raw child output of its own, persisted the same way, -/// so it has to apply a bound too -- which is why this is `pub(crate)` rather -/// than private: [`crate::amd_smi::amd_smi_json`] bounds its non-zero-exit -/// quote (the one #623 tracked) with *this* constant instead of declaring a -/// second one free to drift from it. The daemon's -/// `parse_sandbox_child_output` still predates both and is outside this scope. -/// -/// The bound matters because the message does not stop at a terminal. `rocmd` -/// renders the whole error chain into the bridge snapshot's `note`, and the -/// GPU-metrics watcher copies that note into the automation-event and audit -/// JSONL files once a minute — stores that are never rotated or capped. -/// Unbounded, a child that prints a megabyte and then stalls past its -/// timeout grows both logs by that much per minute, forever. A 151K-character -/// JSONL line was reproduced from exactly that poller. -/// -/// Sized to still hold a real stderr backtrace: the point is to stop a -/// payload-sized quote, not to shorten diagnostics. -pub(crate) const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; - -/// One captured stream: the bytes so far, whether its reader saw EOF, and the -/// read error that stopped it, if one did. -/// -/// # Lifetime of the reader thread -/// -/// The thread outlives the call that started it whenever the drain grace runs -/// out — that is the point of the grace — and it then lives until the pipe's -/// last write end closes, which is the abandoned grandchild's business, not -/// ours. What it must *not* do is keep growing: `take()` marks the capture -/// abandoned, and from that moment the reader drains to EOF and discards, so a -/// buffer nobody will read again cannot grow without bound in a daemon that -/// probes every 60 seconds. -/// -/// What that bounds is the memory, and only the memory. The *number* of these -/// threads is not bounded here: each abandoned capture leaves one parked in -/// `read` until the grandchild lets go, so every probe that meets one costs two -/// until it does. The trade is deliberate — a parked reader holds a stack and -/// no CPU, and the alternative is the one below. -/// -/// Discarding rather than closing the read end is deliberate. Dropping the pipe -/// would send the still-writing grandchild `EPIPE`/`SIGPIPE`, which for the -/// background helpers a sandboxed installer legitimately leaves behind would -/// turn a leak whose cost we know into a behaviour change we cannot predict. -#[derive(Clone)] -struct Capture { - buffer: Arc>>, - finished: Arc, - failure: Arc>>, - /// Set by `take()`: nothing will read this capture's bytes again. - /// - /// Read and written under `buffer`'s mutex, so "emptied and marked" is one - /// step as far as the reader is concerned. Checking it outside the lock - /// would leave the window where a reader that has just passed the check - /// appends to the buffer `take()` is in the middle of emptying. - abandoned: Arc, -} - -impl Capture { - /// Start draining `pipe` on its own thread. - /// - /// Publishing in chunks as they arrive, rather than returning the whole - /// buffer at EOF, is what lets the caller take partial output from a reader - /// that is still blocked. - fn draining(pipe: Option) -> Self { - let capture = Self { - buffer: Arc::new(Mutex::new(Vec::new())), - finished: Arc::new(AtomicBool::new(false)), - failure: Arc::new(Mutex::new(None)), - abandoned: Arc::new(AtomicBool::new(false)), - }; - let Some(mut pipe) = pipe else { - capture.finished.store(true, Ordering::Release); - return capture; - }; - let sink = capture.clone(); - thread::spawn(move || { - let mut chunk = [0_u8; READ_CHUNK]; - loop { - match pipe.read(&mut chunk) { - Ok(0) => break, - Ok(read) => { - // Keep draining once abandoned, but stop accumulating: - // see the type's doc. The check shares the buffer's - // lock with `take()`, so after `take()` returns this - // buffer cannot grow by another byte. - let mut buffer = sink.lock(); - if !sink.abandoned.load(Ordering::Acquire) { - buffer.extend_from_slice(&chunk[..read]); - } - } - // `read_to_end` — what `wait_with_output` used before this - // function existed — retries an interrupted read and - // returns every other error. Treating either as EOF would - // silently shorten the output instead, and the callers - // parse these bytes as JSON, where that surfaces as a - // parse error pointing at the wrong thing. - // Round again rather than break: not the end of anything. - Err(error) if error.kind() == ErrorKind::Interrupted => {} - Err(error) => { - // Recorded only while someone is still listening. Once - // `take()` has fully returned this is merely tidy — - // nothing calls `take()` twice, so a failure stored - // afterwards would just sit unread. - // - // It is load-bearing for one window, and that window is - // inside `take()` itself: `take()` marks `abandoned` - // and empties the buffer under `buffer`'s lock, then - // releases it and drains the failure slot under a - // *different* lock. A read error that lands in between - // would otherwise be handed back attached to bytes the - // caller has already been given, and `assemble_output` - // turns any failure on a non-timed-out run into a hard - // `failed to read the output of ...` — a cleanly - // finished command reported as a read failure. Taking - // `buffer`'s lock before the check is what closes it; - // a bare load could pass the check, then have `take()` - // run the whole sequence before the store. - let _buffer = sink.lock(); - if !sink.abandoned.load(Ordering::Acquire) { - *sink.failure() = Some(error); - } - break; - } - } - } - sink.finished.store(true, Ordering::Release); - }); - capture - } - - fn lock(&self) -> std::sync::MutexGuard<'_, Vec> { - self.buffer - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - fn failure(&self) -> std::sync::MutexGuard<'_, Option> { - self.failure - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - fn is_finished(&self) -> bool { - self.finished.load(Ordering::Acquire) - } - - /// Take what has been captured, leaving a still-running reader unblocked. - /// - /// The bytes come back alongside any read error rather than instead of it, - /// so a caller already reporting something else — a timeout — can still - /// quote what it managed to collect. - /// - /// This is also the point of no return for the reader: whatever it collects - /// after this is discarded, because there is no second caller to hand it to. - fn take(&self) -> (Vec, Option) { - let bytes = { - // Marked while the buffer is held, so the reader cannot append - // between the mark and the emptying and strand bytes in a buffer - // that is never read again. - let mut buffer = self.lock(); - self.abandoned.store(true, Ordering::Release); - std::mem::take(&mut *buffer) - }; - (bytes, self.failure().take()) - } - - /// Mark the capture abandoned and release what it holds, for an error path - /// that returns before `take()` would. - /// - /// `take()` is the only place that sets `abandoned`, so going through it - /// keeps "marked" and "emptied" a single step with one implementation. The - /// bytes and any read error are dropped because a caller on this path is - /// already reporting something else. - fn abandon(&self) { - drop(self.take()); - } -} - -/// Run `command` to completion with both output streams captured, giving up -/// after `timeout`. -/// -/// A thread per pipe drains stdout and stderr, and both are started *before* -/// the wait loop. That ordering is the whole point. Waiting for the child to -/// exit and only then calling `wait_with_output` — the shape this replaced — -/// deadlocks: the pipes are bounded by the OS (commonly ~64KiB, less on -/// Windows), so a child that writes more than that blocks in `write`, which -/// means it never exits, which means the drain is never reached. The only way -/// out was the timeout, so it measured how much the child printed rather than -/// how long it took, and killed successful commands mid-flight. -/// -/// Both pipes need their own reader for the same reason: draining stdout to -/// completion and only then reading stderr stalls on a child that fills stderr -/// first. -/// -/// The call returns within roughly `timeout` either way: once the child is no -/// longer running the readers get `DRAIN_GRACE` to finish and are then abandoned -/// rather than joined. Joining unconditionally would hand the schedule to a -/// grandchild the child left behind, which holds the pipe's write end open so no -/// EOF ever arrives — that is how an abandoned `sleep` comes to decide how long -/// a 2-second probe takes. -/// -/// The stdio configuration is this function's own: it sets null stdin and piped -/// stdout/stderr on `command`, overriding whatever the caller set, because -/// capturing both streams is the whole contract. Callers need not — and should -/// not — set them. The stdin half is a promise in its own right rather than -/// housekeeping: a child that reads from an inherited stdin waits for input -/// that is never coming, so it is the deadline that ends it. Pinned by -/// `a_child_that_reads_stdin_is_given_an_empty_one`, on both platforms. -/// -/// `label` names the subject in error messages (e.g. `"process"`, -/// `"sandbox process"`). -pub fn run_with_timeout(mut command: Command, timeout: Duration, label: &str) -> Result { - let mut child = command - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .with_context(|| format!("failed to spawn {label}"))?; - - // Taking the pipes here is what obliges us to assemble `Output` by hand - // below: `wait_with_output` reads the handles it owns, and these are gone. - let stdout = Capture::draining(child.stdout.take()); - let stderr = Capture::draining(child.stderr.take()); - - run_captured(child, stdout, stderr, timeout, label) -} - -/// The rest of [`run_with_timeout`] once the child is spawned and both readers -/// are running. -/// -/// Generic over [`Waitable`] rather than taking a `Child` so that the wait -/// loop's two failure paths — which are the only ways out of this function that -/// skip the normal `take()` — are reachable from a test. -fn run_captured( - mut child: C, - stdout: Capture, - stderr: Capture, - timeout: Duration, - label: &str, -) -> Result { - // The single error path between `Capture::draining` above and the `take()` - // pair below, and it must abandon both captures before it propagates. - // `take()` is what marks a reader abandoned, and an abandoned reader is - // what stops a buffer nobody will read again from growing for as long as a - // grandchild holds the pipe open. Returning the error straight out of the - // wait loop — the shape this replaced — left both readers appending - // forever, in a daemon that probes every 60 seconds. - let (status, timed_out) = match wait_bounded(&mut child, timeout, label) { - Ok(outcome) => outcome, - Err(error) => { - stdout.abandon(); - stderr.abandon(); - return Err(error); - } - }; - - let drained_by = Instant::now() + DRAIN_GRACE; - while !(stdout.is_finished() && stderr.is_finished()) && Instant::now() < drained_by { - thread::sleep(POLL_INTERVAL.min(DRAIN_GRACE / 4)); - } - assemble_output( - status, - timed_out, - stdout.take(), - stderr.take(), - label, - timeout, - ) -} - -/// Wait for `child` to exit, killing it once `timeout` has passed. -/// -/// Returns the status and whether the deadline is what ended the wait. Kept -/// free of the captures so that it has exactly one caller to get the -/// abandon-on-error handling right, rather than one per `?`. -fn wait_bounded( - child: &mut C, - timeout: Duration, - label: &str, -) -> Result<(ExitStatus, bool)> { - let deadline = Instant::now() + timeout; - loop { - let polled = child.try_wait(); - if polled.is_err() { - // The only exit from this function that is not already followed by - // a kill. Whatever stopped us polling, we are about to stop - // waiting, and the caller drops the child without ever reaping it; - // leaving it running would hand back a process nobody is watching. - // `let _` to match the timeout branch: the kill is best-effort, and - // the poll error is the one worth reporting. - let _ = child.kill(); - } - if let Some(status) = polled.with_context(|| format!("failed to poll {label}"))? { - return Ok((status, false)); - } - if Instant::now() >= deadline { - let _ = child.kill(); - let status = child - .wait() - .with_context(|| format!("failed to reap timed-out {label}"))?; - return Ok((status, true)); - } - thread::sleep(POLL_INTERVAL); - } -} - -/// The three child operations [`wait_bounded`] needs. -/// -/// A trait only so a test can fail `try_wait` or the post-kill `wait` on -/// demand: both are error paths that skip the normal `take()`, and provoking -/// either from a real child needs the kernel to refuse to reap a process. -trait Waitable { - fn try_wait(&mut self) -> std::io::Result>; - fn kill(&mut self) -> std::io::Result<()>; - fn wait(&mut self) -> std::io::Result; -} - -// `Child::` rather than `Self::` on purpose: inside this impl `Self` is -// `Child`, so `Self::try_wait` reads as the trait method being defined -// here rather than the inherent one being forwarded to. -#[allow(clippy::use_self)] -impl Waitable for Child { - fn try_wait(&mut self) -> std::io::Result> { - Child::try_wait(self) - } - - fn kill(&mut self) -> std::io::Result<()> { - Child::kill(self) - } - - fn wait(&mut self) -> std::io::Result { - Child::wait(self) - } -} - -/// Turn what the wait loop and the two readers produced into the call's result. -/// -/// Split out from [`run_with_timeout`] because it is the only way to test it. -/// Every branch here is plain logic over values already in hand — which error -/// wins, which stream gets quoted — but reaching it through a real child means -/// making a pipe fail on demand, which needs a kernel having a bad day. Given -/// the two `(bytes, error)` pairs directly, each branch is an ordinary unit -/// test; see the cases at the bottom of this file. -fn assemble_output( - status: ExitStatus, - timed_out: bool, - (stdout, stdout_failure): (Vec, Option), - (stderr, stderr_failure): (Vec, Option), - label: &str, - timeout: Duration, -) -> Result { - let output = Output { - status, - stdout, - stderr, - }; - - if timed_out { - // `command_failure_detail` is the crate's one "which stream explains - // this failure" rule — stderr, else stdout, else "no output" — shared - // since #584 so callers cannot quietly disagree about the order. - // - // Truncated, not dropped: what the child managed to say before it - // stalled is usually the whole explanation, and the timeout is named - // first either way, so a reader who only sees the head still learns - // which command gave up and why. - bail!( - "{label} exceeded {}s timeout: {}", - timeout.as_secs(), - truncate_to_chars( - crate::command_failure_detail(&output), - MAX_QUOTED_OUTPUT_CHARS, - ) - ); - } - - // Only once the timeout has had its say: a killed child whose pipe then - // fails to read is a timeout, and reporting it as a read error would name - // the symptom instead of the cause. - if let Some(error) = stdout_failure.or(stderr_failure) { - return Err(anyhow::Error::new(error)) - .with_context(|| format!("failed to read the output of {label}")); - } - - Ok(output) -} - /// Property tests for [`walk_process_tree`]; a child module so it can reach /// the walk while it stays private. #[cfg(all(test, not(windows)))] @@ -1785,932 +1360,6 @@ mod tests { let _ = fs::remove_dir_all(&dir); } - - /// The shells every grandchild case below is run against. - /// - /// Whether a shell forks or execs decides whether a grandchild exists at - /// all, so it decides whether these tests test anything: dash forks `sleep` - /// in `sh -c "sleep 30"`, bash execs it into the shell's own process and no - /// grandchild is left holding the pipe. A test written against one is dead - /// weight on a host running the other, and which one `/bin/sh` is varies - /// across the distributions this ships on. So the scripts below are written - /// to fork under either, and each case runs under every shell present to - /// keep that true. - #[cfg(unix)] - fn grandchild_shells() -> Vec<&'static str> { - let shells: Vec<&'static str> = ["/bin/sh", "/bin/bash", "/bin/dash"] - .into_iter() - .filter(|shell| std::path::Path::new(shell).exists()) - .collect(); - assert!( - !shells.is_empty(), - "no shell to run the grandchild cases in" - ); - shells - } - - #[test] - fn a_command_that_succeeds_reports_its_output() { - let mut command = Command::new(if cfg!(windows) { "cmd" } else { "sh" }); - if cfg!(windows) { - // No space around `&`: cmd would carry a trailing one into the - // echoed text, which `trim` hides on stdout but is easier not to - // write than to explain. - command.args(["/C", "echo hello&echo oops 1>&2"]); - } else { - command.args(["-c", "echo hello; echo oops >&2"]); - } - - let output = run_with_timeout(command, Duration::from_secs(10), "process").expect("ran"); - - assert!(output.status.success()); - assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "hello"); - // Asserted on Windows too: `1>&2` is a cmd redirection like any other, - // and skipping it there left the second pipe — the one whose separate - // reader this function exists to provide — unchecked on that platform. - assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), "oops"); - } - - /// What a stdin-reading fixture prints when its read reaches EOF. - const STDIN_EOF_MARKER: &str = "eof"; - - /// The assertions shared by both platforms' empty-stdin cases. - /// - /// `command` must run a child that reads one line from stdin and prints - /// [`STDIN_EOF_MARKER`] if that read reaches EOF instead. - /// - /// The stdin set on `command` here is the read end of a pipe whose write end - /// this function holds open across the whole call, so it never reaches EOF - /// of its own accord. That is what makes the assertion two-sided rather than - /// a tautology: were `run_with_timeout` to stop setting null stdin, the - /// child would sit in `read` until the deadline killed it — exactly how a - /// child that prompts behaves against an inherited terminal — and the call - /// would come back as a timeout rather than with the marker. - /// - /// Setting stdin on the caller's side is also the contract under test: this - /// function's own stdio overrides whatever the caller set, which is what - /// lets all four callers hand over a bare `Command` and say nothing about - /// stdin at all. - /// - /// One thing it does not pin, said plainly rather than left to be - /// discovered: `null` against `inherit`. Telling those two apart needs this - /// test process's own fd 0 to be something other than EOF, and arranging - /// that is a `dup2` racing every other test in the binary. An `inherit` - /// mutant was run and does pass here. What is caught is the regression that - /// can actually happen -- no stdin setting at all, which is the line this - /// function took over from its callers. - /// - /// The race is avoidable by re-execing this test binary under a stdin pipe - /// the outer test holds open and asserting in that child, which would fail - /// the `inherit` mutant too. Not done: it buys one mutant that no plausible - /// edit produces -- nothing would replace `null` with `inherit`, the stdio - /// mode this helper exists to stop callers from being handed -- at the cost - /// of a self-re-execing test whose own harness plumbing is then the thing - /// most likely to break. - fn assert_the_child_is_given_an_empty_stdin(mut command: Command) { - let (stdin_reader, stdin_writer) = - std::io::pipe().expect("failed to create a pipe for the child's stdin"); - command.stdin(Stdio::from(stdin_reader)); - - let started = Instant::now(); - let output = run_with_timeout(command, Duration::from_secs(5), "process").expect( - "a child given an empty stdin must read EOF rather than block until the deadline", - ); - let elapsed = started.elapsed(); - // Dropped only here: releasing the write end any earlier would close the - // pipe, so the child would reach EOF through the caller's stdin and the - // test would pass whatever this function set. - drop(stdin_writer); - - assert!(output.status.success(), "{output:?}"); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - STDIN_EOF_MARKER, - "the child must be given an empty stdin rather than the one the caller set" - ); - assert!( - elapsed < Duration::from_secs(3), - "the read must hit EOF at once rather than waiting out the deadline, but the call \ - took {elapsed:?}" - ); - } - - #[cfg(unix)] - #[test] - fn a_child_that_reads_stdin_is_given_an_empty_one() { - // `read` reports EOF by returning non-zero, so the `||` branch is the - // one an empty stdin takes. A stdin left open takes neither. - let script = format!("read line || echo {STDIN_EOF_MARKER}"); - let mut command = Command::new("sh"); - command.args(["-c", script.as_str()]); - - assert_the_child_is_given_an_empty_stdin(command); - } - - #[cfg(unix)] - #[test] - fn a_genuinely_slow_command_times_out_on_time() { - for shell in grandchild_shells() { - // `sleep 30 & wait` rather than plain `sleep 30`: a lone final - // command is exec-ed into the shell itself by bash, which would - // leave nothing behind to hold the pipe and quietly turn this into - // a test of the easy case. Backgrounding forces a real grandchild - // under every shell. - // - // Killing the shell at the deadline does not kill that grandchild, - // so it keeps the inherited write end of both pipes open and no EOF - // arrives. An implementation that joins its readers therefore waits - // out the full 30 seconds and the timeout bounds nothing — which - // for the 2-second amd-smi probe is the difference between a snappy - // telemetry read and a stalled one. - let mut command = Command::new(shell); - command.args(["-c", "sleep 30 & wait"]); - - let started = Instant::now(); - let error = run_with_timeout(command, Duration::from_secs(1), "process") - .expect_err("a command that outlives its timeout must fail"); - let elapsed = started.elapsed(); - - assert!( - error.to_string().contains("exceeded 1s timeout"), - "the timeout must be reported as one, under {shell}: {error}" - ); - assert!( - elapsed < Duration::from_secs(5), - "the call must return at its deadline rather than waiting for an abandoned \ - grandchild to exit, but under {shell} it took {elapsed:?}" - ); - } - } - - #[cfg(unix)] - #[test] - fn a_grandchild_holding_the_pipe_does_not_hold_a_successful_call() { - for shell in grandchild_shells() { - // The timeout case above covers a child that is killed. This is the - // same hazard on the path that has no timeout to blame: the child - // exits 0 straight away, having left a background `sleep` holding - // the write end of both pipes, so there is no EOF for another five - // seconds. Joining the readers — or giving them an unbounded grace - // — hands the schedule to that `sleep` and a command that finished - // instantly takes 5s. `DRAIN_GRACE` is what bounds it. - let mut command = Command::new(shell); - command.args(["-c", "echo ready; sleep 5 &"]); - - let started = Instant::now(); - let output = run_with_timeout(command, Duration::from_secs(20), "process") - .expect("a child that exits cleanly must not be reported as a timeout"); - let elapsed = started.elapsed(); - - assert!(output.status.success(), "under {shell}: {output:?}"); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - "ready", - "what the child wrote before exiting must still arrive, under {shell}" - ); - // Generous against a loaded machine, and still a third of the - // grandchild's lifetime: the regression this guards takes the full - // five seconds, not a few hundred milliseconds more than the grace. - assert!( - elapsed < Duration::from_secs(3), - "the call must not wait for the grandchild, but under {shell} it took {elapsed:?}" - ); - } - } - - #[cfg(unix)] - #[test] - fn output_arriving_after_the_child_exits_is_still_collected() { - for shell in grandchild_shells() { - // What `DRAIN_GRACE` is for, and the only way to ask for it on - // demand. A child that writes and exits normally cannot pin it: its - // reader is already running and drains the pipe microseconds later, - // long before the wait loop's next poll even notices the exit, so - // taking the buffer the instant the child is seen to have exited - // still collects everything. Here a grandchild writes once the - // child is gone instead. - // - // The 0.1s sits between the two bounds that decide the outcome: - // past `POLL_INTERVAL` (50ms), the longest the wait loop can take - // to notice the exit, so taking the output there misses - // `late-canary` outright; and well inside `DRAIN_GRACE` (200ms), so - // draining collects it. Both margins are ~2x, and the grandchild - // exits immediately after writing, so a working implementation sees - // EOF and returns rather than sitting out the whole grace. - let mut command = Command::new(shell); - command.args(["-c", "echo early-canary; (sleep 0.1; echo late-canary) &"]); - - let started = Instant::now(); - let output = run_with_timeout(command, Duration::from_secs(20), "process") - .expect("a child that exits cleanly must not be reported as a timeout"); - let elapsed = started.elapsed(); - let stdout = String::from_utf8_lossy(&output.stdout); - - assert!( - stdout.contains("early-canary"), - "under {shell}, the child's own output is missing: {stdout:?}" - ); - assert!( - stdout.contains("late-canary"), - "under {shell}, output still in flight when the child exited was dropped \ - instead of drained: {stdout:?}" - ); - assert!( - elapsed < Duration::from_secs(3), - "draining must end at EOF rather than running long, but under {shell} the call \ - took {elapsed:?}" - ); - } - } - - #[test] - fn an_interrupted_read_is_retried_and_a_failed_one_is_reported() { - // Neither branch is reachable through a real child here — EINTR needs a - // signal to land mid-read, and a pipe read failing outright needs the - // kernel to be having a bad day — so the reader is fed a scripted - // stream directly. The property is the one `read_to_end` gave before - // this function replaced it: an interrupted read is not the end of the - // stream, and a failed one is not a successful short read. - struct ScriptedPipe(std::collections::VecDeque>); - - impl Read for ScriptedPipe { - fn read(&mut self, out: &mut [u8]) -> std::io::Result { - match self.0.pop_front() { - Some(Ok(bytes)) => { - out[..bytes.len()].copy_from_slice(bytes); - Ok(bytes.len()) - } - Some(Err(error)) => Err(error), - None => Ok(0), - } - } - } - - fn drain(script: Vec>) -> (Vec, Option) { - let capture = Capture::draining(Some(ScriptedPipe(script.into()))); - let deadline = Instant::now() + Duration::from_secs(10); - while !capture.is_finished() && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!(capture.is_finished(), "the reader never finished"); - capture.take() - } - - let (bytes, failure) = drain(vec![ - Ok(b"before"), - Err(std::io::Error::from(ErrorKind::Interrupted)), - Ok(b"after"), - Ok(b""), - ]); - assert_eq!( - String::from_utf8_lossy(&bytes), - "beforeafter", - "an interrupted read must be retried, not taken for the end of the stream" - ); - assert!(failure.is_none(), "{failure:?}"); - - let (bytes, failure) = drain(vec![ - Ok(b"partial"), - Err(std::io::Error::from(ErrorKind::BrokenPipe)), - Ok(b"unreachable"), - ]); - assert_eq!(String::from_utf8_lossy(&bytes), "partial"); - assert_eq!( - failure.map(|error| error.kind()), - Some(ErrorKind::BrokenPipe), - "a read that failed must be reported rather than passed off as EOF" - ); - } - - /// A pipe that never ends until the test says so. - /// - /// Stoppable rather than genuinely endless so the reader thread finishes - /// with the test instead of spinning for the life of the test binary. - struct EndlessPipe { - reads: Arc, - stop: Arc, - } - - impl Read for EndlessPipe { - fn read(&mut self, out: &mut [u8]) -> std::io::Result { - if self.stop.load(Ordering::Acquire) { - return Ok(0); - } - // Paced so the test can interleave with the reader rather than - // racing a tight loop, and so a regression accumulates a size the - // assertion can see without the test allocating gigabytes. - thread::sleep(Duration::from_millis(1)); - let filled = out.len().min(4096); - out[..filled].fill(b'x'); - self.reads.fetch_add(1, Ordering::Release); - Ok(filled) - } - } - - #[test] - fn an_abandoned_reader_stops_filling_a_buffer_nobody_will_read() { - // The shape this guards: the child exited but a grandchild still holds - // the write end, so `DRAIN_GRACE` expires, `take()` hands the caller - // what there is and the reader is left running. In `rocmd` — a daemon - // that probes amd-smi every 60s and runs MCP captures besides — a - // reader that kept appending would grow a `Vec` that nothing will ever - // read, for as long as the grandchild lives. There is no way to ask - // `run_with_timeout` for that state on demand, so the capture is driven - // directly, which is also where the invariant lives. - let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let stop = Arc::new(AtomicBool::new(false)); - let capture = Capture::draining(Some(EndlessPipe { - reads: Arc::clone(&reads), - stop: Arc::clone(&stop), - })); - - let wait_for_reads = |target: usize| { - let deadline = Instant::now() + Duration::from_secs(10); - while reads.load(Ordering::Acquire) < target && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!( - reads.load(Ordering::Acquire) >= target, - "the reader stalled before {target} reads" - ); - }; - - wait_for_reads(4); - let (bytes, failure) = capture.take(); - // Establishes that the reader really was accumulating up to this point, - // so the emptiness asserted below is the flag working rather than a - // reader that never ran. - assert!( - !bytes.is_empty(), - "the reader must collect while someone is still listening" - ); - assert!(failure.is_none(), "{failure:?}"); - - // Well past what fits in anything the implementation might buffer - // internally: 50 more 4KiB reads is ~200KB, which a regression leaves - // sitting in the buffer and a correct implementation discards. - wait_for_reads(reads.load(Ordering::Acquire) + 50); - let leaked = capture.lock().len(); - assert_eq!( - leaked, 0, - "after `take()` nothing will read this buffer again, so a reader that keeps \ - appending grows it without bound for the life of the daemon; {leaked} bytes \ - accumulated" - ); - - stop.store(true, Ordering::Release); - let deadline = Instant::now() + Duration::from_secs(10); - while !capture.is_finished() && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!(capture.is_finished(), "the reader never finished"); - } - - /// A child that never exits on its own and fails whichever of the wait - /// loop's two calls the test asks it to. - struct FailingChild { - /// `true`: `try_wait` reports an error rather than "still running", so - /// the loop's first `?` fires. `false`: it stays running until the - /// deadline, and the post-kill `wait` is what fails. - fail_poll: bool, - /// Raised by `kill`, so the test can tell a wait loop that leaves the - /// child running from one that does not. This is the whole reason - /// `kill` is on the trait rather than called on a concrete `Child`. - killed: Arc, - } - - impl Waitable for FailingChild { - fn try_wait(&mut self) -> std::io::Result> { - if self.fail_poll { - return Err(std::io::Error::from(ErrorKind::PermissionDenied)); - } - Ok(None) - } - - fn kill(&mut self) -> std::io::Result<()> { - self.killed.store(true, Ordering::Release); - Ok(()) - } - - fn wait(&mut self) -> std::io::Result { - Err(std::io::Error::from(ErrorKind::PermissionDenied)) - } - } - - /// Drive `run_captured` with a child that fails one of the wait loop's two - /// calls, and assert the error is reported, both readers are left - /// abandoned, *and* the child was killed on the way out. - /// - /// `fail_poll` picks which call fails; see [`FailingChild`]. - fn assert_wait_failure_abandons_both_readers( - fail_poll: bool, - timeout: Duration, - expected: &str, - ) { - let killed = Arc::new(AtomicBool::new(false)); - let child = FailingChild { - fail_poll, - killed: Arc::clone(&killed), - }; - let stop = Arc::new(AtomicBool::new(false)); - let out_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let err_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let stdout = Capture::draining(Some(EndlessPipe { - reads: Arc::clone(&out_reads), - stop: Arc::clone(&stop), - })); - let stderr = Capture::draining(Some(EndlessPipe { - reads: Arc::clone(&err_reads), - stop: Arc::clone(&stop), - })); - - let wait_for_reads = |counter: &Arc, target: usize| { - let deadline = Instant::now() + Duration::from_secs(10); - while counter.load(Ordering::Acquire) < target && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } - assert!( - counter.load(Ordering::Acquire) >= target, - "the reader stalled before {target} reads" - ); - }; - - // Both readers are accumulating before the call, so an empty buffer - // afterwards is the abandon working rather than a reader that never ran. - wait_for_reads(&out_reads, 4); - wait_for_reads(&err_reads, 4); - - let error = run_captured(child, stdout.clone(), stderr.clone(), timeout, "process") - .expect_err("a failed wait must be reported, not swallowed"); - assert!( - format!("{error:#}").contains(expected), - "expected {expected:?} in {error:#}" - ); - - // ~200KB more per stream: what a reader that was never marked abandoned - // accumulates into a buffer nobody will ever read again. - wait_for_reads(&out_reads, out_reads.load(Ordering::Acquire) + 50); - wait_for_reads(&err_reads, err_reads.load(Ordering::Acquire) + 50); - for (name, capture) in [("stdout", &stdout), ("stderr", &stderr)] { - let leaked = capture.lock().len(); - assert_eq!( - leaked, 0, - "an error out of the wait loop returns before the `take()` pair, so it must \ - abandon both readers itself or they grow a buffer nobody will read for as \ - long as a grandchild holds the pipe; {name} accumulated {leaked} bytes" - ); - } - - // Abandoning the readers is only half of it: the caller drops the child - // on this path without reaping it, so a wait loop that returns without - // killing leaves a process running that nobody will ever wait on again. - assert!( - killed.load(Ordering::Acquire), - "every exit from the wait loop must kill the child first; this one returned the \ - error with the child still running" - ); - - stop.store(true, Ordering::Release); - } - - #[test] - fn a_failure_to_poll_the_child_still_abandons_both_readers() { - assert_wait_failure_abandons_both_readers( - true, - Duration::from_secs(30), - "failed to poll process", - ); - } - - #[test] - fn a_failure_to_reap_a_timed_out_child_still_abandons_both_readers() { - // Zero timeout so the loop reaches the post-kill reap — the other `?` — - // on its first pass. - assert_wait_failure_abandons_both_readers( - false, - Duration::ZERO, - "failed to reap timed-out process", - ); - } - - /// A successful `ExitStatus` to feed `assemble_output`. - /// - /// `ExitStatus` has no portable public constructor; the per-platform - /// `ExitStatusExt` is the supported way to make one without spawning a - /// process purely to obtain it. - fn success_status() -> ExitStatus { - #[cfg(unix)] - use std::os::unix::process::ExitStatusExt as _; - #[cfg(windows)] - use std::os::windows::process::ExitStatusExt as _; - ExitStatus::from_raw(0) - } - - fn read_failure() -> Option { - Some(std::io::Error::from(ErrorKind::BrokenPipe)) - } - - #[test] - fn a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output() { - // The silent short read this function exists to prevent: handing back - // `Ok` with a truncated buffer sends the caller off to parse half a - // JSON document and blame the document. - let error = assemble_output( - success_status(), - false, - (b"partial".to_vec(), read_failure()), - (Vec::new(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a stream that failed to read must not come back as success"); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("failed to read the output of process"), - "{rendered:?}" - ); - } - - #[test] - fn a_failure_on_either_stream_is_reported() { - // stdout is checked first, so a stderr-only failure is a separate - // branch: `stdout_failure.or(stderr_failure)` reduced to - // `stdout_failure` would still pass the case above. - let error = assemble_output( - success_status(), - false, - (Vec::new(), None), - (b"partial".to_vec(), read_failure()), - "process", - Duration::from_secs(2), - ) - .expect_err("a stderr stream that failed to read must not come back as success"); - - assert!( - format!("{error:#}").contains("failed to read the output of process"), - "{error:#}" - ); - } - - #[test] - fn a_timeout_outranks_a_read_failure() { - // Killing the child is what broke the pipe, so a read error here is the - // symptom. Reporting it would name the consequence and hide the cause, - // and the caller would go looking for a kernel problem. - let error = assemble_output( - success_status(), - true, - (Vec::new(), read_failure()), - (b"the real explanation".to_vec(), read_failure()), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("exceeded 2s timeout"), - "the timeout is the cause and must be what is reported: {rendered:?}" - ); - assert!( - rendered.contains("the real explanation"), - "the child's own account must survive: {rendered:?}" - ); - assert!( - !rendered.contains("failed to read the output of"), - "the read error is downstream of the kill and must not displace it: {rendered:?}" - ); - } - - #[test] - fn a_timeout_quotes_stderr_in_preference_to_stdout() { - let error = assemble_output( - success_status(), - true, - (b"stdout-data".to_vec(), None), - (b"stderr-explanation".to_vec(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("stderr-explanation"), - "stderr is where a child explains itself: {rendered:?}" - ); - assert!( - !rendered.contains("stdout-data"), - "quoting both would push the explanation behind a payload: {rendered:?}" - ); - } - - #[test] - fn a_timeout_falls_back_to_stdout_and_then_to_saying_there_was_none() { - // Every other timeout fixture writes to stderr, so without these two - // the fallback chain could be reordered or deleted unnoticed — and a - // child whose whole output is stdout is exactly the amd-smi case. - let error = assemble_output( - success_status(), - true, - (b"stdout-only".to_vec(), None), - (Vec::new(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - assert!( - format!("{error:#}").contains("stdout-only"), - "a child that only wrote stdout must still be quoted: {error:#}" - ); - - let error = assemble_output( - success_status(), - true, - (Vec::new(), None), - (Vec::new(), None), - "process", - Duration::from_secs(2), - ) - .expect_err("a timed-out call must fail"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("no output"), - "a silent child must say so rather than trailing a bare colon: {rendered:?}" - ); - assert!(rendered.contains("exceeded 2s timeout"), "{rendered:?}"); - } - - #[test] - fn a_clean_run_returns_both_streams() { - let output = assemble_output( - success_status(), - false, - (b"out".to_vec(), None), - (b"err".to_vec(), None), - "process", - Duration::from_secs(2), - ) - .expect("a clean run must succeed"); - - assert!(output.status.success()); - assert_eq!(output.stdout, b"out"); - assert_eq!(output.stderr, b"err"); - } - - #[cfg(unix)] - #[test] - fn a_child_that_floods_both_pipes_does_not_deadlock() { - // The defect this function exists to prevent. Both pipes are bounded by - // the OS (commonly ~64KiB), so a child writing past that blocks in - // `write` and never exits — and a version of this function that drains - // only after the child exits never reads a byte, so the child is killed - // at the timeout and a successful command is reported as too slow. - // - // Flooding *both* streams is deliberate: a single-pipe child still - // passes against an implementation that drains stdout fully and only - // then starts on stderr, which is its own deadlock. - // - // `sh` rather than a real subject program (amd-smi, an SDK install), - // because the property is about this function's pipe handling and - // nothing else, and a shell makes it a unit test rather than one that - // needs a multi-GPU host. - let mut command = Command::new("sh"); - command.args([ - "-c", - "yes deadlock-canary | head -c 2000000; yes deadlock-canary | head -c 2000000 >&2", - ]); - - // Short, because a regression fails by exhausting this timeout: ten - // seconds is far past the ~0.1s a working drain takes, and far short of - // stalling the suite. - let output = run_with_timeout(command, Duration::from_secs(10), "process") - .expect("a child that outruns the pipe buffer must not be killed as a timeout"); - - assert!(output.status.success(), "{output:?}"); - // Without these the test would keep passing if the child ever stopped - // producing more than one buffer, which is to say it would stop testing - // the hazard. - assert!( - output.stdout.len() > 64 * 1024, - "stdout must exceed one pipe buffer: {} bytes", - output.stdout.len() - ); - assert!( - output.stderr.len() > 64 * 1024, - "stderr must exceed one pipe buffer: {} bytes", - output.stderr.len() - ); - } - - /// How large a quoted-output assertion lets the whole error message get. - /// - /// Comfortably above `MAX_QUOTED_OUTPUT_CHARS` plus the label and marker, - /// and two orders of magnitude below the ~200KB the fixtures below feed in, - /// so the assertion answers "is it bounded at all" rather than pinning the - /// exact constant — which is a tuning decision, not a contract. - const QUOTE_ASSERTION_CEILING: usize = 4000; - - /// The flooding fixtures below write this many bytes, past one pipe buffer - /// and ~100x the quote bound. - const FLOOD_BYTES: usize = 200_000; - - const FLOOD_CANARY: &str = "flood-canary"; - - /// The assertions shared by both platforms' timeout-quote cases. - /// - /// Checked together because they trade off against each other: a bound that - /// kept nothing would satisfy the size check while making the message - /// useless, and quoting everything keeps it useful while costing the logs - /// the thing this bound exists to protect. - fn assert_timeout_quote_is_bounded_and_useful(message: &str, timeout_secs: u64) { - assert!( - message.contains(&format!("exceeded {timeout_secs}s timeout")), - "the timeout must still be named: {message:?}" - ); - assert!( - message.contains(FLOOD_CANARY), - "the child's own output must still be quoted, not dropped: {message:?}" - ); - assert!( - message.contains("[truncated]"), - "a shortened quote must say so, or a reader takes the head for the whole: {message:?}" - ); - let length = message.chars().count(); - assert!( - length <= QUOTE_ASSERTION_CEILING, - "the quote must be bounded: the message is {length} characters, from a child that \ - printed {FLOOD_BYTES} bytes" - ); - } - - #[cfg(unix)] - #[test] - fn a_timed_out_child_has_its_output_quoted_within_a_bound() { - for shell in grandchild_shells() { - // Both halves are load-bearing. The flood is what makes the quote - // large; the backgrounded `sleep` is what makes this a timeout, - // which is the only path that quotes the child's output at all. - // - // Unbounded, this message travels: `rocmd` renders the error chain - // into the bridge snapshot's `note`, and the GPU-metrics watcher - // writes that note into two JSONL logs every 60s, neither ever - // rotated or capped. A real amd-smi on a multi-GPU host prints - // well past this fixture's size. - let mut command = Command::new(shell); - command.args([ - "-c", - &format!("yes {FLOOD_CANARY} | head -c {FLOOD_BYTES} >&2; sleep 30 & wait"), - ]); - - let error = run_with_timeout(command, Duration::from_secs(1), "process") - .expect_err("a command that outlives its timeout must fail"); - - assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 1); - } - } - - /// A scratch directory holding a Windows fixture's script, and for - /// [`Self::flooding`] fixtures the payload beside it. - /// - /// Per-process unique so concurrent test binaries cannot collide on it, and - /// removed on drop so a failing assertion does not leave either behind. - #[cfg(windows)] - struct WindowsFixture { - dir: std::path::PathBuf, - } - - #[cfg(windows)] - impl WindowsFixture { - /// Write `script` as a batch file in a scratch directory of its own. - /// - /// A batch file rather than `cmd /C ""`: cmd's rules for - /// stripping the quotes Rust adds around an argument containing spaces - /// and redirections are subtle enough that the fixture would become the - /// thing under test. A bare path has none of that, and `%~dp0` inside - /// the script resolves anything beside it without the test passing a - /// path in. - fn script(name: &str, script: &str) -> Self { - let dir = std::env::temp_dir() - .join(format!("rocm-cli-process-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).expect("failed to create the fixture directory"); - std::fs::write(dir.join("fixture.bat"), script).expect("failed to write the script"); - - Self { dir } - } - - /// A [`Self::script`] fixture with a `FLOOD_BYTES` payload beside it for - /// the script to print. - fn flooding(name: &str, script: &str) -> Self { - let fixture = Self::script(name, script); - - let line = format!("{FLOOD_CANARY}\r\n"); - let mut payload = String::with_capacity(FLOOD_BYTES + line.len()); - while payload.len() < FLOOD_BYTES { - payload.push_str(&line); - } - std::fs::write(fixture.dir.join("payload.txt"), payload) - .expect("failed to write the payload"); - - fixture - } - - fn command(&self) -> Command { - let mut command = Command::new("cmd"); - command.args(["/C"]).arg(self.dir.join("fixture.bat")); - command - } - } - - #[cfg(windows)] - impl Drop for WindowsFixture { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.dir); - } - } - - #[cfg(windows)] - #[test] - fn a_child_that_floods_both_pipes_does_not_deadlock() { - // The Windows half of the unix case above, and the same property: both - // pipes are bounded by the OS here too — more tightly than on Linux — - // so an implementation that drains only after the child exits, or - // drains stdout to completion before starting on stderr, never lets - // this child finish. `type` twice is the shell-free equivalent of the - // unix `yes | head`; cmd has no `yes`. - let fixture = WindowsFixture::flooding( - "flood", - "@echo off\r\ntype \"%~dp0payload.txt\"\r\ntype \"%~dp0payload.txt\" 1>&2\r\n", - ); - - let output = run_with_timeout(fixture.command(), Duration::from_secs(30), "process") - .expect("a child that outruns the pipe buffer must not be killed as a timeout"); - - assert!(output.status.success(), "{output:?}"); - assert!( - output.stdout.len() > 64 * 1024, - "stdout must exceed one pipe buffer: {} bytes", - output.stdout.len() - ); - assert!( - output.stderr.len() > 64 * 1024, - "stderr must exceed one pipe buffer: {} bytes", - output.stderr.len() - ); - } - - #[cfg(windows)] - #[test] - fn a_timed_out_child_has_its_output_quoted_within_a_bound() { - // Covers on Windows what three separate unix cases cover there, because - // one fixture happens to exhibit all of it: `ping -n 30` runs for ~29 - // seconds, so the call times out; `ping` is a grandchild of the `cmd` - // this function kills at the deadline, and killing a process on Windows - // does not kill its children, so it keeps the inherited write ends open - // and no EOF arrives — an implementation that joins its readers waits - // the full 29 seconds; and the preceding flood makes the resulting - // quote large enough for the bound to matter. - let fixture = WindowsFixture::flooding( - "timeout", - "@echo off\r\ntype \"%~dp0payload.txt\" 1>&2\r\nping -n 30 127.0.0.1\r\n", - ); - - let started = Instant::now(); - let error = run_with_timeout(fixture.command(), Duration::from_secs(2), "process") - .expect_err("a command that outlives its timeout must fail"); - let elapsed = started.elapsed(); - - assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 2); - assert!( - elapsed < Duration::from_secs(10), - "the call must return at its deadline rather than waiting for an abandoned \ - grandchild to exit, but it took {elapsed:?}" - ); - } - - #[cfg(windows)] - #[test] - fn a_child_that_reads_stdin_is_given_an_empty_one() { - // `set /p` leaves the variable untouched when its read reaches EOF, so - // pre-setting it to the marker is cmd's equivalent of the unix - // `read || echo`. A stdin left open blocks in `set /p` instead. - // - // Both callers in `rocmd` run on Windows -- the amd-smi probe in - // `apps/rocmd/src/host_snapshot.rs` and the `rocm` capture that - // `apps/rocmd/src/common.rs` makes through `run_rocm_capture_with_env` - // -- and `run_with_timeout` has no `cfg(windows)` branch, so this covers - // the same promise with the same two-sidedness as the unix case. - let fixture = WindowsFixture::script( - "stdin", - &format!( - "@echo off\r\nset \"LINE={STDIN_EOF_MARKER}\"\r\nset /p LINE=\r\necho %LINE%\r\n" - ), - ); - - assert_the_child_is_given_an_empty_stdin(fixture.command()); - } } #[cfg(test)] diff --git a/crates/rocm-core/src/process/bounded.rs b/crates/rocm-core/src/process/bounded.rs new file mode 100644 index 0000000000..ab2b3131d0 --- /dev/null +++ b/crates/rocm-core/src/process/bounded.rs @@ -0,0 +1,1938 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The bounded subprocess executor behind [`super::run_with_timeout`] and +//! [`super::run_bounded`]. +//! +//! Run a command to completion with both streams captured and a real time +//! bound. A thread per pipe drains stdout and stderr from the moment the child +//! starts, so output size cannot stall a command into its own timeout, and a +//! reader that cannot finish is abandoned rather than joined, so a grandchild +//! holding the pipe's write end open cannot outlast the deadline either. The +//! timeout error quotes a bounded head of the child's own output, because that +//! text reaches persisted automation logs. `Capture`, `settle`, `run_captured`, +//! `wait_bounded`, `Waitable` and `assemble_output` exist to serve it. +//! +//! `run_bounded` is the same executor for a caller that wants the run as a +//! [`CommandCapture`] with a typed [`CommandOutcome`] and the raw bytes, a +//! timeout included, instead of an `Err`. The host probes' `run` and +//! `capture_optional_command*` are thin adapters over it. It is not yet the only +//! such helper in the tree; converging the others onto it is tracked in +//! [#623](https://github.com/ROCm/rocm-cli/issues/623). +//! +//! It lives in its own file, split out of `process.rs` to keep that file under +//! the module size limit; the public paths are unchanged, `process.rs` +//! re-exports them. + +use crate::command_capture::{self, CommandCapture, CommandOutcome}; +use crate::probe_exec::truncate_to_chars; +use anyhow::{Context, Result, bail}; +use std::io::{ErrorKind, Read}; +use std::process::{Child, Command, ExitStatus, Output, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +/// The longest the wait loop sleeps between polls for the child's exit. +const POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// The first sleep of the wait loop, before [`next_poll_interval`] backs it off. +/// +/// A host probe is usually a child that exits within a few milliseconds, and a +/// flat 50 ms sleep made every such call take 50 ms: the probes that older +/// helpers polled every 20 to 25 ms would then each have got slower for taking +/// this executor on, and `rocm examine` runs dozens of them one after another. +const FIRST_POLL: Duration = Duration::from_millis(5); + +/// The sleep after `current`: doubled, up to [`POLL_INTERVAL`]. +const fn next_poll_interval(current: Duration) -> Duration { + let doubled = current.saturating_mul(2); + if doubled.as_millis() > POLL_INTERVAL.as_millis() { + POLL_INTERVAL + } else { + doubled + } +} + +/// How much a reader accumulates before publishing it to the shared buffer. +const READ_CHUNK: usize = 16 * 1024; + +/// How long the readers get to finish once the child is no longer running. +/// +/// Reaching this means something other than the child still holds the pipe's +/// write end — a grandchild the child left behind — so there is no EOF coming +/// and waiting longer would make `timeout` meaningless. +/// +/// For a child that genuinely finished the grace has only the tail to absorb: a +/// child cannot exit before writing everything, and the readers are started +/// before the wait loop. What they are not is *scheduled* before it — +/// `wait_bounded` polls `try_wait` once with no sleep in front of it, so a child +/// fast enough to exit between spawn and that first poll can be observed exited +/// while neither reader thread has run a single instruction. So the grace covers +/// reader start-up as well as the last buffer or two. 200ms absorbs both with +/// room to spare, including on the no-GPU e2e lane's 64 concurrent scenarios. +/// +/// The one value for every caller, and what it costs is output: a descendant +/// that writes later than this after the child exited contributes nothing, and +/// the capture's `detail` says only that output was still arriving. Whether a +/// caller whose tool is known to do that should be able to ask for a longer +/// grace is open, listed in `docs/host-probing.md`. +const DRAIN_GRACE: Duration = Duration::from_millis(200); + +/// How much of a timed-out child's own output the timeout error quotes. +/// +/// Bounded here rather than at any one caller because the quote is assembled +/// here: the timeout message is this function's own text, and a caller sees it +/// only as an opaque error, so this is the only place that can still tell which +/// of its characters came from the child. All four callers inherit it. +/// +/// It bounds this message by itself. A caller that quotes the `Output` handed +/// back to it is quoting raw child output of its own, persisted the same way, +/// so it has to apply a bound too -- which is why this is `pub(crate)` rather +/// than private: [`crate::amd_smi::amd_smi_json`] bounds its non-zero-exit +/// quote (the one #623 tracked) with *this* constant instead of declaring a +/// second one free to drift from it. The daemon's +/// `parse_sandbox_child_output` still predates both and is outside this scope. +/// +/// The bound matters because the message does not stop at a terminal. `rocmd` +/// renders the whole error chain into the bridge snapshot's `note`, and the +/// GPU-metrics watcher copies that note into the automation-event and audit +/// JSONL files once a minute — stores that are never rotated or capped. +/// Unbounded, a child that prints a megabyte and then stalls past its +/// timeout grows both logs by that much per minute, forever. A 151K-character +/// JSONL line was reproduced from exactly that poller. +/// +/// Sized to still hold a real stderr backtrace: the point is to stop a +/// payload-sized quote, not to shorten diagnostics. +pub(crate) const MAX_QUOTED_OUTPUT_CHARS: usize = 2000; + +/// One captured stream: the bytes so far, whether its reader saw EOF, and the +/// read error that stopped it, if one did. +/// +/// # Lifetime of the reader thread +/// +/// The thread outlives the call that started it whenever the drain grace runs +/// out — that is the point of the grace — and it then lives until the pipe's +/// last write end closes, which is the abandoned grandchild's business, not +/// ours. What it must *not* do is keep growing: `take()` marks the capture +/// abandoned, and from that moment the reader drains to EOF and discards, so a +/// buffer nobody will read again cannot grow without bound in a daemon that +/// probes every 60 seconds. +/// +/// What that bounds is the memory, and only the memory. The *number* of these +/// threads is not bounded here: each abandoned capture leaves one parked in +/// `read` until the grandchild lets go, so every probe that meets one costs two +/// until it does. The trade is deliberate — a parked reader holds a stack and +/// no CPU, and the alternative is the one below. +/// +/// Discarding rather than closing the read end is deliberate. Dropping the pipe +/// would send the still-writing grandchild `EPIPE`/`SIGPIPE`, which for the +/// background helpers a sandboxed installer legitimately leaves behind would +/// turn a leak whose cost we know into a behaviour change we cannot predict. +#[derive(Clone)] +struct Capture { + buffer: Arc>>, + finished: Arc, + failure: Arc>>, + /// Set by `take()`: nothing will read this capture's bytes again. + /// + /// Read and written under `buffer`'s mutex, so "emptied and marked" is one + /// step as far as the reader is concerned. Checking it outside the lock + /// would leave the window where a reader that has just passed the check + /// appends to the buffer `take()` is in the middle of emptying. + abandoned: Arc, +} + +impl Capture { + /// Start draining `pipe` on its own thread. + /// + /// Publishing in chunks as they arrive, rather than returning the whole + /// buffer at EOF, is what lets the caller take partial output from a reader + /// that is still blocked. + fn draining(pipe: Option) -> Self { + let capture = Self { + buffer: Arc::new(Mutex::new(Vec::new())), + finished: Arc::new(AtomicBool::new(false)), + failure: Arc::new(Mutex::new(None)), + abandoned: Arc::new(AtomicBool::new(false)), + }; + let Some(mut pipe) = pipe else { + capture.finished.store(true, Ordering::Release); + return capture; + }; + let sink = capture.clone(); + thread::spawn(move || { + let mut chunk = [0_u8; READ_CHUNK]; + loop { + match pipe.read(&mut chunk) { + Ok(0) => break, + Ok(read) => { + // Keep draining once abandoned, but stop accumulating: + // see the type's doc. The check shares the buffer's + // lock with `take()`, so after `take()` returns this + // buffer cannot grow by another byte. + let mut buffer = sink.lock(); + if !sink.abandoned.load(Ordering::Acquire) { + buffer.extend_from_slice(&chunk[..read]); + } + } + // `read_to_end` — what `wait_with_output` used before this + // function existed — retries an interrupted read and + // returns every other error. Treating either as EOF would + // silently shorten the output instead, and the callers + // parse these bytes as JSON, where that surfaces as a + // parse error pointing at the wrong thing. + // Round again rather than break: not the end of anything. + Err(error) if error.kind() == ErrorKind::Interrupted => {} + Err(error) => { + // Recorded only while someone is still listening. Once + // `take()` has fully returned this is merely tidy — + // nothing calls `take()` twice, so a failure stored + // afterwards would just sit unread. + // + // It is load-bearing for one window, and that window is + // inside `take()` itself: `take()` marks `abandoned` + // and empties the buffer under `buffer`'s lock, then + // releases it and drains the failure slot under a + // *different* lock. A read error that lands in between + // would otherwise be handed back attached to bytes the + // caller has already been given, and `assemble_output` + // turns any failure on a non-timed-out run into a hard + // `failed to read the output of ...` — a cleanly + // finished command reported as a read failure. Taking + // `buffer`'s lock before the check is what closes it; + // a bare load could pass the check, then have `take()` + // run the whole sequence before the store. + let _buffer = sink.lock(); + if !sink.abandoned.load(Ordering::Acquire) { + *sink.failure() = Some(error); + } + break; + } + } + } + sink.finished.store(true, Ordering::Release); + }); + capture + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Vec> { + self.buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn failure(&self) -> std::sync::MutexGuard<'_, Option> { + self.failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn is_finished(&self) -> bool { + self.finished.load(Ordering::Acquire) + } + + /// Take what has been captured, leaving a still-running reader unblocked. + /// + /// The bytes come back alongside any read error rather than instead of it, + /// so a caller already reporting something else — a timeout — can still + /// quote what it managed to collect. + /// + /// This is also the point of no return for the reader: whatever it collects + /// after this is discarded, because there is no second caller to hand it to. + fn take(&self) -> (Vec, Option) { + let bytes = { + // Marked while the buffer is held, so the reader cannot append + // between the mark and the emptying and strand bytes in a buffer + // that is never read again. + let mut buffer = self.lock(); + self.abandoned.store(true, Ordering::Release); + std::mem::take(&mut *buffer) + }; + (bytes, self.failure().take()) + } +} + +/// Run `command` to completion with both output streams captured, giving up +/// after `timeout`. +/// +/// A thread per pipe drains stdout and stderr, and both are started *before* +/// the wait loop. That ordering is the whole point. Waiting for the child to +/// exit and only then calling `wait_with_output` — the shape this replaced — +/// deadlocks: the pipes are bounded by the OS (commonly ~64KiB, less on +/// Windows), so a child that writes more than that blocks in `write`, which +/// means it never exits, which means the drain is never reached. The only way +/// out was the timeout, so it measured how much the child printed rather than +/// how long it took, and killed successful commands mid-flight. +/// +/// Both pipes need their own reader for the same reason: draining stdout to +/// completion and only then reading stderr stalls on a child that fills stderr +/// first. +/// +/// The call returns within roughly `timeout` either way: once the child is no +/// longer running the readers get `DRAIN_GRACE` to finish and are then abandoned +/// rather than joined. Joining unconditionally would hand the schedule to a +/// grandchild the child left behind, which holds the pipe's write end open so no +/// EOF ever arrives — that is how an abandoned `sleep` comes to decide how long +/// a 2-second probe takes. +/// +/// The stdio configuration is this function's own: it sets null stdin and piped +/// stdout/stderr on `command`, overriding whatever the caller set, because +/// capturing both streams is the whole contract. Callers need not — and should +/// not — set them. The stdin half is a promise in its own right rather than +/// housekeeping: a child that reads from an inherited stdin waits for input +/// that is never coming, so it is the deadline that ends it. Pinned by +/// `a_child_that_reads_stdin_is_given_an_empty_one`, on both platforms. +/// +/// `label` names the subject in error messages (e.g. `"process"`, +/// `"sandbox process"`). +pub fn run_with_timeout(mut command: Command, timeout: Duration, label: &str) -> Result { + let (child, stdout, stderr) = + spawn_captured(&mut command).with_context(|| format!("failed to spawn {label}"))?; + run_captured(child, stdout, stderr, timeout, label) +} + +/// [`run_with_timeout`] for a caller that wants to know how the run ended +/// rather than to have a timeout or a failure turned into an error: the same +/// executor, with the result as a [`CommandCapture`]. +/// +/// Nothing is an `Err` here. A program that cannot be started, a timeout and a +/// failure to wait are each an [`CommandOutcome`] of the capture, and a timeout +/// keeps what the child had written, so a caller that reads `stdout_raw` after +/// one gets the partial output [`run_with_timeout`] only quotes. The bounds are +/// [`run_with_timeout`]'s: the readers are abandoned after [`DRAIN_GRACE`], so +/// a descendant that keeps a pipe open does not extend the call. +pub(crate) fn run_bounded(mut command: Command, timeout: Duration, label: &str) -> CommandCapture { + let argv = std::iter::once(command.get_program()) + .chain(command.get_args()) + .map(|part| part.to_string_lossy().into_owned()) + .collect(); + match spawn_captured(&mut command) { + Err(error) => CommandCapture::from_run( + argv, + CommandOutcome::from_spawn_error(error.kind()), + None, + Vec::new(), + Vec::new(), + Some(error.to_string()), + ), + Ok((child, stdout, stderr)) => { + capture_of(argv, settle(child, stdout, stderr, timeout, label)) + } + } +} + +/// Start `command` with null stdin and both output pipes drained from the first +/// moment, which is what lets the wait loop be bounded at all. +fn spawn_captured(command: &mut Command) -> std::io::Result<(Child, Capture, Capture)> { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + // Taking the pipes here is what obliges us to assemble `Output` by hand + // later: `wait_with_output` reads the handles it owns, and these are gone. + let stdout = Capture::draining(child.stdout.take()); + let stderr = Capture::draining(child.stderr.take()); + Ok((child, stdout, stderr)) +} + +/// How a run ended and what it left behind, before it is turned into either a +/// `Result` or a [`CommandCapture`]. +enum Settled { + /// The child was waited for: it exited, or the deadline killed it. + Waited { + status: ExitStatus, + timed_out: bool, + /// Whether both readers reached end of output before the drain grace + /// ended. When not, something other than the child still holds a pipe + /// and the bytes are what arrived by then. + drained: bool, + stdout: (Vec, Option), + stderr: (Vec, Option), + }, + /// Polling or reaping the child failed, and it was killed. + WaitFailed { + error: anyhow::Error, + stdout: (Vec, Option), + stderr: (Vec, Option), + }, +} + +/// The wait, the drain grace and the `take()` of both captures: everything +/// between a child being spawned and its result being formed. +/// +/// Generic over [`Waitable`] rather than taking a `Child` so that the wait +/// loop's two failure paths — which are the only ways out of this function that +/// skip the drain grace — are reachable from a test. +fn settle( + mut child: C, + stdout: Capture, + stderr: Capture, + timeout: Duration, + label: &str, +) -> Settled { + // The single error path between `Capture::draining` and the `take()` pair + // below, and it must take both captures before it returns. `take()` is + // what marks a reader abandoned, and an abandoned reader is what stops a + // buffer nobody will read again from growing for as long as a grandchild + // holds the pipe open. Returning straight out of the wait loop — the shape + // this replaced — left both readers appending forever, in a daemon that + // probes every 60 seconds. + let (status, timed_out) = match wait_bounded(&mut child, timeout, label, thread::sleep) { + Ok(outcome) => outcome, + Err(error) => { + return Settled::WaitFailed { + error, + stdout: stdout.take(), + stderr: stderr.take(), + }; + } + }; + + let drained = wait_for_readers(&stdout, &stderr, DRAIN_GRACE, thread::sleep); + Settled::Waited { + status, + timed_out, + drained, + stdout: stdout.take(), + stderr: stderr.take(), + } +} + +/// Give both readers up to `grace` to reach end of output, looking again after +/// each `sleep` of the back-off schedule, and say whether they did. +/// +/// `sleep` is a parameter so a test can see the schedule the loop really +/// follows instead of inferring it from [`next_poll_interval`] alone. +fn wait_for_readers( + stdout: &Capture, + stderr: &Capture, + grace: Duration, + mut sleep: impl FnMut(Duration), +) -> bool { + let drained_by = Instant::now() + grace; + let mut pause = FIRST_POLL; + while !(stdout.is_finished() && stderr.is_finished()) && Instant::now() < drained_by { + sleep(pause); + pause = next_poll_interval(pause); + } + stdout.is_finished() && stderr.is_finished() +} + +/// The rest of [`run_with_timeout`] once the child is spawned and both readers +/// are running. +fn run_captured( + child: C, + stdout: Capture, + stderr: Capture, + timeout: Duration, + label: &str, +) -> Result { + match settle(child, stdout, stderr, timeout, label) { + Settled::WaitFailed { error, .. } => Err(error), + Settled::Waited { + status, + timed_out, + stdout, + stderr, + .. + } => assemble_output(status, timed_out, stdout, stderr, label, timeout), + } +} + +/// A [`Settled`] run as a [`CommandCapture`], keeping the bytes a timeout or a +/// read error would otherwise have turned into an error. +fn capture_of(argv: Vec, settled: Settled) -> CommandCapture { + match settled { + Settled::WaitFailed { + error, + stdout, + stderr, + } => CommandCapture::from_run( + argv, + CommandOutcome::WaitFailed, + None, + stdout.0, + stderr.0, + Some(format!("{error:#}")), + ), + Settled::Waited { + status, + timed_out, + drained, + stdout: (stdout, stdout_failure), + stderr: (stderr, stderr_failure), + } => { + let outcome = if timed_out { + CommandOutcome::TimedOut + } else { + command_capture::outcome_of_status(status) + }; + // What cut the output short, if anything: a pipe that failed to + // read, and/or a descendant still writing when the grace ended. The + // outcome stays what the child did, and the bytes are what arrived. + let notes: Vec = [ + stdout_failure.map(|error| format!("failed to read stdout: {error}")), + stderr_failure.map(|error| format!("failed to read stderr: {error}")), + (!drained).then(|| STILL_ARRIVING.to_owned()), + ] + .into_iter() + .flatten() + .collect(); + let detail = (!notes.is_empty()).then(|| notes.join("; ")); + CommandCapture::from_run(argv, outcome, Some(status), stdout, stderr, detail) + } + } +} + +/// Wait for `child` to exit, killing it once `timeout` has passed. +/// +/// Returns the status and whether the deadline is what ended the wait. Kept +/// free of the captures so that it has exactly one caller to get the +/// abandon-on-error handling right, rather than one per `?`. `sleep` is what +/// the loop waits with between polls, a parameter so a test can see the +/// back-off schedule it follows. +fn wait_bounded( + child: &mut C, + timeout: Duration, + label: &str, + mut sleep: impl FnMut(Duration), +) -> Result<(ExitStatus, bool)> { + let deadline = Instant::now() + timeout; + let mut pause = FIRST_POLL; + loop { + let polled = child.try_wait(); + if polled.is_err() { + // The only exit from this function that is not already followed by + // a kill. Whatever stopped us polling, we are about to stop + // waiting, and the caller drops the child without ever reaping it; + // leaving it running would hand back a process nobody is watching. + // `let _` to match the timeout branch: the kill is best-effort, and + // the poll error is the one worth reporting. + let _ = child.kill(); + } + if let Some(status) = polled.with_context(|| format!("failed to poll {label}"))? { + return Ok((status, false)); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let status = child + .wait() + .with_context(|| format!("failed to reap timed-out {label}"))?; + return Ok((status, true)); + } + sleep(pause); + pause = next_poll_interval(pause); + } +} + +/// What a capture's `detail` says when the drain grace ended with a reader still +/// waiting for end of output, so the caller can tell a complete capture from a +/// short one. +const STILL_ARRIVING: &str = "output was still arriving when the drain grace ended"; + +/// The three child operations [`wait_bounded`] needs. +/// +/// A trait only so a test can fail `try_wait` or the post-kill `wait` on +/// demand: both are error paths that skip the drain grace, and provoking either +/// from a real child needs the kernel to refuse to reap a process. +trait Waitable { + fn try_wait(&mut self) -> std::io::Result>; + fn kill(&mut self) -> std::io::Result<()>; + fn wait(&mut self) -> std::io::Result; +} + +// `Child::` rather than `Self::` on purpose: inside this impl `Self` is +// `Child`, so `Self::try_wait` reads as the trait method being defined +// here rather than the inherent one being forwarded to. +#[allow(clippy::use_self)] +impl Waitable for Child { + fn try_wait(&mut self) -> std::io::Result> { + Child::try_wait(self) + } + + fn kill(&mut self) -> std::io::Result<()> { + Child::kill(self) + } + + fn wait(&mut self) -> std::io::Result { + Child::wait(self) + } +} + +/// Turn what the wait loop and the two readers produced into the call's result. +/// +/// Split out from [`run_with_timeout`] because it is the only way to test it. +/// Every branch here is plain logic over values already in hand — which error +/// wins, which stream gets quoted — but reaching it through a real child means +/// making a pipe fail on demand, which needs a kernel having a bad day. Given +/// the two `(bytes, error)` pairs directly, each branch is an ordinary unit +/// test; see the cases at the bottom of this file. +fn assemble_output( + status: ExitStatus, + timed_out: bool, + (stdout, stdout_failure): (Vec, Option), + (stderr, stderr_failure): (Vec, Option), + label: &str, + timeout: Duration, +) -> Result { + let output = Output { + status, + stdout, + stderr, + }; + + if timed_out { + // `command_failure_detail` is the crate's one "which stream explains + // this failure" rule — stderr, else stdout, else "no output" — shared + // since #584 so callers cannot quietly disagree about the order. + // + // Truncated, not dropped: what the child managed to say before it + // stalled is usually the whole explanation, and the timeout is named + // first either way, so a reader who only sees the head still learns + // which command gave up and why. + bail!( + "{label} exceeded {}s timeout: {}", + timeout.as_secs(), + truncate_to_chars( + crate::command_failure_detail(&output), + MAX_QUOTED_OUTPUT_CHARS, + ) + ); + } + + // Only once the timeout has had its say: a killed child whose pipe then + // fails to read is a timeout, and reporting it as a read error would name + // the symptom instead of the cause. + if let Some(error) = stdout_failure.or(stderr_failure) { + return Err(anyhow::Error::new(error)) + .with_context(|| format!("failed to read the output of {label}")); + } + + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Stdio; + + /// The shells every grandchild case below is run against. + /// + /// Whether a shell forks or execs decides whether a grandchild exists at + /// all, so it decides whether these tests test anything: dash forks `sleep` + /// in `sh -c "sleep 30"`, bash execs it into the shell's own process and no + /// grandchild is left holding the pipe. A test written against one is dead + /// weight on a host running the other, and which one `/bin/sh` is varies + /// across the distributions this ships on. So the scripts below are written + /// to fork under either, and each case runs under every shell present to + /// keep that true. + #[cfg(unix)] + fn grandchild_shells() -> Vec<&'static str> { + let shells: Vec<&'static str> = ["/bin/sh", "/bin/bash", "/bin/dash"] + .into_iter() + .filter(|shell| std::path::Path::new(shell).exists()) + .collect(); + assert!( + !shells.is_empty(), + "no shell to run the grandchild cases in" + ); + shells + } + + #[test] + fn a_command_that_succeeds_reports_its_output() { + let mut command = Command::new(if cfg!(windows) { "cmd" } else { "sh" }); + if cfg!(windows) { + // No space around `&`: cmd would carry a trailing one into the + // echoed text, which `trim` hides on stdout but is easier not to + // write than to explain. + command.args(["/C", "echo hello&echo oops 1>&2"]); + } else { + command.args(["-c", "echo hello; echo oops >&2"]); + } + + let output = run_with_timeout(command, Duration::from_secs(10), "process").expect("ran"); + + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "hello"); + // Asserted on Windows too: `1>&2` is a cmd redirection like any other, + // and skipping it there left the second pipe — the one whose separate + // reader this function exists to provide — unchecked on that platform. + assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), "oops"); + } + + /// What a stdin-reading fixture prints when its read reaches EOF. + const STDIN_EOF_MARKER: &str = "eof"; + + /// The assertions shared by both platforms' empty-stdin cases. + /// + /// `command` must run a child that reads one line from stdin and prints + /// [`STDIN_EOF_MARKER`] if that read reaches EOF instead. + /// + /// The stdin set on `command` here is the read end of a pipe whose write end + /// this function holds open across the whole call, so it never reaches EOF + /// of its own accord. That is what makes the assertion two-sided rather than + /// a tautology: were `run_with_timeout` to stop setting null stdin, the + /// child would sit in `read` until the deadline killed it — exactly how a + /// child that prompts behaves against an inherited terminal — and the call + /// would come back as a timeout rather than with the marker. + /// + /// Setting stdin on the caller's side is also the contract under test: this + /// function's own stdio overrides whatever the caller set, which is what + /// lets all four callers hand over a bare `Command` and say nothing about + /// stdin at all. + /// + /// One thing it does not pin, said plainly rather than left to be + /// discovered: `null` against `inherit`. Telling those two apart needs this + /// test process's own fd 0 to be something other than EOF, and arranging + /// that is a `dup2` racing every other test in the binary. An `inherit` + /// mutant was run and does pass here. What is caught is the regression that + /// can actually happen -- no stdin setting at all, which is the line this + /// function took over from its callers. + /// + /// The race is avoidable by re-execing this test binary under a stdin pipe + /// the outer test holds open and asserting in that child, which would fail + /// the `inherit` mutant too. Not done: it buys one mutant that no plausible + /// edit produces -- nothing would replace `null` with `inherit`, the stdio + /// mode this helper exists to stop callers from being handed -- at the cost + /// of a self-re-execing test whose own harness plumbing is then the thing + /// most likely to break. + fn assert_the_child_is_given_an_empty_stdin(mut command: Command) { + let (stdin_reader, stdin_writer) = + std::io::pipe().expect("failed to create a pipe for the child's stdin"); + command.stdin(Stdio::from(stdin_reader)); + + let started = Instant::now(); + let output = run_with_timeout(command, Duration::from_secs(5), "process").expect( + "a child given an empty stdin must read EOF rather than block until the deadline", + ); + let elapsed = started.elapsed(); + // Dropped only here: releasing the write end any earlier would close the + // pipe, so the child would reach EOF through the caller's stdin and the + // test would pass whatever this function set. + drop(stdin_writer); + + assert!(output.status.success(), "{output:?}"); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + STDIN_EOF_MARKER, + "the child must be given an empty stdin rather than the one the caller set" + ); + assert!( + elapsed < Duration::from_secs(3), + "the read must hit EOF at once rather than waiting out the deadline, but the call \ + took {elapsed:?}" + ); + } + + #[cfg(unix)] + #[test] + fn a_child_that_reads_stdin_is_given_an_empty_one() { + // `read` reports EOF by returning non-zero, so the `||` branch is the + // one an empty stdin takes. A stdin left open takes neither. + let script = format!("read line || echo {STDIN_EOF_MARKER}"); + let mut command = Command::new("sh"); + command.args(["-c", script.as_str()]); + + assert_the_child_is_given_an_empty_stdin(command); + } + + #[cfg(unix)] + #[test] + fn a_genuinely_slow_command_times_out_on_time() { + for shell in grandchild_shells() { + // `sleep 30 & wait` rather than plain `sleep 30`: a lone final + // command is exec-ed into the shell itself by bash, which would + // leave nothing behind to hold the pipe and quietly turn this into + // a test of the easy case. Backgrounding forces a real grandchild + // under every shell. + // + // Killing the shell at the deadline does not kill that grandchild, + // so it keeps the inherited write end of both pipes open and no EOF + // arrives. An implementation that joins its readers therefore waits + // out the full 30 seconds and the timeout bounds nothing — which + // for the 2-second amd-smi probe is the difference between a snappy + // telemetry read and a stalled one. + let mut command = Command::new(shell); + command.args(["-c", "sleep 30 & wait"]); + + let started = Instant::now(); + let error = run_with_timeout(command, Duration::from_secs(1), "process") + .expect_err("a command that outlives its timeout must fail"); + let elapsed = started.elapsed(); + + assert!( + error.to_string().contains("exceeded 1s timeout"), + "the timeout must be reported as one, under {shell}: {error}" + ); + assert!( + elapsed < Duration::from_secs(5), + "the call must return at its deadline rather than waiting for an abandoned \ + grandchild to exit, but under {shell} it took {elapsed:?}" + ); + } + } + + #[cfg(unix)] + #[test] + fn a_grandchild_holding_the_pipe_does_not_hold_a_successful_call() { + for shell in grandchild_shells() { + // The timeout case above covers a child that is killed. This is the + // same hazard on the path that has no timeout to blame: the child + // exits 0 straight away, having left a background `sleep` holding + // the write end of both pipes, so there is no EOF for another five + // seconds. Joining the readers — or giving them an unbounded grace + // — hands the schedule to that `sleep` and a command that finished + // instantly takes 5s. `DRAIN_GRACE` is what bounds it. + let mut command = Command::new(shell); + command.args(["-c", "echo ready; sleep 5 &"]); + + let started = Instant::now(); + let output = run_with_timeout(command, Duration::from_secs(20), "process") + .expect("a child that exits cleanly must not be reported as a timeout"); + let elapsed = started.elapsed(); + + assert!(output.status.success(), "under {shell}: {output:?}"); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "ready", + "what the child wrote before exiting must still arrive, under {shell}" + ); + // Generous against a loaded machine, and still a third of the + // grandchild's lifetime: the regression this guards takes the full + // five seconds, not a few hundred milliseconds more than the grace. + assert!( + elapsed < Duration::from_secs(3), + "the call must not wait for the grandchild, but under {shell} it took {elapsed:?}" + ); + } + } + + #[cfg(unix)] + #[test] + fn output_arriving_after_the_child_exits_is_still_collected() { + for shell in grandchild_shells() { + // What `DRAIN_GRACE` is for, and the only way to ask for it on + // demand. A child that writes and exits normally cannot pin it: its + // reader is already running and drains the pipe microseconds later, + // long before the wait loop's next poll even notices the exit, so + // taking the buffer the instant the child is seen to have exited + // still collects everything. Here a grandchild writes once the + // child is gone instead. + // + // The 0.1s sits between the two bounds that decide the outcome: + // past `POLL_INTERVAL` (50ms), the longest the wait loop can take + // to notice the exit, so taking the output there misses + // `late-canary` outright; and well inside `DRAIN_GRACE` (200ms), so + // draining collects it. Both margins are ~2x, and the grandchild + // exits immediately after writing, so a working implementation sees + // EOF and returns rather than sitting out the whole grace. + let mut command = Command::new(shell); + command.args(["-c", "echo early-canary; (sleep 0.1; echo late-canary) &"]); + + let started = Instant::now(); + let output = run_with_timeout(command, Duration::from_secs(20), "process") + .expect("a child that exits cleanly must not be reported as a timeout"); + let elapsed = started.elapsed(); + let stdout = String::from_utf8_lossy(&output.stdout); + + assert!( + stdout.contains("early-canary"), + "under {shell}, the child's own output is missing: {stdout:?}" + ); + assert!( + stdout.contains("late-canary"), + "under {shell}, output still in flight when the child exited was dropped \ + instead of drained: {stdout:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "draining must end at EOF rather than running long, but under {shell} the call \ + took {elapsed:?}" + ); + } + } + + #[test] + fn an_interrupted_read_is_retried_and_a_failed_one_is_reported() { + // Neither branch is reachable through a real child here — EINTR needs a + // signal to land mid-read, and a pipe read failing outright needs the + // kernel to be having a bad day — so the reader is fed a scripted + // stream directly. The property is the one `read_to_end` gave before + // this function replaced it: an interrupted read is not the end of the + // stream, and a failed one is not a successful short read. + struct ScriptedPipe(std::collections::VecDeque>); + + impl Read for ScriptedPipe { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + match self.0.pop_front() { + Some(Ok(bytes)) => { + out[..bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + Some(Err(error)) => Err(error), + None => Ok(0), + } + } + } + + fn drain(script: Vec>) -> (Vec, Option) { + let capture = Capture::draining(Some(ScriptedPipe(script.into()))); + let deadline = Instant::now() + Duration::from_secs(10); + while !capture.is_finished() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!(capture.is_finished(), "the reader never finished"); + capture.take() + } + + let (bytes, failure) = drain(vec![ + Ok(b"before"), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"after"), + Ok(b""), + ]); + assert_eq!( + String::from_utf8_lossy(&bytes), + "beforeafter", + "an interrupted read must be retried, not taken for the end of the stream" + ); + assert!(failure.is_none(), "{failure:?}"); + + let (bytes, failure) = drain(vec![ + Ok(b"partial"), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"unreachable"), + ]); + assert_eq!(String::from_utf8_lossy(&bytes), "partial"); + assert_eq!( + failure.map(|error| error.kind()), + Some(ErrorKind::BrokenPipe), + "a read that failed must be reported rather than passed off as EOF" + ); + } + + /// A pipe that never ends until the test says so. + /// + /// Stoppable rather than genuinely endless so the reader thread finishes + /// with the test instead of spinning for the life of the test binary. + struct EndlessPipe { + reads: Arc, + stop: Arc, + } + + impl Read for EndlessPipe { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.stop.load(Ordering::Acquire) { + return Ok(0); + } + // Paced so the test can interleave with the reader rather than + // racing a tight loop, and so a regression accumulates a size the + // assertion can see without the test allocating gigabytes. + thread::sleep(Duration::from_millis(1)); + let filled = out.len().min(4096); + out[..filled].fill(b'x'); + self.reads.fetch_add(1, Ordering::Release); + Ok(filled) + } + } + + #[test] + fn an_abandoned_reader_stops_filling_a_buffer_nobody_will_read() { + // The shape this guards: the child exited but a grandchild still holds + // the write end, so `DRAIN_GRACE` expires, `take()` hands the caller + // what there is and the reader is left running. In `rocmd` — a daemon + // that probes amd-smi every 60s and runs MCP captures besides — a + // reader that kept appending would grow a `Vec` that nothing will ever + // read, for as long as the grandchild lives. There is no way to ask + // `run_with_timeout` for that state on demand, so the capture is driven + // directly, which is also where the invariant lives. + let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let capture = Capture::draining(Some(EndlessPipe { + reads: Arc::clone(&reads), + stop: Arc::clone(&stop), + })); + + let wait_for_reads = |target: usize| { + let deadline = Instant::now() + Duration::from_secs(10); + while reads.load(Ordering::Acquire) < target && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!( + reads.load(Ordering::Acquire) >= target, + "the reader stalled before {target} reads" + ); + }; + + wait_for_reads(4); + let (bytes, failure) = capture.take(); + // Establishes that the reader really was accumulating up to this point, + // so the emptiness asserted below is the flag working rather than a + // reader that never ran. + assert!( + !bytes.is_empty(), + "the reader must collect while someone is still listening" + ); + assert!(failure.is_none(), "{failure:?}"); + + // Well past what fits in anything the implementation might buffer + // internally: 50 more 4KiB reads is ~200KB, which a regression leaves + // sitting in the buffer and a correct implementation discards. + wait_for_reads(reads.load(Ordering::Acquire) + 50); + let leaked = capture.lock().len(); + assert_eq!( + leaked, 0, + "after `take()` nothing will read this buffer again, so a reader that keeps \ + appending grows it without bound for the life of the daemon; {leaked} bytes \ + accumulated" + ); + + stop.store(true, Ordering::Release); + let deadline = Instant::now() + Duration::from_secs(10); + while !capture.is_finished() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!(capture.is_finished(), "the reader never finished"); + } + + /// A child that never exits on its own and fails whichever of the wait + /// loop's two calls the test asks it to. + struct FailingChild { + /// `true`: `try_wait` reports an error rather than "still running", so + /// the loop's first `?` fires. `false`: it stays running until the + /// deadline, and the post-kill `wait` is what fails. + fail_poll: bool, + /// Raised by `kill`, so the test can tell a wait loop that leaves the + /// child running from one that does not. This is the whole reason + /// `kill` is on the trait rather than called on a concrete `Child`. + killed: Arc, + } + + impl Waitable for FailingChild { + fn try_wait(&mut self) -> std::io::Result> { + if self.fail_poll { + return Err(std::io::Error::from(ErrorKind::PermissionDenied)); + } + Ok(None) + } + + fn kill(&mut self) -> std::io::Result<()> { + self.killed.store(true, Ordering::Release); + Ok(()) + } + + fn wait(&mut self) -> std::io::Result { + Err(std::io::Error::from(ErrorKind::PermissionDenied)) + } + } + + /// Drive `run_captured` with a child that fails one of the wait loop's two + /// calls, and assert the error is reported, both readers are left + /// abandoned, *and* the child was killed on the way out. + /// + /// `fail_poll` picks which call fails; see [`FailingChild`]. + fn assert_wait_failure_abandons_both_readers( + fail_poll: bool, + timeout: Duration, + expected: &str, + ) { + let killed = Arc::new(AtomicBool::new(false)); + let child = FailingChild { + fail_poll, + killed: Arc::clone(&killed), + }; + let stop = Arc::new(AtomicBool::new(false)); + let out_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let err_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let stdout = Capture::draining(Some(EndlessPipe { + reads: Arc::clone(&out_reads), + stop: Arc::clone(&stop), + })); + let stderr = Capture::draining(Some(EndlessPipe { + reads: Arc::clone(&err_reads), + stop: Arc::clone(&stop), + })); + + let wait_for_reads = |counter: &Arc, target: usize| { + let deadline = Instant::now() + Duration::from_secs(10); + while counter.load(Ordering::Acquire) < target && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!( + counter.load(Ordering::Acquire) >= target, + "the reader stalled before {target} reads" + ); + }; + + // Both readers are accumulating before the call, so an empty buffer + // afterwards is the abandon working rather than a reader that never ran. + wait_for_reads(&out_reads, 4); + wait_for_reads(&err_reads, 4); + + let error = run_captured(child, stdout.clone(), stderr.clone(), timeout, "process") + .expect_err("a failed wait must be reported, not swallowed"); + assert!( + format!("{error:#}").contains(expected), + "expected {expected:?} in {error:#}" + ); + + // ~200KB more per stream: what a reader that was never marked abandoned + // accumulates into a buffer nobody will ever read again. + wait_for_reads(&out_reads, out_reads.load(Ordering::Acquire) + 50); + wait_for_reads(&err_reads, err_reads.load(Ordering::Acquire) + 50); + for (name, capture) in [("stdout", &stdout), ("stderr", &stderr)] { + let leaked = capture.lock().len(); + assert_eq!( + leaked, 0, + "an error out of the wait loop returns before the `take()` pair, so it must \ + abandon both readers itself or they grow a buffer nobody will read for as \ + long as a grandchild holds the pipe; {name} accumulated {leaked} bytes" + ); + } + + // Abandoning the readers is only half of it: the caller drops the child + // on this path without reaping it, so a wait loop that returns without + // killing leaves a process running that nobody will ever wait on again. + assert!( + killed.load(Ordering::Acquire), + "every exit from the wait loop must kill the child first; this one returned the \ + error with the child still running" + ); + + stop.store(true, Ordering::Release); + } + + #[test] + fn a_failure_to_poll_the_child_still_abandons_both_readers() { + assert_wait_failure_abandons_both_readers( + true, + Duration::from_secs(30), + "failed to poll process", + ); + } + + #[test] + fn a_failure_to_reap_a_timed_out_child_still_abandons_both_readers() { + // Zero timeout so the loop reaches the post-kill reap — the other `?` — + // on its first pass. + assert_wait_failure_abandons_both_readers( + false, + Duration::ZERO, + "failed to reap timed-out process", + ); + } + + /// A successful `ExitStatus` to feed `assemble_output`. + /// + /// `ExitStatus` has no portable public constructor; the per-platform + /// `ExitStatusExt` is the supported way to make one without spawning a + /// process purely to obtain it. + fn success_status() -> ExitStatus { + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt as _; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt as _; + ExitStatus::from_raw(0) + } + + #[test] + fn the_poll_interval_backs_off_from_a_few_milliseconds_to_the_flat_one() { + // The flat 50 ms this loop had would make a probe that exits at once take + // 50 ms, where the helpers it replaces polled every 20 to 25 ms. + let mut pause = FIRST_POLL; + let mut seen = vec![pause]; + for _ in 0..5 { + pause = next_poll_interval(pause); + seen.push(pause); + } + let millis: Vec = seen.iter().map(Duration::as_millis).collect(); + assert_eq!(millis, [5, 10, 20, 40, 50, 50]); + assert!(FIRST_POLL < Duration::from_millis(20)); + } + + /// A child that has not exited for its first `remaining` polls. + struct ExitsAfterPolls { + remaining: usize, + } + + impl Waitable for ExitsAfterPolls { + fn try_wait(&mut self) -> std::io::Result> { + if self.remaining == 0 { + return Ok(Some(success_status())); + } + self.remaining -= 1; + Ok(None) + } + + fn kill(&mut self) -> std::io::Result<()> { + Ok(()) + } + + fn wait(&mut self) -> std::io::Result { + Ok(success_status()) + } + } + + fn millis(slept: &[Duration]) -> Vec { + slept.iter().map(Duration::as_millis).collect() + } + + #[test] + fn the_wait_loop_sleeps_the_back_off_schedule_between_polls() { + // The sleeps are recorded, not taken, so the schedule the loop really + // follows is what is asserted. A loop that slept `POLL_INTERVAL` every + // time would charge a probe that exits at once the full 50 ms. + let mut slept = Vec::new(); + let mut child = ExitsAfterPolls { remaining: 6 }; + + let (status, timed_out) = + wait_bounded(&mut child, Duration::from_mins(1), "process", |pause| { + slept.push(pause); + }) + .expect("the child exits on the seventh poll"); + + assert!(status.success() && !timed_out); + assert_eq!(millis(&slept), [5, 10, 20, 40, 50, 50]); + } + + /// Two readers over [`EndlessPipe`]s that stop when `stop` is raised. + fn two_endless_captures(stop: &Arc) -> (Capture, Capture) { + let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let pipe = || EndlessPipe { + reads: Arc::clone(&reads), + stop: Arc::clone(stop), + }; + ( + Capture::draining(Some(pipe())), + Capture::draining(Some(pipe())), + ) + } + + #[test] + fn the_drain_wait_sleeps_the_back_off_schedule_until_both_readers_finish() { + let stop = Arc::new(AtomicBool::new(false)); + let (stdout, stderr) = two_endless_captures(&stop); + let mut slept = Vec::new(); + + let drained = wait_for_readers(&stdout, &stderr, Duration::from_secs(30), |pause| { + slept.push(pause); + if slept.len() == 3 { + // End the pipes, and give the reader threads real time to see it. + stop.store(true, Ordering::Release); + thread::sleep(Duration::from_millis(100)); + } + }); + + assert!(drained, "both readers reached the end of their pipes"); + assert_eq!(millis(&slept[..3]), [5, 10, 20]); + } + + #[test] + fn a_drain_wait_that_runs_out_of_grace_says_the_readers_were_still_going() { + let stop = Arc::new(AtomicBool::new(false)); + let (stdout, stderr) = two_endless_captures(&stop); + + let drained = wait_for_readers(&stdout, &stderr, Duration::ZERO, |_| { + unreachable!("no grace, so no sleep") + }); + + stop.store(true, Ordering::Release); + assert!(!drained); + } + + /// Drive `settle` for a child that fails the wait loop the way + /// [`FailingChild`] says, over two readers that are accumulating. + fn settle_a_failing_child(fail_poll: bool, timeout: Duration) -> (Settled, bool) { + let killed = Arc::new(AtomicBool::new(false)); + let child = FailingChild { + fail_poll, + killed: Arc::clone(&killed), + }; + let stop = Arc::new(AtomicBool::new(false)); + // One counter per stream: a single shared one is satisfied by one reader + // doing all the reads, which leaves the other stream's bytes to chance. + let out_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let err_reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let pipe = |reads: &Arc| EndlessPipe { + reads: Arc::clone(reads), + stop: Arc::clone(&stop), + }; + let stdout = Capture::draining(Some(pipe(&out_reads))); + let stderr = Capture::draining(Some(pipe(&err_reads))); + let deadline = Instant::now() + Duration::from_secs(10); + while (out_reads.load(Ordering::Acquire) < 4 || err_reads.load(Ordering::Acquire) < 4) + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(5)); + } + assert!( + out_reads.load(Ordering::Acquire) >= 4 && err_reads.load(Ordering::Acquire) >= 4, + "both readers must have started" + ); + + let settled = settle(child, stdout, stderr, timeout, "process"); + + stop.store(true, Ordering::Release); + (settled, killed.load(Ordering::Acquire)) + } + + #[test] + fn a_failure_to_wait_is_a_capture_that_keeps_the_bytes_that_had_arrived() { + // `run_raw` returned `124` with the bytes joined after a failed poll, so + // the capture keeps them rather than dropping them with the error, and + // the child is killed on the way out. + for (fail_poll, timeout, expected) in [ + (true, Duration::from_secs(30), "failed to poll process"), + (false, Duration::ZERO, "failed to reap timed-out process"), + ] { + let (settled, killed) = settle_a_failing_child(fail_poll, timeout); + assert!(killed, "{expected}: the child must be killed"); + + let capture = capture_of(vec!["fake".to_owned()], settled); + + assert_eq!(capture.outcome, CommandOutcome::WaitFailed, "{expected}"); + assert!(capture.status.is_none(), "{expected}"); + assert!( + capture.stdout_raw.len() >= 4096 && capture.stdout_raw.iter().all(|&b| b == b'x'), + "{expected}: stdout bytes: {}", + capture.stdout_raw.len() + ); + assert!( + capture.stderr_raw.len() >= 4096 && capture.stderr_raw.iter().all(|&b| b == b'x'), + "{expected}: stderr bytes: {}", + capture.stderr_raw.len() + ); + assert_eq!(capture.exit_status, 1, "{expected}"); + let detail = capture.detail.expect("the wait error is kept in words"); + assert!(detail.contains(expected), "{detail}"); + } + } + + fn waited( + timed_out: bool, + drained: bool, + stdout: (Vec, Option), + stderr: (Vec, Option), + ) -> Settled { + Settled::Waited { + status: success_status(), + timed_out, + drained, + stdout, + stderr, + } + } + + #[test] + fn a_timed_out_settle_is_a_capture_with_the_partial_output_and_the_outcome() { + let capture = capture_of( + vec!["fake".to_owned()], + waited( + true, + true, + (b"partial".to_vec(), None), + (b"warning".to_vec(), None), + ), + ); + + assert_eq!(capture.outcome, CommandOutcome::TimedOut); + assert_eq!(capture.stdout_raw, b"partial"); + assert_eq!(capture.stderr, "warning"); + assert_eq!(capture.exit_status, 1, "a timeout has no exit code"); + assert!(capture.detail.is_none()); + } + + #[test] + fn a_read_failure_on_either_stream_keeps_the_bytes_and_the_outcome_the_child_had() { + // `run_with_timeout` turns this into an error and drops the bytes + // (`a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output`); + // the capture keeps both, as the older helpers did with `let _ =`. + let stdout_only = capture_of( + vec!["fake".to_owned()], + waited( + false, + true, + (b"partial".to_vec(), read_failure()), + (Vec::new(), None), + ), + ); + assert_eq!(stdout_only.outcome, CommandOutcome::Exited(0)); + assert_eq!(stdout_only.stdout, "partial"); + let detail = stdout_only.detail.expect("the read error is kept in words"); + assert!(detail.contains("failed to read stdout"), "{detail}"); + assert!(!detail.contains("stderr"), "{detail}"); + + let stderr_only = capture_of( + vec!["fake".to_owned()], + waited( + false, + true, + (b"out".to_vec(), None), + (b"partial".to_vec(), read_failure()), + ), + ); + assert_eq!(stderr_only.outcome, CommandOutcome::Exited(0)); + assert_eq!(stderr_only.stderr, "partial"); + let detail = stderr_only.detail.expect("the read error is kept in words"); + assert!(detail.contains("failed to read stderr"), "{detail}"); + assert!(!detail.contains("stdout"), "{detail}"); + + let both = capture_of( + vec!["fake".to_owned()], + waited( + false, + true, + (Vec::new(), read_failure()), + (Vec::new(), read_failure()), + ), + ); + let detail = both.detail.expect("both read errors are kept in words"); + assert!( + detail.contains("failed to read stdout") && detail.contains("failed to read stderr"), + "{detail}" + ); + } + + #[test] + fn a_capture_says_when_a_reader_was_still_going_at_the_end_of_the_grace() { + let short = capture_of( + vec!["fake".to_owned()], + waited(false, false, (b"early".to_vec(), None), (Vec::new(), None)), + ); + assert_eq!(short.outcome, CommandOutcome::Exited(0)); + assert_eq!(short.stdout, "early"); + assert_eq!(short.detail.as_deref(), Some(STILL_ARRIVING)); + + let complete = capture_of( + vec!["fake".to_owned()], + waited(false, true, (b"early".to_vec(), None), (Vec::new(), None)), + ); + assert!(complete.detail.is_none()); + } + + #[cfg(unix)] + #[test] + fn output_written_after_the_grace_is_missing_and_the_capture_says_so() { + // The descendant writes 0.5 s after the child exits, which is past the + // 200 ms grace: the capture holds `early`, and `detail` says it is short. + let mut command = Command::new("sh"); + command.args(["-c", "echo early; (sleep 0.5; echo late) &"]); + + let capture = run_bounded(command, Duration::from_secs(20), "process"); + + assert_eq!(capture.outcome, CommandOutcome::Exited(0)); + assert_eq!(capture.stdout, "early\n"); + assert_eq!(capture.detail.as_deref(), Some(STILL_ARRIVING)); + } + + #[test] + fn a_program_that_cannot_be_started_is_a_capture_not_an_error() { + let missing = "/nonexistent-dir-for-the-runner-pair/tool"; + let mut command = Command::new(missing); + command.arg("--flag"); + + let capture = run_bounded(command, Duration::from_secs(5), "process"); + + assert_eq!(capture.outcome, CommandOutcome::NotFound); + assert_eq!(capture.argv, [missing, "--flag"]); + assert!(capture.stdout_raw.is_empty() && capture.stderr_raw.is_empty()); + assert_eq!(capture.exit_status, 1); + assert!(capture.status.is_none()); + assert!(capture.detail.is_some(), "the OS error is kept in words"); + } + + #[cfg(unix)] + #[test] + fn a_program_that_exists_but_cannot_be_executed_is_a_spawn_failure_not_not_found() { + use std::os::unix::fs::PermissionsExt as _; + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("not-executable"); + std::fs::write(&path, "#!/bin/sh\n").expect("write"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).expect("chmod"); + + // Not skipped for root: `execve` refuses a file with no execute bit for + // every user. + let capture = run_bounded(Command::new(&path), Duration::from_secs(5), "process"); + + assert_eq!( + capture.outcome, + CommandOutcome::SpawnFailed(ErrorKind::PermissionDenied) + ); + } + + #[cfg(unix)] + #[test] + fn a_run_is_a_capture_with_its_exit_code_or_its_signal_and_both_streams() { + let capture = run_bounded( + { + let mut command = Command::new("sh"); + command.args(["-c", "echo out; echo err >&2; exit 3"]); + command + }, + Duration::from_secs(10), + "process", + ); + assert_eq!(capture.outcome, CommandOutcome::Exited(3)); + assert_eq!(capture.exit_status, 3); + assert_eq!(capture.stdout, "out\n"); + assert_eq!(capture.stderr_raw, b"err\n"); + assert_eq!(capture.argv, ["sh", "-c", "echo out; echo err >&2; exit 3"]); + assert_eq!(capture.status.and_then(|status| status.code()), Some(3)); + assert!(capture.detail.is_none()); + + let capture = run_bounded( + { + let mut command = Command::new("sh"); + command.args(["-c", "echo partial; kill -9 $$"]); + command + }, + Duration::from_secs(10), + "process", + ); + assert_eq!(capture.outcome, CommandOutcome::Signaled(Some(9))); + assert_eq!(capture.exit_status, 1); + assert_eq!(capture.stdout, "partial\n"); + } + + #[cfg(windows)] + #[test] + fn a_run_is_a_capture_with_its_exit_code_and_both_streams() { + let mut command = Command::new("cmd"); + command.args(["/C", "echo out&echo err 1>&2&exit 3"]); + + let capture = run_bounded(command, Duration::from_secs(10), "process"); + + assert_eq!(capture.outcome, CommandOutcome::Exited(3)); + assert_eq!(capture.stdout.trim(), "out"); + assert_eq!(capture.stderr.trim(), "err"); + } + + #[cfg(unix)] + #[test] + fn a_timeout_is_a_capture_with_what_the_child_had_written() { + // `exec` so killing the shell leaves nothing on the pipe: what is under + // test is the partial output and the outcome, not the join. + let mut command = Command::new("sh"); + command.args(["-c", "echo before; exec sleep 30"]); + + // Room for a loaded host to start the shell and run its `echo` before the + // deadline, since the assertion is on what the child had written by then. + let started = Instant::now(); + let capture = run_bounded(command, Duration::from_secs(3), "process"); + + assert_eq!(capture.outcome, CommandOutcome::TimedOut); + assert_eq!(capture.stdout, "before\n"); + assert!(started.elapsed() < Duration::from_secs(10)); + } + + fn read_failure() -> Option { + Some(std::io::Error::from(ErrorKind::BrokenPipe)) + } + + #[test] + fn a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output() { + // The silent short read this function exists to prevent: handing back + // `Ok` with a truncated buffer sends the caller off to parse half a + // JSON document and blame the document. + let error = assemble_output( + success_status(), + false, + (b"partial".to_vec(), read_failure()), + (Vec::new(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a stream that failed to read must not come back as success"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains("failed to read the output of process"), + "{rendered:?}" + ); + } + + #[test] + fn a_failure_on_either_stream_is_reported() { + // stdout is checked first, so a stderr-only failure is a separate + // branch: `stdout_failure.or(stderr_failure)` reduced to + // `stdout_failure` would still pass the case above. + let error = assemble_output( + success_status(), + false, + (Vec::new(), None), + (b"partial".to_vec(), read_failure()), + "process", + Duration::from_secs(2), + ) + .expect_err("a stderr stream that failed to read must not come back as success"); + + assert!( + format!("{error:#}").contains("failed to read the output of process"), + "{error:#}" + ); + } + + #[test] + fn a_timeout_outranks_a_read_failure() { + // Killing the child is what broke the pipe, so a read error here is the + // symptom. Reporting it would name the consequence and hide the cause, + // and the caller would go looking for a kernel problem. + let error = assemble_output( + success_status(), + true, + (Vec::new(), read_failure()), + (b"the real explanation".to_vec(), read_failure()), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains("exceeded 2s timeout"), + "the timeout is the cause and must be what is reported: {rendered:?}" + ); + assert!( + rendered.contains("the real explanation"), + "the child's own account must survive: {rendered:?}" + ); + assert!( + !rendered.contains("failed to read the output of"), + "the read error is downstream of the kill and must not displace it: {rendered:?}" + ); + } + + #[test] + fn a_timeout_quotes_stderr_in_preference_to_stdout() { + let error = assemble_output( + success_status(), + true, + (b"stdout-data".to_vec(), None), + (b"stderr-explanation".to_vec(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + + let rendered = format!("{error:#}"); + assert!( + rendered.contains("stderr-explanation"), + "stderr is where a child explains itself: {rendered:?}" + ); + assert!( + !rendered.contains("stdout-data"), + "quoting both would push the explanation behind a payload: {rendered:?}" + ); + } + + #[test] + fn a_timeout_falls_back_to_stdout_and_then_to_saying_there_was_none() { + // Every other timeout fixture writes to stderr, so without these two + // the fallback chain could be reordered or deleted unnoticed — and a + // child whose whole output is stdout is exactly the amd-smi case. + let error = assemble_output( + success_status(), + true, + (b"stdout-only".to_vec(), None), + (Vec::new(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + assert!( + format!("{error:#}").contains("stdout-only"), + "a child that only wrote stdout must still be quoted: {error:#}" + ); + + let error = assemble_output( + success_status(), + true, + (Vec::new(), None), + (Vec::new(), None), + "process", + Duration::from_secs(2), + ) + .expect_err("a timed-out call must fail"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("no output"), + "a silent child must say so rather than trailing a bare colon: {rendered:?}" + ); + assert!(rendered.contains("exceeded 2s timeout"), "{rendered:?}"); + } + + #[test] + fn a_clean_run_returns_both_streams() { + let output = assemble_output( + success_status(), + false, + (b"out".to_vec(), None), + (b"err".to_vec(), None), + "process", + Duration::from_secs(2), + ) + .expect("a clean run must succeed"); + + assert!(output.status.success()); + assert_eq!(output.stdout, b"out"); + assert_eq!(output.stderr, b"err"); + } + + #[cfg(unix)] + #[test] + fn a_child_that_floods_both_pipes_does_not_deadlock() { + // The defect this function exists to prevent. Both pipes are bounded by + // the OS (commonly ~64KiB), so a child writing past that blocks in + // `write` and never exits — and a version of this function that drains + // only after the child exits never reads a byte, so the child is killed + // at the timeout and a successful command is reported as too slow. + // + // Flooding *both* streams is deliberate: a single-pipe child still + // passes against an implementation that drains stdout fully and only + // then starts on stderr, which is its own deadlock. + // + // `sh` rather than a real subject program (amd-smi, an SDK install), + // because the property is about this function's pipe handling and + // nothing else, and a shell makes it a unit test rather than one that + // needs a multi-GPU host. + let mut command = Command::new("sh"); + command.args([ + "-c", + "yes deadlock-canary | head -c 2000000; yes deadlock-canary | head -c 2000000 >&2", + ]); + + // Short, because a regression fails by exhausting this timeout: ten + // seconds is far past the ~0.1s a working drain takes, and far short of + // stalling the suite. + let output = run_with_timeout(command, Duration::from_secs(10), "process") + .expect("a child that outruns the pipe buffer must not be killed as a timeout"); + + assert!(output.status.success(), "{output:?}"); + // Without these the test would keep passing if the child ever stopped + // producing more than one buffer, which is to say it would stop testing + // the hazard. + assert!( + output.stdout.len() > 64 * 1024, + "stdout must exceed one pipe buffer: {} bytes", + output.stdout.len() + ); + assert!( + output.stderr.len() > 64 * 1024, + "stderr must exceed one pipe buffer: {} bytes", + output.stderr.len() + ); + } + + /// How large a quoted-output assertion lets the whole error message get. + /// + /// Comfortably above `MAX_QUOTED_OUTPUT_CHARS` plus the label and marker, + /// and two orders of magnitude below the ~200KB the fixtures below feed in, + /// so the assertion answers "is it bounded at all" rather than pinning the + /// exact constant — which is a tuning decision, not a contract. + const QUOTE_ASSERTION_CEILING: usize = 4000; + + /// The flooding fixtures below write this many bytes, past one pipe buffer + /// and ~100x the quote bound. + const FLOOD_BYTES: usize = 200_000; + + const FLOOD_CANARY: &str = "flood-canary"; + + /// The assertions shared by both platforms' timeout-quote cases. + /// + /// Checked together because they trade off against each other: a bound that + /// kept nothing would satisfy the size check while making the message + /// useless, and quoting everything keeps it useful while costing the logs + /// the thing this bound exists to protect. + fn assert_timeout_quote_is_bounded_and_useful(message: &str, timeout_secs: u64) { + assert!( + message.contains(&format!("exceeded {timeout_secs}s timeout")), + "the timeout must still be named: {message:?}" + ); + assert!( + message.contains(FLOOD_CANARY), + "the child's own output must still be quoted, not dropped: {message:?}" + ); + assert!( + message.contains("[truncated]"), + "a shortened quote must say so, or a reader takes the head for the whole: {message:?}" + ); + let length = message.chars().count(); + assert!( + length <= QUOTE_ASSERTION_CEILING, + "the quote must be bounded: the message is {length} characters, from a child that \ + printed {FLOOD_BYTES} bytes" + ); + } + + #[cfg(unix)] + #[test] + fn a_timed_out_child_has_its_output_quoted_within_a_bound() { + for shell in grandchild_shells() { + // Both halves are load-bearing. The flood is what makes the quote + // large; the backgrounded `sleep` is what makes this a timeout, + // which is the only path that quotes the child's output at all. + // + // Unbounded, this message travels: `rocmd` renders the error chain + // into the bridge snapshot's `note`, and the GPU-metrics watcher + // writes that note into two JSONL logs every 60s, neither ever + // rotated or capped. A real amd-smi on a multi-GPU host prints + // well past this fixture's size. + let mut command = Command::new(shell); + command.args([ + "-c", + &format!("yes {FLOOD_CANARY} | head -c {FLOOD_BYTES} >&2; sleep 30 & wait"), + ]); + + let error = run_with_timeout(command, Duration::from_secs(1), "process") + .expect_err("a command that outlives its timeout must fail"); + + assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 1); + } + } + + /// A scratch directory holding a Windows fixture's script, and for + /// [`Self::flooding`] fixtures the payload beside it. + /// + /// Per-process unique so concurrent test binaries cannot collide on it, and + /// removed on drop so a failing assertion does not leave either behind. + #[cfg(windows)] + struct WindowsFixture { + dir: std::path::PathBuf, + } + + #[cfg(windows)] + impl WindowsFixture { + /// Write `script` as a batch file in a scratch directory of its own. + /// + /// A batch file rather than `cmd /C ""`: cmd's rules for + /// stripping the quotes Rust adds around an argument containing spaces + /// and redirections are subtle enough that the fixture would become the + /// thing under test. A bare path has none of that, and `%~dp0` inside + /// the script resolves anything beside it without the test passing a + /// path in. + fn script(name: &str, script: &str) -> Self { + let dir = std::env::temp_dir() + .join(format!("rocm-cli-process-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("failed to create the fixture directory"); + std::fs::write(dir.join("fixture.bat"), script).expect("failed to write the script"); + + Self { dir } + } + + /// A [`Self::script`] fixture with a `FLOOD_BYTES` payload beside it for + /// the script to print. + fn flooding(name: &str, script: &str) -> Self { + let fixture = Self::script(name, script); + + let line = format!("{FLOOD_CANARY}\r\n"); + let mut payload = String::with_capacity(FLOOD_BYTES + line.len()); + while payload.len() < FLOOD_BYTES { + payload.push_str(&line); + } + std::fs::write(fixture.dir.join("payload.txt"), payload) + .expect("failed to write the payload"); + + fixture + } + + fn command(&self) -> Command { + let mut command = Command::new("cmd"); + command.args(["/C"]).arg(self.dir.join("fixture.bat")); + command + } + } + + #[cfg(windows)] + impl Drop for WindowsFixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + #[cfg(windows)] + #[test] + fn a_child_that_floods_both_pipes_does_not_deadlock() { + // The Windows half of the unix case above, and the same property: both + // pipes are bounded by the OS here too — more tightly than on Linux — + // so an implementation that drains only after the child exits, or + // drains stdout to completion before starting on stderr, never lets + // this child finish. `type` twice is the shell-free equivalent of the + // unix `yes | head`; cmd has no `yes`. + let fixture = WindowsFixture::flooding( + "flood", + "@echo off\r\ntype \"%~dp0payload.txt\"\r\ntype \"%~dp0payload.txt\" 1>&2\r\n", + ); + + let output = run_with_timeout(fixture.command(), Duration::from_secs(30), "process") + .expect("a child that outruns the pipe buffer must not be killed as a timeout"); + + assert!(output.status.success(), "{output:?}"); + assert!( + output.stdout.len() > 64 * 1024, + "stdout must exceed one pipe buffer: {} bytes", + output.stdout.len() + ); + assert!( + output.stderr.len() > 64 * 1024, + "stderr must exceed one pipe buffer: {} bytes", + output.stderr.len() + ); + } + + #[cfg(windows)] + #[test] + fn a_timed_out_child_has_its_output_quoted_within_a_bound() { + // Covers on Windows what three separate unix cases cover there, because + // one fixture happens to exhibit all of it: `ping -n 30` runs for ~29 + // seconds, so the call times out; `ping` is a grandchild of the `cmd` + // this function kills at the deadline, and killing a process on Windows + // does not kill its children, so it keeps the inherited write ends open + // and no EOF arrives — an implementation that joins its readers waits + // the full 29 seconds; and the preceding flood makes the resulting + // quote large enough for the bound to matter. + let fixture = WindowsFixture::flooding( + "timeout", + "@echo off\r\ntype \"%~dp0payload.txt\" 1>&2\r\nping -n 30 127.0.0.1\r\n", + ); + + let started = Instant::now(); + let error = run_with_timeout(fixture.command(), Duration::from_secs(2), "process") + .expect_err("a command that outlives its timeout must fail"); + let elapsed = started.elapsed(); + + assert_timeout_quote_is_bounded_and_useful(&format!("{error:#}"), 2); + assert!( + elapsed < Duration::from_secs(10), + "the call must return at its deadline rather than waiting for an abandoned \ + grandchild to exit, but it took {elapsed:?}" + ); + } + + #[cfg(windows)] + #[test] + fn a_child_that_reads_stdin_is_given_an_empty_one() { + // `set /p` leaves the variable untouched when its read reaches EOF, so + // pre-setting it to the marker is cmd's equivalent of the unix + // `read || echo`. A stdin left open blocks in `set /p` instead. + // + // Both callers in `rocmd` run on Windows -- the amd-smi probe in + // `apps/rocmd/src/host_snapshot.rs` and the `rocm` capture that + // `apps/rocmd/src/common.rs` makes through `run_rocm_capture_with_env` + // -- and `run_with_timeout` has no `cfg(windows)` branch, so this covers + // the same promise with the same two-sidedness as the unix case. + let fixture = WindowsFixture::script( + "stdin", + &format!( + "@echo off\r\nset \"LINE={STDIN_EOF_MARKER}\"\r\nset /p LINE=\r\necho %LINE%\r\n" + ), + ); + + assert_the_child_is_given_an_empty_stdin(fixture.command()); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 124a86c6e7..a44a608b5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -154,7 +154,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`anthropic_proxy.rs`](../apps/rocmd/src/anthropic_proxy.rs) — the Anthropic-compatible `/v1/messages` front for a served model: `spawn_alongside_engine` waits for the engine to list its model, then binds a translation proxy, even for an engine that has its own `/v1/messages` route because that route only accepts the served model's name (`anthropic_to_openai`, `openai_to_anthropic`, `StreamTranslator` for `stream: true` requests, with tool use translated both ways for non-streaming ones, an axum router over `/v1/messages` and `/v1/models`) on a port of its own, publishing the result in the engine state file for the service record to adopt. It is a library module, not a daemon task: `rocm`'s `run_builtin_engine_serve_http` in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) starts it on a thread of the process that owns the engine, so it lives and dies with the service. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). +- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal; with no watcher enabled it exits by itself, deciding that under the autostart lock and publishing "stopped" and clearing its spawn claim before releasing it — `record_daemon_stop`, `release_own_autostart_claim`) and `rocmd status` (`print_status`), with the watcher tick interval, which `LoopHooks` makes injectable for tests. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. - [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs) — which binary a daemon spawn runs. The daemon runs as the `rocmd` binary or embedded in `rocm` (`rocm daemon`, which enters through `run_from_args` and marks the process), and `supervise` is a command of `rocmd` alone. The hidden engine commands (`__engine-serve-http`, `__engine-stdio`) are `rocm`'s alone. `DaemonHost` holds the one executable path every choice and sibling lookup is made from (`rocm_core::sibling_binary_path_from`). `rocmd_binary` returns the current executable when the daemon is `rocmd` and the `rocmd` installed beside `rocm` when it is embedded; `rocm_binary` is the mirror, the current executable when embedded and the `rocm` installed beside `rocmd` when not. Each names what was missing when there is none. `detect` returns the real host. Under test it returns the test binary for both commands and looks for no sibling, and the two stand-in hosts (`test_binary_standing_in_for_rocm` and `test_binary_standing_in_for_rocmd`) name only the binary they stand in for and fail when asked for the other, so none of them can run a `rocm` or `rocmd` built beside the test binary; a test that needs another binary passes the code it exercises a host of its own. A leaf: it imports no other crate module. - [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process, run through the same client `rocm` uses, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs); `rocmd` chooses what to run, `__engine-stdio ` on the `rocm` binary that `DaemonHost::rocm_binary` finds, since `__engine-stdio` is `rocm`'s command, and what to ask of the client: its own app paths, the service's endpoint key file when it holds a usable key, and stderr kept for the error rather than forwarded), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate but [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs); [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. @@ -189,7 +189,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151) automation-daemon runtime state and its on-disk event/proposal/audit records, and the policy applied to each store (which are rotated, and the proposal compaction) — see the module's own doc comment for what it owns. The generic JSONL machinery under those stores lives in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs). The dependency runs one way: this module reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), and [`config.rs`](../crates/rocm-core/src/config.rs) imports nothing from here, because the built-in watcher catalog (whose entries carry a default `WatcherMode`, and which `RocmCliConfig::watcher_enabled` and `effective_watcher_mode` take as an argument) lives in [`config.rs`](../crates/rocm-core/src/config.rs) beside `WatcherMode`. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. -- [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs) — (ROCMAI-720) running the `rocm` binary as a captured child: `CommandCapture`, `run_rocm_capture_with_env` and the `command_capture_text` formatter, shared by `rocmd`'s sandbox/MCP clusters and `rocm`'s chat tool bridge. Each caller passes the environment the child should get rather than this module choosing it; the bounded-execution half is [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`. +- [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs) — (ROCMAI-720) running the `rocm` binary as a captured child: `CommandCapture`, `run_rocm_capture_with_env` and the `command_capture_text` formatter, shared by `rocmd`'s sandbox/MCP clusters and `rocm`'s chat tool bridge. Each caller passes the environment the child should get rather than this module choosing it; the bounded-execution half is [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_with_timeout` (re-exported from [`process.rs`](../crates/rocm-core/src/process.rs)). `CommandCapture` is also the result carrier of that executor for the host probes ([`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded`): besides the lossy text and the legacy `exit_status` it holds the raw bytes of both streams, a typed `CommandOutcome` (not found, spawn failed, timed out, wait failed, exited with a code, killed by a signal) and the status and error text the executor saw. A capture answered without a child is built with `CommandCapture::from_text`. - [`config.rs`](../crates/rocm-core/src/config.rs) — (ROCMAI-151) the unified `RocmCliConfig` and its nested sections, including the persisted `WatcherMode` enum, the built-in watcher catalog (`BuiltinWatcherSpec`, `builtin_watchers`, `builtin_watcher`) and the rocm-dash dashboard sub-config — see the module's own doc comment for what it owns. - [`confirm.rs`](../crates/rocm-core/src/confirm.rs) — (issue #769) the shared `[y/N]` prompt primitives: `parse_answer`, which sorts a typed line into `Answer::Yes`, `Answer::No` (`n`, `no` or an empty line) or `Answer::Unrecognized`, plus `print_prompt` and `read_answer` (each generic over the `Write`/`BufRead` stream so the sequence is unit-testable without a controlled terminal). `read_answer` takes the output stream as well: it ends the prompt line when the input ended without a newline (Ctrl-D, or piped input with no final newline) and says so when an answer is unrecognized and treated as no, showing at most the first 40 characters of the answer, best effort, so a failed write never replaces the verdict. Deduplicates the print-prompt/read-line/parse sequence that [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs)'s `confirm()`, [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs)'s `confirm_uninstall` and [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs)'s `prompt_yes_no` each used to hand-roll. [`uninstall.rs`](../apps/rocm/src/uninstall.rs), [`storage.rs`](../apps/rocm/src/storage.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) reach the prompt only through `confirm_uninstall`. `print_prompt` and `read_answer` stay separate rather than one bundled call because [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs)'s `confirm()` attempts the read even when the prompt couldn't be written, while the other callers treat a failed write as fatal; [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs)'s `confirm()` also keeps its own `--yes`/non-interactive short-circuit and refusal wording on top of these, since those are specific to the `fix` recipe flow. The module also holds `interactive_terminal`, whether stdin and stdout are both terminals, which callers check before they prompt, and `require_yes_or_interactive`, the `--yes`/terminal guard that [`uninstall.rs`](../apps/rocm/src/uninstall.rs), [`storage.rs`](../apps/rocm/src/storage.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) put in front of that prompt; `interactive_terminal` is re-exported from `lib.rs` as `rocm_core::interactive_terminal` and from `host_gpu.rs` as `rocm_core::host_gpu::interactive_terminal`, the path it had before it moved here. - [`crypto.rs`](../crates/rocm-core/src/crypto.rs) — (ROCMAI-152, Phase 7c) RSASSA-PKCS#1 v1.5/SHA-256 sign/verify, generic — not model-catalog-specific. @@ -203,7 +203,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) — pure arithmetic over that header: weights + KV cache at a context size and KV type + compute buffer + runtime reserve against dedicated VRAM, the largest context that fits, and what would make a model that does not fit, fit. - [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) — where a header comes from: a local file, or a Hugging Face file, found by one repository-listing request (and a `HEAD` request for a file the listing does not size) and read by HTTP range requests (`HF_ENDPOINT` selects a mirror; redirects are followed by hand, up to five, and never from https to plain http). The only code in the estimate that reads a file or the network. - [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). -- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). It is a snapshot, to be retired when the convergence tickets close. +- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). The first step of the subprocess runners is delivered too: [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded` is the executor behind `run`, `capture_optional_command*` and the Windows diagnostic capture. It is a snapshot, to be retired when the convergence tickets close. - [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and nothing in this module depends on it, in production or in its own tests: those drive the engine with a small record type of their own. The module is private; its engine items are `pub(crate)`. - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. @@ -211,8 +211,9 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`net.rs`](../crates/rocm-core/src/net.rs) — (ROCMAI-154) networking primitives: the default local host and port, URL/endpoint formatting and parsing, and the hand-rolled loopback HTTP/TCP client (`http_get_text*`, `http_post_json_with_auth`, `read_http_response_bounded`). It depends on no other service module: the managed-service readiness probes built on it live in [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) and the streaming downloads in [`download.rs`](../crates/rocm-core/src/download.rs). - [`paths.rs`](../crates/rocm-core/src/paths.rs) — `AppPaths` and its filesystem-location primitives, including `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, and the constants that name the variables (`CONFIG_DIR_ENV`, `DATA_DIR_ENV`, `CACHE_DIR_ENV`, `ENGINE_ENVS_ROOT_ENV`). - [`placeholder.rs`](../crates/rocm-core/src/placeholder.rs) — the `` placeholder rendered reports print for an empty value: `empty_as_unknown` (returns the value untouched) and `trimmed_or_unknown` (trims first), shared by `apps/rocm`'s driver-install plan and chat tool summaries and by [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs)'s Windows GPU diagnostics, replacing the copy those two `apps/rocm` callers shared in [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and `host_gpu.rs`'s own trimming copy. -- [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) — the host probes' subprocess and file-read helpers, split out of [`examine.rs`](../crates/rocm-core/src/examine.rs) so `fix.rs`, `process.rs` and the child modules of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) reach them without importing the module that owns the `Examination` document: `run`/`run_with_env`/`run_raw` (run a command under a timeout; `rc` is `127` when it cannot be spawned and `124` on timeout), `run_utf16le`/`decode_utf16le` (for `wsl.exe`'s UTF-16LE output), `read_text`/`read_host_text`, `which` (the one `PATH` lookup, used by both host-probing stacks: it skips an empty `PATH` entry, searches a `PATH` that is not valid Unicode and follows `PATHEXT` on Windows) with `tool_path_candidates`, the spellings it tries and the capture runner in [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) starts in turn, the `SHORT`/`MEDIUM` timeouts and `truncate_to_chars`. All are `pub(crate)` except `run_raw`, which stays private to the module. The module itself is private to the crate and is not re-exported. -- [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus `run_with_timeout`: it drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. It is not yet the *only* such helper — [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded`, [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `run_raw`, and [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs)'s `capture_optional_command_candidate_with_timeout` (plus the Windows-only `capture_diagnostic_command` in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs)) still join their reader threads with no time limit, so a grandchild holding the pipe can outlast their timeout; converging them onto `run_with_timeout` is follow-up work to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). +- [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) — the host probes' subprocess and file-read helpers, split out of [`examine.rs`](../crates/rocm-core/src/examine.rs) so `fix.rs`, `process/bounded.rs` and the child modules of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) reach them without importing the module that owns the `Examination` document: `run`/`run_with_env`/`run_raw` (run a command under a timeout, as thin adapters over [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded`; `rc` is `127` when it cannot be spawned, `124` on timeout or a failure to wait and `-1` for a child killed by a signal), `run_utf16le`/`decode_utf16le` (for `wsl.exe`'s UTF-16LE output), `read_text`/`read_host_text`, `which` (the one `PATH` lookup, used by both host-probing stacks: it skips an empty `PATH` entry, searches a `PATH` that is not valid Unicode and follows `PATHEXT` on Windows) with `tool_path_candidates`, the spellings it tries and the capture runner in [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) starts in turn, the `SHORT`/`MEDIUM` timeouts and `truncate_to_chars`. All are `pub(crate)` except `run_raw`, which stays private to the module. The module itself is private to the crate and is not re-exported. +- [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus a re-export of the bounded subprocess executor, `run_with_timeout` and `run_bounded`, which lives in [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs) (split out of this file to keep it under the module size limit, with its tests, and re-exported here so every caller's path is unchanged). +- [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs) — the bounded subprocess executor behind `run_with_timeout` and `run_bounded`, a private child of [`process.rs`](../crates/rocm-core/src/process.rs) whose items are re-exported from there. It drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. The same executor is also reachable as `run_bounded`, which returns the run as a [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs) `CommandCapture` (raw bytes, a typed `CommandOutcome`, the partial output of a timed-out child) instead of an `Err`, and is what [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `run_raw`, [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs)'s `capture_optional_command_candidate_with_timeout` and the Windows-only `capture_diagnostic_command` in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) now run on, so none of them waits for a descendant that holds a pipe beyond the 200 ms drain grace. Its wait loop starts at a few milliseconds and backs off to 50 ms, so a probe that exits at once is not charged the full interval. It is not yet the *only* such helper: [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded` still joins its reader threads with no time limit, so a grandchild holding the pipe can outlast its timeout; converging it is the remaining part of the follow-up to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). - [`quoting.rs`](../crates/rocm-core/src/quoting.rs) — (ROCMAI-727) argument-quoting helpers with real execution stakes (POSIX shell, PowerShell), consolidated out of `apps/rocm`-side near-duplicates — see the module's own doc comment for what it owns. `posix_shell_quote` also renders the command lines the CLI shows (through `apps/rocm`'s `shell_quote`); only `comfyui.rs`'s log-line quoter stays separate. - [`report_rollup.rs`](../crates/rocm-core/src/report_rollup.rs) — (ROCMAI-492) counts read Doctor reports at every level of `Report::group_key`, with refused and unread reports kept apart. Pure, with no transport: the mailbox adapter that feeds it is maintainer tooling in `xtask`, behind `cargo xtask report-rollup`. - [`report_trend.rs`](../crates/rocm-core/src/report_trend.rs) — (ROCMAI-745) the same reports counted per week, and the weeks in which a known problem rose sharply. Pure like the rollup: it is handed a `Week`, which is all a date reduces to, and holds no time finer than that. @@ -227,7 +228,7 @@ What else stays in [`lib.rs`](../crates/rocm-core/src/lib.rs), beside module dec - **Permanent crate-root primitives.** Each is consumed from more than one module or crate, and none belongs to a domain an existing module owns; a dedicated module would add a path segment and nothing else. - `unix_time_millis` — the crate-wide clock helper, used across this crate's own modules and throughout `rocm` and `rocmd`. - `require_nonempty` — non-empty field validation, used by this crate's automation, binary-path and model-catalog modules and by all three engine crates. - - `command_failure_detail` — picks the best detail string from a failed subprocess's output, used by [`process.rs`](../crates/rocm-core/src/process.rs), the vLLM engine, and the AIM engine's [`docker.rs`](../engines/aim/src/docker.rs). + - `command_failure_detail` — picks the best detail string from a failed subprocess's output, used by [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs), the vLLM engine, and the AIM engine's [`docker.rs`](../engines/aim/src/docker.rs). - `sanitize_component` — the lowercase-alphanumeric slugger shared by [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs)'s `generate_service_id` and [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs)'s artifact cache-marker names. It is not model-catalog-specific, so it does not move there. - `SDK_TORCH_STACK_PACKAGES` (the packages the opt-out spares, read by both the CLI and the vLLM engine) beside `TORCH_ALIGNMENT_DISABLED_ENV`/`torch_alignment_disabled` — the one read of the torch-alignment opt-out, shared by the CLI and the vLLM engine so the two cannot disagree. - `VLLM_GPU_MEMORY_UTILIZATION_HINT` — a single wording contract between the CLI, the vLLM engine, and this crate's own `fix-16-vllm-oom` diagnosis; `rocm-core` is the only crate all of them can reach. diff --git a/docs/host-probing.md b/docs/host-probing.md index cc67ef900c..b084c12705 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: MIT # Host probing: which stack owns each fact -Status: proposal, with the first three pairs delivered. This note changes no code and no behaviour itself, and what it proposes for a pair is decided by the change that converges that pair, which then records the decision here (the PATH lookup, pair 1, the kernel release, pair 2, and the CPU model, pair 3, have: see "PATH lookup", "Kernel version" and "CPU model"). It lists the host facts that `rocm-core` reads twice (on the hosts each is read on, see "Which host reads which fact"), says how the two reads differ, and proposes which one should survive and in what order a later series of changes could remove the other. Each pair is meant to be a separate change with its own review. +Status: proposal, with the first three pairs and the first step of the last delivered. This note changes no code and no behaviour itself, and what it proposes for a pair is decided by the change that converges that pair, which then records the decision here (the PATH lookup, pair 1, the kernel release, pair 2, and the CPU model, pair 3, have: see "PATH lookup", "Kernel version" and "CPU model"; the first step of the subprocess runners, pair 6, has: see "Decided and delivered, first step" under "Subprocess runners"). It lists the host facts that `rocm-core` reads twice (on the hosts each is read on, see "Which host reads which fact"), says how the two reads differ, and proposes which one should survive and in what order a later series of changes could remove the other. Each pair is meant to be a separate change with its own review. This note is a snapshot of `main` at the time of writing, not maintained documentation. It is to be retired (the file deleted, together with its entry in [architecture.md](architecture.md)) when the convergence tickets for these pairs close, so the inventory does not keep pointing at a stale design record. @@ -23,7 +23,7 @@ For each pair, the proposed survivor and the biggest difference a user can see t | [Windows GPU query](#windows-gpu-query) | the summary stack's inventory, with `probe_gpus_windows` building `gpus[]` from it and NVIDIA kept in the JSON | an NVIDIA-only host reads "AMD GPU not detected yet" in the text beside `has_nvidia_gpu: true` in the JSON | | [Marketing name to gfx target](#marketing-name-to-gfx-target) | the `AMD_MARKETING_GFX_TARGETS` table, gaining the codenames `classify_amd_marketing_name` knows (finish [#521](https://github.com/ROCm/rocm-cli/pull/521)) | on Windows without a HIP SDK an RX 7900 XTX has an empty `gpus[].gfx_target` while `summary.detected_gfx_target` is `gfx1100` | | [PATH lookup](#path-lookup) | delivered: the behaviour of `tool_on_path`, housed in `probe_exec.rs` under the name `which`; an empty `PATH` entry is skipped | before the change, an empty `PATH` entry or a `PATH` that is not UTF-8 changed which tools read as present (for example `wsl.rocminfo`); now there is one answer, and the differences it moved are listed under "PATH lookup" | -| [Subprocess runners](#subprocess-runners) | `run_with_timeout` as the one bounded executor, with `CommandCapture` extended by a typed outcome | a probe behind a hung grandchild returns at its deadline, and a successful capture can lose output a descendant writes more than 200 ms after the child exits | +| [Subprocess runners](#subprocess-runners) | first step delivered: `run_with_timeout` as the one bounded executor (it shares its spawn and wait with `run_bounded`, which returns a `CommandCapture`), with `CommandCapture` extended by raw bytes and a typed outcome; `run`, `capture_optional_command*` and the Windows diagnostic capture are adapters over it | before the change, a probe behind a hung grandchild outlasted its deadline (or, for a child that had exited, waited for the grandchild); now it returns at its deadline, and a successful capture can lose output a descendant writes more than 200 ms after the child exits | ## The two stacks @@ -236,20 +236,20 @@ No `which` call site passes a name with an extension other than `wsl.exe` (found ## Subprocess runners -Verdict: confirmed duplicate, and a family rather than a pair. +Verdict: confirmed duplicate, and a family rather than a pair. The first step is delivered (see "Decided and delivered, first step" below). The comparison that follows is how the helpers differed before it, kept as the record of what it moved, and so are the paragraphs after it up to the proposal. | Helper | Streams | Result | Default timeout | Reader threads | |---|---|---|---|---| | `capture_optional_command_with_timeout` ([capture.rs](../crates/rocm-core/src/host_gpu/capture.rs)) | stdout only; stderr discarded | `Option`: only on success and valid UTF-8 | 1.5 s, 5 s for inventory queries | joined with no limit; tries each candidate spelling in turn after a spawn failure, a non-zero exit or a timeout | | `run`, `run_raw` ([probe_exec.rs](../crates/rocm-core/src/probe_exec.rs)) | both | `(rc, stdout, stderr)`, `127` for spawn failure, `124` for timeout, lossy UTF-8 | 5 s, 8 s, 15 s | joined with no limit | | `capture_diagnostic_command` ([diagnostics.rs](../crates/rocm-core/src/host_gpu/diagnostics.rs), Windows only) | both | status, text, error and a timed-out flag | per call | joined with no limit | -| `run_with_timeout` ([process.rs](../crates/rocm-core/src/process.rs)) | both | `Result`; a timeout is an `Err` that quotes bounded output | per call | abandoned after a short grace | +| `run_with_timeout` ([bounded.rs](../crates/rocm-core/src/process/bounded.rs), re-exported by [process.rs](../crates/rocm-core/src/process.rs)) | both | `Result`; a timeout is an `Err` that quotes bounded output | per call | abandoned after a short grace | -The first three join their reader threads with no limit, so a grandchild that keeps the pipe open can outlast their timeout; [process.rs](../crates/rocm-core/src/process.rs) documents why `run_with_timeout` does not. That defect is already tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623), which lists the same helpers. Separately, the candidate loop in `capture_optional_command_with_timeout` keeps going after any failure of a candidate, a timeout included, so on Windows a hung `powershell` may be started again under several spellings (UNCONFIRMED; the loop does not stop on a timeout). `capture_diagnostic_command` moves to the next candidate only when a spawn fails. +The first three join their reader threads with no limit, so a grandchild that keeps the pipe open can outlast their timeout; [bounded.rs](../crates/rocm-core/src/process/bounded.rs) documents why `run_with_timeout` does not. That defect is already tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623), which lists the same helpers. Separately, the candidate loop in `capture_optional_command_with_timeout` keeps going after any failure of a candidate, a timeout included, so on Windows a hung `powershell` may be started again under several spellings (UNCONFIRMED; the loop does not stop on a timeout). `capture_diagnostic_command` moves to the next candidate only when a spawn fails. The helpers differ on a successful run too, not only on a timeout. Each of these is derived from the code and not run: -- Late descendant output. After the child exits, `run_with_timeout` (`run_captured` in [process.rs](../crates/rocm-core/src/process.rs)) gives the readers `DRAIN_GRACE` (200 ms) and then takes the buffer and abandons them, so a descendant that writes later than that after the child exited successfully contributes nothing. `run_raw`, `capture_optional_command_candidate_with_timeout` and `capture_diagnostic_command` join the readers and so wait for EOF, keeping whatever the descendant writes however late, and waiting without a bound if it never closes the pipe. Output the child itself wrote before exiting is kept by both. The existing tests `output_arriving_after_the_child_exits_is_still_collected` and `a_grandchild_holding_the_pipe_does_not_hold_a_successful_call` in [process.rs](../crates/rocm-core/src/process.rs) cover `run_with_timeout` only, and the first uses a 0.1 s delay inside the grace, so neither shows the difference. +- Late descendant output. After the child exits, `run_with_timeout` (`run_captured` in [bounded.rs](../crates/rocm-core/src/process/bounded.rs)) gives the readers `DRAIN_GRACE` (200 ms) and then takes the buffer and abandons them, so a descendant that writes later than that after the child exited successfully contributes nothing. `run_raw`, `capture_optional_command_candidate_with_timeout` and `capture_diagnostic_command` join the readers and so wait for EOF, keeping whatever the descendant writes however late, and waiting without a bound if it never closes the pipe. Output the child itself wrote before exiting is kept by both. The existing tests `output_arriving_after_the_child_exits_is_still_collected` and `a_grandchild_holding_the_pipe_does_not_hold_a_successful_call` in [bounded.rs](../crates/rocm-core/src/process/bounded.rs) cover `run_with_timeout` only, and the first uses a 0.1 s delay inside the grace, so neither shows the difference. - A read error on a pipe. The three older helpers discard it (`let _ = read_to_end`) and keep the bytes read so far with the real exit status; `run_with_timeout` turns a read failure on a run that did not time out into an error and drops the bytes. - A failure to wait. `run_raw` treats a `try_wait` error as `124` without killing the child and then joins the readers; `capture_optional_command_candidate_with_timeout` and `run_with_timeout` kill it first. - Spawn failure and encoding. `run` reports `127` with empty output where `run_with_timeout` returns an `Err` (a spawn failure there is an `std::io::Error` under a context message); `capture_optional_command*` returns `None` for output that is not UTF-8 where `run` keeps it lossily, and discards stderr where the others capture it, so after the change a descendant that holds only stderr open costs it up to the 200 ms grace, where it cost nothing before. @@ -259,6 +259,32 @@ The capture type from the earlier `rocm` capture work is on `main`: `CommandCapt Proposal: `run_with_timeout` survives as the one bounded executor, with `CommandCapture` as the shared result carrier extended with a typed outcome (not found, timed out, exited with a code) rather than a second capture type. `run` and `capture_optional_command*` become thin adapters that keep their current return shapes, so the first change touches no call site. That is not the same as keeping every observable result: the timeout path and the successful-run differences listed above are where they differ (see the last item of the order below). Keeping the strict-UTF-8 and lossy decodings, and the partial output after a timeout that `run` returns, needs the raw bytes from the executor, which is a second reason the carrier has to hold bytes rather than lossy text. The adapters can then be retired site by site as the pairs above remove call sites. This pair overlaps [#623](https://github.com/ROCm/rocm-cli/issues/623); extend that issue's scope with the typed outcome rather than opening a parallel one. +### Decided and delivered, first step + +The first step took the proposal, touching no call site of the helpers. `run_with_timeout` (in [bounded.rs](../crates/rocm-core/src/process/bounded.rs), re-exported from [process.rs](../crates/rocm-core/src/process.rs)) is still the one bounded executor, and its core is now also reachable as `run_bounded`, which returns the run as a `CommandCapture` instead of turning a timeout or a failure into an `Err`. `CommandCapture` in [command_capture.rs](../crates/rocm-core/src/command_capture.rs) is the shared carrier, extended rather than duplicated: it holds the raw bytes of both streams (the text fields are their lossy form), a typed `CommandOutcome` (not found, spawn failed with an error kind, timed out, wait failed, exited with a code, killed by a signal), the `ExitStatus` of a reaped child and the error text the executor saw. Its legacy `exit_status` keeps reading `1` where there is no code, so the typed outcome is how a caller tells a signal from an exit with `1`. `run_raw` (under `run`, `run_with_env` and `run_utf16le`) and `capture_optional_command_candidate_with_timeout` are thin adapters that keep their return shapes, and the Windows-only `capture_diagnostic_command` runs on the executor too: its candidate loop and the translation of each outcome are in `capture_diagnostic_with`, which takes the runner as an argument, so its unit tests run on Linux and on the Windows job. The change touches no call site of those helpers; the hand-built captures in `rocm` and `rocmd` (the chat bridge, the watchers, the sandbox tests) moved to `CommandCapture::from_text` because the struct gained fields, and nothing else there changed. The typed outcome belongs on `CommandCapture`, not beside it, which settles that item of the open list. + +The wait loop also changed, so that taking the executor on does not slow the probes that used to poll every 20 to 25 ms: it sleeps 5 ms and doubles up to the 50 ms it used to sleep flat, and the drain-grace wait does the same. + +Nothing here is output-preserving without qualification. The differences that moved, each with the test that pins it (the unit cases are in [capture.rs](../crates/rocm-core/src/host_gpu/capture.rs) module `subprocess_runners_pair`, which runs the same fake program through `run`, `capture_optional_command_with_timeout` and `run_with_timeout`, unless another file is named). The adapters keep their old answer where the last column says so: + +| Difference | Before | Now | Adapter keeps the old answer | Pinned by | +|---|---|---|---|---| +| Output a descendant writes more than `DRAIN_GRACE` (200 ms) after the child exited 0 | `run` and the optional capture: kept (they join their readers and wait for EOF); `run_with_timeout`: lost | lost by all three; output written within the grace is still kept by all three. The capture's `detail` says output was still arriving when the grace ended; the optional capture writes it to its opt-in debug channel (`ROCM_CLI_DEBUG_COMMAND_CAPTURE`) as a line saying its output may be incomplete, and `run` and `run_with_timeout` do not surface it today | no | `output_written_more_than_the_drain_grace_after_the_child_exits_is_lost_to_all_three`, with the control `output_written_within_the_drain_grace_after_the_child_exits_is_kept_by_all_three`; the `detail` is `output_written_after_the_grace_is_missing_and_the_capture_says_so` and `a_capture_says_when_a_reader_was_still_going_at_the_end_of_the_grace` (bounded.rs), and the debug line is `a_short_answer_from_the_optional_capture_comes_with_a_debug_line_and_a_whole_one_does_not`, `a_failed_run_gets_its_failure_line_and_no_incomplete_line` and `only_a_run_that_exited_0_with_something_cutting_its_output_short_is_called_incomplete` (host_gpu/capture.rs) | +| A descendant holds the pipe open after the child exited 0 | `run` and the optional capture: the call lasts as long as the descendant; `run_with_timeout`: returns after the grace | all three return after the grace, with what the child wrote | no | `a_descendant_holding_the_pipe_no_longer_extends_run_or_the_optional_capture`, scenario `examine-48` | +| A descendant holds the pipe open past a timeout ([#623](https://github.com/ROCm/rocm-cli/issues/623)) | `run` and the optional capture: the reader joins wait for the descendant, so the timeout bounds nothing; `run_with_timeout`: returns at the deadline | all three return at the deadline plus the grace; `run` still returns `124` with the partial output that arrived by then, and the optional capture still returns `None` | the result shape yes, the content of the partial output no (what arrived by the deadline and the grace, not everything until EOF) | `a_descendant_holding_the_pipe_no_longer_outlasts_the_timeout_of_run_or_the_optional_capture`; the plain timeout with no descendant is `a_timeout_keeps_partial_output_for_run_and_quotes_it_for_the_bounded_executor`, unchanged | +| A descendant holds only stderr | the optional capture sends stderr to the null device, so it was not held at all; `run` waited for the descendant | the optional capture pipes stderr like the others and the descendant costs it up to the grace; `run` returns after the grace without the descendant's late stderr | the answer yes, the promptness of the optional capture by at most the grace | `a_descendant_holding_only_stderr_costs_the_optional_capture_at_most_the_drain_grace` (bounds the call by the descendant's life; the extra 200 ms is too small for a timing assertion) | +| A pipe that fails to read | the older helpers discard the error (`let _ = read_to_end`) and keep the bytes read with the real status; `run_with_timeout` turns it into an error and drops the bytes | unchanged for `run_with_timeout`; the capture keeps the bytes and the real outcome and puts the error in `detail`, and both adapters ignore it as before, except that the optional capture also writes the error to its opt-in debug channel when the run exited 0 | yes | `a_read_failure_on_either_stream_keeps_the_bytes_and_the_outcome_the_child_had` (bounded.rs, a failure on stdout, on stderr and on both), `a_pipe_that_failed_to_read_does_not_change_what_run_returns` (probe_exec.rs), `a_pipe_that_failed_to_read_does_not_change_the_answer` (host_gpu/capture.rs), and for `run_with_timeout` `a_read_failure_without_a_timeout_is_reported_rather_than_returned_as_output` (bounded.rs). The older helpers' own behaviour cannot be provoked with a real child (it needs the kernel to fail a pipe read), so that column of the table is derived from the code, not run | +| A failure to wait | `run_raw`: `124` without killing the child, then an unbounded join; the optional capture: kills, `None`; the diagnostic capture: kills, `failed to wait: ` | all of them kill the child. `run`: `124` with the bytes that had arrived (not joined to EOF); the optional capture: `None`; the diagnostic capture: `failed to wait: ` followed by the executor's own wording (`failed to poll process: ` or `failed to reap timed-out process: `) | the return shapes yes; `run` no longer leaves the child running, and the diagnostic text gains the executor's prefix | `a_failure_to_wait_is_a_capture_that_keeps_the_bytes_that_had_arrived` (both the poll and the reap failure, bounded.rs), `a_failure_to_wait_is_final_and_reports_no_output` (diagnostics.rs), `a_run_reads_as_the_rc_run_raw_has_always_returned` (probe_exec.rs), and the existing `a_failure_to_poll_the_child_still_abandons_both_readers` and `a_failure_to_reap_a_timed_out_child_still_abandons_both_readers` | +| A program that cannot be started | `run`: `127` with empty output; the optional capture: `None` and a debug line with the OS error; the diagnostic capture: next candidate; `run_with_timeout`: an error | the same for all of them; the executor now also says whether the program was missing (`NotFound`) or could not be executed (`SpawnFailed` with the kind) | yes | `a_program_that_cannot_be_started_is_127_empty_none_or_an_error` and `a_spawn_failure_reports_the_error_the_os_gave` (host_gpu/capture.rs), `a_program_that_never_starts_reports_the_last_launch_error` (host_gpu/diagnostics.rs), and for the typed outcome `a_program_that_cannot_be_started_is_a_capture_not_an_error`, `a_program_that_exists_but_cannot_be_executed_is_a_spawn_failure_not_not_found` (bounded.rs) and `only_a_missing_program_is_not_found` (command_capture.rs) | +| Decoding | `run` lossy UTF-8, the optional capture strict UTF-8 (`None` for a stray byte), `run_utf16le` from the raw bytes | the same: the carrier holds the bytes and each adapter decodes as before | yes | `output_that_is_not_utf8_is_lossy_for_run_none_for_the_optional_capture_and_raw_for_the_executor`, `utf16le_output_is_decoded_by_declaration_from_the_raw_bytes`, `a_program_that_exited_0_with_output_that_is_not_utf8_gives_no_answer_and_no_debug_line` | +| A child killed by a signal | `run`: `-1`; the optional capture: `None`; `capture_program`'s `exit_status`: `1` | the same, and the capture's `outcome` now says `Signaled` with the signal number on Unix | yes | `a_program_killed_by_a_signal_has_no_exit_code`, `a_child_killed_by_a_signal_reads_as_exit_status_1` (command_capture.rs), `a_run_is_a_capture_with_its_exit_code_or_its_signal_and_both_streams` (bounded.rs), `an_exit_code_outranks_a_signal_and_no_code_is_a_signal` and `a_status_is_an_exit_code_or_a_signal` (command_capture.rs, every host) | +| Poll interval | 20 ms in `run_raw`, 25 ms in the two capture loops, 50 ms flat in `run_with_timeout` | 5 ms doubling to 50 ms in every one of them, so a child that exits at once is noticed within a few milliseconds, one that runs long up to 50 ms after it exits, and a deadline is overshot by up to 50 ms | not the timing | `the_poll_interval_backs_off_from_a_few_milliseconds_to_the_flat_one` pins the schedule, and `the_wait_loop_sleeps_the_back_off_schedule_between_polls` and `the_drain_wait_sleeps_the_back_off_schedule_until_both_readers_finish` (bounded.rs) record the sleeps each loop really takes; the latency itself is not asserted, because a timing assertion on a few milliseconds is flaky | +| The diagnostic capture on a timeout | joins the readers after the kill, so unbounded | returns the output that arrived and `timed_out`, bounded | the report yes | `a_timeout_is_final_keeps_the_output_that_arrived_and_says_so`; the candidate rule, next spelling only when a program could not be started, is `the_next_spelling_is_tried_only_when_a_program_could_not_be_started`, and the status text is the platform's `ExitStatus` as before (`exit code: 3` on Windows) | + +Not changed, and still open below: the candidate loop of `capture_optional_command_with_timeout` still starts the next spelling after a timeout, so a hung `powershell` can still be started under several spellings; `wait_with_output_bounded` in `apps/rocm` still joins its readers with no limit; and the runners still start a bare program name through the operating system, which resolves an empty `PATH` entry as the current directory where `which` does not. + +The scenario `examine-48` in [examine.feature](../tests/e2e-cucumber/features/examine.feature) drives the second row through `rocm examine --json` on the Linux lane: a stand-in `lspci` on a simulated native Linux root prints one AMD GPU, exits 0 and leaves a `sleep` holding its output for 30 seconds; the inspection must finish in under 15 seconds and still list the GPU. That shows the output the child wrote before it exited is still collected; it does not pin the length of the grace, which the unit cases do. The loss of late output (the first row) has no scenario: it needs a probe tool that writes more than 200 ms after it exits, which no tool the CLI runs is known to do, so it is pinned by the unit cases above only. The Windows paths cannot run on the Linux lane: the candidate loop and the translation of each outcome run through `capture_diagnostic_with` on the Windows job, and a `cmd` case (`a_run_is_a_capture_with_its_exit_code_and_both_streams`) runs the executor itself there. UNCONFIRMED: whether a Windows child can be arranged to leave a late-writing or pipe-holding descendant in the unit harness, so the grandchild and late-output rows are covered on Linux only (shell fixtures); on Windows the executor's bounds rest on the existing `a_timed_out_child_has_its_output_quoted_within_a_bound` and are not re-pinned per adapter. Nothing was run on a Windows host here; the Windows target was only type-checked (clippy) on Linux. + ## Proposed order One pair per change. Each begins with a commit that adds a characterisation test and is green against the code as it is: it pins what both copies return for the same input, and where they differ it pins each copy's answer, with a comment naming the pair. The convergence commit then flips the assertions that are meant to change and the PR text lists each one as a user-visible difference. Tests that need a program on `PATH` use the fake-executable helper in [fake_executable.rs](../crates/rocm-core/src/fake_executable.rs); Windows-only logic goes behind an injectable function so a unit test runs on the Windows job, as the inventory already does with `WindowsInventorySources`. @@ -268,7 +294,14 @@ One pair per change. Each begins with a commit that adds a characterisation test 3. CPU model (delivered: see "Decided and delivered" under "CPU model"). Adds the model-and-vendor reader; needs the Windows inventory row, which already exists. Its PR says which WSL option under "CPU model" it takes, (a) or (b); a PR that does not is not ready. It took (a). Not output-preserving: the differences listed under "CPU model" change a model's spelling, a `Hardware`-only board's `cpu_model`, and the time the Windows reads wait. 4. Marketing name to gfx target, by finishing [#521](https://github.com/ROCm/rocm-cli/pull/521). It must land before the next item, so that `gpus[]` is not rebuilt on top of two name tables. 5. Windows GPU query. The riskiest: it needs an answer on NVIDIA and on adapter names, and a Windows lane. Scenario `examine-windows-driver-version-date-and-age` already runs there; a new scenario should compare the text's `detected_gfx_target` with the JSON `gpus[]` on a named adapter. Native Windows only; no scenario can run this on the Linux or WSL lanes. -6. Subprocess runners. Last, because items 1 to 5 reduce the number of call sites, but independent: the bounded-join fix in [#623](https://github.com/ROCm/rocm-cli/issues/623) does not wait for it and can land at any time. It has two intended differences, and the second is easy to miss because it needs no timeout: a probe behind a hung grandchild returns at its deadline instead of late (with possibly different output kept after a timeout), and a successful capture can keep less, because a descendant that writes more than `DRAIN_GRACE` (200 ms) after the child exited loses that later output where the older helpers waited for EOF. Each of the other successful-run differences listed under "Subprocess runners" needs its own line in the PR text. The characterisation test for this item comes first and must include late descendant output: a child that prints `early`, exits 0 and leaves a descendant that prints `late` after about 0.5 s, run through each of `run` (over `run_raw`, which `run_utf16le` shares), `capture_optional_command*` and `run_with_timeout`, pinning that the first two return both lines and `run_with_timeout` only `early`. Hosts covered: Linux and WSL with shell fixtures (the same shape the `process.rs` tests already use for the grace); `capture_diagnostic_command` and the Windows spellings are Windows only, so on the Windows job they get the injectable-function test of the adapter's result translation instead. UNCONFIRMED whether a Windows child can be arranged to leave a late-writing descendant in the test harness; if not, the PR says that path is untested and why. +6. Subprocess runners. The first step is delivered (see "Decided and delivered, first step" under "Subprocess runners": the executor, the carrier and the three adapters, with every difference it moved listed there); the later steps stay open and are listed after this paragraph. What follows is the item as it was planned, kept as the record. Last, because items 1 to 5 reduce the number of call sites, but independent: the bounded-join fix in [#623](https://github.com/ROCm/rocm-cli/issues/623) does not wait for it and can land at any time. It has two intended differences, and the second is easy to miss because it needs no timeout: a probe behind a hung grandchild returns at its deadline instead of late (with possibly different output kept after a timeout), and a successful capture can keep less, because a descendant that writes more than `DRAIN_GRACE` (200 ms) after the child exited loses that later output where the older helpers waited for EOF. Each of the other successful-run differences listed under "Subprocess runners" needs its own line in the PR text. The characterisation test for this item comes first and must include late descendant output: a child that prints `early`, exits 0 and leaves a descendant that prints `late` after about 0.5 s, run through each of `run` (over `run_raw`, which `run_utf16le` shares), `capture_optional_command*` and `run_with_timeout`, pinning that the first two return both lines and `run_with_timeout` only `early`. Hosts covered: Linux and WSL with shell fixtures (the same shape the `bounded.rs` tests already use for the grace); `capture_diagnostic_command` and the Windows spellings are Windows only, so on the Windows job they get the injectable-function test of the adapter's result translation instead. UNCONFIRMED whether a Windows child can be arranged to leave a late-writing descendant in the test harness; if not, the PR says that path is untested and why. + +The later steps of item 6, each its own change and all still open: + +- Retire the adapters site by site, as items 2 to 5 remove call sites: a probe that calls `run_bounded` directly reads the typed outcome, so "not installed" stays quiet and "timed out" becomes a `probe_failures` entry without parsing a message, which the adapters cannot do because they keep their return shapes. +- Move `wait_with_output_bounded` in [process.rs](../apps/rocm/src/therock/process.rs) of `apps/rocm` onto the executor, which is the last helper of [#623](https://github.com/ROCm/rocm-cli/issues/623) that still joins its readers with no limit, and decide where `rocmd`'s `parse_sandbox_child_output` quote is bounded. +- Make the candidate loop of `capture_optional_command_with_timeout` stop on a timeout instead of starting the next spelling of a program that hung. +- Decide whether the runners refuse to resolve a bare program name through an empty `PATH` entry, and whether the adapters take a per-call drain grace: two items of the open list below. Items 2 and 3 should add a scenario (item 3 did: `examine-45` to `examine-47`) that compares `summary.kernel` with `kernel_release` and `summary.cpu` with `cpu_model` in one `--json` run, since no existing scenario does. Each comparison is scoped to the hosts where both sides are filled: kernel on native Linux and WSL, CPU on native Linux and Windows (option (a) under "CPU model"). The Linux lane can also drive a WSL variant with a simulated host root (`ROCM_CLI_TEST_HOST_ROOT` in an `e2e-test-hooks` build, with a `/dev/dxg`, a `/proc/version` that reads as WSL and a `/proc/cpuinfo`, as [hardware_root.rs](../crates/rocm-core/src/hardware_root.rs) describes), which pins the empty WSL CPU fields; `uname` still runs for real there, so a kernel value is compared rather than fixed. Assert the specific values, not only that both are non-empty. @@ -281,9 +314,9 @@ No item is output-preserving without qualification, and the PR for each says whi - What an absent Windows inventory does once there is one inventory: remember the absence for the run, or drop the per-fact fallbacks. - Whether `gpus[].name` should come from `pnputil` or from WMI. - Whether `kernel_release` should be filled on Windows (the kernel timeout is decided, see "Kernel version"; the CPU read took one 5 s limit, see "CPU model"). -- Whether the typed outcome belongs on `CommandCapture` or beside it. - Whether CPU facts should be collected on WSL at all (option (b) under "CPU model", which the CPU-model change did not take), and if so how the Windows-hosted distro probe obtains them. Scenario `examine-47` and `on_wsl_the_summary_reads_the_cpu_and_the_examination_does_not` pin the empty WSL fields until it is decided. - Whether the Windows reads should be given more or less time than 5 s once they can be measured on a host, and whether the examination should reuse the summary's inventory instead of running the query itself (the CPU read is still one query per stack). - Whether the adapters over `run_with_timeout` take a per-call drain grace, so that a probe whose tool is known to leave a late-writing descendant keeps its output, or whether the 200 ms grace is accepted for every probe. +- Whether the number of reader threads an abandoned capture leaves behind needs a bound. The executor bounds the memory (a capture that has been taken stops accumulating) but not the threads: each stream still held open by a descendant when the grace ends leaves one reader parked in `read` until the descendant lets go (two when it holds both), and `rocmd` probes every 60 seconds. A parked reader costs a stack and no CPU, and closing the read end instead would send the descendant `SIGPIPE`, so the first step accepts it; the reasoning is in the doc comment of `Capture` in [bounded.rs](../crates/rocm-core/src/process/bounded.rs). - Whether the near copy of `tool_path_candidates` in `apps/rocm` joins the PATH lookup change. - Whether the runners that replace the subprocess helpers should refuse to resolve a bare program name through an empty `PATH` entry, so that what the lookup calls absent is also what they do not start. diff --git a/docs/testing.md b/docs/testing.md index 099e834a28..95befb1724 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4108,6 +4108,12 @@ is what the verdict turns on. RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-3[5-8] ' ``` +`examine-48` plants a native Linux machine (a simulated host root that is not WSL, because the PCI probe is skipped on WSL) whose `lspci` is a stand-in on `PATH` that prints one AMD GPU, exits 0 and leaves a `sleep` holding its output pipes for 30 seconds. The host probes now stop reading a tool's output shortly after the tool exits, so `rocm examine --json` must finish in under 15 seconds, well inside the background process's life, and must still list the GPU the stand-in printed at `0000:fe:00.0`, so what the tool printed before it exited is still collected (the length of the grace is not pinned here). It runs on the Linux lanes and needs no GPU; the unit tests behind it (`subprocess_runners_pair` in `host_gpu/capture.rs`) pin the same bound for each of the three runners. + +```bash +RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-48 ' +``` + `serve-42` plants the same WSL machine with `LD_LIBRARY_PATH` empty and runs `rocm serve`: it is refused, its advice names the WSL `driver_status` and not `/dev/kfd`, and `rocm examine` on that machine reports `wsl_rocdxg_missing`, so the command the refusal names exists and shows the condition. The launch that follows when `LD_LIBRARY_PATH` does reach the library has no scenario, because it would start preparing a real engine; the serve gate's decision there is covered by unit tests in `gpu_visibility.rs`. ```bash diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 630b4c5ec4..9e0dc5af00 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -715,3 +715,20 @@ Feature: GPU detection and system inspection Given a WSL machine whose cpuinfo names an AMD CPU When the user inspects that machine in machine-readable form Then the summary reports that CPU model, and the examination reports no CPU model and an unknown vendor + + # The host probes' subprocess helpers used to read a tool's output until every + # holder of the pipe had closed it, so a tool that left a background process + # behind (a helper the tool starts and does not wait for) held the whole + # inspection until that process exited, however quickly the tool itself did. + # They now stop reading shortly after the tool exits, and the inspection keeps + # what the tool printed before that. The stand-in `lspci` prints one AMD GPU, + # exits 0 and leaves a `sleep` holding its output for 30 seconds; the + # inspection must finish well inside that and still list the GPU, so what the + # tool printed before it exited is still collected. It does not pin the length + # of the grace: that is the unit tests' (`subprocess_runners_pair`). + @id:examine-lspci-leaving-a-background-process-does-not-hold-up-the-inspection @requires-os:linux + Scenario: examine-48 - A PCI listing tool that leaves a background process behind does not hold up the inspection + Given a machine whose lspci prints a GPU and leaves a background process holding its output open + When the user inspects that machine in machine-readable form and the time it takes is measured + Then the inspection finishes long before the background process does + And the inspection lists the GPU that lspci printed diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index f433460695..0bd56572ce 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -96,10 +96,12 @@ pub struct E2eWorld { /// Wall-clock time the last measured `rocm` invocation took, for scenarios /// where the *duration* bounds the behaviour under test — /// `service-cleanup-07`, where a prune that returned while the managed-launch - /// lock was still held cannot have waited for it. It is a one-sided bound: - /// the measurement brackets the whole child process, so a long elapsed time - /// does not establish that the prune blocked. Set by the When step that - /// measures it; `None` everywhere else. + /// lock was still held cannot have waited for it, and `examine-48`, where an + /// inspection that finished long before a probe tool's background process + /// did cannot have waited for it. Both assert the short side of a one-sided + /// bound: the measurement brackets the whole child process, so a long elapsed + /// time does not establish that the command blocked. Set by the When step + /// that measures it; `None` everywhere else. pub cli_elapsed: Option, /// The `ROCM_CLI_DATA_DIR` value a refusal told the user to set, read from /// its stderr by the first step that follows that advice, so later steps diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 49fd18295a..a986bc22f7 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3027,6 +3027,100 @@ async fn assert_only_the_summary_reports_the_cpu_on_wsl(world: &mut E2eWorld) { ); } +// ── A probe tool that leaves a background process behind ─────────────────── + +/// How long the stand-in `lspci`'s background process keeps the pipes it +/// inherited open, which is how long a probe that waits for end of output waits. +const LEFTOVER_PROCESS_SECONDS: u64 = 30; + +/// An inspection that finishes within this did not wait for that process. Half +/// its life, so a loaded runner has the whole of the other half to be slow in. +const INSPECTION_BUDGET: std::time::Duration = std::time::Duration::from_secs(15); + +/// The PCI address and the line the stand-in `lspci` prints for its one GPU. The +/// bus number is one no real machine's first adapter has, so the line cannot be +/// mistaken for a card of the runner's own. +const LEFTOVER_LSPCI_ADDRESS: &str = "0000:fe:00.0"; +const LEFTOVER_LSPCI_LINE: &str = "0000:fe:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi 31 [Radeon RX 7900 XTX] [1002:744c] (rev c8)"; + +#[given( + "a machine whose lspci prints a GPU and leaves a background process holding its output open" +)] +async fn machine_whose_lspci_leaves_a_background_process(world: &mut E2eWorld) { + // A simulated native Linux root: the PCI probe is skipped on WSL, so a runner + // that is a WSL host would otherwise never reach the stand-in. + plant_os_release(world, "ID=ubuntu\nVERSION_ID=\"24.04\"\n"); + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("leftover-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the stand-in lspci's directory"); + // The `sleep` starts in the background of the script, so it inherits the + // script's stdout and stderr, the pipes `rocm` reads the listing from, and + // outlives the script, which exits 0 straight after printing. + rocm_core::fake_executable::write_fake_executable( + &bin.join("lspci"), + format!( + "#!/bin/sh\nprintf '%s\\n' '{LEFTOVER_LSPCI_LINE}'\nsleep {LEFTOVER_PROCESS_SECONDS} &\n" + ), + ) + .expect("failed to write the stand-in lspci"); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.retain(|(key, _)| *key != "PATH"); + world.command_env.push(("PATH", path)); +} + +#[when("the user inspects that machine in machine-readable form and the time it takes is measured")] +async fn user_times_an_inspection_of_the_planted_machine(world: &mut E2eWorld) { + let started = std::time::Instant::now(); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip", "--json"]); + world.cli_elapsed = Some(started.elapsed()); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the inspection finishes long before the background process does")] +async fn assert_the_inspection_did_not_wait_for_the_background_process(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let elapsed = world.cli_elapsed.expect("the inspection was not timed"); + assert!( + elapsed < INSPECTION_BUDGET, + "the inspection took {elapsed:?}, so it waited for the {LEFTOVER_PROCESS_SECONDS}s \ + background process that held `lspci`'s output open" + ); +} + +#[then("the inspection lists the GPU that lspci printed")] +async fn assert_the_gpu_lspci_printed_is_listed(world: &mut E2eWorld) { + let json = parsed_json(world); + let listed = json + .pointer("/gpus") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("`examine --json` has no `gpus` array:\n{json:#}")) + .iter() + .find(|gpu| { + gpu.pointer("/pci_id").and_then(serde_json::Value::as_str) + == Some(LEFTOVER_LSPCI_ADDRESS) + }) + .unwrap_or_else(|| { + panic!("the GPU `lspci` printed at {LEFTOVER_LSPCI_ADDRESS} is not listed:\n{json:#}") + }); + assert_eq!( + listed + .pointer("/is_amd") + .and_then(serde_json::Value::as_bool), + Some(true), + "{listed:#}" + ); +} + // ── `rocm serve` on a WSL host whose ROCDXG library the loader cannot find ── #[when("the user serves a model under the GPU-required default on that machine")]