diff --git a/Cargo.lock b/Cargo.lock index 7071a703eb..b856bfcdc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3905,6 +3905,8 @@ dependencies = [ name = "rocm-engine-protocol" version = "0.1.0" dependencies = [ + "anyhow", + "libc", "rocm-core", "serde", "serde_json", @@ -3930,6 +3932,7 @@ dependencies = [ "anyhow", "axum", "clap", + "libc", "rocm-core", "rocm-engine-protocol", "serde", diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index e48eb7cf8a..f4eb6b32ec 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -74,6 +74,8 @@ apple-native-keyring-store = { version = "1.0", features = ["keychain"] } zbus-secret-service-keyring-store = { version = "1.0", features = ["rt-tokio-crypto-rust"] } [dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } # Property-based tests for the byte-size formatter behind the `download: # approved up to …` line. "A size is printed in its own unit" has to hold at # every unit boundary, and the narrow band where `{:.1}` rounding reaches 1024.0 diff --git a/apps/rocm/src/chat/mcp_bridge.rs b/apps/rocm/src/chat/mcp_bridge.rs index 85af28b5aa..01328cf8d2 100644 --- a/apps/rocm/src/chat/mcp_bridge.rs +++ b/apps/rocm/src/chat/mcp_bridge.rs @@ -28,7 +28,7 @@ use crate::managed_service_status::{load_managed_services, managed_service_is_li use crate::rocm_argv::{chat_cli_arg_value, chat_cli_has_flag}; use crate::runtimes_cmd::render_runtimes_text; use crate::services_text::{ - read_tail_lines, render_logs_browser_text, render_logs_text, render_service_logs_text, + read_log_tail, render_logs_browser_text, render_logs_text, render_service_logs_text, render_services_text, }; use crate::tool_policy::{ @@ -213,16 +213,14 @@ pub(crate) fn run_internal_mcp_call( .into_iter() .find(|service| service.service_id == service_id) .with_context(|| format!("managed service `{service_id}` not found"))?; - let tail = read_tail_lines(&record.log_path, lines, "service log")?.join("\n"); + let log_tail = read_log_tail(&record.log_path, lines, "service log")?; Ok(internal_mcp_tool_success( - format!( - "Read the last {} line(s) from service `{}`.", - lines, record.service_id - ), + rocm_core::service_log_tool_summary(&record.service_id, &log_tail), serde_json::json!({ "service": record, "lines": lines, - "tail": tail, + "newest_line_exceeds_read_bound": log_tail.newest_line_exceeds_the_read_bound(), + "tail": log_tail.lines.join("\n"), }), )) } diff --git a/apps/rocm/src/comfyui.rs b/apps/rocm/src/comfyui.rs index bf3de3349a..99efe4f55c 100644 --- a/apps/rocm/src/comfyui.rs +++ b/apps/rocm/src/comfyui.rs @@ -519,13 +519,20 @@ fn render_logs_with_options( if show_file_locations { writeln!(output, " saved file: {}", path.display())?; } - let lines = read_tail_lines(path, limit) + let tail = read_log_tail(path, limit) .with_context(|| format!("failed to read {}", path.display()))?; - if lines.is_empty() { + if tail.newest_line_exceeds_the_read_bound() { + // The file has content; saying it is empty would be false. + writeln!( + output, + " No output shown: {}.", + rocm_core::newest_line_exceeds_bound_notice() + )?; + } else if tail.lines.is_empty() { writeln!(output, " The saved log is empty.")?; } else { writeln!(output, " latest output:")?; - for line in lines { + for line in tail.lines { writeln!(output, " {line}")?; } } @@ -668,66 +675,17 @@ fn comfyui_log_title(path: &Path) -> &'static str { } } -fn read_tail_lines(path: &Path, limit: usize) -> Result> { - let bytes = fs::read(path)?; - let text = strip_ansi_sequences(&decode_log_bytes(&bytes)); - let mut lines = text - .lines() - .rev() - .take(limit) - .map(str::to_owned) - .collect::>(); - lines.reverse(); - Ok(lines) -} - -fn decode_log_bytes(bytes: &[u8]) -> String { - if bytes.starts_with(&[0xff, 0xfe]) || looks_like_utf16_le(bytes) { - let start = if bytes.starts_with(&[0xff, 0xfe]) { - 2 - } else { - 0 - }; - let units = bytes[start..] - .chunks_exact(2) - .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) - .collect::>(); - return String::from_utf16_lossy(&units); - } - String::from_utf8_lossy(bytes).into_owned() -} - -fn looks_like_utf16_le(bytes: &[u8]) -> bool { - if bytes.len() < 8 { - return false; - } - let sample_len = bytes.len().min(512); - let zero_odd_bytes = bytes[..sample_len] - .iter() - .enumerate() - .filter(|(index, byte)| index % 2 == 1 && **byte == 0) - .count(); - zero_odd_bytes * 4 >= sample_len -} - -fn strip_ansi_sequences(text: &str) -> String { - let mut cleaned = String::with_capacity(text.len()); - let mut chars = text.chars().peekable(); - while let Some(ch) = chars.next() { - if ch != '\x1b' { - cleaned.push(ch); - continue; - } - if chars.peek() == Some(&'[') { - let _ = chars.next(); - for next in chars.by_ref() { - if ('@'..='~').contains(&next) { - break; - } - } - } - } - cleaned +/// The last `limit` lines of a saved log, read through the shared bounded tail +/// routine. A ComfyUI log may have been redirected by a Windows shell as +/// UTF-16 and carries the colour codes of the Python process, so it is +/// decoded the console way: UTF-16 detected, ANSI sequences stripped. +fn read_log_tail(path: &Path, limit: usize) -> Result { + rocm_core::tail_log_decoded( + path, + limit, + rocm_core::LOG_VIEW_MAX_TAIL_READ, + rocm_core::LogDecode::CONSOLE, + ) } fn load_manifest(paths: &AppPaths) -> Result> { @@ -1162,6 +1120,222 @@ mod tests { Ok(()) } + /// The lines of [`read_log_tail`], for the tests that compare lines. + fn read_tail_lines(path: &Path, limit: usize) -> Result> { + Ok(read_log_tail(path, limit)?.lines) + } + + fn tail_fixture(name: &str, bytes: &[u8]) -> Result { + let paths = test_paths(name); + let dir = app_root(&paths).join("logs"); + fs::create_dir_all(&dir)?; + let path = dir.join("start-fixture.log"); + fs::write(&path, bytes)?; + Ok(path) + } + + fn utf16_le(text: &str, bom: bool) -> Vec { + let mut bytes = if bom { vec![0xff, 0xfe] } else { Vec::new() }; + for unit in text.encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + bytes + } + + /// 5 MiB: a marked first line, 5120 lines of 1 KiB, a marked last line. + fn large_log_with_head_marker() -> Vec { + let mut bytes = b"HEAD-MARKER\n".to_vec(); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..5120 { + bytes.extend_from_slice(filler.as_bytes()); + } + bytes.extend_from_slice(b"TAIL-MARKER\n"); + bytes + } + + /// What `render_logs` prints for a lone saved log with these bytes, and + /// the log's size on disk, so a message is asserted with the state it + /// describes. + fn render_lone_log(name: &str, bytes: &[u8]) -> Result<(String, u64)> { + let paths = test_paths(name); + let logs = app_root(&paths).join("logs"); + fs::create_dir_all(&logs)?; + let log = logs.join("start-fixture.log"); + fs::write(&log, bytes)?; + let size = fs::metadata(&log)?.len(); + Ok((render_logs(&paths, 5)?, size)) + } + + #[test] + fn logs_say_a_log_is_empty_only_when_it_is_and_name_the_bound_when_it_is_not() -> Result<()> { + let bound = rocm_core::LOG_VIEW_MAX_TAIL_READ; + assert_eq!( + bound, + 4 * 1024 * 1024, + "the notice below names this many MiB" + ); + let notice = "No output shown: the newest line is longer than the 4 MiB a log view reads."; + let over_the_bound = usize::try_from(bound)? + 1024; + + // An empty file: the log really is empty, and says so. + let (rendered, size) = render_lone_log("comfyui-logs-empty", b"")?; + assert_eq!(size, 0); + assert!(rendered.contains("The saved log is empty."), "{rendered}"); + assert!(!rendered.contains(notice), "{rendered}"); + + // One line longer than the bound, with and without a final newline, + // and as sixteen-bit text: the file has content, so it is not empty, + // and the notice names the bound. + let one_line = "x".repeat(over_the_bound).into_bytes(); + let mut with_newline = one_line.clone(); + with_newline.push(b'\n'); + let wide = utf16_le(&"y".repeat(over_the_bound / 2 + 1), true); + for (name, bytes) in [ + ("comfyui-logs-long-line", one_line), + ("comfyui-logs-long-line-newline", with_newline), + ("comfyui-logs-long-line-wide", wide), + ] { + let (rendered, size) = render_lone_log(name, &bytes)?; + assert!(size > bound, "{name}: the log is bigger than the bound"); + assert!(rendered.contains(notice), "{name}: {rendered}"); + assert!(!rendered.contains("is empty"), "{name}: {rendered}"); + assert!(!rendered.contains("latest output:"), "{name}: {rendered}"); + } + + // A big log whose newest lines are short is shown, not described. + let (rendered, size) = render_lone_log("comfyui-logs-big", &large_log_with_head_marker())?; + assert!(size > bound); + assert!(rendered.contains("TAIL-MARKER"), "{rendered}"); + assert!(!rendered.contains(notice), "{rendered}"); + Ok(()) + } + + #[test] + fn read_tail_lines_a_large_file_is_read_through_a_bounded_window() -> Result<()> { + // A line far outside the tail window is not returned however large the + // limit: the log is not read whole. + let log = tail_fixture("comfyui-tail-large", &large_log_with_head_marker())?; + + let lines = read_tail_lines(&log, 10_000)?; + fs::remove_file(&log)?; + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + assert!(lines.len() < 5122, "got {} lines", lines.len()); + assert!(lines.len() > 3500, "the window holds about 4 MiB of lines"); + Ok(()) + } + + #[test] + fn read_tail_lines_a_large_sixteen_bit_log_is_read_through_a_bounded_window() -> Result<()> { + // The window of a UTF-16 log must start on a character boundary or + // every following character decodes as garbage. + let mut text = String::from("HEAD-MARKER\r\n"); + let filler = format!("\u{1b}[33m{}\u{1b}[0m\r\n", "y".repeat(1000)); + for _ in 0..3000 { + text.push_str(&filler); + } + text.push_str("TAIL-MARKER\r\n"); + let log = tail_fixture("comfyui-tail-large-utf16", &utf16_le(&text, true))?; + assert!(fs::metadata(&log)?.len() > 6_000_000); + + let lines = read_tail_lines(&log, 10_000)?; + fs::remove_file(&log)?; + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + let whole = "y".repeat(1000); + for line in &lines[..lines.len() - 1] { + assert_eq!(line, &whole, "a partial leading fragment was kept"); + } + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_a_bomless_utf16_log_and_strips_ansi() -> Result<()> { + let text = "\u{1b}[32m[INFO]\u{1b}[0m one\r\n\u{1b}[1;31mtwo\u{1b}[0m\r\n"; + let log = tail_fixture("comfyui-tail-utf16-nobom", &utf16_le(text, false))?; + + let lines = read_tail_lines(&log, 5)?; + + assert_eq!(lines, vec!["[INFO] one", "two"]); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_ansi_from_an_eight_bit_log() -> Result<()> { + let log = tail_fixture("comfyui-tail-ansi-utf8", b"\x1b[32mgreen\x1b[0m done\n")?; + + assert_eq!(read_tail_lines(&log, 5)?, vec!["green done"]); + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_invalid_eight_bit_text_lossily() -> Result<()> { + let log = tail_fixture("comfyui-tail-invalid", b"ok\nbad\xffbyte\nlast\n")?; + + assert_eq!( + read_tail_lines(&log, 5)?, + vec!["ok", "bad\u{FFFD}byte", "last"] + ); + Ok(()) + } + + #[test] + fn read_tail_lines_missing_file_is_a_not_found_error_naming_the_path() { + let paths = test_paths("comfyui-tail-missing"); + let missing = app_root(&paths).join("logs").join("absent.log"); + + let error = read_tail_lines(&missing, 5).expect_err("a missing log must not read"); + + assert_eq!( + error.to_string(), + format!("failed to open {}", missing.display()) + ); + let io_error = error + .downcast_ref::() + .expect("the io error is kept as the cause"); + assert_eq!(io_error.kind(), io::ErrorKind::NotFound); + // A limit of 0 returns before the file is looked at. + assert!( + read_tail_lines(&missing, 0) + .expect("limit 0 never opens the file") + .is_empty() + ); + } + + #[test] + fn read_tail_lines_limit_zero_returns_nothing_for_an_existing_log() -> Result<()> { + let log = tail_fixture("comfyui-tail-zero", b"a\nb\n")?; + + assert!(read_tail_lines(&log, 0)?.is_empty()); + Ok(()) + } + + #[test] + fn read_tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let log = tail_fixture("comfyui-tail-few", b"a\nb\n")?; + + assert_eq!(read_tail_lines(&log, 50)?, vec!["a", "b"]); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let log = tail_fixture("comfyui-tail-no-newline", b"a\nb\nc")?; + + assert_eq!(read_tail_lines(&log, 2)?, vec!["b", "c"]); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_crlf_line_endings() -> Result<()> { + let log = tail_fixture("comfyui-tail-crlf", b"a\r\nb\r\n")?; + + assert_eq!(read_tail_lines(&log, 5)?, vec!["a", "b"]); + Ok(()) + } + #[test] fn unmanaged_running_url_reports_reachable_loopback_endpoint() -> Result<()> { let listener = TcpListener::bind("127.0.0.1:0")?; diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 3e9565d00e..db883adcf5 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -4,35 +4,34 @@ //! Engine plugin binary resolution and the engine IPC request/response //! envelope: locating an engine's executable (packaged plugin vs. the -//! platform-packaged fallback), encoding/decoding the JSON-over-stdio -//! envelope used to talk to it, and the built-in (in-process) fast path for -//! `lemonade`/`vllm` that skips spawning a child process entirely. +//! platform-packaged fallback), talking to it over JSON-over-stdio, and the +//! built-in (in-process) fast path for `lemonade`/`vllm`/`aim` that skips +//! spawning a child process entirely. +//! +//! The stdio conversation itself is not implemented here. The request framing, +//! the child's environment, stderr handling, the SIGPIPE guard around the write +//! and the reading of the answer are the client in +//! [`rocm_engine_protocol`]'s `stdio_client` module, which `rocmd` uses too. +//! This module chooses what to run (the built-in handler, or the resolved +//! engine binary with `stdio`) and what `rocm` asks of the client for it +//! ([`engine_stdio_child_request`]). //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-213) of the `main.rs` modularization (see `docs/architecture.md`). //! `dispatch()` calls directly into this module at two points — //! `Command::EngineStdio { engine } => run_builtin_engine_stdio(&engine)` and //! the `__engine-serve-http` arm's `run_builtin_engine_serve_http(...)` call — //! both re-imported via `use crate::engine_ipc::{run_builtin_engine_serve_http, -//! run_builtin_engine_stdio};`; `dispatch()`'s own body is otherwise -//! unaffected. +//! run_builtin_engine_stdio};`. //! //! This module owns: resolving an engine's binary path //! (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, -//! `find_engine_plugin_binary`), the stdio request/response envelope used to -//! talk to an external engine binary or the in-process built-in handler +//! `find_engine_plugin_binary`), the request entry points //! (`engine_request`, `engine_request_with_env_root`, -//! `decode_engine_response`, the `ScopedEnvVar`/ +//! `engine_stdio_child_request`), the `ScopedEnvVar`/ //! `with_scoped_builtin_engine_env` env-scoping helpers and the //! `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, -//! `run_builtin_engine_stdio`), and the built-in HTTP-serving path +//! `run_builtin_engine_stdio`, and the built-in HTTP-serving path //! (`builtin_engine_available`, `run_builtin_engine_serve_http`, -//! `builtin_engine_serve_http_args`). One item sits physically inside this -//! span in `main.rs` but stays there rather than moving: `impl -//! From for WatcherMode` is unrelated clap-arg-to-domain-type -//! conversion that only happens to sit between `find_engine_plugin_binary` and -//! `engine_request` in the original file (it has since moved to `cli.rs`, -//! next to the enum it converts). +//! `builtin_engine_serve_http_args`). //! //! `main.rs` calls into it for `resolve_engine_binary_path`, //! `resolve_engine_binary_path_with_paths`, `engine_request`, @@ -42,39 +41,32 @@ //! `runtime_install_finalize.rs` call into it for //! `engine_request_with_env_root`; `service_launch.rs` calls into it for //! `builtin_engine_serve_http_args`. `find_engine_plugin_binary`, -//! `decode_engine_response`, `with_scoped_builtin_engine_env`, -//! `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this -//! module and stay private. Shared helpers this module calls back into at the -//! crate root — `with_sigpipe_ignored` and `device_policy_name` — stay in -//! `main.rs` and are reached unqualified via `use crate::{...}`, same as for -//! other mechanically relocated modules. `parse_device_policy` moved out of -//! `main.rs` into `gpu_policy.rs` in ROCMAI-214 (Phase 6f); it is reached via -//! `use crate::gpu_policy::parse_device_policy;` instead. `apply_app_path_env`, -//! `app_path_env_vars`, `optional_arg`, and `engine_recipe_json_arg` moved out -//! of `main.rs` into `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f); this -//! file's `use` list below points there now instead of at the crate root. +//! `with_scoped_builtin_engine_env`, `builtin_engine_request`, and +//! `ScopedEnvVar` have no callers outside this module and stay private. The +//! one helper this module calls back into at the crate root, `device_policy_name`, +//! stays in `main.rs` and is reached via `use crate::device_policy_name;`. +//! `parse_device_policy` is in `gpu_policy.rs`; `app_path_env_vars` and +//! `engine_recipe_json_arg` are in `engine_runtime_select.rs`, and +//! `optional_arg` is in `rocm_engine_protocol`. use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; -use std::process::{Command as ProcessCommand, Stdio}; use std::sync::{Mutex, OnceLock}; -use std::thread; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, engine_binary_path, engine_plugin_dirs}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, engine_binary_path, engine_plugin_dirs}; use rocm_engine_protocol::{ DevicePolicy, EngineMethod, EnginePluginDescriptor, EngineRecipeHint, EngineRequestEnvelope, - EngineResponseEnvelope, + EngineResponseEnvelope, EngineStdioOptions, decode_engine_response, engine_request_envelope, + optional_arg, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use crate::engine_runtime_select::{ - app_path_env_vars, apply_app_path_env, engine_recipe_json_arg, optional_arg, -}; +use crate::device_policy_name; +use crate::engine_runtime_select::{app_path_env_vars, engine_recipe_json_arg}; use crate::gpu_policy::parse_device_policy; -use crate::{device_policy_name, with_sigpipe_ignored}; static BUILTIN_ENGINE_ENV_LOCK: OnceLock> = OnceLock::new(); @@ -130,11 +122,7 @@ where R: DeserializeOwned, { let stream_progress = matches!(&method, EngineMethod::Install); - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request) - .context("failed to encode engine request payload")?, - }; + let envelope = engine_request_envelope(method, request)?; if let Some(envelope) = with_scoped_builtin_engine_env(paths, env_root, || { builtin_engine_request(engine, &envelope) }) { @@ -146,118 +134,53 @@ where "unable to locate engine binary for {engine}; build the workspace or install the engine package" ) })?; - let mut command = ProcessCommand::new(engine_binary); - command.arg("stdio"); - if let Some(paths) = paths { - apply_app_path_env(&mut command, paths); - } - if let Some(env_root) = env_root { - command.env("ROCM_CLI_ENGINE_ENVS_ROOT", env_root); - } - if stream_progress { - command.env("ROCM_ENGINE_PROGRESS_STDERR", "1"); - } - let mut child = command - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn engine stdio process")?; - - { - let mut stdin = child - .stdin - .take() - .context("engine stdio child did not expose stdin")?; - // Ignore SIGPIPE for the duration of the write: with SIG_DFL in effect - // process-wide, an engine that exits before reading its stdin would - // otherwise kill `rocm` with SIGPIPE before the exit-status diagnostics - // below can run. Under SIG_IGN the write returns EPIPE instead, which we - // surface as the normal "failed to write engine request" error and then - // let `wait()` report why the engine exited. - with_sigpipe_ignored(|| { - serde_json::to_writer(&mut stdin, &envelope) - .context("failed to write engine request")?; - stdin.write_all(b"\n")?; - Ok::<(), anyhow::Error>(()) - })?; - } - - let stderr_handle = child.stderr.take().map(|stderr| { - thread::spawn(move || { - let mut reader = io::BufReader::new(stderr); - let mut collected = Vec::new(); - let mut line = Vec::new(); - loop { - line.clear(); - match reader.read_until(b'\n', &mut line) { - Ok(0) => break, - Ok(_) => { - if let Ok(text) = std::str::from_utf8(&line) { - eprint!("{text}"); - } else { - let _ = io::stderr().write_all(&line); - } - collected.extend_from_slice(&line); - } - Err(error) => { - let message = format!("failed to read engine progress: {error}\n"); - eprint!("{message}"); - collected.extend_from_slice(message.as_bytes()); - break; - } - } - } - collected - }) - }); - let mut stdout = Vec::new(); - child - .stdout - .take() - .context("engine stdio child did not expose stdout")? - .read_to_end(&mut stdout) - .context("failed to read engine stdio response")?; - let status = child - .wait() - .context("failed waiting for engine stdio response")?; - let stderr = stderr_handle - .map(|handle| handle.join().unwrap_or_else(|_| Vec::new())) - .unwrap_or_default(); - - if !status.success() && stdout.is_empty() { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {status}"); - } - bail!("engine stdio process exited with status {status}: {stderr}"); - } - let envelope: EngineResponseEnvelope = serde_json::from_slice(&stdout).with_context(|| { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - "failed to parse engine response envelope".to_owned() - } else { - format!("failed to parse engine response envelope; stderr: {stderr}") - } - })?; - decode_engine_response(envelope) + engine_stdio_child_request(&engine_binary, paths, &envelope, env_root, stream_progress) } -fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result +/// The out-of-process half of [`engine_request_with_env_root`]: run +/// `engine_binary stdio` through the client `rocm` shares with `rocmd`, send +/// `envelope` on its stdin and decode its answer. +/// +/// What `rocm` asks of that client: the child runs on this process's app paths +/// when it has them and on the engine environments root when the request names +/// one, an install asks for progress on stderr, and the child's stderr is +/// written to this process's as it arrives, which is how a terminal user +/// watches an install. No endpoint key is handed over, because none of the +/// requests `rocm` makes is about a running service's endpoint. Split out so a +/// test can point it at a stand-in binary. +fn engine_stdio_child_request( + engine_binary: &Path, + paths: Option<&AppPaths>, + envelope: &EngineRequestEnvelope, + env_root: Option<&Path>, + stream_progress: bool, +) -> Result where R: DeserializeOwned, { - if !envelope.ok { - let error = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |value| format!("{}: {}", value.code, value.message), - ); - bail!("{error}"); + let options = stdio_options(paths, env_root, stream_progress); + decode_engine_response(run_engine_stdio( + engine_binary, + ["stdio"], + &options, + envelope, + )?) +} + +/// What `rocm` asks of the shared stdio client; see +/// [`engine_stdio_child_request`]. +const fn stdio_options<'a>( + paths: Option<&'a AppPaths>, + env_root: Option<&'a Path>, + stream_progress: bool, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: paths, + env_root, + endpoint_key_file: None, + request_progress: stream_progress, + forward_stderr: true, } - let data = envelope - .data - .context("engine response envelope did not contain data")?; - serde_json::from_value(data).context("failed to decode engine response payload") } struct ScopedEnvVar { @@ -305,10 +228,7 @@ fn with_scoped_builtin_engine_env( } } if let Some(env_root) = env_root { - vars.push(ScopedEnvVar::set_path( - "ROCM_CLI_ENGINE_ENVS_ROOT", - env_root, - )); + vars.push(ScopedEnvVar::set_path(ENGINE_ENVS_ROOT_ENV, env_root)); } let result = f(); drop(vars); @@ -490,4 +410,232 @@ mod tests { assert_eq!(discovered, Some(primary_path)); Ok(()) } + + // ---- What `rocm` asks of the shared stdio client. ---- + // + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocm`'s entry point against a stand-in engine child and pin what + // `rocm` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocmd/src/engine_ipc.rs`. + + #[test] + fn rocm_asks_for_forwarded_stderr_and_no_endpoint_key() { + let (root, paths) = crate::tests::test_paths("engine-ipc-options"); + let env_root = root.join("envs"); + + let options = stdio_options(Some(&paths), Some(&env_root), true); + assert_eq!( + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) + ); + assert_eq!(options.env_root, Some(env_root.as_path())); + assert!(options.request_progress); + // The terminal user watches an install through the child's stderr. + assert!(options.forward_stderr); + // No request `rocm` makes is about a running service's endpoint. + assert_eq!(options.endpoint_key_file, None); + + let bare = stdio_options(None, None, false); + assert!(bare.app_paths.is_none() && bare.env_root.is_none()); + assert!(!bare.request_progress); + assert!(bare.forward_stderr); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn envelope(method: EngineMethod, payload: serde_json::Value) -> EngineRequestEnvelope { + EngineRequestEnvelope { method, payload } + } + + fn stop_request() -> EngineRequestEnvelope { + envelope(EngineMethod::Stop, json!({"service_id": "svc"})) + } + + fn request( + engine: &FakeEngine, + paths: Option<&AppPaths>, + env_root: Option<&Path>, + stream_progress: bool, + ) -> Result { + engine_stdio_child_request::( + engine.path(), + paths, + &stop_request(), + env_root, + stream_progress, + ) + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_gets_the_stdio_verb_and_one_line_of_request() { + let engine = FakeEngine::answering("rocm-framing", STOPPED, 0); + let answer = request(&engine, None, None, false).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn the_child_runs_on_the_callers_app_paths_when_given_them() { + let (root, paths) = crate::tests::test_paths("engine-ipc-app-paths"); + let engine = FakeEngine::answering("rocm-app-paths", STOPPED, 0); + request(&engine, Some(&paths), None, false).unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!( + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() + ); + } + + #[test] + fn the_child_inherits_the_callers_environment_when_no_app_paths_are_given() { + let engine = FakeEngine::answering("rocm-no-app-paths", STOPPED, 0); + request(&engine, None, None, false).unwrap(); + + for key in [ + "ROCM_CLI_CONFIG_DIR", + "ROCM_CLI_DATA_DIR", + "ROCM_CLI_CACHE_DIR", + ] { + assert_eq!(engine.env(key), ambient(key), "{key}"); + } + } + + #[test] + fn an_env_root_reaches_the_child_as_the_engine_envs_root() { + let engine = FakeEngine::answering("rocm-env-root", STOPPED, 0); + let env_root = Path::new("/an/engine/envs/root"); + request(&engine, None, Some(env_root), false).unwrap(); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/an/engine/envs/root") + ); + + let without = FakeEngine::answering("rocm-no-env-root", STOPPED, 0); + request(&without, None, None, false).unwrap(); + assert_eq!( + without.env("ROCM_CLI_ENGINE_ENVS_ROOT"), + ambient("ROCM_CLI_ENGINE_ENVS_ROOT") + ); + } + + #[test] + fn progress_streaming_is_asked_for_only_when_the_caller_wants_it() { + let streaming = FakeEngine::answering("rocm-progress-on", STOPPED, 0); + request(&streaming, None, None, true).unwrap(); + assert_eq!( + streaming.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + + let quiet = FakeEngine::answering("rocm-progress-off", STOPPED, 0); + request(&quiet, None, None, false).unwrap(); + assert_eq!( + quiet.env("ROCM_ENGINE_PROGRESS_STDERR"), + ambient("ROCM_ENGINE_PROGRESS_STDERR") + ); + } + + #[test] + fn the_endpoint_key_is_not_passed_to_the_child() { + // `rocm`'s requests are not about a running service's endpoint (detect, + // install, resolve-model, stop), so this client has no service to find + // a key for. `rocmd`'s healthcheck is the call that threads one. + let engine = FakeEngine::answering("rocm-no-key", STOPPED, 0); + request(&engine, None, None, false).unwrap(); + assert_eq!( + engine.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), + ambient(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV) + ); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + // The engine's own error envelope is the useful diagnostic, so an + // answer on stdout wins over the exit status. + let engine = FakeEngine::answering( + "rocm-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = request(&engine, None, None, false).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("rocm-answers-then-exits-2", STOPPED, 2); + assert_eq!( + request(&engine, None, None, false).unwrap(), + json!({"stopped": true}) + ); + } + + /// The write of a request to an engine that has already exited must come + /// back as an error even where SIGPIPE is at its default action, which is + /// how `main` runs `rocm`. Re-executes this test in a fresh process that + /// installs `SIG_DFL` first (the test harness leaves SIGPIPE ignored, and + /// changing it process-wide would race sibling tests). + #[test] + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_ENGINE_SIGPIPE_ARM").is_some() { + crate::reset_sigpipe(); + let engine = FakeEngine::new("rocm-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let result = engine_stdio_child_request::( + engine.path(), + None, + &envelope(EngineMethod::Stop, json!({ "blob": big })), + None, + false, + ); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_ENGINE_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } } diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 793a82f3ab..08994faf12 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -7,8 +7,8 @@ //! to (`EngineSelection`, `resolve_engine_selection`, //! `validate_engine_selection_runtime`, `single_ready_runtime_key`, //! `resolve_runtime_selector_to_exact_key`), small argv-building helpers used -//! when relaunching a managed engine (`optional_arg`, -//! `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, +//! when relaunching a managed engine (`engine_recipe_json_arg`, +//! `parse_engine_recipe_json_arg`, //! `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` //! environment variables every managed child inherits //! (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/ @@ -61,8 +61,7 @@ //! it does not decide which runtime counts as "the one ready runtime" itself, //! it asks `runtime_adopt::select_single_ready_runtime`, the rule //! `rocm comfyui install` shares. -//! `optional_arg` and -//! `engine_recipe_json_arg` are `pub(crate)`, read from `engine_ipc.rs`'s +//! `engine_recipe_json_arg` is `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read //! from `main.rs`'s `dispatch()` and `service_control.rs`'s `restart_child_argv`. //! `app_path_env_vars`/`apply_app_path_env` are `pub(crate)`, read from @@ -246,13 +245,6 @@ pub(crate) fn resolve_runtime_selector_to_exact_key( } } -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } -} - pub(crate) fn engine_recipe_json_arg( engine_recipe: Option<&EngineRecipeHint>, ) -> Result> { @@ -276,11 +268,7 @@ pub(crate) fn parse_engine_recipe_json_arg( } pub(crate) fn app_path_env_vars(paths: &AppPaths) -> [(&'static str, &Path); 3] { - [ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ] + paths.env_vars() } #[cfg_attr(not(windows), allow(dead_code))] diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index e879060b5c..b80184f394 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -99,13 +99,11 @@ mod update_cmd; // `update_cmd.rs`'s `apply_runtime_update`/`append_update_activate_summary` // call into `runtimes_cmd` directly too, via their own imports rather than // this re-export. `env_root_for_service`, `attach_background_stdio`, -// `detach_background_command`, `managed_engine_startup_failure_detail`, -// `managed_service_process_command`, and -// `ensure_public_service_has_endpoint_key` are no longer needed here either: -// their only caller outside `dispatch()`, `restart_internal_managed_service` -// (and the endpoint-key check inside it), moved out to `services_text.rs` in -// ROCMAI-209 (Phase 6f) and has since moved on to `service_control.rs`, which -// reaches them via its own imports instead. +// `detach_background_command`, `managed_engine_startup_failure_detail`, and +// `managed_service_process_command` are no longer needed here either: their +// only caller outside `dispatch()`, `restart_internal_managed_service`, moved +// out to `services_text.rs` in ROCMAI-209 (Phase 6f) and has since moved on to +// `service_control.rs`, which reaches them via its own imports instead. // `record_cli_audit_event`, `cli_lifecycle_log_path` and `sanitize_log_value` // (from `cli_audit.rs`) are re-imported so other modules keep reaching them as // `crate::`. @@ -190,10 +188,6 @@ use std::fs; use std::io::{self, Write}; use std::path::PathBuf; use std::process::ExitCode; -#[cfg(all(test, not(windows)))] -use std::process::Stdio; -#[cfg(all(test, not(windows)))] -use std::process::{Command as ProcessCommand, ExitStatus}; #[cfg(test)] use std::sync::Mutex; @@ -222,33 +216,6 @@ fn reset_sigpipe() { #[allow(clippy::missing_const_for_fn)] fn reset_sigpipe() {} -/// Run `f` with SIGPIPE temporarily ignored, restoring the previous -/// disposition afterwards. -/// -/// [`reset_sigpipe`] sets `SIG_DFL` process-wide so stdout/stderr pipe writes -/// terminate the process conventionally. But a write to a spawned child's stdin -/// whose reader has already exited would then kill `rocm` with SIGPIPE before -/// the caller can turn the resulting `EPIPE` into a diagnostic. Wrap such writes -/// in this helper so the failure surfaces as an `io::Error` (`BrokenPipe`) -/// instead of a signal. -#[cfg(unix)] -#[allow(unsafe_code)] // libc FFI -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - // SAFETY: reading/installing a signal disposition is always safe. - let previous = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; - let result = f(); - // SAFETY: restoring the disposition captured above is always safe. - unsafe { - libc::signal(libc::SIGPIPE, previous); - } - result -} - -#[cfg(not(unix))] -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - f() -} - /// Marker error carrying `rocm fix`'s exit code back through `main()`'s /// ordinary return path, instead of calling `std::process::exit` mid-stack /// and skipping the `_log_guard` destructor held in `run()`. @@ -1448,89 +1415,6 @@ pub(crate) mod tests { } } - /// Regression test for the engine child-stdin write under the process-wide - /// `SIG_DFL` that `main` installs via [`reset_sigpipe`]. If an engine child - /// exits before reading its stdin, the parent's write to that pipe must - /// surface as a `BrokenPipe` error — handled by the caller's diagnostics — - /// rather than killing `rocm` with SIGPIPE before those diagnostics run. - /// - /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE - /// at Rust's default `SIG_IGN`, and flipping it process-wide would race - /// sibling tests. So this re-execs itself in fresh processes that install - /// `SIG_DFL` first, and checks BOTH directions: - /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — - /// this is the regression the plain `SIG_DFL` reset introduced; - /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns - /// an error instead — this is the fix. - /// - /// Each child reaps its short-lived grandchild before writing, so the read - /// end is closed deterministically and the outcome does not race. - #[cfg(unix)] - #[test] - fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { - use std::os::unix::process::ExitStatusExt as _; - - // Grandchild helper: reproduce main()'s SIG_DFL, spawn a process that - // exits immediately, reap it, then write to its now-closed stdin. The - // "guarded" arm wraps the write; the "unguarded" arm does not. - if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { - reset_sigpipe(); - let mut grandchild = ProcessCommand::new("true") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn a process that exits immediately"); - let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); - grandchild.wait().expect("reap grandchild"); - // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; - // it must exceed the pipe buffer so the write actually reaches the - // closed read end rather than being swallowed by kernel buffering. - let payload = vec![b'x'; 64 * 1024]; - if mode == "guarded" { - let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); - assert!( - result.is_err(), - "guarded write to a dead pipe should return BrokenPipe, not succeed" - ); - } else { - // Unguarded: under SIG_DFL this write delivers SIGPIPE and never - // returns. If the process somehow survives, exit 0 so the parent's - // "should have been signalled" assertion fails loudly. - let _ = stdin.write_all(&payload); - } - return; - } - - let exe = std::env::current_exe().expect("current test executable"); - let run_arm = |mode: &str| -> ExitStatus { - ProcessCommand::new(&exe) - .args([ - "tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", - "--exact", - "--nocapture", - ]) - .env("ROCM_TEST_SIGPIPE_ARM", mode) - .status() - .expect("re-exec the test arm in a child process") - }; - - let unguarded = run_arm("unguarded"); - assert_eq!( - unguarded.signal(), - Some(libc::SIGPIPE), - "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ - SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" - ); - - let guarded = run_arm("guarded"); - assert!( - guarded.success(), - "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ - SIGPIPE reached the process instead of surfacing as an error" - ); - } - #[test] fn completions_generate_for_every_shell() { use clap_complete::Shell; diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 2f501506fc..0fb537a75a 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -465,12 +465,12 @@ mod tests { fn dash_loopback_hosts_match_the_clis_non_public_hosts() { use rocm_dash_tui::ui::serve_wizard::{LOOPBACK_HOSTS, is_loopback_host}; for host in LOOPBACK_HOSTS { - assert!(!rocm_engine_protocol::is_public_bind_host(host), "{host}"); + assert!(!rocm_core::is_public_bind_host(host), "{host}"); } for host in ["LOCALHOST", "[::1]", "127.0.0.2", "0.0.0.0", "::"] { assert_eq!( is_loopback_host(host), - !rocm_engine_protocol::is_public_bind_host(host), + !rocm_core::is_public_bind_host(host), "{host}" ); } diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 2f4db4617e..007edf94b1 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -8,9 +8,9 @@ //! Mechanically relocated from `services_text.rs` with no behavior change. //! //! This module owns: the sandbox-tool runner (`run_internal_sandbox_tool`), the -//! stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` -//! constant, the private `RecordedPidOutcome`/`RecordedStopTermination` types -//! and the `scripted_stop_outcome` end-to-end hook, `stop_internal_managed_service`, +//! stop path (`stop_recorded_processes`, a thin wrapper over the termination +//! routine `rocmd` shares (`rocm-core`'s `proc_lifecycle.rs`) that adds the +//! `scripted_stop_outcome` end-to-end hook, `stop_internal_managed_service`, //! `stop_internal_managed_service_with`, //! `stop_internal_managed_service_confirmed_with`, //! `unload_lemonade_service_model`), and the restart path @@ -32,9 +32,9 @@ //! (`load_managed_service`, `read_managed_service_manifest`) and shapes the //! `stop_server` answer with its `internal_stop_server_value`, which stays there //! next to `service_stop_unconfirmed` and the renderer that reads the same -//! `stop_unconfirmed` status. This module takes -//! `ensure_public_service_has_endpoint_key` and the spawn helpers from -//! `service_launch.rs`, which does not import this module back. +//! `stop_unconfirmed` status. This module takes the spawn helpers from +//! `service_launch.rs`, which does not import this module back; the respawn +//! guard it applies is `rocm_core::ensure_public_service_has_endpoint_key`. //! //! The regression tests for the stop and restart paths live in this module's own //! `mod tests`. They reach shared fixtures (`test_paths`, `ScopedTestEnv`, @@ -68,7 +68,6 @@ use crate::engines_cmd::env_root_for_service; use crate::gpu_policy::parse_device_policy; use crate::managed_service_status::load_managed_services; use crate::service_hints; -use crate::service_launch::ensure_public_service_has_endpoint_key; #[cfg(not(windows))] use crate::service_launch::{ attach_background_stdio, detach_background_command, managed_engine_startup_failure_detail, @@ -126,119 +125,14 @@ pub(crate) fn run_internal_sandbox_tool( })) } -/// How long a managed stop waits for each recorded process to actually exit -/// after `SIGTERM` before escalating to `SIGKILL` and then reporting a timeout. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// One recorded PID's fate in a managed stop: the role the record gave it, and -/// what [`rocm_core::terminate_verified`] did about it. Kept per PID because the -/// outcomes that leave a stop unconfirmed are reported differently: a -/// `TimedOut` PID was signalled and it or a process under it outlived the -/// deadline, while an `Unverified` one is not signalled by this stop, and the -/// report names a command to check on it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct RecordedPidOutcome { - pid: u32, - /// `supervisor`, `engine`, or `supervisor_and_engine` when the record names - /// one PID for both — the same labels `rocmd`'s `pid_outcomes` uses. - role: &'static str, - outcome: rocm_core::TerminationOutcome, -} - -/// What a managed stop did to the processes its record names. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(crate) struct RecordedStopTermination { - /// Every PID that received a signal, including one that then outlived the - /// deadline. - signaled_pids: Vec, - /// Whether every recorded process is confirmed no longer running. - all_stopped: bool, - pid_outcomes: Vec, -} - -impl RecordedStopTermination { - /// A stop stand-in that confirms every recorded process gone without - /// signalling any, for tests in other modules that drive the real stop path. - #[cfg(test)] - pub(crate) fn confirmed_stopped() -> Self { - Self { - all_stopped: true, - ..Self::default() - } - } -} - -/// Terminate the processes recorded for a managed service, verifying each PID's -/// identity first so a recycled PID never causes an unrelated process tree to be -/// killed. -/// -/// The supervisor (launcher) and the engine server are distinct processes, so -/// each PID is paired with **its own** start-time token (`supervisor_start_ticks` -/// vs `engine_start_ticks`). When they resolve to the same PID (e.g. before the -/// engine state has been refreshed) the entry is de-duplicated, preferring a -/// known token. Returns the PIDs actually signalled, whether every recorded -/// process is confirmed no longer running, and each PID's own outcome. -fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> RecordedStopTermination { - // Build the (pid, own-token, role) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. - let mut entries: Vec<(u32, Option, &'static str)> = Vec::new(); - for (pid, ticks, role) in [ - ( - Some(record.supervisor_pid), - record.supervisor_start_ticks, - "supervisor", - ), - (record.engine_pid, record.engine_start_ticks, "engine"), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - if existing.1.is_none() { - existing.1 = ticks; - } - existing.2 = "supervisor_and_engine"; - } else { - entries.push((pid, ticks, role)); - } - } - - let mut termination = RecordedStopTermination { - all_stopped: true, - ..RecordedStopTermination::default() - }; - for (pid, ticks, role) in entries { - let identity = rocm_core::ProcessIdentity::new(pid, ticks); - // The managed stop contract is definitive: after the adapter gets its - // graceful opportunity, this cleanup pass forces any verified survivor - // so the command does not report success while a GPU worker remains. - let outcome = scripted_stop_outcome().unwrap_or_else(|| { - rocm_core::terminate_verified( - &identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) - }); - if !outcome.stopped() { - termination.all_stopped = false; - } - // Report every PID that received a signal. TimedOut belongs here because - // signalling was attempted even though the process was not confirmed - // stopped; `all_stopped` separately carries the truthful completion state. - if matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ) { - termination.signaled_pids.push(pid); - } - termination - .pid_outcomes - .push(RecordedPidOutcome { pid, role, outcome }); - } - termination +/// Terminate the processes recorded for a managed service through the routine +/// `rocmd` shares, [`rocm_core::terminate_recorded_service_pids`], with +/// [`rocm_core::terminate_recorded_pid`] as the termination of each PID. The +/// one thing this adds is the e2e seam below. +fn stop_recorded_processes(record: &ManagedServiceRecord) -> Vec { + rocm_core::terminate_recorded_service_pids(record, |identity| { + scripted_stop_outcome().unwrap_or_else(|| rocm_core::terminate_recorded_pid(identity)) + }) } /// E2E hook: when `ROCM_E2E_STOP_OUTCOME` is `timed_out` or `unverified`, a @@ -266,12 +160,12 @@ const fn scripted_stop_outcome() -> Option { } fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_internal_managed_service_with(paths, service_id, terminate_recorded_service_pids) + stop_internal_managed_service_with(paths, service_id, stop_recorded_processes) } /// [`stop_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller. Production passes -/// [`terminate_recorded_service_pids`]; a test passes a stand-in that observes +/// [`stop_recorded_processes`]; a test passes a stand-in that observes /// what is on disk at the moment the processes would be signalled. Everything /// else — the engine stop, the manifest writes, the key cleanup — runs for real. /// @@ -280,7 +174,7 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { stop_internal_managed_service_confirmed_with(paths, service_id, terminate) .map(|(report, _)| report) @@ -294,7 +188,7 @@ pub(crate) fn stop_internal_managed_service_with( fn stop_internal_managed_service_confirmed_with( paths: &AppPaths, service_id: &str, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result<(serde_json::Value, bool)> { let mut stopping = load_managed_service(paths, service_id)?; // Record the request on disk before anything is stopped, not only after. @@ -322,11 +216,17 @@ fn stop_internal_managed_service_confirmed_with( }, ) }; - let RecordedStopTermination { - signaled_pids, - all_stopped: terminated_all, - pid_outcomes, - } = terminate(&stopping); + let stops = terminate(&stopping); + // Every PID that received a signal, including one that then outlived the + // deadline: signalling was attempted even though the process was not + // confirmed stopped, and `terminated_all` separately carries the truthful + // completion state. + let signaled_pids: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); + let terminated_all = stops.iter().all(|stop| stop.stopped()); // Re-read the record rather than write back the copy loaded before the // stop, which can be some forty seconds old by now: `write` replaces the // whole file, and a supervisor, the daemon's restart or another stop may @@ -355,6 +255,15 @@ fn stop_internal_managed_service_confirmed_with( if all_stopped { record.status = "stopped".to_owned(); record.stop_requested_unix_ms = None; + // The processes this stop confirmed gone are not the service's any more, + // and their PIDs are stale the instant they exited. Left in the record, + // a later stop would signal them again: a PID recorded without a + // start-time token (every one a Windows record had before it recorded + // a creation time) cannot be refuted, so if the + // OS had given it to another process the Windows tree kill would take + // that process and its subtree. The same call, on the same confirmed + // path, as `rocmd`'s stop. + record.clear_recorded_processes(); } else { // The marker records that a stop was *asked for* even though it could // not be confirmed. `refresh_managed_service_runtime_liveness` needs it @@ -401,15 +310,9 @@ fn stop_internal_managed_service_confirmed_with( "status": record.status, "engine_stop": engine_stop, "signaled_pids": signaled_pids, - "pid_outcomes": pid_outcomes + "pid_outcomes": stops .iter() - .map(|entry| { - serde_json::json!({ - "pid": entry.pid, - "role": entry.role, - "outcome": entry.outcome.as_str(), - }) - }) + .map(|stop| stop.report_entry()) .collect::>(), "record_changed": record_changed, // A container engine is confirmed stopped by its own `Stop`; when that @@ -517,12 +420,7 @@ pub(crate) fn restart_internal_managed_service( service_id: &str, runtime_pin: Option<&str>, ) -> Result { - restart_internal_managed_service_with( - paths, - service_id, - runtime_pin, - terminate_recorded_service_pids, - ) + restart_internal_managed_service_with(paths, service_id, runtime_pin, stop_recorded_processes) } /// [`restart_internal_managed_service`], with the termination of the recorded @@ -538,7 +436,7 @@ fn restart_internal_managed_service_with( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { restart_internal_managed_service_inner(paths, service_id, runtime_pin, terminate, None) } @@ -573,7 +471,7 @@ fn restart_internal_managed_service_inner( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, spawn_override: Option<&RestartSpawnOverride<'_>>, ) -> Result { #[cfg(windows)] @@ -586,10 +484,11 @@ fn restart_internal_managed_service_inner( let preserved_endpoint_key = endpoint_keys::endpoint_api_key(paths, service_id); // Checked before the stop, so a refused restart leaves a running service // running instead of stopping it and then failing to bring it back. - ensure_public_service_has_endpoint_key( + rocm_core::ensure_public_service_has_endpoint_key( &record.host, preserved_endpoint_key.is_some(), record.requires_api_key, + rocm_core::ServiceStart::Restart, )?; let (stop_report, stopped) = stop_internal_managed_service_confirmed_with(paths, service_id, terminate) @@ -1033,6 +932,24 @@ mod tests { !rendered.contains("--require-api-key"), "a service that never passed the flag must not be told it did: {rendered}" ); + // The restart's refusal is the shared guard's, worded for a restart — + // the same text a fresh launch and the daemon print, bar the action. A + // private copy here, or the wrong action passed, would differ. + assert_eq!( + rendered, + rocm_core::ensure_public_service_has_endpoint_key( + "0.0.0.0", + false, + false, + rocm_core::ServiceStart::Restart, + ) + .unwrap_err() + .to_string() + ); + assert!( + rendered.contains("restarting it would reopen it without authentication"), + "{rendered}" + ); // The refusal happens before the stop, so the service is left alone. assert!( @@ -1075,7 +992,7 @@ mod tests { // Proves the *ordering*, not just the refusal: had the guard run after // `stop_internal_managed_service`, the stop would have written // "stopped". It reaches that state here because the record's only pid is - // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` + // the test's own (`engine_pid` is None and the shared stop routine // skips the caller's pid), so the stop confirms termination trivially. let after = load_managed_service(&paths, service_id).unwrap(); assert_ne!( @@ -1372,27 +1289,41 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the endpoint key is kept"); } - /// A stop stand-in's bare verdict, without per-PID detail. - fn bare_termination(signaled_pids: Vec, all_stopped: bool) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids, - all_stopped, - pid_outcomes: Vec::new(), + /// One recorded process's stop, for a stand-in to return. + fn stop_of(pid: u32, outcome: rocm_core::TerminationOutcome) -> rocm_core::RecordedPidStop { + rocm_core::RecordedPidStop { + pid, + role: "supervisor_and_engine", + outcome: rocm_core::RecordedPidOutcome::Terminated(outcome), } } + /// A stop stand-in's bare verdict, without per-PID detail: `signaled_pids` + /// were signalled and `all_stopped` says whether they exited. A stop that + /// signalled nothing and is not confirmed has one unverifiable process. + fn bare_termination( + signaled_pids: Vec, + all_stopped: bool, + ) -> Vec { + use rocm_core::TerminationOutcome::{Graceful, TimedOut, Unverified}; + if signaled_pids.is_empty() { + return if all_stopped { + Vec::new() + } else { + vec![stop_of(999_999_999, Unverified)] + }; + } + let outcome = if all_stopped { Graceful } else { TimedOut }; + signaled_pids + .into_iter() + .map(|pid| stop_of(pid, outcome)) + .collect() + } + /// A stop stand-in whose one recorded process (the record's shared /// supervisor/engine PID) was signalled and outlived the deadline. - fn timed_out_termination(pid: u32) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids: vec![pid], - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::TimedOut, - }], - } + fn timed_out_termination(pid: u32) -> Vec { + vec![stop_of(pid, rocm_core::TerminationOutcome::TimedOut)] } /// The most recent audit event written under `paths`. @@ -1412,17 +1343,13 @@ mod tests { stop_test_record(&paths, service_id).write().unwrap(); endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: Vec::new(), - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::Unverified, + )] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); @@ -1485,31 +1412,26 @@ mod tests { record.engine_pid = Some(888_888_888); record.write().unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: vec![999_999_999], - all_stopped: false, - pid_outcomes: vec![ - RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor", - outcome: rocm_core::TerminationOutcome::TimedOut, - }, - RecordedPidOutcome { - pid: 888_888_888, - role: "engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }, - ], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + role: "supervisor", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(888_888_888, rocm_core::TerminationOutcome::Unverified) + }, + ] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); let _ = fs::remove_dir_all(root); let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); assert_no_retry_line(&rendered); assert!( - rendered.contains("check: `ps -p 888888888 -o pid,args`"), + rendered.contains(&format!("check: `{}`", process_check_command(888_888_888))), "{rendered}" ); assert!(!rendered.contains("the record changed"), "{rendered}"); @@ -1610,6 +1532,110 @@ mod tests { assert_eq!(key, None, "a confirmed stop drops the key"); } + /// A confirmed stop must not leave the processes it stopped named in the + /// record. Their PIDs are stale the moment they exit, and a recorded PID + /// with no start-time token is signalled on trust alone once the OS has + /// handed it to another process; on Windows the tree kill then takes that + /// process and everything under it. `rocmd`'s stop clears them for the same + /// reason (`stop_managed_service_clears_recorded_pids_once_the_processes_are_gone`). + #[test] + fn a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing() { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("services-stop-clears-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-clears-pids"; + let mut record = stop_test_record(&paths, service_id); + // The dangerous shape: a supervisor recorded without a start-time token, + // and an engine with one. + record.supervisor_pid = 4242; + record.supervisor_start_ticks = None; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + // The termination of each recorded PID, observed: the real routine runs + // (which PIDs it visits, in which order), only the signalling is replaced. + let signalled = std::cell::RefCell::new(Vec::new()); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + signalled.borrow_mut().push(*identity); + Graceful + }) + }; + + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let signalled_by_first = signalled.borrow().clone(); + + let second = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_second = raw_manifest(&paths, service_id); + let signalled_in_all = signalled.borrow().clone(); + let _ = fs::remove_dir_all(root); + + assert_eq!(first["stopped"], true, "{first}"); + assert_eq!( + signalled_by_first, + vec![ + rocm_core::ProcessIdentity::new(4242, None), + rocm_core::ProcessIdentity::new(4243, Some(77)), + ] + ); + // The persisted state after the confirmed stop names no process. + assert_eq!(after_first.status, "stopped"); + assert_eq!(after_first.stop_requested_unix_ms, None); + assert_eq!(after_first.supervisor_pid, 0); + assert_eq!(after_first.supervisor_start_ticks, None); + assert_eq!(after_first.engine_pid, None); + assert_eq!(after_first.engine_start_ticks, None); + // So a stop of the same record signals nothing, and still reads stopped. + assert_eq!( + signalled_in_all, signalled_by_first, + "the second stop reached a PID the first one had already stopped" + ); + assert_eq!(second["stopped"], true, "{second}"); + assert_eq!(second["signaled_pids"], serde_json::json!([]), "{second}"); + assert_eq!(second["pid_outcomes"], serde_json::json!([]), "{second}"); + assert_eq!(after_second.status, "stopped"); + assert_eq!(after_second.supervisor_pid, 0); + assert_eq!(after_second.engine_pid, None); + } + + #[test] + fn an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop() { + use rocm_core::TerminationOutcome::TimedOut; + let (root, paths) = test_paths("services-stop-keeps-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-keeps-pids"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + let signalled = std::cell::Cell::new(0_usize); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |_| { + signalled.set(signalled.get() + 1); + TimedOut + }) + }; + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let _ = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let signalled_in_all = signalled.get(); + let _ = fs::remove_dir_all(root); + + // Nothing was confirmed gone, so the record still names what survived: + // that is the only thing a later stop can find them by. + assert_eq!(first["stopped"], false, "{first}"); + assert_eq!(after_first.supervisor_pid, 4242); + assert_eq!(after_first.supervisor_start_ticks, Some(5)); + assert_eq!(after_first.engine_pid, Some(4243)); + assert_eq!(after_first.engine_start_ticks, Some(77)); + assert_eq!(signalled_in_all, 4, "each stop tried both processes"); + } + #[test] fn restart_does_not_start_an_engine_over_an_unconfirmed_stop() { // A restart used to discard its stop's result and start the new engine @@ -1909,8 +1935,11 @@ mod tests { // The claim the message makes: the service really is stopped, and no // replacement child was started behind the failed wait. assert_eq!(record.status, "stopped"); + // The confirmed stop cleared the recorded processes (the record names + // none), so a supervisor recorded behind the failed wait would show as a + // nonzero PID. assert_eq!( - record.supervisor_pid, 999_999_999, + record.supervisor_pid, 0, "no new supervisor may be recorded behind the failed wait" ); Ok(()) @@ -2385,7 +2414,7 @@ mod tests { "svc-cpu-policy", "vllm", "starting", - // Skipped by `terminate_recorded_service_pids`, which never + // Skipped by the shared stop routine, which never // signals this process — so the stop runs for real without the // test killing itself. std::process::id(), @@ -2471,15 +2500,14 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real.wrapping_add(1))); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); - assert!(signaled.is_empty(), "must not signal a mismatched identity"); + assert!( + stops.iter().all(|stop| !stop.signaled()), + "must not signal a mismatched identity" + ); // A recycled PID means the recorded service process itself is gone. - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); assert!( rocm_core::process_is_running(pid), "the unrelated live process must survive" @@ -2509,11 +2537,13 @@ mod tests { record.engine_pid = Some(eng_pid); record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); - let RecordedStopTermination { - signaled_pids: mut signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let all_stopped = stops.iter().all(|stop| stop.stopped()); + let mut signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); signaled.sort_unstable(); let mut expected = vec![sup_pid, eng_pid]; expected.sort_unstable(); @@ -2541,14 +2571,15 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real)); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); assert_eq!(signaled, vec![pid]); - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); // Reap our own child (a detached managed process would be reaped by init) // so the liveness check does not observe a not-yet-reaped zombie. let mut child = child; @@ -2559,4 +2590,204 @@ mod tests { ); let _ = fs::remove_dir_all(root); } + + // ---- This module's stop, through the routine it shares with `rocmd`. ---- + // + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. Every PID below is one that cannot be + // running, so nothing is signalled and the tests behave the same on every + // host. + + /// A PID no host has in use, so the stop reports it as already gone. + const ABSENT_SUPERVISOR_PID: u32 = 999_999_998; + const ABSENT_ENGINE_PID: u32 = 999_999_997; + + /// Run the production stop of a record naming `supervisor` and `engine`, + /// and return its report. + fn stop_report_for(name: &str, supervisor: u32, engine: Option) -> serde_json::Value { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let service_id = "svc-stop-through-the-shared-routine"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = supervisor; + record.engine_pid = engine; + record.write().unwrap(); + + let report = + stop_internal_managed_service_with(&paths, service_id, stop_recorded_processes) + .unwrap(); + let _ = fs::remove_dir_all(root); + report + } + + #[test] + fn a_stop_reports_the_supervisor_then_the_engine_in_that_order() { + let report = stop_report_for( + "managed-stop-order", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_ENGINE_PID), + ); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor", "outcome": "already_gone"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!(report["signaled_pids"], serde_json::json!([])); + assert_eq!(report["stopped"], true); + } + + #[test] + fn a_stop_reports_a_pid_recorded_twice_once_under_both_roles() { + let report = stop_report_for( + "managed-stop-aliased", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_SUPERVISOR_PID), + ); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor_and_engine", "outcome": "already_gone"}, + ]) + ); + } + + /// Both binaries report the stopping process's own PID as `self`, never + /// signal it, and do not let it stand in the way of a confirmed stop. `rocm` + /// used to drop it from the report, so `pid_outcomes` showed one entry fewer + /// than `rocmd`'s for the same record. + #[test] + fn a_stop_reports_its_own_pid_as_self() { + let own = std::process::id(); + + let report = stop_report_for("managed-stop-own-pid", own, Some(ABSENT_ENGINE_PID)); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": own, "role": "supervisor", "outcome": "self"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!( + report["signaled_pids"], + serde_json::json!([]), + "the stopping process is never signalled" + ); + assert_eq!(report["stopped"], true); + } + + #[test] + fn a_stop_of_a_record_naming_only_its_own_pid_confirms() { + let own = std::process::id(); + + let report = stop_report_for("managed-stop-only-own-pid", own, None); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([{"pid": own, "role": "supervisor", "outcome": "self"}]) + ); + assert_eq!(report["stopped"], true); + } + + /// The `self` row is not one of the outcomes that leave a stop unconfirmed, + /// so it never reaches the report's `still running` or `not signaled` lines. + #[test] + fn a_self_row_adds_no_line_to_an_unconfirmed_report() { + let (root, paths) = test_paths("managed-stop-self-row-unconfirmed"); + paths.ensure().unwrap(); + let service_id = "svc-self-row-unconfirmed"; + stop_test_record(&paths, service_id).write().unwrap(); + let own = std::process::id(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + pid: own, + role: "supervisor", + outcome: rocm_core::RecordedPidOutcome::OwnProcess, + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + ] + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let _ = fs::remove_dir_all(root); + let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); + + assert!( + rendered.contains("still running: process 999999999 (engine)"), + "{rendered}" + ); + assert!( + !rendered.contains(&format!("process {own}")), + "the stopping process is not a survivor: {rendered}" + ); + } + + #[cfg(unix)] + #[test] + fn managed_stop_signals_the_supervisor_before_the_engine() { + let sup = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn supervisor"); + let eng = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn engine"); + let (sup_pid, eng_pid) = (sup.id(), eng.id()); + let (root, paths) = test_paths("managed-stop-signal-order"); + let mut record = + managed_record_for_pid(&paths, sup_pid, rocm_core::process_start_ticks(sup_pid)); + record.engine_pid = Some(eng_pid); + record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); + + let stops = stop_recorded_processes(&record); + let (mut sup, mut eng) = (sup, eng); + let _ = sup.wait(); + let _ = eng.wait(); + let _ = fs::remove_dir_all(root); + + // Not sorted: the order is the routine's, and a copy that visited the + // engine first would fail here. + assert_eq!( + stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect::>(), + vec![sup_pid, eng_pid] + ); + assert!(stops.iter().all(|stop| stop.stopped())); + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service_control.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } + } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 53f9b5c41d..29ecc4d315 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -75,11 +75,11 @@ pub(crate) fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result< Ok(()) } -/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the -/// policy so `rocmd` classifies a recorded `host` identically when it respawns -/// the service. +/// Inverse of [`rocm_core::is_public_bind_host`], which owns the policy so +/// `rocmd` classifies a recorded `host` identically when it respawns the +/// service. pub(crate) fn is_loopback_host(host: &str) -> bool { - !rocm_engine_protocol::is_public_bind_host(host) + !rocm_core::is_public_bind_host(host) } /// Resolve the API key that will guard this endpoint, applying the @@ -172,57 +172,6 @@ pub(crate) fn ensure_public_bind_engine_supported( Ok(()) } -/// Refuse to (re)spawn a managed service recorded on a public host when its -/// endpoint API key is gone, so a respawn cannot reopen the endpoint anonymously. -/// -/// `serve()` applies the loopback-vs-public policy once, via -/// [`resolve_endpoint_auth`], and persists the resulting key. Every later spawn -/// — `rocm services restart`, and `rocmd`'s recovery supervisor — reads that key -/// file back and would otherwise treat "no key file" as "no auth wanted", -/// silently downgrading a protected public endpoint to an open one. The real -/// invariant is a property of the *host*, not of the file: a non-loopback bind -/// must always be authenticated. -/// -/// The gap is reachable through ordinary commands, because a stop deletes the -/// key file and `rocm services restart` accepts a stopped service id (its help -/// points at `rocm services list --all`). -/// -/// `key_present` is a plain `bool` rather than a path so both branches are -/// unit-testable without touching the filesystem, mirroring `is_windows` in -/// [`ensure_public_bind_engine_supported`]. -pub(crate) fn ensure_public_service_has_endpoint_key( - host: &str, - key_present: bool, - requires_api_key: bool, -) -> Result<()> { - // Two ways a service can need a key. A public bind is the obvious one. The - // other is a service that asked for auth on a loopback bind, because - // something outside this process republishes the port — a tailnet publish - // survives a reboot, let alone a restart, so "loopback" stops meaning - // "only this machine" and the bind address can no longer be trusted to - // answer the question on its own. - if requires_api_key && !key_present { - bail!( - "managed service was launched with `--require-api-key` but has no endpoint API key, \ - so restarting it would reopen it without authentication. Something outside this \ - machine may still be publishing its port. The key is dropped when a service stops \ - and cannot be recovered. Launch it again with \ - `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ - to choose the key instead of generating one)." - ); - } - if rocm_engine_protocol::is_public_bind_host(host) && !key_present { - bail!( - "managed service is bound to the public host `{host}` but has no endpoint API key, \ - so restarting it would reopen it without authentication. The key is dropped when a \ - service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --port --allow-public-bind` (add `--api-key `, or set \ - ROCM_SERVE_API_KEY, to choose the key instead of generating one)." - ); - } - Ok(()) -} - #[cfg(not(windows))] pub(crate) fn detach_background_command(command: &mut ProcessCommand) { rocm_core::detach_command_session(command); @@ -505,7 +454,7 @@ pub(crate) enum ManagedReuseRefusal { /// builds — so a server started without one keeps serving anonymously no /// matter what is written afterwards. Upgrading the record would be worse /// than doing nothing: it would claim auth the live process does not - /// enforce, and `ensure_public_service_has_endpoint_key` would pass on the + /// enforce, and the endpoint-key respawn guard would pass on the /// strength of a key file nothing reads. `rocm remote serve` relies on this: /// it publishes a loopback port onto the tailnet and prints "the API key /// above is what stops anyone else calling it", which reusing an @@ -930,10 +879,11 @@ fn launch_managed_engine_child_locked( // // The field is threaded, never derived from the key file. Deriving it marked // every public bind as having demanded auth — see the assignment above. - ensure_public_service_has_endpoint_key( + rocm_core::ensure_public_service_has_endpoint_key( host, endpoint_key_file.is_some(), record.requires_api_key, + rocm_core::ServiceStart::Launch, )?; // Hand the port over: close the probe socket immediately before the spawn, // still under the caller's launch lock, so no other ROCm transaction can @@ -2678,7 +2628,13 @@ mod tests { finish("attached-unverified", Ok(stop_answer(false, "unverified"))); let error = format!("{:#}", result.expect_err("an unverified stop fails")); assert!(error.contains("not signaled: process 4242"), "{error}"); - assert!(error.contains("check: `ps -p 4242 -o pid,args`"), "{error}"); + assert!( + error.contains(&format!( + "check: `{}`", + crate::services_text::process_check_command(4242) + )), + "{error}" + ); crate::services_text::tests::assert_no_retry_line(&error); assert!(out.is_empty(), "{out}"); assert_eq!(signalled, None); @@ -3154,7 +3110,7 @@ mod tests { // was launched with `--require-api-key`, and its key file is present but // invalid — so `requires_api_key` is true while `key_present` is false, // which is the only way to reach the first branch. A test calling - // `ensure_public_service_has_endpoint_key` directly cannot catch a + // `rocm_core::ensure_public_service_has_endpoint_key` directly cannot catch a // mis-wired call site, which is the defect that has occurred here twice. // // The flag is passed explicitly rather than inferred from the key file. @@ -3238,6 +3194,24 @@ mod tests { message.contains("without authentication"), "the refusal must say what the risk is: {message}" ); + // The spawn's refusal is the shared guard's, worded for a launch — the + // same text `rocm services restart` and the daemon print, bar the + // action. A private copy here, or the wrong action passed, would differ. + assert_eq!( + message, + rocm_core::ensure_public_service_has_endpoint_key( + "127.0.0.1", + false, + true, + rocm_core::ServiceStart::Launch, + ) + .unwrap_err() + .to_string() + ); + assert!( + message.contains("launching it would open it without authentication"), + "{message}" + ); assert!( !manifest_path.exists(), "a pre-spawn failure must not leave a record" @@ -3767,6 +3741,18 @@ mod tests { ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key } + /// The guard `rocm services restart` runs, as that caller words it. The + /// guard itself is `rocm_core`'s and is tested there; these tests are about + /// what a restart is told. + fn restart_guard(host: &str, key_present: bool, requires_api_key: bool) -> Result<()> { + rocm_core::ensure_public_service_has_endpoint_key( + host, + key_present, + requires_api_key, + rocm_core::ServiceStart::Restart, + ) + } + #[test] fn a_public_bind_is_refused_with_the_command_that_restores_it() { // `resolve_endpoint_auth` mints a key for every non-loopback bind whether @@ -3778,7 +3764,7 @@ mod tests { // // A plain `--host 0.0.0.0 --allow-public-bind` launch: key file present, // `--require-api-key` never passed. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false) + let error = restart_guard("0.0.0.0", false, false) .expect_err("a public bind with no key must be refused"); let rendered = format!("{error:#}"); assert!( @@ -3791,7 +3777,7 @@ mod tests { ); // And the loopback-with-forced-auth case still reports its own reason. - let error = ensure_public_service_has_endpoint_key("127.0.0.1", false, true) + let error = restart_guard("127.0.0.1", false, true) .expect_err("a service that demanded auth must not come back without it"); assert!( format!("{error:#}").contains("--require-api-key"), @@ -3803,7 +3789,7 @@ mod tests { fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { // A stop deletes the key file, so a later restart of a public service // would otherwise respawn it with no auth at all. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + let error = restart_guard("0.0.0.0", false, false).unwrap_err(); let message = error.to_string(); assert!(message.contains("0.0.0.0"), "{error:#}"); assert!(message.contains("without authentication"), "{error:#}"); @@ -3811,7 +3797,7 @@ mod tests { assert!(message.contains("--allow-public-bind"), "{error:#}"); // A public service that still has its key restarts normally. - ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); + restart_guard("0.0.0.0", true, false).unwrap(); } #[test] @@ -3819,7 +3805,7 @@ mod tests { // Loopback stays credential-free, so every accepted spelling must pass // the guard with no key present — when nothing asked for auth. for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false, false) + restart_guard(host, false, false) .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); } } @@ -3832,7 +3818,7 @@ mod tests { // reopen a reachable endpoint with no authentication, and the bind // address gives the guard no way to notice. for host in ["127.0.0.1", "localhost", "::1"] { - let error = ensure_public_service_has_endpoint_key(host, false, true) + let error = restart_guard(host, false, true) .expect_err("a service launched with a key must not restart without one"); let message = format!("{error:#}"); assert!(message.contains("without authentication"), "{message}"); @@ -3840,7 +3826,148 @@ mod tests { } // With its key still present it restarts normally. - ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); + restart_guard("127.0.0.1", true, true).unwrap(); + } + + /// The `rocm serve …` command a refusal tells the user to run: the backtick + /// span that starts with `rocm serve`, split into words. The `` + /// placeholder is filled with a port, as a user would, so the command can be + /// parsed; `` is left for each caller to treat as it needs. + fn advised_serve_command(message: &str) -> Vec { + let spans: Vec<&str> = message + .split('`') + .skip(1) + .step_by(2) + .filter(|span| span.starts_with("rocm serve")) + .collect(); + assert_eq!( + spans.len(), + 1, + "the refusal must name exactly one `rocm serve` command: {message}" + ); + spans[0] + .split_whitespace() + .map(|word| match word { + "" => crate::advised_commands::placeholder_value(word, "--port"), + word => word.to_owned(), + }) + .collect() + } + + /// Run what `rocm serve` does with `advised` before it spawns anything — the + /// real clap parse, `validate_bind_host`, `resolve_endpoint_auth`, and the + /// key file `serve()` stores — and return the guard's answer for the service + /// that launch would record. Nothing is spawned. + fn guard_after_following( + advised: &[String], + paths: &AppPaths, + service_id: &str, + start: rocm_core::ServiceStart, + ) -> Result<()> { + use crate::{Cli, Command, cli_command}; + use clap::FromArgMatches as _; + + let matches = cli_command() + .try_get_matches_from(advised) + .unwrap_or_else(|error| panic!("{advised:?} does not parse: {error}")); + let Some(Command::Serve { + host, + port, + allow_public_bind, + require_api_key, + api_key, + .. + }) = Cli::from_arg_matches(&matches)?.command + else { + panic!("{advised:?} is not a `rocm serve` invocation"); + }; + validate_bind_host(&host, allow_public_bind)?; + // A host other than the default loopback needs an explicit port; a + // command that leaves it out is refused here as `rocm serve` would. + crate::serve_port::validate_port_request( + &host, + crate::serve_port::PortRequest::from_flag(port), + )?; + let key = resolve_endpoint_auth(&host, api_key.as_deref(), require_api_key)?; + if let Some(key) = key.as_deref() { + endpoint_keys::store_endpoint_api_key(paths, service_id, key)?; + } + // The expression `spawn_managed_engine_child` computes: a *usable* key. + let key_present = endpoint_keys::endpoint_key_file_if_present(paths, service_id) + .is_some_and(|path| rocm_engine_protocol::endpoint_api_key_from_file(&path).is_some()); + rocm_core::ensure_public_service_has_endpoint_key( + &host, + key_present, + require_api_key, + start, + ) + } + + /// The command a refusal names is the same for every action that can print + /// it — `rocm services restart`, a fresh launch, and the daemon's respawn, + /// whose text is the same shared guard's — so each is followed here, from + /// the refused state, through the real `rocm serve` front half. + #[test] + fn the_command_a_respawn_refusal_names_clears_the_refusal() -> Result<()> { + use rocm_core::ServiceStart; + + for start in [ + ServiceStart::Launch, + ServiceStart::Restart, + ServiceStart::Respawn, + ] { + for (host, requires_api_key) in [ + ("127.0.0.1", true), + ("localhost", true), + ("::1", true), + ("0.0.0.0", false), + ("192.168.1.10", false), + ("gpu-box.example", false), + ] { + let message = rocm_core::ensure_public_service_has_endpoint_key( + host, + false, + requires_api_key, + start, + ) + .expect_err("a keyless service in this state is refused") + .to_string(); + let advised = advised_serve_command(&message); + + // `rocm serve` takes a required : the command names it as + // a placeholder, and parses as printed. + assert_eq!(advised[2], "", "{advised:?}"); + assert_eq!( + crate::advised_commands::rocm_verdict(&advised[1..]), + crate::advised_commands::Verdict::Parses, + "{advised:?}" + ); + + // With a model in the placeholder's place the command asks for + // what the refusal needs (`--require-api-key` for the first rule, + // the public bind for the second), and what it stores satisfies + // the guard that refused. + let mut runnable = advised.clone(); + runnable[2] = "Qwen/Qwen3-0.6B".to_owned(); + let (root, paths) = test_paths("respawn-advice-clears"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let service_id = "lemonade-qwen-3000"; + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none(), + "the refused state has no key on disk" + ); + guard_after_following(&runnable, &paths, service_id, start).unwrap_or_else( + |error| panic!("{runnable:?} does not clear {host} ({start:?}): {error:#}"), + ); + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), + "following the advice leaves the service a key to respawn with" + ); + let _ = fs::remove_dir_all(root); + } + } + Ok(()) } #[test] @@ -4425,7 +4552,7 @@ mod tests { // confirms every recorded process gone without touching the test // process itself. crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { - crate::service_control::RecordedStopTermination::confirmed_stopped() + Vec::new() })?; let decision = resolve_managed_port_in_transaction( diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index f1bee48f29..5623b59b67 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -31,7 +31,7 @@ //! `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the //! CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, //! `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, -//! `read_tail_lines`). +//! `read_log_tail`). //! //! The `LogBrowserEntry` struct moved here with the logs browser and stays //! private. @@ -54,7 +54,7 @@ //! `service_control.rs` and from `managed_service_status.rs`'s own test module; //! `read_managed_service_manifest`, `internal_stop_server_value` and //! `process_check_command` are `pub(crate)` for `service_control.rs` (the last -//! one only for its tests); `read_tail_lines`, `render_logs_text`, +//! one only for its tests); `read_log_tail`, `render_logs_text`, //! `render_logs_browser_text`, `render_services_text` and //! `render_service_logs_text` are also `pub(crate)` and read from //! `chat/mcp_bridge.rs`'s read-only command runner; `validate_service_id` is also @@ -77,10 +77,9 @@ //! classification respectively — stay in `main.rs`'s shared `mod tests` //! instead. -use std::collections::VecDeque; use std::fmt::Write as _; use std::fs; -use std::io::{self, BufRead, Read as _, Seek as _, SeekFrom}; +use std::io::{self, Read as _, Seek as _, SeekFrom}; use std::path::{Path, PathBuf}; use std::thread; use std::time::Duration; @@ -479,8 +478,7 @@ fn render_service_logs_text_with_options( show_file_locations: bool, ) -> Result { let record = load_managed_service(paths, service_id)?; - let recent_lines = - read_optional_tail_lines(&record.log_path, DEFAULT_LOG_TAIL_LINES, "service log"); + let recent = read_optional_log_tail(&record.log_path, DEFAULT_LOG_TAIL_LINES, "service log"); let mut output = String::new(); let _ = writeln!(output, "Service Log"); @@ -506,11 +504,17 @@ fn render_service_logs_text_with_options( let _ = writeln!(output, " Choose Show file locations to see exact paths."); } let _ = writeln!(output); - if recent_lines.is_empty() { + if recent.newest_line_exceeds_the_read_bound() { + let _ = writeln!( + output, + "Recent output: none shown; {}", + rocm_core::newest_line_exceeds_bound_notice() + ); + } else if recent.lines.is_empty() { let _ = writeln!(output, "Recent output: no output yet"); } else { - let _ = writeln!(output, "Recent output: last {} line(s)", recent_lines.len()); - for line in recent_lines { + let _ = writeln!(output, "Recent output: last {} line(s)", recent.lines.len()); + for line in recent.lines { let _ = writeln!(output, " {line}"); } } @@ -1140,11 +1144,22 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { /// stop left alone because it could not verify it. Not a `rocm` command: the /// CLI deliberately does not act on that process, so the user decides. /// -/// A Linux command because only Linux can produce that outcome: it needs a -/// recorded start-time that can no longer be read back, and -/// `rocm_core::process_start_ticks` records one only on Linux. +/// Linux and Windows can both produce that outcome: it needs a recorded +/// start-time that can no longer be read back, and `rocm_core::process_start_ticks` +/// records one on both. So the command is the one the host has: `ps` on Linux, +/// `tasklist` on Windows. pub(crate) fn process_check_command(pid: u64) -> String { - format!("ps -p {pid} -o pid,args") + process_check_command_for(cfg!(windows), pid) +} + +/// [`process_check_command`] for a chosen platform, so a test reads both forms +/// on every host. +fn process_check_command_for(windows: bool, pid: u64) -> String { + if windows { + format!("tasklist /FI \"PID eq {pid}\"") + } else { + format!("ps -p {pid} -o pid,args") + } } /// A managed service's record exactly as it is on disk, without the engine-state @@ -1249,12 +1264,22 @@ fn list_log_files(dir: &Path, limit: usize) -> Vec { } pub(crate) fn read_optional_tail_lines(path: &Path, limit: usize, label: &str) -> Vec { + read_optional_log_tail(path, limit, label).lines +} + +/// [`read_optional_tail_lines`], keeping what the byte budget left unread so +/// a caller that describes an empty result can tell an empty log from one +/// whose newest line is too long to show. +pub(crate) fn read_optional_log_tail(path: &Path, limit: usize, label: &str) -> rocm_core::LogTail { if !path.is_file() { - return Vec::new(); + return rocm_core::LogTail::default(); } - match read_tail_lines(path, limit, label) { - Ok(lines) => lines, - Err(error) => vec![format!("")], + match read_log_tail(path, limit, label) { + Ok(tail) => tail, + Err(error) => rocm_core::LogTail { + lines: vec![format!("")], + skipped_bytes: 0, + }, } } @@ -1320,23 +1345,17 @@ fn lifecycle_field<'a>(line: &'a str, key: &str) -> Option<&'a str> { .filter(|value| !value.is_empty()) } -pub(crate) fn read_tail_lines(path: &Path, limit: usize, label: &str) -> Result> { - if limit == 0 { - return Ok(Vec::new()); - } - - let file = fs::File::open(path) - .with_context(|| format!("failed to open {label} {}", path.display()))?; - let reader = io::BufReader::new(file); - let mut lines = VecDeque::with_capacity(limit); - for line in reader.lines() { - let line = line.with_context(|| format!("failed to read {label} {}", path.display()))?; - if lines.len() == limit { - lines.pop_front(); - } - lines.push_back(line); - } - Ok(lines.into_iter().collect()) +/// The last `limit` lines of a log, read through the shared bounded tail +/// routine, with what the byte budget left unread; `label` names the log in the +/// error ("service log", "CLI action log"). +pub(crate) fn read_log_tail(path: &Path, limit: usize, label: &str) -> Result { + rocm_core::tail_log_decoded( + path, + limit, + rocm_core::LOG_VIEW_MAX_TAIL_READ, + rocm_core::LogDecode::PLAIN, + ) + .with_context(|| format!("failed to read {label} {}", path.display())) } #[cfg(test)] pub(crate) mod tests { @@ -1348,6 +1367,166 @@ pub(crate) mod tests { use crate::service_control::run_internal_sandbox_tool; use crate::tests::test_paths; + /// The lines of [`read_log_tail`], for the tests that only look at those. + fn read_tail_lines(path: &Path, limit: usize, label: &str) -> Result> { + Ok(read_log_tail(path, limit, label)?.lines) + } + + /// The check a stop that could not verify a process tells the user to run + /// is a command the host has: `ps` on Linux, `tasklist` on Windows, where a + /// Windows record now carries a creation time that can fail to read back. + /// The wording is pinned here; that the Windows form shows whether a PID is + /// running is what the `service-stop-07` steps rely on, which look a PID up + /// with the same `tasklist /FI "PID eq "` filter. + #[test] + fn the_process_check_command_is_the_one_the_host_has() { + assert_eq!(process_check_command_for(false, 42), "ps -p 42 -o pid,args"); + assert_eq!( + process_check_command_for(true, 42), + "tasklist /FI \"PID eq 42\"" + ); + assert_eq!( + process_check_command(42), + process_check_command_for(cfg!(windows), 42), + "production picks by the host it runs on" + ); + } + + fn tail_fixture(name: &str, bytes: &[u8]) -> Result { + let (root, _paths) = test_paths(name); + fs::create_dir_all(&root)?; + let path = root.join("service.log"); + fs::write(&path, bytes)?; + Ok(path) + } + + /// 5 MiB: a marked first line, 5120 lines of 1 KiB, a marked last line. + fn large_log_with_head_marker() -> Vec { + let mut bytes = b"HEAD-MARKER\n".to_vec(); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..5120 { + bytes.extend_from_slice(filler.as_bytes()); + } + bytes.extend_from_slice(b"TAIL-MARKER\n"); + bytes + } + + #[test] + fn read_tail_lines_a_large_file_is_read_through_a_bounded_window() -> Result<()> { + // A line far outside the tail window is not returned however large the + // limit: the log is not streamed whole. + let log = tail_fixture("services-tail-large", &large_log_with_head_marker())?; + + let lines = read_tail_lines(&log, 10_000, "service log")?; + fs::remove_file(&log)?; + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + assert!(lines.len() < 5122, "got {} lines", lines.len()); + assert!(lines.len() > 3500, "the window holds about 4 MiB of lines"); + Ok(()) + } + + #[test] + fn read_tail_lines_shows_a_sixteen_bit_log_as_lossy_text_instead_of_failing() -> Result<()> { + // Service logs are the child's own bytes, read as UTF-8; a UTF-16 log + // is not detected here (only the ComfyUI log view does that), but it no + // longer aborts the view. + let mut bytes = vec![0xff, 0xfe]; + for unit in "[INFO] one\r\n".encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + let log = tail_fixture("services-tail-utf16", &bytes)?; + + let lines = read_tail_lines(&log, 5, "service log")?; + + assert!( + lines[0].starts_with("\u{FFFD}\u{FFFD}["), + "the byte order mark is replaced, the rest kept: {lines:?}" + ); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_ansi_colour_codes() -> Result<()> { + let log = tail_fixture("services-tail-ansi", b"\x1b[32mgreen\x1b[0m done\n")?; + + assert_eq!( + read_tail_lines(&log, 5, "service log")?, + vec!["\u{1b}[32mgreen\u{1b}[0m done"] + ); + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_invalid_eight_bit_text_lossily() -> Result<()> { + let log = tail_fixture("services-tail-invalid", b"ok\nbad\xffbyte\nlast\n")?; + + let expected = vec![ + "ok".to_owned(), + "bad\u{FFFD}byte".to_owned(), + "last".to_owned(), + ]; + assert_eq!(read_tail_lines(&log, 5, "service log")?, expected); + // The optional reader shows the same lines, not a failure placeholder. + assert_eq!(read_optional_tail_lines(&log, 5, "service log"), expected); + Ok(()) + } + + #[test] + fn read_tail_lines_missing_file_is_a_labelled_error_naming_the_path() { + let (root, _paths) = test_paths("services-tail-missing"); + let missing = root.join("absent.log"); + + let error = + read_tail_lines(&missing, 5, "service log").expect_err("a missing log must not read"); + + assert_eq!( + error.to_string(), + format!("failed to read service log {}", missing.display()) + ); + assert!( + format!("{error:#}").contains(&format!("failed to open {}", missing.display())), + "the cause names the failing step: {error:#}" + ); + // read_optional_tail_lines guards on is_file first and stays quiet. + assert!(read_optional_tail_lines(&missing, 5, "service log").is_empty()); + } + + #[test] + fn read_tail_lines_limit_zero_returns_nothing_without_opening_the_file() { + let (root, _paths) = test_paths("services-tail-zero"); + + let lines = read_tail_lines(&root.join("absent.log"), 0, "service log") + .expect("limit 0 returns before the file is opened"); + + assert!(lines.is_empty()); + } + + #[test] + fn read_tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let log = tail_fixture("services-tail-few", b"a\nb\n")?; + + assert_eq!(read_tail_lines(&log, 50, "service log")?, vec!["a", "b"]); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let log = tail_fixture("services-tail-no-newline", b"a\nb\nc")?; + + assert_eq!(read_tail_lines(&log, 2, "service log")?, vec!["b", "c"]); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_crlf_line_endings() -> Result<()> { + let log = tail_fixture("services-tail-crlf", b"a\r\nb\r\n")?; + + assert_eq!(read_tail_lines(&log, 5, "service log")?, vec!["a", "b"]); + Ok(()) + } + #[test] fn render_logs_text_preserves_directory_summary() { let (_root, paths) = test_paths("logs-summary"); @@ -2372,6 +2551,91 @@ pub(crate) mod tests { Ok(()) } + /// What `rocm services logs` prints for a service whose log holds `bytes` + /// (or has no file when `None`), with the log's size on disk, so a message + /// is asserted together with the state it describes. + fn render_service_log_with(name: &str, bytes: Option<&[u8]>) -> Result<(String, Option)> { + let (root, paths) = test_paths(name); + paths.ensure()?; + let record = ManagedServiceRecord::new( + &paths, + "svc_tail_notice", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_preferred".to_owned()), + ); + record.write()?; + let size = match bytes { + Some(bytes) => { + fs::write(&record.log_path, bytes)?; + Some(fs::metadata(&record.log_path)?.len()) + } + None => None, + }; + let rendered = render_service_logs_text(&paths, "svc_tail_notice")?; + let _ = fs::remove_dir_all(root); + Ok((rendered, size)) + } + + #[test] + fn service_logs_say_no_output_only_for_an_empty_log_and_name_the_bound_otherwise() -> Result<()> + { + let bound = rocm_core::LOG_VIEW_MAX_TAIL_READ; + assert_eq!( + bound, + 4 * 1024 * 1024, + "the notice below names this many MiB" + ); + let notice = + "Recent output: none shown; the newest line is longer than the 4 MiB a log view reads"; + let over_the_bound = usize::try_from(bound)? + 1024; + + // No log file and an empty one: nothing was written, and it says so. + let (rendered, size) = render_service_log_with("service-logs-none", None)?; + assert_eq!(size, None); + assert!( + rendered.contains("Recent output: no output yet"), + "{rendered}" + ); + let (rendered, size) = render_service_log_with("service-logs-empty", Some(b""))?; + assert_eq!(size, Some(0)); + assert!( + rendered.contains("Recent output: no output yet"), + "{rendered}" + ); + + // One line longer than the bound, with and without a final newline: + // the file has content, so "no output yet" would be false. + let one_line = "x".repeat(over_the_bound).into_bytes(); + let mut with_newline = one_line.clone(); + with_newline.push(b'\n'); + for (name, bytes) in [ + ("service-logs-long-line", one_line), + ("service-logs-long-line-newline", with_newline), + ] { + let (rendered, size) = render_service_log_with(name, Some(&bytes))?; + assert!(size.is_some_and(|size| size > bound), "{name}: {size:?}"); + assert!(rendered.contains(notice), "{name}: {rendered}"); + assert!(!rendered.contains("no output yet"), "{name}: {rendered}"); + assert!(!rendered.contains("last 0 line(s)"), "{name}: {rendered}"); + } + + // A big log whose newest lines are short is shown, not described. + let (rendered, size) = + render_service_log_with("service-logs-big", Some(&large_log_with_head_marker()))?; + assert!(size.is_some_and(|size| size > bound)); + assert!(rendered.contains("TAIL-MARKER"), "{rendered}"); + assert!(!rendered.contains(notice), "{rendered}"); + Ok(()) + } + #[test] fn render_services_text_lists_live_services_by_default_and_all_on_request() -> Result<()> { use std::io::{Read, Write}; diff --git a/apps/rocm/src/tool_policy.rs b/apps/rocm/src/tool_policy.rs index d4cae125fb..9634985e45 100644 --- a/apps/rocm/src/tool_policy.rs +++ b/apps/rocm/src/tool_policy.rs @@ -254,7 +254,7 @@ const CHAT_TOOLS_SCREENING_THEIR_OWN_ARGUMENTS: [&str; 1] = ["rocm_command"]; /// /// Scope note, so the guarantee is not overread: this covers model-supplied /// *arguments*, not model-supplied prose. An assistant's reply text is still -/// printed as it comes; `strip_ansi_sequences` in `comfyui.rs` is the tool for +/// printed as it comes; `rocm_core::strip_ansi_sequences` is the tool for /// that surface, and is already used on ComfyUI log output. /// /// Walks the whole arguments object rather than naming fields, so a new diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..7219350fa8 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -27,3 +27,11 @@ ureq = { version = "2.12", features = ["native-certs"] } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } + +[dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test restores in a re-executed child. +libc.workspace = true diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 417491a716..c95ea8d24a 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -27,22 +27,7 @@ pub(crate) fn run_rocm_capture_for_paths( args: &[&str], timeout: Duration, ) -> Result { - rocm_core::run_rocm_capture_with_env( - args, - &[ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ], - timeout, - ) -} - -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } + rocm_core::run_rocm_capture_with_env(args, &paths.env_vars(), timeout) } pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index 32f1834547..ba51dcfd24 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -2,16 +2,16 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result}; use rocm_core::AppPaths; use rocm_engine_protocol::{ - EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, - HealthcheckResponse, + EngineMethod, EngineStdioOptions, HealthcheckRequest, HealthcheckResponse, + decode_engine_response, engine_request_envelope, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use std::io::Write; -use std::process::{Command as ProcessCommand, Stdio}; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; pub(crate) fn engine_healthcheck_response( paths: &AppPaths, @@ -47,7 +47,7 @@ pub(crate) fn healthcheck_response_recoverable(response: &HealthcheckResponse) - /// /// Returns whether a key was applied. Callers that spawn a *listener* must /// check it against the service's host — a public bind with no key would come -/// up anonymous (see [`ensure_public_service_has_endpoint_key`]). Callers that +/// up anonymous (see [`rocm_core::ensure_public_service_has_endpoint_key`]). Callers that /// only make a stdio plugin call can ignore it. /// /// The file must hold a *usable* key, not merely exist: the engine adapters @@ -60,49 +60,25 @@ pub(crate) fn apply_endpoint_key_env( paths: &AppPaths, service_id: &str, ) -> bool { - if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - { + if let Some(key_file) = usable_endpoint_key_file(paths, service_id) { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); return true; } false } -/// Refuse to respawn a recorded service on a public host without its endpoint -/// API key, so daemon recovery cannot reopen a protected endpoint anonymously. -/// -/// Mirrors the guard of the same name in `rocm`; the shared -/// [`rocm_engine_protocol::is_public_bind_host`] keeps the two classifications -/// identical for a given `ManagedServiceRecord::host`. -pub(crate) fn ensure_public_service_has_endpoint_key( - host: &str, - key_present: bool, - requires_api_key: bool, -) -> Result<()> { - // The bind address is not the whole story. A service bound to loopback is - // only private until something republishes the port, and a tailnet publish - // outlives both the process and this daemon. The requirement is recorded on - // the service precisely so recovery can honour it without re-deriving it - // from an address that no longer answers the question. - if requires_api_key && !key_present { - bail!( - "refusing to recover a service that was launched with an endpoint API key but no \ - longer has one: it would come back up without authentication, and something \ - outside this machine may still be publishing its port. Relaunch it with \ - `rocm serve --require-api-key` to issue a new key." - ); - } - if rocm_engine_protocol::is_public_bind_host(host) && !key_present { - bail!( - "refusing to respawn a service bound to the public host `{host}` without an endpoint \ - API key: it would come back up without authentication. Relaunch it with \ - `rocm serve --host {host} --port --allow-public-bind` to issue a new key." - ); - } - Ok(()) +/// The service's endpoint key file, when it exists and holds a usable key. +fn usable_endpoint_key_file(paths: &AppPaths, service_id: &str) -> Option { + rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) } +/// Ask an engine a question about a managed service, through the stdio client +/// `rocmd` shares with `rocm`. +/// +/// The engine is reached by running the `rocm` executable as +/// `__engine-stdio `, the verb `rocm` serves for its built-in engines +/// ([`rocm_executable`] says which executable that is). pub(crate) fn engine_request( paths: &AppPaths, engine: &str, @@ -114,19 +90,52 @@ where T: Serialize, R: DeserializeOwned, { - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request).context("failed to serialize engine request")?, - }; - let engine_binary = + let envelope = engine_request_envelope(method, request)?; + let engine_binary = rocm_executable()?; + engine_request_via(&engine_binary, paths, engine, service_id, &envelope) +} + +/// The `rocm` executable that serves `__engine-stdio`. +/// +/// `rocmd` runs in two shapes. Embedded, as `rocm daemon`, this process +/// is `rocm` and its own executable is the answer. Run as the standalone +/// `rocmd` binary shipped beside `rocm`, its own executable has no such verb +/// (its parser answers "unrecognized subcommand"), so every engine request +/// failed; there the answer is the `rocm` next to it. +fn rocm_executable() -> Result { + let current = std::env::current_exe().context("failed to resolve current rocm executable path")?; - let mut command = ProcessCommand::new(&engine_binary); - command - .arg("__engine-stdio") - .arg(engine) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + rocm_executable_from(current, || rocm_core::sibling_binary_path("rocm")) +} + +/// [`rocm_executable`] given this process's executable and the way to find a +/// sibling `rocm`, so the choice can be tested without either. +fn rocm_executable_from( + current: PathBuf, + sibling_rocm: impl FnOnce() -> Result, +) -> Result { + // Exactly `rocmd` (or `rocmd.exe`). A renamed or test binary is not known to + // lack the verb, and has no `rocm` beside it to prefer. + if current.file_stem().is_some_and(|stem| stem == "rocmd") { + return sibling_rocm().context( + "the standalone rocmd needs the rocm executable beside it to reach an engine", + ); + } + Ok(current) +} + +/// [`engine_request`] with the executable that serves `__engine-stdio` named by +/// the caller, so a test can point it at a stand-in. +fn engine_request_via( + engine_binary: &Path, + paths: &AppPaths, + engine: &str, + service_id: &str, + envelope: &rocm_engine_protocol::EngineRequestEnvelope, +) -> Result +where + R: DeserializeOwned, +{ // A protected public endpoint enforces its key on every request, including // this stdio-transported healthcheck — without the carrier the engine // adapter's HTTP probe is unauthenticated and the endpoint looks @@ -135,50 +144,33 @@ where // // No host to check here: this is a stdio plugin call, not a listener, so a // missing key only weakens this probe rather than opening a port. - let _ = apply_endpoint_key_env(&mut command, paths, service_id); - let mut child = command.spawn().with_context(|| { - format!( - "failed to spawn engine stdio process {}", - engine_binary.display() - ) - })?; - - { - let stdin = child - .stdin - .as_mut() - .context("engine stdio child did not expose stdin")?; - serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; - stdin.write_all(b"\n")?; - } + let key_file = usable_endpoint_key_file(paths, service_id); + let options = stdio_options(paths, key_file.as_deref()); + decode_engine_response(run_engine_stdio( + engine_binary, + ["__engine-stdio", engine], + &options, + envelope, + )?) +} - let output = child - .wait_with_output() - .context("failed waiting for engine stdio response")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {}", output.status); - } - bail!( - "engine stdio process exited with status {}: {}", - output.status, - stderr - ); - } - let envelope: EngineResponseEnvelope = - serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; - if !envelope.ok { - let detail = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |error| format!("{}: {}", error.code, error.message), - ); - bail!(detail); +/// What `rocmd` asks of the shared stdio client; see [`engine_request`]. +/// +/// The child runs on the daemon's own app paths and carries the service's +/// endpoint key file when there is a usable one. Its stderr is kept for the +/// error, not forwarded: nobody watches a daemon's terminal, and a healthcheck +/// asks for no progress. +const fn stdio_options<'a>( + paths: &'a AppPaths, + endpoint_key_file: Option<&'a Path>, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: Some(paths), + env_root: None, + endpoint_key_file, + request_progress: false, + forward_stderr: false, } - let data = envelope - .data - .context("engine response did not include data")?; - serde_json::from_value(data).context("failed to deserialize engine response data") } #[cfg(test)] @@ -189,23 +181,41 @@ mod tests { #[test] fn recovery_refuses_a_service_that_lost_a_key_it_was_launched_with() { - // The daemon keeps its own copy of this guard, and it only knew about - // public binds. A loopback service that something republishes — a + // The daemon once kept its own copy of this guard, and it only knew + // about public binds. A loopback service that something republishes — a // tailnet publish outlives this daemon, let alone the process — would // be recovered without authentication, and the rebuilt record would // then disarm `rocm services restart` too. for host in ["127.0.0.1", "localhost", "::1"] { - let error = ensure_public_service_has_endpoint_key(host, false, true) - .expect_err("a service launched with a key must not be recovered without one"); + let error = rocm_core::ensure_public_service_has_endpoint_key( + host, + false, + true, + rocm_core::ServiceStart::Respawn, + ) + .expect_err("a service launched with a key must not be recovered without one"); + let message = format!("{error:#}"); assert!( - format!("{error:#}").contains("without authentication"), - "{error:#}" + message.contains("respawning it would reopen it without authentication"), + "{message}" ); } // With the key still present, recovery proceeds. - ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); + rocm_core::ensure_public_service_has_endpoint_key( + "127.0.0.1", + true, + true, + rocm_core::ServiceStart::Respawn, + ) + .unwrap(); // And a service that never had one is untouched. - ensure_public_service_has_endpoint_key("127.0.0.1", false, false).unwrap(); + rocm_core::ensure_public_service_has_endpoint_key( + "127.0.0.1", + false, + false, + rocm_core::ServiceStart::Respawn, + ) + .unwrap(); } #[test] @@ -213,16 +223,81 @@ mod tests { // Daemon recovery re-execs `rocmd supervise` for the recorded host. With // the key gone the child would listen on that public host anonymously, // so the spawn must be refused instead. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + let error = rocm_core::ensure_public_service_has_endpoint_key( + "0.0.0.0", + false, + false, + rocm_core::ServiceStart::Respawn, + ) + .unwrap_err(); assert!( - error.to_string().contains("without authentication"), + error + .to_string() + .contains("respawning it would reopen it without authentication"), "{error:#}" ); - ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); + rocm_core::ensure_public_service_has_endpoint_key( + "0.0.0.0", + true, + false, + rocm_core::ServiceStart::Respawn, + ) + .unwrap(); for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false, false) - .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + rocm_core::ensure_public_service_has_endpoint_key( + host, + false, + false, + rocm_core::ServiceStart::Respawn, + ) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + } + } + + #[test] + fn the_key_file_rocm_serve_stores_clears_the_recovery_refusal() { + // The condition both refusals report is a missing *usable* key. Relaunching + // with the command they name makes `rocm serve` write the key file at the + // deterministic path, and the expression recovery uses to decide + // `key_present` must then read it as present. The parse of the named + // command itself is checked in `rocm`, which owns the clap tree, against + // the very text this guard produces + // (`the_command_a_respawn_refusal_names_clears_the_refusal`). + let (_root, paths) = temp_app_paths("recovery-refusal-cleared"); + let service_id = "svc-recovery-refusal-cleared"; + let key_present = || { + rocm_engine_protocol::endpoint_key_file_if_present(&paths, service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) + .is_some() + }; + for (host, requires_api_key) in [("127.0.0.1", true), ("0.0.0.0", false)] { + let _ = fs::remove_file(rocm_engine_protocol::endpoint_key_file_path( + &paths, service_id, + )); + assert!(!key_present()); + rocm_core::ensure_public_service_has_endpoint_key( + host, + key_present(), + requires_api_key, + rocm_core::ServiceStart::Respawn, + ) + .expect_err("no key on disk: refused"); + + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write( + rocm_engine_protocol::endpoint_key_file_path(&paths, service_id), + "freshly-issued-key\n", + ) + .unwrap(); + assert!(key_present()); + rocm_core::ensure_public_service_has_endpoint_key( + host, + key_present(), + requires_api_key, + rocm_core::ServiceStart::Respawn, + ) + .unwrap_or_else(|error| panic!("{host}: a stored key must clear it: {error:#}")); } } @@ -341,4 +416,266 @@ mod tests { }; assert!(!healthcheck_response_ready(&listing_only)); } + + // ---- What `rocmd` asks of the shared stdio client. ---- + // + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocmd`'s entry point against a stand-in engine child and pin what + // `rocmd` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocm/src/engine_ipc.rs`. + + #[test] + fn the_standalone_rocmd_reaches_engines_through_the_rocm_beside_it() { + let beside = || Ok(PathBuf::from("/opt/rocm-cli/bin/rocm")); + + // The standalone binary has no `__engine-stdio` verb, so it must not + // re-execute itself. + for standalone in ["/opt/rocm-cli/bin/rocmd", "C:/rocm-cli/bin/rocmd.exe"] { + assert_eq!( + rocm_executable_from(PathBuf::from(standalone), beside).unwrap(), + Path::new("/opt/rocm-cli/bin/rocm"), + "{standalone}" + ); + } + + // Embedded as `rocm daemon`, this process is `rocm`; a test binary + // or a renamed copy has no `rocm` beside it to prefer. + for own in [ + "/opt/rocm-cli/bin/rocm", + "C:/rocm-cli/bin/rocm.exe", + "/work/target/debug/deps/rocmd-0f3a9c1d", + "/opt/rocm-cli/bin/rocm-0.3", + ] { + let chosen = rocm_executable_from(PathBuf::from(own), || { + panic!("{own} has no use for a sibling") + }) + .unwrap(); + assert_eq!(chosen, Path::new(own)); + } + + // A standalone `rocmd` with no `rocm` beside it fails with a message that + // says why, rather than running something that cannot serve the verb. + let error = rocm_executable_from(PathBuf::from("/opt/rocm-cli/bin/rocmd"), || { + anyhow::bail!("unable to locate sibling binary rocm") + }) + .unwrap_err(); + assert_eq!( + format!("{error:#}"), + "the standalone rocmd needs the rocm executable beside it to reach an engine: \ + unable to locate sibling binary rocm" + ); + } + + #[test] + fn rocmd_runs_the_child_on_its_own_paths_keeps_stderr_and_asks_for_no_progress() { + let (_root, paths) = temp_app_paths("stdio-options"); + let key_file = Path::new("/keys/svc.endpoint-key"); + + let options = stdio_options(&paths, Some(key_file)); + assert_eq!( + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) + ); + assert_eq!(options.endpoint_key_file, Some(key_file)); + // A healthcheck has no install to report progress on, and nobody watches + // a daemon's terminal. + assert!(!options.request_progress); + assert!(!options.forward_stderr); + assert_eq!(options.env_root, None); + + assert_eq!(stdio_options(&paths, None).endpoint_key_file, None); + } + + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::EngineRequestEnvelope; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const SERVICE_ID: &str = "svc-stdio-child"; + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn healthcheck_envelope() -> EngineRequestEnvelope { + EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({"service_id": SERVICE_ID}), + } + } + + fn request(engine: &FakeEngine, paths: &AppPaths) -> Result { + engine_request_via::( + engine.path(), + paths, + "vllm", + SERVICE_ID, + &healthcheck_envelope(), + ) + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_is_asked_to_serve_the_engine_over_stdio_with_one_line_of_request() { + let (_root, paths) = temp_app_paths("stdio-framing"); + let engine = FakeEngine::answering("rocmd-framing", STOPPED, 0); + let answer = request(&engine, &paths).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["__engine-stdio", "vllm"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"healthcheck\",\"payload\":{\"service_id\":\"svc-stdio-child\"}}\n" + ); + } + + #[test] + fn the_endpoint_key_file_reaches_the_child_only_when_it_holds_a_usable_key() { + let (_root, paths) = temp_app_paths("stdio-endpoint-key"); + let key_env = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + + let keyless = FakeEngine::answering("rocmd-keyless", STOPPED, 0); + request(&keyless, &paths).unwrap(); + assert_eq!(keyless.env(key_env), ambient(key_env)); + + let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, SERVICE_ID); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write(&key_path, " \n").unwrap(); + let unusable = FakeEngine::answering("rocmd-unusable-key", STOPPED, 0); + request(&unusable, &paths).unwrap(); + assert_eq!(unusable.env(key_env), ambient(key_env)); + + fs::write(&key_path, "secret-key").unwrap(); + let keyed = FakeEngine::answering("rocmd-keyed", STOPPED, 0); + request(&keyed, &paths).unwrap(); + assert_eq!(keyed.env(key_env).as_deref(), key_path.to_str()); + } + + #[test] + fn the_child_runs_on_the_daemons_app_paths() { + let (_root, paths) = temp_app_paths("stdio-app-paths"); + let engine = FakeEngine::answering("rocmd-app-paths", STOPPED, 0); + request(&engine, &paths).unwrap(); + assert_eq!( + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() + ); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let (_root, paths) = temp_app_paths("stdio-nonzero"); + let answered = FakeEngine::answering("rocmd-answers-then-fails", STOPPED, 2); + assert_eq!( + request(&answered, &paths).unwrap(), + json!({"stopped": true}) + ); + + let failed = FakeEngine::answering( + "rocmd-error-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + assert_eq!( + request(&failed, &paths).unwrap_err().to_string(), + "stop_failed: it broke" + ); + } + + #[test] + fn a_crash_with_no_answer_reports_the_status_and_stderr() { + let (_root, paths) = temp_app_paths("stdio-crash"); + let crash = FakeEngine::new("rocmd-crash", true, "echo boom >&2\nexit 3"); + assert_eq!( + request(&crash, &paths).unwrap_err().to_string(), + "engine stdio process exited with status exit status: 3: boom" + ); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let (_root, paths) = temp_app_paths("stdio-garbage"); + let garbage = + FakeEngine::new("rocmd-garbage", true, "printf 'not json'\necho oops >&2"); + assert_eq!( + request(&garbage, &paths).unwrap_err().to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + } + + /// A request to a child that has already exited is an ordinary error + /// wherever SIGPIPE stands: ignored (Rust's startup default, which the + /// test harness keeps) or at its default action, which is how `rocm` + /// runs and so how `rocm daemon` runs this code. Re-executes this + /// test in a fresh process for each case, since changing the disposition + /// process-wide would race sibling tests. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition() { + if let Some(arm) = std::env::var_os("ROCMD_TEST_ENGINE_SIGPIPE_ARM") { + if arm == "default-action" { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + } + let (_root, paths) = temp_app_paths("stdio-sigpipe"); + let engine = FakeEngine::new("rocmd-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let envelope = EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({ "blob": big }), + }; + let result = engine_request_via::( + engine.path(), + &paths, + "vllm", + SERVICE_ID, + &envelope, + ); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let run_arm = |arm: &str| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition", + "--exact", + "--nocapture", + ]) + .env("ROCMD_TEST_ENGINE_SIGPIPE_ARM", arm) + .status() + .unwrap() + }; + + let ignored = run_arm("ignored"); + assert!(ignored.success(), "{ignored:?}"); + + let default_action = run_arm("default-action"); + assert_eq!( + default_action.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {default_action:?}" + ); + assert!(default_action.success(), "{default_action:?}"); + } + } } diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index 2b8bca1bed..5d81d9ffc2 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -18,8 +18,6 @@ use rocm_core::{ }; use serde_json::Value; use serde_json::json; -use std::collections::VecDeque; -use std::fs; use std::io::{self, BufRead, Write}; use std::time::Duration; @@ -631,16 +629,14 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result Result { - let content = - fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut lines = VecDeque::with_capacity(limit); - for line in content.lines() { - if lines.len() == limit { - lines.pop_front(); - } - lines.push_back(line.to_owned()); - } - Ok(lines.into_iter().collect::>().join("\n")) +/// The last `limit` lines of a service log, read through the shared bounded +/// tail routine, with what the byte budget left unread. +fn read_log_tail(path: &std::path::Path, limit: usize) -> Result { + rocm_core::tail_log_decoded( + path, + limit, + rocm_core::LOG_VIEW_MAX_TAIL_READ, + rocm_core::LogDecode::PLAIN, + ) + .with_context(|| format!("failed to read {}", path.display())) } #[cfg(test)] @@ -869,6 +864,12 @@ mod tests { use std::fs; use std::path::PathBuf; + /// [`read_log_tail`], joined with newlines, for the tests that only look + /// at the text. + fn read_tail_lines(path: &std::path::Path, limit: usize) -> Result { + Ok(read_log_tail(path, limit)?.lines.join("\n")) + } + fn unique_test_path(label: &str) -> PathBuf { let root = workspace_test_artifact_dir(); fs::create_dir_all(&root).expect("create workspace-local test dir"); @@ -937,6 +938,117 @@ mod tests { .expect("explicitly approved direct MCP mutation should pass the helper guard"); } + /// Call the `service_logs` tool for a service whose log holds `log` (no log + /// file at all for `None`), and return the tool's reply with the log's size + /// on disk, so a claim the reply makes can be checked against the file. + fn service_logs_reply(label: &str, log: Option<&[u8]>) -> Result<(Value, Option)> { + let (root, paths) = temp_app_paths(label); + paths.ensure()?; + let service_id = format!("svc-{label}"); + let record = rocm_core::ManagedServiceRecord::new( + &paths, + &service_id, + "llamacpp", + "a-model", + "a-model", + "127.0.0.1", + 18080, + "managed", + std::process::id(), + None, + None, + None, + ); + record.write()?; + let size = match log { + Some(bytes) => { + fs::write(&record.log_path, bytes)?; + Some(fs::metadata(&record.log_path)?.len()) + } + None => None, + }; + let reply = handle_mcp_tool_call( + &paths, + &json!({"name": "service_logs", "arguments": {"service_id": service_id}}), + ); + let _ = fs::remove_dir_all(root); + Ok((reply?, size)) + } + + #[test] + fn service_logs_tell_an_empty_log_from_one_whose_newest_line_exceeds_the_read_bound() + -> Result<()> { + let bound = usize::try_from(rocm_core::LOG_VIEW_MAX_TAIL_READ)?; + let notice = "the newest line is longer than the 4 MiB a log view reads"; + assert_eq!( + rocm_core::LOG_VIEW_MAX_TAIL_READ, + 4 * 1024 * 1024, + "the notice asserted below names this many MiB" + ); + + // An empty log: nothing was written, and the reply says so. + let (reply, size) = service_logs_reply("service-logs-empty", Some(b""))?; + assert_eq!(size, Some(0)); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert_eq!( + text, "Service `svc-service-logs-empty` has no log output yet.", + "{reply}" + ); + assert_eq!(reply["structuredContent"]["tail"], ""); + assert_eq!( + reply["structuredContent"]["newest_line_exceeds_read_bound"], + false + ); + + // One line longer than the bound, with and without a final newline: the + // file has content, so "no output" or "Read the last 0 line(s)" would be + // false, and the structured reply carries the same distinction. + let one_line = vec![b'x'; bound + 1024]; + let mut with_newline = one_line.clone(); + with_newline.push(b'\n'); + for (name, bytes) in [ + ("service-logs-long", one_line), + ("service-logs-long-nl", with_newline), + ] { + let (reply, size) = service_logs_reply(name, Some(&bytes))?; + assert!( + size.is_some_and(|size| size > rocm_core::LOG_VIEW_MAX_TAIL_READ), + "{name}: {size:?}" + ); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert_eq!( + text, + format!("Service `svc-{name}` has log output, but none is shown: {notice}."), + "{name}: {reply}" + ); + assert_eq!(reply["structuredContent"]["tail"], "", "{name}"); + assert_eq!( + reply["structuredContent"]["newest_line_exceeds_read_bound"], true, + "{name}" + ); + assert_eq!(reply["isError"], false, "{name}"); + } + + // Short lines are read, and the count is the number returned. + let (reply, _) = service_logs_reply("service-logs-two", Some(b"a\nb\n"))?; + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert_eq!( + text, + "Read the last 2 line(s) from service `svc-service-logs-two`." + ); + assert_eq!(reply["structuredContent"]["tail"], "a\nb"); + assert_eq!( + reply["structuredContent"]["newest_line_exceeds_read_bound"], + false + ); + + // No log file is an error, not an empty tail. + let error = service_logs_reply("service-logs-absent", None) + .expect_err("a missing log must not read as empty"); + assert!(format!("{error:#}").contains("failed to read"), "{error:#}"); + Ok(()) + } + #[test] fn read_tail_lines_returns_last_lines_only() -> Result<()> { let path = unique_test_path(&format!( @@ -951,6 +1063,151 @@ mod tests { Ok(()) } + fn tail_fixture(label: &str, bytes: &[u8]) -> Result { + let path = unique_test_path(&format!( + "rocmd-tail-{label}-{}.log", + crate::test_support::unique_suffix() + )); + fs::write(&path, bytes)?; + Ok(path) + } + + /// 5 MiB: a marked first line, 5120 lines of 1 KiB, a marked last line. + fn large_log_with_head_marker() -> Vec { + let mut bytes = b"HEAD-MARKER\n".to_vec(); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..5120 { + bytes.extend_from_slice(filler.as_bytes()); + } + bytes.extend_from_slice(b"TAIL-MARKER\n"); + bytes + } + + #[test] + fn read_tail_lines_a_large_file_is_read_through_a_bounded_window() -> Result<()> { + // A line far outside the tail window is not returned however large the + // limit: the log is not read whole into one string. + let path = tail_fixture("large", &large_log_with_head_marker())?; + + let tail = read_tail_lines(&path, 10_000)?; + fs::remove_file(&path)?; + + assert!(!tail.lines().any(|line| line == "HEAD-MARKER")); + assert!(tail.ends_with("\nTAIL-MARKER")); + assert!(tail.lines().count() < 5122); + assert!(tail.lines().count() > 3500, "the window holds about 4 MiB"); + Ok(()) + } + + #[test] + fn read_tail_lines_shows_a_sixteen_bit_log_as_lossy_text_instead_of_failing() -> Result<()> { + // The tool reads the child's own bytes as UTF-8; a UTF-16 log is not + // detected here (only the ComfyUI log view does that), but it no + // longer fails the whole tool call. + let mut bytes = vec![0xff, 0xfe]; + for unit in "[INFO] one\r\n".encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + let path = tail_fixture("utf16", &bytes)?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert!( + tail.starts_with("\u{FFFD}\u{FFFD}["), + "the byte order mark is replaced, the rest kept: {tail:?}" + ); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_ansi_colour_codes() -> Result<()> { + let path = tail_fixture("ansi", b"\x1b[32mgreen\x1b[0m done\n")?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "\u{1b}[32mgreen\u{1b}[0m done"); + Ok(()) + } + + #[test] + fn read_tail_lines_decodes_invalid_eight_bit_text_lossily() -> Result<()> { + let path = tail_fixture("invalid", b"ok\nbad\xffbyte\nlast\n")?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "ok\nbad\u{FFFD}byte\nlast"); + Ok(()) + } + + #[test] + fn read_tail_lines_missing_file_is_a_labelled_read_error() { + let path = unique_test_path(&format!( + "rocmd-tail-missing-{}.log", + crate::test_support::unique_suffix() + )); + + let error = read_tail_lines(&path, 5).expect_err("a missing log must not read"); + + assert_eq!( + error.to_string(), + format!("failed to read {}", path.display()) + ); + assert!( + format!("{error:#}").contains(&format!("failed to open {}", path.display())), + "the cause names the failing step: {error:#}" + ); + } + + #[test] + fn read_tail_lines_limit_zero_returns_nothing() -> Result<()> { + // With limit 0 the old reader never evicted and returned the whole + // file. The tool handler clamps the limit to 1..=500, so only a + // direct call could see it. + let path = tail_fixture("zero", b"a\nb\nc\n")?; + + let tail = read_tail_lines(&path, 0)?; + fs::remove_file(&path)?; + + assert_eq!(tail, ""); + Ok(()) + } + + #[test] + fn read_tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let path = tail_fixture("few", b"a\nb\n")?; + + let tail = read_tail_lines(&path, 50)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "a\nb"); + Ok(()) + } + + #[test] + fn read_tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let path = tail_fixture("no-newline", b"a\nb\nc")?; + + let tail = read_tail_lines(&path, 2)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "b\nc"); + Ok(()) + } + + #[test] + fn read_tail_lines_strips_crlf_line_endings() -> Result<()> { + let path = tail_fixture("crlf", b"a\r\nb\r\n")?; + + let tail = read_tail_lines(&path, 5)?; + fs::remove_file(&path)?; + + assert_eq!(tail, "a\nb"); + Ok(()) + } + fn mcp_stop_server(paths: &AppPaths, service_id: &str) -> Result { handle_mcp_tool_call( paths, diff --git a/apps/rocmd/src/mcp_argv.rs b/apps/rocmd/src/mcp_argv.rs index a052beea42..c7a0d6604c 100644 --- a/apps/rocmd/src/mcp_argv.rs +++ b/apps/rocmd/src/mcp_argv.rs @@ -305,10 +305,10 @@ pub(crate) fn build_watcher_enable_args( Ok(argv) } -/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the -/// policy so `rocm` and `rocmd` never classify the same host differently. +/// Inverse of [`rocm_core::is_public_bind_host`], which owns the policy so +/// `rocm` and `rocmd` never classify the same host differently. fn is_loopback_host(host: &str) -> bool { - !rocm_engine_protocol::is_public_bind_host(host) + !rocm_core::is_public_bind_host(host) } fn system_prefix_requires_ack(prefix: &std::path::Path) -> bool { diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 41e7fcf844..441c0b696d 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -7,12 +7,9 @@ use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use serde_json::Value; use serde_json::json; -#[cfg(windows)] -use std::process::{Command as ProcessCommand, Stdio}; -use std::time::Duration; pub(crate) fn stop_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_managed_service_with(paths, service_id, terminate_recorded_pid) + stop_managed_service_with(paths, service_id, rocm_core::terminate_recorded_pid) } fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result { @@ -23,8 +20,8 @@ fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result Result { let mut stopping = find_managed_service(paths, service_id)?; // Record the request before the stop starts, not only after it ends. A stop - // can take up to `MANAGED_STOP_GRACE` twice per recorded PID, and for that + // can take up to `rocm_core::MANAGED_STOP_GRACE` twice per recorded PID, and for that // whole window the service still reads `ready`/`running` while its endpoint // stops answering — exactly what the daemon's `server-recover` watcher // restarts. The marker is what that watcher skips (`stop_requested`), so it @@ -73,32 +70,27 @@ pub(crate) fn stop_managed_service_via( // container-backed engine (`aim`) this is the only thing that stops the // container, since its recorded PIDs are only the supervisor that follows it. let engine_stop = (stopping.engine != "lemonade").then(|| engine_stop(paths, &stopping)); - let stops = terminate_recorded_service_pids(&stopping, terminate); - let pids_where = |keep: fn(&RecordedPidStop) -> bool| -> Vec { + let stops = rocm_core::terminate_recorded_service_pids(&stopping, terminate); + let pids_where = |keep: fn(rocm_core::RecordedPidStop) -> bool| -> Vec { stops .iter() - .filter(|stop| keep(stop)) - .map(|s| s.pid) + .copied() + .filter(|stop| keep(*stop)) + .map(|stop| stop.pid) .collect() }; - let signaled_pids = pids_where(|stop| stop.signaled); - let force_signaled_pids = pids_where(|stop| stop.forced); + let signaled_pids = pids_where(rocm_core::RecordedPidStop::signaled); + let force_signaled_pids = pids_where(rocm_core::RecordedPidStop::forced); // Everything that was deliberately not signalled, which is no longer one // kind of thing: the stopping process itself; a PID that was already gone; // a PID that is live but provably belongs to somebody else now; and a live // PID whose identity could not be read at all, which may still be the engine // and may still hold the device. Only `pid_outcomes` separates them, and // only `stopped` reports that the last of those is not a completed stop. - let skipped_pids = pids_where(|stop| !stop.signaled); + let skipped_pids = pids_where(|stop| !stop.signaled()); let pid_outcomes = stops .iter() - .map(|stop| { - json!({ - "pid": stop.pid, - "role": stop.role, - "outcome": stop.outcome, - }) - }) + .map(|stop| stop.report_entry()) .collect::>(); // Re-read the record rather than write back the copy loaded before the // stop: `write` replaces the whole file, and other writers — a supervisor @@ -113,7 +105,7 @@ pub(crate) fn stop_managed_service_via( // would each be false. let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.as_ref().is_some_and(Result::is_ok); - let all_stopped = stops.iter().all(|stop| stop.stopped) + let all_stopped = stops.iter().all(|stop| stop.stopped()) && stopping.names_same_processes(&record) && engine_stop_confirms; // A PID whose identity cannot be read at all — `/proc//stat` @@ -135,11 +127,9 @@ pub(crate) fn stop_managed_service_via( record.stop_requested_unix_ms = None; // Recorded PIDs go stale the instant their processes exit, and a stale // PID is exactly what the identity check exists to catch — so do not - // leave one behind for the next stop to find. - record.supervisor_pid = 0; - record.supervisor_start_ticks = None; - record.engine_pid = None; - record.engine_start_ticks = None; + // leave one behind for the next stop to find. The same call + // `rocm services stop` makes on its confirmed path. + record.clear_recorded_processes(); } else { // The recorded PIDs stay: an unconfirmed survivor is still the // service's, and the record is what names it. @@ -208,179 +198,6 @@ pub(crate) fn stop_report_confirmed(report: &Value) -> bool { report.get("stopped").and_then(Value::as_bool) == Some(true) } -/// How long a stop waits for a recorded process tree to exit on its own before -/// escalating to a forced kill. Matches `rocm services stop`, which terminates -/// the same records. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// What a stop did about one PID recorded in a service manifest. -struct RecordedPidStop { - pid: u32, - /// Which recorded PID this is: `supervisor`, `engine`, or - /// `supervisor_and_engine` when the record holds the same PID for both. - role: &'static str, - /// Stable label for what happened, from - /// [`rocm_core::TerminationOutcome::as_str`], plus `self` for the stopping - /// process's own PID. - outcome: &'static str, - signaled: bool, - forced: bool, - /// Whether the recorded process is confirmed to be no longer running. - stopped: bool, -} - -/// Terminate the processes a service manifest records, verifying identity first. -/// -/// PIDs are recycled, so a persisted PID is not by itself evidence that the -/// recorded process is still the one holding it. Each PID is therefore paired -/// with **its own** start-time token — `supervisor_start_ticks` with -/// `supervisor_pid`, `engine_start_ticks` with `engine_pid` — and routed through -/// [`rocm_core::terminate_verified`], which signals nothing when the recorded -/// identity is refuted. `rocm services stop` reads the same records through the -/// same primitive, so neither command can decide differently about one. -/// -/// Descendants are reached from the *verified* root rather than expanded out of -/// a live process listing keyed on a PID nobody checked — which, given a stale -/// root, enumerated a stranger's children and signalled those too. Where the -/// kernel start-time is readable, `terminate_verified` additionally binds each -/// child to its own identity before any forced kill; see -/// [`terminate_recorded_pid`] for what Windows can and cannot do here. -/// -/// The stopping process's own PID is reported but never signalled, and does not -/// count towards "the recorded service is gone": `rocmd` records itself as the -/// supervisor of the services it launches, and it is the launcher, not the -/// service. -/// -/// `terminate` performs the verified termination of one recorded PID; -/// production passes [`terminate_recorded_pid`]. -fn terminate_recorded_service_pids( - record: &ManagedServiceRecord, - terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, -) -> Vec { - // Build the (pid, role, own-token) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. The two PIDs can - // coincide — a launcher that is also the server — and a token must never be - // read across from the PID it does not belong to. - let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); - for (pid, role, ticks) in [ - ( - Some(record.supervisor_pid), - "supervisor", - record.supervisor_start_ticks, - ), - (record.engine_pid, "engine", record.engine_start_ticks), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - // Only the engine entry can land here, aliasing the supervisor: the - // one PID is both, and is reported as both. - existing.1 = "supervisor_and_engine"; - if existing.2.is_none() { - existing.2 = ticks; - } - } else { - entries.push((pid, role, ticks)); - } - } - - entries - .into_iter() - .map(|(pid, role, ticks)| { - if pid == std::process::id() { - return RecordedPidStop { - pid, - role, - outcome: "self", - signaled: false, - forced: false, - stopped: true, - }; - } - let outcome = terminate(&rocm_core::ProcessIdentity::new(pid, ticks)); - RecordedPidStop { - pid, - role, - outcome: outcome.as_str(), - // TimedOut belongs with the signalled: the signal went out, the - // exit was just never observed. `stopped` carries that truth. - signaled: matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ), - // "The stop escalated to a forced kill", not "the forced kill - // worked": a forced stop only reaches `TimedOut` by way of that - // escalation, and `stopped` is what says whether the process - // actually went away. - forced: matches!( - outcome, - rocm_core::TerminationOutcome::Forced | rocm_core::TerminationOutcome::TimedOut - ), - stopped: outcome.stopped(), - } - }) - .collect() -} - -/// Terminate one recorded process after verifying its identity. -/// -/// The stop is forced: its contract is definitive, so the command must not -/// report success while an engine worker still holds the GPU. Only a process -/// whose recorded identity is confirmed is ever reached by it. -fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome { - // Windows needs the descendants taken separately. `rocm_core` has no way to - // walk a process tree there — `process_tree_pids` returns just the root — and - // the engine runs one level below the recorded PID, because `rocmd` launches - // it through an `__engine-serve-http` process. A root-only kill would leave - // the engine running and still holding the device. - // - // Identity verification does not apply on Windows. `process_start_ticks` - // has no `/proc` to read there and returns `None`, so no recorded identity - // can be refuted and the gate below only ever turns away a PID that is not - // running at all. A recorded PID that has been recycled by an unrelated - // process still passes it, and `taskkill /T /F` then takes that process and - // its whole subtree: the stale-PID hazard is unchanged on Windows, exactly - // as it was before this gate existed. Closing it needs `rocm_core` to read a - // Windows process creation time. - #[cfg(windows)] - if matches!( - rocm_core::identity_state(identity), - rocm_core::IdentityState::Matches - ) { - // Taken while the root is still alive: `taskkill /T` resolves children - // through the live parent, so it cannot reach them once the root is gone. - let _ = ProcessCommand::new("taskkill") - .args(["/PID", &identity.pid.to_string(), "/T", "/F"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - // `terminate_verified` still owns the bounded wait and the verdict, but - // a root the tree kill already took now reads back as `AlreadyGone` — - // which would deny a stop this command did perform. - let outcome = rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ); - return if matches!(outcome, rocm_core::TerminationOutcome::AlreadyGone) { - rocm_core::TerminationOutcome::Forced - } else { - outcome - }; - } - rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) -} - #[cfg(test)] mod tests { use super::*; @@ -510,10 +327,11 @@ mod tests { /// a different process: the recorded service already exited and its PID was /// recycled. The stop must refuse it rather than signal a stranger. /// - /// Linux-only: a refutable identity needs a readable start-time, and - /// `rocm_core::process_start_ticks` reads it from `/proc`. On Windows it is - /// always `None`, so no identity can be refuted there and the state this - /// asserts on cannot be constructed. + /// Linux-only: it stages the bystander with `sleep` and offsets its token + /// read from `/proc`. The same refusal on Windows, where the token is the + /// process creation time, is asserted in `rocm-core`'s `proc_lifecycle` + /// tests (`terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` + /// and its Windows-only neighbour) and by scenario `service-stop-09`. #[cfg(target_os = "linux")] #[test] fn stop_managed_service_refuses_a_pid_whose_recorded_identity_no_longer_matches() -> Result<()> @@ -1080,4 +898,199 @@ mod tests { ); Ok(()) } + + // ---- This module's stop, through the routine it shares with `rocm`. ---- + // + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. The termination is a stand-in throughout, so + // no process is signalled and the tests behave the same on every host. + + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + /// Plant a record naming `supervisor` and `engine`, with the given start-time + /// tokens, and return the paths it lives under. + fn plant_record_naming( + name: &str, + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> Result<(std::path::PathBuf, AppPaths, &'static str)> { + let (root, paths) = temp_app_paths(name); + paths.ensure()?; + let service_id = "svc-stop-through-the-shared-routine"; + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record.status = "ready".to_owned(); + record.write()?; + Ok((root, paths, service_id)) + } + + fn pids_listed(report: &Value, key: &str) -> Vec { + report[key] + .as_array() + .unwrap_or_else(|| panic!("no `{key}` in {report}")) + .iter() + .filter_map(Value::as_u64) + .collect() + } + + #[test] + fn a_stop_terminates_the_supervisor_then_the_engine_each_with_its_own_start_time() -> Result<()> + { + let (root, paths, service_id) = plant_record_naming( + "stop-order", + SUPERVISOR_PID, + Some(11), + Some(ENGINE_PID), + Some(22), + )?; + let seen = std::cell::RefCell::new(Vec::new()); + + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Graceful + }); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!( + seen.into_inner(), + vec![ + rocm_core::ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + rocm_core::ProcessIdentity::new(ENGINE_PID, Some(22)), + ] + ); + assert_eq!( + report["pid_outcomes"], + json!([ + {"pid": SUPERVISOR_PID, "role": "supervisor", "outcome": "graceful"}, + {"pid": ENGINE_PID, "role": "engine", "outcome": "graceful"}, + ]) + ); + assert_eq!( + pids_listed(&report, "signaled_pids"), + vec![u64::from(SUPERVISOR_PID), u64::from(ENGINE_PID)] + ); + Ok(()) + } + + /// The stopping process's own PID is reported as `self`, never handed to the + /// termination, and does not stand in the way of a confirmed stop. + #[test] + fn a_stop_reports_its_own_pid_as_self_and_skips_it() -> Result<()> { + let own = std::process::id(); + let (root, paths, service_id) = + plant_record_naming("stop-own-pid", own, None, Some(ENGINE_PID), None)?; + let seen = std::cell::RefCell::new(Vec::new()); + + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(identity.pid); + rocm_core::TerminationOutcome::Graceful + }); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!(seen.into_inner(), vec![ENGINE_PID]); + assert_eq!( + report["pid_outcomes"][0], + json!({"pid": own, "role": "supervisor", "outcome": "self"}) + ); + assert_eq!(pids_listed(&report, "skipped_pids"), vec![u64::from(own)]); + assert_eq!( + pids_listed(&report, "signaled_pids"), + vec![u64::from(ENGINE_PID)] + ); + assert_eq!(report["stopped"], true); + Ok(()) + } + + #[test] + fn a_stop_sorts_each_outcome_into_the_signaled_forced_and_skipped_lists() -> Result<()> { + use rocm_core::TerminationOutcome::{ + AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, + }; + // (outcome, signaled, forced, skipped, stopped) + let expected = [ + (AlreadyGone, false, false, true, true), + (IdentityMismatch, false, false, true, true), + (Unverified, false, false, true, false), + (Graceful, true, false, false, true), + (Forced, true, true, false, true), + (TimedOut, true, true, false, false), + ]; + for (outcome, signaled, forced, skipped, stopped) in expected { + let (root, paths, service_id) = + plant_record_naming("stop-classes", SUPERVISOR_PID, None, None, None)?; + + let report = stop_managed_service_with(&paths, service_id, |_| outcome); + fs::remove_dir_all(root).ok(); + + let report = report?; + let listed = |key| !pids_listed(&report, key).is_empty(); + assert_eq!( + ( + listed("signaled_pids"), + listed("force_signaled_pids"), + listed("skipped_pids"), + report["stopped"].as_bool() + ), + (signaled, forced, skipped, Some(stopped)), + "{outcome:?}: {report}" + ); + } + Ok(()) + } + + #[test] + fn the_production_stop_reports_an_absent_pid_as_already_gone() -> Result<()> { + let (root, paths, service_id) = + plant_record_naming("stop-production-absent", SUPERVISOR_PID, None, None, None)?; + + let report = stop_managed_service(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!(report["pid_outcomes"][0]["outcome"], "already_gone"); + assert_eq!(report["stopped"], true); + Ok(()) + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } + } } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index eb594e2167..c6e057cc89 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -101,12 +101,13 @@ pub(crate) fn supervise_service( // attempt leaves the recorded restart_count and timestamps intact instead of // clobbering them with a record no live process will ever back. The spawn // site below re-checks against the key actually threaded onto the command. - crate::engine_ipc::ensure_public_service_has_endpoint_key( + rocm_core::ensure_public_service_has_endpoint_key( &record.host, rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) .is_some(), record.requires_api_key, + rocm_core::ServiceStart::Respawn, )?; record.write()?; @@ -146,10 +147,11 @@ pub(crate) fn supervise_service( // an anonymous public one is not. let endpoint_key_applied = crate::engine_ipc::apply_endpoint_key_env(&mut command, paths, &record.service_id); - crate::engine_ipc::ensure_public_service_has_endpoint_key( + rocm_core::ensure_public_service_has_endpoint_key( &record.host, endpoint_key_applied, record.requires_api_key, + rocm_core::ServiceStart::Respawn, )?; let mut child = command .spawn() @@ -277,9 +279,12 @@ fn engine_serve_http_args( if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); - args.extend(crate::common::optional_arg("--env-id", env_id)); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( + "--runtime-id", + runtime_id, + )); + args.extend(rocm_engine_protocol::optional_arg("--env-id", env_id)); + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", engine_recipe_json, )); @@ -805,18 +810,18 @@ fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { .unwrap_or("gpu_required") .to_owned(), ]; - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--runtime-id", record.runtime_id.as_deref(), )); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--env-id", record.env_id.as_deref(), )); if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", record.engine_recipe_json.as_deref(), )); @@ -855,7 +860,7 @@ mod tests { /// call site testable at all. The arguments below are the shape a recovery /// re-exec passes: a loopback bind, no GPU, no recipe. /// - /// This exists because testing `ensure_public_service_has_endpoint_key` + /// This exists because testing `rocm_core::ensure_public_service_has_endpoint_key` /// directly with literal arguments cannot catch the defect that actually /// happened twice in this crate's history — the guard being *wired up* with /// the wrong value at its call site. @@ -866,8 +871,14 @@ mod tests { /// suite instead of failing it. Both callers below are regression tests for /// a fail-*open*, which is exactly the shape that turns into a hang. fn supervise_at_the_guard(paths: &AppPaths, service_id: &str) -> Result<()> { + supervise_at_the_guard_on(paths, service_id, "127.0.0.1") + } + + /// [`supervise_at_the_guard`] for a service recorded on `host`. + fn supervise_at_the_guard_on(paths: &AppPaths, service_id: &str, host: &str) -> Result<()> { let paths = paths.clone(); let service_id = service_id.to_owned(); + let host = host.to_owned(); let (sender, receiver) = std::sync::mpsc::channel(); std::thread::spawn(move || { let outcome = supervise_service( @@ -878,7 +889,7 @@ mod tests { "a-model".to_owned(), None, None, - "127.0.0.1".to_owned(), + host, 11434, "gpu_required".to_owned(), None, @@ -1000,8 +1011,22 @@ mod tests { let error = supervise_at_the_guard(&paths, "svc-needs-key") .expect_err("a service that required a key must not be recovered without one"); + // The daemon's refusal is the shared guard's, worded for a respawn: the + // same text `rocm` prints for a restart, bar the action. A private copy + // here, or the wrong action passed, would read differently. + assert_eq!( + format!("{error:#}"), + rocm_core::ensure_public_service_has_endpoint_key( + "127.0.0.1", + false, + true, + rocm_core::ServiceStart::Respawn, + ) + .unwrap_err() + .to_string() + ); assert!( - format!("{error:#}").contains("without authentication"), + format!("{error:#}").contains("respawning it would reopen it without authentication"), "{error:#}" ); @@ -1021,6 +1046,37 @@ mod tests { let _ = fs::remove_dir_all(&root); } + #[test] + fn supervising_a_public_service_refuses_when_its_key_is_gone_whether_or_not_it_asked_for_one() { + // The second rule at the real call site: the bind address alone makes a + // keyless respawn unsafe, so a service that never passed + // `--require-api-key` is refused too, and is not told it did. + let (root, paths) = temp_app_paths("supervise-public-no-key"); + seed_registry(&paths, "svc-public", false); + + let error = supervise_at_the_guard_on(&paths, "svc-public", "0.0.0.0") + .expect_err("a public service must not be respawned without its key"); + let rendered = format!("{error:#}"); + assert_eq!( + rendered, + rocm_core::ensure_public_service_has_endpoint_key( + "0.0.0.0", + false, + false, + rocm_core::ServiceStart::Respawn, + ) + .unwrap_err() + .to_string() + ); + assert!( + rendered.contains("bound to the public host `0.0.0.0`"), + "{rendered}" + ); + assert!(!rendered.contains("`--require-api-key`"), "{rendered}"); + + let _ = fs::remove_dir_all(&root); + } + #[test] fn a_registry_that_cannot_be_read_refuses_recovery_rather_than_assuming_no_key() { // `load_managed_services` already skips records it cannot parse, so an diff --git a/apps/rocmd/src/watchers/server_recover.rs b/apps/rocmd/src/watchers/server_recover.rs index f43402a4e1..d0c30f2dda 100644 --- a/apps/rocmd/src/watchers/server_recover.rs +++ b/apps/rocmd/src/watchers/server_recover.rs @@ -205,12 +205,13 @@ fn handle_server_recover_event_with_record( // Report it and stop, rather than letting a permanent failure // propagate out of `evaluate_watchers` and take the whole daemon — // and every other watcher — down on each 30s recovery tick. - if let Err(error) = crate::engine_ipc::ensure_public_service_has_endpoint_key( + if let Err(error) = rocm_core::ensure_public_service_has_endpoint_key( &record.host, rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) .is_some(), record.requires_api_key, + rocm_core::ServiceStart::Respawn, ) { return crate::persistence::record_event( paths, @@ -920,6 +921,65 @@ mod tests { Ok(()) } + /// A public service whose key is gone is reported, not respawned, and the + /// report is the shared guard's refusal worded for a respawn — the same + /// text `rocm services restart` prints, bar the action. + #[test] + fn server_recover_refuses_a_keyless_public_service_in_the_guards_words() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-keyless-public"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-public-no-key", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "0.0.0.0", + 11452, + "managed", + 123, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Contained, + None, + )]); + + evaluate_server_recover(&paths, WatcherMode::Contained, &mut state)?; + let reported = state.active_watchers[0].last_event.clone(); + let reloaded = load_service_record(&paths, "svc-public-no-key"); + fs::remove_dir_all(root).ok(); + + let expected = rocm_core::ensure_public_service_has_endpoint_key( + "0.0.0.0", + false, + false, + rocm_core::ServiceStart::Respawn, + ) + .unwrap_err() + .to_string(); + let reported = reported.expect("a refused recovery is reported"); + assert!( + reported + .starts_with("cannot recover managed service svc-public-no-key on 0.0.0.0:11452"), + "{reported}" + ); + assert!( + reported.ends_with(&expected), + "the report must carry the guard's own refusal: {reported}" + ); + // The refusal came before any restart: the record was not touched. + let reloaded = reloaded?; + assert_eq!(reloaded.status, "failed"); + assert_eq!(reloaded.last_restart_unix_ms, None); + Ok(()) + } + /// An unconfirmed stop leaves the status at `ready` on purpose, which is /// exactly what the daemon's recovery scan looks for. The stop request it /// records is the only thing telling it apart from a crash, so recovery diff --git a/crates/rocm-core/src/endpoint_key.rs b/crates/rocm-core/src/endpoint_key.rs index bfcb523808..46a3aa733a 100644 --- a/crates/rocm-core/src/endpoint_key.rs +++ b/crates/rocm-core/src/endpoint_key.rs @@ -4,7 +4,9 @@ //! The bearer key that guards a managed service's endpoint. //! -//! The endpoint-API-key generator and its header-safety check. Drawn from a +//! The endpoint-API-key generator, its header-safety check, and the one guard +//! `rocm` and `rocmd` both apply before bringing a recorded service back up +//! (see [`ensure_public_service_has_endpoint_key`]). Drawn from a //! CSPRNG — deliberately never derived from [`crate::service_identity`]'s //! timestamp-based, guessable [`crate::generate_service_id`] — the two meet //! only at the filesystem layer, where a service's endpoint key is stored in a @@ -49,6 +51,110 @@ pub fn endpoint_api_key_has_forbidden_chars(key: &str) -> bool { key.chars().any(char::is_control) } +/// Whether `host` names a public (non-loopback) bind, i.e. one that must be +/// protected by an endpoint API key. +/// +/// Shared so `rocm` (which decides at `serve` time whether to mint a key) and +/// `rocmd` (which respawns recorded services during recovery) apply the same +/// policy to the same `ManagedServiceRecord::host` value, and so +/// [`ensure_public_service_has_endpoint_key`] can apply it itself. +/// +/// Deliberately an exact match against the three loopback spellings the CLI +/// accepts, so anything unrecognized — an IPv6 loopback alias, a differently +/// cased `localhost`, a hostname that happens to resolve locally — is treated +/// as public and therefore *requires* a key. Misclassifying a loopback bind as +/// public costs a key nobody needed; the reverse would open an unauthenticated +/// network listener. +#[must_use] +pub fn is_public_bind_host(host: &str) -> bool { + !matches!(host, "127.0.0.1" | "localhost" | "::1") +} + +/// What is about to bring a recorded service up. +/// +/// It only words the refusal in [`ensure_public_service_has_endpoint_key`] for +/// the action; the rules and the commands the refusal names do not depend on it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ServiceStart { + /// A fresh `rocm serve` spawn. + Launch, + /// `rocm services restart`. + Restart, + /// The daemon's recovery supervisor putting a failed service back up. + Respawn, +} + +impl ServiceStart { + /// The clause that finishes "so …" in a refusal. + const fn would_open(self) -> &'static str { + match self { + Self::Launch => "launching it would open it", + Self::Restart => "restarting it would reopen it", + Self::Respawn => "respawning it would reopen it", + } + } +} + +/// Refuse to bring a managed service up when it needs an endpoint API key and +/// has none, so a respawn cannot reopen a protected endpoint anonymously. +/// +/// `serve()` applies the loopback-vs-public policy once and persists the +/// resulting key. Every later spawn — `rocm services restart`, and `rocmd`'s +/// recovery supervisor — reads that key file back and would otherwise treat +/// "no key file" as "no auth wanted", silently downgrading a protected public +/// endpoint to an open one. A stop deletes the key file and `rocm services +/// restart` accepts a stopped service id, so the gap is reachable through +/// ordinary commands. The real invariant is a property of the *host*, not of +/// the file: a non-loopback bind must always be authenticated. +/// +/// Two ways a service can need a key. A public bind is the obvious one. The +/// other is a service that asked for auth on a loopback bind +/// (`requires_api_key`), because something outside this process republishes the +/// port — a tailnet publish survives a reboot, let alone a restart, so the bind +/// address can no longer be trusted to answer the question on its own. The +/// requirement is recorded on the service precisely so a respawn can honour it. +/// The second is checked first, so a service that asked for auth is never told +/// its public bind is the problem. +/// +/// `key_present` means a *usable* key: the engine adapters enforce nothing when +/// the file is empty or malformed. It is a plain `bool` rather than a path so +/// every branch is unit-testable without touching the filesystem. +/// +/// Both refusals name the command that restores the service — `rocm serve +/// …` — because the key is dropped when a service stops and cannot be +/// recovered, only issued again by a new launch. +pub fn ensure_public_service_has_endpoint_key( + host: &str, + key_present: bool, + requires_api_key: bool, + start: ServiceStart, +) -> anyhow::Result<()> { + if key_present { + return Ok(()); + } + let would_open = start.would_open(); + if requires_api_key { + anyhow::bail!( + "managed service was launched with `--require-api-key` but has no endpoint API key, \ + so {would_open} without authentication. Something outside this machine may still be \ + publishing its port. The key is dropped when a service stops and cannot be \ + recovered. Launch it again with `rocm serve --require-api-key` (add \ + `--api-key `, or set ROCM_SERVE_API_KEY, to choose the key instead of \ + generating one)." + ); + } + if is_public_bind_host(host) { + anyhow::bail!( + "managed service is bound to the public host `{host}` but has no endpoint API key, \ + so {would_open} without authentication. The key is dropped when a service stops and \ + cannot be recovered. Launch it again with `rocm serve --host {host} \ + --port --allow-public-bind` (add `--api-key `, or set \ + ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + ); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -71,6 +177,159 @@ mod tests { )); } + const STARTS: [ServiceStart; 3] = [ + ServiceStart::Launch, + ServiceStart::Restart, + ServiceStart::Respawn, + ]; + + fn refusal(host: &str, key_present: bool, requires: bool, start: ServiceStart) -> String { + ensure_public_service_has_endpoint_key(host, key_present, requires, start) + .expect_err("this state must be refused") + .to_string() + } + + #[test] + fn is_public_bind_host_recognizes_only_the_accepted_loopback_spellings() { + for host in ["127.0.0.1", "localhost", "::1"] { + assert!(!is_public_bind_host(host), "{host} must be loopback"); + } + for host in [ + "0.0.0.0", + "::", + "192.168.1.10", + "example.test", + // Conservative on purpose: an unrecognized spelling of a loopback + // address is classified public, so it requires a key rather than + // opening an unauthenticated listener. + "LOCALHOST", + "127.0.0.2", + "[::1]", + ] { + assert!(is_public_bind_host(host), "{host} must be public"); + } + } + + #[test] + fn a_service_that_requires_a_key_is_refused_without_one_on_any_host() { + // The bind address is not the whole story: a loopback service that + // something republishes is only private until it is respawned keyless. + for start in STARTS { + for host in [ + "127.0.0.1", + "localhost", + "::1", + "0.0.0.0", + "gpu-box.example", + ] { + let message = refusal(host, false, true, start); + assert!( + message.contains("launched with `--require-api-key`"), + "{host} {start:?}: {message}" + ); + assert!( + message.contains("Launch it again with `rocm serve --require-api-key`"), + "{host} {start:?}: {message}" + ); + // It never blames the bind address, whatever the host is. + assert!(!message.contains("public host"), "{message}"); + assert!(!message.contains("--allow-public-bind"), "{message}"); + } + } + } + + #[test] + fn a_public_bind_is_refused_without_a_key_even_if_it_never_asked_for_one() { + for start in STARTS { + for host in [ + "0.0.0.0", + "::", + "192.168.1.10", + "gpu-box.example", + "LOCALHOST", + ] { + let message = refusal(host, false, false, start); + assert!( + message.contains(&format!("bound to the public host `{host}`")), + "{host} {start:?}: {message}" + ); + // The remediation keeps the host and the public-bind opt-in, so + // following it does not bring the service back on loopback. + assert!( + message.contains(&format!( + "Launch it again with `rocm serve --host {host} --port --allow-public-bind`" + )), + "{host} {start:?}: {message}" + ); + // A service that never passed the flag is not told it did. + assert!(!message.contains("`--require-api-key`"), "{message}"); + } + } + } + + #[test] + fn a_service_that_needs_no_key_or_has_one_is_not_refused() { + for start in STARTS { + for host in ["127.0.0.1", "localhost", "::1"] { + ensure_public_service_has_endpoint_key(host, false, false, start) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + } + for (host, requires) in [("0.0.0.0", false), ("127.0.0.1", true), ("0.0.0.0", true)] { + ensure_public_service_has_endpoint_key(host, true, requires, start) + .unwrap_or_else(|error| panic!("{host} has its key: {error:#}")); + } + } + } + + #[test] + fn a_service_that_asked_for_a_key_on_a_public_host_is_told_about_the_key_requirement() { + // The first rule wins, so the remediation does not drop what the user + // asked for. + for start in STARTS { + assert_eq!( + refusal("0.0.0.0", false, true, start), + refusal("127.0.0.1", false, true, start) + ); + } + } + + #[test] + fn the_start_changes_only_what_the_refusal_says_the_action_would_do() { + // The same text is shown by `rocm services restart`, a fresh `rocm serve` + // spawn and the daemon's recovery: only the action's own words differ, so + // the rules and the commands they name cannot drift apart per caller. + for (host, requires) in [("127.0.0.1", true), ("0.0.0.0", false)] { + let restart = refusal(host, false, requires, ServiceStart::Restart); + for (start, clause) in [ + (ServiceStart::Launch, "launching it would open it"), + (ServiceStart::Respawn, "respawning it would reopen it"), + ] { + assert_eq!( + refusal(host, false, requires, start), + restart.replace("restarting it would reopen it", clause), + "{host} {start:?}" + ); + } + } + assert_eq!( + refusal("127.0.0.1", false, true, ServiceStart::Restart), + "managed service was launched with `--require-api-key` but has no endpoint API key, \ + so restarting it would reopen it without authentication. Something outside this \ + machine may still be publishing its port. The key is dropped when a service stops \ + and cannot be recovered. Launch it again with `rocm serve \ + --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, to choose the \ + key instead of generating one)." + ); + assert_eq!( + refusal("0.0.0.0", false, false, ServiceStart::Respawn), + "managed service is bound to the public host `0.0.0.0` but has no endpoint API key, \ + so respawning it would reopen it without authentication. The key is dropped when a \ + service stops and cannot be recovered. Launch it again with `rocm serve \ + --host 0.0.0.0 --port --allow-public-bind` (add `--api-key `, or set \ + ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + ); + } + #[test] fn endpoint_api_key_has_forbidden_chars_flags_control_chars() { // Control characters (notably CR/LF) enable header injection when the key diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index c142c3332f..ef643e33c8 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -52,6 +52,7 @@ pub mod rocm_install; pub mod runtime; pub mod service_identity; pub mod service_record; +mod sigpipe; pub mod terminal; pub mod user_path; @@ -103,7 +104,10 @@ pub use download::{ download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, }; -pub use endpoint_key::{endpoint_api_key_has_forbidden_chars, generate_endpoint_api_key}; +pub use endpoint_key::{ + ServiceStart, endpoint_api_key_has_forbidden_chars, ensure_public_service_has_endpoint_key, + generate_endpoint_api_key, is_public_bind_host, +}; pub use endpoint_readiness::{ EndpointReadiness, EndpointReadinessOutcome, INFERENCE_PROBE_TIMEOUT, engine_state_inference_verified, managed_service_endpoint_model_ready, @@ -138,7 +142,11 @@ use host_gpu::{ detect_wsl_summary, is_wsl1_kernel, ldconfig_cache, ldconfig_lists_librocdxg, }; pub use jsonl_store::jsonl_lock_path; -pub use log_tail::tail_lines; +pub use log_tail::{ + LOG_VIEW_MAX_TAIL_READ, LogDecode, LogTail, newest_line_exceeds_bound_notice, + service_log_tool_summary, strip_ansi_sequences, tail_lines, tail_lines_decoded, + tail_log_decoded, +}; pub use managed_runtime::{ FrameworkInterpreter, ManagedRuntimeEnvironment, active_managed_framework_interpreter, active_managed_therock_channel, active_managed_therock_environment, @@ -164,10 +172,11 @@ pub use net::{ http_get_text_with_auth, http_post_json_with_auth, parse_http_endpoint, read_http_response_bounded, write_all_tcp_stream, }; -pub use paths::{AppPaths, engine_plugin_dirs}; +pub use paths::{AppPaths, ENGINE_ENVS_ROOT_ENV, engine_plugin_dirs}; pub use proc_lifecycle::{ - IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, - process_start_ticks, terminate_verified, + IdentityState, KillScope, MANAGED_STOP_GRACE, ProcessIdentity, RecordedPidOutcome, + RecordedPidStop, TerminationOutcome, identity_state, process_start_ticks, + terminate_recorded_pid, terminate_recorded_service_pids, terminate_verified, }; pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, @@ -219,6 +228,7 @@ pub use service_record::{ INFERENCE_VERIFIED_STATE_KEY, ManagedServiceNotFound, ManagedServiceRecord, engine_stop_decides_confirmation, }; +pub use sigpipe::with_sigpipe_ignored; pub use terminal::{quotable_in_single_quotes, strip_terminal_control_sequences}; pub use uv::{ DEFAULT_UV_TIMEOUT_SECS, DependencyViolation, UV_CACHE_DIR_ENV, UV_CACHE_DIR_OVERRIDE_ENV, diff --git a/crates/rocm-core/src/log_tail.rs b/crates/rocm-core/src/log_tail.rs index 9ab1fb93f9..6dc74fe3c7 100644 --- a/crates/rocm-core/src/log_tail.rs +++ b/crates/rocm-core/src/log_tail.rs @@ -2,83 +2,389 @@ // // SPDX-License-Identifier: MIT +//! The one routine that reads the last lines of a log file. +//! +//! The engine adapters, the ComfyUI log view, the services text module and the +//! `rocmd` MCP `service_logs` tool all go through [`tail_lines_decoded`] +//! (`tail_lines` is the plain-text form of it), so the seek-boundary handling, +//! the lossy decoding and the "never read the whole file" bound each have one +//! implementation. A caller picks how the bytes become text with a +//! [`LogDecode`]; it does not pick how the file is read. + use anyhow::{Context, Result}; use std::collections::VecDeque; use std::fs; use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; use std::path::Path; +/// How many bytes from the end of a log the log views (ComfyUI logs, `rocm +/// services logs`, the MCP `service_logs` tool) read at most. The engine +/// adapters keep their own budgets. +pub const LOG_VIEW_MAX_TAIL_READ: u64 = 4 * 1024 * 1024; + +/// How the bytes of a log become lines of text. +/// +/// Whatever the choice, text that is not valid in the chosen encoding is +/// replaced with U+FFFD rather than failing the read: a log view must show +/// what it can. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LogDecode { + /// Decode the log as UTF-16 little-endian when it carries a byte order + /// mark, or (without one) when at least a quarter of the odd-numbered + /// bytes in its first 512 bytes are zero (mostly-ASCII UTF-16 text), as a + /// Windows shell redirect writes it. Otherwise the log is UTF-8. + pub utf16_le: bool, + /// Remove ANSI CSI escape sequences (colour and cursor codes) from each + /// returned line. + pub strip_ansi: bool, +} + +impl LogDecode { + /// UTF-8 only, escape sequences kept: what the engine adapters and the + /// service logs return. + pub const PLAIN: Self = Self { + utf16_le: false, + strip_ansi: false, + }; + /// Detect UTF-16 little-endian and strip ANSI sequences: for logs a + /// Windows shell may have redirected, shown to a person in a terminal. + pub const CONSOLE: Self = Self { + utf16_le: true, + strip_ansi: true, + }; +} + +/// The lines a bounded tail read returned, and what the bound left unread. +/// +/// A caller that only wants the lines uses [`tail_lines_decoded`]; one that +/// tells a person what it found uses [`tail_log_decoded`], so that "the log +/// has no lines" and "the log's newest line does not fit in the read bound" +/// are not the same empty list. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LogTail { + /// The last lines of the log, oldest first. + pub lines: Vec, + /// How many bytes at the start of the file the read did not look at + /// because of the byte budget; zero when the whole file was read. + pub skipped_bytes: u64, +} + +impl LogTail { + /// True when the read found no complete line even though the log has + /// content: its newest line begins before the byte budget's window, so the + /// window held only a fragment of it, which is dropped. An empty log, or a + /// limit of zero, is not this. + pub const fn newest_line_exceeds_the_read_bound(&self) -> bool { + self.lines.is_empty() && self.skipped_bytes > 0 + } +} + +/// The clause a log view prints when none of a log's content is shown. +/// +/// That is the case when the newest line does not fit the read budget +/// ([`LogTail::newest_line_exceeds_the_read_bound`]). Derived from +/// [`LOG_VIEW_MAX_TAIL_READ`], the budget the views read with, so the number +/// cannot drift from it. +pub fn newest_line_exceeds_bound_notice() -> String { + format!( + "the newest line is longer than the {} MiB a log view reads", + LOG_VIEW_MAX_TAIL_READ / (1024 * 1024) + ) +} + +/// What the `service_logs` tool says about a service's log tail. +/// +/// An empty log and a log whose newest line does not fit the read bound are +/// not both reported as "Read the last 0 line(s)". Shared by the daemon's MCP +/// server and the CLI's own tool bridge, which answer the same tool. +/// +/// The count is the number of lines returned, not the number asked for. +pub fn service_log_tool_summary(service_id: &str, tail: &LogTail) -> String { + if tail.newest_line_exceeds_the_read_bound() { + format!( + "Service `{service_id}` has log output, but none is shown: {}.", + newest_line_exceeds_bound_notice() + ) + } else if tail.lines.is_empty() { + format!("Service `{service_id}` has no log output yet.") + } else { + format!( + "Read the last {} line(s) from service `{service_id}`.", + tail.lines.len() + ) + } +} + +/// Reads the last `limit` lines from a file as UTF-8, seeking near the end +/// instead of reading the whole file for files larger than `max_tail_read`. +/// +/// Shorthand for [`tail_lines_decoded`] with [`LogDecode::PLAIN`]. +pub fn tail_lines(path: &Path, limit: usize, max_tail_read: u64) -> Result> { + tail_lines_decoded(path, limit, max_tail_read, LogDecode::PLAIN) +} + /// Reads the last `limit` lines from a file, seeking near the end instead of /// reading the whole file for files larger than `max_tail_read`. /// -/// Shared by the engine adapters so the seek-boundary classification has one -/// implementation: when the seek lands in the middle of a line, the first -/// line read back is a partial fragment and must be dropped; when it lands -/// exactly on a line boundary (the byte before the seek position is a -/// newline), that first line is complete and must be kept. `max_tail_read` -/// stays a parameter rather than a shared constant because each engine picks -/// its own tail-read budget. -pub fn tail_lines(path: &Path, limit: usize, max_tail_read: u64) -> Result> { +/// Shared by every reader of a log tail so the seek-boundary classification +/// has one implementation: when the seek lands in the middle of a line, the +/// first line read back is a partial fragment and must be dropped; when it +/// lands exactly on a line boundary (the character before the seek position +/// is a newline), that first line is complete and must be kept. +/// `max_tail_read` stays a parameter rather than a shared constant because +/// each engine picks its own tail-read budget (the log views share +/// [`LOG_VIEW_MAX_TAIL_READ`]). +/// +/// A limit of zero returns no lines without opening the file. Line endings +/// are `\n` or `\r\n`. See [`LogDecode`] for the text decoding. +pub fn tail_lines_decoded( + path: &Path, + limit: usize, + max_tail_read: u64, + decode: LogDecode, +) -> Result> { + Ok(tail_log_decoded(path, limit, max_tail_read, decode)?.lines) +} + +/// [`tail_lines_decoded`], also reporting how much of the start of the file +/// the byte budget left unread (see [`LogTail`]). The lines are the same. +pub fn tail_log_decoded( + path: &Path, + limit: usize, + max_tail_read: u64, + decode: LogDecode, +) -> Result { if limit == 0 { - return Ok(Vec::new()); + return Ok(LogTail::default()); } - let mut file = + let file = fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; let metadata = file .metadata() .with_context(|| format!("failed to stat {}", path.display()))?; - let file_size = metadata.len(); + tail_lines_from(file, metadata.len(), path, limit, max_tail_read, decode) +} + +/// The tail read over any seekable source of `size` bytes; `path` only names +/// the source in error messages. Split out so a test can count the bytes the +/// read touches. +fn tail_lines_from( + mut source: R, + size: u64, + path: &Path, + limit: usize, + max_tail_read: u64, + decode: LogDecode, +) -> Result { + let utf16_bom = if decode.utf16_le { + sniff_utf16_le(&mut source, size, path)? + } else { + None + }; + // The width of one character unit, and so of the newline probe below: a + // UTF-16 window must start on a unit boundary. + let unit: u64 = if utf16_bom.is_some() { 2 } else { 1 }; + let bom_len: u64 = if utf16_bom == Some(true) { 2 } else { 0 }; let mut first_line_is_partial = false; - if file_size > max_tail_read { - let seek_pos = file_size - max_tail_read; - // Probe the byte preceding `seek_pos` to classify the first line, then - // leave the cursor at `seek_pos` for the buffered read below. - file.seek(SeekFrom::Start(seek_pos - 1)) + let window_start = if size > max_tail_read { + let mut seek_pos = size - max_tail_read; + if unit == 2 && seek_pos % 2 == 1 { + seek_pos += 1; + } + if seek_pos >= size { + return Ok(LogTail { + lines: Vec::new(), + skipped_bytes: size, + }); + } + // Probe the unit preceding `seek_pos` to classify the first line, then + // leave the cursor at `seek_pos` for the read below. + source + .seek(SeekFrom::Start(seek_pos - unit)) .with_context(|| format!("failed to seek in {}", path.display()))?; - let mut probe = [0u8; 1]; - file.read_exact(&mut probe) + let mut probe = [0u8; 2]; + let probe = &mut probe[..unit as usize]; + source + .read_exact(probe) .with_context(|| format!("failed to read from {}", path.display()))?; - first_line_is_partial = probe[0] != b'\n'; - } + // The unit before the first line of a file with a byte order mark is + // the mark itself, not the end of a previous line. + first_line_is_partial = seek_pos != bom_len && probe != [b'\n'] && probe != [b'\n', 0]; + seek_pos + } else { + // The whole file is the window; skip a byte order mark the way a + // reader of the file from its start would. + source + .seek(SeekFrom::Start(bom_len)) + .with_context(|| format!("failed to seek in {}", path.display()))?; + bom_len + }; - let mut reader = BufReader::new(file); // Capped, not `VecDeque::with_capacity(limit)`: callers may pass // `usize::MAX` to mean "no limit", which would try to allocate that // capacity upfront. Real callers ask for small limits (tens to a few // hundred lines), so capping still gives them a single allocation. let mut lines: VecDeque = VecDeque::with_capacity(limit.min(4096)); - // Read raw bytes rather than `BufRead::lines()`: when the seek above lands - // inside a multi-byte UTF-8 character, the leading fragment is invalid - // UTF-8 on its own. `lines()` would surface that as an `Err` and abort the - // whole read before `first_line_is_partial` gets a chance to drop exactly - // that fragment. Decoding lossily keeps the drop-the-fragment logic in - // charge of it instead. - let mut raw_line = Vec::new(); - loop { - raw_line.clear(); - let bytes_read = reader - .read_until(b'\n', &mut raw_line) + let mut keep = |text: String, terminated: bool| { + // The unterminated tail of a console log can be only the escape + // sequence a program writes after its last newline (a colour reset). + // It is not a line: counting it would push the last real line out + // when the limit is small. A terminated line that strips to nothing + // is a blank line the program wrote, and stays. + if decode.strip_ansi && !terminated && strip_ansi_sequences(&text).is_empty() { + return; + } + if lines.len() == limit { + lines.pop_front(); + } + lines.push_back(text); + }; + + if unit == 2 { + // The window is at most `max_tail_read` bytes (or the whole, smaller, + // file), so it is read in one go and split on the newline unit. + let mut window = Vec::new(); + (&mut source) + .take(size - window_start) + .read_to_end(&mut window) .with_context(|| format!("failed to read lines from {}", path.display()))?; - if bytes_read == 0 { - break; + let units: Vec = window + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(); + let mut rest = units.as_slice(); + while !rest.is_empty() { + let (mut line, terminated) = + if let Some(end) = rest.iter().position(|&unit| unit == 0x000A) { + let line = &rest[..end]; + rest = &rest[end + 1..]; + (line, true) + } else { + let line = rest; + rest = &[]; + (line, false) + }; + if terminated && line.last() == Some(&0x000D) { + line = &line[..line.len() - 1]; + } + if first_line_is_partial { + first_line_is_partial = false; + continue; + } + keep(String::from_utf16_lossy(line), terminated); } - if raw_line.last() == Some(&b'\n') { - raw_line.pop(); - if raw_line.last() == Some(&b'\r') { + } else { + // Capped at the snapshot like the sixteen-bit window above: a log + // whose writer is still appending must not pull the read past the + // budget by however much it grew since `size` was taken. + let mut reader = BufReader::new(source.take(size - window_start)); + // Read raw bytes rather than `BufRead::lines()`: when the seek above + // lands inside a multi-byte UTF-8 character, the leading fragment is + // invalid UTF-8 on its own. `lines()` would surface that as an `Err` + // and abort the whole read before `first_line_is_partial` gets a + // chance to drop exactly that fragment. Decoding lossily keeps the + // drop-the-fragment logic in charge of it instead. + let mut raw_line = Vec::new(); + loop { + raw_line.clear(); + let bytes_read = reader + .read_until(b'\n', &mut raw_line) + .with_context(|| format!("failed to read lines from {}", path.display()))?; + if bytes_read == 0 { + break; + } + let terminated = raw_line.last() == Some(&b'\n'); + if terminated { raw_line.pop(); + if raw_line.last() == Some(&b'\r') { + raw_line.pop(); + } } + if first_line_is_partial { + first_line_is_partial = false; + continue; + } + keep(String::from_utf8_lossy(&raw_line).into_owned(), terminated); + } + } + + let lines = lines + .into_iter() + .map(|line| { + if decode.strip_ansi { + strip_ansi_sequences(&line) + } else { + line + } + }) + .collect(); + Ok(LogTail { + lines, + // Not `window_start`: a byte order mark skipped at the start of a + // whole-file read is not something the budget left unread. + skipped_bytes: if size > max_tail_read { + window_start + } else { + 0 + }, + }) +} + +/// `Some(has_byte_order_mark)` when the start of the source reads as UTF-16 +/// little-endian, `None` when it reads as UTF-8. Looks only at the first 512 +/// bytes, so it costs the same however large the log is. +fn sniff_utf16_le(source: &mut R, size: u64, path: &Path) -> Result> { + const SAMPLE: u64 = 512; + let mut head = Vec::new(); + source + .seek(SeekFrom::Start(0)) + .with_context(|| format!("failed to seek in {}", path.display()))?; + source + .take(size.min(SAMPLE)) + .read_to_end(&mut head) + .with_context(|| format!("failed to read from {}", path.display()))?; + if head.starts_with(&[0xff, 0xfe]) { + return Ok(Some(true)); + } + // Mostly-ASCII UTF-16 has a zero in every other byte. + if size >= 8 { + let zero_odd_bytes = head + .iter() + .enumerate() + .filter(|(index, byte)| index % 2 == 1 && **byte == 0) + .count(); + if zero_odd_bytes * 4 >= head.len() { + return Ok(Some(false)); } - if first_line_is_partial { - first_line_is_partial = false; + } + Ok(None) +} + +/// Removes ANSI CSI sequences from one line of text. +/// +/// A sequence is `ESC [` up to and including a final byte in `@`..=`~`. A bare +/// `ESC` not followed by `[` is dropped alone. The log tail applies this per +/// line, so an unterminated sequence ends with its line instead of swallowing +/// the lines after it. +pub fn strip_ansi_sequences(text: &str) -> String { + let mut cleaned = String::with_capacity(text.len()); + let mut chars = text.chars().peekable(); + while let Some(ch) = chars.next() { + if ch != '\x1b' { + cleaned.push(ch); continue; } - if lines.len() == limit { - lines.pop_front(); + if chars.peek() == Some(&'[') { + let _ = chars.next(); + for next in chars.by_ref() { + if ('@'..='~').contains(&next) { + break; + } + } } - lines.push_back(String::from_utf8_lossy(&raw_line).into_owned()); } - Ok(lines.into_iter().collect()) + cleaned } #[cfg(test)] @@ -219,6 +525,68 @@ mod tests { Ok(()) } + #[test] + fn tail_lines_on_a_large_file_returns_only_whole_lines_from_the_window() -> Result<()> { + // 1 MiB of 1 KiB lines under a 64 KiB budget, with a marked first and + // last line. The first line lies far outside the window, so a read + // that returned it was not bounded; the window starts mid-line (the + // budget is not a multiple of the line length), so a leading fragment + // that was kept would be shorter than a whole line. + let mut content = String::from("HEAD-MARKER\n"); + let filler = format!("{}\n", "x".repeat(1023)); + for _ in 0..1024 { + content.push_str(&filler); + } + content.push_str("TAIL-MARKER\n"); + let path = + std::env::temp_dir().join(format!("rocm-core-tail-large-{}.log", unique_suffix())); + fs::write(&path, &content)?; + + let lines = tail_lines(&path, usize::MAX, 64 * 1024 + 7)?; + fs::remove_file(&path).ok(); + + assert!(!lines.iter().any(|line| line == "HEAD-MARKER")); + assert_eq!(lines.last().map(String::as_str), Some("TAIL-MARKER")); + assert!(lines.len() <= 65, "got {} lines", lines.len()); + for line in &lines[..lines.len() - 1] { + assert_eq!(line.len(), 1023, "a partial leading fragment was kept"); + } + Ok(()) + } + + #[test] + fn tail_lines_missing_file_is_an_open_error_naming_the_path() { + let path = Path::new("/does/not/exist/rocm-core-tail-missing.log"); + + let error = tail_lines(path, 5, TEST_MAX_TAIL_READ).expect_err("missing file"); + + assert_eq!( + error.to_string(), + format!("failed to open {}", path.display()) + ); + } + + #[test] + fn tail_lines_returns_every_line_when_fewer_than_the_limit() -> Result<()> { + let path = std::env::temp_dir().join(format!("rocm-core-tail-few-{}.log", unique_suffix())); + fs::write(&path, "a\nb\n")?; + let lines = tail_lines(&path, 50, TEST_MAX_TAIL_READ)?; + fs::remove_file(&path).ok(); + assert_eq!(lines, vec!["a".to_owned(), "b".to_owned()]); + Ok(()) + } + + #[test] + fn tail_lines_keeps_a_final_line_without_a_trailing_newline() -> Result<()> { + let path = + std::env::temp_dir().join(format!("rocm-core-tail-no-nl-{}.log", unique_suffix())); + fs::write(&path, "a\nb\nc")?; + let lines = tail_lines(&path, 2, TEST_MAX_TAIL_READ)?; + fs::remove_file(&path).ok(); + assert_eq!(lines, vec!["b".to_owned(), "c".to_owned()]); + Ok(()) + } + #[test] fn tail_lines_limit_zero_returns_empty_without_opening_file() -> Result<()> { let path = Path::new("/does/not/exist/rocm-core-tail-limit-zero.log"); @@ -226,4 +594,608 @@ mod tests { assert!(lines.is_empty()); Ok(()) } + + // --- bounded reads ------------------------------------------------- + + /// Counts the bytes its inner reader hands out, so a test can show how + /// much of a source a tail read touched. + struct CountingReader { + inner: R, + bytes_read: std::rc::Rc>, + } + + impl Read for CountingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buf)?; + self.bytes_read.set(self.bytes_read.get() + read as u64); + Ok(read) + } + } + + impl Seek for CountingReader { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.inner.seek(position) + } + } + + /// Runs the tail read over `content` and returns the lines with the number + /// of bytes the read pulled from the source. + fn read_counting( + content: Vec, + limit: usize, + max_tail_read: u64, + decode: LogDecode, + ) -> Result<(Vec, u64)> { + let size = content.len() as u64; + let bytes_read = std::rc::Rc::new(std::cell::Cell::new(0)); + let source = CountingReader { + inner: std::io::Cursor::new(content), + bytes_read: bytes_read.clone(), + }; + let lines = tail_lines_from( + source, + size, + Path::new("counting-source"), + limit, + max_tail_read, + decode, + )? + .lines; + Ok((lines, bytes_read.get())) + } + + fn utf16_le_bytes(text: &str, bom: bool) -> Vec { + let mut bytes = if bom { vec![0xff, 0xfe] } else { Vec::new() }; + for unit in text.encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + bytes + } + + #[test] + fn a_tail_read_touches_no_more_than_its_budget_of_a_large_file() -> Result<()> { + // 8 MiB under a 64 KiB budget: a whole-file read would pull 8 MiB. + let filler = format!("{}\n", "x".repeat(1023)); + let content = filler.repeat(8192).into_bytes(); + let budget = 64 * 1024; + + let (lines, bytes_read) = read_counting(content, usize::MAX, budget, LogDecode::PLAIN)?; + + assert!(!lines.is_empty()); + // The window plus the one byte probed before it. + assert!(bytes_read <= budget + 1, "read {bytes_read} bytes"); + Ok(()) + } + + #[test] + fn a_sixteen_bit_tail_read_touches_no_more_than_its_budget_of_a_large_file() -> Result<()> { + let filler = format!("\u{1b}[33m{}\u{1b}[0m\r\n", "y".repeat(500)); + let content = utf16_le_bytes(&filler.repeat(8000), true); + assert!(content.len() > 8_000_000); + let budget = 64 * 1024 + 1; + + let (lines, bytes_read) = read_counting(content, usize::MAX, budget, LogDecode::CONSOLE)?; + + assert!(!lines.is_empty()); + assert!( + lines.iter().all(|line| line == &"y".repeat(500)), + "decoded from a unit boundary, ANSI stripped, no fragment" + ); + // The window, the unit probed before it, and the 512-byte sniff of the + // start of the file. + assert!(bytes_read <= budget + 2 + 512, "read {bytes_read} bytes"); + Ok(()) + } + + /// For every budget from nothing to the whole file, the lines returned are + /// a suffix of the file's complete lines: never a fragment, never garbage. + fn assert_every_budget_returns_a_suffix_of_whole_lines( + content: &[u8], + expected: &[String], + decode: LogDecode, + ) -> Result<()> { + // A byte order mark is not part of the first line. + let bom_len = if decode.utf16_le && content.starts_with(&[0xff, 0xfe]) { + 2 + } else { + 0 + }; + for budget in 0..=content.len() as u64 + 2 { + let (lines, _) = read_counting(content.to_vec(), usize::MAX, budget, decode)?; + assert!( + lines.len() <= expected.len() && lines == expected[expected.len() - lines.len()..], + "budget {budget}: {lines:?} is not a suffix of {expected:?}" + ); + if budget + bom_len >= content.len() as u64 { + assert_eq!(lines, expected, "budget {budget} covers the whole file"); + } + } + Ok(()) + } + + #[test] + fn every_budget_returns_whole_utf8_lines() -> Result<()> { + let text = "alpha\nbravo\r\ncharlie\n\ndelta\u{2588}\nlast"; + let expected = ["alpha", "bravo", "charlie", "", "delta\u{2588}", "last"] + .map(str::to_owned) + .to_vec(); + assert_every_budget_returns_a_suffix_of_whole_lines( + text.as_bytes(), + &expected, + LogDecode::PLAIN, + )?; + assert_every_budget_returns_a_suffix_of_whole_lines( + text.as_bytes(), + &expected, + LogDecode::CONSOLE, + ) + } + + #[test] + fn every_budget_returns_whole_sixteen_bit_lines_including_odd_ones() -> Result<()> { + // An odd budget puts the seek position on the second byte of a unit, + // which must be rounded to a unit boundary rather than decoded from. + let text = "alpha\nbravo\r\ncharlie\n\ndelta\u{2588}\u{1f600}\nlast"; + let expected = [ + "alpha", + "bravo", + "charlie", + "", + "delta\u{2588}\u{1f600}", + "last", + ] + .map(str::to_owned) + .to_vec(); + for bom in [true, false] { + assert_every_budget_returns_a_suffix_of_whole_lines( + &utf16_le_bytes(text, bom), + &expected, + LogDecode::CONSOLE, + )?; + } + Ok(()) + } + + // --- decoding ------------------------------------------------------ + + fn read_bytes( + name: &str, + bytes: &[u8], + limit: usize, + decode: LogDecode, + ) -> Result> { + let path = + std::env::temp_dir().join(format!("rocm-core-tail-{name}-{}.log", unique_suffix())); + fs::write(&path, bytes)?; + let lines = tail_lines_decoded(&path, limit, TEST_MAX_TAIL_READ, decode); + fs::remove_file(&path).ok(); + lines + } + + #[test] + fn console_decode_reads_a_sixteen_bit_log_with_a_byte_order_mark() -> Result<()> { + let bytes = utf16_le_bytes("\u{1b}[32m[INFO]\u{1b}[0m first\r\nsecond\r\n", true); + + let lines = read_bytes("bom", &bytes, 10, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["[INFO] first", "second"]); + Ok(()) + } + + #[test] + fn console_decode_detects_a_sixteen_bit_log_without_a_byte_order_mark() -> Result<()> { + let bytes = utf16_le_bytes("first\nsecond\nthird", false); + + let lines = read_bytes("nobom", &bytes, 2, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["second", "third"]); + Ok(()) + } + + #[test] + fn console_decode_replaces_a_lone_surrogate_and_ignores_an_odd_trailing_byte() -> Result<()> { + let mut bytes = utf16_le_bytes("ok\n", true); + bytes.extend_from_slice(&[0x00, 0xd8, b'x', 0x00, b'\n', 0x00, 0x41]); + + let lines = read_bytes("surrogate", &bytes, 10, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["ok", "\u{FFFD}x"]); + Ok(()) + } + + #[test] + fn console_decode_leaves_a_short_or_ascii_log_as_utf8() -> Result<()> { + // Under 8 bytes there is too little to call it sixteen-bit; plain + // ASCII has no zero bytes to suggest it. + let short = read_bytes("short", b"a\0b\0c\0", 5, LogDecode::CONSOLE)?; + let ascii = read_bytes("ascii", b"plain ascii text\n", 5, LogDecode::CONSOLE)?; + + assert_eq!(short, vec!["a\0b\0c\0"]); + assert_eq!(ascii, vec!["plain ascii text"]); + Ok(()) + } + + #[test] + fn console_decode_strips_ansi_per_line_so_an_unterminated_sequence_ends_with_its_line() + -> Result<()> { + let bytes = b"\x1b[31mred\x1b[0m\ncut off \x1b[1;3\nnext line\n"; + + let lines = read_bytes("ansi", bytes, 10, LogDecode::CONSOLE)?; + + assert_eq!(lines, vec!["red", "cut off ", "next line"]); + Ok(()) + } + + #[test] + fn plain_decode_neither_detects_sixteen_bit_text_nor_strips_ansi() -> Result<()> { + let ansi = read_bytes("plain-ansi", b"\x1b[32mgreen\x1b[0m\n", 5, LogDecode::PLAIN)?; + let wide = read_bytes( + "plain-wide", + &utf16_le_bytes("hi\n", true), + 5, + LogDecode::PLAIN, + )?; + + assert_eq!(ansi, vec!["\u{1b}[32mgreen\u{1b}[0m"]); + assert_eq!(wide[0], "\u{FFFD}\u{FFFD}h\0i\0"); + Ok(()) + } + + #[test] + fn strip_ansi_sequences_removes_csi_sequences_and_a_bare_escape() { + assert_eq!( + strip_ansi_sequences("\u{1b}[1;31mred\u{1b}[0m \u{1b}[2Kdone \u{1b}x"), + "red done x" + ); + assert_eq!(strip_ansi_sequences("no escapes"), "no escapes"); + } + + #[test] + fn a_sixteen_bit_limit_zero_returns_empty_without_opening_the_file() -> Result<()> { + let path = Path::new("/does/not/exist/rocm-core-tail-limit-zero-wide.log"); + assert!(tail_lines_decoded(path, 0, TEST_MAX_TAIL_READ, LogDecode::CONSOLE)?.is_empty()); + Ok(()) + } + + // --- a source that grows after its size was read ------------------- + + /// A source whose underlying data grows by `appended` on the first read + /// after the size was taken, as a log does when its writer is still busy + /// between the `stat` and the read. + struct GrowsOnFirstRead { + cursor: std::io::Cursor>, + appended: Option>, + bytes_read: std::rc::Rc>, + } + + impl GrowsOnFirstRead { + fn new(content: Vec, appended: Vec) -> (Self, std::rc::Rc>) { + let bytes_read = std::rc::Rc::new(std::cell::Cell::new(0)); + let source = Self { + cursor: std::io::Cursor::new(content), + appended: Some(appended), + bytes_read: bytes_read.clone(), + }; + (source, bytes_read) + } + } + + impl Read for GrowsOnFirstRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(appended) = self.appended.take() { + self.cursor.get_mut().extend_from_slice(&appended); + } + let read = self.cursor.read(buf)?; + self.bytes_read.set(self.bytes_read.get() + read as u64); + Ok(read) + } + } + + impl Seek for GrowsOnFirstRead { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.cursor.seek(position) + } + } + + #[test] + fn a_source_that_grows_after_its_size_was_read_is_not_read_past_that_size() -> Result<()> { + // 64 KiB of 16-byte lines, then 64 KiB more appended after the size + // snapshot. With a budget of a quarter of the snapshot the window + // starts mid-file; with a budget above the snapshot there is no seek. + // Either way the tail must end where the snapshot ended and the read + // must touch no more than the window (plus the one probe byte). + let settled = "settled-line-xx\n".repeat(4096).into_bytes(); + let appended = "APPENDED-LINE-x\n".repeat(4096).into_bytes(); + let size = settled.len() as u64; + for (label, budget) in [ + ("window inside the file", size / 4), + ("whole file", size + 1), + ] { + let (source, bytes_read) = GrowsOnFirstRead::new(settled.clone(), appended.clone()); + + let lines = tail_lines_from( + source, + size, + Path::new("growing-source"), + usize::MAX, + budget, + LogDecode::PLAIN, + )? + .lines; + + assert!( + lines.iter().all(|line| line == "settled-line-xx"), + "{label}: read past the size snapshot into {:?}", + lines.iter().find(|line| line.starts_with("APPENDED")) + ); + assert_eq!(lines.last().map(String::as_str), Some("settled-line-xx")); + assert!( + bytes_read.get() <= budget.min(size) + 1, + "{label}: read {} bytes against a budget of {budget}", + bytes_read.get() + ); + } + Ok(()) + } + + #[test] + fn a_growing_sixteen_bit_source_is_not_read_past_its_size_either() -> Result<()> { + let settled = utf16_le_bytes(&"settled-line-xx\n".repeat(1024), true); + let appended = utf16_le_bytes(&"APPENDED-LINE-x\n".repeat(1024), false); + let size = settled.len() as u64; + let (source, _) = GrowsOnFirstRead::new(settled, appended); + + let lines = tail_lines_from( + source, + size, + Path::new("growing-source"), + usize::MAX, + size / 4, + LogDecode::CONSOLE, + )? + .lines; + + assert!(!lines.is_empty()); + assert!(lines.iter().all(|line| line == "settled-line-xx")); + Ok(()) + } + + // --- a final escape-only fragment ---------------------------------- + + /// What the ComfyUI log view returned before the tail routine replaced its + /// private reader: decode the whole file, strip ANSI sequences from the + /// whole text, then take the last `limit` of `str::lines`. + fn comfyui_reader_before_the_shared_routine(bytes: &[u8], limit: usize) -> Vec { + let looks_utf16 = bytes.len() >= 8 && { + let sample = &bytes[..bytes.len().min(512)]; + let zero_odd = sample + .iter() + .enumerate() + .filter(|(index, byte)| index % 2 == 1 && **byte == 0) + .count(); + zero_odd * 4 >= sample.len() + }; + let text = if bytes.starts_with(&[0xff, 0xfe]) || looks_utf16 { + let start = if bytes.starts_with(&[0xff, 0xfe]) { + 2 + } else { + 0 + }; + let units: Vec = bytes[start..] + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + String::from_utf16_lossy(&units) + } else { + String::from_utf8_lossy(bytes).into_owned() + }; + let text = strip_ansi_sequences(&text); + let mut lines: Vec = text.lines().rev().take(limit).map(str::to_owned).collect(); + lines.reverse(); + lines + } + + /// The same text as UTF-8 and as UTF-16 with and without a byte order mark. + fn console_encodings(text: &str) -> Vec<(&'static str, Vec)> { + vec![ + ("utf-8", text.as_bytes().to_vec()), + ("utf-16 with bom", utf16_le_bytes(text, true)), + ("utf-16 without bom", utf16_le_bytes(text, false)), + ] + } + + #[test] + fn a_colour_reset_after_the_final_newline_does_not_displace_the_last_line() -> Result<()> { + for (label, bytes) in console_encodings("\u{1b}[31mfailed\n\u{1b}[0m") { + let lines = read_bytes("reset", &bytes, 1, LogDecode::CONSOLE)?; + assert_eq!(lines, vec!["failed"], "{label}"); + } + Ok(()) + } + + #[test] + fn a_final_escape_only_fragment_is_not_a_line_but_blank_lines_are() -> Result<()> { + // (text, limit, expected): the unterminated escape-only tail never + // counts; a terminated blank line, even one made only of an escape + // sequence, is a real line and keeps its place. + let cases: [(&str, usize, &[&str]); 5] = [ + ("one\ntwo\n\u{1b}[0m", 2, &["one", "two"]), + ("one\ntwo\n\u{1b}[1;3", 2, &["one", "two"]), + ("one\ntwo\n\n", 2, &["two", ""]), + ("one\ntwo\n\u{1b}[0m\n", 2, &["two", ""]), + ("one\ntwo\n\u{1b}[0mtext", 2, &["two", "text"]), + ]; + for (text, limit, expected) in cases { + for (label, bytes) in console_encodings(text) { + let lines = read_bytes("fragment", &bytes, limit, LogDecode::CONSOLE)?; + assert_eq!(lines, expected, "{label}: {text:?} limit {limit}"); + } + } + Ok(()) + } + + #[test] + fn plain_decode_keeps_a_final_escape_fragment_because_it_strips_nothing() -> Result<()> { + let lines = read_bytes("plain-reset", b"a\n\x1b[0m", 2, LogDecode::PLAIN)?; + + assert_eq!(lines, vec!["a", "\u{1b}[0m"]); + Ok(()) + } + + #[test] + fn console_decode_matches_the_comfyui_reader_it_replaced_on_small_logs() -> Result<()> { + // Inputs on which the old reader and the per-line strip agree: no + // escape sequence runs across a newline. The old reader is the + // oracle, so a difference here is a change in what a user sees. + let texts = [ + "plain\nlines\n", + "\u{1b}[31mfailed\n\u{1b}[0m", + "\u{1b}[31mfailed\u{1b}[0m\n\u{1b}[0m", + "a\n\u{1b}[0m\n\u{1b}[0m", + "a\r\nb\r\n\u{1b}[0m", + "a\n\n\n", + "a\n\u{1b}[2K", + "\u{1b}[0m", + "", + ]; + for text in texts { + for (label, bytes) in console_encodings(text) { + for limit in 1..=4 { + let expected = comfyui_reader_before_the_shared_routine(&bytes, limit); + let actual = read_bytes("oracle", &bytes, limit, LogDecode::CONSOLE)?; + assert_eq!(actual, expected, "{label}: {text:?} limit {limit}"); + } + } + } + Ok(()) + } + + // --- an empty tail: no lines, or no line that fits the bound -------- + + fn read_log_bytes( + name: &str, + bytes: &[u8], + limit: usize, + max_tail_read: u64, + decode: LogDecode, + ) -> Result { + let path = + std::env::temp_dir().join(format!("rocm-core-tail-{name}-{}.log", unique_suffix())); + fs::write(&path, bytes)?; + let tail = tail_log_decoded(&path, limit, max_tail_read, decode); + fs::remove_file(&path).ok(); + tail + } + + #[test] + fn an_empty_tail_says_whether_the_log_is_empty_or_its_newest_line_is_too_long() -> Result<()> { + // The state matrix of an empty `lines`: (content, limit, whether the + // newest line exceeds the bound, bytes the budget left unread). + let long_line = "x".repeat(300); + let long_with_newline = format!("{long_line}\n"); + let long_utf16 = utf16_le_bytes(&long_line, true); + let cases: Vec<(&str, Vec, usize, bool, u64)> = vec![ + ("empty file", Vec::new(), 5, false, 0), + ( + "limit zero on a long line", + long_line.clone().into_bytes(), + 0, + false, + 0, + ), + ( + "long line without a newline", + long_line.into_bytes(), + 5, + true, + 300 - 64, + ), + ( + "long line with a newline", + long_with_newline.clone().into_bytes(), + 5, + true, + 301 - 64, + ), + ("long sixteen-bit line", long_utf16, 5, true, 602 - 64), + ( + "long line that ends a larger log", + format!("first\n{long_with_newline}").into_bytes(), + 5, + true, + 6 + 301 - 64, + ), + ("short log", b"a\nb\n".to_vec(), 5, false, 0), + ]; + for (label, content, limit, exceeds, skipped) in cases { + let tail = read_log_bytes("empty-tail", &content, limit, 64, LogDecode::CONSOLE)?; + assert_eq!( + tail.newest_line_exceeds_the_read_bound(), + exceeds, + "{label}: {tail:?}" + ); + assert_eq!(tail.skipped_bytes, skipped, "{label}: {tail:?}"); + // The flag is only ever set for an empty list, and the empty list + // only ever comes from the cases that name it. + if exceeds || content.is_empty() || limit == 0 { + assert!(tail.lines.is_empty(), "{label}: {tail:?}"); + } + } + Ok(()) + } + + #[test] + fn a_tail_that_found_lines_never_reports_an_exceeded_bound() -> Result<()> { + let content = "line\n".repeat(100).into_bytes(); + + let tail = read_log_bytes("found", &content, 5, 64, LogDecode::PLAIN)?; + + assert_eq!(tail.lines.len(), 5); + assert!(tail.skipped_bytes > 0, "the budget did leave bytes unread"); + assert!(!tail.newest_line_exceeds_the_read_bound()); + Ok(()) + } + + #[test] + fn a_byte_order_mark_is_not_counted_as_bytes_the_budget_skipped() -> Result<()> { + let content = utf16_le_bytes("a\nb\n", true); + + let tail = read_log_bytes("bom-skipped", &content, 5, 1024, LogDecode::CONSOLE)?; + + assert_eq!(tail.lines, vec!["a", "b"]); + assert_eq!(tail.skipped_bytes, 0); + Ok(()) + } + + #[test] + fn the_service_log_summary_names_each_of_the_three_states_apart() { + let none = LogTail::default(); + let too_long = LogTail { + lines: Vec::new(), + skipped_bytes: 1, + }; + let two = LogTail { + lines: vec!["a".to_owned(), "b".to_owned()], + skipped_bytes: 7, + }; + + assert_eq!( + service_log_tool_summary("svc", &none), + "Service `svc` has no log output yet." + ); + assert_eq!( + service_log_tool_summary("svc", &too_long), + "Service `svc` has log output, but none is shown: the newest line is longer than \ + the 4 MiB a log view reads." + ); + assert_eq!( + service_log_tool_summary("svc", &two), + "Read the last 2 line(s) from service `svc`." + ); + assert_eq!( + LOG_VIEW_MAX_TAIL_READ, + 4 * 1024 * 1024, + "the notice above names this many MiB" + ); + } } diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 15840e56e9..0ee01eeb3d 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -34,7 +34,24 @@ pub struct AppPaths { pub cache_dir: PathBuf, } +/// The variable that moves where engine virtualenvs live (see +/// [`AppPaths::engine_envs_root`]), and the one `rocm` sets on an engine child +/// that must install into a particular root. +pub const ENGINE_ENVS_ROOT_ENV: &str = "ROCM_CLI_ENGINE_ENVS_ROOT"; + impl AppPaths { + /// The three `ROCM_CLI_*_DIR` variables that make a child process run on + /// these roots, in the order config, data, cache. [`Self::discover`] reads + /// the same three back, so a child started with these finds the roots its + /// parent was given. + pub fn env_vars(&self) -> [(&'static str, &Path); 3] { + [ + ("ROCM_CLI_CONFIG_DIR", self.config_dir.as_path()), + ("ROCM_CLI_DATA_DIR", self.data_dir.as_path()), + ("ROCM_CLI_CACHE_DIR", self.cache_dir.as_path()), + ] + } + pub fn discover() -> Result { let data_dir_override = env_path_override("ROCM_CLI_DATA_DIR"); let cache_dir_override = env_path_override("ROCM_CLI_CACHE_DIR"); @@ -159,7 +176,7 @@ impl AppPaths { /// dropped; `engine_envs_dir_reads_its_root_from_the_environment` is what /// keeps the lookup honest meanwhile. pub fn engine_envs_root(&self) -> PathBuf { - self.engine_envs_root_from(env_path_override("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref()) + self.engine_envs_root_from(env_path_override(ENGINE_ENVS_ROOT_ENV).as_deref()) } /// [`Self::engine_envs_root`] against a caller-supplied override. @@ -343,6 +360,28 @@ mod tests { use crate::test_support::temp_app_paths; + #[test] + fn env_vars_name_the_three_roots_discover_reads_back() { + let (_root, paths) = temp_app_paths("env-vars"); + assert_eq!( + paths.env_vars(), + [ + ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), + ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), + ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), + ] + ); + // `discover` reads these same names; a rename in one place must not + // leave the other behind. + let source = include_str!("paths.rs"); + for (name, _) in paths.env_vars() { + assert!( + source.contains(&format!("env_path_override(\"{name}\")")), + "{name} is not read by AppPaths::discover" + ); + } + } + #[test] fn audit_events_path_lives_under_data_audit() { let (_root, paths) = temp_app_paths("audit-path"); diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 57da4bb1fe..831c9ef5a0 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -32,8 +32,10 @@ pub enum KillScope { /// A spawned process identified in a way that is robust to PID recycling. /// /// `start_ticks` is the kernel start-time of the process (clock ticks since -/// boot). It is `None` on platforms without `/proc`, where identity cannot be -/// verified and callers fall back to best-effort behaviour. +/// boot) on Linux, and its creation time as a `FILETIME` on Windows. It is +/// `None` on other platforms, and for a record written before Windows recorded +/// one, where identity cannot be verified and callers fall back to best-effort +/// behaviour. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProcessIdentity { pub pid: u32, @@ -88,7 +90,21 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { return IdentityState::Gone; } - match (id.start_ticks, process_start_ticks(id.pid)) { + classify_live_identity(id.start_ticks, process_start_ticks(id.pid)) +} + +/// The identity verdict for a PID that is running: what its recorded token +/// (`recorded`) says about the token read from the live process now (`live`). +/// +/// The one decision every stop path turns on, apart from the system reads, so +/// a test covers each case on every host. A process is refused only when a +/// token was recorded and the live one is provably different. A record with no +/// token (an older record, or one written where no token could be read) cannot +/// be refuted and is a match whatever the live process reads, so its stop and +/// its tree kill go ahead as they always did. A recorded token that cannot be +/// confirmed now is neither signalled nor reported stopped. +const fn classify_live_identity(recorded: Option, live: Option) -> IdentityState { + match (recorded, live) { (Some(expected), Some(actual)) => { if expected == actual { IdentityState::Matches @@ -104,6 +120,18 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { } } +/// The verdict that ends a termination before anything is signalled, when the +/// recorded identity says not to: `None` for a process that matches and so goes +/// on to be stopped. +const fn refusal_before_signalling(state: IdentityState) -> Option { + match state { + IdentityState::Gone => Some(TerminationOutcome::AlreadyGone), + IdentityState::Recycled => Some(TerminationOutcome::IdentityMismatch), + IdentityState::Indeterminate => Some(TerminationOutcome::Unverified), + IdentityState::Matches => None, + } +} + /// Whether `state` means the recorded process is definitively no longer running. /// /// [`IdentityState::Indeterminate`] is intentionally excluded: a process we can @@ -194,11 +222,8 @@ pub fn terminate_verified( grace: Duration, force: bool, ) -> TerminationOutcome { - match identity_state(id) { - IdentityState::Gone => return TerminationOutcome::AlreadyGone, - IdentityState::Recycled => return TerminationOutcome::IdentityMismatch, - IdentityState::Indeterminate => return TerminationOutcome::Unverified, - IdentityState::Matches => {} + if let Some(verdict) = refusal_before_signalling(identity_state(id)) { + return verdict; } let tree = matches!(scope, KillScope::Tree); @@ -261,6 +286,248 @@ pub fn terminate_verified( } } +/// How long a managed stop waits on a recorded process before it escalates. +/// +/// Each recorded process, tree included, gets this long to exit after `SIGTERM` +/// before the stop escalates to `SIGKILL`, and then this long again before it +/// reports a timeout. The one value behind `rocm services stop` and `rocmd`'s +/// `stop_server`: both stop the same records, so neither may wait for a +/// different time. +pub const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); + +/// What a managed stop did about one PID a service record names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedPidOutcome { + /// The PID is the stopping process's own. It is reported but never + /// signalled, and it counts as stopped: `rocmd` and an attached `rocm + /// serve` record themselves as the supervisor of the service they launched, + /// and the launcher is not the service. + OwnProcess, + /// The PID was handed to the verified termination, with this verdict. + Terminated(TerminationOutcome), +} + +impl RecordedPidOutcome { + /// The label a stop report carries: `self` for the stopping process's own + /// PID, otherwise [`TerminationOutcome::as_str`]. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::OwnProcess => "self", + Self::Terminated(outcome) => outcome.as_str(), + } + } +} + +/// One recorded PID's fate in a managed stop. +/// +/// Kept per PID because the outcomes that leave a stop unconfirmed are reported +/// differently: a `timed_out` PID was signalled and it, or a process under it, +/// outlived the deadline, while an `unverified` one was not signalled at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecordedPidStop { + pub pid: u32, + /// Which recorded PID this is: `supervisor`, `engine`, or + /// `supervisor_and_engine` when the record holds the same PID for both. + pub role: &'static str, + pub outcome: RecordedPidOutcome, +} + +impl RecordedPidStop { + /// Whether a signal went out to this PID. A `timed_out` PID counts: the + /// signal was sent and only the exit was never observed, which + /// [`stopped`](Self::stopped) reports separately. + #[must_use] + pub const fn signaled(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::TimedOut + ) + ) + } + + /// Whether the stop escalated to a forced kill for this PID. That is not + /// "the forced kill worked": a forced stop only reaches `timed_out` by way + /// of the escalation, and [`stopped`](Self::stopped) says whether the + /// process actually went away. + #[must_use] + pub const fn forced(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Forced | TerminationOutcome::TimedOut + ) + ) + } + + /// Whether the recorded process is confirmed no longer running. + #[must_use] + pub const fn stopped(self) -> bool { + match self.outcome { + RecordedPidOutcome::OwnProcess => true, + RecordedPidOutcome::Terminated(outcome) => outcome.stopped(), + } + } + + /// The entry a stop report lists for this PID under `pid_outcomes`. + #[must_use] + pub fn report_entry(self) -> serde_json::Value { + serde_json::json!({ + "pid": self.pid, + "role": self.role, + "outcome": self.outcome.as_str(), + }) + } +} + +/// Terminate the processes a service record names, verifying each PID's +/// identity first, and report what happened to each. +/// +/// PIDs are recycled, so a persisted PID is not by itself evidence that the +/// recorded process still holds it. Each PID is paired with **its own** +/// start-time token (`supervisor_start_ticks` with `supervisor_pid`, +/// `engine_start_ticks` with `engine_pid`) and handed to `terminate`, which +/// production supplies as [`terminate_recorded_pid`]. When the two PIDs +/// coincide, the entry is reported once, under the role `supervisor_and_engine`, +/// and keeps a known token in preference to a missing one: a token is never read +/// across from the PID it does not belong to. The supervisor is visited first. +/// +/// A zero PID names nothing and is skipped. The stopping process's own PID is +/// reported as [`RecordedPidOutcome::OwnProcess`] and `terminate` is not called +/// for it. +/// +/// `terminate` is injectable so a caller's test can supply the verdicts it +/// cannot construct on demand (a process that survives `SIGKILL`, one whose +/// identity cannot be read). +pub fn terminate_recorded_service_pids( + record: &crate::ManagedServiceRecord, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> Vec { + let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); + for (pid, role, ticks) in [ + ( + Some(record.supervisor_pid), + "supervisor", + record.supervisor_start_ticks, + ), + (record.engine_pid, "engine", record.engine_start_ticks), + ] { + let Some(pid) = pid.filter(|pid| *pid != 0) else { + continue; + }; + if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { + // Only the engine entry can land here, aliasing the supervisor: the + // one PID is both, and is reported as both. + existing.1 = "supervisor_and_engine"; + if existing.2.is_none() { + existing.2 = ticks; + } + } else { + entries.push((pid, role, ticks)); + } + } + + entries + .into_iter() + .map(|(pid, role, ticks)| { + let outcome = if pid == std::process::id() { + RecordedPidOutcome::OwnProcess + } else { + RecordedPidOutcome::Terminated(terminate(&ProcessIdentity::new(pid, ticks))) + }; + RecordedPidStop { pid, role, outcome } + }) + .collect() +} + +/// Terminate one recorded process, and its descendants, after verifying its +/// identity. The production `terminate` of [`terminate_recorded_service_pids`]. +/// +/// The stop is forced: a managed stop is definitive, so it must not report +/// success while an engine worker still holds the GPU. Only a process whose +/// recorded identity is confirmed is ever reached by it. +/// +/// On Windows the descendants are taken separately, first. [`terminate_verified`] +/// cannot walk a process tree there (`process_tree_pids` returns just the root), +/// and the engine runs one level below the recorded PID, because the daemon +/// launches it through an `__engine-serve-http` process: a root-only kill would +/// leave the engine running and still holding the device. +/// +/// The tree kill runs only for a PID that passes the same identity gate as the +/// rest of the stop. A record written with a start-time token carries the +/// process's creation time on Windows ([`process_start_ticks`]), so a PID that +/// an unrelated process has since taken reads back [`IdentityState::Recycled`] +/// and neither it nor its subtree is touched. A record written before Windows +/// recorded a token has none to refute, and for it the gate only turns away a +/// PID that is not running at all: that old record can still name a recycled +/// PID, and `taskkill /T /F` then takes that process and its subtree. A stop +/// that confirms clears the record's PIDs, so that exposure shrinks to the +/// records written before the token existed and not yet stopped. +pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome { + terminate_recorded_pid_with(identity, platform_tree_kill(), identity_state, |identity| { + terminate_verified(identity, KillScope::Tree, MANAGED_STOP_GRACE, true) + }) +} + +/// [`terminate_recorded_pid`] with its three platform and system touchpoints +/// supplied by the caller, so a test drives the tree-kill step on every host: +/// `tree_kill` takes one PID's descendants down (`None` where the platform +/// needs no separate step), `state_of` reads the PID's identity, and `terminate` +/// is the verified termination of the root. +fn terminate_recorded_pid_with( + identity: &ProcessIdentity, + tree_kill: Option<&dyn Fn(u32)>, + state_of: impl Fn(&ProcessIdentity) -> IdentityState, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> TerminationOutcome { + if let Some(tree_kill) = tree_kill + && matches!(state_of(identity), IdentityState::Matches) + { + // Taken while the root is still alive: `taskkill /T` resolves children + // through the live parent, so it cannot reach them once the root is gone. + tree_kill(identity.pid); + // `terminate` still owns the bounded wait and the verdict, but a root + // the tree kill already took now reads back as `AlreadyGone` — which + // would deny a stop this command did perform. + return match terminate(identity) { + TerminationOutcome::AlreadyGone => TerminationOutcome::Forced, + outcome => outcome, + }; + } + terminate(identity) +} + +/// The separate descendant kill this platform needs before the verified +/// termination of a root, if any: `taskkill /T /F` on Windows, none elsewhere, +/// where [`terminate_verified`] walks the tree itself. +// The `cfg` arms differ in what they return, so the non-Windows arm cannot be a +// `const fn` without giving the two platforms different signatures. +#[allow(clippy::missing_const_for_fn)] +fn platform_tree_kill() -> Option<&'static dyn Fn(u32)> { + #[cfg(windows)] + { + Some(&taskkill_tree) + } + #[cfg(not(windows))] + { + None + } +} + +#[cfg(windows)] +fn taskkill_tree(pid: u32) { + use std::process::{Command, Stdio}; + let _ = Command::new("taskkill") + .args(["/PID", &pid.to_string(), "/T", "/F"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); +} + /// Poll until every process in `members` has exited, or `grace` elapses. /// /// A member is exited once its PID is gone or now belongs to a different process @@ -300,8 +567,9 @@ fn send_signal(pid: u32, signal: Signal, _tree: bool) -> bool { /// Read the kernel start-time (field 22 of `/proc//stat`) for `pid`. /// -/// Returns `None` when the value cannot be read, including on non-Linux -/// platforms, where identity verification degrades to best-effort. +/// Returns `None` when the value cannot be read, including on platforms with +/// neither `/proc` nor a creation time to read (neither Linux nor Windows), +/// where identity verification degrades to best-effort. #[cfg(target_os = "linux")] #[must_use] pub fn process_start_ticks(pid: u32) -> Option { @@ -309,7 +577,56 @@ pub fn process_start_ticks(pid: u32) -> Option { parse_start_ticks(&stat) } -#[cfg(not(target_os = "linux"))] +/// Read the creation time of `pid` (`GetProcessTimes`) as the token that tells +/// the process from a later one that reuses its PID. +/// +/// The value is the creation time as a Windows `FILETIME` (100 ns intervals +/// since 1601), not clock ticks since boot: only equality with a value read +/// earlier from the same PID means anything. `None` when the process cannot be +/// opened for querying (it is gone, or belongs to a more privileged account). +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn process_start_ticks(pid: u32) -> Option { + use windows_sys::Win32::Foundation::{CloseHandle, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + if pid == 0 { + return None; + } + // SAFETY: plain Win32 calls. The handle is checked before use and closed on + // every path, and the four out-parameters are valid for the one call. + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return None; + } + let mut creation = FILETIME::default(); + let mut exit = FILETIME::default(); + let mut kernel = FILETIME::default(); + let mut user = FILETIME::default(); + let read = GetProcessTimes( + handle, + &raw mut creation, + &raw mut exit, + &raw mut kernel, + &raw mut user, + ); + CloseHandle(handle); + (read != 0).then(|| filetime_ticks(creation.dwLowDateTime, creation.dwHighDateTime)) + } +} + +/// The 64-bit value of a Windows `FILETIME`, which a Win32 call returns as two +/// 32-bit halves. Kept apart from the FFI so a test can run it on every host. +#[cfg(any(windows, test))] +fn filetime_ticks(low: u32, high: u32) -> u64 { + (u64::from(high) << 32) | u64::from(low) +} + +#[cfg(not(any(target_os = "linux", windows)))] #[must_use] // Platform stub. The `cfg` sibling calls non-const code, so making only this // arm `const` would give the two platforms different signatures and push @@ -445,7 +762,7 @@ mod tests { let _ = child.wait(); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_matches_our_own_process() { let pid = std::process::id(); @@ -454,7 +771,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Matches); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_mismatch_when_start_ticks_differ() { // Same live PID (ours), but a different recorded start-time: a recycled @@ -474,7 +791,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Gone); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn refuses_to_signal_on_identity_mismatch() { // Our own PID with a wrong start-time. A forced stop must NOT kill us: @@ -614,4 +931,549 @@ mod tests { ); reap(child); } + + // ---- The managed-stop routine `rocm` and `rocmd` share. ---- + + use std::cell::RefCell; + + /// A PID no host has in use. + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + fn record_naming( + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> crate::ManagedServiceRecord { + let paths = crate::AppPaths { + config_dir: std::path::PathBuf::from("config"), + data_dir: std::path::PathBuf::from("data"), + cache_dir: std::path::PathBuf::from("cache"), + }; + let mut record = crate::ManagedServiceRecord::new( + &paths, + "svc-managed-stop", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record + } + + /// The identities `terminate_recorded_service_pids` handed its termination + /// function, in call order, and the stops it reported. + fn stop_with( + record: &crate::ManagedServiceRecord, + outcome: TerminationOutcome, + ) -> (Vec, Vec) { + let seen = RefCell::new(Vec::new()); + let stops = terminate_recorded_service_pids(record, |identity| { + seen.borrow_mut().push(*identity); + outcome + }); + (seen.into_inner(), stops) + } + + #[test] + fn managed_stop_grace_is_ten_seconds() { + assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + } + + #[test] + fn managed_stop_terminates_supervisor_then_engine_each_with_its_own_start_time() { + let record = record_naming(SUPERVISOR_PID, Some(11), Some(ENGINE_PID), Some(22)); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ + ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + ProcessIdentity::new(ENGINE_PID, Some(22)), + ], + "the supervisor first, and no start-time read across from the other PID" + ); + assert_eq!( + stops + .iter() + .map(|stop| (stop.pid, stop.role)) + .collect::>(), + vec![(SUPERVISOR_PID, "supervisor"), (ENGINE_PID, "engine")] + ); + } + + #[test] + fn managed_stop_terminates_a_pid_recorded_twice_once_and_keeps_a_known_start_time() { + // Either entry may be the one that carries the token. + for (supervisor_ticks, engine_ticks, expected) in [ + (None, Some(33), Some(33)), + (Some(44), None, Some(44)), + (Some(44), Some(33), Some(44)), + (None, None, None), + ] { + let record = record_naming( + SUPERVISOR_PID, + supervisor_ticks, + Some(SUPERVISOR_PID), + engine_ticks, + ); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(SUPERVISOR_PID, expected)], + "{supervisor_ticks:?} / {engine_ticks:?}" + ); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + } + } + + #[test] + fn managed_stop_skips_a_zero_pid_and_an_absent_engine() { + let record = record_naming(0, None, Some(0), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert!(stops.is_empty()); + } + + #[test] + fn managed_stop_reports_its_own_pid_as_self_and_never_terminates_it() { + let own = std::process::id(); + let record = record_naming(own, None, Some(ENGINE_PID), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(ENGINE_PID, None)], + "the stopping process is never signalled" + ); + assert_eq!(stops.len(), 2); + assert_eq!(stops[0].pid, own); + assert_eq!(stops[0].role, "supervisor"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + assert_eq!(stops[0].outcome.as_str(), "self"); + assert!(!stops[0].signaled() && !stops[0].forced()); + assert!(stops[0].stopped(), "the launcher is not the service"); + assert_eq!( + stops[0].report_entry(), + serde_json::json!({"pid": own, "role": "supervisor", "outcome": "self"}) + ); + } + + #[test] + fn managed_stop_reports_a_pid_that_is_both_roles_and_its_own_as_self() { + let own = std::process::id(); + let record = record_naming(own, None, Some(own), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + } + + #[test] + fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { + use TerminationOutcome::{ + AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, + }; + let record = record_naming(SUPERVISOR_PID, None, None, None); + + // (outcome, signaled, forced, stopped) + let expected = [ + (AlreadyGone, false, false, true), + (IdentityMismatch, false, false, true), + (Unverified, false, false, false), + (Graceful, true, false, true), + (Forced, true, true, true), + (TimedOut, true, true, false), + ]; + for (outcome, signaled, forced, stopped) in expected { + let (_, stops) = stop_with(&record, outcome); + assert_eq!(stops.len(), 1); + let stop = stops[0]; + assert_eq!(stop.outcome, RecordedPidOutcome::Terminated(outcome)); + assert_eq!(stop.outcome.as_str(), outcome.as_str()); + assert_eq!( + (stop.signaled(), stop.forced(), stop.stopped()), + (signaled, forced, stopped), + "{outcome:?}" + ); + } + } + + /// What one `terminate_recorded_pid_with` call did, in order. + #[derive(Debug, PartialEq, Eq)] + enum Step { + TreeKill(u32), + Terminate(u32), + } + + /// Run `terminate_recorded_pid_with` for `pid`, with a tree-kill step + /// present or not, the identity in `state`, and the root's termination + /// answering `outcome`. Returns the verdict and the steps taken. + fn tree_kill_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| steps.borrow_mut().push(Step::TreeKill(pid)); + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, None), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32)), + |_| state, + |identity| { + steps.borrow_mut().push(Step::Terminate(identity.pid)); + outcome + }, + ); + (verdict, steps.into_inner()) + } + + #[test] + fn the_tree_is_killed_before_the_root_is_terminated() { + let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!( + steps, + vec![ + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID) + ], + "`taskkill /T` resolves children through the live parent, so the \ + tree kill has to come first" + ); + } + + #[test] + fn a_root_the_tree_kill_already_took_is_reported_forced_not_already_gone() { + let (verdict, _) = tree_kill_run( + true, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + verdict, + TerminationOutcome::Forced, + "the stop did perform the kill; reading the root back as gone must not deny it" + ); + } + + #[test] + fn the_verdict_after_a_tree_kill_is_the_terminations_own_for_every_other_outcome() { + use TerminationOutcome::{Forced, Graceful, IdentityMismatch, TimedOut, Unverified}; + for outcome in [Graceful, Forced, TimedOut, Unverified, IdentityMismatch] { + let (verdict, _) = tree_kill_run(true, IdentityState::Matches, outcome); + + assert_eq!(verdict, outcome); + } + } + + #[test] + fn no_tree_kill_is_run_for_a_pid_that_is_gone_recycled_or_unverifiable() { + for state in [ + IdentityState::Gone, + IdentityState::Recycled, + IdentityState::Indeterminate, + ] { + let (verdict, steps) = tree_kill_run(true, state, TerminationOutcome::AlreadyGone); + + assert_eq!( + steps, + vec![Step::Terminate(SUPERVISOR_PID)], + "{state:?}: a stranger's process tree must not be taken" + ); + assert_eq!( + verdict, + TerminationOutcome::AlreadyGone, + "{state:?}: and the verdict is the termination's, unmapped" + ); + } + } + + #[test] + fn without_a_tree_kill_step_the_root_is_terminated_and_its_verdict_unmapped() { + let (verdict, steps) = tree_kill_run( + false, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!(steps, vec![Step::Terminate(SUPERVISOR_PID)]); + assert_eq!(verdict, TerminationOutcome::AlreadyGone); + } + + /// Every way a recorded token and the live process's token can stand, with + /// what the stop does about each: whether the Windows tree kill runs, and the + /// verdict the stop reports (the label and whether it counts as stopped). + /// + /// The live process is running in every row; the tokens go through the same + /// decision production uses ([`classify_live_identity`] and + /// [`refusal_before_signalling`]), with the system reads and the signalling + /// supplied, so each row runs on every host. A process is refused only when + /// a token was recorded and the live one is provably different. + #[test] + fn what_a_stop_does_for_each_recorded_and_live_token() { + const LIVE: u64 = 0x01DC_0000_0000_0001; + struct Row { + name: &'static str, + recorded: Option, + live: Option, + tree_kill_runs: bool, + label: &'static str, + stopped: bool, + } + let row = |name, recorded, live, tree_kill_runs, label, stopped| Row { + name, + recorded, + live, + tree_kill_runs, + label, + stopped, + }; + let rows = [ + // The shape `service-stop-07` records, and any record older than the + // token: nothing to refute, so the whole tree is taken as before. + row("no token recorded", None, Some(LIVE), true, "forced", true), + row( + "no token recorded, none readable", + None, + None, + true, + "forced", + true, + ), + row( + "token matches", + Some(LIVE), + Some(LIVE), + true, + "forced", + true, + ), + // A recycled PID: the process is a stranger, left alone. + row( + "token differs", + Some(LIVE + 1), + Some(LIVE), + false, + "identity_mismatch", + true, + ), + // A recorded token that cannot be read back is neither signalled nor + // reported stopped. + row( + "token unreadable", + Some(LIVE), + None, + false, + "unverified", + false, + ), + ]; + for Row { + name: row, + recorded, + live, + tree_kill_runs, + label, + stopped, + } in rows + { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| steps.borrow_mut().push(Step::TreeKill(pid)); + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + Some(&tree_kill as &dyn Fn(u32)), + |id| classify_live_identity(id.start_ticks, live), + |id| { + steps.borrow_mut().push(Step::Terminate(id.pid)); + // What the verified termination answers: its refusal for a + // process the identity rules out, otherwise the tree kill + // has already taken the root and it reads back as gone. + refusal_before_signalling(classify_live_identity(id.start_ticks, live)) + .unwrap_or(TerminationOutcome::AlreadyGone) + }, + ); + let expected_steps = if tree_kill_runs { + vec![ + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + ] + } else { + vec![Step::Terminate(SUPERVISOR_PID)] + }; + assert_eq!(steps.into_inner(), expected_steps, "{row}: steps"); + assert_eq!(verdict.as_str(), label, "{row}: verdict"); + assert_eq!(verdict.stopped(), stopped, "{row}: counts as stopped"); + } + } + + #[test] + fn a_recorded_token_refutes_a_live_one_only_when_both_are_known_and_differ() { + use IdentityState::{Indeterminate, Matches, Recycled}; + assert_eq!(classify_live_identity(None, None), Matches); + assert_eq!(classify_live_identity(None, Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(8)), Recycled); + assert_eq!(classify_live_identity(Some(7), None), Indeterminate); + assert_eq!(refusal_before_signalling(Matches), None); + } + + #[test] + fn only_windows_has_a_separate_tree_kill_step() { + // Ties the injected tests above to what production actually runs: the + // step they exercise is the one `terminate_recorded_pid` has on Windows + // and does without elsewhere, where `terminate_verified` walks the tree. + assert_eq!(platform_tree_kill().is_some(), cfg!(windows)); + } + + #[test] + fn terminate_recorded_pid_reports_an_absent_pid_as_already_gone() { + let outcome = terminate_recorded_pid(&ProcessIdentity::new(SUPERVISOR_PID, None)); + + assert_eq!(outcome, TerminationOutcome::AlreadyGone); + } + + #[cfg(unix)] + #[test] + fn terminate_recorded_pid_stops_a_running_verified_process() { + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let id = ProcessIdentity::capture(child.id()); + + let outcome = terminate_recorded_pid(&id); + + assert!(outcome.stopped(), "{outcome:?}"); + assert!(matches!( + outcome, + TerminationOutcome::Graceful | TerminationOutcome::Forced + )); + reap(child); + assert!(!crate::process_is_running(id.pid)); + } + + #[test] + fn a_filetime_is_its_high_word_above_its_low_word() { + // The Windows creation time arrives as two 32-bit halves; swapping them + // or dropping the shift would make every process read as a different one. + assert_eq!( + filetime_ticks(0x89AB_CDEF, 0x0123_4567), + 0x0123_4567_89AB_CDEF + ); + assert_eq!(filetime_ticks(1, 0), 1); + assert_eq!(filetime_ticks(0, 1), 1 << 32); + assert_eq!(filetime_ticks(u32::MAX, u32::MAX), u64::MAX); + } + + /// The managed stop's own entry point refuses a PID whose recorded identity no + /// longer matches, and reaches neither it nor its tree. Our own PID with a + /// wrong start-time is what a recycled PID looks like; if the stop went + /// ahead, `taskkill /T /F` on Windows (a plain forced kill elsewhere) would + /// end this test process instead of asserting. On Windows this runs the real + /// tree-kill gate: the verdict has to come from the identity check, before + /// any `taskkill`. + #[cfg(any(target_os = "linux", windows))] + #[test] + fn terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches() { + let pid = std::process::id(); + let real = process_start_ticks(pid).expect("own start-time"); + let stale = ProcessIdentity::new(pid, Some(real.wrapping_add(1))); + + let outcome = terminate_recorded_pid(&stale); + + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!(crate::process_is_running(pid), "nothing was signalled"); + } + + /// The same refusal against a bystander that is a different process holding + /// the recorded PID (the recycled case), then the stop of that process when + /// its identity does match. Windows only: that is where a tree kill by PID + /// follows the identity gate, and where a creation-time token is read. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + let real = process_start_ticks(pid).expect("a live process has a creation time"); + assert_eq!( + process_start_ticks(pid), + Some(real), + "the creation time of one process does not change" + ); + assert_ne!( + Some(real), + process_start_ticks(std::process::id()), + "two processes do not share one" + ); + + let outcome = + terminate_recorded_pid(&ProcessIdentity::new(pid, Some(real.wrapping_add(1)))); + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!( + child.try_wait().expect("check the bystander").is_none(), + "a PID whose recorded creation time differs is a stranger and must keep running" + ); + + let outcome = terminate_recorded_pid(&ProcessIdentity::capture(pid)); + assert!(outcome.stopped(), "{outcome:?}"); + let _ = child.wait(); + assert!(!crate::process_is_running(pid)); + } + + /// A record with no token (written before Windows recorded one, and the shape + /// `service-stop-07` plants) cannot be refuted and still gets the whole stop: + /// the process ends. Windows only, where a real process is stopped by PID. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_stops_a_process_recorded_without_a_token() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + + let outcome = terminate_recorded_pid(&ProcessIdentity::new(pid, None)); + + assert!(outcome.stopped(), "{outcome:?}"); + assert_eq!(outcome.as_str(), "forced"); + assert!( + child.wait().is_ok() && !crate::process_is_running(pid), + "the process recorded without a token must be gone" + ); + } } diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index aecab92d8c..bf405c3728 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -322,6 +322,25 @@ impl ManagedServiceRecord { self.engine_start_ticks = crate::process_start_ticks(pid); } + /// Forget the processes this record names: the supervisor and the engine, + /// each with its start-time token. + /// + /// For a stop that has confirmed every one of them gone, and for nothing + /// else. A recorded PID goes stale the instant its process exits, and a + /// later stop of the same record would signal whatever the OS has since + /// handed that number to; a PID recorded without a token cannot be + /// refuted, and on Windows none can, so the tree kill would take that + /// process and everything under it. `rocm services stop` and `rocmd`'s stop + /// both call this on the confirmed path, so the two leave the same record + /// behind. An unconfirmed stop keeps the PIDs: the record is the only thing + /// that names a survivor. + pub const fn clear_recorded_processes(&mut self) { + self.supervisor_pid = 0; + self.supervisor_start_ticks = None; + self.engine_pid = None; + self.engine_start_ticks = None; + } + pub fn normalize_paths_for_host(&mut self) { self.manifest_path = normalize_runtime_path_for_host(&self.manifest_path); self.log_path = normalize_runtime_path_for_host(&self.log_path); @@ -745,6 +764,35 @@ mod tests { assert_eq!(record.engine_start_ticks, expected); } + /// What a stop that confirmed its processes gone leaves behind: no PID and + /// no token for either role, and nothing else touched. + #[test] + fn clearing_the_recorded_processes_clears_each_pid_and_its_token_and_nothing_else() { + let mut record = probe_test_record(11437); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.status = "ready".to_owned(); + record.restart_count = 3; + let before = record.clone(); + + record.clear_recorded_processes(); + + assert_eq!(record.supervisor_pid, 0); + assert_eq!(record.supervisor_start_ticks, None); + assert_eq!(record.engine_pid, None); + assert_eq!(record.engine_start_ticks, None); + assert_eq!(record.service_id, before.service_id); + assert_eq!(record.status, "ready"); + assert_eq!(record.restart_count, 3); + assert_eq!(record.port, before.port); + // A cleared record no longer names what it named, so a stop that read it + // before and finds this after does not claim the same processes. + assert!(!before.names_same_processes(&record)); + assert!(record.names_same_processes(&record.clone())); + } + #[test] fn engine_state_refresh_keeps_the_runtime_selector_the_launch_requested() -> Result<()> { let (root, paths) = temp_app_paths("record-runtime-pin"); diff --git a/crates/rocm-core/src/sigpipe.rs b/crates/rocm-core/src/sigpipe.rs new file mode 100644 index 0000000000..68192204b1 --- /dev/null +++ b/crates/rocm-core/src/sigpipe.rs @@ -0,0 +1,252 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Writing to a pipe whose reader has gone away, without being killed for it. +//! +//! Rust starts every process with SIGPIPE ignored, so a write to a closed pipe +//! returns `EPIPE`. `rocm` restores the default action at startup so it ends +//! quietly when its own output is piped into `head`, and `rocm daemon` +//! hosts the daemon in that same process. Under the default action a write to +//! a child that has already exited kills the writer before it can report why +//! the child exited, so code that writes to a child's stdin wraps the write in +//! [`with_sigpipe_ignored`]. + +/// Run `f` with SIGPIPE ignored, restoring the previous disposition afterwards. +/// +/// A write that reaches a closed pipe inside `f` then returns an `EPIPE` error +/// instead of delivering a signal. Where the disposition was already ignored +/// (Rust's startup default) this changes nothing. The disposition is +/// process-wide, so keep `f` to the write itself. Windows has no SIGPIPE and +/// runs `f` as is. +/// +/// Guarded regions may overlap, on one thread or on several (the stderr +/// forwarder of one engine request runs while another request writes to its +/// child). The disposition in force before the first region is remembered once +/// and put back when the last region ends, so an overlap can neither restore +/// "ignored" over a default action nor leave "ignored" behind. +#[cfg(unix)] +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + let _guard = unix::IgnoreGuard::enter(); + f() +} + +#[cfg(unix)] +#[allow(unsafe_code)] // libc FFI +mod unix { + use std::sync::{Mutex, PoisonError}; + + /// How many guarded regions are open, and the disposition that was in force + /// before the first of them. + static OPEN: Mutex<(usize, libc::sighandler_t)> = Mutex::new((0, libc::SIG_DFL)); + + /// Holds SIGPIPE ignored from [`IgnoreGuard::enter`] until it is dropped, so + /// a panic inside the guarded region still restores the disposition. + pub(super) struct IgnoreGuard; + + impl IgnoreGuard { + pub(super) fn enter() -> Self { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + if open.0 == 0 { + // SAFETY: reading/installing a signal disposition is always safe. + open.1 = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + } + open.0 += 1; + Self + } + } + + impl Drop for IgnoreGuard { + fn drop(&mut self) { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + open.0 -= 1; + if open.0 == 0 { + // SAFETY: restoring the disposition captured by the first guard. + unsafe { + libc::signal(libc::SIGPIPE, open.1); + } + } + } + } +} + +#[cfg(not(unix))] +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + f() +} + +#[cfg(all(test, unix))] +mod tests { + use super::with_sigpipe_ignored; + use std::io::Write as _; + use std::os::unix::process::ExitStatusExt as _; + use std::process::{Command, ExitStatus, Stdio}; + + /// The disposition SIGPIPE has now (read by installing the same one back). + #[allow(unsafe_code)] // libc::signal + fn current_disposition() -> libc::sighandler_t { + // SAFETY: reading/installing a signal disposition is always safe. + let current = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + // SAFETY: putting back what was just read. + unsafe { + libc::signal(libc::SIGPIPE, current); + } + current + } + + /// Guarded regions that overlap, nested on one thread or interleaved across + /// two, leave SIGPIPE ignored for as long as any of them is open and put the + /// original default action back only when the last one closes. With a guard + /// that saved and restored per call, the region that ended first would + /// restore "ignored" over the second's `SIG_DFL` capture, or restore the + /// default under the region still open. Runs in a re-executed child, since + /// the disposition is process-wide and the harness leaves it ignored. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn overlapping_guards_restore_the_original_disposition_once() { + if std::env::var_os("ROCM_TEST_SIGPIPE_OVERLAP_ARM").is_some() { + use std::sync::Barrier; + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + + // Nested on one thread. + with_sigpipe_ignored(|| { + with_sigpipe_ignored(|| {}); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the inner region ended while the outer one was open" + ); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "nested"); + + // Two threads whose regions overlap, the first to enter leaving + // first. The second thread waits until the first is inside its + // region before it enters its own, so the order is the same on + // every run: were it left to the scheduler, a per-call save and + // restore would pass whenever the "second" closure happened to run + // first. + let first_in = Barrier::new(2); + let entered = Barrier::new(2); + let first_left = Barrier::new(2); + std::thread::scope(|scope| { + scope.spawn(|| { + with_sigpipe_ignored(|| { + first_in.wait(); + entered.wait(); + }); + first_left.wait(); + }); + scope.spawn(|| { + first_in.wait(); + with_sigpipe_ignored(|| { + entered.wait(); + first_left.wait(); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the first region ended while this one was open" + ); + }); + }); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "interleaved"); + return; + } + + let status = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "sigpipe::tests::overlapping_guards_restore_the_original_disposition_once", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_OVERLAP_ARM", "1") + .status() + .expect("re-exec the test arm in a child process"); + assert!(status.success(), "{status:?}"); + } + + /// A write to a dead child's stdin under the default SIGPIPE action is + /// fatal, and the same write inside [`with_sigpipe_ignored`] returns an + /// error instead. + /// + /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE + /// at Rust's default `SIG_IGN`, and flipping it process-wide would race + /// sibling tests. So this re-execs itself in fresh processes that install + /// `SIG_DFL` first, and checks BOTH directions: + /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — + /// this is the regression the plain `SIG_DFL` reset introduced; + /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns + /// an error instead — this is the fix. + /// + /// Each child reaps its short-lived grandchild before writing, so the read + /// end is closed deterministically and the outcome does not race. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { + // Grandchild helper: install the default action, spawn a process that + // exits immediately, reap it, then write to its now-closed stdin. The + // "guarded" arm wraps the write; the "unguarded" arm does not. + if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let mut grandchild = Command::new("true") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a process that exits immediately"); + let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); + grandchild.wait().expect("reap grandchild"); + // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; + // it must exceed the pipe buffer so the write actually reaches the + // closed read end rather than being swallowed by kernel buffering. + let payload = vec![b'x'; 64 * 1024]; + if mode == "guarded" { + let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); + assert!( + result.is_err(), + "guarded write to a dead pipe should return BrokenPipe, not succeed" + ); + } else { + // Unguarded: under SIG_DFL this write delivers SIGPIPE and never + // returns. If the process somehow survives, exit 0 so the parent's + // "should have been signalled" assertion fails loudly. + let _ = stdin.write_all(&payload); + } + return; + } + + let exe = std::env::current_exe().expect("current test executable"); + let run_arm = |mode: &str| -> ExitStatus { + Command::new(&exe) + .args([ + "sigpipe::tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_ARM", mode) + .status() + .expect("re-exec the test arm in a child process") + }; + + let unguarded = run_arm("unguarded"); + assert_eq!( + unguarded.signal(), + Some(libc::SIGPIPE), + "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ + SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" + ); + + let guarded = run_arm("guarded"); + assert!( + guarded.success(), + "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ + SIGPIPE reached the process instead of surfacing as an error" + ); + } +} diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index 19bd5f483a..e0eb049751 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -10,7 +10,17 @@ publish.workspace = true [lints] workspace = true +[features] +# Compiles `fake_engine`, a stand-in engine child for tests, into the `rocm` and +# `rocmd` test builds. Never enabled in a release build. +test-support = [] + [dependencies] +anyhow.workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test installs in a re-executed child. +libc.workspace = true diff --git a/crates/rocm-engine-protocol/src/fake_engine.rs b/crates/rocm-engine-protocol/src/fake_engine.rs new file mode 100644 index 0000000000..dc90a42449 --- /dev/null +++ b/crates/rocm-engine-protocol/src/fake_engine.rs @@ -0,0 +1,174 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A stand-in engine child for tests. +//! +//! It is a `/bin/sh` script that records how it was launched (arguments, +//! environment, stdin) into a scratch directory and then runs a caller-supplied +//! body, so a test can assert what an engine IPC client put on the child's +//! command line, environment and stdin, and shape what comes back (a response +//! envelope, a crash, stderr output). +//! +//! Test-only: compiled for this crate's own tests and for the `rocm` and +//! `rocmd` test builds through the `test-support` feature, so the two engine +//! IPC callers are exercised against the same fake rather than each growing a +//! private one. Unix-only, because the fake is a shell script; the pure parts +//! of the client are tested on every platform without it. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; + +static NEXT_ENGINE: AtomicU32 = AtomicU32::new(0); + +/// `value` as a single-quoted POSIX shell word. Inside single quotes nothing is +/// special except the quote itself, which cannot be escaped there: close the +/// literal, emit an escaped quote, and reopen it (`'\''`). An apostrophe in a +/// response (`can't stop`) or in `TMPDIR` would otherwise end the literal early. +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', r"'\''")) +} + +pub struct FakeEngine { + dir: PathBuf, + script: PathBuf, +} + +impl FakeEngine { + /// Write the script. `reads_stdin` makes the fake drain its stdin into + /// `stdin` first (what a well-behaved engine does); pass `false` for an + /// engine that exits without reading its request. `body` runs after the + /// recording and may use `$dir`, the scratch directory. + /// + /// The script is written through [`rocm_core::fake_executable`], so running + /// it straight away cannot fail with `ETXTBSY` while other tests on parallel + /// threads spawn children of their own. + pub fn new(label: &str, reads_stdin: bool, body: &str) -> Self { + Self::new_in(&std::env::temp_dir(), label, reads_stdin, body) + } + + /// [`FakeEngine::new`] with its scratch directory created under `base` + /// instead of the system temporary directory. + pub fn new_in(base: &Path, label: &str, reads_stdin: bool, body: &str) -> Self { + let dir = base.join(format!( + "rocm-fake-engine-{label}-{}-{}", + std::process::id(), + NEXT_ENGINE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&dir).expect("create the fake engine's scratch directory"); + let script = dir.join("engine.sh"); + let read_stdin = if reads_stdin { + "cat > \"$dir/stdin\"\n" + } else { + "" + }; + rocm_core::fake_executable::write_fake_executable( + &script, + format!( + "#!/bin/sh\ndir={}\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", + shell_quote(&dir.to_string_lossy()) + ), + ) + .expect("write the fake engine script"); + Self { dir, script } + } + + /// A fake that reads its request and answers with `response` (a complete + /// response envelope as JSON text), exiting with `status`. + pub fn answering(label: &str, response: &str, status: i32) -> Self { + Self::new( + label, + true, + &format!("printf '%s' {}\nexit {status}", shell_quote(response)), + ) + } + + pub fn path(&self) -> &Path { + &self.script + } + + /// The arguments the fake was started with, one per entry. + pub fn argv(&self) -> Vec { + fs::read_to_string(self.dir.join("argv")) + .expect("the fake engine never recorded its arguments") + .lines() + .map(str::to_owned) + .collect() + } + + /// Everything the fake read from its stdin, verbatim. + pub fn stdin(&self) -> String { + fs::read_to_string(self.dir.join("stdin")).expect("the fake engine never read its stdin") + } + + /// The value of `key` in the fake's environment, if it was set. + pub fn env(&self, key: &str) -> Option { + let prefix = format!("{key}="); + fs::read_to_string(self.dir.join("env")) + .expect("the fake engine never recorded its environment") + .lines() + .find_map(|line| line.strip_prefix(&prefix).map(str::to_owned)) + } +} + +impl Drop for FakeEngine { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + use std::process::{Command, Stdio}; + + /// Run `engine` the way a client does: request on stdin, answer on stdout. + fn run(engine: &FakeEngine) -> (std::process::ExitStatus, String, String) { + let mut child = Command::new(engine.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"request\n").unwrap(); + let output = child.wait_with_output().unwrap(); + ( + output.status, + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) + } + + #[test] + fn shell_quote_survives_each_character_a_literal_can_hold() { + assert_eq!(shell_quote("plain"), "'plain'"); + assert_eq!(shell_quote("can't"), r"'can'\''t'"); + assert_eq!(shell_quote("''"), r"''\'''\'''"); + assert_eq!(shell_quote(""), "''"); + } + + #[test] + fn a_response_with_an_apostrophe_is_printed_verbatim() { + let response = r#"{"ok":false,"error":{"code":"x","message":"can't stop"}}"#; + let engine = FakeEngine::answering("fake-apostrophe-response", response, 0); + let (status, stdout, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stdout, response); + assert_eq!(stderr, "", "the script must parse"); + } + + #[test] + fn a_scratch_directory_with_an_apostrophe_still_records_the_launch() { + let base = + std::env::temp_dir().join(format!("rocm-fake-engine-it's-{}", std::process::id())); + let engine = FakeEngine::new_in(&base, "fake-apostrophe-dir", true, "exit 0"); + let (status, _, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stderr, "", "the script must parse"); + assert_eq!(engine.stdin(), "request\n"); + drop(engine); + let _ = fs::remove_dir_all(&base); + } +} diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6ca773ccd6..026d0d772e 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -2,6 +2,15 @@ // // SPDX-License-Identifier: MIT +#[cfg(all(unix, any(test, feature = "test-support")))] +pub mod fake_engine; + +mod stdio_client; +pub use stdio_client::{ + ENGINE_PROGRESS_STDERR_ENV, EngineStdioOptions, decode_engine_response, + engine_request_envelope, run_engine_stdio, +}; + use serde::Deserialize; use serde::Serialize; use serde_json::Value; @@ -30,24 +39,6 @@ pub const DEFAULT_LOG_TAIL_LINES: usize = 80; /// (e.g. a loopback bind). pub const ENDPOINT_API_KEY_FILE_ENV: &str = "ROCM_SERVE_API_KEY_FILE"; -/// Whether `host` names a public (non-loopback) bind, i.e. one that must be -/// protected by an endpoint API key. -/// -/// Shared so `rocm` (which decides at `serve` time whether to mint a key) and -/// `rocmd` (which respawns recorded services during recovery) apply the same -/// policy to the same `ManagedServiceRecord::host` value. -/// -/// Deliberately an exact match against the three loopback spellings the CLI -/// accepts, so anything unrecognized — an IPv6 loopback alias, a differently -/// cased `localhost`, a hostname that happens to resolve locally — is treated -/// as public and therefore *requires* a key. Misclassifying a loopback bind as -/// public costs a key nobody needed; the reverse would open an unauthenticated -/// network listener. -#[must_use] -pub fn is_public_bind_host(host: &str) -> bool { - !matches!(host, "127.0.0.1" | "localhost" | "::1") -} - /// Resolve the endpoint API key an engine adapter must enforce, if any. /// /// Reads the key file named by [`ENDPOINT_API_KEY_FILE_ENV`]. Returns `None` when @@ -280,6 +271,17 @@ impl GpuSelection { } } +/// The two arguments `flag value` when `value` is set, none when it is not, for +/// building the command line of an engine or supervisor child. Shared so `rocm` +/// and `rocmd` spell optional flags the same way. +#[must_use] +pub fn optional_arg(flag: &str, value: Option<&str>) -> Vec { + match value { + Some(value) => vec![flag.to_owned(), value.to_owned()], + None => Vec::new(), + } +} + /// Render an explicit GPU ordinal as a comma-separated `HIP_VISIBLE_DEVICES` /// value. Returns `None` when the slice is empty (no pinning requested). #[must_use] @@ -822,27 +824,6 @@ mod tests { std::fs::remove_dir_all(&paths.data_dir).ok(); } - #[test] - fn is_public_bind_host_recognizes_only_the_accepted_loopback_spellings() { - for host in ["127.0.0.1", "localhost", "::1"] { - assert!(!is_public_bind_host(host), "{host} must be loopback"); - } - for host in [ - "0.0.0.0", - "::", - "192.168.1.10", - "example.test", - // Conservative on purpose: an unrecognized spelling of a loopback - // address is classified public, so it requires a key rather than - // opening an unauthenticated listener. - "LOCALHOST", - "127.0.0.2", - "[::1]", - ] { - assert!(is_public_bind_host(host), "{host} must be public"); - } - } - #[test] fn request_roundtrip_preserves_method() { let envelope = EngineRequestEnvelope { @@ -892,6 +873,15 @@ mod tests { assert!(GpuSelection::parse_cli_value("0,2,3").is_err()); } + #[test] + fn optional_arg_is_the_flag_and_value_or_nothing() { + assert_eq!( + optional_arg("--runtime-id", Some("therock-release:gfx120X-all")), + ["--runtime-id", "therock-release:gfx120X-all"] + ); + assert!(optional_arg("--runtime-id", None).is_empty()); + } + #[test] fn gpu_indices_helpers_handle_auto_and_pinned() { assert_eq!(gpu_indices_to_csv(&[]), None); diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs new file mode 100644 index 0000000000..ed7c9c1fd4 --- /dev/null +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -0,0 +1,781 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The client side of the engine stdio protocol: run an engine as a child +//! process, send it one [`EngineRequestEnvelope`] on stdin as a single line of +//! JSON, and read one [`EngineResponseEnvelope`] back from stdout. +//! +//! `rocm` and `rocmd` both talk to engines this way and share this one +//! implementation. They differ in *which* process they start (`rocm` runs the +//! resolved engine binary with `stdio`, or answers a built-in engine in its own +//! process without a child at all; `rocmd` runs `rocm __engine-stdio `), +//! and that stays with each caller. What is shared is everything that follows +//! the choice of program: the request framing, the environment, how stderr is +//! handled, the write guard against SIGPIPE, what a failing exit status or an +//! unusable answer is reported as, and decoding the response. The few things +//! the callers want differently are the fields of [`EngineStdioOptions`]. +//! +//! The request and the child's lifetime are not bounded by a timeout: an engine +//! install legitimately runs for minutes, and no caller has ever set one. + +use std::ffi::OsStr; +use std::io::{self, BufRead, Read, Write}; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::thread; + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, with_sigpipe_ignored}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::{ + ENDPOINT_API_KEY_FILE_ENV, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, +}; + +/// Set to `1` on an engine child whose caller wants its progress lines on +/// stderr as they happen (an engine install). +pub const ENGINE_PROGRESS_STDERR_ENV: &str = "ROCM_ENGINE_PROGRESS_STDERR"; + +/// What a caller wants different about the child [`run_engine_stdio`] starts. +/// +/// The default asks for nothing beyond the request itself: the child inherits +/// the caller's environment and its stderr is only collected. +#[derive(Debug, Clone, Copy, Default)] +pub struct EngineStdioOptions<'a> { + /// Run the child on these roots, by setting the `ROCM_CLI_*_DIR` variables + /// ([`AppPaths::env_vars`]). `None` leaves the child to inherit whatever the + /// caller's own environment says. + pub app_paths: Option<&'a AppPaths>, + /// The root engine virtualenvs go under for this request, set as + /// [`ENGINE_ENVS_ROOT_ENV`]. Only an install into a particular root needs it. + pub env_root: Option<&'a Path>, + /// A key file to hand the engine through [`ENDPOINT_API_KEY_FILE_ENV`], so + /// its HTTP probe of a protected endpoint authenticates. The caller decides + /// whether a file holds a usable key; this only carries it. + pub endpoint_key_file: Option<&'a Path>, + /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). + pub request_progress: bool, + /// Write the child's stderr to this process's stderr as it arrives, so a + /// terminal user watches an install. Either way the whole of it is kept and + /// shown in the error when the request fails. + pub forward_stderr: bool, +} + +/// Build the request envelope for `method` from `request`. +pub fn engine_request_envelope( + method: EngineMethod, + request: &T, +) -> Result { + Ok(EngineRequestEnvelope { + method, + payload: serde_json::to_value(request) + .context("failed to encode engine request payload")?, + }) +} + +/// The command [`run_engine_stdio`] runs: `program` with `args`, all three +/// standard streams piped, and the environment `options` asks for. Separate so +/// a test can read the command line and environment without starting anything. +fn engine_stdio_command(program: &Path, args: I, options: &EngineStdioOptions<'_>) -> Command +where + I: IntoIterator, + S: AsRef, +{ + let mut command = Command::new(program); + command + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(paths) = options.app_paths { + for (key, value) in paths.env_vars() { + command.env(key, value); + } + } + if let Some(env_root) = options.env_root { + command.env(ENGINE_ENVS_ROOT_ENV, env_root); + } + if let Some(key_file) = options.endpoint_key_file { + command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); + } + if options.request_progress { + command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); + } + command +} + +/// Send `envelope` to the engine child `program args…` and return what it +/// answered. +/// +/// A failing exit status is an error only when the child printed no answer: an +/// engine that reports its own failure in an error envelope and then exits +/// non-zero is read as that envelope, which says more than the status does. +/// Every other failure names what went wrong and carries the child's stderr. +pub fn run_engine_stdio( + program: &Path, + args: I, + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, +) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let mut child = engine_stdio_command(program, args, options) + .spawn() + .with_context(|| format!("failed to spawn engine stdio process {}", program.display()))?; + + let mut stdin = child + .stdin + .take() + .context("engine stdio child did not expose stdin")?; + // Ignore SIGPIPE for the duration of the write. Under the default action, an + // engine that exits before reading its stdin would kill this process with + // SIGPIPE (`rocm` runs that way, and so does the daemon hosted in it); + // ignored, the write returns EPIPE and fails as an ordinary error. + let written = with_sigpipe_ignored(|| { + serde_json::to_writer(&mut stdin, envelope).context("failed to write engine request")?; + stdin.write_all(b"\n")?; + Ok::<(), anyhow::Error>(()) + }); + if let Err(error) = written { + // The engine will never get its request, so do not leave it running or + // unreaped: a long-lived caller would collect one zombie per failure. + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + // End of the request: the engine reads to EOF before it answers. + drop(stdin); + + let forward_stderr = options.forward_stderr; + let stderr_reader = child.stderr.take().map(|stderr| { + thread::spawn(move || collect_stderr(stderr, forward_stderr.then(io::stderr))) + }); + let stdout_pipe = child + .stdout + .take() + .context("engine stdio child did not expose stdout")?; + let stdout = read_response(&mut child, stdout_pipe)?; + let status = child + .wait() + .context("failed waiting for engine stdio response")?; + let stderr = stderr_reader + .map(|reader| reader.join().unwrap_or_default()) + .unwrap_or_default(); + + if !status.success() && stdout.is_empty() { + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if stderr.is_empty() { + bail!("engine stdio process exited with status {status}"); + } + bail!("engine stdio process exited with status {status}: {stderr}"); + } + serde_json::from_slice(&stdout).with_context(|| { + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if stderr.is_empty() { + "failed to parse engine response envelope".to_owned() + } else { + format!("failed to parse engine response envelope; stderr: {stderr}") + } + }) +} + +/// Read the child's whole answer from `stdout`. +/// +/// A failed read would leave the child running with nobody waiting on it, and +/// this client lives in the long-lived daemon, so the child is killed and +/// reaped before the error is returned, as it is when the request cannot be +/// written. The stderr collector is not joined on that path: a grandchild that +/// inherited the pipe could hold it open, and the thread ends by itself once +/// the pipe closes. +fn read_response(child: &mut Child, mut stdout: impl Read) -> Result> { + let mut response = Vec::new(); + if let Err(error) = stdout.read_to_end(&mut response) { + let _ = child.kill(); + let _ = child.wait(); + return Err(anyhow::Error::new(error).context("failed to read engine stdio response")); + } + Ok(response) +} + +/// Read `stderr` to the end line by line, optionally writing each line to +/// `forward` (this process's stderr, in production) as it arrives, and return +/// everything read. A failed forward is ignored: a closed terminal must not +/// cost the request its answer. That holds for the signal as well as the error: +/// under SIGPIPE's default action (`rocm` runs that way) a write to a closed +/// pipe ends the whole process before `write_all` can return, so each forward +/// runs inside [`with_sigpipe_ignored`] (see [`forward_best_effort`]). A read error ends the +/// collection and is itself recorded as the last line. +fn collect_stderr(stderr: impl Read, mut forward: Option) -> Vec { + let mut reader = io::BufReader::new(stderr); + let mut collected = Vec::new(); + let mut line = Vec::new(); + loop { + line.clear(); + match reader.read_until(b'\n', &mut line) { + Ok(0) => break, + Ok(_) => { + if let Some(sink) = forward.as_mut() { + forward_best_effort(sink, &line); + } + collected.extend_from_slice(&line); + } + Err(error) => { + let message = format!("failed to read engine progress: {error}\n"); + if let Some(sink) = forward.as_mut() { + forward_best_effort(sink, message.as_bytes()); + } + collected.extend_from_slice(message.as_bytes()); + break; + } + } + } + collected +} + +/// Write `bytes` to the forwarding sink and drop the result: a sink that has +/// gone away must not cost the request its answer. SIGPIPE is ignored for the +/// write, so a closed pipe comes back as the error that is then discarded +/// rather than as a signal that kills the process. +fn forward_best_effort(sink: &mut impl Write, bytes: &[u8]) { + let _ = with_sigpipe_ignored(|| sink.write_all(bytes)); +} + +/// Turn an engine's answer into the response type the caller asked for: the +/// engine's own error when it reported one, otherwise the decoded `data`. +pub fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result +where + R: DeserializeOwned, +{ + if !envelope.ok { + let error = envelope.error.map_or_else( + || "unknown engine error".to_owned(), + |value| format!("{}: {}", value.code, value.message), + ); + bail!("{error}"); + } + let data = envelope + .data + .context("engine response envelope did not contain data")?; + serde_json::from_value(data).context("failed to decode engine response payload") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EngineErrorDetail, StopResponse}; + use serde_json::{Value, json}; + use std::collections::BTreeMap; + + /// The environment `command` would add to its child, as plain strings. + fn added_env(command: &Command) -> BTreeMap { + command + .get_envs() + .filter_map(|(key, value)| { + Some(( + key.to_string_lossy().into_owned(), + value?.to_string_lossy().into_owned(), + )) + }) + .collect() + } + + fn args_of(command: &Command) -> Vec { + command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect() + } + + #[test] + fn the_default_options_add_nothing_to_the_childs_environment() { + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions::default(), + ); + assert_eq!(command.get_program(), "/an/engine"); + assert_eq!(args_of(&command), ["stdio"]); + assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); + } + + #[test] + fn each_option_sets_exactly_its_own_variable() { + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + let command = engine_stdio_command( + Path::new("/an/engine"), + ["__engine-stdio", "vllm"], + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), + request_progress: true, + forward_stderr: true, + }, + ); + assert_eq!(args_of(&command), ["__engine-stdio", "vllm"]); + assert_eq!( + added_env(&command), + BTreeMap::from( + [ + ("ROCM_CLI_CONFIG_DIR", "/c"), + ("ROCM_CLI_DATA_DIR", "/d"), + ("ROCM_CLI_CACHE_DIR", "/k"), + ("ROCM_CLI_ENGINE_ENVS_ROOT", "/envs"), + ("ROCM_SERVE_API_KEY_FILE", "/keys/svc.endpoint-key"), + ("ROCM_ENGINE_PROGRESS_STDERR", "1"), + ] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ) + ); + + for (name, options) in [ + ( + "ROCM_CLI_ENGINE_ENVS_ROOT", + EngineStdioOptions { + env_root: Some(Path::new("/envs")), + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_SERVE_API_KEY_FILE", + EngineStdioOptions { + endpoint_key_file: Some(Path::new("/keys/k")), + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_ENGINE_PROGRESS_STDERR", + EngineStdioOptions { + request_progress: true, + ..EngineStdioOptions::default() + }, + ), + ] { + let command = engine_stdio_command(Path::new("/an/engine"), ["stdio"], &options); + let env = added_env(&command); + assert_eq!( + env.keys().map(String::as_str).collect::>(), + [name], + "an option must set only its own variable" + ); + } + } + + #[test] + fn the_variable_names_are_the_ones_the_engines_read() { + assert_eq!(ENGINE_ENVS_ROOT_ENV, "ROCM_CLI_ENGINE_ENVS_ROOT"); + assert_eq!(ENDPOINT_API_KEY_FILE_ENV, "ROCM_SERVE_API_KEY_FILE"); + assert_eq!(ENGINE_PROGRESS_STDERR_ENV, "ROCM_ENGINE_PROGRESS_STDERR"); + } + + #[test] + fn the_request_envelope_is_the_method_and_the_encoded_payload() { + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap(); + assert_eq!(envelope.method, EngineMethod::Stop); + assert_eq!(envelope.payload, json!({"service_id": "svc"})); + assert_eq!( + serde_json::to_string(&envelope).unwrap(), + r#"{"method":"stop","payload":{"service_id":"svc"}}"# + ); + } + + #[test] + fn decode_engine_response_reports_each_way_an_answer_can_be_unusable() { + let failed = EngineResponseEnvelope { + ok: false, + data: None, + error: Some(EngineErrorDetail { + code: "stop_failed".to_owned(), + message: "it broke".to_owned(), + }), + }; + let error = decode_engine_response::(failed).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let unexplained = EngineResponseEnvelope { + ok: false, + data: None, + error: None, + }; + let error = decode_engine_response::(unexplained).unwrap_err(); + assert_eq!(error.to_string(), "unknown engine error"); + + let empty = EngineResponseEnvelope { + ok: true, + data: None, + error: None, + }; + let error = decode_engine_response::(empty).unwrap_err(); + assert_eq!( + error.to_string(), + "engine response envelope did not contain data" + ); + + let wrong_shape = EngineResponseEnvelope::success(json!({"stopped": "yes"})); + let error = decode_engine_response::(wrong_shape).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to decode engine response payload" + ); + } + + /// A stderr whose reads fail. + struct Fails; + impl Read for Fails { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("pipe went away")) + } + } + + /// A sink that refuses every write, like a closed terminal. + struct Closed; + impl Write for Closed { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn stderr_is_collected_whole_and_a_read_error_is_recorded() { + let collected = collect_stderr(&b"first\nsecond"[..], None::>); + assert_eq!(collected, b"first\nsecond"); + + let collected = String::from_utf8(collect_stderr(Fails, None::>)).unwrap(); + assert_eq!( + collected, + "failed to read engine progress: pipe went away\n" + ); + } + + #[test] + fn forwarded_stderr_is_written_out_and_still_collected() { + let mut shown = Vec::new(); + let collected = collect_stderr(&b"step 1\nstep 2\n"[..], Some(&mut shown)); + assert_eq!(shown, b"step 1\nstep 2\n"); + assert_eq!(collected, b"step 1\nstep 2\n"); + + // A sink that refuses writes (a closed terminal) costs nothing. + let collected = collect_stderr(&b"still kept\n"[..], Some(Closed)); + assert_eq!(collected, b"still kept\n"); + } + + #[cfg(unix)] + mod child { + use super::*; + use crate::fake_engine::FakeEngine; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn stop_request() -> EngineRequestEnvelope { + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap() + } + + fn run(engine: &FakeEngine, options: &EngineStdioOptions<'_>) -> Result { + decode_engine_response(run_engine_stdio( + engine.path(), + ["stdio"], + options, + &stop_request(), + )?) + } + + #[test] + fn a_request_is_one_newline_terminated_line_on_the_childs_stdin() { + let engine = FakeEngine::answering("client-framing", STOPPED, 0); + let answer = run(&engine, &EngineStdioOptions::default()).unwrap(); + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn a_response_with_an_apostrophe_reaches_the_caller_intact() { + // The fake writes its answer through a shell literal; an apostrophe + // in valid JSON must not end that literal early. + let engine = FakeEngine::answering( + "client-apostrophe", + r#"{"ok":false,"error":{"code":"stop_failed","message":"can't stop"}}"#, + 0, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: can't stop"); + } + + #[test] + fn the_options_reach_the_running_child() { + let engine = FakeEngine::answering("client-options", STOPPED, 0); + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + run( + &engine, + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/k")), + request_progress: true, + forward_stderr: false, + }, + ) + .unwrap(); + assert_eq!(engine.env("ROCM_CLI_DATA_DIR").as_deref(), Some("/d")); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/envs") + ); + assert_eq!( + engine.env("ROCM_SERVE_API_KEY_FILE").as_deref(), + Some("/keys/k") + ); + assert_eq!( + engine.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + } + + #[test] + fn a_nonzero_exit_with_no_answer_reports_the_status_and_stderr() { + let engine = FakeEngine::new("client-crash", true, "echo boom >&2\nexit 3"); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with status exit status: 3: boom" + ); + + let silent = FakeEngine::new("client-silent-crash", true, "exit 4"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with status exit status: 4" + ); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let engine = FakeEngine::answering( + "client-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("client-answers-then-exits-2", STOPPED, 2); + assert_eq!( + run(&engine, &EngineStdioOptions::default()).unwrap(), + json!({"stopped": true}) + ); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let engine = FakeEngine::new( + "client-garbage", + true, + "printf 'not json'\necho oops >&2\nexit 0", + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + + let silent = FakeEngine::new("client-garbage-quiet", true, "printf 'not json'"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + + /// An answer whose read fails kills the child and reaps it, so a daemon + /// that hits this repeatedly collects neither a running engine nor a + /// zombie per failure. A zombie keeps its `/proc` entry until it is + /// waited on, so the entry being gone shows the reap and not only the + /// kill (hence Linux only). + #[cfg(target_os = "linux")] + #[test] + fn a_failed_read_of_the_answer_kills_and_reaps_the_child() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + let proc_entry = format!("/proc/{pid}"); + assert!(Path::new(&proc_entry).exists(), "the child is running"); + + let error = read_response(&mut child, Fails).unwrap_err(); + + assert_eq!(error.to_string(), "failed to read engine stdio response"); + assert_eq!( + format!("{error:#}"), + "failed to read engine stdio response: pipe went away" + ); + assert!( + !Path::new(&proc_entry).exists(), + "the child must be killed and reaped, not left running or as a zombie" + ); + // The status was collected by the reap, so it is still reported. + assert_eq!( + child.try_wait().unwrap().and_then(|status| status.signal()), + Some(libc::SIGKILL) + ); + } + + /// The success path of the same helper hands the bytes back and leaves + /// the child alone. + #[test] + fn a_successful_read_of_the_answer_leaves_the_child_to_the_caller() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + + let bytes = read_response(&mut child, &b"{\"ok\":true}"[..]).unwrap(); + + assert_eq!(bytes, b"{\"ok\":true}"); + assert!(child.try_wait().unwrap().is_none(), "still running"); + child.kill().unwrap(); + child.wait().unwrap(); + } + + #[test] + fn a_program_that_cannot_start_is_named_in_the_error() { + let error = run_engine_stdio( + Path::new("/no/such/engine"), + ["stdio"], + &EngineStdioOptions::default(), + &stop_request(), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "failed to spawn engine stdio process /no/such/engine" + ); + } + + /// Progress forwarded to a stderr whose reader has gone away is dropped, not + /// fatal, also where SIGPIPE is at its default action (`rocm` runs that + /// way). The `Closed` sink above cannot show this: it returns the error + /// itself, where the real failure is a signal that ends the process before + /// any error is returned. So this re-executes itself in a fresh process + /// that installs `SIG_DFL` and forwards into one end of a real pipe whose + /// other end is closed. Two arms, one per forwarding write in + /// `collect_stderr`: a line of the child's stderr, and the note recorded + /// when reading that stderr fails. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_FORWARD_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let (reader, writer) = io::pipe().unwrap(); + drop(reader); + let collected = if arm == "line" { + collect_stderr(&b"progress\n"[..], Some(writer)) + } else { + collect_stderr(Fails, Some(writer)) + }; + // Surviving is the point; the text is still kept for the error. + assert!(!collected.is_empty()); + return; + } + + for arm in ["line", "read-error"] { + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_FORWARD_ARM", arm) + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "{arm}: SIGPIPE killed the process instead of the write failing: {status:?}" + ); + assert!(status.success(), "{arm}: {status:?}"); + } + } + + /// A request written to an engine that has already exited comes back as an + /// ordinary error, also where SIGPIPE is at its default action. Re-executes + /// this test in a fresh process that installs `SIG_DFL` first, because the + /// harness leaves SIGPIPE ignored and changing it process-wide would race + /// sibling tests; a write that is not guarded kills that process, and the + /// parent reports the signal. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_CLIENT_SIGPIPE_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let engine = FakeEngine::new("client-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let envelope = engine_request_envelope( + EngineMethod::Stop, + &json!({"blob": "x".repeat(1 << 20)}), + ) + .unwrap(); + let error = run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 487fe32658..91c82baa0e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -56,8 +56,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) — full domain extraction, a child of `driver_plan.rs` that depends on the plan model, `driver_command`, `driver_root_preflight_checks` and `resolve_shell_default_template` in its parent, and on `truthy_env` in [`therock.rs`](../apps/rocm/src/therock.rs) (split out of [`driver_install.rs`](../apps/rocm/src/driver_install.rs)): the WSL2 rocDXG plan. `wsl_rocdxg_driver_plan` builds the plan that installs the `rocdxg-roct` package, `resolve_rocdxg_verification` decides how that download is authenticated (a supplied digest, the digest pinned for the version in `ROCDXG_PINNED_DIGESTS`, or the explicit opt-out), and `wsl_rocdxg_refusal_plan` is the plan that installs nothing and says why when it cannot be authenticated or the version is not a well-formed package version. `ROCDXG_PINNED_DIGESTS` is the trust anchor an unattended root install is checked against, so a change to it is a security-relevant diff; it moved byte for byte. `wsl_rocdxg_driver_plan` is `pub(crate)` and re-exported by `driver_plan.rs` (its builders call it, and tests in `driver_install.rs` use it as a plan fixture); everything else in production code is private. Its tests are the `wsl_rocdxg_*` plan tests, including the one that runs the generated digest step against a real file on Unix. The test-only `scoped_rocdxg_env` helper that clears the rocDXG environment lives here too, `pub(super)` for the plan-builder tests in `driver_plan.rs`. - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. - [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. -- [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the stdio request/response envelope used to talk to an external engine binary or the in-process built-in handler (`engine_request`, `engine_request_with_env_root`, `decode_engine_response`, the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `decode_engine_response`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. +- [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — owns private types: resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the request entry points, which either answer from the in-process built-in handler or run the resolved engine binary with `stdio` (`engine_request`, `engine_request_with_env_root`, and `engine_stdio_child_request`, which hands the child to the stdio client `rocm` shares with `rocmd`, from [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal), the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — owns types: deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`, which is [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs), `app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. - [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. Its tests do not call into `model_catalog_text.rs`: the test of the note the verbose `rocm model` listing prints for a rule-out engine lives beside that renderer. @@ -87,13 +87,13 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. - [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. -- [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, the `RecordedPidOutcome` and `RecordedStopTermination` types, `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which passes it to [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `run_attached_service` as the stop for a Ctrl-C, the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. This module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which does not import it back. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. +- [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`stop_recorded_processes`, a thin wrapper over the termination routine `rocmd` shares, in [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs), that adds `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which passes it to [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `run_attached_service` as the stop for a Ctrl-C, the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. This module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which does not import it back. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. - [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs), call into it; `main.rs` does not. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. - [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). - [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. -- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. +- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_log_tail`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_log_tail`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) — shared CLI-output component (`shell_quote`): the CLI's name for `rocm-core`'s POSIX quoter, [`posix_shell_quote`](../crates/rocm-core/src/quoting.rs). `rocm-dash-tui` keeps a copy as `ui::exec::quote_display_arg` in [`exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs), since the crate-edge check forbids it `rocm-core`; a per-character walk and a property test here hold the two to the same output. It covers the remote `ssh` command line, the chat approval card and the plan for a free-form request, the install dry-run `command:` line, the [`service_hints.rs`](../apps/rocm/src/service_hints.rs) hints, the runtime-activation report, the `rocm services logs ` lines `rocm services` lists and the `rocm comfyui install --runtime-id` advice a failed ComfyUI reinstall prints — not every `rocm …` hint the CLI prints; older ones such as `rocm logs ` still interpolate their value bare. Also holds `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. @@ -142,14 +142,14 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal; with no watcher enabled it exits by itself, deciding that under the autostart lock and publishing "stopped" and clearing its spawn claim before releasing it — `record_daemon_stop`, `release_own_autostart_claim`) and `rocmd status` (`print_status`), with the watcher tick interval, which `LoopHooks` makes injectable for tests. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. -- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and threading its endpoint key: `engine_request` (the stdio request to an engine process, run through the same client `rocm` uses, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs); `rocmd` chooses what to run, `__engine-stdio ` of the `rocm` executable (its own when embedded as `rocm daemon`, the `rocm` beside it when it is the standalone `rocmd`, which has no such verb), and what to ask of the client: its own app paths, the service's endpoint key file when it holds a usable key, and stderr kept for the error rather than forwarded), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helper `apply_endpoint_key_env`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. The guard that refuses to respawn a service that needs an endpoint key and has none is not here: it is `rocm-core`'s `ensure_public_service_has_endpoint_key` (see [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs)), which `supervise.rs` and `watchers/server_recover.rs` call directly with `ServiceStart::Respawn`. - [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). The argument handling the dispatch calls is in [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs). - [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs) — the argument handling for the MCP tools that spawn `rocm`: `normalized_rocm_command_args` and the read-only allowlist `ensure_rocm_command_is_read_only` behind the `rocm_command` tool, and the argv builders for `install_sdk`, `install_engine`, `launch_server` and `watcher_enable` (`build_install_sdk_args`, `build_install_engine_args`, `build_launch_server_args`, `build_watcher_enable_args`, with the public-bind and system-prefix acknowledgement checks they apply). Functions over the tool's JSON arguments: it uses no other module of the crate, spawns nothing and writes nothing, and [`mcp.rs`](../apps/rocmd/src/mcp.rs) is its only caller. Its own `mod tests` holds the allowlist and argv-builder tests. - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). -- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. +- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: the engine `Stop`, the manifest and endpoint-key writes, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. The termination of the recorded processes is the routine shared with `rocm services stop`, in [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs); this module passes it `terminate_recorded_pid` and keeps no copy of it or of its grace constant. - [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — the module root of the built-in watchers: the registry (`reconcile_watcher_snapshots`) and the event collector and dispatcher (`evaluate_watchers`, `collect_automation_events_with_gpu_snapshot`, which takes the GPU snapshot as a closure so tests can inject one, and `evaluate_watchers_for_events`), which route each event to its watcher's handler. `find_recoverable_service` and `stop_requested` are re-exported from here because [`supervise.rs`](../apps/rocmd/src/supervise.rs)'s and [`service.rs`](../apps/rocmd/src/service.rs)'s tests reach them by the `crate::watchers::` path. Each watcher's handler lives in the child below that carries its name, together with the policy they share, and a child reaches only [`policy.rs`](../apps/rocmd/src/watchers/policy.rs) and, for the thermal-protect watcher, [`gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) among the others. - [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs) — what a watcher does in each mode and the proposals it queues: `WatcherPolicyAction` and `watcher_policy_action` (observe, queue a proposal or run contained, chosen by the watcher mode), `queue_proposal` and `queue_proposal_with_arguments` with the action-to-tool and action-to-arguments tables behind them, and `pending_proposal_exists`, the check the thermal-protect and server-recover handlers use so they do not queue a duplicate of a proposal that is still pending. It uses no other watcher module. @@ -177,7 +177,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`crypto.rs`](../crates/rocm-core/src/crypto.rs) — (ROCMAI-152, Phase 7c) RSASSA-PKCS#1 v1.5/SHA-256 sign/verify, generic — not model-catalog-specific. - [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) — `rocm diagnose`: matches an `Examination` and optional symptom text against the closed failure catalog and returns ranked findings. The module root keeps the public types (`Fix`, `Diagnosis`, `Route`, `DiagnoseReport`), the two score thresholds, `diagnose` itself, the two `vllm_*_symptom_is_diagnosable` classifiers the vLLM engine calls (they run the whole catalog, so they cannot live below it) and the re-exports that keep every `rocm_core::diagnose::` path resolving; everything else lives in child modules under `diagnose/`, with `diagnose.rs` kept as the module root so those paths and the links to it stay valid. The platform children do not import each other, and `catalog.rs` is the only module that names every checker. [`catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs) (the `CHECKERS` table that tags each check with the platform families it applies to, in catalog order, plus the platform-family resolution, applicability and the out-of-scope message), [`scoring.rs`](../crates/rocm-core/src/diagnose/scoring.rs) (every keyword table, the scoring helpers over them, and the small helpers that read an `Examination` or build a `Diagnosis`), [`report.rs`](../crates/rocm-core/src/diagnose/report.rs) (where an unmatched report is routed upstream, `upstream_route`, `route_when_no_match` and the tracker table behind them, and `render_report_text`, the human-facing view), [`linux.rs`](../crates/rocm-core/src/diagnose/linux.rs) (the checks that apply on Linux only: kernel and module, device-node access, package repositories and the repo-native install, plus the package-manager helpers and `dkms_driver_plan_is_supported`, which the driver planner calls), [`windows.rs`](../crates/rocm-core/src/diagnose/windows.rs) (the checks that apply on Windows only: the HIP SDK, the Adrenalin driver, the Visual C++ runtime and the display driver date, with the version comparison and scores only they use), [`wsl.rs`](../crates/rocm-core/src/diagnose/wsl.rs) (the checks that apply on WSL2 only: the DXCore handoff, the ROCDXG userspace, the distro floor, the Windows host driver and WSL 1), [`host.rs`](../crates/rocm-core/src/diagnose/host.rs) (the checks on the host's configuration that run on more than one platform: wheel architecture, the HSA override, `PATH`, the wheel's ROCm against the system's, an integrated GPU beside a discrete one, and GPUs ROCm does not support), [`engine.rs`](../crates/rocm-core/src/diagnose/engine.rs) (the checks on the engine runtime that run on Linux and WSL2: the vLLM out-of-memory and import failures, a second copy of the code object manager, and a `/dev/shm` too small for a serving workload, plus the two canonical symptom strings the vLLM engine falls back to) and [`test_support.rs`](../crates/rocm-core/src/diagnose/test_support.rs) (test-only: the `Examination` builders that the tests of more than one `diagnose` module share). - [`download.rs`](../crates/rocm-core/src/download.rs) — streaming file downloads with resume and retry: `DownloadRequest`/`DownloadOutcome`, `download_file_streaming(_with_progress)` and the simpler `download_file_to_path(_with_progress)`, `Backoff`, and the retry policy (`DOWNLOAD_MAX_ATTEMPTS`). A download streams to a per-process `.part-` sibling, resumes with a `Range` request, and enforces the caller's size limit, expected length and SHA-256 digest before publishing the file. It uses `ureq`, not the loopback client in [`net.rs`](../crates/rocm-core/src/net.rs), and depends only on [`disk_space.rs`](../crates/rocm-core/src/disk_space.rs). The module is private and [`lib.rs`](../crates/rocm-core/src/lib.rs) re-exports its public names. -- [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs) — (ROCMAI-662) the bearer key that guards a managed service's endpoint: the CSPRNG-drawn generator and its header-safety check. Split out of [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs): an id and its endpoint key are made by unrelated mechanisms (readable/timestamp-based vs. CSPRNG) and meet only at the filesystem layer, where the key's sidecar is named after the id — not a reason to keep the two in one module. +- [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs) — (ROCMAI-662) the bearer key that guards a managed service's endpoint: the CSPRNG-drawn generator and its header-safety check, and the one guard `rocm` and `rocmd` both apply before bringing a recorded service back up. `ensure_public_service_has_endpoint_key` refuses a service that was launched with `--require-api-key`, or is bound to a public host (`is_public_bind_host`, which lives here too), when it has no usable key. The caller passes only a `ServiceStart` (`Launch` for a fresh `rocm serve` spawn, `Restart` for `rocm services restart`, `Respawn` for the daemon's recovery), which words what the action would do; the rules and the `rocm serve …` command the refusal names are the same for every caller, so the two binaries cannot drift apart. It is in `rocm-core` because `rocmd` must not depend on `rocm`. Split out of [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs): an id and its endpoint key are made by unrelated mechanisms (readable/timestamp-based vs. CSPRNG) and meet only at the filesystem layer, where the key's sidecar is named after the id — not a reason to keep the two in one module. - [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) — how far along a managed service's OpenAI-compatible endpoint is: `managed_service_endpoint_readiness` and `EndpointReadiness` (unreachable, listing the model, or serving a real single-token chat request), the lower-level `openai_chat_completion_probe`/`openai_chat_completion_status`/`openai_models_endpoint_has_model`, the shared inference-verified latch the engine adapters call (`engine_state_inference_verified`, `merge_json_state_file`) and the model-reference matching behind the model-listing check. It depends on [`net.rs`](../crates/rocm-core/src/net.rs), [`service_record.rs`](../crates/rocm-core/src/service_record.rs) and [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs); [`net.rs`](../crates/rocm-core/src/net.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) do not depend on it. The module is private and [`lib.rs`](../crates/rocm-core/src/lib.rs) re-exports its public names, so callers keep using `rocm_core::` paths. - [`examine.rs`](../crates/rocm-core/src/examine.rs) — the host examination probe: the `Examination` document and its types, and `Examination::probe`, which runs the probes in order. The probes live in child modules under `examine/`, with `examine.rs` kept as the module root so the [`examine_driver_proptests.rs`](../crates/rocm-core/src/examine_driver_proptests.rs) `#[path]` attribute and the links to it stay valid: [`linux_gpus.rs`](../crates/rocm-core/src/examine/linux_gpus.rs) (Linux GPU enumeration from `lspci`, `rocminfo` and the kernel topology, with the marketing-name and APU classification they share), [`linux_driver.rs`](../crates/rocm-core/src/examine/linux_driver.rs) (kernel modules, device-node access and Secure Boot), [`rocm_install_probe.rs`](../crates/rocm-core/src/examine/rocm_install_probe.rs) (how ROCm was installed and which repository files configure it), [`library_copies.rs`](../crates/rocm-core/src/examine/library_copies.rs) (the loader library-copy search for comgr and the HIP runtime, and `comgr_matches_runtime`, which `diagnose/engine.rs` reads), [`framework.rs`](../crates/rocm-core/src/examine/framework.rs) (the PyTorch and llama.cpp probes), [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) (the OS and distro, CPU, tracked environment variables, shared memory, container check and kernel log), [`wsl.rs`](../crates/rocm-core/src/examine/wsl.rs) (the WSL probe, including the remote probe that reads a named distribution from the Windows host, and the distro-floor check) and [`windows.rs`](../crates/rocm-core/src/examine/windows.rs) (the native Windows probes: the GPU query, the HIP SDK, the Adrenalin driver and the MSVC redistributable, plus `parse_dotted_version`, which `diagnose/windows.rs` reads). - [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) — gfx-target identification and the TheRock package-family tables, split out of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) so [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) can read the family mapping without importing the module that probes the host: `extract_first_gfx_token`, `normalize_therock_family`/`known_therock_families`, `gfx_target_has_no_rocm_build`, the AMD PCI-device-id and marketing-name tables behind `gfx_target_from_pnp_device_id`, and the parsers for the Windows display text and the kernel's gfx-target version. The module is private; `lib.rs` re-exports four of the public names (all but `gfx_target_from_pnp_device_id`) and `host_gpu.rs`, as before, re-exports all five. @@ -188,7 +188,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. - [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) — the `rocm diagnose --model` fit assessment: curated-recipe lookup against a host's measured GPU/RAM. `rocm model --verbose` renders its own fit labels in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which uses exactly two items from this module: `curated_alternatives`, called directly, and `recipe_display_ref`, reached through a private one-line wrapper of the same name in that file. - [`net.rs`](../crates/rocm-core/src/net.rs) — (ROCMAI-154) networking primitives: the default local host and port, URL/endpoint formatting and parsing, and the hand-rolled loopback HTTP/TCP client (`http_get_text*`, `http_post_json_with_auth`, `read_http_response_bounded`). It depends on no other service module: the managed-service readiness probes built on it live in [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) and the streaming downloads in [`download.rs`](../crates/rocm-core/src/download.rs). -- [`paths.rs`](../crates/rocm-core/src/paths.rs) — (ROCMAI-154, Phase 7b) `AppPaths` and its filesystem-location primitives. +- [`paths.rs`](../crates/rocm-core/src/paths.rs) — `AppPaths` and its filesystem-location primitives, including `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, and `ENGINE_ENVS_ROOT_ENV`. - [`placeholder.rs`](../crates/rocm-core/src/placeholder.rs) — the `` placeholder rendered reports print for an empty value: `empty_as_unknown` (returns the value untouched) and `trimmed_or_unknown` (trims first), shared by `apps/rocm`'s driver-install plan and chat tool summaries and by [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs)'s Windows GPU diagnostics, replacing the copy those two `apps/rocm` callers shared in [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and `host_gpu.rs`'s own trimming copy. - [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) — the host probes' subprocess and file-read helpers, split out of [`examine.rs`](../crates/rocm-core/src/examine.rs) so `fix.rs`, `process.rs` and the child modules of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) reach them without importing the module that owns the `Examination` document: `run`/`run_with_env`/`run_raw` (run a command under a timeout; `rc` is `127` when it cannot be spawned and `124` on timeout), `run_utf16le`/`decode_utf16le` (for `wsl.exe`'s UTF-16LE output), `read_text`/`read_host_text`, `which`, the `SHORT`/`MEDIUM` timeouts and `truncate_to_chars`. All are `pub(crate)` except `run_raw`, which stays private to the module. The module itself is private to the crate and is not re-exported. - [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus `run_with_timeout`: it drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. It is not yet the *only* such helper — [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded`, [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `run_raw`, and [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs)'s `capture_optional_command_candidate_with_timeout` (plus the Windows-only `capture_diagnostic_command` in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs)) still join their reader threads with no time limit, so a grandchild holding the pipe can outlast their timeout; converging them onto `run_with_timeout` is follow-up work to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). @@ -221,11 +221,12 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`fake_executable.rs`](../crates/rocm-core/src/fake_executable.rs) — writing stand-in executables that tests across the workspace then run. - [`fix.rs`](../crates/rocm-core/src/fix.rs) — `rocm fix`: remediations for diagnosed failure modes. Its `RECIPES` list (in [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs)), together with the checkers under `diagnose/` (their table is [`catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs)), is the authoritative failure catalog. [`fix.rs`](../crates/rocm-core/src/fix.rs) stays the module root, as [`examine.rs`](../crates/rocm-core/src/examine.rs) does for `examine/`, so links to it stay valid, with its children in `fix/`; it keeps the dispatch (`apply`, `Plan`, `plan_of`, `act_on`) and re-exports the names the rest of the crate and the apps use, so `crate::fix::exit`, `rocm_core::fix_exit`, `rocm_core::fix::FixClass` and the `crate::fix::` oracles that the tests under `diagnose/` call keep resolving. The children are private modules: [`fix/options.rs`](../crates/rocm-core/src/fix/options.rs) (the named exit codes, `FixOptions` and `FixClass`); [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs) (the consent decision and `confirm`, the run and query timeouts, `current_os`, and the `fail!` helper, which it re-exports by path for its siblings); [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) (the host-mutating runners, one per auto-applicable fix, with their rc-file, PATH and device-pin helpers; only the four entry points the catalog names are visible to the root, and the `*_windows` variants are ordinary functions that every platform compiles); [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs) (`FixRecipe`, the per-platform class table, `RECIPES`, the lookups by fix-id, and the `cfg(test)` oracles [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) and the e2e crate hold their own copies to); [`fix/render.rs`](../crates/rocm-core/src/fix/render.rs) (`list_recipes`, a recipe's plan and verify lines, `format_flags`, and the refusal for a diagnosis position passed as a fix-id); and [`fix/manifest.rs`](../crates/rocm-core/src/fix/manifest.rs) (the published JSON contract, `CATALOG_CONTRACT_VERSION` and `catalog_manifest`). `render` and `manifest` read `RECIPES` from `catalog`; nothing in `catalog` reads either. - [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs) — the root the hardware probes resolve host paths through (`host_path`): the real host in a normal build, a simulated host tree for the E2E suite in an `e2e-test-hooks` build. It also owns `read_os_release` (the `/etc/os-release` read with its `/usr/lib/os-release` fallback) and `real_os_release`, the one reader every package-manager plan and install hint uses, which always reads the real host even under a simulated root; the resolver it passes is private, so a second copy of that reader cannot be written outside this module. -- [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) — `tail_lines`, the bounded log-tail read shared by the lemonade and vLLM adapters (no module doc comment). +- [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) — the one routine that reads the last lines of a log: `tail_lines` (UTF-8) and `tail_lines_decoded`, which takes a `LogDecode` (`PLAIN`, or `CONSOLE`: UTF-16 little-endian detected by a byte order mark or by zero bytes in every other position, plus ANSI sequences stripped by `strip_ansi_sequences`). The read is seek-bounded by the caller's byte budget (never past the file size it measured first, so a log that is still being written cannot pull it further), drops a partial leading line, and decodes invalid text lossily instead of failing, so a log view never refuses to show a log. `CONSOLE` does not count an unterminated final fragment made only of escape sequences (a colour reset after the last newline) as a line. `tail_log_decoded` returns the same lines as a `LogTail`, which also says how many bytes the budget left unread, so a view can tell an empty log from one whose newest line is longer than the budget (`newest_line_exceeds_the_read_bound`) and say which: `newest_line_exceeds_bound_notice` is the one wording of that clause, derived from `LOG_VIEW_MAX_TAIL_READ`, and `service_log_tool_summary` the one sentence the `service_logs` tool answers with (no output yet, newest line too long to show, or how many lines were read). The lemonade and vLLM adapters call it, and so do the log views: [`services_text.rs`](../apps/rocm/src/services_text.rs)'s `read_log_tail` (`rocm services logs`, `rocm logs`, and the CLI's own `service_logs` tool in [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)), [`comfyui.rs`](../apps/rocm/src/comfyui.rs)'s `read_log_tail` (`rocm comfyui logs`, with `CONSOLE`) and [`mcp.rs`](../apps/rocmd/src/mcp.rs)'s `read_log_tail` (the daemon's `service_logs` tool). Those are thin adapters that choose a `LogDecode`, pass the shared `LOG_VIEW_MAX_TAIL_READ` budget and add their own error label, and none has a reader of its own. `rocmd` cannot depend on `rocm`, which is why the routine lives here. +- [`sigpipe.rs`](../crates/rocm-core/src/sigpipe.rs) — `with_sigpipe_ignored`, which turns a write to a pipe whose reader has gone into an ordinary error instead of a signal; `rocm` runs with SIGPIPE at its default action, so the engine stdio client wraps its write to the child, and each write forwarding the child's stderr to its own, in it. Guarded regions may overlap, across threads too; the disposition in force before the first is put back when the last ends. - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). @@ -297,7 +298,9 @@ The other three dashboard crates are described at crate level here, not module b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share. +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. +- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the protocol itself, apart from the two modules above: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with, beside `gpu_indices_to_csv`. ### `crates/rocm-deps` diff --git a/docs/testing.md b/docs/testing.md index eb0fb8671f..ef254bdd90 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -167,6 +167,42 @@ child write the file instead, so this process never holds a write fd on it. The same applies to E2E steps: scenarios run concurrently in one process, and the `rocm` child that runs the shim can be the one refused. +### The engine stdio client + +`rocm` and `rocmd` run an engine child through one client, +[`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) in +`rocm-engine-protocol`. Its conversation (the request line, the environment each +option sets, what a failing exit or an unusable answer is reported as, the +SIGPIPE guard) is tested there against `FakeEngine`, a shell-script engine child +from [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) that +records its argv, environment and stdin; the pure parts (the command a call +builds, `decode_engine_response`, stderr collection) run on every platform, and +the ones that start the fake are `#[cfg(unix)]`. The `rocm` and `rocmd` callers +each run their own entry point against the same fake and pin only what they +choose: the program and verb, the app paths, and whether stderr is forwarded (see +`apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). A caller that +grew its own private copy of the conversation would fail those tests. + +The SIGPIPE tests re-execute the test binary in a fresh process with SIGPIPE at +its default action, which is how `rocm` runs, because the test harness leaves it +ignored and changing it process-wide would race sibling tests. A write that is +not guarded kills that process, and the parent test reports the signal. Two +writes are covered that way: the request to the child's stdin, and the forwarding +of the child's stderr to this process's stderr (a line of progress, and the note +recorded when reading it fails). The forwarding test writes into a real pipe whose +reader is closed, because a mock sink that returns `BrokenPipe` itself never +raises the signal. `FakeEngine` quotes the response and its scratch directory +for the shell, so a response such as `can't stop`, or a `TMPDIR` containing an +apostrophe, reaches the client intact. + +A child whose answer cannot be read is killed and reaped before the error is +returned, as it is when the request cannot be written, so the long-lived daemon +does not collect a running engine or a zombie per failure. +`a_failed_read_of_the_answer_kills_and_reaps_the_child` (Linux only) runs the +read step on a real child with a reader that fails and checks that the child's +`/proc` entry is gone: a killed child that was never waited on would keep it as a +zombie. + ### Advised commands must parse `apps/rocm/src/advised_commands.rs` checks the `rocm`/`rocmd` commands that the @@ -1358,6 +1394,16 @@ second terminal to watch the download. The no-log first-run path should report `no CLI logs found yet` without creating the configured `data/logs` directory. +The `service_logs` tool (what the assistant and MCP clients call to read a +server's log) answers through `rocm_core::service_log_tool_summary`, so an empty +log (`has no log output yet`) and a log whose newest line is longer than the +4 MiB a log view reads (`none is shown: the newest line is longer than ...`, with +`newest_line_exceeds_read_bound` set in the structured reply) are not both +reported as zero lines read. The CLI's copy of the tool is run by scenarios +`logs-13` and `logs-14` through `rocm mcp-call service_logs`; the daemon's copy +is covered by `cargo test -p rocmd --lib service_logs`, because the harness +drives only the `rocm` binary. + TUI log pagination focused tests: ```bash @@ -1413,8 +1459,12 @@ cargo test -p rocm --bin rocm respawn_allows_loopback cargo test -p rocm --bin rocm restart_refuses_a_public_service cargo test -p rocm-core generate_endpoint_api_key cargo test -p rocm-engine-protocol endpoint_api_key -cargo test -p rocm-engine-protocol is_public_bind_host +cargo test -p rocm-core is_public_bind_host +cargo test -p rocm-core endpoint_key +cargo test -p rocm --bin rocm the_command_a_respawn_refusal_names_clears_the_refusal cargo test -p rocmd recovery_respawn_fails_closed +cargo test -p rocmd supervising_a_ +cargo test -p rocmd server_recover_refuses_a_keyless_public_service cargo test -p rocmd apply_endpoint_key_env ``` @@ -1452,7 +1502,13 @@ The same requirement holds across respawns, because the invariant belongs to the recorded *host*, not to the key file: a service recorded on a public host refuses to respawn once its key is gone, rather than coming back unauthenticated. That covers `rocm services restart` and `rocmd`'s recovery supervisor, and the refusal -happens before anything is stopped, so it cannot take down a running service. A +happens before anything is stopped, so it cannot take down a running service. +Both binaries apply the same guard, `rocm-core`'s +`ensure_public_service_has_endpoint_key`, so the refusal reads the same for a +launch, a restart and a daemon respawn bar the action it names; the unit tests +check each caller's text against the guard's, and follow the `rocm serve + …` command it names through the real parser, `serve`'s own key policy +and the key file it stores. A stop therefore drops the key only once it has confirmed the engine is gone — otherwise the CLI would lock itself out of a service that is still running and still enforcing the key — and the deferred cleanup lands on the liveness refresh @@ -1460,8 +1516,9 @@ that later observes the process dead. `service-stop-01` (`features/service_stop.feature`) covers `services stop` succeeding against a running record on the mock lane and the record being persisted as stopped, and `service-stop-02` covers the key being dropped once the daemon's `stop_server` -confirms a stop. There is still no e2e coverage of `services restart`, which -remains unit-tested only. Reusing a running service for a +confirms a stop. `service-stop-08` covers a `services restart` of a public server whose key is +gone being refused, naming a `rocm serve` command that `rocm` accepts, with the +running server left running; a restart that goes ahead remains unit-tested only. Reusing a running service for a `--require-api-key` request it never had is refused rather than silently upgraded — `serve-24` (`features/model_serving.feature`, Lemonade GPU lane, Linux-only since the Windows managed Lemonade spawn can't enforce the key) @@ -1674,6 +1731,78 @@ directly, exits 0, and records whether that signal worked, adding a second stderr `note:` when it did not (`cargo test -p rocm --bin rocm services_stop_` and `cargo test -p rocm --bin rocm attached_ctrl_c_`). +`rocm services stop` and `rocmd`'s `stop_server` terminate the processes a +record names through one routine, +[`terminate_recorded_service_pids`](../crates/rocm-core/src/proc_lifecycle.rs), +and wait `MANAGED_STOP_GRACE` for each; neither binary keeps a copy. The +routine is tested in `rocm-core` (`cargo test -p rocm-core proc_lifecycle::`): +the supervisor is visited before the engine, each PID with its own start-time, +a PID recorded under both roles once, and the stopping process's own PID is +reported as `self` in `pid_outcomes` and never signalled, in both commands. +On Windows the verified root is preceded by `taskkill /PID /T /F`, because +`terminate_verified` cannot walk a process tree there; that step is tested +through injected functions that run on every host (`the_tree_is_killed_before_the_root_is_terminated` +and its neighbours) and, against a real tree, by a Windows-only scenario +(`service-stop-07`, below). The callers' own tests check +that each reaches the routine and reports what it returns +(`cargo test -p rocm --bin rocm service_control::tests::a_stop_` and +`cargo test -p rocmd service::tests::a_stop_`), and each carries a source check +that no private copy of the routine or its constant has come back. A confirmed +stop also clears the PIDs and start-time tokens it stopped from the record +(`ManagedServiceRecord::clear_recorded_processes`, one call from both +binaries), and an unconfirmed one keeps them; a PID recorded without a token is +signalled on trust alone once the OS has recycled it, so a stale one must not +survive for a second stop to find. +`a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` +runs the real routine with the per-PID signal injected, and asserts the +persisted record after the first stop and that a second stop signals nothing; +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop` is the +other direction. `service-stop-07` +(`@requires-os:windows`, no GPU) is the live coverage of the Windows tree kill: +it records a PowerShell parent that has started a `ping` child, runs `rocm +services stop --yes`, and asserts that both processes are gone together with the +report (`Local server stopped`, `stopped processes: 1`) and the record (stopped, +no process, no stop request, key file gone). The child is the tree kill's own +work: with the `taskkill /T` step removed the stop ends only the recorded +process and the scenario fails on the surviving child. The scenario keeps the +child's own identity (`E2eWorld::recorded_descendant`) and teardown ends it when +it is still that process, so a stop that fails to take it down does not leave +`ping` running for its five minutes behind a dead parent; the decision is +`e2e_cucumber::stray_process::leftover_action`, which a table test covers on +every host (ended only while it is still the recorded process, never once gone, +reused or unconfirmed). `service-stop-01` plants +PID 0 and `service-stop-02` is Linux-only, so neither reaches that step; the +injected unit tests above cover its ordering and verdicts on every host. + +The tree kill is behind the same identity gate as the rest of the stop. On +Windows `process_start_ticks` is the process creation time (`GetProcessTimes`, a +`FILETIME`), so a record written by this version refutes a PID that an unrelated +process has taken, and the stop reports `identity_mismatch` without running +`taskkill` on it; only a record written before Windows recorded a token has +nothing to refute: a stop refuses a PID only when a token was recorded and the +live one differs, and `what_a_stop_does_for_each_recorded_and_live_token` pins +that for every case on every host (no token, which is also the shape +`service-stop-07` records; a token that matches; one that differs; one that +cannot be read back), asserting whether the tree kill runs and the verdict the +stop reports together. `a_filetime_is_its_high_word_above_its_low_word` runs the +conversion on every host. `terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` +runs the real stop against this test's own PID with a wrong token on Linux and +Windows (a stop that went ahead would end the test process), and +`terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` +repeats it against a bystander on Windows only and then stops that bystander +with its true token. `service-stop-09` is the live version on every OS: it +records a running process under a token no live process has, stops the server, +and asserts that the process is still running and the record names no process. +Neither Windows-only test nor the Windows half of `service-stop-09` ran where +this was written; the Windows CI jobs are what exercise them. A stop that cannot +verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID +eq "` on Windows (`process_check_command`). +`service-stop-06` (Linux) is the same stop of a real recorded process through +`rocm services stop`, checking the persisted state: the process exited and the +record names none. Run the Windows scenario on a Windows host with +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-07"` +and a prebuilt `ROCM_CLI_BINARY`. + No scenario drives the attached Ctrl-C stop end to end. Attaching needs a real `rocm serve` launch — an engine and a runtime the mock lane does not have, which is why the chat scenarios there plant a record behind a mock server @@ -1698,7 +1827,8 @@ unconfirmed rather than having its PIDs cleared. `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded process and checks the confirmed report against the process, the record and -the endpoint key file. Run it with +the endpoint key file, and that the standalone `rocmd` got an answer to its +engine `Stop` request (it reaches the engine through the `rocm` beside it). Run it with `ROCM_CLI_BINARY=target/debug/rocm ROCM_CLI_ROCMD_BINARY=target/debug/rocmd cargo test -p e2e-cucumber --test e2e -- -n "service-stop-02"`. `service-stop-03`, `-04` and `-05` (Linux, mock lane) cover `rocm services @@ -1708,11 +1838,18 @@ stop report that outcome for the recorded process without signalling it: each checks the report on stderr, exit 1, the kept PID and stop request, and that no stop command is named. -03 and -04 plant an endpoint key and check that the `kept:` line names it and the key file is still there; -05 plants none and -checks that the line omits it and no key file exists. Because the seam +checks that the line omits it and no key file exists. `service-stop-08` plants a +publicly bound server with no key file and runs `rocm services restart --yes`: +it checks the refusal's text and exit status, that the `rocm serve` command it +names parses (its `` filled in, `--help` last, with a bogus flag as the +negative control), and that the recorded process is still running and the +record is not marked stopped. Because the seam replaces `terminate_verified`, they say nothing about what is signalled. They need a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n -"service-stop"`. +"service-stop-0[1-68]"`. Not `-n "service-stop"`: a name filter bypasses the +suite's own host filter, so it would also run the Windows-only +`service-stop-07` on a Linux host, where it fails for want of PowerShell. Manual restricted-tool smoke: diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature index c58430162b..ac68e92fbc 100644 --- a/tests/e2e-cucumber/features/logs.feature +++ b/tests/e2e-cucumber/features/logs.feature @@ -60,3 +60,59 @@ Feature: Log inspection Given a failed local server whose log shows a download at 43 percent When the user follows that server's logs Then the CLI prints the download line, says it is not following, and exits with an error + + # A log is read as text but a stray byte that is not valid text must not hide + # the readable lines around it: the view decodes it lossily and the search + # still finds every matching line. Before, one such byte made `rocm logs` + # show a single "failed to read" placeholder instead of the whole file. + @id:logs-invalid-byte-keeps-matching-lines + Scenario: logs-08 - A log with a byte that is not valid text still yields its matching lines + Given recorded command logs containing several lines about a topic and one byte that is not valid text + When the user searches the logs for that topic + Then the CLI reports the matching recent lines + + # The log views read at most the last 4 MiB of a log. A log whose newest line + # is longer than that has content but no line the read can return whole, and + # reporting it as "no output yet" / "empty" told the user the file held + # nothing. Each pair below plants the two states that look alike (an empty file + # and one huge line) and asserts the wording together with the file on disk. + @id:logs-service-log-longer-than-the-read-bound-is-not-called-empty + Scenario: logs-09 - A server log whose only line is longer than the read bound is not reported as having no output + Given a local server record whose log holds one line longer than the log view reads + When the user reads that server's log + Then the CLI says the newest line is too long to show instead of saying there is no output + + @id:logs-service-log-empty-says-no-output-yet + Scenario: logs-10 - An empty server log is reported as having no output yet + Given a local server record whose log file is empty + When the user reads that server's log + Then the CLI says there is no output yet + + @id:logs-comfyui-log-longer-than-the-read-bound-is-not-called-empty + Scenario: logs-11 - A ComfyUI log whose only line is longer than the read bound is not reported as empty + Given a ComfyUI log holding one line longer than the log view reads + When the user reads the ComfyUI logs + Then the CLI says the ComfyUI log's newest line is too long to show instead of saying it is empty + + @id:logs-comfyui-log-empty-says-empty + Scenario: logs-12 - An empty ComfyUI log is reported as empty + Given a ComfyUI log file that is empty + When the user reads the ComfyUI logs + Then the CLI says the ComfyUI log is empty + + # The `service_logs` tool the assistant and the MCP clients call reads the same + # log through the same bound, and used to answer "Read the last 80 line(s)" with + # an empty tail for both an empty log and one huge line. `rocm mcp-call` runs + # that tool in the CLI; the daemon's copy of it (`rocmd mcp-call`) is covered by + # a `rocmd` unit test, because the harness drives only the `rocm` binary. + @id:logs-service-logs-tool-longer-than-the-read-bound-is-not-called-empty + Scenario: logs-13 - The service_logs tool does not report a log with one over-long line as empty + Given a local server record whose log holds one line longer than the log view reads + When a tool-using client reads that server's log through the service_logs tool + Then the tool says the newest line is too long to show and flags it instead of returning an empty tail unexplained + + @id:logs-service-logs-tool-empty-says-no-output-yet + Scenario: logs-14 - The service_logs tool reports an empty log as having no output yet + Given a local server record whose log file is empty + When a tool-using client reads that server's log through the service_logs tool + Then the tool says the server has no log output yet and does not flag an over-long line diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index cae0e9859a..7edffcf02f 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -59,6 +59,7 @@ Feature: Stopping a managed service Given a managed server whose recorded process is running When the daemon's stop_server tool stops that server Then the tool reports the server stopped, naming the process it stopped + And the tool asked the engine to stop and the engine answered And the recorded process is no longer running And the server's record reads stopped and names no process And the server's endpoint key file is gone @@ -119,3 +120,99 @@ Feature: Stopping a managed service Then the CLI reports the stop as not confirmed and names no stop command And the server's record still names its process and the stop request And the report says only the record was kept, and there is no key file + + # `rocm services stop`'s confirmed half with a real process. service-stop-01 + # plants PID 0, so it never signals anything and cannot show what the stop + # leaves in the record; service-stop-02 shows it for the daemon's tool. This + # runs the CLI's own stop against a real recorded process and checks the + # persisted state it claims: the process exited, and the record names no + # process any more, so a second stop of that record would signal nothing (a + # PID recorded without a start-time token cannot be refuted once the OS + # recycles it). The unit test + # `a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` + # covers the second stop itself. + # + # Linux-only for the same reason as service-stop-02: the recorded process + # carries the start-time token `/proc` gives it. + + @id:service-stop-cli-confirms-a-stop-and-clears-the-record + @requires-os:linux + Scenario: service-stop-06 - `rocm services stop` of a running managed server confirms the stop and leaves no process in its record + Given a managed server whose recorded process is running + When the user stops it with --yes + Then the CLI reports the service as stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + # The Windows half of the stop. A managed engine runs one level below the + # recorded process (the daemon launches it through an `__engine-serve-http` + # process), and `terminate_verified` cannot walk a process tree on Windows, so + # the stop first runs `taskkill /PID /T /F` on the recorded process. + # service-stop-01 plants PID 0 and never reaches that step, and the scenarios + # above need Linux's start-time token, so none of them kills a real tree. + # + # This records a real parent that has started a real child, stops it, and + # checks that BOTH are gone together with the report and the record. Without + # the tree-kill step the stop ends only the recorded process, the child keeps + # running, and the scenario fails on it. No GPU, no engine, no network. The + # record carries no start-time token, like a Windows record written before the + # token was recorded: the case where the PID cannot be refuted, which the tree + # kill and the cleared record exist for. service-stop-09 below is the same stop + # with a token that does not match. + + @id:service-stop-takes-down-the-whole-process-tree-on-windows + @requires-os:windows + Scenario: service-stop-07 - Stopping a managed server on Windows takes down its recorded process together with the process it started + Given a managed server whose recorded process has started another process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the report counts one stopped process + And the recorded process and the process it started are no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + # The restart refusal, which shares this file's planted record. A stop drops a + # server's endpoint key, and `rocm services restart` accepts a stopped id, so a + # restart of a publicly bound server whose key is gone would bring it back up + # without authentication; it is refused instead. The refusal names a `rocm + # serve` command to launch the server again, and the command is checked + # against the real parser with a model and a port filled in for its `` + # and `` placeholders (`--help` ends the parse after every other word has been + # accepted, so nothing is started). That running it clears the condition is + # covered by the unit tests, which follow it through `serve`'s own policy and + # the key file it stores; here it is only shown to be a command `rocm` accepts. + # + # The refusal comes before the stop, so the promise it makes is the one checked + # last: the running server is left running, not marked stopped and with no + # stop request recorded. + # + # Linux-only like the scenarios above: the recorded process carries the + # start-time token `/proc` gives it. + @id:service-stop-restart-refuses-a-public-server-without-its-key + @requires-os:linux + Scenario: service-stop-08 - Restarting a public server whose endpoint key is gone is refused and leaves it running + Given a public managed server whose endpoint key is gone and whose recorded process is running + When the user restarts it with --yes + Then the CLI refuses the restart because it would reopen the public server without authentication + And the refusal names a serve command that rocm accepts + And the server is left running, with no stop recorded and no key file + + # A recorded PID is not proof that the recorded process still holds it: the + # server can exit by itself and the OS hand its PID to an unrelated program. + # The record pairs the PID with the process's start-time token (Linux's + # `/proc` start time, Windows' creation time), and a stop whose token no longer + # matches treats the recorded process as already gone: it reports the stop, + # clears the record and leaves the process that now holds the PID running, + # instead of signalling a stranger (on Windows, with `taskkill /T /F`, its + # whole subtree). The stranger here is a real process, so "left alone" is + # checked on the process itself. Runs on every OS: the planted token is one + # no live process has. + @id:service-stop-leaves-a-process-that-reused-the-recorded-pid-alone + Scenario: service-stop-09 - Stopping a managed server whose recorded PID now belongs to another process leaves that process running + Given a managed server whose recorded process id now belongs to a different process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the process now holding the recorded process id is still running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index acd458f969..98e2dc99ee 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -25,6 +25,7 @@ pub mod send_until; pub mod serve_log; pub mod serve_plan; pub mod shared_runtime; +pub mod stray_process; pub mod terminal_size; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/stray_process.rs b/tests/e2e-cucumber/src/stray_process.rs new file mode 100644 index 0000000000..815b5a9271 --- /dev/null +++ b/tests/e2e-cucumber/src/stray_process.rs @@ -0,0 +1,75 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What scenario teardown does about a process a scenario started and recorded. +//! +//! A scenario that records a process a stop is supposed to take down (a parent +//! and the child it started, say) can fail with one of them left running, and a +//! later stop cannot find a child whose parent is gone. Teardown therefore +//! keeps the child's own identity and ends it itself. Whether to do so is +//! decided here, apart from the system reads, so a table test covers each case +//! on every host: only a process that is still the one recorded is ended, never +//! one that has since exited or whose PID another process has taken. + +/// What the host says about a recorded process now. +/// +/// It is read against the identity the scenario captured for the process. The +/// harness maps the production identity check onto these (`rocm-core` is a +/// dev-dependency of this crate, so the library does not name it). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedState { + /// Running, and the same process (or recorded without a token, which cannot + /// be refuted). + Same, + /// No longer running. + Gone, + /// Running, but provably a different process that took the PID. + Reused, + /// Running, but its identity cannot be confirmed. + Unconfirmed, +} + +/// What teardown does about one recorded process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Leftover { + /// End it: it is still the process the scenario recorded. + Terminate, + /// Leave it: it is gone, or is not (provably) the recorded process. + Leave, +} + +/// The teardown action for a recorded process whose identity reads as `state`. +/// +/// A process that cannot be confirmed as the recorded one is left alone: ending +/// a stranger that happens to hold a recycled PID is worse than leaving a +/// stray five-minute process to time out. +#[must_use] +pub const fn leftover_action(state: RecordedState) -> Leftover { + match state { + RecordedState::Same => Leftover::Terminate, + RecordedState::Gone | RecordedState::Reused | RecordedState::Unconfirmed => Leftover::Leave, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_process_that_is_still_the_recorded_one_is_ended() { + for (state, expected) in [ + // Alive and the same process (or recorded without a token, which + // cannot be refuted): the stray child a failed stop left behind. + (RecordedState::Same, Leftover::Terminate), + // The stop took it down already. + (RecordedState::Gone, Leftover::Leave), + // Its PID has been reused: a stranger. + (RecordedState::Reused, Leftover::Leave), + // Its token cannot be read back, so it cannot be confirmed. + (RecordedState::Unconfirmed, Leftover::Leave), + ] { + assert_eq!(leftover_action(state), expected, "{state:?}"); + } + } +} diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index a9f6a391e6..e32f4051de 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -191,6 +191,11 @@ pub struct E2eWorld { /// reads the record, but a real `rocm services stop e2e-mock --yes` would /// then signal the harness process itself. pub recorded_process: Option, + /// The identity of a process the recorded one started (`service-stop-07`'s + /// `ping`). Teardown ends it on its own account, because a stop that fails + /// to take it down leaves it running with a dead parent, where nothing else + /// can find it. + pub recorded_descendant: Option, /// Whether the Doctor report preview in the dashboard console was the CLI's /// refusal envelope (it carries a `refused` key). Recorded when the preview /// is read, so a later step can check the card against what the CLI said. @@ -356,6 +361,7 @@ impl Default for E2eWorld { skill_symptom: None, helper_identity: None, recorded_process: None, + recorded_descendant: None, dash_report_refused: None, automation_daemon: None, } @@ -704,6 +710,21 @@ impl Drop for E2eWorld { let _ = process.kill(); let _ = process.wait(); } + // The process the recorded one started: ended too when it is still the + // process that was recorded, so a failed scenario does not leave it + // running until its own timeout. + if let Some(descendant) = self.recorded_descendant.take() { + use e2e_cucumber::stray_process::{Leftover, RecordedState, leftover_action}; + let state = match rocm_core::identity_state(&descendant) { + rocm_core::IdentityState::Matches => RecordedState::Same, + rocm_core::IdentityState::Gone => RecordedState::Gone, + rocm_core::IdentityState::Recycled => RecordedState::Reused, + rocm_core::IdentityState::Indeterminate => RecordedState::Unconfirmed, + }; + if leftover_action(state) == Leftover::Terminate { + let _ = rocm_core::terminate_recorded_pid(&descendant); + } + } // A scenario that enabled a watcher started a real detached `rocm daemon`, // which only exits once the watcher is disabled. If a step failed first, // nothing else would stop it, and it would keep polling a config dir diff --git a/tests/e2e-cucumber/tests/e2e/logs_steps.rs b/tests/e2e-cucumber/tests/e2e/logs_steps.rs index e6e31444c2..0c526b323f 100644 --- a/tests/e2e-cucumber/tests/e2e/logs_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/logs_steps.rs @@ -23,26 +23,140 @@ const MATCH_COUNT: usize = 9; #[given("recorded command logs containing several lines about a topic")] async fn plant_command_logs(world: &mut E2eWorld) { + plant_logs(world, false); +} + +#[given( + "recorded command logs containing several lines about a topic and one byte that is not valid text" +)] +async fn plant_command_logs_with_an_invalid_byte(world: &mut E2eWorld) { + plant_logs(world, true); +} + +/// Plant one action log. With `invalid_byte`, an unrelated leading line carries +/// a lone `0xff` — a byte no UTF-8 text contains — which the matching lines +/// after it must survive. +fn plant_logs(world: &E2eWorld, invalid_byte: bool) { let root = world.isolated_root.as_ref().expect("no isolated root"); let cli_logs = root.path().join("data").join("logs").join("cli"); std::fs::create_dir_all(&cli_logs).expect("failed to create cli logs dir"); // Write the matching lines LAST so they fall within the per-file tail window // `rocm logs` reads (a few leading non-matching lines are harmless context). - let mut body = String::new(); + let mut body: Vec = Vec::new(); for i in 0..3 { - let _ = writeln!(body, "2026-01-01T00:00:0{i} unrelated startup line"); + let mut line = String::new(); + let _ = writeln!(line, "2026-01-01T00:00:0{i} unrelated startup line"); + if invalid_byte && i == 1 { + body.extend_from_slice(b"2026-01-01T00:00:01 unrelated line with a stray \xff byte\n"); + } else { + body.extend_from_slice(line.as_bytes()); + } } for i in 1..=MATCH_COUNT { + let mut line = String::new(); let _ = writeln!( - body, + line, "2026-01-01T00:01:00 event {TOPIC} occurred number {i}" ); + body.extend_from_slice(line.as_bytes()); } std::fs::write(cli_logs.join("e2e-probe.log"), body).expect("failed to write log file"); } +/// How far past the 4 MiB a log view reads (`LOG_VIEW_MAX_TAIL_READ` in +/// `rocm-core`) the planted line reaches. +const LOG_VIEW_BOUND_BYTES: usize = 4 * 1024 * 1024; +const OVERSIZE_LINE_BYTES: usize = LOG_VIEW_BOUND_BYTES + 1024; + +/// Plant the record of a stopped local server (the same planting the record +/// cleanup scenarios use) and replace its log with `bytes`. +fn plant_server_record_with_log(world: &E2eWorld, bytes: &[u8]) { + crate::e2e::service_cleanup_steps::plant_record(world, "failed"); + std::fs::write(server_log_path(world), bytes).expect("failed to write the server log"); +} + +fn server_log_path(world: &E2eWorld) -> std::path::PathBuf { + crate::e2e::service_cleanup_steps::services_dir(world).join(format!( + "{}.log", + crate::e2e::service_cleanup_steps::SERVICE_ID + )) +} + +/// The one saved ComfyUI log the ComfyUI log scenarios plant. +fn comfyui_log_path(world: &E2eWorld) -> std::path::PathBuf { + let root = world.isolated_root.as_ref().expect("no isolated root"); + root.path() + .join("data") + .join("apps") + .join("comfyui") + .join("logs") + .join("start-e2e.log") +} + +fn plant_comfyui_log(world: &E2eWorld, bytes: &[u8]) { + let log = comfyui_log_path(world); + std::fs::create_dir_all(log.parent().expect("log has a parent")) + .expect("failed to create the ComfyUI logs dir"); + std::fs::write(&log, bytes).expect("failed to write the ComfyUI log"); +} + +#[given("a ComfyUI log holding one line longer than the log view reads")] +async fn comfyui_with_an_oversize_line(world: &mut E2eWorld) { + plant_comfyui_log(world, &vec![b'x'; OVERSIZE_LINE_BYTES]); +} + +#[given("a ComfyUI log file that is empty")] +async fn comfyui_with_an_empty_log(world: &mut E2eWorld) { + plant_comfyui_log(world, b""); +} + +#[given("a local server record whose log holds one line longer than the log view reads")] +async fn server_with_an_oversize_line(world: &mut E2eWorld) { + // No final newline: a server mid-line when it died, or one that logs + // without line breaks. + plant_server_record_with_log(world, &vec![b'x'; OVERSIZE_LINE_BYTES]); +} + +#[given("a local server record whose log file is empty")] +async fn server_with_an_empty_log(world: &mut E2eWorld) { + plant_server_record_with_log(world, b""); +} + // ── When ─────────────────────────────────────────────────────────── +#[when("the user reads that server's log")] +async fn read_server_log(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "services", + "logs", + crate::e2e::service_cleanup_steps::SERVICE_ID, + ], + ); + record(world, stdout, stderr, rc); +} + +#[when("a tool-using client reads that server's log through the service_logs tool")] +async fn read_server_log_through_the_tool(world: &mut E2eWorld) { + // The tool is read-only, so no `--allow-mutation`. The exit code is checked + // by the `Then` steps that parse the reply. + let arguments = + serde_json::json!({ "service_id": crate::e2e::service_cleanup_steps::SERVICE_ID }) + .to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["mcp-call", "service_logs", "--arguments-json", &arguments], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user reads the ComfyUI logs")] +async fn read_comfyui_logs(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["comfyui", "logs"]); + record(world, stdout, stderr, rc); +} + #[when("the user searches the logs for that topic")] async fn search_topic(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm(world, &["logs", "--search", TOPIC]); @@ -81,6 +195,122 @@ async fn reports_matches(world: &mut E2eWorld) { ); } +#[then("the CLI says the newest line is too long to show instead of saying there is no output")] +async fn says_the_line_is_too_long(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains( + "Recent output: none shown; the newest line is longer than the 4 MiB a log view reads" + ), + "expected the too-long notice, got:\n{out}" + ); + assert!( + !out.contains("no output yet"), + "a log with content must not be reported as having no output:\n{out}" + ); + // The claim and the state it describes: the log on disk holds exactly one + // line, longer than the bound the notice names. + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!( + on_disk.len() > LOG_VIEW_BOUND_BYTES && !on_disk.contains(&b'\n'), + "the log on disk should be one line past the bound, is {} bytes", + on_disk.len() + ); +} + +#[then( + "the CLI says the ComfyUI log's newest line is too long to show instead of saying it is empty" +)] +async fn comfyui_says_the_line_is_too_long(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("No output shown: the newest line is longer than the 4 MiB a log view reads."), + "expected the too-long notice, got:\n{out}" + ); + assert!( + !out.contains("is empty"), + "a log with content must not be reported as empty:\n{out}" + ); + let on_disk = std::fs::read(comfyui_log_path(world)).expect("the ComfyUI log is still there"); + assert!( + on_disk.len() > LOG_VIEW_BOUND_BYTES && !on_disk.contains(&b'\n'), + "the log on disk should be one line past the bound, is {} bytes", + on_disk.len() + ); +} + +#[then("the CLI says the ComfyUI log is empty")] +async fn comfyui_says_empty(world: &mut E2eWorld) { + let out = ok_output(world); + assert!(out.contains("The saved log is empty."), "{out}"); + assert!(!out.contains("longer than"), "{out}"); + let on_disk = std::fs::read(comfyui_log_path(world)).expect("the ComfyUI log is still there"); + assert!(on_disk.is_empty(), "the log on disk should be empty"); +} + +#[then("the CLI says there is no output yet")] +async fn says_no_output_yet(world: &mut E2eWorld) { + let out = ok_output(world); + assert!(out.contains("Recent output: no output yet"), "{out}"); + assert!(!out.contains("longer than"), "{out}"); + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!(on_disk.is_empty(), "the log on disk should be empty"); +} + +/// The reply `rocm mcp-call service_logs` printed, parsed. Asserts the call +/// succeeded and reported no error first, so a failure reads as the CLI's own. +fn service_logs_tool_reply(world: &E2eWorld) -> serde_json::Value { + let out = ok_output(world); + let reply: serde_json::Value = serde_json::from_str(out.trim()) + .unwrap_or_else(|error| panic!("rocm mcp-call did not print JSON ({error}):\n{out}")); + assert_eq!( + reply["isError"], + serde_json::json!(false), + "the tool reported an error:\n{out}" + ); + reply +} + +#[then( + "the tool says the newest line is too long to show and flags it instead of returning an empty tail unexplained" +)] +async fn tool_says_the_line_is_too_long(world: &mut E2eWorld) { + let reply = service_logs_tool_reply(world); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert!( + text.contains("none is shown: the newest line is longer than the 4 MiB a log view reads"), + "expected the too-long notice, got:\n{reply}" + ); + assert!( + !text.contains("no log output yet") && !text.contains("Read the last"), + "a log with content must not be described as empty or as read:\n{reply}" + ); + let content = &reply["structuredContent"]; + assert_eq!(content["tail"], "", "{reply}"); + assert_eq!(content["newest_line_exceeds_read_bound"], true, "{reply}"); + // The claim and the state it describes: the log on disk holds exactly one + // line, longer than the bound the notice names. + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!( + on_disk.len() > LOG_VIEW_BOUND_BYTES && !on_disk.contains(&b'\n'), + "the log on disk should be one line past the bound, is {} bytes", + on_disk.len() + ); +} + +#[then("the tool says the server has no log output yet and does not flag an over-long line")] +async fn tool_says_no_output_yet(world: &mut E2eWorld) { + let reply = service_logs_tool_reply(world); + let text = reply["content"][0]["text"].as_str().unwrap_or_default(); + assert!(text.contains("has no log output yet"), "{reply}"); + assert!(!text.contains("longer than"), "{reply}"); + let content = &reply["structuredContent"]; + assert_eq!(content["tail"], "", "{reply}"); + assert_eq!(content["newest_line_exceeds_read_bound"], false, "{reply}"); + let on_disk = std::fs::read(server_log_path(world)).expect("the server log is still there"); + assert!(on_disk.is_empty(), "the log on disk should be empty"); +} + #[then("the CLI reports no matching lines")] async fn reports_no_matches(world: &mut E2eWorld) { let out = ok_output(world); diff --git a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs index e1bdddbfb3..16f5d589d2 100644 --- a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs @@ -18,7 +18,7 @@ use cucumber::{given, then, when}; use crate::E2eWorld; /// Id of the record every scenario here plants. -const SERVICE_ID: &str = "vllm-e2e-cleanup"; +pub(crate) const SERVICE_ID: &str = "vllm-e2e-cleanup"; /// Engine name, which also picks the engine state directory the record's third /// file lives in (`/engines/vllm/state/`). const ENGINE: &str = "vllm"; @@ -62,7 +62,7 @@ fn data_dir(world: &E2eWorld) -> PathBuf { .join("data") } -fn services_dir(world: &E2eWorld) -> PathBuf { +pub(crate) fn services_dir(world: &E2eWorld) -> PathBuf { data_dir(world).join("services") } @@ -96,7 +96,7 @@ fn record_files(world: &E2eWorld) -> Vec { /// /// The engine state file must agree with `status`: the CLI adopts whatever that /// file says before it decides anything else. -fn plant_record(world: &E2eWorld, status: &str) { +pub(crate) fn plant_record(world: &E2eWorld, status: &str) { let services = services_dir(world); let states = engine_state_dir(world); std::fs::create_dir_all(&services).expect("failed to create services dir"); diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 31df329d2b..3fdc6b37ca 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -158,6 +158,13 @@ async fn keyless_managed_server_with_running_process(world: &mut E2eWorld) { plant_server_with_running_process(world, "127.0.0.1", false); } +/// The server a stop leaves behind when it is bound publicly: its endpoint key +/// is dropped by the stop, so a record read back later has none. +#[given("a public managed server whose endpoint key is gone and whose recorded process is running")] +async fn public_managed_server_without_its_key(world: &mut E2eWorld) { + plant_server_with_running_process(world, "0.0.0.0", false); +} + fn plant_server_with_running_process(world: &mut E2eWorld, host: &str, with_key: bool) { let process = Command::new("sleep") .arg("300") @@ -166,9 +173,161 @@ fn plant_server_with_running_process(world: &mut E2eWorld, host: &str, with_key: .stderr(Stdio::null()) .spawn() .expect("failed to start the process to record"); + let ticks = start_ticks(process.id()); + plant_server_record(world, host, with_key, process, Some(ticks)); +} + +/// A ready, publicly bound server whose recorded PID is a live process that is +/// not the server: what a recycled PID looks like. The record's start-time token +/// is not the process's own, so the stop has to refuse the PID rather than signal +/// it. The token is the process's real one offset by one on Linux, and `1` (a +/// `FILETIME` in 1601, which no live process has) on Windows. +#[given("a managed server whose recorded process id now belongs to a different process")] +async fn managed_server_with_a_recycled_pid(world: &mut E2eWorld) { + let (program, args): (&str, &[&str]) = if cfg!(windows) { + ("ping", &["-n", "300", "127.0.0.1"]) + } else { + ("sleep", &["300"]) + }; + let bystander = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process that holds the recorded PID"); + let stale_ticks = if cfg!(windows) { + 1 + } else { + start_ticks(bystander.id()).wrapping_add(1) + }; + plant_server_record(world, "0.0.0.0", true, bystander, Some(stale_ticks)); +} + +/// The stop's promise about the process that holds the recorded PID, checked on +/// the process: it was not signalled, so it is still running. +#[then("the process now holding the recorded process id is still running")] +async fn process_holding_the_pid_still_runs(world: &mut E2eWorld) { + let process = world + .recorded_process + .as_mut() + .expect("scenario recorded no process"); + assert!( + process + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "the stop signalled a process that is not the recorded server" + ); +} + +/// Where the parent the Windows scenario records writes the PID of the process +/// it started, for the scenario to read. +fn descendant_pid_path(world: &E2eWorld) -> PathBuf { + services_dir(world).join("descendant.pid") +} + +/// A ready, publicly bound server whose recorded supervisor is a parent process +/// that has started a child of its own, recorded the way Windows records it: +/// the PID alone, with no start-time token (there is none to read there). +/// +/// The parent is PowerShell, the child `ping` for five minutes: both ship with +/// Windows, need nothing installed, and are disposable. PowerShell starts the +/// child itself, so the child's parent is the recorded PID, which is the +/// relationship `taskkill /T` follows; it writes the child's PID where the +/// scenario can read it and then stays alive. Both outlive the scenario only if +/// the stop fails to take them, and the World's teardown reaps the parent. +#[given("a managed server whose recorded process has started another process")] +async fn managed_server_whose_process_started_another(world: &mut E2eWorld) { + let services = services_dir(world); + std::fs::create_dir_all(&services).expect("failed to create services dir"); + let pid_file = descendant_pid_path(world); + let process = spawn_parent_with_descendant(&pid_file); + let pid = process.id(); + plant_server_record(world, "0.0.0.0", true, process, None); + + // The premise, so the Then step cannot pass for the wrong reason: the + // descendant exists and is running, and so is the parent. + let descendant = wait_for_descendant_pid(&pid_file); + // Kept first, with its creation time where the host reads one, so teardown + // can end the child on its own even when the stop (or a step) fails and the + // parent is gone: a later stop cannot recover a child from a dead parent. + world.recorded_descendant = Some(rocm_core::ProcessIdentity::capture(descendant)); + assert_ne!(descendant, pid, "the child must be a different process"); + assert!( + windows_process_is_running(descendant), + "premise: the process the recorded one started must be running" + ); + assert!( + world + .recorded_process + .as_mut() + .expect("scenario recorded no process") + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "premise: the recorded process must be running" + ); +} + +fn spawn_parent_with_descendant(pid_file: &std::path::Path) -> std::process::Child { + // A single-quoted PowerShell string: an apostrophe in the path is doubled. + let pid_file = pid_file.display().to_string().replace('\'', "''"); + let script = format!( + "$child = Start-Process -FilePath ping.exe -ArgumentList '-n','300','127.0.0.1' \ + -WindowStyle Hidden -PassThru; \ + Set-Content -LiteralPath '{pid_file}' -Value $child.Id; \ + Start-Sleep -Seconds 300" + ); + Command::new("powershell") + .args(["-NoProfile", "-NonInteractive", "-Command", &script]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process to record") +} + +/// The child's PID, once the parent has written it. +fn wait_for_descendant_pid(pid_file: &std::path::Path) -> u32 { + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); + loop { + if let Some(pid) = std::fs::read_to_string(pid_file) + .ok() + .and_then(|text| text.trim().parse().ok()) + { + return pid; + } + assert!( + std::time::Instant::now() < deadline, + "the recorded process never wrote the PID of the process it started" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } +} + +/// Whether Windows has a process `pid`, by `tasklist`: a match prints a CSV +/// row naming it, and no match prints an INFO line without one. +fn windows_process_is_running(pid: u32) -> bool { + let output = Command::new("tasklist") + .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) + .output() + .expect("failed to run tasklist"); + String::from_utf8_lossy(&output.stdout).contains(&format!("\"{pid}\"")) +} + +/// Write the record for a ready server whose supervisor is `process`, with +/// `ticks` as its start-time token when there is one, and take `process` into +/// the World so the scenario (and its teardown) can check and reap it. +fn plant_server_record( + world: &mut E2eWorld, + host: &str, + with_key: bool, + process: std::process::Child, + ticks: Option, +) { let pid = process.id(); world.recorded_process = Some(process); - let ticks = start_ticks(pid); let services = services_dir(world); std::fs::create_dir_all(&services).expect("failed to create services dir"); @@ -237,6 +396,13 @@ async fn stop_with_yes(world: &mut E2eWorld) { record(world, stdout, stderr, rc); } +#[when("the user restarts it with --yes")] +async fn restart_with_yes(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "restart", SERVICE_ID, "--yes"]); + record(world, stdout, stderr, rc); +} + #[when("the daemon's stop_server tool stops that server")] async fn stop_server_tool(world: &mut E2eWorld) { let mut command = Command::new(crate::rocmd_binary()); @@ -279,6 +445,105 @@ async fn record_marked_stopped(world: &mut E2eWorld) { ); } +#[then( + "the CLI refuses the restart because it would reopen the public server without authentication" +)] +async fn restart_refused_without_authentication(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!( + rc, 0, + "a restart that would reopen the server anonymously must fail:\n{combined}" + ); + assert!( + combined.contains( + "managed service is bound to the public host `0.0.0.0` but has no endpoint API key, \ + so restarting it would reopen it without authentication." + ), + "{combined}" + ); +} + +/// The `rocm serve` command the refusal names, run against the real parser: +/// its `` placeholder filled in, and `--help` last, so the parse ends in +/// help only if every word before it was accepted. A command with a flag `rocm` +/// does not have is shown to be rejected the same way, so the check cannot pass +/// by `--help` alone. +#[then("the refusal names a serve command that rocm accepts")] +async fn refusal_names_an_accepted_serve_command(world: &mut E2eWorld) { + let combined = combined_output(world); + let advised: Vec<&str> = combined + .split('`') + .skip(1) + .step_by(2) + .filter(|span| span.starts_with("rocm serve ")) + .collect(); + assert_eq!( + advised.len(), + 1, + "the refusal must name exactly one `rocm serve ` command:\n{combined}" + ); + assert!( + advised[0].contains("--host 0.0.0.0 --port --allow-public-bind"), + "the command must bring the server back on its public host, with the explicit port a \ + non-default host requires: {}", + advised[0] + ); + let mut args: Vec = advised[0] + .split_whitespace() + .skip(1) + .map(|word| match word { + "" => "Qwen/Qwen3-0.6B".to_owned(), + "" => "8000".to_owned(), + word => word.to_owned(), + }) + .collect(); + args.push("--help".to_owned()); + let words: Vec<&str> = args.iter().map(String::as_str).collect(); + let (stdout, stderr, rc) = crate::run_rocm(world, &words); + assert_eq!(rc, 0, "rocm rejected {words:?}:\n{stdout}\n{stderr}"); + + let mut bogus = words.clone(); + bogus.insert(bogus.len() - 1, "--no-such-flag"); + let (_, _, rc) = crate::run_rocm(world, &bogus); + assert_ne!( + rc, 0, + "a flag rocm does not have must be rejected before --help: {bogus:?}" + ); +} + +#[then("the server is left running, with no stop recorded and no key file")] +async fn server_left_as_it_was(world: &mut E2eWorld) { + let pid = recorded_pid(world); + let process = world + .recorded_process + .as_mut() + .expect("scenario recorded no process"); + assert!( + process + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "the refused restart stopped the running server" + ); + // Not `ready` any more: the CLI's liveness refresh reads the planted server's + // closed port as `starting` before it gets as far as the restart. What a + // refused restart must not do is mark it stopped or drop its process. + let record = read_record(world); + assert_ne!(record["status"], "stopped", "{record}"); + assert_eq!(record["supervisor_pid"], u64::from(pid), "{record}"); + assert!( + record["stop_requested_unix_ms"].is_null(), + "a refused restart stops nothing, so it records no stop request: {record}" + ); + let key = endpoint_key_path(world); + assert!( + !key.exists(), + "a refused restart must not create a key: {}", + key.display() + ); +} + #[then("the tool reports the server stopped, naming the process it stopped")] async fn tool_reports_stopped(world: &mut E2eWorld) { let stdout = world.cli_output.as_deref().unwrap_or_default(); @@ -311,21 +576,83 @@ async fn tool_reports_stopped(world: &mut E2eWorld) { ); } +/// The standalone `rocmd` reaches an engine through the `rocm` beside it (it has +/// no engine verb of its own), so its `Stop` request is answered rather than +/// failing with the standalone binary's "unrecognized subcommand". The report +/// carries the engine's answer or the error that took its place. +#[then("the tool asked the engine to stop and the engine answered")] +async fn engine_answered_the_stop(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + let report: serde_json::Value = + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!(engine_stop["attempted"], true, "{report}"); + assert!( + engine_stop.get("error").is_none(), + "the engine's answer was an error instead: {report}" + ); + assert!(engine_stop["stopped"].is_boolean(), "{report}"); +} + #[then("the recorded process is no longer running")] async fn recorded_process_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); +} + +/// The Windows tree kill, observed: the recorded process AND the one it started +/// are gone after the stop. Without the `taskkill /T` step the stop ends only +/// the recorded process (`TerminateProcess` on the root), and the child keeps +/// running, which is what the second check catches. +#[then("the recorded process and the process it started are no longer running")] +async fn recorded_process_and_its_child_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); + let pid = wait_for_descendant_pid(&descendant_pid_path(world)); + // `taskkill` returns once it has asked for the termination; give the exit + // a moment to land before calling the process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + while windows_process_is_running(pid) { + assert!( + std::time::Instant::now() < deadline, + "the stop took the recorded process down but left the process it \ + started (PID {pid}) running: the tree kill did not reach it" + ); + std::thread::sleep(std::time::Duration::from_millis(250)); + } +} + +/// The report says one process was stopped (the recorded one: a tree kill that +/// already took the root is reported as forced, so it counts), checked next to +/// the processes themselves rather than instead of them. +#[then("the report counts one stopped process")] +async fn report_counts_one_stopped_process(world: &mut E2eWorld) { + let combined = combined_output(world); + assert!(combined.contains("stopped processes: 1"), "{combined}"); +} + +fn assert_recorded_process_exited(world: &mut E2eWorld) { let process = world .recorded_process .as_mut() .expect("scenario recorded no process"); // The report's claim, checked against the process itself: it has exited - // (and is reaped here), rather than merely having been signalled. - let exited = process - .try_wait() - .expect("failed to check the recorded process"); - assert!( - exited.is_some(), - "the stop reported the process gone, but it is still running" - ); + // (and is reaped here), rather than merely having been signalled. The stop + // confirms an exit from the OS's exit code, and this handle is signalled by + // the process object's teardown, so allow that teardown a few seconds on a + // loaded host before calling a process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let exited = process + .try_wait() + .expect("failed to check the recorded process"); + if exited.is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the stop reported the process gone, but it is still running" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } } #[then("the server's record reads stopped and names no process")]