diff --git a/Cargo.lock b/Cargo.lock index 7629aa9554..c4edc8c252 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3905,6 +3905,8 @@ dependencies = [ name = "rocm-engine-protocol" version = "0.1.0" dependencies = [ + "anyhow", + "libc", "proptest", "rocm-core", "serde", @@ -3932,6 +3934,7 @@ dependencies = [ "async-stream", "axum", "clap", + "libc", "rocm-core", "rocm-engine-protocol", "serde", diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index d4315371ae..c964198b5a 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -73,6 +73,8 @@ apple-native-keyring-store = { version = "1.0", features = ["keychain"] } zbus-secret-service-keyring-store = { version = "1.0", features = ["rt-tokio-crypto-rust"] } [dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } # Property-based tests for the byte-size formatter behind the `download: # approved up to …` line. "A size is printed in its own unit" has to hold at # every unit boundary, and the narrow band where `{:.1}` rounding reaches 1024.0 diff --git a/apps/rocm/src/chat/mcp_bridge.rs b/apps/rocm/src/chat/mcp_bridge.rs index 92d7bc05bd..a4a7238f94 100644 --- a/apps/rocm/src/chat/mcp_bridge.rs +++ b/apps/rocm/src/chat/mcp_bridge.rs @@ -22,7 +22,6 @@ use super::summaries::managed_service_running_state; use crate::automations::{render_automations_text, render_daemon_text}; -use crate::engine_runtime_select::app_path_env_vars; use crate::freeform_planner::{format_structured_tool_call, plan_freeform_request_with_context}; use crate::managed_service_status::{load_managed_services, managed_service_is_live}; use crate::rocm_argv::{chat_cli_arg_value, chat_cli_has_flag}; @@ -458,7 +457,7 @@ fn run_rocm_capture_for_paths( args: &[&str], timeout: Duration, ) -> Result { - rocm_core::run_rocm_capture_with_env(args, &app_path_env_vars(paths), timeout) + rocm_core::run_rocm_capture_with_env(args, &paths.env_vars(), timeout) } fn run_rocm_command_for_paths( diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index d1f4829f9d..d36318c7ae 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -4,35 +4,34 @@ //! Engine plugin binary resolution and the engine IPC request/response //! envelope: locating an engine's executable (packaged plugin vs. the -//! platform-packaged fallback), encoding/decoding the JSON-over-stdio -//! envelope used to talk to it, and the built-in (in-process) fast path for -//! `lemonade`/`vllm` that skips spawning a child process entirely. +//! platform-packaged fallback), talking to it over JSON-over-stdio, and the +//! built-in (in-process) fast path for `lemonade`/`vllm`/`aim` that skips +//! spawning a child process entirely. +//! +//! The stdio conversation itself is not implemented here. The request framing, +//! the child's environment, stderr handling, the SIGPIPE guard around the write +//! and the reading of the answer are the client in +//! [`rocm_engine_protocol`]'s `stdio_client` module, which `rocmd` uses too. +//! This module chooses what to run (the built-in handler, or the resolved +//! engine binary with `stdio`) and what `rocm` asks of the client for it +//! ([`engine_stdio_child_request`]). //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-213) of the `main.rs` modularization (see `docs/architecture.md`). //! `dispatch()` calls directly into this module at two points — //! `Command::EngineStdio { engine } => run_builtin_engine_stdio(&engine)` and //! the `__engine-serve-http` arm's `run_builtin_engine_serve_http(...)` call — //! both re-imported via `use crate::engine_ipc::{run_builtin_engine_serve_http, -//! run_builtin_engine_stdio};`; `dispatch()`'s own body is otherwise -//! unaffected. +//! run_builtin_engine_stdio};`. //! //! This module owns: resolving an engine's binary path //! (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, -//! `find_engine_plugin_binary`), the stdio request/response envelope used to -//! talk to an external engine binary or the in-process built-in handler +//! `find_engine_plugin_binary`), the request entry points //! (`engine_request`, `engine_request_with_env_root`, -//! `decode_engine_response`, the `ScopedEnvVar`/ +//! `engine_stdio_child_request`), the `ScopedEnvVar`/ //! `with_scoped_builtin_engine_env` env-scoping helpers and the //! `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, -//! `run_builtin_engine_stdio`), and the built-in HTTP-serving path +//! `run_builtin_engine_stdio`, and the built-in HTTP-serving path //! (`builtin_engine_available`, `run_builtin_engine_serve_http`, -//! `builtin_engine_serve_http_args`). One item sits physically inside this -//! span in `main.rs` but stays there rather than moving: `impl -//! From for WatcherMode` is unrelated clap-arg-to-domain-type -//! conversion that only happens to sit between `find_engine_plugin_binary` and -//! `engine_request` in the original file (it has since moved to `cli.rs`, -//! next to the enum it converts). +//! `builtin_engine_serve_http_args`). //! //! `main.rs` calls into it for `resolve_engine_binary_path`, //! `resolve_engine_binary_path_with_paths`, `engine_request`, @@ -42,39 +41,31 @@ //! `runtime_install_finalize.rs` call into it for //! `engine_request_with_env_root`; `service_launch.rs` calls into it for //! `builtin_engine_serve_http_args`. `find_engine_plugin_binary`, -//! `decode_engine_response`, `with_scoped_builtin_engine_env`, -//! `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this -//! module and stay private. Shared helpers this module calls back into at the -//! crate root — `with_sigpipe_ignored` and `device_policy_name` — stay in -//! `main.rs` and are reached unqualified via `use crate::{...}`, same as for -//! other mechanically relocated modules. `parse_device_policy` moved out of -//! `main.rs` into `gpu_policy.rs` in ROCMAI-214 (Phase 6f); it is reached via -//! `use crate::gpu_policy::parse_device_policy;` instead. `apply_app_path_env`, -//! `app_path_env_vars`, `optional_arg`, and `engine_recipe_json_arg` moved out -//! of `main.rs` into `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f); this -//! file's `use` list below points there now instead of at the crate root. +//! `with_scoped_builtin_engine_env`, `builtin_engine_request`, and +//! `ScopedEnvVar` have no callers outside this module and stay private. The +//! one helper this module calls back into at the crate root, `device_policy_name`, +//! stays in `main.rs` and is reached via `use crate::device_policy_name;`. +//! `parse_device_policy` is in `gpu_policy.rs`, `engine_recipe_json_arg` is in +//! `engine_runtime_select.rs`, and `optional_arg` is in `rocm_engine_protocol`. use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; -use std::process::{Command as ProcessCommand, Stdio}; use std::sync::{Mutex, OnceLock}; -use std::thread; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, engine_binary_path, engine_plugin_dirs}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, engine_binary_path, engine_plugin_dirs}; use rocm_engine_protocol::{ DevicePolicy, EngineMethod, EnginePluginDescriptor, EngineRecipeHint, EngineRequestEnvelope, - EngineResponseEnvelope, + EngineResponseEnvelope, EngineStdioOptions, decode_engine_response, engine_request_envelope, + optional_arg, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use crate::engine_runtime_select::{ - app_path_env_vars, apply_app_path_env, engine_recipe_json_arg, optional_arg, -}; +use crate::device_policy_name; +use crate::engine_runtime_select::engine_recipe_json_arg; use crate::gpu_policy::parse_device_policy; -use crate::{device_policy_name, with_sigpipe_ignored}; static BUILTIN_ENGINE_ENV_LOCK: OnceLock> = OnceLock::new(); @@ -130,11 +121,7 @@ where R: DeserializeOwned, { let stream_progress = matches!(&method, EngineMethod::Install); - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request) - .context("failed to encode engine request payload")?, - }; + let envelope = engine_request_envelope(method, request)?; if let Some(envelope) = with_scoped_builtin_engine_env(paths, env_root, || { builtin_engine_request(engine, &envelope) }) { @@ -146,118 +133,54 @@ where "unable to locate engine binary for {engine}; build the workspace or install the engine package" ) })?; - let mut command = ProcessCommand::new(engine_binary); - command.arg("stdio"); - if let Some(paths) = paths { - apply_app_path_env(&mut command, paths); - } - if let Some(env_root) = env_root { - command.env("ROCM_CLI_ENGINE_ENVS_ROOT", env_root); - } - if stream_progress { - command.env("ROCM_ENGINE_PROGRESS_STDERR", "1"); - } - let mut child = command - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn engine stdio process")?; - - { - let mut stdin = child - .stdin - .take() - .context("engine stdio child did not expose stdin")?; - // Ignore SIGPIPE for the duration of the write: with SIG_DFL in effect - // process-wide, an engine that exits before reading its stdin would - // otherwise kill `rocm` with SIGPIPE before the exit-status diagnostics - // below can run. Under SIG_IGN the write returns EPIPE instead, which we - // surface as the normal "failed to write engine request" error and then - // let `wait()` report why the engine exited. - with_sigpipe_ignored(|| { - serde_json::to_writer(&mut stdin, &envelope) - .context("failed to write engine request")?; - stdin.write_all(b"\n")?; - Ok::<(), anyhow::Error>(()) - })?; - } - - let stderr_handle = child.stderr.take().map(|stderr| { - thread::spawn(move || { - let mut reader = io::BufReader::new(stderr); - let mut collected = Vec::new(); - let mut line = Vec::new(); - loop { - line.clear(); - match reader.read_until(b'\n', &mut line) { - Ok(0) => break, - Ok(_) => { - if let Ok(text) = std::str::from_utf8(&line) { - eprint!("{text}"); - } else { - let _ = io::stderr().write_all(&line); - } - collected.extend_from_slice(&line); - } - Err(error) => { - let message = format!("failed to read engine progress: {error}\n"); - eprint!("{message}"); - collected.extend_from_slice(message.as_bytes()); - break; - } - } - } - collected - }) - }); - let mut stdout = Vec::new(); - child - .stdout - .take() - .context("engine stdio child did not expose stdout")? - .read_to_end(&mut stdout) - .context("failed to read engine stdio response")?; - let status = child - .wait() - .context("failed waiting for engine stdio response")?; - let stderr = stderr_handle - .map(|handle| handle.join().unwrap_or_else(|_| Vec::new())) - .unwrap_or_default(); - - if !status.success() && stdout.is_empty() { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {status}"); - } - bail!("engine stdio process exited with status {status}: {stderr}"); - } - let envelope: EngineResponseEnvelope = serde_json::from_slice(&stdout).with_context(|| { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - "failed to parse engine response envelope".to_owned() - } else { - format!("failed to parse engine response envelope; stderr: {stderr}") - } - })?; - decode_engine_response(envelope) + engine_stdio_child_request(&engine_binary, paths, &envelope, env_root, stream_progress) } -fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result +/// The out-of-process half of [`engine_request_with_env_root`]: run +/// `engine_binary stdio` through the client `rocm` shares with `rocmd`, send +/// `envelope` on its stdin and decode its answer. +/// +/// What `rocm` asks of that client: the child runs on this process's app paths +/// when it has them and on the engine environments root when the request names +/// one, an install asks for progress on stderr, and the child's stderr is +/// written to this process's as it arrives, which is how a terminal user +/// watches an install. No endpoint key is handed over, because none of the +/// requests `rocm` makes is about a running service's endpoint. Split out so a +/// test can point it at a stand-in binary. +fn engine_stdio_child_request( + engine_binary: &Path, + paths: Option<&AppPaths>, + envelope: &EngineRequestEnvelope, + env_root: Option<&Path>, + stream_progress: bool, +) -> Result where R: DeserializeOwned, { - if !envelope.ok { - let error = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |value| format!("{}: {}", value.code, value.message), - ); - bail!("{error}"); + let options = stdio_options(paths, env_root, stream_progress); + decode_engine_response(run_engine_stdio( + engine_binary, + ["stdio"], + &options, + envelope, + )?) +} + +/// What `rocm` asks of the shared stdio client; see +/// [`engine_stdio_child_request`]. +const fn stdio_options<'a>( + paths: Option<&'a AppPaths>, + env_root: Option<&'a Path>, + stream_progress: bool, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: paths, + env_root, + endpoint_key_file: None, + request_progress: stream_progress, + forward_stderr: true, + note_failing_exit_after_answer: false, } - let data = envelope - .data - .context("engine response envelope did not contain data")?; - serde_json::from_value(data).context("failed to decode engine response payload") } struct ScopedEnvVar { @@ -300,15 +223,12 @@ fn with_scoped_builtin_engine_env( let _guard = lock.lock().expect("builtin engine env lock poisoned"); let mut vars = Vec::new(); if let Some(paths) = paths { - for (key, value) in app_path_env_vars(paths) { + for (key, value) in paths.env_vars() { vars.push(ScopedEnvVar::set_path(key, value)); } } if let Some(env_root) = env_root { - vars.push(ScopedEnvVar::set_path( - "ROCM_CLI_ENGINE_ENVS_ROOT", - env_root, - )); + vars.push(ScopedEnvVar::set_path(ENGINE_ENVS_ROOT_ENV, env_root)); } let result = f(); drop(vars); @@ -500,4 +420,289 @@ mod tests { assert_eq!(discovered, Some(primary_path)); Ok(()) } + + // ---- What `rocm` asks of the shared stdio client. ---- + // + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocm`'s entry point against a stand-in engine child and pin what + // `rocm` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocmd/src/engine_ipc.rs`. + + #[test] + fn rocm_asks_for_forwarded_stderr_and_no_endpoint_key() { + let (root, paths) = crate::tests::test_paths("engine-ipc-options"); + let env_root = root.join("envs"); + + let options = stdio_options(Some(&paths), Some(&env_root), true); + assert_eq!( + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) + ); + assert_eq!(options.env_root, Some(env_root.as_path())); + assert!(options.request_progress); + // The terminal user watches an install through the child's stderr. + assert!(options.forward_stderr); + assert!(!options.note_failing_exit_after_answer); + // No request `rocm` makes is about a running service's endpoint. + assert_eq!(options.endpoint_key_file, None); + + let bare = stdio_options(None, None, false); + assert!(bare.app_paths.is_none() && bare.env_root.is_none()); + assert!(!bare.request_progress); + assert!(bare.forward_stderr); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn envelope(method: EngineMethod, payload: serde_json::Value) -> EngineRequestEnvelope { + EngineRequestEnvelope { method, payload } + } + + fn stop_request() -> EngineRequestEnvelope { + envelope(EngineMethod::Stop, json!({"service_id": "svc"})) + } + + fn request( + engine: &FakeEngine, + paths: Option<&AppPaths>, + env_root: Option<&Path>, + stream_progress: bool, + ) -> Result { + engine_stdio_child_request::( + engine.path(), + paths, + &stop_request(), + env_root, + stream_progress, + ) + } + + /// Run `f` holding the lock that serialises every test that sets + /// `ROCM_CLI_*` process-wide (`with_scoped_builtin_engine_env`), so that + /// what a child inherits when `f` starts it and what [`ambient`] reads + /// inside `f` come from the same instant. + fn under_env_lock(f: impl FnOnce() -> R) -> R { + let lock = BUILTIN_ENGINE_ENV_LOCK.get_or_init(|| Mutex::new(())); + let _guard = lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f() + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_gets_the_stdio_verb_and_one_line_of_request() { + let engine = FakeEngine::answering("rocm-framing", STOPPED, 0); + let answer = request(&engine, None, None, false).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn the_child_runs_on_the_callers_app_paths_when_given_them() { + let (root, paths) = crate::tests::test_paths("engine-ipc-app-paths"); + let engine = FakeEngine::answering("rocm-app-paths", STOPPED, 0); + request(&engine, Some(&paths), None, false).unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!( + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() + ); + } + + #[test] + fn the_child_inherits_the_callers_environment_when_no_app_paths_are_given() { + const KEYS: [&str; 3] = [ + "ROCM_CLI_CONFIG_DIR", + "ROCM_CLI_DATA_DIR", + "ROCM_CLI_CACHE_DIR", + ]; + let engine = FakeEngine::answering("rocm-no-app-paths", STOPPED, 0); + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + KEYS.map(ambient) + }); + + for (key, expected) in KEYS.into_iter().zip(expected) { + assert_eq!(engine.env(key), expected, "{key}"); + } + } + + /// The comparison of a child's inherited environment with the ambient one + /// holds while another thread scopes `ROCM_CLI_*` process-wide, which + /// other tests of this module do: the spawn and the read happen under the + /// same lock. Without the lock the child can start while the variables + /// are scoped and the read see them restored (or the other way round). + #[test] + fn the_inherited_environment_is_compared_at_the_instant_the_child_starts() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let (root, paths) = crate::tests::test_paths("engine-ipc-env-race"); + let done = Arc::new(AtomicBool::new(false)); + let flipper = { + let done = Arc::clone(&done); + std::thread::spawn(move || { + while !done.load(Ordering::Relaxed) { + with_scoped_builtin_engine_env(Some(&paths), None, || { + std::thread::sleep(std::time::Duration::from_millis(2)); + }); + // Outside the lock, so the test's own turn is not starved. + std::thread::sleep(std::time::Duration::from_millis(2)); + } + }) + }; + + for round in 0..25 { + let engine = FakeEngine::answering("rocm-env-race", STOPPED, 0); + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + ambient("ROCM_CLI_DATA_DIR") + }); + assert_eq!(engine.env("ROCM_CLI_DATA_DIR"), expected, "round {round}"); + } + + done.store(true, Ordering::Relaxed); + flipper.join().unwrap(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn an_env_root_reaches_the_child_as_the_engine_envs_root() { + let engine = FakeEngine::answering("rocm-env-root", STOPPED, 0); + let env_root = Path::new("/an/engine/envs/root"); + request(&engine, None, Some(env_root), false).unwrap(); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/an/engine/envs/root") + ); + + let without = FakeEngine::answering("rocm-no-env-root", STOPPED, 0); + let expected = under_env_lock(|| { + request(&without, None, None, false).unwrap(); + ambient("ROCM_CLI_ENGINE_ENVS_ROOT") + }); + assert_eq!(without.env("ROCM_CLI_ENGINE_ENVS_ROOT"), expected); + } + + #[test] + fn progress_streaming_is_asked_for_only_when_the_caller_wants_it() { + let streaming = FakeEngine::answering("rocm-progress-on", STOPPED, 0); + request(&streaming, None, None, true).unwrap(); + assert_eq!( + streaming.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + + let quiet = FakeEngine::answering("rocm-progress-off", STOPPED, 0); + let expected = under_env_lock(|| { + request(&quiet, None, None, false).unwrap(); + ambient("ROCM_ENGINE_PROGRESS_STDERR") + }); + assert_eq!(quiet.env("ROCM_ENGINE_PROGRESS_STDERR"), expected); + } + + #[test] + fn the_endpoint_key_is_not_passed_to_the_child() { + // `rocm`'s requests are not about a running service's endpoint (detect, + // install, resolve-model, stop), so this client has no service to find + // a key for. `rocmd`'s healthcheck is the call that threads one. + let engine = FakeEngine::answering("rocm-no-key", STOPPED, 0); + let key = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + ambient(key) + }); + assert_eq!(engine.env(key), expected); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + // The engine's own error envelope is the useful diagnostic, so an + // answer on stdout wins over the exit status. + let engine = FakeEngine::answering( + "rocm-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = request(&engine, None, None, false).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("rocm-answers-then-exits-2", STOPPED, 2); + assert_eq!( + request(&engine, None, None, false).unwrap(), + json!({"stopped": true}) + ); + } + + /// The write of a request to an engine that has already exited must come + /// back as an error even where SIGPIPE is at its default action, which is + /// how `main` runs `rocm`. Re-executes this test in a fresh process that + /// installs `SIG_DFL` first (the test harness leaves SIGPIPE ignored, and + /// changing it process-wide would race sibling tests). + #[test] + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_ENGINE_SIGPIPE_ARM").is_some() { + crate::reset_sigpipe(); + let engine = FakeEngine::new("rocm-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let result = engine_stdio_child_request::( + engine.path(), + None, + &envelope(EngineMethod::Stop, json!({ "blob": big })), + None, + false, + ); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_ENGINE_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } } diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 793a82f3ab..aa65af269d 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -7,12 +7,12 @@ //! to (`EngineSelection`, `resolve_engine_selection`, //! `validate_engine_selection_runtime`, `single_ready_runtime_key`, //! `resolve_runtime_selector_to_exact_key`), small argv-building helpers used -//! when relaunching a managed engine (`optional_arg`, -//! `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, +//! when relaunching a managed engine (`engine_recipe_json_arg`, +//! `parse_engine_recipe_json_arg`, //! `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` //! environment variables every managed child inherits -//! (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/ -//! `apply_app_path_env`), and polling a freshly launched service until it can +//! (`app_path_env_var_values`/`app_path_env_var_refs`, over +//! [`AppPaths::env_vars`]; a command gets them from `AppPaths::apply_env`), and polling a freshly launched service until it can //! actually answer inference requests (`wait_for_service_http_ready`, //! `wait_for_service_http_ready_with_progress`, //! `service_http_readiness_paths`, `http_get_local_service`, @@ -61,16 +61,10 @@ //! it does not decide which runtime counts as "the one ready runtime" itself, //! it asks `runtime_adopt::select_single_ready_runtime`, the rule //! `rocm comfyui install` shares. -//! `optional_arg` and -//! `engine_recipe_json_arg` are `pub(crate)`, read from `engine_ipc.rs`'s +//! `engine_recipe_json_arg` is `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read //! from `main.rs`'s `dispatch()` and `service_control.rs`'s `restart_child_argv`. -//! `app_path_env_vars`/`apply_app_path_env` are `pub(crate)`, read from -//! `engine_ipc.rs`, `app_path_env_vars` is also read from -//! `chat/mcp_bridge.rs`, and `apply_app_path_env` is also read from -//! `engines_cmd.rs`, -//! `service_launch.rs`, and `service_control.rs`'s -//! `restart_internal_managed_service_with`; `app_path_env_var_values`/ +//! `app_path_env_var_values`/ //! `app_path_env_var_refs` are `pub(crate)`, read from `service_launch.rs` //! and `service_control.rs`'s `restart_internal_managed_service_with`. //! `daemon_run_argv` is `pub(crate)`, read from `main.rs`'s `dispatch()`; @@ -96,7 +90,6 @@ use std::ffi::OsString; use std::path::{Path, PathBuf}; -use std::process::Command as ProcessCommand; use std::thread; use std::time::Duration; @@ -246,13 +239,6 @@ pub(crate) fn resolve_runtime_selector_to_exact_key( } } -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } -} - pub(crate) fn engine_recipe_json_arg( engine_recipe: Option<&EngineRecipeHint>, ) -> Result> { @@ -275,20 +261,13 @@ pub(crate) fn parse_engine_recipe_json_arg( .transpose() } -pub(crate) fn app_path_env_vars(paths: &AppPaths) -> [(&'static str, &Path); 3] { - [ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ] -} - #[cfg_attr(not(windows), allow(dead_code))] pub(crate) fn app_path_env_var_values( paths: &AppPaths, env_root: Option<&Path>, ) -> Vec<(&'static str, PathBuf)> { - let mut vars = app_path_env_vars(paths) + let mut vars = paths + .env_vars() .into_iter() .map(|(key, value)| (key, value.to_path_buf())) .collect::>(); @@ -325,12 +304,6 @@ pub(crate) fn managed_service_launcher_path() -> Result { Ok(current_exe) } -pub(crate) fn apply_app_path_env(command: &mut ProcessCommand, paths: &AppPaths) { - for (key, value) in app_path_env_vars(paths) { - command.env(key, value); - } -} - pub(crate) fn wait_for_service_http_ready( engine: &str, host: &str, @@ -923,16 +896,6 @@ mod tests { Ok(()) } - #[test] - fn app_path_env_vars_include_config_data_and_cache() { - let paths = test_app_paths(); - let vars = app_path_env_vars(&paths); - - assert_eq!(vars[0], ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path())); - assert_eq!(vars[1], ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path())); - assert_eq!(vars[2], ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path())); - } - #[test] fn app_path_env_var_values_include_engine_env_root_when_needed() { let paths = test_app_paths(); diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index eaee862b45..dab1629cba 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -41,7 +41,7 @@ use crate::engine_install_env::{ }; use crate::engine_ipc::{engine_request, engine_request_with_env_root}; use crate::engine_runtime_select::{ - apply_app_path_env, resolve_engine_selection, resolve_runtime_selector_to_exact_key, + resolve_engine_selection, resolve_runtime_selector_to_exact_key, validate_engine_selection_runtime, }; use crate::runtime_install_finalize::{ @@ -469,7 +469,7 @@ fn engine_shell( .env("ROCM_CLI_ENV_ID", &resolved.env_id) .env("ROCM_CLI_RUNTIME_ID", &resolved.runtime_id) .env("ROCM_CLI_PYTHON", &resolved.python_executable); - apply_app_path_env(&mut command, &paths); + paths.apply_env(&mut command); let prompt = format!("(rocm:{engine}) "); let mut prompt_marked = false; diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 52a3614f76..fcb31d21e7 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -191,10 +191,6 @@ use std::fs; use std::io::{self, Write}; use std::path::PathBuf; use std::process::ExitCode; -#[cfg(all(test, not(windows)))] -use std::process::Stdio; -#[cfg(all(test, not(windows)))] -use std::process::{Command as ProcessCommand, ExitStatus}; #[cfg(test)] use std::sync::Mutex; @@ -223,33 +219,6 @@ fn reset_sigpipe() { #[allow(clippy::missing_const_for_fn)] fn reset_sigpipe() {} -/// Run `f` with SIGPIPE temporarily ignored, restoring the previous -/// disposition afterwards. -/// -/// [`reset_sigpipe`] sets `SIG_DFL` process-wide so stdout/stderr pipe writes -/// terminate the process conventionally. But a write to a spawned child's stdin -/// whose reader has already exited would then kill `rocm` with SIGPIPE before -/// the caller can turn the resulting `EPIPE` into a diagnostic. Wrap such writes -/// in this helper so the failure surfaces as an `io::Error` (`BrokenPipe`) -/// instead of a signal. -#[cfg(unix)] -#[allow(unsafe_code)] // libc FFI -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - // SAFETY: reading/installing a signal disposition is always safe. - let previous = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; - let result = f(); - // SAFETY: restoring the disposition captured above is always safe. - unsafe { - libc::signal(libc::SIGPIPE, previous); - } - result -} - -#[cfg(not(unix))] -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - f() -} - /// Marker error carrying `rocm fix`'s exit code back through `main()`'s /// ordinary return path, instead of calling `std::process::exit` mid-stack /// and skipping the `_log_guard` destructor held in `run()`. @@ -1467,89 +1436,6 @@ pub(crate) mod tests { } } - /// Regression test for the engine child-stdin write under the process-wide - /// `SIG_DFL` that `main` installs via [`reset_sigpipe`]. If an engine child - /// exits before reading its stdin, the parent's write to that pipe must - /// surface as a `BrokenPipe` error — handled by the caller's diagnostics — - /// rather than killing `rocm` with SIGPIPE before those diagnostics run. - /// - /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE - /// at Rust's default `SIG_IGN`, and flipping it process-wide would race - /// sibling tests. So this re-execs itself in fresh processes that install - /// `SIG_DFL` first, and checks BOTH directions: - /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — - /// this is the regression the plain `SIG_DFL` reset introduced; - /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns - /// an error instead — this is the fix. - /// - /// Each child reaps its short-lived grandchild before writing, so the read - /// end is closed deterministically and the outcome does not race. - #[cfg(unix)] - #[test] - fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { - use std::os::unix::process::ExitStatusExt as _; - - // Grandchild helper: reproduce main()'s SIG_DFL, spawn a process that - // exits immediately, reap it, then write to its now-closed stdin. The - // "guarded" arm wraps the write; the "unguarded" arm does not. - if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { - reset_sigpipe(); - let mut grandchild = ProcessCommand::new("true") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn a process that exits immediately"); - let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); - grandchild.wait().expect("reap grandchild"); - // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; - // it must exceed the pipe buffer so the write actually reaches the - // closed read end rather than being swallowed by kernel buffering. - let payload = vec![b'x'; 64 * 1024]; - if mode == "guarded" { - let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); - assert!( - result.is_err(), - "guarded write to a dead pipe should return BrokenPipe, not succeed" - ); - } else { - // Unguarded: under SIG_DFL this write delivers SIGPIPE and never - // returns. If the process somehow survives, exit 0 so the parent's - // "should have been signalled" assertion fails loudly. - let _ = stdin.write_all(&payload); - } - return; - } - - let exe = std::env::current_exe().expect("current test executable"); - let run_arm = |mode: &str| -> ExitStatus { - ProcessCommand::new(&exe) - .args([ - "tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", - "--exact", - "--nocapture", - ]) - .env("ROCM_TEST_SIGPIPE_ARM", mode) - .status() - .expect("re-exec the test arm in a child process") - }; - - let unguarded = run_arm("unguarded"); - assert_eq!( - unguarded.signal(), - Some(libc::SIGPIPE), - "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ - SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" - ); - - let guarded = run_arm("guarded"); - assert!( - guarded.success(), - "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ - SIGPIPE reached the process instead of surfacing as an error" - ); - } - #[test] fn completions_generate_for_every_shell() { use clap_complete::Shell; diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 2f4db4617e..d2b9128d2b 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -56,8 +56,6 @@ use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use crate::endpoint_keys; use crate::engine_ipc::{builtin_engine_serve_http_args, engine_request}; -#[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; use crate::engine_runtime_select::{ @@ -715,7 +713,7 @@ fn restart_internal_managed_service_inner( command.stdin(Stdio::null()); attach_background_stdio(&mut command, Some(&record.log_path))?; detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); + paths.apply_env(&mut command); if let Some(engine_envs_root) = engine_envs_root.as_deref() { command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); } diff --git a/apps/rocm/src/service_launch/helper_autostart.rs b/apps/rocm/src/service_launch/helper_autostart.rs index 7a54cb5134..430d152097 100644 --- a/apps/rocm/src/service_launch/helper_autostart.rs +++ b/apps/rocm/src/service_launch/helper_autostart.rs @@ -10,8 +10,6 @@ use super::spawn::{ attach_background_stdio, detach_background_command, managed_service_process_command, }; use crate::background_helper::background_helper_already_running; -#[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; use crate::engine_runtime_select::managed_service_launcher_path; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; @@ -151,7 +149,7 @@ fn spawn_background_helper(paths: &AppPaths) -> Result> { command.stdin(Stdio::null()); attach_background_stdio(&mut command, None)?; detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); + paths.apply_env(&mut command); command .spawn() .map(|child| child.id()) diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs index 0bc3d02a34..9b3252efc0 100644 --- a/apps/rocm/src/service_launch/spawn.rs +++ b/apps/rocm/src/service_launch/spawn.rs @@ -13,8 +13,6 @@ use super::launch_transaction::{ }; use crate::endpoint_keys; use crate::engine_ipc::builtin_engine_serve_http_args; -#[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; use crate::engine_runtime_select::managed_service_launcher_path; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; @@ -510,7 +508,7 @@ fn launch_managed_engine_child_locked( command.stdin(Stdio::null()); attach_background_stdio(&mut command, Some(&record.log_path))?; detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); + paths.apply_env(&mut command); if let Some(engine_envs_root) = engine_envs_root.as_deref() { command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); } diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index cac907e7e0..70587d7456 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -25,3 +25,11 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } + +[dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test restores in a re-executed child. +libc.workspace = true diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 417491a716..c95ea8d24a 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -27,22 +27,7 @@ pub(crate) fn run_rocm_capture_for_paths( args: &[&str], timeout: Duration, ) -> Result { - rocm_core::run_rocm_capture_with_env( - args, - &[ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ], - timeout, - ) -} - -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } + rocm_core::run_rocm_capture_with_env(args, &paths.env_vars(), timeout) } pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index b8ccefc354..f4749c3eb2 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -3,16 +3,16 @@ // SPDX-License-Identifier: MIT use crate::daemon_host::DaemonHost; -use anyhow::{Context, Result, bail}; +use anyhow::{Result, bail}; use rocm_core::AppPaths; use rocm_engine_protocol::{ - EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, - HealthcheckResponse, + EngineMethod, EngineStdioOptions, HealthcheckRequest, HealthcheckResponse, + decode_engine_response, engine_request_envelope, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use std::io::Write; -use std::process::{Command as ProcessCommand, Stdio}; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; pub(crate) fn engine_healthcheck_response( paths: &AppPaths, @@ -73,15 +73,19 @@ pub(crate) fn apply_endpoint_key_env( paths: &AppPaths, service_id: &str, ) -> bool { - if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - { + if let Some(key_file) = usable_endpoint_key_file(paths, service_id) { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); return true; } false } +/// The service's endpoint key file, when it exists and holds a usable key. +fn usable_endpoint_key_file(paths: &AppPaths, service_id: &str) -> Option { + rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) +} + /// Refuse to respawn a recorded service on a public host without its endpoint /// API key, so daemon recovery cannot reopen a protected endpoint anonymously. /// @@ -116,6 +120,12 @@ pub(crate) fn ensure_public_service_has_endpoint_key( Ok(()) } +/// Ask an engine a question about a managed service, through the stdio client +/// `rocmd` shares with `rocm`. +/// +/// The engine is reached by running the `rocm` executable as +/// `__engine-stdio `, the verb `rocm` serves for its built-in engines +/// ([`DaemonHost::rocm_binary`] says which executable that is). pub(crate) fn engine_request( paths: &AppPaths, engine: &str, @@ -152,18 +162,23 @@ where T: Serialize, R: DeserializeOwned, { - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request).context("failed to serialize engine request")?, - }; + let envelope = engine_request_envelope(method, request)?; let engine_binary = daemon_host.rocm_binary()?; - let mut command = ProcessCommand::new(&engine_binary); - command - .arg("__engine-stdio") - .arg(engine) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + engine_request_via(&engine_binary, paths, engine, service_id, &envelope) +} + +/// [`engine_request`] with the executable that serves `__engine-stdio` named by +/// the caller, so a test can point it at a stand-in. +fn engine_request_via( + engine_binary: &Path, + paths: &AppPaths, + engine: &str, + service_id: &str, + envelope: &rocm_engine_protocol::EngineRequestEnvelope, +) -> Result +where + R: DeserializeOwned, +{ // A protected public endpoint enforces its key on every request, including // this stdio-transported healthcheck — without the carrier the engine // adapter's HTTP probe is unauthenticated and the endpoint looks @@ -172,50 +187,34 @@ where // // No host to check here: this is a stdio plugin call, not a listener, so a // missing key only weakens this probe rather than opening a port. - let _ = apply_endpoint_key_env(&mut command, paths, service_id); - let mut child = command.spawn().with_context(|| { - format!( - "failed to spawn engine stdio process {}", - engine_binary.display() - ) - })?; - - { - let stdin = child - .stdin - .as_mut() - .context("engine stdio child did not expose stdin")?; - serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; - stdin.write_all(b"\n")?; - } + let key_file = usable_endpoint_key_file(paths, service_id); + let options = stdio_options(paths, key_file.as_deref()); + decode_engine_response(run_engine_stdio( + engine_binary, + ["__engine-stdio", engine], + &options, + envelope, + )?) +} - let output = child - .wait_with_output() - .context("failed waiting for engine stdio response")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {}", output.status); - } - bail!( - "engine stdio process exited with status {}: {}", - output.status, - stderr - ); - } - let envelope: EngineResponseEnvelope = - serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; - if !envelope.ok { - let detail = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |error| format!("{}: {}", error.code, error.message), - ); - bail!(detail); +/// What `rocmd` asks of the shared stdio client; see [`engine_request`]. +/// +/// The child runs on the daemon's own app paths and carries the service's +/// endpoint key file when there is a usable one. Its stderr is kept for the +/// error, not forwarded: nobody watches a daemon's terminal, and a healthcheck +/// asks for no progress. +const fn stdio_options<'a>( + paths: &'a AppPaths, + endpoint_key_file: Option<&'a Path>, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: Some(paths), + env_root: None, + endpoint_key_file, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: true, } - let data = envelope - .data - .context("engine response did not include data")?; - serde_json::from_value(data).context("failed to deserialize engine response data") } #[cfg(test)] @@ -379,6 +378,228 @@ mod tests { assert!(!healthcheck_response_ready(&listing_only)); } + // ---- What `rocmd` asks of the shared stdio client. ---- + // + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocmd`'s entry point against a stand-in engine child and pin what + // `rocmd` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocm/src/engine_ipc.rs`. + + #[test] + fn rocmd_runs_the_child_on_its_own_paths_keeps_stderr_and_asks_for_no_progress() { + let (_root, paths) = temp_app_paths("stdio-options"); + let key_file = Path::new("/keys/svc.endpoint-key"); + + let options = stdio_options(&paths, Some(key_file)); + assert_eq!( + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) + ); + assert_eq!(options.endpoint_key_file, Some(key_file)); + // A healthcheck has no install to report progress on, and nobody watches + // a daemon's terminal. + assert!(!options.request_progress); + assert!(!options.forward_stderr); + // A daemon's stderr is its log, so a failing exit after an answer leaves a note there. + assert!(options.note_failing_exit_after_answer); + assert_eq!(options.env_root, None); + + assert_eq!(stdio_options(&paths, None).endpoint_key_file, None); + } + + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::EngineRequestEnvelope; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const SERVICE_ID: &str = "svc-stdio-child"; + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn healthcheck_envelope() -> EngineRequestEnvelope { + EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({"service_id": SERVICE_ID}), + } + } + + fn request(engine: &FakeEngine, paths: &AppPaths) -> Result { + engine_request_via::( + engine.path(), + paths, + "vllm", + SERVICE_ID, + &healthcheck_envelope(), + ) + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_is_asked_to_serve_the_engine_over_stdio_with_one_line_of_request() { + let (_root, paths) = temp_app_paths("stdio-framing"); + let engine = FakeEngine::answering("rocmd-framing", STOPPED, 0); + let answer = request(&engine, &paths).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["__engine-stdio", "vllm"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"healthcheck\",\"payload\":{\"service_id\":\"svc-stdio-child\"}}\n" + ); + } + + #[test] + fn the_endpoint_key_file_reaches_the_child_only_when_it_holds_a_usable_key() { + let (_root, paths) = temp_app_paths("stdio-endpoint-key"); + let key_env = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + + let keyless = FakeEngine::answering("rocmd-keyless", STOPPED, 0); + request(&keyless, &paths).unwrap(); + assert_eq!(keyless.env(key_env), ambient(key_env)); + + let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, SERVICE_ID); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write(&key_path, " \n").unwrap(); + let unusable = FakeEngine::answering("rocmd-unusable-key", STOPPED, 0); + request(&unusable, &paths).unwrap(); + assert_eq!(unusable.env(key_env), ambient(key_env)); + + fs::write(&key_path, "secret-key").unwrap(); + let keyed = FakeEngine::answering("rocmd-keyed", STOPPED, 0); + request(&keyed, &paths).unwrap(); + assert_eq!(keyed.env(key_env).as_deref(), key_path.to_str()); + } + + #[test] + fn the_child_runs_on_the_daemons_app_paths() { + let (_root, paths) = temp_app_paths("stdio-app-paths"); + let engine = FakeEngine::answering("rocmd-app-paths", STOPPED, 0); + request(&engine, &paths).unwrap(); + assert_eq!( + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() + ); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let (_root, paths) = temp_app_paths("stdio-nonzero"); + let answered = FakeEngine::answering("rocmd-answers-then-fails", STOPPED, 2); + assert_eq!( + request(&answered, &paths).unwrap(), + json!({"stopped": true}) + ); + + let failed = FakeEngine::answering( + "rocmd-error-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + assert_eq!( + request(&failed, &paths).unwrap_err().to_string(), + "stop_failed: it broke" + ); + } + + #[test] + fn a_crash_with_no_answer_reports_the_status_and_stderr() { + let (_root, paths) = temp_app_paths("stdio-crash"); + let crash = FakeEngine::new("rocmd-crash", true, "echo boom >&2\nexit 3"); + assert_eq!( + request(&crash, &paths).unwrap_err().to_string(), + "engine stdio process exited with code 3: boom" + ); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let (_root, paths) = temp_app_paths("stdio-garbage"); + let garbage = + FakeEngine::new("rocmd-garbage", true, "printf 'not json'\necho oops >&2"); + assert_eq!( + request(&garbage, &paths).unwrap_err().to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + } + + /// A request to a child that has already exited is an ordinary error + /// wherever SIGPIPE stands: ignored (Rust's startup default, which the + /// test harness keeps) or at its default action, which is how `rocm` + /// runs and so how `rocm daemon` runs this code. Re-executes this + /// test in a fresh process for each case, since changing the disposition + /// process-wide would race sibling tests. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition() { + if let Some(arm) = std::env::var_os("ROCMD_TEST_ENGINE_SIGPIPE_ARM") { + if arm == "default-action" { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + } + let (_root, paths) = temp_app_paths("stdio-sigpipe"); + let engine = FakeEngine::new("rocmd-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let envelope = EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({ "blob": big }), + }; + let result = engine_request_via::( + engine.path(), + &paths, + "vllm", + SERVICE_ID, + &envelope, + ); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let run_arm = |arm: &str| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition", + "--exact", + "--nocapture", + ]) + .env("ROCMD_TEST_ENGINE_SIGPIPE_ARM", arm) + .status() + .unwrap() + }; + + let ignored = run_arm("ignored"); + assert!(ignored.success(), "{ignored:?}"); + + let default_action = run_arm("default-action"); + assert_eq!( + default_action.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {default_action:?}" + ); + assert!(default_action.success(), "{default_action:?}"); + } + } + /// The stdio command is `rocm`'s alone: a daemon run as `rocmd` asks the /// engine through the `rocm` beside it, which `rocmd` answered with an /// unrecognised-subcommand error, so its readiness checks and its engine diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index 0076e641a6..a19f9d680b 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -316,9 +316,12 @@ fn engine_serve_http_args( if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); - args.extend(crate::common::optional_arg("--env-id", env_id)); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( + "--runtime-id", + runtime_id, + )); + args.extend(rocm_engine_protocol::optional_arg("--env-id", env_id)); + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", engine_recipe_json, )); @@ -886,18 +889,18 @@ fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { .unwrap_or("gpu_required") .to_owned(), ]; - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--runtime-id", record.runtime_id.as_deref(), )); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--env-id", record.env_id.as_deref(), )); if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", record.engine_recipe_json.as_deref(), )); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 2147a8e00c..ae46f662f8 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -55,6 +55,7 @@ pub mod rocm_install; pub mod runtime; pub mod service_identity; pub mod service_record; +mod sigpipe; pub mod terminal; pub mod user_path; @@ -179,7 +180,9 @@ pub use net::{ http_get_text_with_auth, http_get_with_auth, http_post_json_with_auth, parse_http_endpoint, read_http_response_bounded, write_all_tcp_stream, }; -pub use paths::{AppPaths, engine_plugin_dirs}; +pub use paths::{ + AppPaths, CACHE_DIR_ENV, CONFIG_DIR_ENV, DATA_DIR_ENV, ENGINE_ENVS_ROOT_ENV, engine_plugin_dirs, +}; pub use proc_lifecycle::{ IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, process_start_ticks, terminate_verified, @@ -234,6 +237,7 @@ pub use service_record::{ INFERENCE_VERIFIED_STATE_KEY, ManagedServiceNotFound, ManagedServiceRecord, engine_stop_decides_confirmation, }; +pub use sigpipe::with_sigpipe_ignored; pub use terminal::{quotable_in_single_quotes, strip_terminal_control_sequences}; pub use uv::{ DEFAULT_UV_TIMEOUT_SECS, DependencyViolation, UV_CACHE_DIR_ENV, UV_CACHE_DIR_OVERRIDE_ENV, diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index d8eecc41c9..ee4a6f1a07 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -35,7 +35,40 @@ pub struct AppPaths { pub cache_dir: PathBuf, } +/// The variable that moves where engine virtualenvs live (see +/// [`AppPaths::engine_envs_root`]), and the one `rocm` sets on an engine child +/// that must install into a particular root. +pub const ENGINE_ENVS_ROOT_ENV: &str = "ROCM_CLI_ENGINE_ENVS_ROOT"; + +/// The variable that moves the config root. +pub const CONFIG_DIR_ENV: &str = "ROCM_CLI_CONFIG_DIR"; +/// The variable that moves the data root (and, unless it is set too, the cache +/// root that follows it). +pub const DATA_DIR_ENV: &str = "ROCM_CLI_DATA_DIR"; +/// The variable that moves the cache root. +pub const CACHE_DIR_ENV: &str = "ROCM_CLI_CACHE_DIR"; + impl AppPaths { + /// The three `ROCM_CLI_*_DIR` variables that make a child process run on + /// these roots, in the order config, data, cache. [`Self::discover`] reads + /// the same three back, so a child started with these finds the roots its + /// parent was given. + pub fn env_vars(&self) -> [(&'static str, &Path); 3] { + [ + (CONFIG_DIR_ENV, self.config_dir.as_path()), + (DATA_DIR_ENV, self.data_dir.as_path()), + (CACHE_DIR_ENV, self.cache_dir.as_path()), + ] + } + + /// Set [`Self::env_vars`] on `command`, so the child it starts runs on these + /// roots. + pub fn apply_env(&self, command: &mut std::process::Command) { + for (key, value) in self.env_vars() { + command.env(key, value); + } + } + /// Resolve the config, data and cache dirs from the environment and /// `config.json`. /// @@ -48,10 +81,10 @@ impl AppPaths { /// does, and the AIM engine then keeps its state file in the working /// directory, as it already did when no data dir could be found. pub fn discover() -> Result { - let data_dir_override = env_path_override("ROCM_CLI_DATA_DIR"); - let cache_dir_override = env_path_override("ROCM_CLI_CACHE_DIR"); + let data_dir_override = env_path_override(DATA_DIR_ENV); + let cache_dir_override = env_path_override(CACHE_DIR_ENV); let paths = Self::from_overrides(PathOverrides { - config: env_path_override("ROCM_CLI_CONFIG_DIR"), + config: env_path_override(CONFIG_DIR_ENV), data: data_dir_override.clone(), cache: cache_dir_override.clone(), })?; @@ -174,7 +207,7 @@ impl AppPaths { /// dropped; `engine_envs_dir_reads_its_root_from_the_environment` is what /// keeps the lookup honest meanwhile. pub fn engine_envs_root(&self) -> PathBuf { - self.engine_envs_root_from(env_path_override("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref()) + self.engine_envs_root_from(env_path_override(ENGINE_ENVS_ROOT_ENV).as_deref()) } /// [`Self::engine_envs_root`] against a caller-supplied override. @@ -577,6 +610,42 @@ mod tests { use crate::RocmCliConfig; use crate::test_support::temp_app_paths; + /// `env_vars` and `discover` share one constant per name, so they cannot + /// disagree; the names themselves are what a child process and a user's + /// shell see, so they are pinned here. + #[test] + fn apply_env_sets_exactly_the_three_roots_on_a_command() { + let (_root, paths) = temp_app_paths("apply-env"); + let mut command = std::process::Command::new("true"); + paths.apply_env(&mut command); + let mut set: Vec<_> = command + .get_envs() + .map(|(key, value)| (key.to_owned(), value.map(std::ffi::OsStr::to_owned))) + .collect(); + let expected: Vec<_> = paths + .env_vars() + .iter() + .map(|(key, value)| ((*key).into(), Some(value.as_os_str().to_owned()))) + .collect(); + let mut expected = expected; + set.sort(); + expected.sort(); + assert_eq!(set, expected); + } + + #[test] + fn env_vars_pair_each_root_with_its_public_variable_name() { + let (_root, paths) = temp_app_paths("env-vars"); + assert_eq!( + paths.env_vars(), + [ + ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), + ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), + ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), + ] + ); + } + /// For every `config.json` that `load` accepts, discovery reads the /// recorded folder as `load` does: the saved `therock_venv`, trimmed, and /// nothing when it is absent or blank. diff --git a/crates/rocm-core/src/sigpipe.rs b/crates/rocm-core/src/sigpipe.rs new file mode 100644 index 0000000000..2a28b0a223 --- /dev/null +++ b/crates/rocm-core/src/sigpipe.rs @@ -0,0 +1,258 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Writing to a pipe whose reader has gone away, without being killed for it. +//! +//! Rust starts every process with SIGPIPE ignored, so a write to a closed pipe +//! returns `EPIPE`. `rocm` restores the default action at startup so it ends +//! quietly when its own output is piped into `head`, and `rocm daemon` +//! hosts the daemon in that same process. Under the default action a write to +//! a child that has already exited kills the writer before it can report why +//! the child exited, so code that writes to a child's stdin wraps the write in +//! [`with_sigpipe_ignored`]. + +/// Run `f` with SIGPIPE ignored, putting the previous disposition back afterwards. +/// +/// A write that reaches a closed pipe inside `f` then returns an `EPIPE` error +/// instead of delivering a signal. Where the disposition was already ignored +/// (Rust's startup default) this changes nothing. The disposition is +/// process-wide, so keep `f` to the write itself. Windows has no SIGPIPE and +/// runs `f` as is. +/// +/// The disposition is read and put back with `signal()`, which reports only the +/// handler (`SIG_DFL`, `SIG_IGN` or a function address). One that was installed +/// with `sigaction()` and carries flags or a signal mask would come back without +/// them. That is not a case here: this process only ever sets `SIG_DFL` (`rocm` +/// at startup) and `SIG_IGN` (this guard) for SIGPIPE. +/// +/// Guarded regions may overlap, on one thread or on several (the stderr +/// forwarder of one engine request runs while another request writes to its +/// child). The disposition in force before the first region is remembered once +/// and put back when the last region ends, so an overlap can neither restore +/// "ignored" over a default action nor leave "ignored" behind. +#[cfg(unix)] +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + let _guard = unix::IgnoreGuard::enter(); + f() +} + +#[cfg(unix)] +#[allow(unsafe_code)] // libc FFI +mod unix { + use std::sync::{Mutex, PoisonError}; + + /// How many guarded regions are open, and the disposition that was in force + /// before the first of them. + static OPEN: Mutex<(usize, libc::sighandler_t)> = Mutex::new((0, libc::SIG_DFL)); + + /// Holds SIGPIPE ignored from [`IgnoreGuard::enter`] until it is dropped, so + /// a panic inside the guarded region still restores the disposition. + pub(super) struct IgnoreGuard; + + impl IgnoreGuard { + pub(super) fn enter() -> Self { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + if open.0 == 0 { + // SAFETY: reading/installing a signal disposition is always safe. + open.1 = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + } + open.0 += 1; + Self + } + } + + impl Drop for IgnoreGuard { + fn drop(&mut self) { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + open.0 -= 1; + if open.0 == 0 { + // SAFETY: restoring the disposition captured by the first guard. + unsafe { + libc::signal(libc::SIGPIPE, open.1); + } + } + } + } +} + +#[cfg(not(unix))] +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + f() +} + +#[cfg(all(test, unix))] +mod tests { + use super::with_sigpipe_ignored; + use std::io::Write as _; + use std::os::unix::process::ExitStatusExt as _; + use std::process::{Command, ExitStatus, Stdio}; + + /// The disposition SIGPIPE has now (read by installing the same one back). + #[allow(unsafe_code)] // libc::signal + fn current_disposition() -> libc::sighandler_t { + // SAFETY: reading/installing a signal disposition is always safe. + let current = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + // SAFETY: putting back what was just read. + unsafe { + libc::signal(libc::SIGPIPE, current); + } + current + } + + /// Guarded regions that overlap, nested on one thread or interleaved across + /// two, leave SIGPIPE ignored for as long as any of them is open and put the + /// original default action back only when the last one closes. With a guard + /// that saved and restored per call, the region that ended first would + /// restore "ignored" over the second's `SIG_DFL` capture, or restore the + /// default under the region still open. Runs in a re-executed child, since + /// the disposition is process-wide and the harness leaves it ignored. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn overlapping_guards_restore_the_original_disposition_once() { + if std::env::var_os("ROCM_TEST_SIGPIPE_OVERLAP_ARM").is_some() { + use std::sync::Barrier; + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + + // Nested on one thread. + with_sigpipe_ignored(|| { + with_sigpipe_ignored(|| {}); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the inner region ended while the outer one was open" + ); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "nested"); + + // Two threads whose regions overlap, the first to enter leaving + // first. The second thread waits until the first is inside its + // region before it enters its own, so the order is the same on + // every run: were it left to the scheduler, a per-call save and + // restore would pass whenever the "second" closure happened to run + // first. + let first_in = Barrier::new(2); + let entered = Barrier::new(2); + let first_left = Barrier::new(2); + std::thread::scope(|scope| { + scope.spawn(|| { + with_sigpipe_ignored(|| { + first_in.wait(); + entered.wait(); + }); + first_left.wait(); + }); + scope.spawn(|| { + first_in.wait(); + with_sigpipe_ignored(|| { + entered.wait(); + first_left.wait(); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the first region ended while this one was open" + ); + }); + }); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "interleaved"); + return; + } + + let status = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "sigpipe::tests::overlapping_guards_restore_the_original_disposition_once", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_OVERLAP_ARM", "1") + .status() + .expect("re-exec the test arm in a child process"); + assert!(status.success(), "{status:?}"); + } + + /// A write to a dead child's stdin under the default SIGPIPE action is + /// fatal, and the same write inside [`with_sigpipe_ignored`] returns an + /// error instead. + /// + /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE + /// at Rust's default `SIG_IGN`, and flipping it process-wide would race + /// sibling tests. So this re-execs itself in fresh processes that install + /// `SIG_DFL` first, and checks BOTH directions: + /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — + /// this is the regression the plain `SIG_DFL` reset introduced; + /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns + /// an error instead — this is the fix. + /// + /// Each child reaps its short-lived grandchild before writing, so the read + /// end is closed deterministically and the outcome does not race. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { + // Grandchild helper: install the default action, spawn a process that + // exits immediately, reap it, then write to its now-closed stdin. The + // "guarded" arm wraps the write; the "unguarded" arm does not. + if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let mut grandchild = Command::new("true") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a process that exits immediately"); + let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); + grandchild.wait().expect("reap grandchild"); + // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; + // it must exceed the pipe buffer so the write actually reaches the + // closed read end rather than being swallowed by kernel buffering. + let payload = vec![b'x'; 64 * 1024]; + if mode == "guarded" { + let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); + assert!( + result.is_err(), + "guarded write to a dead pipe should return BrokenPipe, not succeed" + ); + } else { + // Unguarded: under SIG_DFL this write delivers SIGPIPE and never + // returns. If the process somehow survives, exit 0 so the parent's + // "should have been signalled" assertion fails loudly. + let _ = stdin.write_all(&payload); + } + return; + } + + let exe = std::env::current_exe().expect("current test executable"); + let run_arm = |mode: &str| -> ExitStatus { + Command::new(&exe) + .args([ + "sigpipe::tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_ARM", mode) + .status() + .expect("re-exec the test arm in a child process") + }; + + let unguarded = run_arm("unguarded"); + assert_eq!( + unguarded.signal(), + Some(libc::SIGPIPE), + "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ + SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" + ); + + let guarded = run_arm("guarded"); + assert!( + guarded.success(), + "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ + SIGPIPE reached the process instead of surfacing as an error" + ); + } +} diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index 1d60c6db3e..cdb00339fa 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -10,7 +10,13 @@ publish.workspace = true [lints] workspace = true +[features] +# Compiles `fake_engine`, a stand-in engine child for tests, into the `rocm` and +# `rocmd` test builds. Never enabled in a release build. +test-support = [] + [dependencies] +anyhow.workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true @@ -18,3 +24,7 @@ serde_json.workspace = true [dev-dependencies] # Property tests for the engine protocol decoders (`tests/protocol_properties.rs`). proptest = { version = "1", default-features = false, features = ["std"] } + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test installs in a re-executed child. +libc.workspace = true diff --git a/crates/rocm-engine-protocol/src/fake_engine.rs b/crates/rocm-engine-protocol/src/fake_engine.rs new file mode 100644 index 0000000000..dc90a42449 --- /dev/null +++ b/crates/rocm-engine-protocol/src/fake_engine.rs @@ -0,0 +1,174 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A stand-in engine child for tests. +//! +//! It is a `/bin/sh` script that records how it was launched (arguments, +//! environment, stdin) into a scratch directory and then runs a caller-supplied +//! body, so a test can assert what an engine IPC client put on the child's +//! command line, environment and stdin, and shape what comes back (a response +//! envelope, a crash, stderr output). +//! +//! Test-only: compiled for this crate's own tests and for the `rocm` and +//! `rocmd` test builds through the `test-support` feature, so the two engine +//! IPC callers are exercised against the same fake rather than each growing a +//! private one. Unix-only, because the fake is a shell script; the pure parts +//! of the client are tested on every platform without it. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; + +static NEXT_ENGINE: AtomicU32 = AtomicU32::new(0); + +/// `value` as a single-quoted POSIX shell word. Inside single quotes nothing is +/// special except the quote itself, which cannot be escaped there: close the +/// literal, emit an escaped quote, and reopen it (`'\''`). An apostrophe in a +/// response (`can't stop`) or in `TMPDIR` would otherwise end the literal early. +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', r"'\''")) +} + +pub struct FakeEngine { + dir: PathBuf, + script: PathBuf, +} + +impl FakeEngine { + /// Write the script. `reads_stdin` makes the fake drain its stdin into + /// `stdin` first (what a well-behaved engine does); pass `false` for an + /// engine that exits without reading its request. `body` runs after the + /// recording and may use `$dir`, the scratch directory. + /// + /// The script is written through [`rocm_core::fake_executable`], so running + /// it straight away cannot fail with `ETXTBSY` while other tests on parallel + /// threads spawn children of their own. + pub fn new(label: &str, reads_stdin: bool, body: &str) -> Self { + Self::new_in(&std::env::temp_dir(), label, reads_stdin, body) + } + + /// [`FakeEngine::new`] with its scratch directory created under `base` + /// instead of the system temporary directory. + pub fn new_in(base: &Path, label: &str, reads_stdin: bool, body: &str) -> Self { + let dir = base.join(format!( + "rocm-fake-engine-{label}-{}-{}", + std::process::id(), + NEXT_ENGINE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&dir).expect("create the fake engine's scratch directory"); + let script = dir.join("engine.sh"); + let read_stdin = if reads_stdin { + "cat > \"$dir/stdin\"\n" + } else { + "" + }; + rocm_core::fake_executable::write_fake_executable( + &script, + format!( + "#!/bin/sh\ndir={}\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", + shell_quote(&dir.to_string_lossy()) + ), + ) + .expect("write the fake engine script"); + Self { dir, script } + } + + /// A fake that reads its request and answers with `response` (a complete + /// response envelope as JSON text), exiting with `status`. + pub fn answering(label: &str, response: &str, status: i32) -> Self { + Self::new( + label, + true, + &format!("printf '%s' {}\nexit {status}", shell_quote(response)), + ) + } + + pub fn path(&self) -> &Path { + &self.script + } + + /// The arguments the fake was started with, one per entry. + pub fn argv(&self) -> Vec { + fs::read_to_string(self.dir.join("argv")) + .expect("the fake engine never recorded its arguments") + .lines() + .map(str::to_owned) + .collect() + } + + /// Everything the fake read from its stdin, verbatim. + pub fn stdin(&self) -> String { + fs::read_to_string(self.dir.join("stdin")).expect("the fake engine never read its stdin") + } + + /// The value of `key` in the fake's environment, if it was set. + pub fn env(&self, key: &str) -> Option { + let prefix = format!("{key}="); + fs::read_to_string(self.dir.join("env")) + .expect("the fake engine never recorded its environment") + .lines() + .find_map(|line| line.strip_prefix(&prefix).map(str::to_owned)) + } +} + +impl Drop for FakeEngine { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + use std::process::{Command, Stdio}; + + /// Run `engine` the way a client does: request on stdin, answer on stdout. + fn run(engine: &FakeEngine) -> (std::process::ExitStatus, String, String) { + let mut child = Command::new(engine.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"request\n").unwrap(); + let output = child.wait_with_output().unwrap(); + ( + output.status, + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) + } + + #[test] + fn shell_quote_survives_each_character_a_literal_can_hold() { + assert_eq!(shell_quote("plain"), "'plain'"); + assert_eq!(shell_quote("can't"), r"'can'\''t'"); + assert_eq!(shell_quote("''"), r"''\'''\'''"); + assert_eq!(shell_quote(""), "''"); + } + + #[test] + fn a_response_with_an_apostrophe_is_printed_verbatim() { + let response = r#"{"ok":false,"error":{"code":"x","message":"can't stop"}}"#; + let engine = FakeEngine::answering("fake-apostrophe-response", response, 0); + let (status, stdout, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stdout, response); + assert_eq!(stderr, "", "the script must parse"); + } + + #[test] + fn a_scratch_directory_with_an_apostrophe_still_records_the_launch() { + let base = + std::env::temp_dir().join(format!("rocm-fake-engine-it's-{}", std::process::id())); + let engine = FakeEngine::new_in(&base, "fake-apostrophe-dir", true, "exit 0"); + let (status, _, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stderr, "", "the script must parse"); + assert_eq!(engine.stdin(), "request\n"); + drop(engine); + let _ = fs::remove_dir_all(&base); + } +} diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6ca773ccd6..6b3f4065bf 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -2,6 +2,15 @@ // // SPDX-License-Identifier: MIT +#[cfg(all(unix, any(test, feature = "test-support")))] +pub mod fake_engine; + +mod stdio_client; +pub use stdio_client::{ + ENGINE_PROGRESS_STDERR_ENV, EngineStdioOptions, decode_engine_response, + engine_request_envelope, run_engine_stdio, +}; + use serde::Deserialize; use serde::Serialize; use serde_json::Value; @@ -280,6 +289,17 @@ impl GpuSelection { } } +/// The two arguments `flag value` when `value` is set, none when it is not, for +/// building the command line of an engine or supervisor child. Shared so `rocm` +/// and `rocmd` spell optional flags the same way. +#[must_use] +pub fn optional_arg(flag: &str, value: Option<&str>) -> Vec { + match value { + Some(value) => vec![flag.to_owned(), value.to_owned()], + None => Vec::new(), + } +} + /// Render an explicit GPU ordinal as a comma-separated `HIP_VISIBLE_DEVICES` /// value. Returns `None` when the slice is empty (no pinning requested). #[must_use] @@ -892,6 +912,15 @@ mod tests { assert!(GpuSelection::parse_cli_value("0,2,3").is_err()); } + #[test] + fn optional_arg_is_the_flag_and_value_or_nothing() { + assert_eq!( + optional_arg("--runtime-id", Some("therock-release:gfx120X-all")), + ["--runtime-id", "therock-release:gfx120X-all"] + ); + assert!(optional_arg("--runtime-id", None).is_empty()); + } + #[test] fn gpu_indices_helpers_handle_auto_and_pinned() { assert_eq!(gpu_indices_to_csv(&[]), None); diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs new file mode 100644 index 0000000000..5a8989e5a7 --- /dev/null +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -0,0 +1,1970 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The client side of the engine stdio protocol: run an engine as a child +//! process, send it one [`EngineRequestEnvelope`] on stdin as a single line of +//! JSON, and read one [`EngineResponseEnvelope`] back from stdout. +//! +//! `rocm` and `rocmd` both talk to engines this way and share this one +//! implementation. They differ in *which* process they start (`rocm` runs the +//! resolved engine binary with `stdio`, or answers a built-in engine in its own +//! process without a child at all; `rocmd` runs `rocm __engine-stdio `), +//! and that stays with each caller. What is shared is everything that follows +//! the choice of program: the request framing, the environment, how stderr is +//! handled, the write guard against SIGPIPE, what a failing exit status or an +//! unusable answer is reported as, and decoding the response. The few things +//! the callers want differently are the fields of [`EngineStdioOptions`]. +//! +//! The request and the child's lifetime are not bounded by a timeout: an engine +//! install legitimately runs for minutes, and no caller has ever set one. +//! +//! Nor is the wait for the pipes after the child has exited, and that is a known +//! limit rather than an oversight. The answer is read to the end of the child's +//! stdout, and the stderr collector is joined, so an engine that leaves a +//! grandchild holding either pipe open makes the request wait for that grandchild +//! to exit. Both clients this one replaces behaved the same way (`rocm` read +//! stdout to the end then joined its stderr thread; `rocmd` used +//! `wait_with_output`), so nothing new is exposed to it, but the daemon is long +//! lived and a hung request holds its caller. Bounding it exactly would mean +//! reading both pipes on their own threads, waiting for the child itself without a +//! limit and for the pipes only up to a deadline after it exits, and choosing what +//! to report when the deadline passes (an answer already complete? a partial +//! one?). That is a timeout policy for an engine contract nothing here sets, so +//! this change leaves it to be decided on its own. The one wait this client adds +//! itself, for the stderr of a child whose request could not be written, is +//! bounded by `EXITED_STDERR_GRACE`. +//! +//! A request that cannot be written is the other place the answer-wins rule does +//! not apply: if the engine answered and exited without draining a request larger +//! than a pipe buffer, the write fails, the child is killed and its stdout is +//! never read, so the error carries its stderr and not its answer. Every engine +//! here (`lemonade`, `vllm`, `aim`, and the `rocm __engine-stdio` verb that serves +//! the built-in ones) reads stdin to the end before it answers, so reaching this +//! needs an engine that breaks the protocol. Decoding such an answer would mean +//! waiting, up to a deadline, for the child to finish printing it and to exit +//! before deciding it is complete, and an engine that has stopped reading is no +//! engine to wait for, so it is left as a documented limit. +//! +//! Forwarding the child's stderr guards each write against SIGPIPE on its own +//! (see `forward_best_effort`), and the guard is held for as long as that write +//! blocks. If the sink is stalled rather than closed (a terminal that has stopped +//! reading), SIGPIPE therefore stays ignored process-wide for that time, and a +//! concurrent `print!` to a closed pipe on another thread fails with an error +//! (which `print!` turns into a panic) instead of ending the process quietly. The +//! guard is not held across the reads between lines, so this needs a stalled +//! sink, not just a quiet engine. It is accepted: a guard that could be taken +//! back mid-write does not exist, and without one a closed sink would end +//! `rocm` before the request could report why. +//! +//! These three are deliberate tradeoffs, not oversights: the wait on pipes held +//! by a grandchild is unbounded, an answer from an engine that exits without +//! draining a large request is not decoded, and SIGPIPE stays ignored +//! process-wide while a stalled stderr forward blocks. Only the first is +//! partly cheap to bound: the stderr side would be one deadline on +//! `StderrCollector::finish`, but stdout is read to its end on the calling thread, +//! so a bound there needs a second reader thread and a decision on what to report +//! when it fires, and bounding one pipe and not the other would be a half +//! measure. The stderr is read from the moment the child starts, on its own +//! thread, so an engine that fills it before draining a large request cannot +//! deadlock the write. + +use std::ffi::OsStr; +use std::io::{self, Read, Write}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc; +use std::sync::{Arc, Mutex, PoisonError}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, with_sigpipe_ignored}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::{ + ENDPOINT_API_KEY_FILE_ENV, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, +}; + +/// Set to `1` on an engine child whose caller wants its progress lines on +/// stderr as they happen (an engine install). +pub const ENGINE_PROGRESS_STDERR_ENV: &str = "ROCM_ENGINE_PROGRESS_STDERR"; + +/// What a caller wants different about the child [`run_engine_stdio`] starts. +/// +/// The default asks for nothing beyond the request itself: the child inherits +/// the caller's environment and its stderr is only collected. +#[derive(Debug, Clone, Copy, Default)] +pub struct EngineStdioOptions<'a> { + /// Run the child on these roots, by setting the `ROCM_CLI_*_DIR` variables + /// ([`AppPaths::env_vars`]). `None` leaves the child to inherit whatever the + /// caller's own environment says. + pub app_paths: Option<&'a AppPaths>, + /// The root engine virtualenvs go under for this request, set as + /// [`ENGINE_ENVS_ROOT_ENV`]. Only an install into a particular root needs it. + pub env_root: Option<&'a Path>, + /// A key file to hand the engine through [`ENDPOINT_API_KEY_FILE_ENV`], so + /// its HTTP probe of a protected endpoint authenticates. The caller decides + /// whether a file holds a usable key; this only carries it. + pub endpoint_key_file: Option<&'a Path>, + /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). + pub request_progress: bool, + /// Write the child's stderr to this process's stderr as it arrives, so a + /// terminal user watches an install. Either way the whole of it is kept and + /// shown in the error when the request fails. + pub forward_stderr: bool, + /// Say, on this process's stderr, when the child answered and then exited + /// with a failing status. The answer is used either way; this only leaves a + /// trace of the status, which the caller would otherwise never see. For a + /// daemon, whose stderr is its log; a terminal user has no use for it. + pub note_failing_exit_after_answer: bool, +} + +/// Build the request envelope for `method` from `request`. +pub fn engine_request_envelope( + method: EngineMethod, + request: &T, +) -> Result { + Ok(EngineRequestEnvelope { + method, + payload: serde_json::to_value(request) + .context("failed to encode engine request payload")?, + }) +} + +/// The command [`run_engine_stdio`] runs: `program` with `args`, all three +/// standard streams piped, and the environment `options` asks for. Separate so +/// a test can read the command line and environment without starting anything. +fn engine_stdio_command(program: &Path, args: I, options: &EngineStdioOptions<'_>) -> Command +where + I: IntoIterator, + S: AsRef, +{ + let mut command = Command::new(program); + command + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(paths) = options.app_paths { + paths.apply_env(&mut command); + } + if let Some(env_root) = options.env_root { + command.env(ENGINE_ENVS_ROOT_ENV, env_root); + } + if let Some(key_file) = options.endpoint_key_file { + command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); + } + if options.request_progress { + command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); + } + command +} + +/// Send `envelope` to the engine child `program args…` and return what it +/// answered. +/// +/// A failing exit status is an error only when the child printed no answer: an +/// engine that reports its own failure in an error envelope and then exits +/// non-zero is read as that envelope, which says more than the status does. The +/// same holds for a success envelope (`ok: true`): a child that answers +/// successfully and then exits non-zero is reported as the success it answered, +/// and the failing exit is not reported or logged anywhere, by design, because +/// the answer is what the caller asked for. +/// Every other failure names what went wrong and carries the child's stderr. +pub fn run_engine_stdio( + program: &Path, + args: I, + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, +) -> Result +where + I: IntoIterator, + S: AsRef, +{ + run_engine_stdio_forwarding( + program, + args, + options, + envelope, + options.forward_stderr.then(io::stderr), + ) +} + +/// [`run_engine_stdio`] with the sink the child's stderr is forwarded to chosen +/// by the caller (`None` forwards nothing), so a test can see what is forwarded +/// without capturing this process's own stderr. [`run_engine_stdio`] is the one +/// place the `forward_stderr` option becomes a sink. +fn run_engine_stdio_forwarding( + program: &Path, + args: I, + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, + forward: Option, +) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let mut child = engine_stdio_command(program, args, options) + .spawn() + .with_context(|| format!("failed to spawn engine stdio process {}", program.display()))?; + + let stdin = child.stdin.take(); + let mut stdin = require_pipe(&mut child, stdin, "stdin")?; + // Read the child's stderr from the start, on its own thread: an engine that + // fills the stderr pipe before it has drained a request larger than a pipe + // buffer would otherwise block on that write while this process blocks on + // its own, with nobody reading either side. + let stderr_collector = child + .stderr + .take() + .map(|stderr| StderrCollector::start(stderr, forward)); + // Ignore SIGPIPE for the duration of the write. Under the default action, an + // engine that exits before reading its stdin would kill this process with + // SIGPIPE (`rocm` runs that way, and so does the daemon hosted in it); + // ignored, the write returns EPIPE and fails as an ordinary error. + let written = with_sigpipe_ignored(|| { + serde_json::to_writer(&mut stdin, envelope).map_err(io::Error::from)?; + stdin.write_all(b"\n") + }); + if let Err(error) = written { + return Err(abandon_unwritten_request( + child, + stderr_collector, + &error, + EXITED_STDERR_GRACE, + )); + } + // End of the request: the engine reads to EOF before it answers. + drop(stdin); + + let stdout_pipe = child.stdout.take(); + let stdout_pipe = require_pipe(&mut child, stdout_pipe, "stdout")?; + let stdout = match read_response(&mut child, stdout_pipe) { + Ok(stdout) => stdout, + Err(error) => { + return Err(read_failure_with_stderr( + error, + stderr_collector, + EXITED_STDERR_GRACE, + )); + } + }; + let status = child + .wait() + .context("failed waiting for engine stdio response")?; + let stderr = stderr_collector + .map(|collector| collector.finish(None)) + .unwrap_or_default(); + + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if !status.success() && stdout.is_empty() { + let exited = exit_note(status); + if stderr.is_empty() { + bail!("{exited}"); + } + bail!("{exited}: {stderr}"); + } + let answer = serde_json::from_slice(&stdout) + .with_context(|| unparseable_answer_message(status, &stderr)); + // The note says the answer was used, so it is only made for an answer that + // parsed; an unparseable one is reported by the error above, with the status. + if answer.is_ok() && !status.success() && options.note_failing_exit_after_answer { + note_failing_exit_after_answer(&mut io::stderr(), status); + } + answer +} + +/// Say on `sink` (this process's stderr, in production) that the child answered +/// and then failed. Written like the stderr forwarding, through the SIGPIPE guard +/// and ignoring a failed write: a closed stderr must neither end a daemon hosted +/// in `rocm` (SIGPIPE at its default action) nor panic it, as `eprintln!` would. +fn note_failing_exit_after_answer(sink: &mut impl Write, status: ExitStatus) { + let note = format!( + "{} after printing its answer; the answer was used\n", + exit_note(status) + ); + forward_best_effort(sink, note.as_bytes()); +} + +/// How a failing exit is worded wherever the status is reported. Built from the +/// exit code (or the signal that ended the process) rather than from the status's +/// own `Display`, which already says "exit status: N" on Unix and "exit code: N" +/// on Windows. +fn exit_note(status: ExitStatus) -> String { + if let Some(code) = status.code() { + return format!("engine stdio process exited with code {code}"); + } + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + if let Some(signal) = status.signal() { + return format!("engine stdio process was terminated by signal {signal}"); + } + } + "engine stdio process ended without an exit code".to_owned() +} + +/// The error for an answer that is not a response envelope. A child that also +/// exited with a failing status says so, because that is often why the answer is +/// cut short; the stderr it wrote follows when there is any. +fn unparseable_answer_message(status: ExitStatus, stderr: &str) -> String { + let mut message = "failed to parse engine response envelope".to_owned(); + if !status.success() { + message = format!("{message} ({})", exit_note(status)); + } + if !stderr.is_empty() { + message = format!("{message}; stderr: {stderr}"); + } + message +} + +/// `pipe` of `child`, or an error that also kills and reaps the child: a child +/// whose request cannot be sent must not be left running, as on every other +/// failure path. Unreachable while the command pipes all three streams, which is +/// why it is a function of its own, testable with a pipe that is missing. +fn require_pipe(child: &mut Child, pipe: Option, name: &str) -> Result { + pipe.ok_or_else(|| { + let _ = child.kill(); + let _ = child.wait(); + anyhow::anyhow!("engine stdio child did not expose {name}") + }) +} + +/// The error for a stdout that could not be read (the child is already killed +/// and reaped), with what its stderr had said, as every other failure carries +/// it: a crashing or signalled engine is where the explanation matters most. +fn read_failure_with_stderr( + error: anyhow::Error, + stderr: Option, + grace: Duration, +) -> anyhow::Error { + let stderr = stderr + .map(|collector| collector.finish(Some(Instant::now() + grace))) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if stderr.is_empty() { + return error; + } + anyhow::anyhow!("{error:#}; stderr: {stderr}") +} + +/// How long the stderr of a child that was killed, or had exited, is waited for +/// when its request could not be written. +const EXITED_STDERR_GRACE: Duration = Duration::from_secs(1); + +/// The request could not be written to `child`'s stdin, so it will never answer: +/// do not leave it running or unreaped (a long-lived caller would collect one +/// zombie per failure), and report why it was not listening. +/// +/// The child is killed (nothing to do for one that has already exited) and +/// waited for, then whatever it wrote to stderr is added to the error: an engine +/// that exits at once usually says why there, and that is the explanation the +/// caller wants. The kill does not wait on `try_wait` showing an exit first, +/// because the pipe closes while the process is still exiting, so a check made +/// right after the failed write can still see it running. The read is bounded +/// by `grace`: a grandchild that inherited the pipe keeps it open past the exit, +/// and this error path must not block on it. +fn abandon_unwritten_request( + mut child: Child, + stderr: Option, + write_error: &io::Error, + grace: Duration, +) -> anyhow::Error { + let _ = child.kill(); + let _ = child.wait(); + let deadline = Instant::now() + grace; + let stderr = stderr + .map(|collector| collector.finish(Some(deadline))) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + let mut message = format!("failed to write engine request: {write_error}"); + if !stderr.is_empty() { + message = format!("{message}; stderr: {stderr}"); + } + anyhow::anyhow!(message) +} + +/// A child's stderr being collected on a thread of its own, from the moment it +/// is started, whatever the main thread is doing meanwhile. +/// +/// The thread outlives the collector when a request fails and a descendant of +/// the engine still holds the pipe: nobody will read what it collects, but it +/// must keep draining the pipe (closing it would hand that descendant a SIGPIPE +/// it did not get before). So dropping the collector marks the capture +/// abandoned, which releases what was retained and makes the thread discard +/// whatever arrives later, in chunks of a fixed size, so a long-lived daemon does +/// not grow with every failed request. +struct StderrCollector { + shared: Arc>, + /// Receives one message when the stderr has been read to its end. + finished: mpsc::Receiver<()>, +} + +/// What the collecting thread has read, and whether anyone still wants it. +#[derive(Default)] +struct Captured { + bytes: Vec, + /// Set when the collector is dropped. Read and written under the same lock as + /// `bytes`, so "emptied and marked" is one step as far as the reader is + /// concerned and the buffer cannot grow after the drop returns. + abandoned: bool, +} + +impl StderrCollector { + /// Start reading `stderr`, forwarding what arrives to `forward` when there is + /// one (see [`collect_stderr_into`]). + fn start( + stderr: impl Read + Send + 'static, + forward: Option, + ) -> Self { + let shared = Arc::new(Mutex::new(Captured::default())); + let (done, finished) = mpsc::channel(); + let sink = Arc::clone(&shared); + thread::spawn(move || { + collect_stderr_into(stderr, forward, &sink); + let _ = done.send(()); + }); + Self { shared, finished } + } + + /// What was collected, once the stderr has ended, or what has arrived by + /// `deadline` when there is one and the stderr has not ended by then (a + /// grandchild holding the pipe open). With no deadline this waits for the + /// end, which is the unbounded wait the module comment states. + fn finish(&self, deadline: Option) -> Vec { + match deadline { + None => { + let _ = self.finished.recv(); + } + Some(deadline) => { + let _ = self + .finished + .recv_timeout(deadline.saturating_duration_since(Instant::now())); + } + } + std::mem::take(&mut lock(&self.shared).bytes) + } +} + +impl Drop for StderrCollector { + fn drop(&mut self) { + let mut captured = lock(&self.shared); + captured.abandoned = true; + captured.bytes = Vec::new(); + } +} + +fn lock(shared: &Mutex) -> std::sync::MutexGuard<'_, Captured> { + shared.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Read the child's whole answer from `stdout`. +/// +/// A failed read would leave the child running with nobody waiting on it, and +/// this client lives in the long-lived daemon, so the child is killed and +/// reaped before the error is returned, as it is when the request cannot be +/// written. The stderr collector is not joined on that path: a grandchild that +/// inherited the pipe could hold it open, and the thread ends by itself once +/// the pipe closes. +fn read_response(child: &mut Child, mut stdout: impl Read) -> Result> { + let mut response = Vec::new(); + if let Err(error) = stdout.read_to_end(&mut response) { + let _ = child.kill(); + let _ = child.wait(); + return Err(anyhow::Error::new(error).context("failed to read engine stdio response")); + } + Ok(response) +} + +/// Read `stderr` to the end line by line, optionally writing each line to +/// `forward` (this process's stderr, in production) as it arrives, and return +/// everything read. A failed forward is ignored: a closed terminal must not +/// cost the request its answer. That holds for the signal as well as the error: +/// under SIGPIPE's default action (`rocm` runs that way) a write to a closed +/// pipe ends the whole process before `write_all` can return, so each forward +/// runs inside [`with_sigpipe_ignored`] (see [`forward_best_effort`]). A read error ends the +/// collection and is itself recorded as the last line. +#[cfg(test)] +fn collect_stderr(stderr: impl Read, forward: Option) -> Vec { + let shared = Mutex::new(Captured::default()); + collect_stderr_into(stderr, forward, &shared); + shared + .into_inner() + .unwrap_or_else(PoisonError::into_inner) + .bytes +} + +/// [`collect_stderr`], publishing into `shared` as each chunk arrives, not when a +/// line is complete, so a caller that stops waiting has everything that had been +/// written (a diagnostic with no trailing newline included) and not only the +/// complete lines. Once the capture is abandoned the thread keeps reading, so the +/// pipe stays drained, but keeps and forwards nothing. +fn collect_stderr_into( + mut stderr: impl Read, + mut forward: Option, + shared: &Mutex, +) { + let mut chunk = [0_u8; 4096]; + let mut carry = Utf8Carry::default(); + loop { + let bytes: Vec = match stderr.read(&mut chunk) { + Ok(0) => break, + Ok(read) => chunk[..read].to_vec(), + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => { + let message = format!("failed to read engine progress: {error}\n").into_bytes(); + publish(shared, forward.as_mut(), &mut carry, &message); + break; + } + }; + publish(shared, forward.as_mut(), &mut carry, &bytes); + } + // What the pipe ended in the middle of a character of is forwarded lossily. + if let Some(sink) = forward.as_mut() + && !lock(shared).abandoned + { + forward_best_effort(sink, &carry.finish()); + } +} + +/// Keep `bytes` and forward the complete UTF-8 of them, unless the capture has +/// been abandoned. The kept copy is the raw bytes; only what is forwarded is +/// held back at an incomplete trailing character. +fn publish( + shared: &Mutex, + forward: Option<&mut impl Write>, + carry: &mut Utf8Carry, + bytes: &[u8], +) { + { + let mut captured = lock(shared); + if captured.abandoned { + return; + } + captured.bytes.extend_from_slice(bytes); + } + if let Some(sink) = forward { + let complete = carry.push(bytes); + if !complete.is_empty() { + forward_best_effort(sink, &complete); + } + } +} + +/// U+FFFD in UTF-8, as bytes: a literal here would be a non-ASCII string in +/// production code, which the output contract forbids. +const REPLACEMENT_CHARACTER: &[u8] = &[0xEF, 0xBF, 0xBD]; + +/// Turns arbitrary chunks of a byte stream into text that is valid UTF-8 at every +/// boundary, for a sink that requires it. Rust's stdio on a Windows console +/// refuses a write that is not valid UTF-8, so a chunk that ends in the middle of +/// a multi-byte character would fail, and the forwarding discards that error, +/// losing valid output. The incomplete trailing sequence is carried to the next +/// chunk instead; bytes that can never be valid are replaced with U+FFFD, and +/// what is still carried when the stream ends is flushed the same way. +#[derive(Default)] +struct Utf8Carry { + pending: Vec, +} + +impl Utf8Carry { + /// The part of `pending + bytes` that is complete UTF-8 (with invalid bytes + /// replaced), keeping an incomplete trailing sequence for the next call. + fn push(&mut self, bytes: &[u8]) -> Vec { + self.pending.extend_from_slice(bytes); + let mut out = Vec::with_capacity(self.pending.len()); + let mut rest = std::mem::take(&mut self.pending); + loop { + match std::str::from_utf8(&rest) { + Ok(valid) => { + out.extend_from_slice(valid.as_bytes()); + return out; + } + Err(error) => { + let valid_up_to = error.valid_up_to(); + out.extend_from_slice(&rest[..valid_up_to]); + match error.error_len() { + // The input ends inside a character: wait for the rest. + None => { + self.pending = rest[valid_up_to..].to_vec(); + return out; + } + Some(len) => { + out.extend_from_slice(REPLACEMENT_CHARACTER); + rest.drain(..valid_up_to + len); + } + } + } + } + } + } + + /// What is carried at the end of the stream, replaced lossily. + fn finish(&mut self) -> Vec { + let pending = std::mem::take(&mut self.pending); + String::from_utf8_lossy(&pending).into_owned().into_bytes() + } +} + +/// Write `bytes` to the forwarding sink and drop the result: a sink that has +/// gone away must not cost the request its answer. SIGPIPE is ignored for the +/// write, so a closed pipe comes back as the error that is then discarded +/// rather than as a signal that kills the process. +fn forward_best_effort(sink: &mut impl Write, bytes: &[u8]) { + let _ = with_sigpipe_ignored(|| sink.write_all(bytes)); +} + +/// Turn an engine's answer into the response type the caller asked for: the +/// engine's own error when it reported one, otherwise the decoded `data`. +pub fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result +where + R: DeserializeOwned, +{ + if !envelope.ok { + let error = envelope.error.map_or_else( + || "unknown engine error".to_owned(), + |value| format!("{}: {}", value.code, value.message), + ); + bail!("{error}"); + } + let data = envelope + .data + .context("engine response envelope did not contain data")?; + serde_json::from_value(data).context("failed to decode engine response payload") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EngineErrorDetail, StopResponse}; + use serde_json::{Value, json}; + use std::collections::BTreeMap; + + /// The environment `command` would add to its child, as plain strings. + fn added_env(command: &Command) -> BTreeMap { + command + .get_envs() + .filter_map(|(key, value)| { + Some(( + key.to_string_lossy().into_owned(), + value?.to_string_lossy().into_owned(), + )) + }) + .collect() + } + + fn args_of(command: &Command) -> Vec { + command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect() + } + + #[test] + fn the_default_options_add_nothing_to_the_childs_environment() { + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions::default(), + ); + assert_eq!(command.get_program(), "/an/engine"); + assert_eq!(args_of(&command), ["stdio"]); + assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); + } + + #[test] + fn each_option_sets_exactly_its_own_variable() { + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + let command = engine_stdio_command( + Path::new("/an/engine"), + ["__engine-stdio", "vllm"], + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), + request_progress: true, + forward_stderr: true, + note_failing_exit_after_answer: false, + }, + ); + assert_eq!(args_of(&command), ["__engine-stdio", "vllm"]); + assert_eq!( + added_env(&command), + BTreeMap::from( + [ + ("ROCM_CLI_CONFIG_DIR", "/c"), + ("ROCM_CLI_DATA_DIR", "/d"), + ("ROCM_CLI_CACHE_DIR", "/k"), + ("ROCM_CLI_ENGINE_ENVS_ROOT", "/envs"), + ("ROCM_SERVE_API_KEY_FILE", "/keys/svc.endpoint-key"), + ("ROCM_ENGINE_PROGRESS_STDERR", "1"), + ] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ) + ); + + for (name, options) in [ + ( + "ROCM_CLI_ENGINE_ENVS_ROOT", + EngineStdioOptions { + env_root: Some(Path::new("/envs")), + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_SERVE_API_KEY_FILE", + EngineStdioOptions { + endpoint_key_file: Some(Path::new("/keys/k")), + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_ENGINE_PROGRESS_STDERR", + EngineStdioOptions { + request_progress: true, + ..EngineStdioOptions::default() + }, + ), + ] { + let command = engine_stdio_command(Path::new("/an/engine"), ["stdio"], &options); + let env = added_env(&command); + assert_eq!( + env.keys().map(String::as_str).collect::>(), + [name], + "an option must set only its own variable" + ); + } + } + + #[test] + fn the_variable_names_are_the_ones_the_engines_read() { + assert_eq!(ENGINE_ENVS_ROOT_ENV, "ROCM_CLI_ENGINE_ENVS_ROOT"); + assert_eq!(ENDPOINT_API_KEY_FILE_ENV, "ROCM_SERVE_API_KEY_FILE"); + assert_eq!(ENGINE_PROGRESS_STDERR_ENV, "ROCM_ENGINE_PROGRESS_STDERR"); + } + + #[test] + fn the_request_envelope_is_the_method_and_the_encoded_payload() { + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap(); + assert_eq!(envelope.method, EngineMethod::Stop); + assert_eq!(envelope.payload, json!({"service_id": "svc"})); + assert_eq!( + serde_json::to_string(&envelope).unwrap(), + r#"{"method":"stop","payload":{"service_id":"svc"}}"# + ); + } + + #[test] + fn decode_engine_response_reports_each_way_an_answer_can_be_unusable() { + let failed = EngineResponseEnvelope { + ok: false, + data: None, + error: Some(EngineErrorDetail { + code: "stop_failed".to_owned(), + message: "it broke".to_owned(), + }), + }; + let error = decode_engine_response::(failed).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let unexplained = EngineResponseEnvelope { + ok: false, + data: None, + error: None, + }; + let error = decode_engine_response::(unexplained).unwrap_err(); + assert_eq!(error.to_string(), "unknown engine error"); + + let empty = EngineResponseEnvelope { + ok: true, + data: None, + error: None, + }; + let error = decode_engine_response::(empty).unwrap_err(); + assert_eq!( + error.to_string(), + "engine response envelope did not contain data" + ); + + let wrong_shape = EngineResponseEnvelope::success(json!({"stopped": "yes"})); + let error = decode_engine_response::(wrong_shape).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to decode engine response payload" + ); + } + + /// A source fed by the test, one chunk per message, that says when it is + /// dropped (the collecting thread has ended). + struct Fed { + chunks: mpsc::Receiver>, + dropped: mpsc::Sender<()>, + } + impl Read for Fed { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.chunks.recv() { + Ok(chunk) => { + buffer[..chunk.len()].copy_from_slice(&chunk); + Ok(chunk.len()) + } + Err(_) => Ok(0), + } + } + } + impl Drop for Fed { + fn drop(&mut self) { + let _ = self.dropped.send(()); + } + } + + /// After a request has failed and its collector is gone, a descendant that + /// still writes to the pipe must not make the daemon grow: what was retained + /// is released at once, and what arrives later is drained and discarded (and + /// not forwarded), in chunks of a fixed size. The test keeps a handle on the + /// shared buffer to see that. + #[test] + fn an_abandoned_collector_releases_what_it_held_and_discards_what_arrives_later() { + let (feed, chunks) = mpsc::channel(); + let (dropped, ended) = mpsc::channel(); + let forwarded = Arc::new(Mutex::new(Vec::new())); + let sink = { + let forwarded = Arc::clone(&forwarded); + FnSink(move |bytes: &[u8]| { + forwarded.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + }) + }; + let collector = StderrCollector::start(Fed { chunks, dropped }, Some(sink)); + let shared = Arc::clone(&collector.shared); + + feed.send(b"before the failure\n".to_vec()).unwrap(); + // Wait for the thread to have published it, not for the stderr to end. + let deadline = Instant::now() + Duration::from_secs(10); + while lock(&shared).bytes.is_empty() { + assert!(Instant::now() < deadline, "the chunk was never published"); + thread::sleep(Duration::from_millis(2)); + } + assert_eq!(lock(&shared).bytes, b"before the failure\n"); + + drop(collector); + assert!(lock(&shared).bytes.is_empty(), "what was held is released"); + assert!(lock(&shared).abandoned); + + for _ in 0..50 { + feed.send(vec![b'x'; 4096]).unwrap(); + } + drop(feed); + ended + .recv_timeout(Duration::from_secs(10)) + .expect("the thread kept draining until the pipe closed"); + let retained = lock(&shared).bytes.len(); + assert_eq!( + retained, 0, + "bytes retained after the capture was abandoned" + ); + assert_eq!( + forwarded.lock().unwrap().as_slice(), + b"before the failure\n", + "nothing is forwarded after the request has ended" + ); + } + + /// The multi-byte text the tests split: two, three and four byte characters. + const MULTIBYTE: &str = "caf\u{e9} \u{2192} \u{1f4e6} done\n"; + + /// A stream split at every possible boundary is forwarded whole: every chunk + /// handed to the sink is valid UTF-8 on its own, and together they are the + /// input. Pure, so it runs on the Windows job too. + #[test] + fn text_split_across_chunks_is_forwarded_whole_and_valid_at_every_boundary() { + let bytes = MULTIBYTE.as_bytes(); + for first in 1..bytes.len() { + for second in first..bytes.len() { + let mut carry = Utf8Carry::default(); + let mut forwarded = Vec::new(); + for part in [&bytes[..first], &bytes[first..second], &bytes[second..]] { + let out = carry.push(part); + assert!( + std::str::from_utf8(&out).is_ok(), + "split {first}/{second}: a chunk was not valid UTF-8: {out:?}" + ); + forwarded.extend(out); + } + forwarded.extend(carry.finish()); + assert_eq!(forwarded, bytes, "split {first}/{second}: bytes were lost"); + } + } + } + + /// Bytes that can never be text are replaced, and a character cut off by the + /// end of the stream is flushed lossily, never dropped. + #[test] + fn bytes_that_are_not_text_are_replaced_and_a_cut_off_tail_is_flushed() { + let mut carry = Utf8Carry::default(); + assert_eq!( + carry.push(b"ok \xff\xfe done"), + "ok \u{fffd}\u{fffd} done".as_bytes() + ); + // A lead byte of a three-byte character with nothing after it. + assert_eq!(carry.push(b"tail \xe2\x86"), b"tail "); + assert_eq!(carry.finish(), "\u{fffd}".as_bytes()); + assert!(carry.finish().is_empty()); + } + + /// The collector forwards a character split across reads in one piece, and + /// keeps the raw bytes for the error. The source hands out one byte at a time. + #[test] + fn the_collector_forwards_a_split_character_whole_and_keeps_the_raw_bytes() { + struct OneByte(std::vec::IntoIter); + impl Read for OneByte { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.0.next() { + Some(byte) => { + buffer[0] = byte; + Ok(1) + } + None => Ok(0), + } + } + } + let chunks = std::cell::RefCell::new(Vec::>::new()); + let sink = FnSink(|bytes: &[u8]| { + chunks.borrow_mut().push(bytes.to_vec()); + Ok(bytes.len()) + }); + let collected = collect_stderr( + OneByte(MULTIBYTE.as_bytes().to_vec().into_iter()), + Some(sink), + ); + assert_eq!(collected, MULTIBYTE.as_bytes()); + let chunks = chunks.into_inner(); + assert!( + chunks + .iter() + .all(|chunk| std::str::from_utf8(chunk).is_ok()) + ); + assert_eq!(chunks.concat(), MULTIBYTE.as_bytes()); + } + + /// A status that is neither an exit nor a signal (a stopped process, on + /// Unix) has neither a code nor a signal to name, and is reported as that. + #[cfg(unix)] + #[test] + fn a_status_with_no_code_and_no_signal_gets_the_neutral_wording() { + use std::os::unix::process::ExitStatusExt as _; + // Stopped by SIGSTOP: WIFSTOPPED, which is neither WIFEXITED nor WIFSIGNALED. + let stopped = ExitStatus::from_raw(0x137f); + assert_eq!(stopped.code(), None); + assert_eq!(stopped.signal(), None); + assert_eq!( + exit_note(stopped), + "engine stdio process ended without an exit code" + ); + } + + /// A source that does not answer until released, then answers once and ends. + struct Gated { + release: mpsc::Receiver<()>, + released: bool, + } + impl Read for Gated { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.released { + return Ok(0); + } + self.release.recv().unwrap(); + self.released = true; + buffer[..5].copy_from_slice(b"late\n"); + Ok(5) + } + } + + /// A collector whose stderr is still open at the deadline reports what had + /// arrived and is not waited for; the reading thread is left alone and the + /// rest is there once the stderr does end. + #[test] + fn a_collector_stops_waiting_at_its_deadline_and_keeps_collecting() { + let (release, released) = mpsc::channel(); + let collector = StderrCollector::start( + Gated { + release: released, + released: false, + }, + None::, + ); + let started = Instant::now(); + let early = collector.finish(Some(Instant::now() + Duration::from_millis(50))); + assert!(early.is_empty()); + assert!(started.elapsed() < Duration::from_secs(5)); + + release.send(()).unwrap(); + assert_eq!(collector.finish(None), b"late\n"); + } + + /// An expired deadline returns at once with what is there, however much is + /// queued behind it. + #[test] + fn an_expired_deadline_returns_what_is_collected_without_waiting() { + let (release, released) = mpsc::channel(); + let collector = StderrCollector::start( + Gated { + release: released, + released: false, + }, + None::, + ); + let started = Instant::now(); + assert!(collector.finish(Some(Instant::now())).is_empty()); + assert!(started.elapsed() < Duration::from_secs(1)); + drop(release); + } + + /// A stderr whose reads fail. + struct Fails; + impl Read for Fails { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("pipe went away")) + } + } + + /// A sink that hands every write to a closure: one that refuses them is a + /// closed terminal, one that records them shows what was forwarded. + struct FnSink(F); + impl io::Result> Write for FnSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + (self.0)(bytes) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn a_closure_sink_writes_through_and_flushes_nothing() { + let mut sink = FnSink(|bytes: &[u8]| Ok(bytes.len())); + assert_eq!(sink.write(b"abc").unwrap(), 3); + sink.flush().unwrap(); + } + + #[test] + fn stderr_is_collected_whole_and_a_read_error_is_recorded() { + let collected = collect_stderr(&b"first\nsecond"[..], None::>); + assert_eq!(collected, b"first\nsecond"); + + let collected = String::from_utf8(collect_stderr(Fails, None::>)).unwrap(); + assert_eq!( + collected, + "failed to read engine progress: pipe went away\n" + ); + } + + #[test] + fn forwarded_stderr_is_written_out_and_still_collected() { + let mut shown = Vec::new(); + let collected = collect_stderr(&b"step 1\nstep 2\n"[..], Some(&mut shown)); + assert_eq!(shown, b"step 1\nstep 2\n"); + assert_eq!(collected, b"step 1\nstep 2\n"); + + // A sink that refuses writes (a closed terminal) costs nothing. + let collected = collect_stderr( + &b"still kept\n"[..], + Some(FnSink(|_: &[u8]| { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + })), + ); + assert_eq!(collected, b"still kept\n"); + } + + #[cfg(unix)] + mod child { + use super::*; + use crate::fake_engine::FakeEngine; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn stop_request() -> EngineRequestEnvelope { + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap() + } + + fn run(engine: &FakeEngine, options: &EngineStdioOptions<'_>) -> Result { + decode_engine_response(run_engine_stdio( + engine.path(), + ["stdio"], + options, + &stop_request(), + )?) + } + + #[test] + fn a_request_is_one_newline_terminated_line_on_the_childs_stdin() { + let engine = FakeEngine::answering("client-framing", STOPPED, 0); + let answer = run(&engine, &EngineStdioOptions::default()).unwrap(); + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn a_response_with_an_apostrophe_reaches_the_caller_intact() { + // The fake writes its answer through a shell literal; an apostrophe + // in valid JSON must not end that literal early. + let engine = FakeEngine::answering( + "client-apostrophe", + r#"{"ok":false,"error":{"code":"stop_failed","message":"can't stop"}}"#, + 0, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: can't stop"); + } + + #[test] + fn the_options_reach_the_running_child() { + let engine = FakeEngine::answering("client-options", STOPPED, 0); + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + run( + &engine, + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/k")), + request_progress: true, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ) + .unwrap(); + assert_eq!(engine.env("ROCM_CLI_DATA_DIR").as_deref(), Some("/d")); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/envs") + ); + assert_eq!( + engine.env("ROCM_SERVE_API_KEY_FILE").as_deref(), + Some("/keys/k") + ); + assert_eq!( + engine.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + } + + #[test] + fn a_nonzero_exit_with_no_answer_reports_the_status_and_stderr() { + let engine = FakeEngine::new("client-crash", true, "echo boom >&2\nexit 3"); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with code 3: boom" + ); + + let silent = FakeEngine::new("client-silent-crash", true, "exit 4"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "engine stdio process exited with code 4"); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let engine = FakeEngine::answering( + "client-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("client-answers-then-exits-2", STOPPED, 2); + assert_eq!( + run(&engine, &EngineStdioOptions::default()).unwrap(), + json!({"stopped": true}) + ); + } + + /// The status of a child that answered and then failed is not part of the + /// answer, so a caller that asks for it (the daemon) gets a note on this + /// process's stderr, and one that does not (the CLI) gets nothing. Run in + /// a re-executed test binary so the parent can read that stderr; the + /// decision itself is unchanged: the answer is used. + #[test] + fn a_failing_exit_after_an_answer_is_noted_only_when_asked_for() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_NOTE_ARM") { + let engine = FakeEngine::answering("client-note", STOPPED, 7); + let options = EngineStdioOptions { + note_failing_exit_after_answer: arm == "on", + ..EngineStdioOptions::default() + }; + let answer = run(&engine, &options).unwrap(); + assert_eq!(answer, json!({"stopped": true})); + return; + } + for (arm, expected) in [("on", true), ("off", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::a_failing_exit_after_an_answer_is_noted_only_when_asked_for", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + let note = "engine stdio process exited with code 7 after printing its answer; the answer was used"; + assert_eq!(stderr.contains(note), expected, "{arm}:\n{stderr}"); + } + } + + /// The note is only made for an answer that parsed: a child that prints + /// something unreadable and fails is reported by the error (with its + /// status), and must not also log that "the answer was used". Both cases + /// assert the outcome and the stderr together, in a re-executed binary. + #[test] + fn the_note_is_made_only_for_an_answer_that_parsed() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_NOTE_PARSE_ARM") { + let (body, status) = if arm == "parsed" { + (format!("printf '%s' '{STOPPED}'"), 7) + } else { + ("printf 'not json'".to_owned(), 7) + }; + let engine = + FakeEngine::new("client-note-parse", true, &format!("{body}\nexit {status}")); + let options = EngineStdioOptions { + note_failing_exit_after_answer: true, + ..EngineStdioOptions::default() + }; + let outcome = run(&engine, &options); + if arm == "parsed" { + assert_eq!(outcome.unwrap(), json!({"stopped": true})); + } else { + assert_eq!( + outcome.unwrap_err().to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with code 7)" + ); + } + return; + } + for (arm, expected) in [("parsed", true), ("unparseable", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::the_note_is_made_only_for_an_answer_that_parsed", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_PARSE_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + stderr.contains("the answer was used"), + expected, + "{arm}:\n{stderr}" + ); + } + } + + /// With stderr a closed pipe and SIGPIPE at its default action (how `rocm` + /// runs, and the daemon with it), the note must not end the process or + /// panic: the answer is still returned. Re-executes this test in a process + /// whose stderr is a pipe with no reader. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process() { + if std::env::var_os("ROCM_TEST_CLIENT_NOTE_CLOSED_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let engine = FakeEngine::answering("client-note-closed", STOPPED, 7); + let options = EngineStdioOptions { + note_failing_exit_after_answer: true, + ..EngineStdioOptions::default() + }; + let answer = run(&engine, &options).expect("the answer is still returned"); + assert_eq!(answer, json!({"stopped": true})); + return; + } + let (reader, writer) = io::pipe().unwrap(); + drop(reader); + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_CLOSED_ARM", "1") + .stderr(Stdio::from(writer)) + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE ended the process: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + + /// A child whose stdin or stdout pipe is missing is killed and reaped, as + /// on every other failure path, not left running. Linux-only for the + /// `/proc` entry a zombie would keep. + #[cfg(target_os = "linux")] + #[test] + fn a_missing_pipe_kills_and_reaps_the_child() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let proc_entry = format!("/proc/{}", child.id()); + assert!(Path::new(&proc_entry).exists()); + + let error = + require_pipe(&mut child, None::, "stdin").unwrap_err(); + + assert_eq!(error.to_string(), "engine stdio child did not expose stdin"); + assert!(!Path::new(&proc_entry).exists(), "killed and reaped"); + } + + /// A pipe that is there is handed back and the child left alone. + #[test] + fn a_present_pipe_is_returned_and_the_child_left_running() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + assert_eq!(require_pipe(&mut child, Some(5), "stdout").unwrap(), 5); + assert!(child.try_wait().unwrap().is_none()); + child.kill().unwrap(); + child.wait().unwrap(); + } + + /// A failed read of the answer reports what the engine had said on + /// stderr, with the read error, like every other failure; the message and + /// its content are asserted together. + #[test] + fn a_failed_read_of_the_answer_reports_the_stderr_too() { + let collector = StderrCollector::start( + io::Cursor::new(b"engine crashed: out of memory\n".to_vec()), + None::, + ); + // The thread ends at once on a cursor; wait for its text. + let error = read_failure_with_stderr( + anyhow::Error::new(io::Error::other("pipe went away")) + .context("failed to read engine stdio response"), + Some(collector), + Duration::from_secs(5), + ); + assert_eq!( + error.to_string(), + "failed to read engine stdio response: pipe went away; stderr: engine crashed: out of memory" + ); + + // With nothing on stderr the error is the read error, unchanged. + let quiet = read_failure_with_stderr( + anyhow::anyhow!("failed to read engine stdio response"), + None, + Duration::from_secs(1), + ); + assert_eq!(quiet.to_string(), "failed to read engine stdio response"); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let engine = FakeEngine::new( + "client-garbage", + true, + "printf 'not json'\necho oops >&2\nexit 0", + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + + let silent = FakeEngine::new("client-garbage-quiet", true, "printf 'not json'"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + + /// A child ended by a signal has no exit code; it is reported by the signal, + /// with or without a half-written answer, and never as a code of its own + /// invention. + #[test] + fn a_child_ended_by_a_signal_is_reported_by_the_signal() { + let killed = FakeEngine::new("client-killed", true, "kill -KILL $$"); + let error = run(&killed, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process was terminated by signal 9" + ); + + let cut_short = FakeEngine::new( + "client-killed-mid-answer", + true, + "printf '{\"ok\"'\nkill -KILL $$", + ); + let error = run(&cut_short, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process was terminated by signal 9)" + ); + } + + /// A child that fails and also leaves something that is not an envelope on + /// stdout is reported with its exit status (the status says the answer was + /// cut short, where the parse error alone would not), as a child that + /// printed nothing is. The same unreadable answer from a child that exited + /// cleanly carries no status, so the note is the condition's own and not + /// added to every parse failure. + #[test] + fn an_unparseable_answer_from_a_failing_child_names_the_exit_status() { + let crashed = FakeEngine::new( + "client-garbage-crash", + true, + "printf 'not json'\necho oops >&2\nexit 4", + ); + let error = run(&crashed, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with code 4); stderr: oops" + ); + + let silent = FakeEngine::new( + "client-garbage-crash-quiet", + true, + "printf 'not json'\nexit 5", + ); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with code 5)" + ); + + let clean = FakeEngine::new("client-garbage-clean", true, "printf 'not json'\nexit 0"); + let error = run(&clean, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + + /// Runs `work` on its own thread and fails the test if it has not finished + /// within `limit`, so a step that should not block turns red instead of + /// hanging the run. + fn finishes_within( + limit: Duration, + work: impl FnOnce() -> T + Send + 'static, + ) -> T { + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let _ = sender.send(work()); + }); + receiver + .recv_timeout(limit) + .unwrap_or_else(|_| panic!("did not finish within {limit:?}")) + } + + /// An engine that fills its stderr before it has drained a request larger + /// than a pipe buffer does not deadlock the client: the stderr is read from + /// the start, so the engine's write completes while the request is still + /// being written. (With the collector started after the write, the engine + /// blocks on a full stderr pipe and the client on a full stdin pipe.) + #[test] + fn an_engine_that_writes_a_lot_of_stderr_before_reading_its_request_does_not_deadlock() { + let engine = FakeEngine::new( + "client-stderr-first", + false, + &format!( + "head -c 300000 /dev/zero | tr '\\0' x >&2\ncat > /dev/null\nprintf '%s' '{STOPPED}'" + ), + ); + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"blob": "y".repeat(1 << 20)})) + .unwrap(); + let program = engine.path().to_path_buf(); + let answer = finishes_within(Duration::from_secs(20), move || { + run_engine_stdio( + &program, + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + }) + .expect("the engine answered"); + assert!(answer.ok); + drop(engine); + } + + /// A request that cannot be written because the engine is gone reports + /// what the engine said on stderr, not only that the write failed: the + /// engine's own words are the explanation. The request is larger than a + /// pipe buffer, so the write reaches the closed read end. + #[test] + fn a_request_to_an_engine_that_exited_reports_what_it_said_on_stderr() { + let engine = FakeEngine::new("client-no-listen", false, "echo 'no thanks' >&2\nexit 3"); + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"blob": "x".repeat(1 << 20)})) + .unwrap(); + let error = run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .expect_err("the engine never read its request"); + let message = error.to_string(); + assert!( + message.starts_with("failed to write engine request: "), + "{message}" + ); + assert!(message.ends_with("; stderr: no thanks"), "{message}"); + } + + /// The child of a failed write is killed and reaped when it is still + /// running, which a failed write does not show it to have stopped. + /// Linux-only for the `/proc` entry, which a zombie keeps until waited on. + #[cfg(target_os = "linux")] + #[test] + fn a_request_that_cannot_be_written_kills_and_reaps_a_child_that_is_still_running() { + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let proc_entry = format!("/proc/{}", child.id()); + assert!(Path::new(&proc_entry).exists(), "the child is running"); + + let error = finishes_within(Duration::from_secs(10), move || { + let mut child = child; + let stderr = child + .stderr + .take() + .map(|pipe| StderrCollector::start(pipe, None::)); + abandon_unwritten_request( + child, + stderr, + &io::Error::from(io::ErrorKind::BrokenPipe), + Duration::from_secs(1), + ) + }); + + assert!( + error + .to_string() + .starts_with("failed to write engine request: "), + "{error}" + ); + assert!( + !error.to_string().contains("stderr"), + "a child that said nothing adds no stderr: {error}" + ); + assert!( + !Path::new(&proc_entry).exists(), + "the child must be killed and reaped, not left running or as a zombie" + ); + } + + /// A grandchild that holds the child's stderr open past its exit cannot + /// make the error path wait for it: what had been written is reported + /// and the rest is not waited for. The diagnostic has no trailing newline, + /// as an engine's last words often do not, and is still reported: the + /// collector publishes what has arrived, not only complete lines. + #[test] + fn the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace() { + let mut child = Command::new("sh") + .args(["-c", "printf said >&2; sleep 5 & exit 3"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + // The engine has said its piece and exited before the failure is + // handled, as it has in the case this bounds. + child.wait().unwrap(); + let started = Instant::now(); + + let stderr = child + .stderr + .take() + .map(|pipe| StderrCollector::start(pipe, None::)); + let error = finishes_within(Duration::from_secs(4), move || { + abandon_unwritten_request( + child, + stderr, + &io::Error::from(io::ErrorKind::BrokenPipe), + Duration::from_millis(300), + ) + }); + + let waited = started.elapsed(); + assert!( + waited < Duration::from_secs(4), + "waited for the grandchild: {waited:?}" + ); + assert!(error.to_string().ends_with("; stderr: said"), "{error}"); + } + + /// An answer whose read fails kills the child and reaps it, so a daemon + /// that hits this repeatedly collects neither a running engine nor a + /// zombie per failure. A zombie keeps its `/proc` entry until it is + /// waited on, so the entry being gone shows the reap and not only the + /// kill (hence Linux only). + #[cfg(target_os = "linux")] + #[test] + fn a_failed_read_of_the_answer_kills_and_reaps_the_child() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + let proc_entry = format!("/proc/{pid}"); + assert!(Path::new(&proc_entry).exists(), "the child is running"); + + let error = read_response(&mut child, Fails).unwrap_err(); + + assert_eq!(error.to_string(), "failed to read engine stdio response"); + assert_eq!( + format!("{error:#}"), + "failed to read engine stdio response: pipe went away" + ); + assert!( + !Path::new(&proc_entry).exists(), + "the child must be killed and reaped, not left running or as a zombie" + ); + // The status was collected by the reap, so it is still reported. + assert_eq!( + child.try_wait().unwrap().and_then(|status| status.signal()), + Some(libc::SIGKILL) + ); + } + + /// The success path of the same helper hands the bytes back and leaves + /// the child alone. + #[test] + fn a_successful_read_of_the_answer_leaves_the_child_to_the_caller() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + + let bytes = read_response(&mut child, &b"{\"ok\":true}"[..]).unwrap(); + + assert_eq!(bytes, b"{\"ok\":true}"); + assert!(child.try_wait().unwrap().is_none(), "still running"); + child.kill().unwrap(); + child.wait().unwrap(); + } + + #[test] + fn a_program_that_cannot_start_is_named_in_the_error() { + let error = run_engine_stdio( + Path::new("/no/such/engine"), + ["stdio"], + &EngineStdioOptions::default(), + &stop_request(), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "failed to spawn engine stdio process /no/such/engine" + ); + } + + /// What the running child writes to its stderr reaches the forwarding + /// sink as it arrives. (That nothing is forwarded without the option is + /// the `off` arm of the test below.) + #[test] + fn a_childs_stderr_reaches_the_forwarding_sink_when_there_is_one() { + let progress = "downloading 1/2\ndownloading 2/2\n"; + let body = format!("printf '%s' '{progress}' >&2\nprintf '%s' '{STOPPED}'"); + let engine = FakeEngine::new("client-forward-sink", true, &body); + + let forwarded = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let sink = { + let forwarded = std::sync::Arc::clone(&forwarded); + FnSink(move |bytes: &[u8]| { + forwarded.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + }) + }; + let answer = run_engine_stdio_forwarding( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &stop_request(), + Some(sink), + ) + .unwrap(); + assert!(answer.ok); + assert_eq!(forwarded.lock().unwrap().as_slice(), progress.as_bytes()); + } + + /// The public entry point sends the child's stderr to THIS process's + /// stderr when `forward_stderr` is set, and does not when it is not. That + /// is the one place the option takes effect, and it is what shows a user + /// an engine install's progress. The test binary re-executes itself so the + /// parent can read the stderr the arm writes. + #[test] + fn forward_stderr_sends_the_childs_stderr_to_this_processs_stderr() { + const LINE: &str = "engine progress line for the terminal"; + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_FORWARD_OPTION_ARM") { + let engine = FakeEngine::new( + "client-forward-option", + true, + &format!("echo '{LINE}' >&2\nprintf '%s' '{STOPPED}'"), + ); + let options = EngineStdioOptions { + forward_stderr: arm == "on", + ..EngineStdioOptions::default() + }; + run_engine_stdio(engine.path(), ["stdio"], &options, &stop_request()).unwrap(); + return; + } + + for (arm, expected) in [("on", true), ("off", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forward_stderr_sends_the_childs_stderr_to_this_processs_stderr", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_FORWARD_OPTION_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + stderr.contains(LINE), + expected, + "forward_stderr {arm}: the child's stderr on this process's stderr:\n{stderr}" + ); + } + } + + /// Progress forwarded to a stderr whose reader has gone away is dropped, not + /// fatal, also where SIGPIPE is at its default action (`rocm` runs that + /// way). The `Closed` sink above cannot show this: it returns the error + /// itself, where the real failure is a signal that ends the process before + /// any error is returned. So this re-executes itself in a fresh process + /// that installs `SIG_DFL` and forwards into one end of a real pipe whose + /// other end is closed. Two arms, one per forwarding write in + /// `collect_stderr`: a line of the child's stderr, and the note recorded + /// when reading that stderr fails. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_FORWARD_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let (reader, writer) = io::pipe().unwrap(); + drop(reader); + let collected = if arm == "line" { + collect_stderr(&b"progress\n"[..], Some(writer)) + } else { + collect_stderr(Fails, Some(writer)) + }; + // Surviving is the point; the text is still kept for the error. + assert!(!collected.is_empty()); + return; + } + + for arm in ["line", "read-error"] { + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_FORWARD_ARM", arm) + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "{arm}: SIGPIPE killed the process instead of the write failing: {status:?}" + ); + assert!(status.success(), "{arm}: {status:?}"); + } + } + + /// Whether SIGPIPE is ignored right now, read by installing the same + /// disposition back. + #[allow(unsafe_code)] // libc::signal + fn sigpipe_is_ignored() -> bool { + // SAFETY: reading/installing a signal disposition is always safe. + let current = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + // SAFETY: putting back what was just read. + unsafe { + libc::signal(libc::SIGPIPE, current); + } + current == libc::SIG_IGN + } + + /// Forwarding guards each write on its own and never holds SIGPIPE ignored + /// across the reads between them. The reads block for as long as the engine + /// is quiet (an install runs for minutes), and the disposition is + /// process-wide, so a guard around the whole loop would leave every other + /// thread's write to a closed stdout returning an error for that whole time + /// instead of ending `rocm` quietly. Each line costs one pair of + /// `signal` calls, which is nothing next to the write itself. A fresh + /// process with `SIG_DFL` records the disposition seen inside every read + /// and every write, and after the loop. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn forwarding_guards_each_write_and_never_the_reads_between_them() { + use std::cell::RefCell; + use std::rc::Rc; + + type Seen = Rc>>; + + struct Source { + lines: Vec<&'static [u8]>, + seen: Seen, + } + impl Read for Source { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + self.seen.borrow_mut().push(("read", sigpipe_is_ignored())); + if self.lines.is_empty() { + return Ok(0); + } + let line = self.lines.remove(0); + buffer[..line.len()].copy_from_slice(line); + Ok(line.len()) + } + } + + if std::env::var_os("ROCM_TEST_CLIENT_GUARD_SCOPE_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let seen: Seen = Rc::default(); + let source = Source { + lines: vec![b"one\n", b"two\n", b"three\n"], + seen: Rc::clone(&seen), + }; + let sink_seen = Rc::clone(&seen); + let sink = FnSink(move |bytes: &[u8]| { + sink_seen.borrow_mut().push(("write", sigpipe_is_ignored())); + Ok(bytes.len()) + }); + collect_stderr(source, Some(sink)); + let seen = seen.borrow(); + let writes = seen.iter().filter(|(what, _)| *what == "write").count(); + assert_eq!(writes, 3, "{seen:?}"); + for (what, ignored) in seen.iter() { + assert_eq!( + *ignored, + *what == "write", + "SIGPIPE must be ignored inside each write and only there: {seen:?}" + ); + } + assert!( + !sigpipe_is_ignored(), + "the default action is back after the loop" + ); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forwarding_guards_each_write_and_never_the_reads_between_them", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_GUARD_SCOPE_ARM", "1") + .status() + .unwrap(); + assert!(status.success(), "{status:?}"); + } + + /// A request written to an engine that has already exited comes back as an + /// ordinary error, also where SIGPIPE is at its default action. Re-executes + /// this test in a fresh process that installs `SIG_DFL` first, because the + /// harness leaves SIGPIPE ignored and changing it process-wide would race + /// sibling tests; a write that is not guarded kills that process, and the + /// parent reports the signal. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_CLIENT_SIGPIPE_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let engine = FakeEngine::new("client-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let envelope = engine_request_envelope( + EngineMethod::Stop, + &json!({"blob": "x".repeat(1 << 20)}), + ) + .unwrap(); + let error = run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 4e88c8dddc..bdf38fdcb2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,8 +59,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) — full domain extraction, a child of `driver_plan.rs` that depends on the plan model, `driver_command`, `driver_root_preflight_checks` and `resolve_shell_default_template` in its parent, and on `truthy_env` in [`therock.rs`](../apps/rocm/src/therock.rs) (split out of [`driver_install.rs`](../apps/rocm/src/driver_install.rs)): the WSL2 rocDXG plan. `wsl_rocdxg_driver_plan` builds the plan that installs the `rocdxg-roct` package, `resolve_rocdxg_verification` decides how that download is authenticated (a supplied digest, the digest pinned for the version in `ROCDXG_PINNED_DIGESTS`, or the explicit opt-out), and `wsl_rocdxg_refusal_plan` is the plan that installs nothing and says why when it cannot be authenticated or the version is not a well-formed package version. `ROCDXG_PINNED_DIGESTS` is the trust anchor an unattended root install is checked against, so a change to it is a security-relevant diff; it moved byte for byte. `wsl_rocdxg_driver_plan` is `pub(crate)` and re-exported by `driver_plan.rs` (its builders call it, and tests in `driver_install.rs` use it as a plan fixture); everything else in production code is private. Its tests are the `wsl_rocdxg_*` plan tests, including the one that runs the generated digest step against a real file on Unix. The test-only `scoped_rocdxg_env` helper that clears the rocDXG environment lives here too, `pub(super)` for the plan-builder tests in `driver_plan.rs`. - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. - [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. -- [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the stdio request/response envelope used to talk to an external engine binary or the in-process built-in handler (`engine_request`, `engine_request_with_env_root`, `decode_engine_response`, the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `decode_engine_response`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. +- [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the request entry points, which either answer from the in-process built-in handler or run the resolved engine binary with `stdio` (`engine_request`, `engine_request_with_env_root`, and `engine_stdio_child_request`, which hands the child to the stdio client `rocm` shares with `rocmd`, from [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal), the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_var_values`/`app_path_env_var_refs`, over [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs)), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. - [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. Its tests do not call into `model_catalog_text.rs`: the test of the note the verbose `rocm model` listing prints for a rule-out engine lives beside that renderer. @@ -156,7 +156,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal; with no watcher enabled it exits by itself, deciding that under the autostart lock and publishing "stopped" and clearing its spawn claim before releasing it — `record_daemon_stop`, `release_own_autostart_claim`) and `rocmd status` (`print_status`), with the watcher tick interval, which `LoopHooks` makes injectable for tests. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. - [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs) — which binary a daemon spawn runs. The daemon runs as the `rocmd` binary or embedded in `rocm` (`rocm daemon`, which enters through `run_from_args` and marks the process), and `supervise` is a command of `rocmd` alone. The hidden engine commands (`__engine-serve-http`, `__engine-stdio`) are `rocm`'s alone. `DaemonHost` holds the one executable path every choice and sibling lookup is made from (`rocm_core::sibling_binary_path_from`). `rocmd_binary` returns the current executable when the daemon is `rocmd` and the `rocmd` installed beside `rocm` when it is embedded; `rocm_binary` is the mirror, the current executable when embedded and the `rocm` installed beside `rocmd` when not. Each names what was missing when there is none. `detect` returns the real host. Under test it returns the test binary for both commands and looks for no sibling, and the two stand-in hosts (`test_binary_standing_in_for_rocm` and `test_binary_standing_in_for_rocmd`) name only the binary they stand in for and fail when asked for the other, so none of them can run a `rocm` or `rocmd` built beside the test binary; a test that needs another binary passes the code it exercises a host of its own. A leaf: it imports no other crate module. -- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process, run on the `rocm` binary that `DaemonHost::rocm_binary` finds, since `__engine-stdio` is `rocm`'s command), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate but [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs); [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process, run through the same client `rocm` uses, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs); `rocmd` chooses what to run, `__engine-stdio ` on the `rocm` binary that `DaemonHost::rocm_binary` finds, since `__engine-stdio` is `rocm`'s command, and what to ask of the client: its own app paths, the service's endpoint key file when it holds a usable key, and stderr kept for the error rather than forwarded), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate but [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs); [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. - [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. - [`main.rs`](../apps/rocmd/src/main.rs) — the `rocmd` binary's entry point: a synchronous `main` that calls `rocmd::run_bin_cli()`, which builds the Tokio runtime itself (`run_bin_cli` is `#[tokio::main]`) and dispatches the parsed `Cli`, so the binary holds no logic of its own. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). The argument handling the dispatch calls is in [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs). @@ -208,7 +208,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. - [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) — the `rocm diagnose --model` fit assessment: curated-recipe lookup against a host's measured GPU/RAM. `rocm model --verbose` renders its own fit labels in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which uses exactly two items from this module: `curated_alternatives`, called directly, and `recipe_display_ref`, reached through a private one-line wrapper of the same name in that file. For a GGUF outside the catalog it also attaches a context-size estimate (the three `gguf*` modules above). - [`net.rs`](../crates/rocm-core/src/net.rs) — (ROCMAI-154) networking primitives: the default local host and port, URL/endpoint formatting and parsing, and the hand-rolled loopback HTTP/TCP client (`http_get_text*`, `http_post_json_with_auth`, `read_http_response_bounded`). It depends on no other service module: the managed-service readiness probes built on it live in [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) and the streaming downloads in [`download.rs`](../crates/rocm-core/src/download.rs). -- [`paths.rs`](../crates/rocm-core/src/paths.rs) — (ROCMAI-154, Phase 7b) `AppPaths` and its filesystem-location primitives. +- [`paths.rs`](../crates/rocm-core/src/paths.rs) — `AppPaths` and its filesystem-location primitives, including `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, and the constants that name the variables (`CONFIG_DIR_ENV`, `DATA_DIR_ENV`, `CACHE_DIR_ENV`, `ENGINE_ENVS_ROOT_ENV`). - [`placeholder.rs`](../crates/rocm-core/src/placeholder.rs) — the `` placeholder rendered reports print for an empty value: `empty_as_unknown` (returns the value untouched) and `trimmed_or_unknown` (trims first), shared by `apps/rocm`'s driver-install plan and chat tool summaries and by [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs)'s Windows GPU diagnostics, replacing the copy those two `apps/rocm` callers shared in [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and `host_gpu.rs`'s own trimming copy. - [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) — the host probes' subprocess and file-read helpers, split out of [`examine.rs`](../crates/rocm-core/src/examine.rs) so `fix.rs`, `process.rs` and the child modules of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) reach them without importing the module that owns the `Examination` document: `run`/`run_with_env`/`run_raw` (run a command under a timeout; `rc` is `127` when it cannot be spawned and `124` on timeout), `run_utf16le`/`decode_utf16le` (for `wsl.exe`'s UTF-16LE output), `read_text`/`read_host_text`, `which` (the one `PATH` lookup, used by both host-probing stacks: it skips an empty `PATH` entry, searches a `PATH` that is not valid Unicode and follows `PATHEXT` on Windows) with `tool_path_candidates`, the spellings it tries and the capture runner in [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) starts in turn, the `SHORT`/`MEDIUM` timeouts and `truncate_to_chars`. All are `pub(crate)` except `run_raw`, which stays private to the module. The module itself is private to the crate and is not re-exported. - [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus `run_with_timeout`: it drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. It is not yet the *only* such helper — [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded`, [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `run_raw`, and [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs)'s `capture_optional_command_candidate_with_timeout` (plus the Windows-only `capture_diagnostic_command` in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs)) still join their reader threads with no time limit, so a grandchild holding the pipe can outlast their timeout; converging them onto `run_with_timeout` is follow-up work to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). @@ -249,6 +249,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). +- [`sigpipe.rs`](../crates/rocm-core/src/sigpipe.rs) — `with_sigpipe_ignored`, which turns a write to a pipe whose reader has gone into an ordinary error instead of a signal; `rocm` runs with SIGPIPE at its default action, so the engine stdio client wraps its write to the child, and each write forwarding the child's stderr to its own, in it. Guarded regions may overlap, across threads too; the disposition in force before the first is put back when the last ends. - [`terminal.rs`](../crates/rocm-core/src/terminal.rs) — one ECMA-48 walk over untrusted terminal output, shared by every caller that has to quote or re-render it (`serve_summary.rs`, `vllm/startup_diagnostics.rs`, `vllm_oom.rs`, `diagnose/engine.rs`). - [`user_path.rs`](../crates/rocm-core/src/user_path.rs) — removing the install directory from the per-user Windows `Path` that the installer added it to. - [`uv.rs`](../crates/rocm-core/src/uv.rs) — acquiring the `uv` package manager into the managed cache, plus the argument/environment helpers `apps/rocm` and the vLLM engine share. @@ -394,7 +395,9 @@ The other three dashboard crates each have a crate-level bullet, followed by a b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share. +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. It sets no timeout, and does not bound the wait for the child's pipes after it exits (a grandchild holding one makes the request wait); its module comment states that limit. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. +- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the protocol contract: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. ### `crates/rocm-deps` diff --git a/docs/testing.md b/docs/testing.md index df3e3936f8..f048f1b863 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -270,6 +270,170 @@ child write the file instead, so this process never holds a write fd on it. The same applies to E2E steps: scenarios run concurrently in one process, and the `rocm` child that runs the shim can be the one refused. +### The engine stdio client + +`rocm` and `rocmd` run an engine child through one client, +[`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) in +`rocm-engine-protocol`. Its conversation (the request line, the environment each +option sets, what a failing exit or an unusable answer is reported as, the +SIGPIPE guard) is tested there against `FakeEngine`, a shell-script engine child +from [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) that +records its argv, environment and stdin; the pure parts (the command a call +builds, `decode_engine_response`, stderr collection) run on every platform, and +the ones that start the fake are `#[cfg(unix)]`. The `rocm` and `rocmd` callers +each run their own entry point against the same fake and pin only what they +choose: the program and verb, the app paths, and whether stderr is forwarded (see +`apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). These tests +catch behavioural drift: a caller whose own copy of the conversation behaved +differently from the shared client in what they pin (the framing, the +environment, the exit and answer handling, the guards) would fail them. A copy +that behaves the same passes them, so a second guard is on the existence of a +second copy: +[engine_stdio_client_contract.rs](../xtask/src/engine_stdio_client_contract.rs) +is a static text scan over every `.rs` file in the tree (comments stripped) that +fails when a file other than the shared client both pipes a child's stdin and +names an engine request or response envelope, which is what an engine stdio +client is made of. Both are looked for anywhere in the file, not on one line, so +a copy that builds its `Stdio` in one place and passes it to `.stdin(..)` in +another is seen too (a fixture covers it); the price is that a file which pipes +some child's stdin for another reason and also names an envelope type is +reported, and the answer to that is a reviewed allowlist entry. A caller that reaches an engine through `run_engine_stdio` +has neither. Pasting the old rocmd copy back into `apps/rocmd/src/engine_ipc.rs` +fails it by name (and, separately, five of rocmd's pinning tests: the +app-paths, crash, unparseable-answer, nonzero-with-answer and SIGPIPE ones). Like +the other allowlists it is bidirectional, so an exemption cannot outlive its +reason. + +```bash +cargo test -p xtask engine_stdio_client_contract +``` + +It is a source-text scan, not a call-graph analysis: a copy that makes its pipe +by hand, or reaches the envelope types under another name, is outside what it +can see, and it does not check that a caller uses the shared client correctly +(the callers' tests do). It closes the cheap, likely regression of a +spawn-write-read helper pasted back into a binary. + +An answer that is not a response envelope is reported as that, and when the +child also exited with a failing status the error says so, because the status is +often why the answer was cut short (a clean exit adds nothing). A failing child +that printed nothing is reported by its status alone. The status is worded from +the exit code (`exited with code N`) or, for a child ended by a signal on Unix, +the signal (`was terminated by signal N`), not from `ExitStatus`'s own text, which +already reads `exit status: N` and would double up (`a_child_ended_by_a_signal_is_reported_by_the_signal`). Both carry the child's +stderr. `an_unparseable_answer_from_a_failing_child_names_the_exit_status` pins +the three cases against the fake, and `engine-ipc-02` below shows the same +message in a `stop_server` report next to what the stop did. + +The SIGPIPE tests re-execute the test binary in a fresh process with SIGPIPE at +its default action, which is how `rocm` runs, because the test harness leaves it +ignored and changing it process-wide would race sibling tests. A write that is +not guarded kills that process, and the parent test reports the signal. Two +writes are covered that way: the request to the child's stdin, and the forwarding +of the child's stderr to this process's stderr (a line of progress, and the note +recorded when reading it fails). The forwarding test writes into a real pipe whose +reader is closed, because a mock sink that returns `BrokenPipe` itself never +raises the signal. Each forwarding write is guarded on its own, never the loop +around it, because the reads between lines block for as long as the engine is +quiet and the disposition is process-wide; a third re-executed test, +`forwarding_guards_each_write_and_never_the_reads_between_them`, records the +disposition inside every read and every write and after the loop. `FakeEngine` quotes the response and its scratch directory +for the shell, so a response such as `can't stop`, or a `TMPDIR` containing an +apostrophe, reaches the client intact. + +The `forward_stderr` option, which is what shows a user an engine install's +progress, is tested at both ends: the client runs against a private entry point +that takes the sink (`a_childs_stderr_reaches_the_forwarding_sink_when_there_is_one`, +a `Vec`-backed writer), and the public entry point is run in a re-executed test +binary whose own stderr the parent reads +(`forward_stderr_sends_the_childs_stderr_to_this_processs_stderr`: the child's +line is there with the option and absent without it). Replacing the one +`forward_stderr.then(io::stderr)` with `None` turns the second red. + +A child that answers and then exits with a failing status is read as having +answered (see the doc comment of `run_engine_stdio`), and its status is not part +of the answer. The daemon asks the client to leave a note on its stderr, which is +its log (`note_failing_exit_after_answer`); the CLI does not, since its stderr is a +terminal. `a_failing_exit_after_an_answer_is_noted_only_when_asked_for` reads the +stderr of a re-executed test binary for both settings. The note is made only +for an answer that parsed (an unreadable answer with a failing exit is reported +by the error, with the status, and is not also logged as used), and it is written +through the same SIGPIPE-guarded, error-ignoring write as the stderr forwarding, +so a closed stderr neither ends a daemon hosted in `rocm` nor panics it +(`the_note_is_made_only_for_an_answer_that_parsed` and +`the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process`, the latter +with stderr a pipe without a reader under SIGPIPE's default action). A failed read of the +answer carries the child's stderr like every other failure +(`a_failed_read_of_the_answer_reports_the_stderr_too`), and a child whose stdin or +stdout pipe is missing is killed and reaped, not left running +(`a_missing_pipe_kills_and_reaps_the_child`, Linux only). + +What is forwarded to the terminal is complete UTF-8: stderr is read in chunks +that can end in the middle of a multi-byte character, and Rust's stdio on a +Windows console refuses a write that is not valid UTF-8, an error the best-effort +forwarding discards, so a split character would silently lose output. The +collector carries an incomplete trailing sequence to the next chunk (invalid +bytes become U+FFFD, and a character cut off by the end of the stream is flushed +lossily) while still publishing the raw bytes at once for the error. The logic is +the pure `Utf8Carry`, tested on every platform by +`text_split_across_chunks_is_forwarded_whole_and_valid_at_every_boundary` (every +two-point split of a string with two-, three- and four-byte characters), +`bytes_that_are_not_text_are_replaced_and_a_cut_off_tail_is_flushed` and +`the_collector_forwards_a_split_character_whole_and_keeps_the_raw_bytes` (a source +that hands out one byte at a time). + +A child whose answer cannot be read is killed and reaped before the error is +returned, as it is when the request cannot be written, so the long-lived daemon +does not collect a running engine or a zombie per failure. +`a_failed_read_of_the_answer_kills_and_reaps_the_child` (Linux only) runs the +read step on a real child with a reader that fails and checks that the child's +`/proc` entry is gone: a killed child that was never waited on would keep it as a +zombie. + +A request that cannot be written (the engine exited, or closed its stdin, before +reading it) is reported with what the engine wrote to stderr, since that is +usually the explanation, after the child has been killed and reaped. The child is +killed whether or not a `try_wait` would show it gone, because its pipe closes +while it is still exiting; the read of its stderr is bounded (a grandchild that +holds the pipe cannot hold the error path). `a_request_to_an_engine_that_exited_reports_what_it_said_on_stderr` +runs the whole client against a fake that exits without reading; +`a_request_that_cannot_be_written_kills_and_reaps_a_child_that_is_still_running` +(Linux only) checks the `/proc` entry of one that is still running, and +`the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` one whose +grandchild keeps the pipe open. The last two run the step on a real child with an +injected write error, because whether a real write fails before the child has +closed its stdin is a race. The answer of an engine that exits without draining a request larger than a pipe buffer is not read on this path (every engine here reads stdin to the end before answering, so that needs one that breaks the protocol); the module comment in `stdio_client.rs` states the limit. The stderr of a failed +write is collected by a thread that starts before the request is written and is +waited for with one deadline (`StderrCollector::finish`), returning what had +arrived when it passes; `a_collector_stops_waiting_at_its_deadline_and_keeps_collecting` +and `an_expired_deadline_returns_what_is_collected_without_waiting` run that on +every platform. It publishes each chunk as it arrives, not each complete line, so +a diagnostic with no trailing newline that arrived before the deadline is +reported (`the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` uses +`printf` without a newline for that reason). When the request has failed and the +collector is dropped, the capture is marked abandoned: what it held is released, +and the thread, which must keep draining the pipe so a surviving descendant is not +sent a SIGPIPE, discards and no longer forwards what arrives later, in chunks of +a fixed size, so a long-lived daemon does not grow with every failed request +(`an_abandoned_collector_releases_what_it_held_and_discards_what_arrives_later`, +red without the drop handling). + +Starting the collector before the write also means an engine that fills its +stderr pipe before it has drained a request larger than a pipe buffer cannot +deadlock the client (the engine blocked on a full stderr, the client on a full +stdin, nobody reading either). +`an_engine_that_writes_a_lot_of_stderr_before_reading_its_request_does_not_deadlock` +runs that against a fake that writes 300 kB to stderr first and is red when the +collector is started after the write. + +One wait is deliberately not bounded and has no test: after the child exits, the +answer is read to the end of its stdout and its stderr collector is joined, so an +engine that leaves a grandchild holding either pipe makes the request wait for +it. The clients this one replaced had the same exposure, and bounding it would +mean reading both pipes on their own threads and choosing what to report when a +deadline passes, which is a timeout policy the engine contract does not have +today. The module comment in `stdio_client.rs` states the limit and its tradeoff. + ### Advised commands must parse `apps/rocm/src/advised_commands.rs` checks the `rocm`/`rocmd` commands that the @@ -2111,7 +2275,8 @@ unconfirmed rather than having its PIDs cleared. `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded process and checks the confirmed report against the process, the record and -the endpoint key file. Run it with +the endpoint key file, and that the standalone `rocmd` got an answer to its +engine `Stop` request (it reaches the engine through the `rocm` beside it). Run it with `ROCM_CLI_BINARY=target/debug/rocm ROCM_CLI_ROCMD_BINARY=target/debug/rocmd cargo test -p e2e-cucumber --test e2e -- -n "service-stop-02"`. `service-stop-03`, `-04` and `-05` (Linux, mock lane) cover `rocm services @@ -2127,6 +2292,24 @@ a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n "service-stop"`. +`features/engine_ipc.feature` (`engine-ipc-01` and `-02`, Linux, mock lane) +covers how the daemon reads an engine child's answer, through the same +`stop_server` tool. Each scenario copies the real `rocmd` into a scratch +directory and puts a stand-in `rocm` shell script beside it, which is the +executable the standalone daemon runs for an engine request; the stand-in +records its argv and the request it was sent, and prints what the scenario +says. `engine-ipc-01` has the engine answer the stop and then exit non-zero: the +report carries the engine's answer (not an error built from the exit status). +`engine-ipc-02` has it print something that is not an answer, write to stderr +and exit non-zero: the report's engine error is the parse failure, naming the +exit status the engine ended with, followed by its stderr. Both check the report together with the server's process, record and +endpoint key, and that the stand-in was run once as `__engine-stdio vllm` with +one `stop` request line. The server is a `vllm` one, for which the engine's +`Stop` is advisory, so in `-02` the stop still completes; an engine whose stop +decides the confirmation (`aim`) is not staged here, because teardown would +then run the real engine's stop. Run them with a hooks build and `-n +"engine-ipc"`. + Manual restricted-tool smoke: ```bash diff --git a/tests/e2e-cucumber/features/engine_ipc.feature b/tests/e2e-cucumber/features/engine_ipc.feature new file mode 100644 index 0000000000..96eb773636 --- /dev/null +++ b/tests/e2e-cucumber/features/engine_ipc.feature @@ -0,0 +1,47 @@ +Feature: How the daemon reads an engine's answer + + # `rocmd` asks an engine to do things (here: stop a server) by running + # `rocm __engine-stdio ` and reading what the child prints. The + # standalone `rocmd` binary has no such verb of its own and runs the `rocm` + # beside it, which is what makes these scenarios stageable from outside: each + # one copies the real `rocmd` into a scratch directory and puts a stand-in + # `rocm` next to it, so the daemon's own tool reaches a child whose answer the + # scenario controls. The stand-in records the argv and the request it was + # given and prints what the scenario says. + # + # service-stop-02 covers the engine that answers cleanly; the two below are the + # other ways an engine child ends. In each, the engine's message in the tool's + # report is checked together with what the stop did to the server: its process, + # its record and its endpoint key. The server is a `vllm` one, for which the + # engine's `Stop` is advisory (the recorded process decides whether the stop + # is confirmed), so the stop going through is part of what is shown, not a + # side effect. + # + # Linux-only, for the same reason as service-stop-02: the recorded process + # carries the start-time token `/proc` gives it, and the stand-in is a shell + # script. No GPU and no real engine are involved, so these run on the mock lane. + + @id:engine-ipc-reads-an-answer-printed-before-a-failing-exit + @requires-os:linux + Scenario: engine-ipc-01 - An engine that answers the stop and then exits non-zero is read as having answered it + Given a managed server whose recorded process is running + And the engine answers the stop request and then exits with a failing status + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer to the stop, not the status it exited with + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-reports-an-unreadable-answer-with-the-engines-stderr + @requires-os:linux + Scenario: engine-ipc-02 - An engine that prints something unreadable and fails reports its stderr, and the stop still completes + Given a managed server whose recorded process is running + And the engine prints something that is not an answer, complains on stderr and exits with a failing status + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the stop request failed, with the engine's exit status and stderr + And the tool still reports the server stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index cae0e9859a..b32ef7e933 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -41,7 +41,11 @@ Feature: Stopping a managed service # # The recorded process is a real one the scenario starts, recorded with the # kernel's own start-time token for it, so the stop's identity check confirms - # it and goes on to terminate it. No GPU and no engine are involved. + # it and goes on to terminate it. No GPU and no running engine are involved: the + # daemon does ask the engine to stop (a real `vllm` engine, reached through the + # `rocm` beside the standalone `rocmd`), and since the planted record names no + # engine process the engine answers that it stopped nothing, which the scenario + # checks along with the report. # # Linux-only: the token is read from `/proc`, which is where `rocmd` reads it # too; without one the identity check is not exercised at all. @@ -59,6 +63,7 @@ Feature: Stopping a managed service Given a managed server whose recorded process is running When the daemon's stop_server tool stops that server Then the tool reports the server stopped, naming the process it stopped + And the tool asked the engine to stop and the engine answered And the recorded process is no longer running And the server's record reads stopped and names no process And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 31df329d2b..32153a060d 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -222,6 +222,72 @@ async fn next_stop_unverified(world: &mut E2eWorld) { script_next_stop_outcome(world, "unverified"); } +// ── Given: an engine child the scenario controls ──────────────────── + +/// The standalone daemon's file name: `rocmd`, with the platform's suffix. +const STANDALONE_ROCMD: &str = if cfg!(windows) { "rocmd.exe" } else { "rocmd" }; +/// What the stand-in engine launcher records, one line per time it is run. +const ENGINE_INVOCATIONS: &str = "engine-invocations"; +/// What the stand-in engine launcher read on its stdin, over every run. +const ENGINE_REQUESTS: &str = "engine-requests"; + +/// The directory holding the standalone daemon copy and the stand-in `rocm`. +fn standalone_dir(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("standalone-daemon") +} + +/// Plant a copy of the real `rocmd` with a stand-in `rocm` beside it. +/// +/// A `rocmd` named exactly that reaches an engine through the `rocm` next to it, +/// so the engine child the daemon runs for its `Stop` request is `behaviour`: a +/// shell script that first records its argv and what it read on stdin (into +/// [`ENGINE_INVOCATIONS`] and [`ENGINE_REQUESTS`] beside it), then does what the +/// scenario says. The copy is a plain file, not a link, because the daemon looks +/// for its sibling next to where its executable really is. +fn install_standalone_daemon(world: &E2eWorld, behaviour: &str) { + let dir = standalone_dir(world); + std::fs::create_dir_all(&dir).expect("failed to create the standalone daemon dir"); + let rocmd = std::fs::read(crate::rocmd_binary()).expect("failed to read rocmd"); + rocm_core::fake_executable::write_fake_executable(&dir.join(STANDALONE_ROCMD), rocmd) + .expect("failed to plant the standalone daemon"); + let script = format!( + "#!/bin/sh\n\ + dir=$(dirname \"$0\")\n\ + echo \"$*\" >> \"$dir/{ENGINE_INVOCATIONS}\"\n\ + cat >> \"$dir/{ENGINE_REQUESTS}\"\n\ + {behaviour}\n" + ); + rocm_core::fake_executable::write_fake_executable(&dir.join("rocm"), script) + .expect("failed to plant the stand-in rocm"); +} + +#[given("the engine answers the stop request and then exits with a failing status")] +async fn engine_answers_then_fails(world: &mut E2eWorld) { + install_standalone_daemon( + world, + "echo '{\"ok\":true,\"data\":{\"stopped\":true,\"graceful\":true}}'\n\ + echo 'stand-in engine: exiting with a failing status after answering' >&2\n\ + exit 3", + ); +} + +#[given( + "the engine prints something that is not an answer, complains on stderr and exits with a failing status" +)] +async fn engine_prints_garbage_and_fails(world: &mut E2eWorld) { + install_standalone_daemon( + world, + "printf 'this is not an answer'\n\ + echo 'engine exploded: no container runtime' >&2\n\ + exit 1", + ); +} + // ── When ─────────────────────────────────────────────────────────── /// Runs the pinned, directly-runnable form `serve_summary.rs`'s `stop` row @@ -239,7 +305,25 @@ async fn stop_with_yes(world: &mut E2eWorld) { #[when("the daemon's stop_server tool stops that server")] async fn stop_server_tool(world: &mut E2eWorld) { - let mut command = Command::new(crate::rocmd_binary()); + run_stop_server_tool(world, crate::rocmd_binary()); +} + +/// The same tool, run from the copy of `rocmd` that has a stand-in `rocm` beside +/// it (see [`install_standalone_daemon`]), so the engine's `Stop` request reaches +/// a child the scenario controls. +#[when("the standalone daemon's stop_server tool stops that server")] +async fn standalone_stop_server_tool(world: &mut E2eWorld) { + let rocmd = standalone_dir(world).join(STANDALONE_ROCMD); + assert!( + rocmd.is_file(), + "no standalone daemon was installed: {}", + rocmd.display() + ); + run_stop_server_tool(world, rocmd); +} + +fn run_stop_server_tool(world: &mut E2eWorld, rocmd: PathBuf) { + let mut command = Command::new(rocmd); command.args(["sandbox-tool", "stop_server", "--service-id", SERVICE_ID]); world.isolate_cmd(&mut command); let output = command.output().expect("failed to run rocmd"); @@ -311,6 +395,117 @@ async fn tool_reports_stopped(world: &mut E2eWorld) { ); } +/// The standalone `rocmd` reaches an engine through the `rocm` beside it (it has +/// no engine verb of its own), so its `Stop` request is answered rather than +/// failing with the standalone binary's "unrecognized subcommand". The report +/// carries the engine's answer or the error that took its place. +/// +/// The answer is `stopped: false`, and that is the right one here, not a gap: the +/// planted record names no engine state with a process id, so the real `vllm` +/// engine has nothing it can confirm stopping (`stop_service` answers `false` +/// when no PID is recorded). The recorded supervisor, which the daemon itself +/// signals, is what the other steps check. Asserting the whole answer, not just +/// that it is a boolean, is what tells it from an engine that was never reached. +#[then("the tool asked the engine to stop and the engine answered")] +async fn engine_answered_the_stop(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + let report: serde_json::Value = + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!(engine_stop["attempted"], true, "{report}"); + assert!( + engine_stop.get("error").is_none(), + "the engine's answer was an error instead: {report}" + ); + assert_eq!( + *engine_stop, + serde_json::json!({"attempted": true, "stopped": false, "graceful": false}), + "{report}" + ); +} + +/// The stop report `rocmd sandbox-tool stop_server` printed, with the tool's own +/// exit status checked first: both engine endings below leave the stop itself +/// succeeding. +fn tool_report(world: &E2eWorld) -> serde_json::Value { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "rocmd failed:\n{stdout}\n{}", + world.cli_stderr.as_deref().unwrap_or_default() + ); + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")) +} + +/// The stand-in engine was run once, as `__engine-stdio vllm`, and was sent one +/// line: an engine `stop` request for the planted server. Reads what the +/// stand-in itself recorded, so it holds for the child the daemon ran, not for +/// what the daemon says it ran. +#[then("the engine received one stop request for that server over stdio")] +async fn engine_received_one_stop_request(world: &mut E2eWorld) { + let dir = standalone_dir(world); + let invocations = std::fs::read_to_string(dir.join(ENGINE_INVOCATIONS)) + .expect("the stand-in engine was never run"); + assert_eq!( + invocations, + format!("__engine-stdio {ENGINE}\n"), + "the daemon must run the rocm beside it once, as `__engine-stdio `" + ); + let requests = std::fs::read_to_string(dir.join(ENGINE_REQUESTS)) + .expect("the stand-in engine recorded no request"); + assert_eq!( + requests.lines().count(), + 1, + "one request, one line: {requests:?}" + ); + let request: serde_json::Value = serde_json::from_str(&requests) + .unwrap_or_else(|error| panic!("the request is not JSON ({error}): {requests:?}")); + assert_eq!(request["method"], "stop", "{request}"); + assert_eq!(request["payload"]["service_id"], SERVICE_ID, "{request}"); +} + +/// What the engine printed before it exited non-zero is what the report carries: +/// a stop it answered, with no error taken from the exit status. +#[then("the tool reports the engine's answer to the stop, not the status it exited with")] +async fn engine_answer_survives_a_failing_exit(world: &mut E2eWorld) { + let report = tool_report(world); + assert_eq!( + report["result"]["engine_stop"], + serde_json::json!({"attempted": true, "stopped": true, "graceful": true}), + "{report}" + ); + assert_eq!(report["status"], "stopped", "{report}"); + assert_eq!(report["result"]["stopped"], true, "{report}"); +} + +/// An answer that cannot be read is reported as that, together with the status +/// the engine exited with and what it said on stderr: the status says the answer +/// was cut short, the stderr why. +#[then("the tool reports the stop request failed, with the engine's exit status and stderr")] +async fn unreadable_answer_reported_with_stderr(world: &mut E2eWorld) { + let report = tool_report(world); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!(engine_stop["attempted"], true, "{report}"); + assert_eq!( + engine_stop["error"], + "failed to parse engine response envelope \ + (engine stdio process exited with code 1); \ + stderr: engine exploded: no container runtime", + "{report}" + ); + assert!(engine_stop.get("stopped").is_none(), "{report}"); +} + +/// For a `vllm` server the engine's `Stop` is advisory: the recorded process +/// decides, so a failed request does not leave the server unconfirmed. +#[then("the tool still reports the server stopped")] +async fn tool_still_reports_stopped(world: &mut E2eWorld) { + let report = tool_report(world); + assert_eq!(report["status"], "stopped", "{report}"); + assert_eq!(report["result"]["stopped"], true, "{report}"); +} + #[then("the recorded process is no longer running")] async fn recorded_process_gone(world: &mut E2eWorld) { let process = world diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index ab4ad13c8e..bb1f06cae9 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -33,6 +33,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("diagnose.feature", "diagnose"), ("download_progress_pty.feature", "download-progress"), ("driver_install.feature", "driver-install"), + ("engine_ipc.feature", "engine-ipc"), ("engine_shell.feature", "engine-shell"), ("examine.feature", "examine"), ("install_lifecycle.feature", "lifecycle"), diff --git a/xtask/src/engine_stdio_client_contract.rs b/xtask/src/engine_stdio_client_contract.rs new file mode 100644 index 0000000000..f311abe55c --- /dev/null +++ b/xtask/src/engine_stdio_client_contract.rs @@ -0,0 +1,301 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: only the shared client may send an engine child its +//! request. +//! +//! `rocm` and `rocmd` used to carry their own copies of the JSON-over-stdio +//! request to an engine process, and the two drifted (different environment, +//! different treatment of a failing exit, a SIGPIPE guard in only one). The +//! single client in `crates/rocm-engine-protocol/src/stdio_client.rs` replaced +//! both. The behavioural tests of that client and of each caller catch a copy +//! that *behaves* differently; none of them can see that a second copy exists, +//! because the regression is a new function nobody calls from a test. +//! +//! So this guard is on the ingredient. A client has to put a request on a +//! child's stdin, which takes a piped stdin, and the request is an engine +//! envelope, so a file that does both is a client. [`CLIENT`] is the only file +//! allowed to; a caller reaches an engine through `run_engine_stdio` and has +//! neither a piped stdin nor a need to name an envelope for a child. A second +//! client has to either use the shared one or add itself to [`ALLOWED`] in a +//! reviewed diff that says why, as for [`crate::crate_edges::ALLOWLIST`]. +//! +//! The two ingredients are looked for anywhere in a file, not on one line, so a +//! copy that builds its `Stdio` in one place and passes it to `.stdin(..)` in +//! another is still seen. That makes the scan coarse on purpose: a file that +//! pipes some child's stdin for any reason and also names an envelope type is +//! reported, and the answer to a false positive is a reviewed [`ALLOWED`] entry. +//! +//! What this does **not** claim: it is a source-text scan, not a call-graph +//! analysis. A copy that builds its stdin another way (a pipe made by hand), or +//! reaches the envelope types under another name, is outside what it can see, +//! and it says nothing about whether a caller then uses the shared client +//! correctly (the callers' own tests pin that). It closes the cheap, likely +//! regression: somebody pasting a spawn-write-read helper back into a binary. +//! +//! Bidirectional, like the other allowlists: an entry that no longer matches +//! fails too, so the exemption does not outlive its reason. + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + /// The one file that is the client. + const CLIENT: &str = "crates/rocm-engine-protocol/src/stdio_client.rs"; + + /// Every file that may both pipe a child's stdin and name an engine + /// envelope, with the reason it is allowed to. + const ALLOWED: &[(&str, &str)] = &[( + CLIENT, + "the shared client: it spawns the engine child, writes the request line \ + to its stdin and reads the answer", + )]; + + /// This file, which the scan must skip: its fixtures quote both ingredients + /// as data, and a text scan cannot tell a fixture string from code. + const SELF_PATH: &str = "xtask/src/engine_stdio_client_contract.rs"; + + /// What names an engine envelope: the request and response types, or the + /// helper that builds a request. + const ENVELOPE_TOKENS: &[&str] = &[ + "EngineRequestEnvelope", + "EngineResponseEnvelope", + "engine_request_envelope", + ]; + + fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + /// Every `.rs` file in the repository as repo-relative, `/`-separated + /// paths. `target/` and dot-directories are skipped so the verdict never + /// depends on state outside the commit under test (the same two exclusions + /// as the other contract tests here). + fn workspace_rust_sources(root: &Path) -> Vec { + let mut found = Vec::new(); + collect_rust_files(root, root, &mut found); + assert!( + !found.is_empty(), + "found no Rust sources to scan -- the walk is broken, not the tree" + ); + found.sort(); + found + } + + fn collect_rust_files(root: &Path, dir: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let skip = path + .file_name() + .and_then(|name| name.to_str()) + .is_none_or(|name| name == "target" || name.starts_with('.')); + if skip { + continue; + } + collect_rust_files(root, &path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") + && let Ok(relative) = path.strip_prefix(root) + { + found.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } + + /// `line` with any `//` comment tail removed, so prose that names the + /// ingredients is not read as code. Crude on purpose (it does not track + /// string literals): that fails open only for a pathological line. + fn strip_comment(line: &str) -> &str { + match line.find("//") { + Some(at) => &line[..at], + None => line, + } + } + + /// Whether `text` pipes a child's stdin *and* names an engine envelope, + /// ignoring comments: the two ingredients of an engine stdio client. + fn is_a_client(text: &str) -> bool { + let code: Vec<&str> = text.lines().map(strip_comment).collect(); + // Anywhere in the file, not on one line: a `Stdio` built on one line and + // handed to `.stdin(..)` on another (a helper that takes it as a + // parameter) is the same ingredient. + let pipes_stdin = code.iter().any(|line| line.contains(".stdin(")) + && code.iter().any(|line| line.contains("piped()")); + let names_envelope = code + .iter() + .any(|line| ENVELOPE_TOKENS.iter().any(|token| line.contains(token))); + pipes_stdin && names_envelope + } + + fn client_files(root: &Path) -> Vec { + workspace_rust_sources(root) + .into_iter() + .filter(|relative| relative != SELF_PATH) + .filter(|relative| { + std::fs::read_to_string(root.join(relative)).is_ok_and(|text| is_a_client(&text)) + }) + .collect() + } + + /// Check a set of client files against [`ALLOWED`] in both directions. + fn check(clients: &[String]) -> Vec { + let unexpected = clients + .iter() + .filter(|file| !ALLOWED.iter().any(|(allowed, _)| allowed == file)) + .map(|file| { + format!( + "`{file}` pipes a child's stdin and names an engine envelope, which is an \ + engine stdio client. Use `rocm_engine_protocol::run_engine_stdio` (in \ + {CLIENT}), or add `{file}` to ALLOWED in \ + xtask/src/engine_stdio_client_contract.rs with the reason a second client \ + is needed" + ) + }); + let stale = ALLOWED + .iter() + .filter(|(allowed, _)| !clients.iter().any(|file| file == allowed)) + .map(|(allowed, _)| { + format!( + "ALLOWED entry `{allowed}` is stale: it no longer pipes a child's stdin and \ + names an engine envelope. Remove it, or the exemption outlives its reason" + ) + }); + unexpected.chain(stale).collect() + } + + #[test] + fn there_is_only_one_engine_stdio_client() { + let violations = check(&client_files(&repo_root())); + assert!( + violations.is_empty(), + "rocm and rocmd share one engine stdio client; a second copy can drift from it \ + (ROCMAI-875):\n{}", + violations.join("\n") + ); + } + + #[test] + fn every_allowed_client_carries_a_reason() { + for (file, reason) in ALLOWED { + assert!( + reason.len() > 40, + "ALLOWED entry `{file}` has no real justification: {reason:?}" + ); + } + } + + /// The shape the two replaced copies had: spawn with a piped stdin, write an + /// envelope to it, read the answer back. + const OLD_STYLE_COPY: &str = " +fn engine_request(engine_binary: &Path, envelope: &EngineRequestEnvelope) -> Result<()> { + let mut command = ProcessCommand::new(engine_binary); + command + .arg(\"__engine-stdio\") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()); + let mut child = command.spawn()?; + serde_json::to_writer(child.stdin.as_mut().unwrap(), envelope)?; + let output = child.wait_with_output()?; + let _answer: EngineResponseEnvelope = serde_json::from_slice(&output.stdout)?; + Ok(()) +} +"; + + #[test] + fn a_pasted_back_copy_of_the_old_client_is_recognised_and_reported() { + assert!(is_a_client(OLD_STYLE_COPY)); + let mut clients: Vec = ALLOWED.iter().map(|(file, _)| (*file).to_owned()).collect(); + clients.push("apps/rocmd/src/engine_ipc.rs".to_owned()); + let violations = check(&clients); + assert_eq!(violations.len(), 1, "{violations:?}"); + assert!(violations[0].contains("apps/rocmd/src/engine_ipc.rs")); + assert!(violations[0].contains("run_engine_stdio")); + } + + #[test] + fn a_copy_that_builds_its_stdio_on_one_line_and_uses_it_on_another_is_recognised() { + let split = " +fn spawn_engine(engine: &Path, stdin: Stdio) -> Result { + Ok(Command::new(engine).stdin(stdin).stdout(Stdio::null()).spawn()?) +} +fn ask(engine: &Path, envelope: &EngineRequestEnvelope) -> Result<()> { + let mut child = spawn_engine(engine, Stdio::piped())?; + serde_json::to_writer(child.stdin.as_mut().unwrap(), envelope)?; + Ok(()) +} +"; + assert!(is_a_client(split)); + } + + #[test] + fn a_caller_of_the_shared_client_is_not_a_client() { + let caller = " +fn engine_request(engine_binary: &Path, envelope: &EngineRequestEnvelope) -> Result { + decode_engine_response(run_engine_stdio(engine_binary, [\"stdio\"], &options, envelope)?) +} +"; + assert!(!is_a_client(caller)); + // Piping stdin for something that is not an engine request is fine. + let unrelated = "let child = Command::new(\"tar\").stdin(Stdio::piped()).spawn()?;"; + assert!(!is_a_client(unrelated)); + // Naming the envelope without a child is fine too (the serving side). + let serving = "fn handle(envelope: EngineRequestEnvelope) -> EngineResponseEnvelope {}"; + assert!(!is_a_client(serving)); + } + + #[test] + fn a_mention_in_a_comment_is_not_a_client() { + let prose = " +// builds an EngineRequestEnvelope and pipes it with .stdin(Stdio::piped()) +/// see EngineResponseEnvelope; the child gets `.stdin(Stdio::piped())` +fn nothing() {} +"; + assert!(!is_a_client(prose)); + } + + #[test] + fn the_client_that_is_allowed_stops_being_allowed_if_it_stops_matching() { + let violations = check(&[]); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("is stale")); + assert!(violations[0].contains(CLIENT)); + } + + #[test] + fn the_self_exclusion_names_a_file_that_exists_and_would_otherwise_be_reported() { + // A rename would leave SELF_PATH pointing at nothing, and the exclusion + // would silently stop applying. + let root = repo_root(); + let text = std::fs::read_to_string(root.join(SELF_PATH)) + .unwrap_or_else(|e| panic!("SELF_PATH `{SELF_PATH}` does not name a real file: {e}")); + assert!( + text.contains("OLD_STYLE_COPY"), + "`{SELF_PATH}` no longer holds the fixture the exclusion exists for" + ); + } + + #[test] + fn the_walk_reaches_the_client_and_both_callers() { + // Without this the contract could pass by scanning nothing. + let files = workspace_rust_sources(&repo_root()); + for expected in [ + CLIENT, + "apps/rocm/src/engine_ipc.rs", + "apps/rocmd/src/engine_ipc.rs", + "xtask/src/engine_stdio_client_contract.rs", + ] { + assert!( + files.iter().any(|file| file == expected), + "the source walk missed `{expected}`, so the contract is scanning less than \ + the tree" + ); + } + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index f4cdcb60d0..3082470dc4 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,6 +23,7 @@ mod demos; mod e2e; mod e2e_prewarm; mod e2e_report; +mod engine_stdio_client_contract; mod env_mutation_contract; mod eval_fixtures; mod lychee;